diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4900f6..56773e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,8 +56,9 @@ Policyserv is written to: ## Definitions / Architecture -**Filter**: Code which analyzes an event in isolation from other filters, producing a series of classifications and a -"neutral" or "spammy" determination. +**Filter**: Code which analyzes an event in isolation from other filters, producing a series of harm identifiers/content +classifications. The content classifications are used to determine whether an event is "spammy", neutral, or not spammy. +Content is by default classified as "neutral". **Set Group**: The precise filters to run on an event for a given stage. All filters in a set group are executed concurrently. Can be used to "pre" and "post" process events. For example, within a filter set, the first set group may @@ -68,11 +69,12 @@ group may inspect the result from the prior set groups to have future events be or room into the filters that community would like to run against events. Internally, the set groups which make up the filter set are ordered to provide some priority filtering and efficient processing. -**Confidence Vector**: A float value between 0 and 1 to carry confidence information about a classification. Zero is low/no -confidence that the classification applies while 1 is high/total confidence. +**Harm** (Identifier): A specific classification of prohibited content. Using a harm identifier implies the content is +not allowed (spammy). Most harm identifiers are defined by [MSC4456](https://github.com/matrix-org/matrix-spec-proposals/pull/4456), +though some are policyserv-specific. -**Classification**: A measurable and useful datum about an event. The most common being "spam". Events may have multiple -classifications through confidence vectors. +**Content Class**: Whether the content is allowed, neutral, or prohibited (spammy). A neutral designation does not mean +that the content is spam-free, but rather that the content doesn't trip any flags (positive or negative). ## PR review process diff --git a/README.md b/README.md index 8e58a34..b2a647d 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,6 @@ server are trusted by the server administrator(s). ### General -* `PS_SPAM_THRESHOLD` (default `0.8`) - A value between 0 and 1 denoting how much "confidence" is needed before an event - is considered spammy. Note that policyserv can currently only generate scores of 0, 0.5, and 1 - a future version will - allow for more precise scores, so it's recommended to keep this value as a decimal. * `PS_MODERATION_BOT_USER_ID` (default empty value) - The user ID of the bot account where policyserv can send redaction commands to. A device on the user *must* implement the [policyserv to-device protocol](./docs/to_device.md) for this to work. Set to an empty value to disable this feature. diff --git a/ai/openai_omni_moderation.go b/ai/openai_omni_moderation.go index 732caa7..800f1f8 100644 --- a/ai/openai_omni_moderation.go +++ b/ai/openai_omni_moderation.go @@ -8,7 +8,7 @@ import ( "github.com/matrix-org/policyserv/config" "github.com/matrix-org/policyserv/event" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" ) @@ -35,7 +35,7 @@ func NewOpenAIOmniModeration(cnf *config.InstanceConfig, additionalClientOptions }, nil } -func (m *OpenAIOmniModeration) CheckEvent(ctx context.Context, cnf *OpenAIOmniModerationConfig, input *Input) ([]classification.Classification, error) { +func (m *OpenAIOmniModeration) CheckEvent(ctx context.Context, cnf *OpenAIOmniModerationConfig, input *Input) (*harms.ContentInfo, error) { messages, err := event.RenderToText(input.Event) if err != nil { return nil, err @@ -53,25 +53,25 @@ func (m *OpenAIOmniModeration) CheckEvent(ctx context.Context, cnf *OpenAIOmniMo log.Printf("[%s | %s] Error checking message: %s", input.Event.EventID(), input.Event.RoomID(), err) if cnf.FailSecure { log.Printf("[%s | %s] Returning spam response to block events and discourage retries", input.Event.EventID(), input.Event.RoomID()) - return []classification.Classification{classification.Spam, classification.Frequency}, nil + return harms.ProhibitedContent(harms.OtherGeneral), nil } else { log.Printf("[%s | %s] Returning neutral response despite error, per config", input.Event.EventID(), input.Event.RoomID()) - return nil, nil + return harms.NeutralContent(), nil } } for _, r := range res.Results { // Note: we compress JSON here because the OpenAI library tends to return *a lot* of redundant detail, including JSON with newlines in it. log.Printf("[%s | %s] Result for sender %s: Flagged=%t Flags=%s Scores=%s", input.Event.EventID(), input.Event.RoomID(), input.Event.SenderID(), r.Flagged, compressJsonResponse(r.Categories), compressJsonResponse(r.CategoryScores)) if r.Flagged { - flags := []classification.Classification{classification.Spam} + harmIds := []harms.Harm{harms.SpamGeneral} if r.Categories.SexualMinors { - flags = append(flags, classification.CSAM) + harmIds = append(harmIds, harms.ChildSafetyCSAM) } - return flags, nil + return harms.ProhibitedContent(harmIds...), nil } } } - return nil, nil + return harms.NeutralContent(), nil } type compressible interface { diff --git a/ai/openai_omni_moderation_test.go b/ai/openai_omni_moderation_test.go index 7a5ea3a..149c668 100644 --- a/ai/openai_omni_moderation_test.go +++ b/ai/openai_omni_moderation_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/openai/openai-go/v3/option" "github.com/stretchr/testify/assert" @@ -38,35 +38,32 @@ func TestOpenAIOmniModeration(t *testing.T) { spammyEvent1 := test.MustMakeKeywordEvent(test.KeywordSpammy) ret, err := provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: spammyEvent1}) assert.NoError(t, err) - assert.Equal(t, []classification.Classification{ - classification.Spam, - }, ret) + test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.SpamGeneral), ret) spammyEvent2 := test.MustMakeKeywordEvent(test.KeywordSpammyCSAM) ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: spammyEvent2}) assert.NoError(t, err) - assert.Equal(t, []classification.Classification{ - classification.Spam, - classification.CSAM, // should have been detected - }, ret) + test.AssertEqualContentInfo(t, harms.ProhibitedContent( + harms.SpamGeneral, + harms.ChildSafetyCSAM, // should have been detected + ), ret) neutralEvent := test.MustMakeKeywordEvent(test.KeywordNeutral) ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: neutralEvent}) assert.NoError(t, err) - assert.Nil(t, ret) // no classifications + test.AssertEqualContentInfo(t, harms.NeutralContent(), ret) // no classifications failEvent := test.MustMakeKeywordEvent(test.KeywordIntentionalFail) // First test that when FailSecure: true we return a spam classification ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: true}, &Input{Event: failEvent}) assert.NoError(t, err) - assert.Equal(t, []classification.Classification{ - classification.Spam, - classification.Frequency, // also added by the provider - }, ret) + test.AssertEqualContentInfo(t, harms.ProhibitedContent( + harms.OtherGeneral, // added by the provider + ), ret) // Now when FailSecure: false, we should get no classifications (but also no errors) ret, err = provider.CheckEvent(context.Background(), &OpenAIOmniModerationConfig{FailSecure: false}, &Input{Event: failEvent}) assert.NoError(t, err) - assert.Nil(t, ret) + test.AssertEqualContentInfo(t, harms.NeutralContent(), ret) } diff --git a/ai/types.go b/ai/types.go index f063c91..847a8f4 100644 --- a/ai/types.go +++ b/ai/types.go @@ -4,7 +4,7 @@ import ( "context" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/media" ) @@ -14,5 +14,5 @@ type Input struct { } type Provider[ConfigT any] interface { - CheckEvent(ctx context.Context, cnf ConfigT, input *Input) ([]classification.Classification, error) + CheckEvent(ctx context.Context, cnf ConfigT, input *Input) (*harms.ContentInfo, error) } diff --git a/api/errors.go b/api/errors.go index c083f56..f370853 100644 --- a/api/errors.go +++ b/api/errors.go @@ -4,6 +4,7 @@ import ( "log" "net/http" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/homeserver" "github.com/matrix-org/policyserv/metrics" ) @@ -12,10 +13,10 @@ type errorResponder struct { action string w http.ResponseWriter r *http.Request - harms []string + harms []harms.Harm } -func (e *errorResponder) addHarm(harm string) *errorResponder { +func (e *errorResponder) addHarm(harm harms.Harm) *errorResponder { e.harms = append(e.harms, harm) return e } @@ -49,6 +50,6 @@ func newErrorResponder(action string, w http.ResponseWriter, r *http.Request) *e action: action, w: w, r: r, - harms: make([]string, 0), + harms: make([]harms.Harm, 0), } } diff --git a/api/http_server_api_check.go b/api/http_server_api_check.go index f242212..a7a9fdf 100644 --- a/api/http_server_api_check.go +++ b/api/http_server_api_check.go @@ -5,7 +5,7 @@ import ( "log" "net/http" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics" "github.com/matrix-org/policyserv/queue" "github.com/matrix-org/policyserv/storage" @@ -36,16 +36,15 @@ func httpCheckTextCommunityApi(api *Api, community *storage.StoredCommunity, w h } textToCheck := string(b) - vecs, err := set.CheckText(r.Context(), textToCheck) + info, err := set.CheckText(r.Context(), textToCheck) if err != nil { errs.err(http.StatusInternalServerError, "M_UNKNOWN", err) return } - if set.IsSpamResponse(r.Context(), vecs) { - errs.addHarm("org.matrix.msc4387.spam") - if vecs.GetVector(classification.CSAM) > 0.5 { - errs.addHarm("org.matrix.msc4387.child_safety.csam") + if info.Class() == harms.ContentClassProhibited { + for _, h := range info.Harms() { + errs.addHarm(h) } errs.text(http.StatusBadRequest, "ORG.MATRIX.MSC4387_SAFETY", "Text is probably spammy") } else { @@ -87,7 +86,7 @@ func httpCheckEventIdCommunityApi(api *Api, community *storage.StoredCommunity, return } if event != nil { - renderEventResult(event.IsProbablySpam, w, r, errs) + renderEventResult(event.ContentInfo, w, r, errs) return } @@ -122,11 +121,14 @@ func httpCheckEventIdCommunityApi(api *Api, community *storage.StoredCommunity, errs.err(http.StatusInternalServerError, "M_UNKNOWN", res.Err) return } - renderEventResult(res.IsProbablySpam, w, r, errs) + renderEventResult(res.ContentInfo, w, r, errs) } -func renderEventResult(isProbablySpam bool, w http.ResponseWriter, r *http.Request, errs *errorResponder) { - if isProbablySpam { +func renderEventResult(info *harms.ContentInfo, w http.ResponseWriter, r *http.Request, errs *errorResponder) { + if info.Class() == harms.ContentClassProhibited { + for _, h := range info.Harms() { + errs.addHarm(h) + } errs.text(http.StatusBadRequest, "M_FORBIDDEN", "This message is not allowed by the policy server") } else { err := respondJson("httpCheckEventIdCommunityApi", r, w, map[string]any{}) diff --git a/api/http_server_api_check_test.go b/api/http_server_api_check_test.go index 94bd1d9..38311c0 100644 --- a/api/http_server_api_check_test.go +++ b/api/http_server_api_check_test.go @@ -11,8 +11,7 @@ import ( "testing" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/homeserver" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/storage" @@ -51,7 +50,7 @@ func TestHttpCheckTextCommunityApi(t *testing.T) { httpCheckTextCommunityApi(api, serverCommunity, w, r) assert.Equal(t, http.StatusBadRequest, w.Code) test.AssertApiError(t, w, "ORG.MATRIX.MSC4387_SAFETY", "Text is probably spammy") - test.AssertApiErrorHarms(t, w, []string{"org.matrix.msc4387.spam"}) + test.AssertApiErrorHarms(t, w, []string{string(harms.SpamGeneral)}) // Then test a non-match w = httptest.NewRecorder() @@ -81,15 +80,15 @@ func TestHttpCheckEventIdCommunityApiWithKnownEvent(t *testing.T) { // For this test, cache a couple event results to ensure we can avoid federation calls err := api.storage.UpsertEventResult(context.Background(), &storage.StoredEventResult{ - EventId: "$spam", - IsProbablySpam: true, - ConfidenceVectors: confidence.Vectors{classification.Spam: 1.0}, + EventId: "$spam", + IsProbablySpam: true, + ContentInfo: harms.ProhibitedContent(harms.SpamGeneral), }) assert.NoError(t, err) err = api.storage.UpsertEventResult(context.Background(), &storage.StoredEventResult{ - EventId: "$neutral", - IsProbablySpam: false, - ConfidenceVectors: confidence.NewConfidenceVectors(), + EventId: "$neutral", + IsProbablySpam: false, + ContentInfo: harms.NeutralContent(), }) assert.NoError(t, err) diff --git a/community/manager.go b/community/manager.go index e981e19..910ccd0 100644 --- a/community/manager.go +++ b/community/manager.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/policyserv/config" "github.com/matrix-org/policyserv/content" "github.com/matrix-org/policyserv/filter" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/notifiers" "github.com/matrix-org/policyserv/pubsub" @@ -97,7 +98,7 @@ func (m *Manager) GetFilterSetForCommunityId(ctx context.Context, communityId st prefilters := []string{filter.ProtectLocalUserFilterName} hellbanPrefilters := make([]string, 0) // these run after the prefilters, but before the other filters filters := make([]string, 0) - postfilters := make([]string, 0) + postfilterSilences := make([]string, 0) if len(internal.Dereference(communityConfig.KeywordFilterKeywords)) > 0 { filters = append(filters, filter.KeywordFilterName) } @@ -139,7 +140,7 @@ func (m *Manager) GetFilterSetForCommunityId(ctx context.Context, communityId st } if internal.Dereference(communityConfig.HellbanPostfilterMinutes) > 0 { hellbanPrefilters = append(hellbanPrefilters, filter.HellbanPrefilterName) - postfilters = append(postfilters, filter.HellbanPostfilterName) + postfilterSilences = append(postfilterSilences, filter.HellbanPostfilterName) } if m.instanceConfig.OpenAIApiKey != "" { // Access to this filter is gated by further instance config (namely, the room IDs allowed to use it) @@ -180,34 +181,31 @@ func (m *Manager) GetFilterSetForCommunityId(ctx context.Context, communityId st InstanceConfig: m.instanceConfig, Groups: []*filter.SetGroupConfig{{ // The first set group replaces the concept of "prefilters". We want this group to capture all events, - // so we set the min and max to cover the full range. - EnabledNames: prefilters, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + // so we set the classes appropriately. + EnabledNames: prefilters, + // Micro optimization: Put neutral first so we can skip CPU cycles in a loop in the general case. We also + // probably don't need to specify allowed and prohibited here because we have no code which pushes such + // classes right away, but for safety we might as well. + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral, harms.ContentClassAllowed, harms.ContentClassProhibited}, }, { // This set group is similar to prefilters, but only contains the hellban prefilters. This is to prevent // users being denied abilities that are intended to be granted to them via other prefilters (like an // ability to leave the room). EnabledNames: hellbanPrefilters, - // We want the min/max values to catch "maybe spam" from the prefilters level, but explicitly not events - // that were flagged as not-spam by the same layer. This is why the minimum is not quite zero. - MinimumSpamVectorValue: 0.1, - MaximumSpamVectorValue: 1.0, + // We want to capture "maybe spam", but not events that were already flagged as (not) spam. + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, }, { // The third set group is what was previously the middle layer for filters. This is where the bulk of // the work happens. We only want it to run if the prefilters didn't already declare an event spammy or // neutral though, so we narrow the min/max range a bit. EnabledNames: filters, - // We want the min/max to be slightly less than the extremes to avoid doing work - // when the event is already flagged as (not) spam. - MinimumSpamVectorValue: 0.3, - MaximumSpamVectorValue: 0.7, + // Skip this group for events that are already (not) spam. + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, }, { - // The last set group replaces "postfilters", and should be run regardless of whether the previous groups - // flagged the event as spam. As such, we set the min/max to the widest possible range. - EnabledNames: postfilters, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + // The last set group is for telling the hellban postfilter if any previous filter flagged an event as + // spammy so it can put a silence in place. + EnabledNames: postfilterSilences, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, }}, } filterSet, err := filter.NewSet(setConfig, m.storage, m.pubsubClient, m.notifier, scanner) diff --git a/community/manager_test.go b/community/manager_test.go index be7ef73..5474df8 100644 --- a/community/manager_test.go +++ b/community/manager_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/matrix-org/policyserv/config" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/pubsub" "github.com/matrix-org/policyserv/storage" @@ -83,10 +84,10 @@ func TestGetFilterSetForRoomId(t *testing.T) { "body": "keyword1", }, }) - vecs, err := set.CheckEvent(ctx, keyword1Event, nil) + info, err := set.CheckEvent(ctx, keyword1Event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.True(t, set.IsSpamResponse(ctx, vecs)) + assert.NotNil(t, info) + assert.Equal(t, harms.ContentClassProhibited, info.Class()) // Now, create a second room and community and ensure that the keyword1 event above // is *not* spam, but a new keyword *is* spam (proving we can have independent configs) @@ -112,9 +113,9 @@ func TestGetFilterSetForRoomId(t *testing.T) { set, err = manager.GetFilterSetForRoomId(ctx, roomId2) assert.NoError(t, err) assert.NotNil(t, set) - vecs, err = set.CheckEvent(ctx, keyword1Event, nil) + info, err = set.CheckEvent(ctx, keyword1Event, nil) assert.NoError(t, err) - assert.False(t, set.IsSpamResponse(ctx, vecs)) + assert.NotEqual(t, harms.ContentClassProhibited, info.Class()) keyword2Event := test.MustMakePDU(&test.BaseClientEvent{ EventId: "$event2", RoomId: roomId, @@ -124,9 +125,9 @@ func TestGetFilterSetForRoomId(t *testing.T) { "body": "keyword2", }, }) - vecs, err = set.CheckEvent(ctx, keyword2Event, nil) + info, err = set.CheckEvent(ctx, keyword2Event, nil) assert.NoError(t, err) - assert.True(t, set.IsSpamResponse(ctx, vecs)) + assert.Equal(t, harms.ContentClassProhibited, info.Class()) } func TestCommunityManagerCacheInvalidation(t *testing.T) { @@ -176,10 +177,10 @@ func TestCommunityManagerCacheInvalidation(t *testing.T) { "body": "keyword1", }, }) - vecs, err := set.CheckEvent(ctx, keyword1Event, nil) + info, err := set.CheckEvent(ctx, keyword1Event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.True(t, set.IsSpamResponse(ctx, vecs)) + assert.NotNil(t, info) + assert.Equal(t, harms.ContentClassProhibited, info.Class()) // Now, change the config but *don't* notify the manager about it to prove the cache is working cnf.KeywordFilterKeywords = &[]string{"keyword2"} @@ -192,10 +193,10 @@ func TestCommunityManagerCacheInvalidation(t *testing.T) { set, err = manager.GetFilterSetForRoomId(ctx, roomId) assert.NoError(t, err) assert.NotNil(t, set) - vecs, err = set.CheckEvent(ctx, keyword1Event, nil) + info, err = set.CheckEvent(ctx, keyword1Event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.True(t, set.IsSpamResponse(ctx, vecs)) // should still be spam + assert.NotNil(t, info) + assert.Equal(t, harms.ContentClassProhibited, info.Class()) // should still be spam // Now, notify the manager and re-test err = manager.pubsubClient.Publish(ctx, pubsub.TopicCommunityConfig, communityId) @@ -204,10 +205,10 @@ func TestCommunityManagerCacheInvalidation(t *testing.T) { set, err = manager.GetFilterSetForRoomId(ctx, roomId) assert.NoError(t, err) assert.NotNil(t, set) - vecs, err = set.CheckEvent(ctx, keyword1Event, nil) + info, err = set.CheckEvent(ctx, keyword1Event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.False(t, set.IsSpamResponse(ctx, vecs)) // should not be spam + assert.NotNil(t, info) + assert.NotEqual(t, harms.ContentClassProhibited, info.Class()) // should not be spam // Create a second community and assign the existing room to it, but again: don't // notify the manager of that room ID to community ID change @@ -241,10 +242,10 @@ func TestCommunityManagerCacheInvalidation(t *testing.T) { set, err = manager.GetFilterSetForRoomId(ctx, roomId) assert.NoError(t, err) assert.NotNil(t, set) - vecs, err = set.CheckEvent(ctx, keyword3Event, nil) + info, err = set.CheckEvent(ctx, keyword3Event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.False(t, set.IsSpamResponse(ctx, vecs)) + assert.NotNil(t, info) + assert.NotEqual(t, harms.ContentClassProhibited, info.Class()) // Now notify the manager and re-check the event (which should now be spam) err = manager.pubsubClient.Publish(ctx, pubsub.TopicRoomCommunityId, roomId) @@ -253,8 +254,8 @@ func TestCommunityManagerCacheInvalidation(t *testing.T) { set, err = manager.GetFilterSetForRoomId(ctx, roomId) assert.NoError(t, err) assert.NotNil(t, set) - vecs, err = set.CheckEvent(ctx, keyword3Event, nil) + info, err = set.CheckEvent(ctx, keyword3Event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.True(t, set.IsSpamResponse(ctx, vecs)) // now it's spam + assert.NotNil(t, info) + assert.Equal(t, harms.ContentClassProhibited, info.Class()) // now it's spam } diff --git a/config/community.go b/config/community.go index ddceae4..99ceede 100644 --- a/config/community.go +++ b/config/community.go @@ -37,7 +37,6 @@ type CommunityConfig struct { EventTypePrefilterAllowedStateEventTypes *[]string `json:"event_type_prefilter_allowed_state_event_types,omitempty" envconfig:"event_type_prefilter_allowed_state_event_types" default:"m.room.power_levels,m.room.avatar,m.room.name,m.room.topic,m.room.join_rules,m.room.history_visibility,m.room.create,m.room.server_acl,m.room.tombstone,m.room.encryption,m.room.canonical_alias"` HellbanPostfilterMinutes *int `json:"hellban_postfilter_minutes,omitempty" envconfig:"hellban_postfilter_minutes" default:"60"` MjolnirFilterEnabled *bool `json:"mjolnir_filter_enabled,omitempty" envconfig:"mjolnir_filter_enabled" default:"true"` - SpamThreshold *float64 `json:"spam_threshold,omitempty" envconfig:"spam_threshold" default:"0.8"` WebhookUrl *string `json:"webhook_url,omitempty" envconfig:"webhook_url" default:""` OpenAIFilterFailSecure *bool `json:"openai_filter_fail_secure,omitempty" envconfig:"openai_filter_fail_secure" default:"true"` StickyEventsFilterAllowStickyEvents *bool `json:"sticky_events_filter_allow_sticky_events,omitempty" envconfig:"sticky_events_filter_allow_sticky_events" default:"true"` diff --git a/config/layering_test.go b/config/layering_test.go index 6bae681..f1edc44 100644 --- a/config/layering_test.go +++ b/config/layering_test.go @@ -11,16 +11,16 @@ func TestNewCommunityConfigForJSON(t *testing.T) { // NewCommunityConfigForJSON calls newCommunityConfigForJSONWithBase internally testBase := &CommunityConfig{ - SpamThreshold: internal.Pointer(0.2), - KeywordFilterKeywords: &[]string{"spammy spam", "example"}, + InlineEmojiSizeFilterMaxHeightPixels: internal.Pointer(100), + KeywordFilterKeywords: &[]string{"spammy spam", "example"}, } testJSON := []byte(`{ - "spam_threshold": 0.5, + "inline_emoji_size_filter_max_height_pixels": 36, "length_filter_max_length": 12 }`) testConfig, err := newCommunityConfigForJSONWithBase(testBase, testJSON) assert.NoError(t, err) - assert.Equal(t, 0.5, *testConfig.SpamThreshold) + assert.Equal(t, 36, *testConfig.InlineEmojiSizeFilterMaxHeightPixels) assert.Equal(t, 12, *testConfig.LengthFilterMaxLength) assert.Equal(t, []string{"spammy spam", "example"}, *testConfig.KeywordFilterKeywords) } @@ -31,27 +31,27 @@ func TestEnvconfigAssumptions(t *testing.T) { // will use defaults instead of "instance config". t.Cleanup(func() { - t.Setenv("PS_SPAM_THRESHOLD", "") + t.Setenv("PS_INLINE_EMOJI_SIZE_FILTER_MAX_HEIGHT_PIXELS", "") t.Setenv("PS_KEYWORD_FILTER_KEYWORDS", "") }) - t.Setenv("PS_SPAM_THRESHOLD", "0.2") + t.Setenv("PS_INLINE_EMOJI_SIZE_FILTER_MAX_HEIGHT_PIXELS", "100") t.Setenv("PS_KEYWORD_FILTER_KEYWORDS", "spammy spam,example") buildBaseConfig() - assert.Equal(t, 0.2, *baseConfigRaw.SpamThreshold) + assert.Equal(t, 100, *baseConfigRaw.InlineEmojiSizeFilterMaxHeightPixels) assert.Equal(t, []string{"spammy spam", "example"}, *baseConfigRaw.KeywordFilterKeywords) assert.Equal(t, 10000, *baseConfigRaw.LengthFilterMaxLength) // default assert.Equal(t, 20, *baseConfigRaw.ManyAtsFilterMaxAts) // default testJSON := []byte(`{ - "spam_threshold": 0.5, + "inline_emoji_size_filter_max_height_pixels": 36, "length_filter_max_length": 12 }`) testConfig, err := newCommunityConfigForJSONWithBase(baseConfigRaw, testJSON) assert.NoError(t, err) // JSON overrides - assert.Equal(t, 0.5, *testConfig.SpamThreshold) + assert.Equal(t, 36, *testConfig.InlineEmojiSizeFilterMaxHeightPixels) assert.Equal(t, 12, *testConfig.LengthFilterMaxLength) // Base config overrides @@ -85,14 +85,14 @@ func TestNewDefaultCommunityConfig(t *testing.T) { // When calling the function, we shouldn't be getting any envconfig values t.Cleanup(func() { - t.Setenv("PS_SPAM_THRESHOLD", "") + t.Setenv("PS_INLINE_EMOJI_SIZE_FILTER_MAX_HEIGHT_PIXELS", "") t.Setenv("PS_KEYWORD_FILTER_KEYWORDS", "") }) - t.Setenv("PS_SPAM_THRESHOLD", "0.2") + t.Setenv("PS_INLINE_EMOJI_SIZE_FILTER_MAX_HEIGHT_PIXELS", "100") t.Setenv("PS_KEYWORD_FILTER_KEYWORDS", "spammy spam,example") cnf, err := NewDefaultCommunityConfig() assert.NoError(t, err) - assert.NotEqual(t, 0.2, *cnf.SpamThreshold) + assert.NotEqual(t, 100, *cnf.InlineEmojiSizeFilterMaxHeightPixels) assert.NotEqual(t, []string{"spammy spam", "example"}, *cnf.KeywordFilterKeywords) } diff --git a/content/hma.go b/content/hma.go index e384f87..3aa1caa 100644 --- a/content/hma.go +++ b/content/hma.go @@ -13,7 +13,7 @@ import ( "net/url" "sync" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics" ) @@ -34,7 +34,7 @@ func NewHMAScanner(apiBaseUrl string, apiKey string, enabledBankNames []string) }, nil } -func (s *HMAScanner) Scan(ctx context.Context, contentType Type, content []byte) ([]classification.Classification, error) { +func (s *HMAScanner) Scan(ctx context.Context, contentType Type, content []byte) (*harms.ContentInfo, error) { hash, err := s.hash(contentType, content) if err != nil { return nil, err @@ -63,10 +63,10 @@ func (s *HMAScanner) Scan(ctx context.Context, contentType Type, content []byte) if hadMatch { // TODO: Support labeling the banks rather than assuming it's always CSAM - return []classification.Classification{classification.Spam, classification.CSAM}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.ChildSafetyCSAM, harms.PolicyservMedia), nil } - return nil, nil + return harms.NeutralContent(), nil } func (s *HMAScanner) hash(contentType Type, content []byte) (hashResponse, error) { diff --git a/content/scanning.go b/content/scanning.go index 303d57b..69005aa 100644 --- a/content/scanning.go +++ b/content/scanning.go @@ -3,9 +3,9 @@ package content import ( "context" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" ) type Scanner interface { - Scan(ctx context.Context, contentType Type, content []byte) ([]classification.Classification, error) + Scan(ctx context.Context, contentType Type, content []byte) (*harms.ContentInfo, error) } diff --git a/docker-compose.yml b/docker-compose.yml index cf2927a..5bfc14d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,6 @@ services: PS_JOIN_SERVER: hs1 PS_HTTP_BIND: "0.0.0.0:8080" PS_API_KEY: dontuseinproduction - # Adjust spam threshold so that not every message is considered spam (policyserv sets default confidence to 50%) - PS_SPAM_THRESHOLD: "0.8" # Otherwise we'll never find the keys for hs1/hs2 as they only exist locally PS_DIRECT_KEY_FETCHING: "true" # Policyserv denies private ips by default which would block the other containers https://github.com/matrix-org/policyserv/blob/9b0687b877b326b55cbdd2dee79a4cc46114a9f8/config/config.go#L35. diff --git a/filter/audit.go b/filter/audit.go index ef95d16..e4cab2d 100644 --- a/filter/audit.go +++ b/filter/audit.go @@ -11,14 +11,14 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/notifiers" ) type auditContext struct { Event gomatrixserverlib.PDU IsSpam bool - FilterResponses map[string][]classification.Classification + FilterResponses map[string][]string CommunityId string lock sync.Mutex // use a lock instead of a sync.Map because sync.Map doesn't support generics (and library support appears lacking in quality) @@ -28,7 +28,7 @@ type auditContext struct { func newAuditContext(notifier notifiers.MatrixNotifier, communityId string, event gomatrixserverlib.PDU) (*auditContext, error) { return &auditContext{ Event: event, - FilterResponses: make(map[string][]classification.Classification), + FilterResponses: make(map[string][]string), CommunityId: communityId, // Populated later @@ -40,10 +40,13 @@ func newAuditContext(notifier notifiers.MatrixNotifier, communityId string, even }, nil } -func (c *auditContext) AppendFilterResponse(filterName string, classifications []classification.Classification) { +func (c *auditContext) AppendFilterResponse(filterName string, contentInfo *harms.ContentInfo) { c.lock.Lock() defer c.lock.Unlock() - c.FilterResponses[filterName] = classifications + c.FilterResponses[filterName] = []string{contentInfo.Class().String()} + for _, h := range contentInfo.Harms() { + c.FilterResponses[filterName] = append(c.FilterResponses[filterName], string(h)) + } } func (c *auditContext) Publish() error { @@ -69,7 +72,7 @@ func (c *auditContext) Publish() error { } contentJson := contentBuf.String() - wasHellban := c.FilterResponses[HellbanPrefilterName] != nil && slices.Contains(c.FilterResponses[HellbanPrefilterName], classification.Spam) + wasHellban := c.FilterResponses[HellbanPrefilterName] != nil && slices.Contains(c.FilterResponses[HellbanPrefilterName], string(harms.SpamGeneral)) htmlAudit := "A user has had an event of theirs flagged as spam by policyserv:
" htmlAudit += fmt.Sprintf("User ID: %s
", html.EscapeString(string(c.Event.SenderID()))) diff --git a/filter/classification/classifications.go b/filter/classification/classifications.go deleted file mode 100644 index 878fed8..0000000 --- a/filter/classification/classifications.go +++ /dev/null @@ -1,49 +0,0 @@ -package classification - -import "strings" - -type Classification string - -// Spam - When closer to 1, the event should be considered spam. When closer to 0, the event is neutral -// and should still be considered as "potentially spammy, though unlikely". -const Spam Classification = "spam" - -// CSAM - When closer to 1, the event likely (or definitely) contains CSAM. -const CSAM Classification = "csam" - -// Volumetric - When closer to 1, the event attempts to disrupt the timeline of the room. -const Volumetric Classification = "volumetric" - -// Frequency - When closer to 1, the event (or similar) may have already been repeated a lot. -const Frequency Classification = "frequency" - -// Mentions - When closer to 1, the event's mechanism for spam includes mentioning other users. -const Mentions Classification = "mentions" - -// DAGAbuse - When closer to 1, the event appears to be attempting to disrupt the core structure of the room. -const DAGAbuse Classification = "dag_abuse" - -// NonCompliance - When closer to 1, the event does not comply with the specification. Events classified as such -// should have already been rejected by "real" homeservers. -const NonCompliance Classification = "non_compliance" - -// Unsafe - When closer to 1, the event was not able to be fully checked by the filter engine and therefore may be spammy. -const Unsafe Classification = "unsafe" - -func (c Classification) String() string { - if c.IsInverted() { - return strings.TrimPrefix(string(c), "inverted_") - } - return string(c) -} - -func (c Classification) IsInverted() bool { - return strings.HasPrefix(string(c), "inverted_") -} - -func (c Classification) Invert() Classification { - if c.IsInverted() { - return Classification(c.String()) - } - return Classification("inverted_" + c.String()) -} diff --git a/filter/confidence/consts.go b/filter/confidence/consts.go deleted file mode 100644 index ac960be..0000000 --- a/filter/confidence/consts.go +++ /dev/null @@ -1,4 +0,0 @@ -package confidence - -const UnlikelyButNotCertain float64 = 0.2 // arbitrarily set as "not quite 0.0" -const LikelyButNotCertain float64 = 0.9 // arbitrarily set as "not quite 1.0" diff --git a/filter/confidence/vectors.go b/filter/confidence/vectors.go deleted file mode 100644 index 03c3c7e..0000000 --- a/filter/confidence/vectors.go +++ /dev/null @@ -1,56 +0,0 @@ -package confidence - -import ( - "fmt" - - "github.com/matrix-org/policyserv/filter/classification" -) - -type Vectors map[classification.Classification]float64 - -func NewConfidenceVectors() Vectors { - return make(map[classification.Classification]float64) -} - -func (v Vectors) SetVector(classification classification.Classification, vector float64) { - if vector > 1.0 || vector < 0.0 { - // "Should never happen" - programmer error - panic(fmt.Errorf("vector out of range: %f must be between 0.0 and 1.0", vector)) - } - // We store non-inverted values. - if classification.IsInverted() { - vector = 1.0 - vector - classification = classification.Invert() - } - v[classification] = vector -} - -func (v Vectors) GetVector(classification classification.Classification) float64 { - // We store non-inverted values. - if classification.IsInverted() { - classification = classification.Invert() - return 1.0 - v[classification] - } - return v[classification] -} - -func AverageVectors(vectors ...Vectors) Vectors { - // Create a container - combined := NewConfidenceVectors() - - // Add up all the vectors - for _, other := range vectors { - for cls, vec := range other { - // Note: we don't need to check for inverted values here, because we store non-inverted values. - combined[cls] += vec - } - } - - // Average them - for cls, _ := range combined { - combined[cls] /= float64(len(vectors)) - } - - // Return the result - return combined -} diff --git a/filter/confidence/vectors_test.go b/filter/confidence/vectors_test.go deleted file mode 100644 index 8769119..0000000 --- a/filter/confidence/vectors_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package confidence - -import ( - "fmt" - "testing" - - "github.com/matrix-org/policyserv/filter/classification" - "github.com/stretchr/testify/assert" -) - -func TestGetSetVectors(t *testing.T) { - v := NewConfidenceVectors() - - // Set some values to ensure the map can be used right away - v.SetVector(classification.Spam, 1.0) - v.SetVector(classification.Mentions, 0.8) - v.SetVector(classification.Frequency, 0.2) - v.SetVector(classification.NonCompliance, 0.0) - v.SetVector(classification.Volumetric.Invert(), 1.0) - - // Make sure those values were stored - assert.Equal(t, 1.0, v.GetVector(classification.Spam), "spam should be 1.0") - assert.Equal(t, 0.8, v.GetVector(classification.Mentions), "mentions should be 0.8") - assert.Equal(t, 0.2, v.GetVector(classification.Frequency), "frequency should be 0.2") - assert.Equal(t, 0.0, v.GetVector(classification.NonCompliance), "non_compliance should be 0.0") - assert.Equal(t, 0.0, v.GetVector(classification.Volumetric), "volumetric should be 0.0") - assert.Equal(t, 1.0, v.GetVector(classification.Volumetric.Invert()), "inverted_volumetric should be 1.0") - - // Overwrite a value and ensure it gets read back fine - v.SetVector(classification.Spam, 0.2) - assert.Equal(t, 0.2, v.GetVector(classification.Spam), "overwritten spam should be 0.2") - - // Get a value that hasn't been set - assert.Equal(t, 0.0, v.GetVector(classification.Volumetric), "unset value should be 0.0") -} - -func TestSetVectorRangeTooHigh(t *testing.T) { - vals := []float64{1.1, 2.0, 1.00000000001, -0.1, -1.0, -0.00000000001} - for _, val := range vals { - func(val float64) { - v := NewConfidenceVectors() - defer func() { - if r := recover(); r != nil { - if err, ok := r.(error); ok { - assert.ErrorContains(t, err, "vector out of range", fmt.Sprintf("expected out of range on %f", val)) - } else { - t.Errorf("Unexpected panic on %f: %#v %s", val, r, r) - } - } else { - t.Errorf("Expected panic on %f", val) - } - }() - v.SetVector(classification.Spam, val) - }(val) - } -} - -func TestAverageVectors(t *testing.T) { - v1 := NewConfidenceVectors() - v1.SetVector(classification.Spam, 1.0) - - v2 := NewConfidenceVectors() - v2.SetVector(classification.Spam, 1.0) - v2.SetVector(classification.Mentions, 0.5) - v2.SetVector(classification.Frequency.Invert(), 0.75) - - avg := AverageVectors(v1, v2) - assert.Equal(t, 1.0, avg.GetVector(classification.Spam), "spam should be 1.0") - assert.Equal(t, 0.25, avg.GetVector(classification.Mentions), "mentions should be 0.25") - assert.Equal(t, 0.125, avg.GetVector(classification.Frequency), "frequency should be 0.125") -} diff --git a/filter/filter_ai_executor.go b/filter/filter_ai_executor.go index 7821416..e3a348e 100644 --- a/filter/filter_ai_executor.go +++ b/filter/filter_ai_executor.go @@ -5,7 +5,7 @@ import ( "slices" "github.com/matrix-org/policyserv/ai" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" ) type InstancedAIExecutorFilter[ConfigT any] struct { @@ -30,9 +30,9 @@ func (f *InstancedAIExecutorFilter[ConfigT]) Name() string { return f.name } -func (f *InstancedAIExecutorFilter[ConfigT]) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedAIExecutorFilter[ConfigT]) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if !slices.Contains(f.inRoomIds, input.Event.RoomID().String()) { - return nil, nil // this filter isn't allowed to execute in here + return harms.NeutralContent(), nil // this filter isn't allowed to execute in here } return f.aiProvider.CheckEvent(ctx, f.config, &ai.Input{ Event: input.Event, diff --git a/filter/filter_ai_executor_test.go b/filter/filter_ai_executor_test.go index 8da5f40..d329c02 100644 --- a/filter/filter_ai_executor_test.go +++ b/filter/filter_ai_executor_test.go @@ -7,7 +7,7 @@ import ( "github.com/matrix-org/policyserv/ai" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" ) @@ -22,11 +22,11 @@ type TestAIProvider[ConfigT any] struct { T *testing.T Called bool ExpectedConfig ConfigT - Return []classification.Classification + Return *harms.ContentInfo ReturnErr error } -func (p *TestAIProvider[ConfigT]) CheckEvent(ctx context.Context, cnf ConfigT, input *ai.Input) ([]classification.Classification, error) { +func (p *TestAIProvider[ConfigT]) CheckEvent(ctx context.Context, cnf ConfigT, input *ai.Input) (*harms.ContentInfo, error) { assert.NotNil(p.T, ctx, "context is required") assert.NotNil(p.T, cnf, "config is required") assert.NotNil(p.T, input, "input is required") @@ -51,6 +51,7 @@ func TestInstancedAIExecutorFilter(t *testing.T) { provider := &TestAIProvider[*arbitraryConfig]{ T: t, ExpectedConfig: &arbitraryConfig{SomeVal: true}, + Return: harms.NeutralContent(), } set := &Set{ communityConfig: &config.CommunityConfig{}, @@ -70,9 +71,9 @@ func TestInstancedAIExecutorFilter(t *testing.T) { "msgtype": "m.text", }, }) - vecs, err := instance.CheckEvent(ctx, &EventInput{Event: event}) + info, err := instance.CheckEvent(ctx, &EventInput{Event: event}) assert.NoError(t, err) - assert.Equal(t, 0, len(vecs)) + test.AssertEqualContentInfo(t, harms.NeutralContent(), info) assert.False(t, provider.Called) // Ensure the AI Provider is called for rooms it's supposed to @@ -86,9 +87,9 @@ func TestInstancedAIExecutorFilter(t *testing.T) { "msgtype": "m.text", }, }) - vecs, err = instance.CheckEvent(ctx, &EventInput{Event: event}) + info, err = instance.CheckEvent(ctx, &EventInput{Event: event}) assert.NoError(t, err) - assert.Equal(t, 0, len(vecs)) + test.AssertEqualContentInfo(t, harms.NeutralContent(), info) assert.True(t, provider.Called) // Ensure errors are passed through @@ -98,10 +99,10 @@ func TestInstancedAIExecutorFilter(t *testing.T) { assert.Equal(t, retErr, err) // Ensure classifications are passed through - ret := []classification.Classification{classification.Spam, classification.Volumetric} + ret := harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding) provider.Return = ret provider.ReturnErr = nil - vecs, err = instance.CheckEvent(ctx, &EventInput{Event: event}) + info, err = instance.CheckEvent(ctx, &EventInput{Event: event}) assert.NoError(t, err) - assert.Equal(t, ret, vecs) + test.AssertEqualContentInfo(t, ret, info) } diff --git a/filter/filter_density.go b/filter/filter_density.go index 6d8ccb5..b75825a 100644 --- a/filter/filter_density.go +++ b/filter/filter_density.go @@ -7,7 +7,7 @@ import ( "log" "regexp" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -40,13 +40,12 @@ func (f *InstancedDensityFilter) Name() string { return DensityFilterName } -func (f *InstancedDensityFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedDensityFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { eventId := input.Event.EventID() roomId := input.Event.RoomID().String() if input.Event.Type() != "m.room.message" { - // no-op and return the same vectors - return nil, nil + return harms.NeutralContent(), nil } content := &bodyOnly{} @@ -59,14 +58,14 @@ func (f *InstancedDensityFilter) CheckEvent(ctx context.Context, input *EventInp return f.checkTextWithLogging(ctx, content.Body, fmt.Sprintf("%s | %s", eventId, roomId)) } -func (f *InstancedDensityFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedDensityFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { return f.checkTextWithLogging(ctx, text, "CheckText") } -func (f *InstancedDensityFilter) checkTextWithLogging(ctx context.Context, text string, logPrefix string) ([]classification.Classification, error) { +func (f *InstancedDensityFilter) checkTextWithLogging(ctx context.Context, text string, logPrefix string) (*harms.ContentInfo, error) { if len(text) < f.minTriggerLength { // no-op - return nil, nil + return harms.NeutralContent(), nil } beforeTrim := float64(len(text)) @@ -76,13 +75,10 @@ func (f *InstancedDensityFilter) checkTextWithLogging(ctx context.Context, text log.Printf("[%s] Density is %f", logPrefix, density) if density >= f.maxDensity { - return []classification.Classification{ - classification.Spam, - classification.Volumetric, - }, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } type bodyOnly struct { diff --git a/filter/filter_density_test.go b/filter/filter_density_test.go index 7b2cd46..ddf5390 100644 --- a/filter/filter_density_test.go +++ b/filter/filter_density_test.go @@ -1,16 +1,13 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" - "github.com/tidwall/gjson" ) func TestDensityFilter(t *testing.T) { @@ -20,9 +17,8 @@ func TestDensityFilter(t *testing.T) { DensityFilterMinTriggerLength: internal.Pointer(10), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{DensityFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{DensityFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -82,43 +78,10 @@ func TestDensityFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) - assertSpamVector(noopEvent1, false) - assertSpamVector(noopEvent2, false) - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := gjson.Get(string(event.Content()), "body").String() - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertTextSpamVector(spammyEvent1, true) - assertTextSpamVector(spammyEvent2, true) - assertTextSpamVector(neutralEvent1, false) - assertTextSpamVector(neutralEvent2, false) - //assertTextSpamVector(noopEvent1, false) // text doesn't have a concept of event types, so skip this one - assertTextSpamVector(noopEvent2, false) + AssertCheckTextAndEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckTextAndEvent(t, set, neutralEvent2, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) // text doesn't have a concept of event types, so only check events + AssertCheckTextAndEvent(t, set, noopEvent2, harms.NeutralContent()) } diff --git a/filter/filter_event_type.go b/filter/filter_event_type.go index 2954929..1b401e2 100644 --- a/filter/filter_event_type.go +++ b/filter/filter_event_type.go @@ -3,7 +3,7 @@ package filter import ( "context" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -34,7 +34,7 @@ func (f *InstancedEventTypeFilter) Name() string { return EventTypeFilterName } -func (f *InstancedEventTypeFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedEventTypeFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { eventTypeSet := f.allowedEventTypes if input.Event.StateKey() != nil { eventTypeSet = f.allowedStateEventTypes @@ -42,9 +42,9 @@ func (f *InstancedEventTypeFilter) CheckEvent(ctx context.Context, input *EventI for _, allowedType := range eventTypeSet { if allowedType == input.Event.Type() { - return []classification.Classification{classification.Spam.Invert()}, nil // we expect to be run as a prefilter, so explicitly return not spam + return harms.AllowedContent(), nil } } - return nil, nil // no opinions when allow-listed + return harms.NeutralContent(), nil // no opinions when not explicitly allowed } diff --git a/filter/filter_event_type_test.go b/filter/filter_event_type_test.go index b657acb..7017f24 100644 --- a/filter/filter_event_type_test.go +++ b/filter/filter_event_type_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" ) @@ -18,9 +16,8 @@ func TestEventTypeFilter(t *testing.T) { EventTypePrefilterAllowedStateEventTypes: &[]string{"m.room.topic"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{EventTypeFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{EventTypeFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -68,18 +65,9 @@ func TestEventTypeFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - // Because the filter doesn't flag things as "spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } else { - assert.Equal(t, 0.0, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) + // This filter emits either Neutral or Allowed, so spam becomes Neutral + AssertCheckEvent(t, set, spammyEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, spammyEvent2, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent1, harms.AllowedContent()) + AssertCheckEvent(t, set, neutralEvent2, harms.AllowedContent()) } diff --git a/filter/filter_fixed_test.go b/filter/filter_fixed_test.go index 31b0e00..d523eab 100644 --- a/filter/filter_fixed_test.go +++ b/filter/filter_fixed_test.go @@ -5,7 +5,7 @@ import ( "errors" "testing" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/stretchr/testify/assert" ) @@ -30,19 +30,19 @@ func (f *FixedCanBeInstancedFilter) MakeFor(set *Set) (Instanced, error) { // FixedInstancedFilter - An Instanced filter which exists for testing purposes type FixedInstancedFilter struct { - T *testing.T - Set *Set - Expect *EventInput - ExpectText string - ReturnClasses []classification.Classification - ReturnErr error + T *testing.T + Set *Set + Expect *EventInput + ExpectText string + ReturnInfo *harms.ContentInfo + ReturnErr error } func (f *FixedInstancedFilter) Name() string { return FixedFilterName } -func (f *FixedInstancedFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *FixedInstancedFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { assert.NotNil(f.T, ctx, "context is required") if f.Expect != nil { @@ -59,17 +59,17 @@ func (f *FixedInstancedFilter) CheckEvent(ctx context.Context, input *EventInput assert.Equal(f.T, f.Expect, input) } - return f.ReturnClasses, f.ReturnErr + return f.ReturnInfo, f.ReturnErr } -func (f *FixedInstancedFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *FixedInstancedFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { assert.NotNil(f.T, ctx, "context is required") if f.ExpectText != "" { assert.Equal(f.T, f.ExpectText, text) } - return f.ReturnClasses, f.ReturnErr + return f.ReturnInfo, f.ReturnErr } type ErrorFilter struct { diff --git a/filter/filter_frequency.go b/filter/filter_frequency.go index a07470f..e53d379 100644 --- a/filter/filter_frequency.go +++ b/filter/filter_frequency.go @@ -7,8 +7,8 @@ import ( "log" "time" - "github.com/matrix-org/policyserv/filter/classification" "github.com/matrix-org/policyserv/frequency" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -50,7 +50,7 @@ func (f *InstancedFrequencyFilter) Close() error { return f.counter.Close() } -func (f *InstancedFrequencyFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedFrequencyFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { doProcess := false for _, t := range f.eventTypes { if t == input.Event.Type() { @@ -59,7 +59,7 @@ func (f *InstancedFrequencyFilter) CheckEvent(ctx context.Context, input *EventI } } if !doProcess { - return nil, nil // no opinion + return harms.NeutralContent(), nil // no opinion } // First, increment the counter for this user @@ -76,11 +76,7 @@ func (f *InstancedFrequencyFilter) CheckEvent(ctx context.Context, input *EventI rate := float64(eventsLastMinute+1) / float64(60) log.Printf("[%s | %s] Rate for user %s is %f (limit: %f)", input.Event.EventID(), input.Event.RoomID().String(), input.Event.SenderID(), rate, f.rateLimit) if rate > f.rateLimit { - return []classification.Classification{ - classification.Spam, - classification.Frequency, - }, nil - } else { - return nil, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } + return harms.NeutralContent(), nil } diff --git a/filter/filter_frequency_test.go b/filter/filter_frequency_test.go index cb11aa9..7c22184 100644 --- a/filter/filter_frequency_test.go +++ b/filter/filter_frequency_test.go @@ -1,12 +1,11 @@ package filter import ( - "context" "testing" "time" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/storage" "github.com/matrix-org/policyserv/test" @@ -23,9 +22,8 @@ func TestFrequencyFilter(t *testing.T) { FrequencyFilterRateLimit: internal.Pointer(1.0 / 60.0), // 1 message per minute }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FrequencyFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FrequencyFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -66,26 +64,17 @@ func TestFrequencyFilter(t *testing.T) { // No-op events shouldn't affect frequency. Because our rate limit is a single event, if this is handled improperly // then the next event will fail its test. - vecs, err := set.CheckEvent(context.Background(), noopEvent1, nil) - assert.NoError(t, err) - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) // original value survives due to "no opinion" - assert.Equal(t, 0.0, vecs.GetVector(classification.Frequency)) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) // Now we send an event that's in scope, but is technically going to be neutral. This is because we increment at a // different point, so we might "miss" the first spammy event. - vecs, err = set.CheckEvent(context.Background(), spammyEvent1, nil) - assert.NoError(t, err) - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) // original value survives due to "no opinion" - assert.Equal(t, 0.0, vecs.GetVector(classification.Frequency)) + AssertCheckEvent(t, set, spammyEvent1, harms.NeutralContent()) // Give a little bit of time for the notifier to settle time.Sleep(100 * time.Millisecond) // Now try to send another event that's in scope. This time it should exceed the rate limit as spam. - vecs, err = set.CheckEvent(context.Background(), spammyEvent2, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Frequency)) + AssertCheckEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamFlooding)) // Allow the goroutines to settle before concluding the test time.Sleep(100 * time.Millisecond) diff --git a/filter/filter_hellban.go b/filter/filter_hellban.go index 1befdc3..67d05fa 100644 --- a/filter/filter_hellban.go +++ b/filter/filter_hellban.go @@ -9,7 +9,7 @@ import ( "time" cache "github.com/Code-Hex/go-generics-cache" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/pubsub" ) @@ -116,7 +116,7 @@ func (f *InstancedHellbanFilter) Name() string { return mode } -func (f *InstancedHellbanFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedHellbanFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { mode := f.Name() eventId := input.Event.EventID() @@ -127,27 +127,21 @@ func (f *InstancedHellbanFilter) CheckEvent(ctx context.Context, input *EventInp if mode == HellbanPrefilterName { if _, ok := f.userIdsCache.Get(senderUserId); ok { log.Printf("[%s | %s | %s] Sender '%s' is hellbanned", eventId, roomId, mode, senderUserId) - return []classification.Classification{ - classification.Spam, - classification.Frequency, - }, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } } else { - if f.set.IsSpamResponse(ctx, input.IncrementalConfidenceVectors) { - log.Printf("[%s | %s | %s] Sender '%s' sent a spammy event", eventId, roomId, mode, senderUserId) - err := f.set.pubsub.Publish(ctx, pubsub.TopicHellban, mustEncodeHellban(f.set.communityId, senderUserId)) - if err != nil { - return nil, err - } - return []classification.Classification{ - classification.Spam, - classification.Frequency, - }, nil + // The community manager/filter set group will only call this filter if the event was prohibited, so we can + // safely make the assumption that the event is spammy. + log.Printf("[%s | %s | %s] Sender '%s' sent a spammy event", eventId, roomId, mode, senderUserId) + err := f.set.pubsub.Publish(ctx, pubsub.TopicHellban, mustEncodeHellban(f.set.communityId, senderUserId)) + if err != nil { + return nil, err } + return harms.ProhibitedContent(harms.SpamFlooding), nil } // If we reached here, the sender isn't spammy by our metrics. - return nil, nil + return harms.NeutralContent(), nil } func (f *InstancedHellbanFilter) Close() error { diff --git a/filter/filter_hellban_test.go b/filter/filter_hellban_test.go index 0cd04e1..d0ceed9 100644 --- a/filter/filter_hellban_test.go +++ b/filter/filter_hellban_test.go @@ -5,10 +5,8 @@ import ( "testing" "time" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/media" "github.com/matrix-org/policyserv/pubsub" @@ -44,9 +42,8 @@ func TestHellbanPrefilterDoesntEternallyExtend(t *testing.T) { CommunityId: "TestHellbanPrefilterDoesntEternallyExtend", CommunityConfig: &config.CommunityConfig{}, Groups: []*SetGroupConfig{{ - EnabledNames: []string{fastHellbanPrefilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{fastHellbanPrefilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -83,10 +80,7 @@ func TestHellbanPrefilterDoesntEternallyExtend(t *testing.T) { }, }) - vecs, err := set.CheckEvent(context.Background(), neutralEvent1, nil) - assert.NoError(t, err) - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) } func TestHellbanPrefilter(t *testing.T) { @@ -98,9 +92,8 @@ func TestHellbanPrefilter(t *testing.T) { HellbanPostfilterMinutes: internal.Pointer(10), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{HellbanPrefilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{HellbanPrefilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -147,37 +140,22 @@ func TestHellbanPrefilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Frequency)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) } func TestHellbanPostfilter(t *testing.T) { ctx := context.Background() cnf := &SetConfig{ - CommunityId: "TestHellbanPostfilter", - CommunityConfig: &config.CommunityConfig{ - SpamThreshold: internal.Pointer(0.8), // set to a value where the hellban filter will detect the fixed filter's output as spam - }, + CommunityId: "TestHellbanPostfilter", + CommunityConfig: &config.CommunityConfig{}, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything starts as neutral by default in the test }, { - EnabledNames: []string{HellbanPostfilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{HellbanPostfilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // but we only want to detect spam for the postfilter test }}, } memStorage := test.NewMemoryStorage(t) @@ -216,45 +194,30 @@ func TestHellbanPostfilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - fixedFilter.Expect = &EventInput{ - Event: event, - Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, - } - if isSpam { - fixedFilter.ReturnClasses = []classification.Classification{classification.Spam} - } else { - fixedFilter.ReturnClasses = []classification.Classification{classification.Spam.Invert()} - } - - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Frequency)) - } else { - assert.Equal(t, 0.0, vecs.GetVector(classification.Spam)) - } - - if isSpam { - select { - case recv := <-subCh: - assert.Equal(t, mustEncodeHellban(cnf.CommunityId, spammerUserId), recv) - case <-time.After(5 * time.Second): // if after a little bit we haven't heard anything, fail - assert.Fail(t, "didn't receive a subscription event") - } - } else { - select { - case <-subCh: - assert.Fail(t, "should not have received a subscription event") - case <-time.After(1 * time.Second): // we use 1 second for the same reason as the prefilter above - // passing case - we want this to happen - } - } + // Check the spammy event first (should cause a hellban) + fixedFilter.Expect = &EventInput{ + Event: spammyEvent1, + Medias: make([]*media.Item, 0), + } + fixedFilter.ReturnInfo = harms.ProhibitedContent(harms.SpamGeneral) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding)) + select { + case recv := <-subCh: + assert.Equal(t, mustEncodeHellban(cnf.CommunityId, spammerUserId), recv) + case <-time.After(5 * time.Second): // if after a little bit we haven't heard anything, fail + assert.Fail(t, "didn't receive a subscription event") + } + + // Neutral events shouldn't cause a hellban + fixedFilter.Expect.Event = neutralEvent1 + fixedFilter.ReturnInfo = harms.NeutralContent() + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) + select { + case <-subCh: + assert.Fail(t, "should not have received a subscription event") + case <-time.After(1 * time.Second): // we use 1 second for the same reason as the prefilter above + // passing case - we want this to happen } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) } func TestHellbanFiltersCombined(t *testing.T) { @@ -264,20 +227,16 @@ func TestHellbanFiltersCombined(t *testing.T) { CommunityId: "TestHellbanFiltersCombined", CommunityConfig: &config.CommunityConfig{ HellbanPostfilterMinutes: internal.Pointer(10), - SpamThreshold: internal.Pointer(0.1), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{HellbanPrefilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{HellbanPrefilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything starts as neutral by default in the test }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // for later: don't run on spammy events }, { - EnabledNames: []string{HellbanPostfilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{HellbanPostfilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // *only* run on spammy events }}, } memStorage := test.NewMemoryStorage(t) @@ -323,18 +282,14 @@ func TestHellbanFiltersCombined(t *testing.T) { // Step 2 prep fixedFilter.Expect = &EventInput{ - Event: spammyEvent1, - Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, // just the starting value + Event: spammyEvent1, + Medias: make([]*media.Item, 0), } - // We set a Mentions classification so we can detect that the filter ran - fixedFilter.ReturnClasses = []classification.Classification{classification.Spam, classification.Mentions} + // We set a media harm so we can detect that the filter ran + fixedFilter.ReturnInfo = harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia) // Invoke steps 1 through 3 - vecs, err := set.CheckEvent(ctx, spammyEvent1, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Mentions)) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding, harms.PolicyservMedia)) select { case recv := <-subCh: assert.Equal(t, mustEncodeHellban(cnf.CommunityId, spammerUserId), recv) @@ -349,16 +304,11 @@ func TestHellbanFiltersCombined(t *testing.T) { fixedFilter.Expect = &EventInput{ Event: spammyEvent1, Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{ - classification.Spam: 1.0, - classification.Frequency: 1.0, - }, } // Invoke steps 5 and 6 (step 4 is implied by the high spam figure) - vecs, err = set.CheckEvent(ctx, spammyEvent1, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) + // Note: "media" shouldn't appear here because of the RunOnClasses config. We expect Flooding from the prefilter. + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) // Step 7 states that we should (probably) get a subscription event select { diff --git a/filter/filter_inline_emoji_size.go b/filter/filter_inline_emoji_size.go index 8faf108..6fff7d2 100644 --- a/filter/filter_inline_emoji_size.go +++ b/filter/filter_inline_emoji_size.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/matrix-org/policyserv/event" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "golang.org/x/net/html" ) @@ -47,7 +47,7 @@ func (i *InstancedInlineEmojiSizeFilter) Name() string { return InlineEmojiSizeFilterName } -func (i *InstancedInlineEmojiSizeFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (i *InstancedInlineEmojiSizeFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { htmlRepresentations, err := event.ExtractHtmlRepresentations(input.Event) if err != nil { return nil, err @@ -96,7 +96,7 @@ func (i *InstancedInlineEmojiSizeFilter) CheckEvent(ctx context.Context, input * if strings.ToLower(string(key)) == "height" { if alreadySawHeight { log.Printf("[%s | %s] Multiple height attributes found on inline image", input.Event.EventID(), input.Event.RoomID().String()) - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance), nil } alreadySawHeight = true @@ -104,7 +104,7 @@ func (i *InstancedInlineEmojiSizeFilter) CheckEvent(ctx context.Context, input * if !sizeRegex.MatchString(size) { // We don't log the size we saw because it's potentially spammy log.Printf("[%s | %s] Invalid height attribute on inline image", input.Event.EventID(), input.Event.RoomID().String()) - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance), nil } // Parse the integer @@ -112,13 +112,13 @@ func (i *InstancedInlineEmojiSizeFilter) CheckEvent(ctx context.Context, input * if err != nil { // Don't log the size we saw because it's potentially spammy log.Printf("[%s | %s] Failed to parse height attribute on inline image", input.Event.EventID(), input.Event.RoomID().String()) - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance), nil } if height > i.maxHeightPixels { // Don't log the size we saw because it's potentially spammy log.Printf("[%s | %s] Height attribute on inline image is too large", input.Event.EventID(), input.Event.RoomID().String()) - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance), nil } } if !moreAttrs { @@ -128,11 +128,11 @@ func (i *InstancedInlineEmojiSizeFilter) CheckEvent(ctx context.Context, input * // If the tag isn't an emoji or we didn't see a height attribute, flag as spam if !isEmoticon || !alreadySawHeight { - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance), nil } } } } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_inline_emoji_size_test.go b/filter/filter_inline_emoji_size_test.go index 10c02f4..fe2a21d 100644 --- a/filter/filter_inline_emoji_size_test.go +++ b/filter/filter_inline_emoji_size_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -212,9 +212,8 @@ func TestInlineEmojiSizeFilter(t *testing.T) { InlineEmojiSizeFilterMaxHeightPixels: internal.Pointer(inlineEmojiSizeFilterMaxPixels), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{InlineEmojiSizeFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{InlineEmojiSizeFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -240,13 +239,10 @@ func TestInlineEmojiSizeFilter(t *testing.T) { }) t.Log("Testing event with HTML:", tc.html) - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) if tc.isSpammy { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) + AssertCheckEvent(t, set, event, harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance)) } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) + AssertCheckEvent(t, set, event, harms.NeutralContent()) } }) } @@ -258,9 +254,8 @@ func TestInlineEmojiSizeFilterRejectsInvalidHtml(t *testing.T) { InlineEmojiSizeFilterMaxHeightPixels: internal.Pointer(inlineEmojiSizeFilterMaxPixels), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{InlineEmojiSizeFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{InlineEmojiSizeFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) diff --git a/filter/filter_keyword_template.go b/filter/filter_keyword_template.go index 2b05858..115372a 100644 --- a/filter/filter_keyword_template.go +++ b/filter/filter_keyword_template.go @@ -10,7 +10,7 @@ import ( "time" "github.com/matrix-org/policyserv/event" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/pslib" ) @@ -64,10 +64,9 @@ func (f *InstancedKeywordTemplateFilter) Name() string { return KeywordTemplateFilterName } -func (f *InstancedKeywordTemplateFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedKeywordTemplateFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if input.Event.Type() != "m.room.message" { - // no-op and return the same vectors - return nil, nil + return harms.NeutralContent(), nil } var toScan string @@ -87,12 +86,12 @@ func (f *InstancedKeywordTemplateFilter) CheckEvent(ctx context.Context, input * return f.checkTextWithLogging(ctx, toScan, fmt.Sprintf("%s | %s", input.Event.EventID(), input.Event.RoomID().String())) } -func (f *InstancedKeywordTemplateFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedKeywordTemplateFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { return f.checkTextWithLogging(ctx, text, "CheckText") } -func (f *InstancedKeywordTemplateFilter) checkTextWithLogging(ctx context.Context, text string, logPrefix string) ([]classification.Classification, error) { - harms := make([]string, 0) +func (f *InstancedKeywordTemplateFilter) checkTextWithLogging(ctx context.Context, text string, logPrefix string) (*harms.ContentInfo, error) { + harmIds := make([]harms.Harm, 0) for _, tmpl := range f.templates { log.Printf("[%s] Checking template '%s'", logPrefix, tmpl.Name) returnedHarms, err := tmpl.IdentifyHarms(text) @@ -101,15 +100,16 @@ func (f *InstancedKeywordTemplateFilter) checkTextWithLogging(ctx context.Contex } if len(returnedHarms) > 0 { log.Printf("[%s] Template '%s' matched: %v", logPrefix, tmpl.Name, returnedHarms) - harms = append(harms, returnedHarms...) + for _, id := range returnedHarms { + harmIds = append(harmIds, harms.Harm(id)) + } } else { log.Printf("[%s] Template '%s' matched nothing", logPrefix, tmpl.Name) } } - if len(harms) > 0 { - // Our classification system doesn't (yet?) support MSC4387 harms, so just return "spam" - return []classification.Classification{classification.Spam}, nil + if len(harmIds) > 0 { + return harms.ProhibitedContent(harmIds...), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_keyword_template_test.go b/filter/filter_keyword_template_test.go index a683046..135df09 100644 --- a/filter/filter_keyword_template_test.go +++ b/filter/filter_keyword_template_test.go @@ -4,14 +4,12 @@ import ( "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/storage" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" - "github.com/tidwall/gjson" ) func TestKeywordTemplateFilter(t *testing.T) { @@ -24,9 +22,8 @@ func TestKeywordTemplateFilter(t *testing.T) { KeywordTemplateFilterTemplateNames: &[]string{"example", "this_one_doesnt_exist_but_thats_okay"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{KeywordTemplateFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{KeywordTemplateFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -46,7 +43,7 @@ func TestKeywordTemplateFilter(t *testing.T) { {{ range $bodyWord := .BodyWords }} {{ $bodyWord := $bodyWord | RemovePunctuation | ToLower }} {{ if StrSliceContains $badWords $bodyWord }} - org.matrix.msc4387.spam + org.example.keyword.spam {{ end }} {{ end }} {{ $rawBody := .BodyRaw | ToUpper }} @@ -54,7 +51,7 @@ func TestKeywordTemplateFilter(t *testing.T) { {{ range $badWord := $badWords }} {{ if StringContains $rawBody $badWord }} {{/* use "flooding" for variety, and so we can see which of these branches activated */}} - org.matrix.msc4387.spam.flooding + org.example.keyword.spam.flooding {{ end }} {{ end }} `, @@ -99,38 +96,20 @@ func TestKeywordTemplateFilter(t *testing.T) { "body": "this is not spam", // doesn't use `badWords` }, }) + noopEvent := test.MustMakePDU(&test.BaseClientEvent{ + EventId: "$noop", + RoomId: "!foo:example.org", + Type: "org.example.wrong_event_type_for_filter", + Content: map[string]any{ + "body": "doesn't matter", + }, + }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(spammyEvent3, true) - assertSpamVector(neutralEvent, false) - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := gjson.Get(string(event.Content()), "body").String() - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertTextSpamVector(spammyEvent1, true) - assertTextSpamVector(spammyEvent2, true) - //assertTextSpamVector(spammyEvent3, true) // this tests HTML, which we're not extracting here, so skip it - assertTextSpamVector(neutralEvent, false) + AssertCheckTextAndEvent(t, set, spammyEvent1, harms.ProhibitedContent("org.example.keyword.spam", "org.example.keyword.spam.flooding")) + AssertCheckTextAndEvent(t, set, spammyEvent2, harms.ProhibitedContent("org.example.keyword.spam", "org.example.keyword.spam.flooding")) + AssertCheckEvent(t, set, spammyEvent3, harms.ProhibitedContent("org.example.keyword.spam", "org.example.keyword.spam.flooding")) // skip text check because it's HTML and we're not extracting that + AssertCheckTextAndEvent(t, set, neutralEvent, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent, harms.NeutralContent()) // skip text because text isn't aware of event types } func TestKeywordTemplateFilterWithFullEvent(t *testing.T) { @@ -140,9 +119,8 @@ func TestKeywordTemplateFilterWithFullEvent(t *testing.T) { KeywordTemplateFilterUseFullEvent: internal.Pointer(true), // this is what we're testing }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{KeywordTemplateFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{KeywordTemplateFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -155,7 +133,7 @@ func TestKeywordTemplateFilterWithFullEvent(t *testing.T) { Body: ` {{/* In a real filter, we wouldn't be doing simple "contains" checks. */}} {{ if StringContains .BodyRaw "user_id_has_the_keyword_instead" }} - org.matrix.msc4387.spam + org.example.keyword.spam {{ end }} `, }) @@ -175,7 +153,5 @@ func TestKeywordTemplateFilterWithFullEvent(t *testing.T) { }, }) - vecs, err := set.CheckEvent(context.Background(), spammyEvent, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) + AssertCheckEvent(t, set, spammyEvent, harms.ProhibitedContent("org.example.keyword.spam")) } diff --git a/filter/filter_keywords.go b/filter/filter_keywords.go index ef03e7e..270e4df 100644 --- a/filter/filter_keywords.go +++ b/filter/filter_keywords.go @@ -4,7 +4,7 @@ import ( "context" "strings" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -35,7 +35,7 @@ func (f *InstancedKeywordFilter) Name() string { return KeywordFilterName } -func (f *InstancedKeywordFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedKeywordFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { toScan := string(input.Event.Content()) if f.useFullEvent { toScan = string(input.Event.JSON()) @@ -43,12 +43,12 @@ func (f *InstancedKeywordFilter) CheckEvent(ctx context.Context, input *EventInp return f.CheckText(ctx, toScan) } -func (f *InstancedKeywordFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedKeywordFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { for _, k := range f.keywords { if strings.Contains(text, k) { - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral), nil } } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_keywords_test.go b/filter/filter_keywords_test.go index 544781e..34fea1d 100644 --- a/filter/filter_keywords_test.go +++ b/filter/filter_keywords_test.go @@ -1,16 +1,13 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" - "github.com/tidwall/gjson" ) func TestKeywordsFilter(t *testing.T) { @@ -21,9 +18,8 @@ func TestKeywordsFilter(t *testing.T) { KeywordFilterKeywords: &[]string{"spammy spam", "example"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{KeywordFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{KeywordFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -59,35 +55,9 @@ func TestKeywordsFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(neutralEvent, false) - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := gjson.Get(string(event.Content()), "body").String() - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertTextSpamVector(spammyEvent1, true) - assertTextSpamVector(spammyEvent2, true) - assertTextSpamVector(neutralEvent, false) + AssertCheckTextAndEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamGeneral)) + AssertCheckTextAndEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamGeneral)) + AssertCheckTextAndEvent(t, set, neutralEvent, harms.NeutralContent()) } func TestKeywordsFilterWithFullEvent(t *testing.T) { @@ -97,9 +67,8 @@ func TestKeywordsFilterWithFullEvent(t *testing.T) { KeywordFilterUseFullEvent: internal.Pointer(true), // this is what we're testing }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{KeywordFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{KeywordFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -120,7 +89,5 @@ func TestKeywordsFilterWithFullEvent(t *testing.T) { }, }) - vecs, err := set.CheckEvent(context.Background(), spammyEvent, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) + AssertCheckEvent(t, set, spammyEvent, harms.ProhibitedContent(harms.SpamGeneral)) } diff --git a/filter/filter_length.go b/filter/filter_length.go index 9ca64c8..ab9aba1 100644 --- a/filter/filter_length.go +++ b/filter/filter_length.go @@ -3,7 +3,7 @@ package filter import ( "context" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -32,23 +32,20 @@ func (f *InstancedLengthFilter) Name() string { return LengthFilterName } -func (f *InstancedLengthFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedLengthFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if input.Event.Type() != "m.room.message" { // not an event we're interested in - return nil, nil + return harms.NeutralContent(), nil } b := input.Event.JSON() return f.CheckText(ctx, string(b)) } -func (f *InstancedLengthFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedLengthFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { if len(text) > f.maxLength { - return []classification.Classification{ - classification.Spam, - classification.Volumetric, - }, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_length_test.go b/filter/filter_length_test.go index 9967d4c..3e9d2db 100644 --- a/filter/filter_length_test.go +++ b/filter/filter_length_test.go @@ -1,17 +1,14 @@ package filter import ( - "context" "strings" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" - "github.com/tidwall/gjson" ) func TestLengthFilter(t *testing.T) { @@ -22,9 +19,8 @@ func TestLengthFilter(t *testing.T) { LengthFilterMaxLength: internal.Pointer(300), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{LengthFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{LengthFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -60,37 +56,7 @@ func TestLengthFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(noopEvent1, false) - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := gjson.Get(string(event.Content()), "body").String() - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertTextSpamVector(spammyEvent1, true) - assertTextSpamVector(neutralEvent1, false) - //assertTextSpamVector(noopEvent1, false) // text doesn't have a concept of event types, so skip this one + AssertCheckTextAndEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) // text doesn't have a concept of event types, so only check events } diff --git a/filter/filter_links.go b/filter/filter_links.go index 252345b..3b17201 100644 --- a/filter/filter_links.go +++ b/filter/filter_links.go @@ -4,7 +4,7 @@ import ( "context" "regexp" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/ryanuber/go-glob" ) @@ -40,15 +40,15 @@ func (f *InstancedLinkFilter) Name() string { return LinkFilterName } -func (f *InstancedLinkFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedLinkFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { content := string(input.Event.Content()) return f.CheckText(ctx, content) } -func (f *InstancedLinkFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedLinkFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { // If neither list is configured, this filter has no opinion. if len(f.allowedUrlGlobs) == 0 && len(f.deniedUrlGlobs) == 0 { - return nil, nil + return harms.NeutralContent(), nil } // Find all of the URLs in the text @@ -56,16 +56,18 @@ func (f *InstancedLinkFilter) CheckText(ctx context.Context, text string) ([]cla // No URLs found, so nothing to check. if len(urls) == 0 { - return nil, nil + return harms.NeutralContent(), nil } for _, url := range urls { if !f.isUrlAllowed(url) { - return []classification.Classification{classification.Spam}, nil + // Links are probably phishing (SpamFraud), but we don't *really* know how people are using this filter + // to be confident enough to *always* return that. + return harms.ProhibitedContent(harms.SpamGeneral), nil } } - return nil, nil + return harms.NeutralContent(), nil } func (f *InstancedLinkFilter) isUrlAllowed(url string) bool { diff --git a/filter/filter_links_test.go b/filter/filter_links_test.go index 777d26b..39a078e 100644 --- a/filter/filter_links_test.go +++ b/filter/filter_links_test.go @@ -1,15 +1,12 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" - "github.com/tidwall/gjson" ) func TestLinkFilter(t *testing.T) { @@ -21,9 +18,8 @@ func TestLinkFilter(t *testing.T) { LinkFilterDeniedUrlGlobs: &[]string{"https://allowed.example.org/blocked/*"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{LinkFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{LinkFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -94,40 +90,11 @@ func TestLinkFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - - assertSpamVector(allowedEvent, false) - assertSpamVector(deniedEvent, true) // deny wins over allow - assertSpamVector(notAllowedEvent, true) - assertSpamVector(noUrlEvent, false) - assertSpamVector(mixedEvent, true) //contains a default-denied URL. - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := gjson.Get(string(event.Content()), "body").String() - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertTextSpamVector(allowedEvent, false) - assertTextSpamVector(deniedEvent, true) // deny wins over allow - assertTextSpamVector(notAllowedEvent, true) - assertTextSpamVector(noUrlEvent, false) - assertTextSpamVector(mixedEvent, true) //contains a default-denied URL. + AssertCheckTextAndEvent(t, set, allowedEvent, harms.NeutralContent()) + AssertCheckTextAndEvent(t, set, deniedEvent, harms.ProhibitedContent(harms.SpamGeneral)) // deny wins over allow + AssertCheckTextAndEvent(t, set, notAllowedEvent, harms.ProhibitedContent(harms.SpamGeneral)) + AssertCheckTextAndEvent(t, set, noUrlEvent, harms.NeutralContent()) + AssertCheckTextAndEvent(t, set, mixedEvent, harms.ProhibitedContent(harms.SpamGeneral)) // contains a default-denied URL. } func TestLinkFilterDenyListOnly(t *testing.T) { @@ -139,9 +106,8 @@ func TestLinkFilterDenyListOnly(t *testing.T) { LinkFilterDeniedUrlGlobs: &[]string{"*denied.example.org/path*"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{LinkFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{LinkFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -174,23 +140,6 @@ func TestLinkFilterDenyListOnly(t *testing.T) { }, }) - vecs, err := set.CheckEvent(context.Background(), deniedEvent, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - - vecs, err = set.CheckEvent(context.Background(), allowedEvent, nil) - assert.NoError(t, err) - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - - // Also test the text filter implementation - - body := gjson.Get(string(deniedEvent.Content()), "body").String() - vecs, err = set.CheckText(context.Background(), body) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - - body = gjson.Get(string(allowedEvent.Content()), "body").String() - vecs, err = set.CheckText(context.Background(), body) - assert.NoError(t, err) - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) + AssertCheckTextAndEvent(t, set, deniedEvent, harms.ProhibitedContent(harms.SpamGeneral)) + AssertCheckTextAndEvent(t, set, allowedEvent, harms.NeutralContent()) } diff --git a/filter/filter_many_ats.go b/filter/filter_many_ats.go index bb6f95b..1b27631 100644 --- a/filter/filter_many_ats.go +++ b/filter/filter_many_ats.go @@ -4,7 +4,7 @@ import ( "context" "strings" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -33,17 +33,14 @@ func (f *InstancedManyAtsFilter) Name() string { return ManyAtsFilterName } -func (f *InstancedManyAtsFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedManyAtsFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { return f.CheckText(ctx, string(input.Event.Content())) } -func (f *InstancedManyAtsFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedManyAtsFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { if strings.Count(text, "@") >= f.maxAts { - return []classification.Classification{ - classification.Spam, - classification.Mentions, - }, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_many_ats_test.go b/filter/filter_many_ats_test.go index 749edce..d2324bc 100644 --- a/filter/filter_many_ats_test.go +++ b/filter/filter_many_ats_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -15,12 +13,11 @@ import ( func TestManyAtsFilter(t *testing.T) { cnf := &SetConfig{ CommunityConfig: &config.CommunityConfig{ - ManyAtsFilterMaxAts: internal.Pointer(5), + ManyAtsFilterMaxAts: internal.Pointer(2), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{ManyAtsFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{ManyAtsFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -36,7 +33,20 @@ func TestManyAtsFilter(t *testing.T) { RoomId: "!foo:example.org", Type: "m.room.message", Content: map[string]any{ - "body": "@_@", // 2 ats + "body": "@_@", // 2 ats (enough to trip the text test) + "m.mentions": []string{ // 3 ats + "@user1:example.org", + "@user2:example.org", + "@user3:example.org", + }, + }, + }) + spammyEvent2 := test.MustMakePDU(&test.BaseClientEvent{ + EventId: "$spam1", + RoomId: "!foo:example.org", + Type: "m.room.message", + Content: map[string]any{ + "body": "test m.mentions", "m.mentions": []string{ // 3 ats "@user1:example.org", "@user2:example.org", @@ -49,39 +59,11 @@ func TestManyAtsFilter(t *testing.T) { RoomId: "!foo:example.org", Type: "m.room.message", Content: map[string]any{ - "body": "no ats", + "body": "one @", // 1 is less than 2 }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Mentions)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Mentions)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := string(event.Content()) // ideally we'd pull just the `body`, but our test uses `m.mentions` too - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Mentions)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Mentions)) - } - } - assertTextSpamVector(spammyEvent1, true) - assertTextSpamVector(neutralEvent1, false) + AssertCheckTextAndEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamFlooding)) // `m.mentions` isn't checked by text, so check event only + AssertCheckTextAndEvent(t, set, neutralEvent1, harms.NeutralContent()) } diff --git a/filter/filter_media.go b/filter/filter_media.go index 2c8aa0f..d7fc10e 100644 --- a/filter/filter_media.go +++ b/filter/filter_media.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -33,17 +33,17 @@ func (f *InstancedMediaFilter) Name() string { return MediaFilterName } -func (f *InstancedMediaFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedMediaFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { // Check event type (for stickers) first for _, mediaType := range f.mediaTypes { if mediaType == input.Event.Type() { - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia), nil } } // the msgtype check only applies to regular room messages, so return early if we can if input.Event.Type() != "m.room.message" { - return nil, nil + return harms.NeutralContent(), nil } content := &msgtypeOnly{} @@ -55,13 +55,11 @@ func (f *InstancedMediaFilter) CheckEvent(ctx context.Context, input *EventInput for _, mediaType := range f.mediaTypes { if mediaType == content.Msgtype { - return []classification.Classification{ - classification.Spam, - }, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia), nil } } - return nil, nil + return harms.NeutralContent(), nil } type msgtypeOnly struct { diff --git a/filter/filter_media_scanning.go b/filter/filter_media_scanning.go index d5993ff..50ef9ab 100644 --- a/filter/filter_media_scanning.go +++ b/filter/filter_media_scanning.go @@ -11,7 +11,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/content" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/media" "github.com/matrix-org/policyserv/storage" ) @@ -46,16 +46,16 @@ func (f *InstancedMediaScanningFilter) Name() string { return MediaScanningFilterName } -func (f *InstancedMediaScanningFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedMediaScanningFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if len(input.Medias) == 0 { - return nil, nil // return early to avoid doing work + return harms.NeutralContent(), nil // return early to avoid doing work } - retChans := make([]chan []classification.Classification, len(input.Medias)) + retChans := make([]chan *harms.ContentInfo, len(input.Medias)) // Create the channels & async the scanning work for i, m := range input.Medias { - ch := make(chan []classification.Classification, 1) + ch := make(chan *harms.ContentInfo, 1) retChans[i] = ch // scanMedia will close the channel when it's done - we don't need to do it here go f.scanMedia(ctx, input.Event, m, ch) @@ -66,31 +66,37 @@ func (f *InstancedMediaScanningFilter) CheckEvent(ctx context.Context, input *Ev defer cancel() // Collect all of the scan results, up to a timeout - classifications := make([]classification.Classification, 0) + contentClass := harms.ContentClassNeutral + harmIds := make([]harms.Harm, 0) for _, ch := range retChans { select { case <-readTimeout.Done(): log.Printf("[%s | %s] Media scanning timed out", input.Event.EventID(), input.Event.RoomID().String()) - return []classification.Classification{classification.Spam, classification.Unsafe}, nil - case subClassifications := <-ch: - if subClassifications != nil { - classifications = append(classifications, subClassifications...) + return harms.ProhibitedContent(harms.OtherGeneral), nil + case mediaInfo := <-ch: + if mediaInfo != nil { + if contentClass < mediaInfo.Class() { + contentClass = mediaInfo.Class() + } + for _, h := range mediaInfo.Harms() { + harmIds = append(harmIds, h) + } } } } // Return those results - return classifications, nil + return harms.NewContentInfo(contentClass, harmIds...), nil } -func (f *InstancedMediaScanningFilter) scanMedia(ctx context.Context, event gomatrixserverlib.PDU, media *media.Item, ch chan<- []classification.Classification) { +func (f *InstancedMediaScanningFilter) scanMedia(ctx context.Context, event gomatrixserverlib.PDU, media *media.Item, ch chan<- *harms.ContentInfo) { cached, err := f.set.storage.GetMediaClassification(ctx, media.String(), f.set.communityId) if err != nil && !errors.Is(err, sql.ErrNoRows) { log.Printf("[%s | %s] Non-fatal error getting cached media classification: %s", event.EventID(), event.RoomID().String(), err) } if err == nil { log.Printf("[%s | %s] Using cached media classification for %s (%v)", event.EventID(), event.RoomID().String(), media, cached.Classifications) - ch <- cached.Classifications + ch <- cached.Classifications.ContentInfo return } @@ -98,7 +104,7 @@ func (f *InstancedMediaScanningFilter) scanMedia(ctx context.Context, event goma b, err := media.Download() if err != nil { log.Printf("[%s | %s] Error downloading media: %s", event.EventID(), event.RoomID().String(), err) - ch <- []classification.Classification{classification.Spam, classification.Unsafe} // Consider errors to be spam for now. + ch <- harms.ProhibitedContent(harms.OtherGeneral) // Consider errors to be spam for now. return } @@ -113,14 +119,18 @@ func (f *InstancedMediaScanningFilter) scanMedia(ctx context.Context, event goma res, err := f.scanner.Scan(ctx, contentType, b) if err != nil { log.Printf("[%s | %s] Error scanning media: %s", event.EventID(), event.RoomID().String(), err) + ch <- harms.ProhibitedContent(harms.OtherGeneral) // Consider errors to be spam for now. + return } log.Printf("[%s | %s] Media scan result on %s: %v", event.EventID(), event.RoomID().String(), media, res) err = f.set.storage.UpsertMediaClassification(ctx, &storage.StoredMediaClassification{ - MxcUri: media.String(), - CommunityId: f.set.communityId, - Classifications: res, + MxcUri: media.String(), + CommunityId: f.set.communityId, + Classifications: storage.StoredClassifications{ + ContentInfo: res, + }, }) if err != nil { log.Printf("[%s | %s] Non-fatal error caching media classification: %s", event.EventID(), event.RoomID().String(), err) diff --git a/filter/filter_media_scanning_test.go b/filter/filter_media_scanning_test.go index 9e55c50..6ecce64 100644 --- a/filter/filter_media_scanning_test.go +++ b/filter/filter_media_scanning_test.go @@ -8,7 +8,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" "github.com/matrix-org/policyserv/content" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/media" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -22,9 +22,8 @@ func TestMediaScanningFilter(t *testing.T) { MediaFilterMediaTypes: &[]string{"m.sticker", "m.image"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{MediaScanningFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{MediaScanningFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -41,8 +40,8 @@ func TestMediaScanningFilter(t *testing.T) { // Note: we set the CSAM classification in both expectations so we can detect that the filter actually ran the scanner. // Only the first will result in a spam response though because it sets the spam classification. - scanner.Expect(content.TypePhoto, spammyBytes, []classification.Classification{classification.CSAM, classification.Spam}, nil) - scanner.Expect(content.TypePhoto, neutralBytes, []classification.Classification{classification.CSAM}, nil) + scanner.Expect(content.TypePhoto, spammyBytes, harms.ProhibitedContent(harms.ChildSafetyCSAM, harms.SpamGeneral), nil) + scanner.Expect(content.TypePhoto, neutralBytes, harms.ProhibitedContent(harms.ChildSafetyCSAM), nil) spammyMxcUri1 := "mxc://example.org/spam1" spammyMxcUri2 := "mxc://example.org/spam2" @@ -98,24 +97,26 @@ func TestMediaScanningFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool, expectedDownloadCalls int) { - before := downloader.DownloadCalls - vecs, err := set.CheckEvent(context.Background(), event, downloader) + // We can't use AssertCheckEvent here because we need to give it a downloader to use. + assertCheckEvent := func(event gomatrixserverlib.PDU, expected *harms.ContentInfo) { + info, err := set.CheckEvent(context.Background(), event, downloader) assert.NoError(t, err) - assert.Equal(t, before+expectedDownloadCalls, downloader.DownloadCalls) - assert.Equal(t, 1.0, vecs.GetVector(classification.CSAM)) // always set regardless of spam/neutral - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } + test.AssertEqualContentInfo(t, expected, info) } - assertSpamVector(spammyEvent1, true, 1) - assertSpamVector(spammyEvent2, true, 1) - assertSpamVector(spammyEvent3, true, 0) // should have cached the result in spammyEvent2 - assertSpamVector(neutralEvent1, false, 1) - assertSpamVector(neutralEvent2, false, 0) // should have been cached above too + + assertCheckEvent(spammyEvent1, harms.ProhibitedContent(harms.ChildSafetyCSAM, harms.SpamGeneral)) + assert.Equal(t, 1, downloader.DownloadCalls) + assertCheckEvent(spammyEvent2, harms.ProhibitedContent(harms.ChildSafetyCSAM, harms.SpamGeneral)) + assert.Equal(t, 2, downloader.DownloadCalls) // +1 call + assertCheckEvent(spammyEvent3, harms.ProhibitedContent(harms.ChildSafetyCSAM, harms.SpamGeneral)) + assert.Equal(t, 2, downloader.DownloadCalls) // should have already cached the result + + // We always return a CSAM harm for testing, even on neutral media + + assertCheckEvent(neutralEvent1, harms.ProhibitedContent(harms.ChildSafetyCSAM)) + assert.Equal(t, 3, downloader.DownloadCalls) // +1 call + assertCheckEvent(neutralEvent2, harms.ProhibitedContent(harms.ChildSafetyCSAM)) + assert.Equal(t, 3, downloader.DownloadCalls) // should have already cached the result too } func TestMediaScanningFilterGracefullyHandlesDownloadTimeouts(t *testing.T) { @@ -145,17 +146,59 @@ func TestMediaScanningFilterGracefullyHandlesDownloadTimeouts(t *testing.T) { }, }) - before := downloader.DownloadCalls mediaItem, err := media.NewItem("mxc://example.org/media", downloader) assert.NoError(t, err) assert.NotNil(t, mediaItem) - vecs, err := f.CheckEvent(context.Background(), &EventInput{ + info, err := f.CheckEvent(context.Background(), &EventInput{ Event: event, Medias: []*media.Item{mediaItem}, }) assert.NoError(t, err) - assert.Equal(t, []classification.Classification{classification.Spam, classification.Unsafe}, vecs) - assert.Equal(t, before+1, downloader.DownloadCalls) + test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.OtherGeneral), info) + assert.Equal(t, 1, downloader.DownloadCalls) synctest.Wait() }) } + +func TestMediaScanningFilterGracefullyHandlesScannerError(t *testing.T) { + t.Parallel() + + cnf := &SetConfig{ + Groups: []*SetGroupConfig{{ + EnabledNames: []string{MediaScanningFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, + }}, + } + memStorage := test.NewMemoryStorage(t) + defer memStorage.Close() + ps := test.NewMemoryPubsub(t) + defer ps.Close() + scanner := test.NewMemoryContentScanner(t) + set, err := NewSet(cnf, memStorage, ps, test.NewMatrixNotifier(t), scanner) + assert.NoError(t, err) + + mediaBytes := []byte("some media") + mxcUri := "mxc://example.org/error" + downloader := test.MustMakeMediaDownloader(t).Set("example.org", "error", mediaBytes) + + event := test.MustMakePDU(&test.BaseClientEvent{ + EventId: "$error1", + RoomId: "!foo:example.org", + Type: "m.room.message", + Content: map[string]any{"url": mxcUri}, + }) + + // Simulate scanner error + scanner.Expect(content.TypePhoto, mediaBytes, nil, assert.AnError) + + info, err := set.CheckEvent(context.Background(), event, downloader) + assert.NoError(t, err) + + // Failed scanning should present as prohibited content (with general harm) + test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.OtherGeneral), info) + + // Ensure it didn't cache the nil result + cached, err := memStorage.GetMediaClassification(context.Background(), mxcUri, "") + assert.Error(t, err) + assert.Nil(t, cached) +} diff --git a/filter/filter_media_test.go b/filter/filter_media_test.go index bc13f60..1d2025c 100644 --- a/filter/filter_media_test.go +++ b/filter/filter_media_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" ) @@ -17,9 +15,8 @@ func TestMediaFilter(t *testing.T) { MediaFilterMediaTypes: &[]string{"m.sticker", "m.image"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{MediaFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{MediaFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -71,19 +68,9 @@ func TestMediaFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) - assertSpamVector(noopEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia)) + AssertCheckEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent2, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) } diff --git a/filter/filter_mentions.go b/filter/filter_mentions.go index 08095a3..80ac270 100644 --- a/filter/filter_mentions.go +++ b/filter/filter_mentions.go @@ -7,7 +7,7 @@ import ( goSet "github.com/deckarep/golang-set" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -105,20 +105,17 @@ func (f *InstancedMentionsFilter) CountMentionsToLimit(ctx context.Context, even return numMentionedUserIds, nil } -func (f *InstancedMentionsFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedMentionsFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { numMentionedUserIds, err := f.CountMentionsToLimit(ctx, input.Event, f.maxMentions) if err != nil { return nil, err } if numMentionedUserIds >= f.maxMentions { - return []classification.Classification{ - classification.Spam, - classification.Mentions, - }, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } type mentionsContent struct { diff --git a/filter/filter_mentions_frequency.go b/filter/filter_mentions_frequency.go index ae2471c..40c736c 100644 --- a/filter/filter_mentions_frequency.go +++ b/filter/filter_mentions_frequency.go @@ -8,8 +8,8 @@ import ( "math" "time" - "github.com/matrix-org/policyserv/filter/classification" "github.com/matrix-org/policyserv/frequency" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -57,7 +57,7 @@ func (f *InstancedMentionsFrequencyFilter) Close() error { return f.counter.Close() } -func (f *InstancedMentionsFrequencyFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedMentionsFrequencyFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { numMentions, err := f.mentionsFilter.CountMentionsToLimit(ctx, input.Event, f.mentionsFilter.maxMentions) if err != nil { return nil, err @@ -82,12 +82,7 @@ func (f *InstancedMentionsFrequencyFilter) CheckEvent(ctx context.Context, input rate := float64(eventsLastMinute+numMentions) / float64(60) log.Printf("[%s | %s] Rate for user %s is %f (limit: %f)", input.Event.EventID(), input.Event.RoomID().String(), input.Event.SenderID(), rate, f.rateLimit) if rate > f.rateLimit { - return []classification.Classification{ - classification.Spam, - classification.Frequency, - classification.Mentions, - }, nil - } else { - return nil, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } + return harms.NeutralContent(), nil } diff --git a/filter/filter_mentions_frequency_test.go b/filter/filter_mentions_frequency_test.go index 639d9ff..328bcb3 100644 --- a/filter/filter_mentions_frequency_test.go +++ b/filter/filter_mentions_frequency_test.go @@ -6,7 +6,7 @@ import ( "time" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/storage" "github.com/matrix-org/policyserv/test" @@ -23,9 +23,8 @@ func TestMentionsFrequencyFilter(t *testing.T) { MentionFrequencyFilterRateLimit: internal.Pointer(5.0 / 60.0), // 5 mentions per minute }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{MentionsFrequencyFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{MentionsFrequencyFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -80,29 +79,17 @@ func TestMentionsFrequencyFilter(t *testing.T) { // No-op events shouldn't affect frequency. Because our rate limit is a single event, if this is handled improperly // then the next event will fail its test. - vecs, err := set.CheckEvent(context.Background(), noopEvent1, nil) - assert.NoError(t, err) - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) // original value survives due to "no opinion" - assert.Equal(t, 0.0, vecs.GetVector(classification.Frequency)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Mentions)) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) // Now we send an event that's in scope, but is technically going to be neutral. This is because we increment at a // different point, so we might "miss" the first spammy event. - vecs, err = set.CheckEvent(context.Background(), spammyEvent1, nil) - assert.NoError(t, err) - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) // original value survives due to "no opinion" - assert.Equal(t, 0.0, vecs.GetVector(classification.Frequency)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Mentions)) + AssertCheckEvent(t, set, spammyEvent1, harms.NeutralContent()) // Give a little bit of time for the notifier to settle time.Sleep(100 * time.Millisecond) // Now try to send another event that's in scope. This time it should exceed the rate limit as spam. - vecs, err = set.CheckEvent(context.Background(), spammyEvent2, nil) - assert.NoError(t, err) - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Frequency)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Mentions)) + AssertCheckEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamFlooding)) // Allow the goroutines to settle before concluding the test time.Sleep(100 * time.Millisecond) diff --git a/filter/filter_mentions_test.go b/filter/filter_mentions_test.go index c3dd0e5..42a5d4f 100644 --- a/filter/filter_mentions_test.go +++ b/filter/filter_mentions_test.go @@ -2,13 +2,12 @@ package filter import ( "context" - "fmt" "math" "strings" "testing" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -39,9 +38,8 @@ func TestMentionsFilter(t *testing.T) { MentionFilterMinPlaintextLength: internal.Pointer(5), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{MentionsFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{MentionsFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -149,36 +147,33 @@ func TestMentionsFilter(t *testing.T) { roomId := "!foo:example.org" for _, tc := range testCases { - event := test.MustMakePDU(&test.BaseClientEvent{ - EventId: "$testcase", - RoomId: roomId, - Type: tc.EventType, - Content: map[string]any{ - "m.mentions": map[string]any{ - "user_ids": tc.Members.UserIds, + t.Run(tc.TestName, func(t *testing.T) { + event := test.MustMakePDU(&test.BaseClientEvent{ + EventId: "$testcase", + RoomId: roomId, + Type: tc.EventType, + Content: map[string]any{ + "m.mentions": map[string]any{ + "user_ids": tc.Members.UserIds, + }, + "body": tc.Body, + "formatted_body": tc.FormattedBody, }, - "body": tc.Body, - "formatted_body": tc.FormattedBody, - }, - }) - err = memStorage.SetUserIdsAndDisplayNamesByRoomId(ctx, roomId, tc.Members.UserIds, tc.Members.DisplayNames) - assert.NoError(t, err, fmt.Sprintf("%s => wanted no error", tc.TestName)) + }) + err = memStorage.SetUserIdsAndDisplayNamesByRoomId(ctx, roomId, tc.Members.UserIds, tc.Members.DisplayNames) + assert.NoError(t, err) - mentionsFilter, ok := set.groups[0].filters[0].(*InstancedMentionsFilter) - assert.True(t, ok) - numMentions, err := mentionsFilter.CountMentionsToLimit(context.Background(), event, math.MaxInt) - assert.NoError(t, err) - assert.Equal(t, tc.MentionsCount, numMentions, fmt.Sprintf("%s => wanted %d mentions", tc.TestName, tc.MentionsCount)) + mentionsFilter, ok := set.groups[0].filters[0].(*InstancedMentionsFilter) + assert.True(t, ok) + numMentions, err := mentionsFilter.CountMentionsToLimit(context.Background(), event, math.MaxInt) + assert.NoError(t, err) + assert.Equal(t, tc.MentionsCount, numMentions) - vecs, err := set.CheckEvent(ctx, event, nil) - assert.NoError(t, err) - if tc.WantNeutral { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam), fmt.Sprintf("%s => wanted spam vector of 0.5", tc.TestName)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Mentions), fmt.Sprintf("%s => wanted mentions vector of 0.0", tc.TestName)) - } else { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam), fmt.Sprintf("%s => wanted spam vector of 1.0", tc.TestName)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Mentions), fmt.Sprintf("%s => wanted mentions vector of 0.0", tc.TestName)) - } + if tc.WantNeutral { + AssertCheckEvent(t, set, event, harms.NeutralContent()) + } else { + AssertCheckEvent(t, set, event, harms.ProhibitedContent(harms.SpamFlooding)) + } + }) } } diff --git a/filter/filter_mjolnir.go b/filter/filter_mjolnir.go index d797bfc..fdcb6a6 100644 --- a/filter/filter_mjolnir.go +++ b/filter/filter_mjolnir.go @@ -3,7 +3,7 @@ package filter import ( "context" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" ) const MjolnirFilterName = "MjolnirFilter" @@ -31,10 +31,10 @@ func (f *InstancedMjolnirFilter) Name() string { return MjolnirFilterName } -func (f *InstancedMjolnirFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedMjolnirFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { // Return early on non-message events if input.Event.Type() != "m.room.message" { - return nil, nil + return harms.NeutralContent(), nil } banned, err := f.set.storage.IsUserBannedInList(ctx, f.policyRoomId, string(input.Event.SenderID())) @@ -43,8 +43,8 @@ func (f *InstancedMjolnirFilter) CheckEvent(ctx context.Context, input *EventInp } if banned { - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.OtherGeneral), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_mjolnir_test.go b/filter/filter_mjolnir_test.go index abd6e91..eb010f8 100644 --- a/filter/filter_mjolnir_test.go +++ b/filter/filter_mjolnir_test.go @@ -4,9 +4,8 @@ import ( "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -24,9 +23,8 @@ func TestMjolnirFilter(t *testing.T) { MjolnirFilterRoomID: mjolnirRoomId, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{MjolnirFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{MjolnirFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -88,19 +86,9 @@ func TestMjolnirFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) - assertSpamVector(noopEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.OtherGeneral)) + AssertCheckEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.OtherGeneral)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent2, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) } diff --git a/filter/filter_openai_omni_test.go b/filter/filter_openai_omni_test.go index 165ab9f..a6d8e46 100644 --- a/filter/filter_openai_omni_test.go +++ b/filter/filter_openai_omni_test.go @@ -5,6 +5,7 @@ import ( "github.com/matrix-org/policyserv/ai" "github.com/matrix-org/policyserv/config" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -30,9 +31,8 @@ func TestOpenAIOmniFilter(t *testing.T) { OpenAIFilterFailSecure: internal.Pointer(true), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{OpenAIOmniFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{OpenAIOmniFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -50,7 +50,7 @@ func TestOpenAIOmniFilter(t *testing.T) { assert.NotNil(t, instanced) // Verify that the config/setup of the executor are carried through correctly - assert.Equal(t, set, instanced.set) // should have been set during filter creation + assert.Equal(t, set, instanced.set) // should have been set during filter creation assert.Equal(t, OpenAIOmniFilterName, instanced.Name()) // should have been set during filter creation assert.Equal(t, &ai.OpenAIOmniModerationConfig{ FailSecure: true, // should have been set by pulling in the community config above diff --git a/filter/filter_protect_local_user.go b/filter/filter_protect_local_user.go index 8b6e545..dc31f6a 100644 --- a/filter/filter_protect_local_user.go +++ b/filter/filter_protect_local_user.go @@ -6,7 +6,7 @@ import ( "log" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" ) // Developer note: this filter is force-enabled and cannot be disabled. @@ -36,7 +36,7 @@ func (p *InstancedProtectLocalUserFilter) Name() string { return ProtectLocalUserFilterName } -func (p *InstancedProtectLocalUserFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (p *InstancedProtectLocalUserFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if input.Event.Type() == "m.room.member" { // If the state key is for our user ID, but the sender is different, then someone is trying to invite, kick, or ban our user. We // don't want them to do that because it can break things - we should be managing it ourselves. The state key and sender will be @@ -44,8 +44,8 @@ func (p *InstancedProtectLocalUserFilter) CheckEvent(ctx context.Context, input // XXX: This also denies invites, but we don't support them yet anyway - https://github.com/matrix-org/policyserv/issues/91 if input.Event.StateKeyEquals(p.localUserId.String()) && input.Event.SenderID().ToUserID().String() != p.localUserId.String() { log.Printf("[%s | %s] %s tried to add/remove policyserv user improperly", input.Event.EventID(), input.Event.RoomID(), input.Event.SenderID()) - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.OtherGeneral), nil } } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_protect_local_user_test.go b/filter/filter_protect_local_user_test.go index 11272c9..b9ee03e 100644 --- a/filter/filter_protect_local_user_test.go +++ b/filter/filter_protect_local_user_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -23,9 +21,8 @@ func TestProtectLocalUserFilter(t *testing.T) { }, Groups: []*SetGroupConfig{{ // The filter is force-enabled in the community manager, which we need to mimic here - EnabledNames: []string{ProtectLocalUserFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{ProtectLocalUserFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -57,16 +54,6 @@ func TestProtectLocalUserFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.OtherGeneral)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) } diff --git a/filter/filter_sender.go b/filter/filter_sender.go index 044fde8..f163b00 100644 --- a/filter/filter_sender.go +++ b/filter/filter_sender.go @@ -3,7 +3,7 @@ package filter import ( "context" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -32,14 +32,12 @@ func (f *InstancedSenderFilter) Name() string { return SenderFilterName } -func (f *InstancedSenderFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedSenderFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { for _, s := range f.allowedUserIds { if s == string(input.Event.SenderID()) { - return []classification.Classification{ - classification.Spam.Invert(), // we expect to be run as a prefilter, so explicitly return not spam - }, nil + return harms.AllowedContent(), nil } } - return nil, nil // no opinions when not allow-listed + return harms.NeutralContent(), nil // no opinions when not allow-listed } diff --git a/filter/filter_sender_test.go b/filter/filter_sender_test.go index fadd795..be997fd 100644 --- a/filter/filter_sender_test.go +++ b/filter/filter_sender_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" ) @@ -17,9 +15,8 @@ func TestSenderFilter(t *testing.T) { SenderPrefilterAllowedSenders: &[]string{"@alice:example.org"}, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{SenderFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{SenderFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -49,16 +46,6 @@ func TestSenderFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - // Because the filter doesn't flag things as "spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } else { - assert.Equal(t, 0.0, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.NeutralContent()) // this filter actually emits Allowed or Neutral, so spam is Neutral + AssertCheckEvent(t, set, neutralEvent1, harms.AllowedContent()) } diff --git a/filter/filter_sticky_events.go b/filter/filter_sticky_events.go index 4b1a210..bd72fc7 100644 --- a/filter/filter_sticky_events.go +++ b/filter/filter_sticky_events.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" ) const StickyEventsFilterName = "StickyEventsFilter" @@ -30,9 +30,9 @@ func (f *InstancedStickyEventsFilter) Name() string { return StickyEventsFilterName } -func (f *InstancedStickyEventsFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedStickyEventsFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if input.Event.IsSticky(time.Now(), time.Now()) { - return []classification.Classification{classification.Spam, classification.Volumetric}, nil + return harms.ProhibitedContent(harms.OtherGeneral), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_sticky_events_test.go b/filter/filter_sticky_events_test.go index eecfd4f..23a2ae6 100644 --- a/filter/filter_sticky_events_test.go +++ b/filter/filter_sticky_events_test.go @@ -1,13 +1,11 @@ package filter import ( - "context" "testing" "time" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -19,9 +17,8 @@ func TestStickyEventsFilter(t *testing.T) { StickyEventsFilterAllowStickyEvents: internal.Pointer(false), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{StickyEventsFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1, + EnabledNames: []string{StickyEventsFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } @@ -64,19 +61,7 @@ func TestStickyEventsFilter(t *testing.T) { StickyUntil: time.Now().Add(-1 * time.Hour), // sticky a long time ago, but not anymore, so neutral }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.OtherGeneral)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent2, harms.NeutralContent()) } diff --git a/filter/filter_trim_length.go b/filter/filter_trim_length.go index ce76df5..1c8c572 100644 --- a/filter/filter_trim_length.go +++ b/filter/filter_trim_length.go @@ -5,7 +5,7 @@ import ( "encoding/json" "strings" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -34,10 +34,10 @@ func (f *InstancedTrimLengthFilter) Name() string { return TrimLengthFilterName } -func (f *InstancedTrimLengthFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedTrimLengthFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { // Return early on non-message events if input.Event.Type() != "m.room.message" { - return nil, nil + return harms.NeutralContent(), nil } content := &bodyOnly{} @@ -49,16 +49,13 @@ func (f *InstancedTrimLengthFilter) CheckEvent(ctx context.Context, input *Event return f.CheckText(ctx, content.Body) } -func (f *InstancedTrimLengthFilter) CheckText(ctx context.Context, text string) ([]classification.Classification, error) { +func (f *InstancedTrimLengthFilter) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { beforeTrim := len(text) afterTrim := len(strings.TrimSpace(text)) if (beforeTrim - afterTrim) >= f.maxDifference { - return []classification.Classification{ - classification.Spam, - classification.Volumetric, - }, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_trim_length_test.go b/filter/filter_trim_length_test.go index 224e2ce..d0f1060 100644 --- a/filter/filter_trim_length_test.go +++ b/filter/filter_trim_length_test.go @@ -1,16 +1,13 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" - "github.com/tidwall/gjson" ) func TestTrimLengthFilter(t *testing.T) { @@ -19,9 +16,8 @@ func TestTrimLengthFilter(t *testing.T) { TrimLengthFilterMaxDifference: internal.Pointer(5), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{TrimLengthFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{TrimLengthFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -81,43 +77,10 @@ func TestTrimLengthFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(spammyEvent3, true) - assertSpamVector(spammyEvent4, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(noopEvent1, false) - - // Also test the text filter implementation - assertTextSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - body := gjson.Get(string(event.Content()), "body").String() - vecs, err := set.CheckText(context.Background(), body) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Volumetric)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Volumetric)) - } - } - assertTextSpamVector(spammyEvent1, true) - assertTextSpamVector(spammyEvent2, true) - assertTextSpamVector(spammyEvent3, true) - assertTextSpamVector(spammyEvent4, true) - assertTextSpamVector(neutralEvent1, false) - //assertTextSpamVector(noopEvent1, false) // text doesn't have a concept of event types, so skip this one + AssertCheckTextAndEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, spammyEvent3, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, spammyEvent4, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckTextAndEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) // text doesn't have a concept of event types, so only check events } diff --git a/filter/filter_unsafe_signing_key.go b/filter/filter_unsafe_signing_key.go index e8a1f13..4ec02ac 100644 --- a/filter/filter_unsafe_signing_key.go +++ b/filter/filter_unsafe_signing_key.go @@ -11,7 +11,7 @@ import ( "slices" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" ) const UnsafeSigningKeyFilterName = "UnsafeSigningKeyFilter" @@ -54,7 +54,7 @@ func (f *InstancedUnsafeSigningKeyFilter) Name() string { return UnsafeSigningKeyFilterName } -func (f *InstancedUnsafeSigningKeyFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedUnsafeSigningKeyFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { // Start by generating our own signature for the event. This requires redaction, so do that first. roomVer := gomatrixserverlib.MustGetRoomVersion(input.Event.Version()) redacted, err := roomVer.RedactEventJSON(input.Event.JSON()) @@ -97,14 +97,11 @@ func (f *InstancedUnsafeSigningKeyFilter) CheckEvent(ctx context.Context, input if slices.Equal(compareSignature, rawSignature) { log.Printf("[%s | %s] Unsafe signing key used by %s as key ID %s", input.Event.EventID(), input.Event.RoomID().String(), serverName, keyId) - return []classification.Classification{ - classification.Spam, - classification.Unsafe, - }, nil + return harms.ProhibitedContent(harms.OtherGeneral), nil } } } } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_unsafe_signing_key_test.go b/filter/filter_unsafe_signing_key_test.go index f873345..88adfbf 100644 --- a/filter/filter_unsafe_signing_key_test.go +++ b/filter/filter_unsafe_signing_key_test.go @@ -1,14 +1,12 @@ package filter import ( - "context" "crypto/ed25519" "math/rand" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" ) @@ -19,9 +17,8 @@ func TestUnsafeSigningKeyFilter(t *testing.T) { UnsafeSigningKeyFilterEnabled: true, }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{UnsafeSigningKeyFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{UnsafeSigningKeyFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -55,18 +52,6 @@ func TestUnsafeSigningKeyFilter(t *testing.T) { }, }).Sign("example.org", "ed25519:testing", realKey) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - assert.Equal(t, 1.0, vecs.GetVector(classification.Unsafe)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - assert.Equal(t, 0.0, vecs.GetVector(classification.Unsafe)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.OtherGeneral)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) } diff --git a/filter/filter_untrusted_media.go b/filter/filter_untrusted_media.go index ef72c2c..6dfeb07 100644 --- a/filter/filter_untrusted_media.go +++ b/filter/filter_untrusted_media.go @@ -4,7 +4,7 @@ import ( "context" "log" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/trust" ) @@ -74,7 +74,7 @@ func (f *InstancedUntrustedMediaFilter) Name() string { return UntrustedMediaFilterName } -func (f *InstancedUntrustedMediaFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedUntrustedMediaFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { userId := input.Event.SenderID().ToUserID().String() isTrusted := false for _, source := range f.trustSources { @@ -97,7 +97,7 @@ func (f *InstancedUntrustedMediaFilter) CheckEvent(ctx context.Context, input *E } if isTrusted { - return nil, nil // no useful opinion on this event - the sender is trusted to send whatever we're about to check for + return harms.NeutralContent(), nil // no useful opinion on this event - the sender is trusted to send whatever we're about to check for } // copy logic from the regular media filter diff --git a/filter/filter_untrusted_media_test.go b/filter/filter_untrusted_media_test.go index 6d751db..d194cd3 100644 --- a/filter/filter_untrusted_media_test.go +++ b/filter/filter_untrusted_media_test.go @@ -4,9 +4,8 @@ import ( "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/matrix-org/policyserv/trust" @@ -23,9 +22,8 @@ func TestUntrustedMediaFilter(t *testing.T) { UntrustedMediaFilterUseMuninn: internal.Pointer(true), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{UntrustedMediaFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{UntrustedMediaFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -138,21 +136,11 @@ func TestUntrustedMediaFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(spammyEvent2, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) - assertSpamVector(neutralEvent3, false) - assertSpamVector(neutralEvent4, false) - assertSpamVector(noopEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia)) + AssertCheckEvent(t, set, spammyEvent2, harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservMedia)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent2, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent3, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent4, harms.NeutralContent()) + AssertCheckEvent(t, set, noopEvent1, harms.NeutralContent()) } diff --git a/filter/filter_user_id_contains_words.go b/filter/filter_user_id_contains_words.go index ee5b6ca..1edfbf7 100644 --- a/filter/filter_user_id_contains_words.go +++ b/filter/filter_user_id_contains_words.go @@ -4,7 +4,7 @@ import ( "context" "regexp" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -35,7 +35,7 @@ func (f *InstancedUserIdContainsWordsFilter) Name() string { return UserIdContainsWordsFilterName } -func (f *InstancedUserIdContainsWordsFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedUserIdContainsWordsFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { localpart := input.Event.SenderID().ToUserID().Local() words := findWordsInLocalpartRegex.FindAllString(localpart, -1) @@ -48,8 +48,8 @@ func (f *InstancedUserIdContainsWordsFilter) CheckEvent(ctx context.Context, inp } if len(nonEmptyWords) > f.maxWords { - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_user_id_contains_words_test.go b/filter/filter_user_id_contains_words_test.go index e2779af..3a705ea 100644 --- a/filter/filter_user_id_contains_words_test.go +++ b/filter/filter_user_id_contains_words_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -18,9 +16,8 @@ func TestUserIdContainsWordsFilter(t *testing.T) { UserIdContainsWordsFilterMaxWords: internal.Pointer(3), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{UserIdContainsWordsFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{UserIdContainsWordsFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -59,17 +56,7 @@ func TestUserIdContainsWordsFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) - assertSpamVector(neutralEvent2, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) + AssertCheckEvent(t, set, neutralEvent2, harms.NeutralContent()) } diff --git a/filter/filter_user_id_length.go b/filter/filter_user_id_length.go index 2badfbe..f76f25d 100644 --- a/filter/filter_user_id_length.go +++ b/filter/filter_user_id_length.go @@ -3,7 +3,7 @@ package filter import ( "context" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" ) @@ -32,9 +32,9 @@ func (f *InstancedUserIdLengthFilter) Name() string { return UserIdLengthFilterName } -func (f *InstancedUserIdLengthFilter) CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) { +func (f *InstancedUserIdLengthFilter) CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) { if len(input.Event.SenderID()) > f.maxLength { - return []classification.Classification{classification.Spam}, nil + return harms.ProhibitedContent(harms.SpamFlooding), nil } - return nil, nil + return harms.NeutralContent(), nil } diff --git a/filter/filter_user_id_length_test.go b/filter/filter_user_id_length_test.go index 0c64984..36bc3d5 100644 --- a/filter/filter_user_id_length_test.go +++ b/filter/filter_user_id_length_test.go @@ -1,12 +1,10 @@ package filter import ( - "context" "testing" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -18,9 +16,8 @@ func TestUserIdLengthFilter(t *testing.T) { UserIdLengthFilterMaxLength: internal.Pointer(40), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{UserIdLengthFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{UserIdLengthFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -50,16 +47,6 @@ func TestUserIdLengthFilter(t *testing.T) { }, }) - assertSpamVector := func(event gomatrixserverlib.PDU, isSpam bool) { - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - if isSpam { - assert.Equal(t, 1.0, vecs.GetVector(classification.Spam)) - } else { - // Because the filter doesn't flag things as "not spam", the seed value should survive - assert.Equal(t, 0.5, vecs.GetVector(classification.Spam)) - } - } - assertSpamVector(spammyEvent1, true) - assertSpamVector(neutralEvent1, false) + AssertCheckEvent(t, set, spammyEvent1, harms.ProhibitedContent(harms.SpamFlooding)) + AssertCheckEvent(t, set, neutralEvent1, harms.NeutralContent()) } diff --git a/filter/foundation_test.go b/filter/foundation_test.go index caf3d3a..00d8bad 100644 --- a/filter/foundation_test.go +++ b/filter/foundation_test.go @@ -2,7 +2,7 @@ package filter import ( "context" - "log" + "fmt" "strconv" "strings" "testing" @@ -10,7 +10,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -22,12 +22,10 @@ func TestMatrixFoundationIntents(t *testing.T) { // Note: this test doesn't need to use real config values or events from production. Its purpose is to // ensure that it's *possible* to separate spam from neutral events when layering multiple filters. - spamThreshold := 0.8 mjolnirRoomId := "!mjolnir:example.org" communityConfig, err := config.NewDefaultCommunityConfig() assert.NoError(t, err) - communityConfig.SpamThreshold = &spamThreshold communityConfig.SenderPrefilterAllowedSenders = &[]string{foundationAdmin} communityConfig.MjolnirFilterEnabled = internal.Pointer(true) communityConfig.DensityFilterMaxDensity = internal.Pointer(0.95) @@ -41,19 +39,16 @@ func TestMatrixFoundationIntents(t *testing.T) { CommunityConfig: communityConfig, Groups: []*SetGroupConfig{{ // Prefilters - EnabledNames: []string{HellbanPrefilterName, SenderFilterName, EventTypeFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{HellbanPrefilterName, SenderFilterName, EventTypeFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything starts as neutral by default in the test }, { // Normal filters - EnabledNames: []string{DensityFilterName, LengthFilterName, ManyAtsFilterName, MediaFilterName, MentionsFilterName, MjolnirFilterName, TrimLengthFilterName}, - MinimumSpamVectorValue: 0.2, - MaximumSpamVectorValue: 0.7, + EnabledNames: []string{DensityFilterName, LengthFilterName, ManyAtsFilterName, MediaFilterName, MentionsFilterName, MjolnirFilterName, TrimLengthFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // only run on still-neutral events }, { // Postfilters - EnabledNames: []string{HellbanPostfilterName}, - MinimumSpamVectorValue: spamThreshold, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{HellbanPostfilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // only run hellban on spammy events }}, } memStorage := test.NewMemoryStorage(t) @@ -69,19 +64,13 @@ func TestMatrixFoundationIntents(t *testing.T) { assert.NoError(t, memStorage.SetUserIdsAndDisplayNamesByRoomId(context.Background(), foundationRoomId, foundationMentionIds, foundationMentionNames)) for i, run := range foundationEvents { - log.Printf("Running test case %d - %s", i, run.Event.EventID()) - vecs, err := set.CheckEvent(context.Background(), run.Event, nil) - assert.NoError(t, err) - assert.NotNil(t, vecs) - if run.IsSpam { - assert.LessOrEqualf(t, spamThreshold, vecs.GetVector(classification.Spam), "%d (%s) should be spam, but was %f", i, run.Event.EventID(), vecs.GetVector(classification.Spam)) - } else { - assert.Greaterf(t, spamThreshold, vecs.GetVector(classification.Spam), "%d (%s) should NOT be spam, but was %f", i, run.Event.EventID(), vecs.GetVector(classification.Spam)) - } + t.Run(fmt.Sprintf("%d (%s)", i, run.Event.EventID()), func(t *testing.T) { + AssertCheckEvent(t, set, run.Event, run.Expected) - // Give things some time to settle before moving on to the next test case. This is primarily important for the - // hellban test cases to ensure the cause event creates a hellban before the effect event. - time.Sleep(100 * time.Millisecond) + // Give things some time to settle before moving on to the next test case. This is primarily important for the + // hellban test cases to ensure the cause event creates a hellban before the effect event. + time.Sleep(100 * time.Millisecond) + }) } // We deferred a bunch of close operations, but sometimes these operations can race with the test cases above causing @@ -90,8 +79,8 @@ func TestMatrixFoundationIntents(t *testing.T) { } type foundationTestCase struct { - Event gomatrixserverlib.PDU - IsSpam bool + Event gomatrixserverlib.PDU + Expected *harms.ContentInfo } const foundationRoomId = "!foo:example.org" @@ -119,7 +108,7 @@ func init() { "msgtype": "m.image", // try to activate the filter rules }, }), - IsSpam: false, + Expected: harms.AllowedContent(), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ // This event is completely neutral: sender isn't special, event type is a message, and the body is fine (but long, to try triggering a few filters) @@ -132,7 +121,7 @@ func init() { "body": "hello world! I'm trying to get help with running policyserv. Can someone help me understand what these logs mean?\n```text\n2025/09/09 15:42:05 Registered DensityFilter as &filter.DensityFilter{}\n2025/09/09 15:42:05 Registered EventTypeFilter as &filter.EventTypeFilter{}\n2025/09/09 15:42:05 Registered HellbanPrefilter as &filter.HellbanPrefilter{}\n2025/09/09 15:42:05 Registered HellbanPostfilter as &filter.HellbanPostfilter{}\n2025/09/09 15:42:05 Registered KeywordFilter as &filter.KeywordFilter{}\n2025/09/09 15:42:05 Registered LengthFilter as &filter.LengthFilter{}\n2025/09/09 15:42:05 Registered ManyAtsFilter as &filter.ManyAtsFilter{}\n2025/09/09 15:42:05 Registered MediaFilter as &filter.MediaFilter{}\n2025/09/09 15:42:05 Registered MentionsFilter as &filter.MentionsFilter{}\n2025/09/09 15:42:05 Registered MjolnirFilter as &filter.MjolnirFilter{}\n2025/09/09 15:42:05 Registered SenderFilter as &filter.SenderFilter{}\n2025/09/09 15:42:05 Registered TrimLengthFilter as &filter.TrimLengthFilter{}\n2025/09/09 15:42:05 Registered FixedFilter as &filter.FixedCanBeInstancedFilter{}\n2025/09/09 15:42:05 Registered ErrorFilter as &filter.ErrorFilter{}```\nThanks!", }, }), - IsSpam: false, + Expected: harms.NeutralContent(), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$banned_sender1", @@ -143,7 +132,7 @@ func init() { "body": "doesn't matter", }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.OtherGeneral, harms.SpamFlooding), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$media1", @@ -155,7 +144,7 @@ func init() { "body": "hello world", }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding, harms.PolicyservMedia), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$mentions_neutral1", @@ -167,7 +156,7 @@ func init() { "body": strings.Join(foundationMentionNames[0:5], " "), // not enough to trigger filter }, }), - IsSpam: false, + Expected: harms.NeutralContent(), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$mentions_spam1", @@ -179,7 +168,7 @@ func init() { "body": strings.Join(foundationMentionNames, " "), // should be enough to trigger filter }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.SpamFlooding), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$length_spam1", @@ -191,7 +180,7 @@ func init() { "body": strings.Repeat("spammy spam", 10000), }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.SpamFlooding), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$density_spam1", @@ -203,7 +192,7 @@ func init() { "body": strings.Repeat("spammy-spam", 100), }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.SpamFlooding), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$hellban_cause", @@ -215,7 +204,7 @@ func init() { "body": strings.Repeat("spammy-spam", 100), // anything to cause a ban }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.SpamFlooding), }, { Event: test.MustMakePDU(&test.BaseClientEvent{ EventId: "$hellban_effect", @@ -227,7 +216,7 @@ func init() { "body": "fine, but hellbanned", // a normal message, but the user should be picked up by hellban }, }), - IsSpam: true, + Expected: harms.ProhibitedContent(harms.SpamFlooding), }, } } diff --git a/filter/set.go b/filter/set.go index 5c4145f..39a6a4a 100644 --- a/filter/set.go +++ b/filter/set.go @@ -11,9 +11,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" "github.com/matrix-org/policyserv/content" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" - "github.com/matrix-org/policyserv/internal" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/media" "github.com/matrix-org/policyserv/notifiers" "github.com/matrix-org/policyserv/pubsub" @@ -55,9 +53,8 @@ func NewSet(config *SetConfig, storage storage.PersistentStorage, pubsub pubsub. } for i, groupCnf := range config.Groups { set.groups[i] = &setGroup{ - filters: make([]Instanced, 0), - minSpamVectorValue: groupCnf.MinimumSpamVectorValue, - maxSpamVectorValue: groupCnf.MaximumSpamVectorValue, + filters: make([]Instanced, 0), + runOnClasses: groupCnf.RunOnClasses, } for _, name := range groupCnf.EnabledNames { f, err := findByName(name) @@ -76,29 +73,25 @@ func NewSet(config *SetConfig, storage storage.PersistentStorage, pubsub pubsub. // CheckEvent - Checks an event over all of the set groups in order. If a set group errors, execution stops there. // Note: the mediaDownloader may be nil to prevent parsing and downloading of media. This should only be done in test environments. -func (s *Set) CheckEvent(ctx context.Context, event gomatrixserverlib.PDU, mediaDownloader media.Downloader) (confidence.Vectors, error) { +func (s *Set) CheckEvent(ctx context.Context, event gomatrixserverlib.PDU, mediaDownloader media.Downloader) (*harms.ContentInfo, error) { log.Printf("[%s | %s | %s] Checking event", event.EventID(), event.RoomID().String(), s.communityId) if !event.SenderID().IsUserID() || event.SenderID().ToUserID() == nil { log.Printf("[%s | %s] Skipping event and flagging as spam because sender is not a user", event.EventID(), event.RoomID().String()) - return confidence.Vectors{ - classification.Spam: 1, - classification.NonCompliance: 1, - }, nil + return harms.ProhibitedContent(harms.SpamGeneral, harms.PolicyservSpecNonCompliance), nil } - vecs := confidence.NewConfidenceVectors() - vecs.SetVector(classification.Spam, 0.5) // per docs elsewhere, start by assuming 50% likelihood of spam + contentClass := harms.ContentClassNeutral + harmIds := make([]harms.Harm, 0) auditCtx, err := newAuditContext(s.notifier, s.communityId, event) if err != nil { return nil, err } for i, group := range s.groups { input := &EventInput{ - Event: event, - IncrementalConfidenceVectors: vecs, - auditContext: auditCtx, - Medias: make([]*media.Item, 0), + Event: event, + auditContext: auditCtx, + Medias: make([]*media.Item, 0), } if mediaDownloader != nil { @@ -127,52 +120,43 @@ func (s *Set) CheckEvent(ctx context.Context, event gomatrixserverlib.PDU, media log.Printf("[%s | %s] Skipping media extraction as mediaDownloader is nil", event.EventID(), event.RoomID().String()) } - v, err := group.checkEvent(ctx, input) + info, err := group.checkEvent(ctx, harms.NewContentInfo(contentClass, harmIds...), input) if err != nil { return nil, errors.Join(fmt.Errorf("error at group %d", i), err) } - vecs = s.combineVectors(vecs, v) + if info.Class() > contentClass { + contentClass = info.Class() + } + harmIds = append(harmIds, info.Harms()...) } - auditCtx.IsSpam = s.IsSpamResponse(ctx, vecs) + + info := harms.NewContentInfo(contentClass, harmIds...) + auditCtx.IsSpam = info.Class() == harms.ContentClassProhibited go func(auditCtx *auditContext, s *Set) { // run the audit publishing async to avoid blocking the hot path any more than required err := auditCtx.Publish() if err != nil { log.Printf("[%s | %s] Non-fatal error publishing audit: %s", auditCtx.Event.EventID(), auditCtx.Event.RoomID().String(), err) } }(auditCtx, s) - return vecs, nil + return info, nil } -func (s *Set) CheckText(ctx context.Context, text string) (confidence.Vectors, error) { +func (s *Set) CheckText(ctx context.Context, text string) (*harms.ContentInfo, error) { log.Printf("[CheckText | %s] Checking text", s.communityId) - vecs := confidence.NewConfidenceVectors() - vecs.SetVector(classification.Spam, 0.5) // per docs elsewhere, start by assuming 50% likelihood of spam + contentClass := harms.ContentClassNeutral + harmIds := make([]harms.Harm, 0) // TODO: Audit context/webhooks for i, group := range s.groups { - v, err := group.checkText(ctx, vecs, text) + info, err := group.checkText(ctx, harms.NewContentInfo(contentClass, harmIds...), text) if err != nil { return nil, errors.Join(fmt.Errorf("error at group %d", i), err) } - vecs = s.combineVectors(vecs, v) - } - return vecs, nil -} - -func (s *Set) combineVectors(incremental confidence.Vectors, group confidence.Vectors) confidence.Vectors { - for cls, val := range group { - // If we've already flagged some content as spam, don't allow that to be un-flagged. - // Note: we compare using .String() because it returns the uninverted value, if inverted. - if cls.String() == classification.Spam.String() && incremental.GetVector(classification.Spam) > 0.5 { // 0.5 to escape the seed value - continue + if info.Class() > contentClass { + contentClass = info.Class() } - incremental.SetVector(cls, val) // overwrite rather than average + harmIds = append(harmIds, info.Harms()...) } - return incremental -} - -func (s *Set) IsSpamResponse(ctx context.Context, vecs confidence.Vectors) bool { - val := vecs.GetVector(classification.Spam) - return val >= internal.Dereference(s.communityConfig.SpamThreshold) + return harms.NewContentInfo(contentClass, harmIds...), nil } func (s *Set) Close() error { diff --git a/filter/set_group.go b/filter/set_group.go index bfbe65d..89249f7 100644 --- a/filter/set_group.go +++ b/filter/set_group.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" "log" + "slices" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics" ) @@ -15,82 +15,86 @@ type SetGroupConfig struct { // The filter names this group enables. All filters within a group are executed concurrently. EnabledNames []string - // The minimum vector value for classification.Spam required for this set group to process events. - // Note: The first set group will receive a classification.Spam vector value of 0.5 - MinimumSpamVectorValue float64 - - // The maximum vector value for classification.Spam allowed for this set group to process events. - MaximumSpamVectorValue float64 + // Which content classes are checked by this set group. + RunOnClasses []harms.ContentClass } type setGroup struct { - filters []Instanced - minSpamVectorValue float64 - maxSpamVectorValue float64 + filters []Instanced + runOnClasses []harms.ContentClass } -// checkEvent - If the input's spam vector is within range, processes the event through the group's filters and returns -// the confidence vectors of all filters combined. If the input's spam vector is out of range, returns empty confidence -// vectors. Errors are collated into a single error. Filters are run concurrently. -func (g *setGroup) checkEvent(ctx context.Context, input *EventInput) (confidence.Vectors, error) { - spamVec := input.IncrementalConfidenceVectors.GetVector(classification.Spam) - return g.runFilters(spamVec, func(unknownFilter Instanced, ch chan setGroupRet) { +// checkEvent - If the group is meant to be run against the content class/info, processes the event through the group's +// filters. The resulting content info is the "most severe" outcome of all filters combined. Errors are collated into a +// single error. Filters are run concurrently. +func (g *setGroup) checkEvent(ctx context.Context, infoSoFar *harms.ContentInfo, input *EventInput) (*harms.ContentInfo, error) { + return g.runFilters(infoSoFar, func(unknownFilter Instanced, ch chan setGroupRet) { filter, ok := unknownFilter.(InstancedEventFilter) if !ok { log.Printf("[%s | %s] Filter %T is not an InstancedEventFilter - skipping", input.Event.EventID(), input.Event.RoomID().String(), unknownFilter) - // we force a nil response rather than an error to ensure we simply skip it - ch <- setGroupRet{unknownFilter, nil, nil} + // we force a neutral response rather than an error to ensure we simply skip it + ch <- setGroupRet{unknownFilter, harms.NeutralContent(), nil} return } log.Printf("[%s | %s] Running filter %T", input.Event.EventID(), input.Event.RoomID().String(), filter) t := metrics.StartFilterTimer(input.Event.RoomID().String(), filter.Name()) - classifications, err := filter.CheckEvent(ctx, input) + info, err := filter.CheckEvent(ctx, input) t.ObserveDuration() - input.auditContext.AppendFilterResponse(filter.Name(), classifications) - g.logFilterClassifications(fmt.Sprintf("%s | %s", input.Event.EventID(), input.Event.RoomID().String()), filter, classifications, err) - ch <- setGroupRet{filter, classifications, err} + // If the `info` is nil, the developer forgot to return a ContentInfo. We're only *really* concerned about this + // if the filter also didn't return an error as that indicates a lack of decision in the filter. + if info == nil { + info = harms.NeutralContent() // set so we don't fully explode with nil dereference errors + if err == nil { + err = fmt.Errorf("developer error: filter %s returned nil content info", filter.Name()) + } + } + input.auditContext.AppendFilterResponse(filter.Name(), info) + g.logFilterClassifications(fmt.Sprintf("%s | %s", input.Event.EventID(), input.Event.RoomID().String()), filter, info, err) + ch <- setGroupRet{filter, info, err} }) } // checkText - The same as checkEvent, but for text content. -func (g *setGroup) checkText(ctx context.Context, incrementalVectors confidence.Vectors, input string) (confidence.Vectors, error) { - spamVec := incrementalVectors.GetVector(classification.Spam) - return g.runFilters(spamVec, func(unknownFilter Instanced, ch chan setGroupRet) { +func (g *setGroup) checkText(ctx context.Context, infoSoFar *harms.ContentInfo, input string) (*harms.ContentInfo, error) { + return g.runFilters(infoSoFar, func(unknownFilter Instanced, ch chan setGroupRet) { filter, ok := unknownFilter.(InstancedTextFilter) if !ok { log.Printf("[CheckText] Filter %T is not an InstancedTextFilter - skipping", unknownFilter) - // we force a nil response rather than an error to ensure we simply skip it - ch <- setGroupRet{unknownFilter, nil, nil} + // we force a neutral response rather than an error to ensure we simply skip it + ch <- setGroupRet{unknownFilter, harms.NeutralContent(), nil} return } log.Printf("[CheckText] Running filter %T", filter) // TODO: Metrics // TODO: Audit context/webhooks - classifications, err := filter.CheckText(ctx, input) - g.logFilterClassifications("CheckText", filter, classifications, err) - ch <- setGroupRet{filter, classifications, err} + info, err := filter.CheckText(ctx, input) + // If the `info` is nil, the developer forgot to return a ContentInfo. We're only *really* concerned about this + // if the filter also didn't return an error as that indicates a lack of decision in the filter. + if info == nil { + info = harms.NeutralContent() // set so we don't fully explode with nil dereference errors + if err == nil { + err = fmt.Errorf("developer error: filter %s returned nil content info", filter.Name()) + } + } + g.logFilterClassifications("CheckText", filter, info, err) + ch <- setGroupRet{filter, info, err} }) } -func (g *setGroup) logFilterClassifications(prefix string, filter Instanced, classifications []classification.Classification, err error) { - strClassifications := make([]string, 0, len(classifications)) - for _, cls := range classifications { - // We don't use .String() here because that will uninvert values, making logs harder to follow. - strClassifications = append(strClassifications, string(cls)) - } - log.Printf("[%s] Filter %T returned %v", prefix, filter, strClassifications) +func (g *setGroup) logFilterClassifications(prefix string, filter Instanced, info *harms.ContentInfo, err error) { + log.Printf("[%s] Filter %T returned %s %v", prefix, filter, info.Class(), info.Harms()) if err != nil { log.Printf("[%s] Filter %T returned error: %s", prefix, filter, err) } } -func (g *setGroup) runFilters(spamVec float64, checkFn func(f Instanced, ch chan setGroupRet)) (confidence.Vectors, error) { +func (g *setGroup) runFilters(infoSoFar *harms.ContentInfo, checkFn func(f Instanced, ch chan setGroupRet)) (*harms.ContentInfo, error) { // First, are we within range to actually process anything? - if spamVec < g.minSpamVectorValue || spamVec > g.maxSpamVectorValue { // don't use equality here because it'll exclude "max: 1.0" configs - // No - return nothing - return confidence.NewConfidenceVectors(), nil + if !slices.Contains(g.runOnClasses, infoSoFar.Class()) { + // No - return nothing/neutral + return harms.NeutralContent(), nil } ch := make(chan setGroupRet, len(g.filters)) @@ -105,34 +109,36 @@ func (g *setGroup) runFilters(spamVec float64, checkFn func(f Instanced, ch chan rets := make([]setGroupRet, len(g.filters)) for i := 0; i < len(g.filters); i++ { rets[i] = <-ch + + // the check functions also look for this, but they also have short circuits that can result in nil content info. + if rets[i].Err == nil && rets[i].Info == nil { + rets[i].Err = fmt.Errorf("developer error: filter %s (%d) returned no error and no content info", rets[i].Filter.Name(), i) + } } - // Scan for errors and prepare for a final confidence vectors result - vecs := confidence.NewConfidenceVectors() + // Scan for errors and prepare a merged content info result + contentClass := harms.ContentClassNeutral + harmIds := make([]harms.Harm, 0) errs := make([]error, 0) for _, r := range rets { if r.Err != nil { errs = append(errs, r.Err) continue } - for _, cls := range r.Classifications { - // If we've already flagged an event as spam, don't allow that to be un-flagged. - // Note: we compare using .String() because it returns the uninverted value, if inverted. - if cls.String() == classification.Spam.String() && vecs.GetVector(classification.Spam) > 0.5 { - continue - } - vecs.SetVector(cls, 1.0) + harmIds = append(harmIds, r.Info.Harms()...) + if contentClass < r.Info.Class() { + contentClass = r.Info.Class() } } if len(errs) > 0 { // explode return nil, errors.Join(errs...) } - return vecs, nil + return harms.NewContentInfo(contentClass, harmIds...), nil } type setGroupRet struct { - Filter Instanced - Classifications []classification.Classification - Err error + Filter Instanced + Info *harms.ContentInfo + Err error } diff --git a/filter/set_group_test.go b/filter/set_group_test.go index beb9143..42e1f9c 100644 --- a/filter/set_group_test.go +++ b/filter/set_group_test.go @@ -2,11 +2,9 @@ package filter import ( "context" - "slices" "testing" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" ) @@ -22,67 +20,42 @@ func TestSetGroupCheck(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, auditCtx) input := &EventInput{ - Event: event, - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, - auditContext: auditCtx, + Event: event, + auditContext: auditCtx, } sg := &setGroup{ filters: []Instanced{&FixedInstancedFilter{ - T: t, - Expect: input, - ExpectText: "hello world", - ReturnClasses: []classification.Classification{classification.Spam, classification.Volumetric}, - ReturnErr: nil, + T: t, + Expect: input, + ExpectText: "hello world", + ReturnInfo: harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding), + ReturnErr: nil, }}, - minSpamVectorValue: 0.3, - maxSpamVectorValue: 0.8, + runOnClasses: []harms.ContentClass{harms.ContentClassAllowed}, // we only want to capture a specific class of events } - // Does it no-op when the spam vector is out of range? - input.IncrementalConfidenceVectors.SetVector(classification.Spam, 0.1) - vecs, err := sg.checkEvent(context.Background(), input) - assert.NoError(t, err) - assert.Equal(t, confidence.NewConfidenceVectors(), vecs) // no-op is to return a zero value - input.IncrementalConfidenceVectors.SetVector(classification.Spam, 0.9) - vecs, err = sg.checkEvent(context.Background(), input) - assert.NoError(t, err) - assert.Equal(t, confidence.NewConfidenceVectors(), vecs) // no-op is to return a zero value + // Note: we test text and events at the same time - // Does it process the event through the filter? - input.IncrementalConfidenceVectors.SetVector(classification.Spam, 0.5) - vecs, err = sg.checkEvent(context.Background(), input) + // Does it no-op when the content class is wrong? + info, err := sg.checkEvent(context.Background(), harms.NeutralContent(), input) // runOnClasses uses ContentClassAllowed assert.NoError(t, err) - classes := make([]classification.Classification, 0) - for cls, _ := range vecs { - classes = append(classes, cls) - } - expectClasses := sg.filters[0].(*FixedInstancedFilter).ReturnClasses - // Note: we sort because the order of values is not guaranteed on our custom types. - slices.Sort(classes) - slices.Sort(expectClasses) - assert.Equal(t, expectClasses, classes) - - // Repeat the above tests, except for checkText instead + test.AssertEqualContentInfo(t, harms.NeutralContent(), info) // no-op is neutral + info, err = sg.checkText(context.Background(), harms.NeutralContent(), "hello world") + assert.NoError(t, err) + test.AssertEqualContentInfo(t, harms.NeutralContent(), info) // no-op is neutral - // no-op when out of range - textIncrementalVectors := confidence.Vectors{classification.Spam: 0.1} - vecs, err = sg.checkText(context.Background(), textIncrementalVectors, "hello world") + info, err = sg.checkEvent(context.Background(), harms.ProhibitedContent(harms.SpamFraud), input) assert.NoError(t, err) - assert.Equal(t, confidence.NewConfidenceVectors(), vecs) // no-op is to return a zero value - textIncrementalVectors.SetVector(classification.Spam, 0.9) - vecs, err = sg.checkText(context.Background(), textIncrementalVectors, "hello world") + test.AssertEqualContentInfo(t, harms.NeutralContent(), info) // no-op is neutral + info, err = sg.checkText(context.Background(), harms.ProhibitedContent(harms.SpamFraud), "hello world") assert.NoError(t, err) - assert.Equal(t, confidence.NewConfidenceVectors(), vecs) // no-op is to return a zero value + test.AssertEqualContentInfo(t, harms.NeutralContent(), info) // no-op is neutral - // does it actually work - textIncrementalVectors.SetVector(classification.Spam, 0.5) - vecs, err = sg.checkText(context.Background(), textIncrementalVectors, "hello world") + // Does it process the event through the filter? + info, err = sg.checkEvent(context.Background(), harms.AllowedContent(), input) // runOnClasses uses ContentClassAllowed assert.NoError(t, err) - classes = make([]classification.Classification, 0) - for cls, _ := range vecs { - classes = append(classes, cls) - } - slices.Sort(classes) - slices.Sort(expectClasses) - assert.Equal(t, expectClasses, classes) + test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding), info) + info, err = sg.checkText(context.Background(), harms.AllowedContent(), "hello world") // runOnClasses uses ContentClassAllowed + assert.NoError(t, err) + test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding), info) } diff --git a/filter/set_test.go b/filter/set_test.go index 539e852..dba47ab 100644 --- a/filter/set_test.go +++ b/filter/set_test.go @@ -12,23 +12,41 @@ import ( "testing" "time" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/media" "github.com/matrix-org/policyserv/notifiers" "github.com/matrix-org/policyserv/storage" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" ) +func AssertCheckEvent(t *testing.T, set *Set, event gomatrixserverlib.PDU, expected *harms.ContentInfo) { + info, err := set.CheckEvent(context.Background(), event, nil) + assert.NoError(t, err) + test.AssertEqualContentInfo(t, expected, info) +} + +func AssertCheckText(t *testing.T, set *Set, eventWithBody gomatrixserverlib.PDU, expected *harms.ContentInfo) { + body := gjson.Get(string(eventWithBody.Content()), "body").String() + info, err := set.CheckText(context.Background(), body) + assert.NoError(t, err) + test.AssertEqualContentInfo(t, expected, info) +} + +func AssertCheckTextAndEvent(t *testing.T, set *Set, eventWithBody gomatrixserverlib.PDU, expected *harms.ContentInfo) { + AssertCheckText(t, set, eventWithBody, expected) + AssertCheckEvent(t, set, eventWithBody, expected) +} + func TestNewSet(t *testing.T) { cnf := &SetConfig{ Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.3, - MaximumSpamVectorValue: 0.8, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -52,9 +70,8 @@ func TestNewSet(t *testing.T) { func TestNewSetUnknownFilter(t *testing.T) { cnf := &SetConfig{ Groups: []*SetGroupConfig{{ - EnabledNames: []string{"this_is_not_a_real_filter_name"}, - MinimumSpamVectorValue: 0.3, - MaximumSpamVectorValue: 0.8, + EnabledNames: []string{"this_is_not_a_real_filter_name"}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -70,9 +87,8 @@ func TestNewSetUnknownFilter(t *testing.T) { func TestNewSetErrorMaking(t *testing.T) { cnf := &SetConfig{ Groups: []*SetGroupConfig{{ - EnabledNames: []string{ErrorFilterName}, - MinimumSpamVectorValue: 0.3, - MaximumSpamVectorValue: 0.8, + EnabledNames: []string{ErrorFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything is neutral by default in the test }}, } memStorage := test.NewMemoryStorage(t) @@ -90,13 +106,11 @@ func TestSetCheckEvent(t *testing.T) { CommunityConfig: &config.CommunityConfig{}, // We want to ensure we call *all* groups, so specify 2 to call Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // events start off neutral by default }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // we're going to test that harms are added }}, } memStorage := test.NewMemoryStorage(t) @@ -114,54 +128,32 @@ func TestSetCheckEvent(t *testing.T) { f2.T = t // Set our expectations and return values - // Note: internally, sets start with a 0.5 spam vector, so things get divided by 3 rather than 2 event := test.MustMakePDU(&test.BaseClientEvent{ RoomId: "!foo:example.org", EventId: "$test", Type: "m.room.message", Content: make(map[string]any), }) - f1.Expect = &EventInput{Event: event, Medias: make([]*media.Item, 0), IncrementalConfidenceVectors: confidence.Vectors{ - classification.Spam: 0.5, - }} - f1.ReturnClasses = []classification.Classification{ - classification.Spam, - classification.Mentions, - } - f2.Expect = &EventInput{Event: event, Medias: make([]*media.Item, 0), IncrementalConfidenceVectors: confidence.Vectors{ - classification.Spam: 1.0, - classification.Mentions: 1.0, - }} - f2.ReturnClasses = []classification.Classification{ - classification.Spam.Invert(), - classification.Volumetric, - } + f1.Expect = &EventInput{Event: event, Medias: make([]*media.Item, 0)} + f2.Expect = f1.Expect // same input + f1.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) + f2.ReturnInfo = harms.ProhibitedContent(harms.SpamFraud) // this should be added on after the filters run - vecs, err := set.CheckEvent(context.Background(), event, nil) - assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.Equal(t, confidence.Vectors{ - classification.Spam: 1.0, - classification.Mentions: 1.0, - classification.Volumetric: 1.0, - }, vecs) + AssertCheckEvent(t, set, event, harms.ProhibitedContent(harms.SpamFlooding, harms.SpamFraud)) } func TestCheckEventWithErrorInGroup(t *testing.T) { cnf := &SetConfig{ CommunityConfig: &config.CommunityConfig{}, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // events start off neutral by default }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // our fixed filter sets everything as prohibited }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // our fixed filter sets everything as prohibited }}, } memStorage := test.NewMemoryStorage(t) @@ -182,15 +174,9 @@ func TestCheckEventWithErrorInGroup(t *testing.T) { { Event: event, Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{ - classification.Spam: 0.5, - }, }, { Event: event, Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{ - classification.Spam: 1.0, // group 0 result - }, }, nil, // should never be called } @@ -199,13 +185,13 @@ func TestCheckEventWithErrorInGroup(t *testing.T) { for _, f := range group.filters { ff := f.(*FixedInstancedFilter) ff.T = t - ff.ReturnClasses = []classification.Classification{classification.Spam} + ff.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) ff.Expect = inputs[i] } } errorFilter := set.groups[1].filters[0].(*FixedInstancedFilter) errorFilter.ReturnErr = errors.New("error within filter group") - errorFilter.ReturnClasses = nil + errorFilter.ReturnInfo = nil vecs, err := set.CheckEvent(context.Background(), event, nil) assert.ErrorContains(t, err, "error at group 1") @@ -217,13 +203,11 @@ func TestSetCheckText(t *testing.T) { CommunityConfig: &config.CommunityConfig{}, // We want to ensure we call *all* groups, so specify 2 to call Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // content start off neutral by default }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // we're going to test that harms are added }}, } memStorage := test.NewMemoryStorage(t) @@ -241,43 +225,28 @@ func TestSetCheckText(t *testing.T) { f2.T = t // Set our expectations and return values - // Note: internally, sets start with a 0.5 spam vector, so things get divided by 3 rather than 2 f1.ExpectText = "Hello world" f2.ExpectText = "Hello world" - f1.ReturnClasses = []classification.Classification{ - classification.Spam, - classification.Mentions, - } - f2.ReturnClasses = []classification.Classification{ - classification.Spam.Invert(), - classification.Volumetric, - } + f1.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) + f2.ReturnInfo = harms.ProhibitedContent(harms.SpamFraud) // this should be added on after the filters run - vecs, err := set.CheckText(context.Background(), "Hello world") + info, err := set.CheckText(context.Background(), "Hello world") assert.NoError(t, err) - assert.NotNil(t, vecs) - assert.Equal(t, confidence.Vectors{ - classification.Spam: 1.0, - classification.Mentions: 1.0, - classification.Volumetric: 1.0, - }, vecs) + test.AssertEqualContentInfo(t, harms.ProhibitedContent(harms.SpamFlooding, harms.SpamFraud), info) } func TestSetCheckTextWithErrorInGroup(t *testing.T) { cnf := &SetConfig{ CommunityConfig: &config.CommunityConfig{}, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // events start off neutral by default }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // our fixed filter sets everything as prohibited }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassProhibited}, // our fixed filter sets everything as prohibited }}, } memStorage := test.NewMemoryStorage(t) @@ -292,7 +261,7 @@ func TestSetCheckTextWithErrorInGroup(t *testing.T) { for _, f := range group.filters { ff := f.(*FixedInstancedFilter) ff.T = t - ff.ReturnClasses = []classification.Classification{classification.Spam} + ff.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) if i > 1 { ff.ReturnErr = errors.New("should never happen") ff.ExpectText = "if you see this in the test output, it broke" @@ -303,45 +272,13 @@ func TestSetCheckTextWithErrorInGroup(t *testing.T) { } errorFilter := set.groups[1].filters[0].(*FixedInstancedFilter) errorFilter.ReturnErr = errors.New("error within filter group") - errorFilter.ReturnClasses = nil + errorFilter.ReturnInfo = nil vecs, err := set.CheckText(context.Background(), "Hello world") assert.ErrorContains(t, err, "error at group 1") assert.Nil(t, vecs) } -func TestSetSpamThreshold(t *testing.T) { - cnf := &SetConfig{ - CommunityConfig: &config.CommunityConfig{ - SpamThreshold: internal.Pointer(0.6), - }, - } - memStorage := test.NewMemoryStorage(t) - defer memStorage.Close() - ps := test.NewMemoryPubsub(t) - defer ps.Close() - - set, err := NewSet(cnf, memStorage, ps, test.NewMatrixNotifier(t), nil) - assert.NoError(t, err) - assert.NotNil(t, set) - - assert.Equal(t, true, set.IsSpamResponse(context.Background(), confidence.Vectors{ - classification.Spam: 0.6, // exact match - })) - assert.Equal(t, true, set.IsSpamResponse(context.Background(), confidence.Vectors{ - classification.Spam: 0.7, // just above - })) - assert.Equal(t, false, set.IsSpamResponse(context.Background(), confidence.Vectors{ - classification.Spam: 0.5, // just below - })) - assert.Equal(t, false, set.IsSpamResponse(context.Background(), confidence.Vectors{ - classification.Spam: 0.0, // very much below - })) - assert.Equal(t, true, set.IsSpamResponse(context.Background(), confidence.Vectors{ - classification.Spam: 1.0, // very much above - })) -} - func TestCallsWebhook(t *testing.T) { t.Parallel() @@ -373,17 +310,14 @@ func TestCallsWebhook(t *testing.T) { cnf := &SetConfig{ CommunityConfig: &config.CommunityConfig{ - SpamThreshold: internal.Pointer(0.6), - WebhookUrl: internal.Pointer(server.URL + "/webhook"), + WebhookUrl: internal.Pointer(server.URL + "/webhook"), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // starts as neutral, stays neutral until the next filter runs }, { - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, }}, } memStorage := test.NewMemoryStorage(t) @@ -417,28 +351,26 @@ func TestCallsWebhook(t *testing.T) { }) // The first group returns a neutral response and the second group returns spam. This is to ensure that the resulting - // webhook shows the groups independently. This also makes writing the test easier because both groups will receive - // the same initial (incremental) vectors. + // webhook shows the groups independently. for i := 0; i < 2; i++ { fixedFilter := set.groups[i].filters[0].(*FixedInstancedFilter) fixedFilter.T = t fixedFilter.Set = set fixedFilter.Expect = &EventInput{ - Event: event, - Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, + Event: event, + Medias: make([]*media.Item, 0), } - fixedFilter.ReturnClasses = []classification.Classification{} + fixedFilter.ReturnInfo = harms.NeutralContent() if i > 0 { - fixedFilter.ReturnClasses = append(fixedFilter.ReturnClasses, classification.Spam, classification.Mentions) + fixedFilter.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) } } - vecs, err := set.CheckEvent(context.Background(), event, nil) + info, err := set.CheckEvent(context.Background(), event, nil) assert.NoError(t, err) - assert.NotNil(t, vecs) // composition checked in other tests + assert.NotNil(t, info) // composition checked in other tests // Wait a bit so the goroutines can settle time.Sleep(1 * time.Second) @@ -467,13 +399,11 @@ func TestCallsWebhookErrorNonFatal(t *testing.T) { cnf := &SetConfig{ CommunityConfig: &config.CommunityConfig{ - SpamThreshold: internal.Pointer(0.6), - WebhookUrl: internal.Pointer(server.URL + "/webhook"), + WebhookUrl: internal.Pointer(server.URL + "/webhook"), }, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything starts as neutral }}, } memStorage := test.NewMemoryStorage(t) @@ -510,11 +440,10 @@ func TestCallsWebhookErrorNonFatal(t *testing.T) { fixedFilter.T = t fixedFilter.Set = set fixedFilter.Expect = &EventInput{ - Event: event, - Medias: make([]*media.Item, 0), - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, + Event: event, + Medias: make([]*media.Item, 0), } - fixedFilter.ReturnClasses = []classification.Classification{classification.Spam} + fixedFilter.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) vecs, err := set.CheckEvent(context.Background(), event, nil) assert.NoError(t, err) @@ -535,9 +464,8 @@ func TestExtractsMedia(t *testing.T) { cnf := &SetConfig{ CommunityConfig: &config.CommunityConfig{}, Groups: []*SetGroupConfig{{ - EnabledNames: []string{FixedFilterName}, - MinimumSpamVectorValue: 0.0, - MaximumSpamVectorValue: 1.0, + EnabledNames: []string{FixedFilterName}, + RunOnClasses: []harms.ContentClass{harms.ContentClassNeutral}, // everything starts as neutral }}, } memStorage := test.NewMemoryStorage(t) @@ -573,8 +501,7 @@ func TestExtractsMedia(t *testing.T) { fixedFilter.T = t fixedFilter.Set = set fixedFilter.Expect = &EventInput{ - Event: event, - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, + Event: event, Medias: []*media.Item{ { Origin: origin1, @@ -586,7 +513,7 @@ func TestExtractsMedia(t *testing.T) { }, }, } - fixedFilter.ReturnClasses = []classification.Classification{classification.Spam} + fixedFilter.ReturnInfo = harms.ProhibitedContent(harms.SpamFlooding) res, err := set.CheckEvent(context.Background(), event, downloader) assert.NoError(t, err) @@ -594,9 +521,8 @@ func TestExtractsMedia(t *testing.T) { // Now test that there are no media items extracted when no downloader is supplied. fixedFilter.Expect = &EventInput{ - Event: event, - IncrementalConfidenceVectors: confidence.Vectors{classification.Spam: 0.5}, - Medias: make([]*media.Item, 0), + Event: event, + Medias: make([]*media.Item, 0), } res, err = set.CheckEvent(context.Background(), event, nil) assert.NoError(t, err) diff --git a/filter/types.go b/filter/types.go index 6b2086c..fe13fd9 100644 --- a/filter/types.go +++ b/filter/types.go @@ -4,8 +4,7 @@ import ( "context" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/media" ) @@ -14,9 +13,6 @@ type EventInput struct { // The event to process/check. Event gomatrixserverlib.PDU - // The confidence.Vectors so far. Note that the first set group will receive a classification.Spam vector of 0.5 - IncrementalConfidenceVectors confidence.Vectors - // Extracted media items from the event. Medias []*media.Item @@ -42,16 +38,16 @@ type CanBeInstanced interface { type InstancedEventFilter interface { Instanced // parent type - // CheckEvent - Processes the given event, returning classifications about it. If an error occurred, the classifications - // array will be nil/empty. - CheckEvent(ctx context.Context, input *EventInput) ([]classification.Classification, error) + // CheckEvent - Processes the given content, returning harm/content classification. The content info may be nil + // if there was an error. + CheckEvent(ctx context.Context, input *EventInput) (*harms.ContentInfo, error) } type InstancedTextFilter interface { Instanced // parent type - // CheckText - Processes the given text, returning classifications about it. If an error occurred, the classifications - // array will be nil/empty. The input text string is assumed to be user-generated (message body, search query, etc) - // rather than structured (JSON, CSV, etc). - CheckText(ctx context.Context, input string) ([]classification.Classification, error) + // CheckText - Processes the given text, returning harm/content classification. The content info may be nil if there + // was an error. The input text string is assumed to be user-generated (message body, search query, etc) rather than + // structured (JSON, CSV, etc). + CheckText(ctx context.Context, input string) (*harms.ContentInfo, error) } diff --git a/harms/ids_msc4456.go b/harms/ids_msc4456.go new file mode 100644 index 0000000..16a3a13 --- /dev/null +++ b/harms/ids_msc4456.go @@ -0,0 +1,115 @@ +package harms + +// *************************************************************************************** +// **Content Warning**: This file contains identifiers for harmful content, but does not +// attempt to describe the harms in detail. This includes identifiers and titles for those +// identifiers for child safety, sexual abuse, self-harm, and other types of harm a user +// may encounter on the open internet. +// --------------------------------------------------------------------------------------- +// The harms defined in this file are defined by MSC4456. +// https://github.com/matrix-org/matrix-spec-proposals/pull/4456 +// *************************************************************************************** + +const idPrefix = "org.matrix.msc4456" + +const ( + // SpamGeneral - Spam - "General/Other" + SpamGeneral Harm = idPrefix + `.spam` + // SpamFraud - Spam - "Fraud/Phishing" + SpamFraud Harm = idPrefix + `.spam.fraud` + // SpamImpersonation - Spam - "Impersonation" + SpamImpersonation Harm = idPrefix + `.spam.impersonation` + // SpamElectionInterference - Spam - "Election Interference" + SpamElectionInterference Harm = idPrefix + `.spam.election_interference` + // SpamFlooding - Spam - "Flooding" + SpamFlooding Harm = idPrefix + `.spam.flooding` +) + +const ( + // AdultGeneral - Adult - "General/Other" + AdultGeneral Harm = idPrefix + `.adult` + // AdultSexualAbuse - Adult - "Sexual Abuse" + AdultSexualAbuse Harm = idPrefix + `.adult.sexual_abuse` + // AdultNCII - Adult - "Non-Consensual Intimate Imagery (NCII)" + AdultNCII Harm = idPrefix + `.adult.ncii` + // AdultDeepfake - Adult - "Deepfake" + AdultDeepfake Harm = idPrefix + `.adult.deepfake` + // AdultAnimalSexualAbuse - Adult - "Animal Sexual Abuse" + AdultAnimalSexualAbuse Harm = idPrefix + `.adult.animal_sexual_abuse` + // AdultSexualViolence - Adult - "Sexual Violence" + AdultSexualViolence Harm = idPrefix + `.adult.sexual_violence` +) + +const ( + // HarassmentGeneral - Harassment - "General/Other" + HarassmentGeneral Harm = idPrefix + `.harassment` + // HarassmentTrolling - Harassment - "Trolling" + HarassmentTrolling Harm = idPrefix + `.harassment.trolling` + // HarassmentTargeted - Harassment - "Targeted" + HarassmentTargeted Harm = idPrefix + `.harassment.targeted` + // HarassmentHate - Harassment - "Hate" + HarassmentHate Harm = idPrefix + `.harassment.hate` + // HarassmentDoxxing - Harassment - "Doxxing/Personal Information" + HarassmentDoxxing Harm = idPrefix + `.harassment.doxxing` +) + +const ( + // ViolenceGeneral - Violence - "General/Other" + ViolenceGeneral Harm = idPrefix + `.violence` + // ViolenceAnimalWelfare - Violence - "Animal Welfare" + ViolenceAnimalWelfare Harm = idPrefix + `.violence.animal_welfare` + // ViolenceThreats - Violence - "Threats/Threatening" + ViolenceThreats Harm = idPrefix + `.violence.threats` + // ViolenceGraphic - Violence - "Graphic/Gore" + ViolenceGraphic Harm = idPrefix + `.violence.graphic` + // ViolenceGlorification - Violence - "Glorification/Promotion" + ViolenceGlorification Harm = idPrefix + `.violence.glorification` + // ViolenceExtremist - Violence - "Extremism" + ViolenceExtremist Harm = idPrefix + `.violence.extremist` + // ViolenceHumanTrafficking - Violence - "Human Trafficking" + ViolenceHumanTrafficking Harm = idPrefix + `.violence.human_trafficking` + // ViolenceDomestic - Violence - "Domestic/Intimate Partner" + ViolenceDomestic Harm = idPrefix + `.violence.domestic` +) + +const ( + // ChildSafetyGeneral - Child Safety - "General/Other" + ChildSafetyGeneral Harm = idPrefix + `.child_safety` + // ChildSafetyCSAM - Child Safety - "Child Sexual Abuse Material (CSAM)" + ChildSafetyCSAM Harm = idPrefix + `.child_safety.csam` + // ChildSafetyGrooming - Child Safety - "Grooming" + ChildSafetyGrooming Harm = idPrefix + `.child_safety.grooming` + // ChildSafetyPrivacyViolation - Child Safety - "Privacy" + ChildSafetyPrivacyViolation Harm = idPrefix + `.child_safety.privacy_violation` + // ChildSafetyHarassment - Child Safety - "Harassment" + ChildSafetyHarassment Harm = idPrefix + `.child_safety.harassment` +) + +const ( + // DangerGeneral - Danger - "General/Other" + DangerGeneral Harm = idPrefix + `.danger` + // DangerSelfHarm - Danger - "Self Harm" + DangerSelfHarm Harm = idPrefix + `.danger.self_harm` + // DangerEatingDisorder - Danger - "Eating Disorder" + DangerEatingDisorder Harm = idPrefix + `.danger.eating_disorder` + // DangerChallenges - Danger - "Challenges, including Social Media Challenges" + DangerChallenges Harm = idPrefix + `.danger.challenges` + // DangerSubstanceAbuse - Danger - "Substance Abuse" + DangerSubstanceAbuse Harm = idPrefix + `.danger.substance_abuse` +) + +const ( + // TOSGeneral - Terms of Service - "General/Other" + TOSGeneral Harm = idPrefix + `.tos` + // TOSHacking - Terms of Service - "Hacking/Computer Misuse" + TOSHacking Harm = idPrefix + `.tos.hacking` + // TOSProhibited - Terms of Service - "Prohibited Items (Drugs, Weapons, etc)" + TOSProhibited Harm = idPrefix + `.tos.prohibited` + // TOSBanEvasion - Terms of Service - "Ban Evasion" + TOSBanEvasion Harm = idPrefix + `.tos.ban_evasion` +) + +const ( + // OtherGeneral - Other - "Other Concern" + OtherGeneral Harm = idPrefix + `.other` +) diff --git a/harms/ids_policyserv.go b/harms/ids_policyserv.go new file mode 100644 index 0000000..b0845a4 --- /dev/null +++ b/harms/ids_policyserv.go @@ -0,0 +1,18 @@ +package harms + +// *************************************************************************************** +// **Content Warning**: This file contains identifiers for harmful content, but does not +// attempt to describe the harms in detail. This includes identifiers and titles for those +// identifiers for harms a user may encounter in a Matrix chat room. +// --------------------------------------------------------------------------------------- +// The harms defined in this file are custom to policyserv. +// *************************************************************************************** + +const psIdPrefix = "org.matrix.policyserv" + +const ( + // PolicyservMedia - policyserv - "Media" + PolicyservMedia Harm = psIdPrefix + `.media` + // PolicyservSpecNonCompliance - policyserv - "Spec Non-Compliance" + PolicyservSpecNonCompliance Harm = psIdPrefix + `.spec_non_compliance` +) diff --git a/harms/types.go b/harms/types.go new file mode 100644 index 0000000..8cc411f --- /dev/null +++ b/harms/types.go @@ -0,0 +1,93 @@ +package harms + +import "slices" + +// Harm - A harm identifier. +type Harm string + +// ContentClass - A broad classification of content, aside from any specific harms. Typically, if +// content is ContentClassProhibited then it will be accompanied by one or more harms too. Defined +// content classes are ordered such that ContentClassNeutral < ContentClassAllowed < ContentClassProhibited. +type ContentClass int + +const ( + // ContentClassNeutral - The default state for any content. + ContentClassNeutral ContentClass = iota + // ContentClassAllowed - The content is explicitly allowed. + ContentClassAllowed + // ContentClassProhibited - The content is explicitly prohibited. + ContentClassProhibited +) + +// String - override to make Sprintf work a bit better. +func (c ContentClass) String() string { + return [...]string{"Neutral", "Allowed", "Prohibited"}[c] +} + +// ContentInfo - Carries harm and class information for a given piece of content. +type ContentInfo struct { + class ContentClass + harms []Harm +} + +// NewContentInfo - Creates a new ContentInfo instance with the given class and harms. If the class is +// ContentClassProhibited and no harms are specified, OtherGeneral is used. If any harms are specified, +// the class is automatically set to ContentClassProhibited. +func NewContentInfo(class ContentClass, harms ...Harm) *ContentInfo { + if len(harms) == 0 && class == ContentClassProhibited { + harms = []Harm{OtherGeneral} + } + if len(harms) > 0 { + class = ContentClassProhibited + } + return &ContentInfo{ + class: class, + harms: deduplicateHarms(harms), + } +} + +func (i *ContentInfo) Class() ContentClass { + return i.class +} + +func (i *ContentInfo) Harms() []Harm { + // Some code will call Harms() and immediately expect that it will have zero or more values, which isn't always + // the case. To add some safety, we force-set harms to an empty slice if it's nil. + if i.harms == nil { + i.harms = make([]Harm, 0) + } + return i.harms +} + +// ProhibitedContent - Creates a ContentInfo instance with the specified harms and ContentClassProhibited. +// If no harms are specified, OtherGeneral is used. +func ProhibitedContent(harms ...Harm) *ContentInfo { + if len(harms) == 0 { + harms = []Harm{OtherGeneral} + } + return &ContentInfo{ + class: ContentClassProhibited, + harms: deduplicateHarms(harms), + } +} + +// NeutralContent - Creates a ContentInfo instance with ContentClassNeutral. +func NeutralContent() *ContentInfo { + return &ContentInfo{ + class: ContentClassNeutral, + } +} + +// AllowedContent - Creates a ContentInfo instance with ContentClassAllowed. +func AllowedContent() *ContentInfo { + return &ContentInfo{ + class: ContentClassAllowed, + } +} + +func deduplicateHarms(harms []Harm) []Harm { + // We sort primarily for tests (they compare ContentInfo instances), but also to make Compact() actually work. + // Compact() removes duplicates when they're right next to each other, and Sort() does that for us. + slices.Sort(harms) + return slices.Compact(harms) +} diff --git a/harms/types_test.go b/harms/types_test.go new file mode 100644 index 0000000..2b5123c --- /dev/null +++ b/harms/types_test.go @@ -0,0 +1,71 @@ +package harms + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContentClassString(t *testing.T) { + assert.Equal(t, "Neutral", ContentClassNeutral.String()) + assert.Equal(t, "Prohibited", ContentClassProhibited.String()) + assert.Equal(t, "Allowed", ContentClassAllowed.String()) +} + +//goland:noinspection GoBoolExpressions (GoLand wants to simplify these tests to assert.True(t, true), which isn't helpful) +func TestContentClassOrder(t *testing.T) { + assert.True(t, ContentClassNeutral < ContentClassAllowed) + assert.True(t, ContentClassAllowed < ContentClassProhibited) +} + +func TestProhibitedContent(t *testing.T) { + c := ProhibitedContent() + assert.NotNil(t, c) + assert.Equal(t, []Harm{OtherGeneral}, c.Harms()) // should default to OtherGeneral + assert.Equal(t, ContentClassProhibited, c.Class()) + + c = ProhibitedContent(SpamFraud, PolicyservMedia) + assert.NotNil(t, c) + assert.Equal(t, []Harm{SpamFraud, PolicyservMedia}, c.Harms()) + assert.Equal(t, ContentClassProhibited, c.Class()) +} + +func TestNeutralContent(t *testing.T) { + c := NeutralContent() + assert.NotNil(t, c) + assert.Equal(t, make([]Harm, 0), c.Harms()) + assert.Equal(t, ContentClassNeutral, c.Class()) +} + +func TestAllowedContent(t *testing.T) { + c := AllowedContent() + assert.NotNil(t, c) + assert.Equal(t, make([]Harm, 0), c.Harms()) + assert.Equal(t, ContentClassAllowed, c.Class()) +} + +func TestContentInfo(t *testing.T) { + c := &ContentInfo{} // zero value testing + assert.Equal(t, ContentClassNeutral, c.Class()) + assert.Equal(t, make([]Harm, 0), c.Harms()) + + c = NewContentInfo(ContentClassAllowed) + assert.Equal(t, ContentClassAllowed, c.Class()) + assert.Equal(t, make([]Harm, 0), c.Harms()) + + c = NewContentInfo(ContentClassAllowed, SpamFlooding) + assert.Equal(t, ContentClassProhibited, c.Class()) // overwritten because we provided a harm to the constructor + assert.Equal(t, []Harm{SpamFlooding}, c.Harms()) + + c = NewContentInfo(ContentClassProhibited) + assert.Equal(t, ContentClassProhibited, c.Class()) + assert.Equal(t, []Harm{OtherGeneral}, c.Harms()) // default when no harms are specified +} + +func TestHarmsDeduplicated(t *testing.T) { + assert.Equal(t, []Harm{SpamGeneral, SpamFlooding}, deduplicateHarms([]Harm{SpamFlooding, SpamGeneral, SpamFlooding, SpamFlooding, SpamGeneral})) + + // Ensure the constructors are also deduplicating + assert.Equal(t, []Harm{SpamGeneral, SpamFlooding}, ProhibitedContent(SpamFlooding, SpamGeneral, SpamFlooding, SpamFlooding, SpamGeneral).Harms()) + assert.Equal(t, []Harm{SpamGeneral, SpamFlooding}, NewContentInfo(ContentClassProhibited, SpamFlooding, SpamGeneral, SpamFlooding, SpamGeneral).Harms()) +} diff --git a/homeserver/api_msc4284_check.go b/homeserver/api_msc4284_check.go index 1327056..2938ff4 100644 --- a/homeserver/api_msc4284_check.go +++ b/homeserver/api_msc4284_check.go @@ -9,6 +9,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics" "github.com/matrix-org/policyserv/queue" "github.com/matrix-org/policyserv/storage" @@ -82,13 +83,13 @@ func httpMSC4284Check(server *Homeserver, w http.ResponseWriter, r *http.Request return } - if res.IsProbablySpam { + if res.ContentInfo.Class() == harms.ContentClassProhibited { redactIfNeeded(r.Context(), server, fedReq.Origin(), event) } defer metrics.RecordHttpResponse(r.Method, "httpMSC4284Check", http.StatusOK) w.Header().Set("Content-Type", "application/json") - if res.IsProbablySpam { + if res.ContentInfo.Class() == harms.ContentClassProhibited { _, _ = w.Write(msc4284SpamResponse) } else { _, _ = w.Write(msc4284NeutralResponse) diff --git a/homeserver/api_msc4284_sign.go b/homeserver/api_msc4284_sign.go index 1095744..5920923 100644 --- a/homeserver/api_msc4284_sign.go +++ b/homeserver/api_msc4284_sign.go @@ -8,6 +8,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics" "github.com/matrix-org/policyserv/queue" ) @@ -37,7 +38,7 @@ func handlePolicySignRequest(server *Homeserver, w http.ResponseWriter, r *http. if room == nil { // we must have a room_id to know if we should sign it. // Notably the create event in v12 rooms will omit this. - refuseToSign(w, r, stable) + refuseToSign(w, r, stable, nil) return } @@ -45,7 +46,7 @@ func handlePolicySignRequest(server *Homeserver, w http.ResponseWriter, r *http. event, err := roomVersion.NewEventFromUntrustedJSON(fedReq.Content()) if err != nil { log.Println("Error parsing event:", err) - refuseToSign(w, r, stable) + refuseToSign(w, r, stable, nil) return } @@ -56,7 +57,7 @@ func handlePolicySignRequest(server *Homeserver, w http.ResponseWriter, r *http. }) if err != nil { log.Printf("Signature verification failed for %s: %s", event.EventID(), err) - refuseToSign(w, r, stable) + refuseToSign(w, r, stable, nil) return } @@ -67,7 +68,7 @@ func handlePolicySignRequest(server *Homeserver, w http.ResponseWriter, r *http. err = server.RunFilters(r.Context(), event, ch) if err != nil { log.Println("Error submitting event:", err) - refuseToSign(w, r, stable) + refuseToSign(w, r, stable, nil) redactIfNeeded(r.Context(), server, fedReq.Origin(), event) return } @@ -85,14 +86,14 @@ func handlePolicySignRequest(server *Homeserver, w http.ResponseWriter, r *http. if res.Err != nil { log.Println("Error receiving event result:", err) - refuseToSign(w, r, stable) + refuseToSign(w, r, stable, nil) redactIfNeeded(r.Context(), server, fedReq.Origin(), event) return } - if res.IsProbablySpam { + if res.ContentInfo.Class() == harms.ContentClassProhibited { log.Printf("🚫 [%s] refusing to sign in %s", event.EventID(), event.RoomID().String()) - refuseToSign(w, r, stable) + refuseToSign(w, r, stable, res.ContentInfo.Harms()) redactIfNeeded(r.Context(), server, fedReq.Origin(), event) return } @@ -123,10 +124,23 @@ func redactIfNeeded(ctx context.Context, server *Homeserver, requestOrigin spec. } } -func refuseToSign(w http.ResponseWriter, r *http.Request, stable bool) { +func refuseToSign(w http.ResponseWriter, r *http.Request, stable bool, harmIds []harms.Harm) { // Stable endpoints allow us to return real errors, so we should do that. if stable { - MatrixHttpError(w, http.StatusBadRequest, "M_FORBIDDEN", "This message is not allowed by the policy server") + if len(harmIds) > 0 { + // TODO: We should be using the api.errorResponder, but that needs extracting out of its package. + // For now we duplicate code. + MustServeError(w, &ClientError{ + HttpCode: http.StatusBadRequest, + Errcode: "M_FORBIDDEN", + Message: "This message is not allowed by the policy server", + AdditionalFields: map[string]any{ + "org.matrix.msc4387.harms": harmIds, + }, + }) + } else { + MatrixHttpError(w, http.StatusBadRequest, "M_FORBIDDEN", "This message is not allowed by the policy server") + } return } diff --git a/homeserver/api_transactions.go b/homeserver/api_transactions.go index 9cd2b6a..b08ff8d 100644 --- a/homeserver/api_transactions.go +++ b/homeserver/api_transactions.go @@ -10,6 +10,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics" "github.com/matrix-org/policyserv/queue" "github.com/matrix-org/policyserv/storage" @@ -137,7 +138,7 @@ func httpTransactionReceive(server *Homeserver, w http.ResponseWriter, r *http.R return } - if res.IsProbablySpam { + if res.ContentInfo.Class() == harms.ContentClassProhibited { redactIfNeeded(ctx, server, "not_a_real_server_to_always_fail_the_included_sender_check", event) } }() diff --git a/homeserver/learn_state.go b/homeserver/learn_state.go index f5612f5..fd12657 100644 --- a/homeserver/learn_state.go +++ b/homeserver/learn_state.go @@ -9,6 +9,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/queue" "github.com/matrix-org/policyserv/storage" ) @@ -86,7 +87,7 @@ func (h *Homeserver) queueLearnStateIfNeeded(ctx context.Context, basedOnResult if basedOnResult.Err != nil { return // we're not interested in this result } - if basedOnResult.IsProbablySpam { + if basedOnResult.ContentInfo.Class() == harms.ContentClassProhibited { return // too spammy for us } diff --git a/homeserver/learn_state_test.go b/homeserver/learn_state_test.go index 84837ec..742e755 100644 --- a/homeserver/learn_state_test.go +++ b/homeserver/learn_state_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/internal" "github.com/matrix-org/policyserv/queue" "github.com/matrix-org/policyserv/storage" @@ -183,8 +184,8 @@ func TestQueueLearnStateIfNeeded(t *testing.T) { // Spammy or errored results should *not* lead to a queued state learn attempt res := &queue.PoolResult{ - Err: errors.New("should fail"), - IsProbablySpam: true, + Err: errors.New("should fail"), + ContentInfo: harms.ProhibitedContent(), } hs.queueLearnStateIfNeeded(context.Background(), res, event) assert.Equal(t, 0, hs.stateLearner.(*expectCreateEventLearner).canLearnCallCount) @@ -222,7 +223,7 @@ func TestQueueLearnStateIfNeeded(t *testing.T) { // If shouldLearnState throws an error, the room's state should not be learned. We simulate this by not // creating a known room for the event, so the function should fail. - res.IsProbablySpam = false // flag the event as not spam + res.ContentInfo = harms.NeutralContent() // flag the event as not spam hs.queueLearnStateIfNeeded(context.Background(), res, event) assert.Equal(t, 0, hs.stateLearner.(*expectCreateEventLearner).canLearnCallCount) assert.Equal(t, 0, hs.stateLearner.(*expectCreateEventLearner).doLearnCallCount) diff --git a/homeserver/run_filters.go b/homeserver/run_filters.go index ec733bb..855756b 100644 --- a/homeserver/run_filters.go +++ b/homeserver/run_filters.go @@ -27,9 +27,8 @@ func (h *Homeserver) RunFilters(ctx context.Context, event gomatrixserverlib.PDU } log.Printf("[%s | %s] Request context cancelled, forcing error response: %s", event.EventID(), event.RoomID().String(), err) res = &queue.PoolResult{ - Vectors: nil, - IsProbablySpam: false, - Err: err, + ContentInfo: nil, + Err: err, } } diff --git a/metrics/processor.go b/metrics/processor.go index 7d2e1eb..70b89b8 100644 --- a/metrics/processor.go +++ b/metrics/processor.go @@ -3,7 +3,7 @@ package metrics import ( "strconv" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -37,18 +37,17 @@ func RecordFailedEventCheck(roomId string) { }).Inc() } -func RecordSuccessfulEventCheck(roomId string, isFirstTimeCheck bool, vecs confidence.Vectors) { +func RecordSuccessfulEventCheck(roomId string, isFirstTimeCheck bool, info *harms.ContentInfo) { EventChecks.With(prometheus.Labels{ "roomId": roomId, "status": "ok", "isFirstTime": strconv.FormatBool(isFirstTimeCheck), }).Inc() - if isFirstTimeCheck { - // We're less concerned about the vector value and more about the types of classifications applied. - for cls, _ := range vecs { + if isFirstTimeCheck && info.Class() == harms.ContentClassProhibited { + for _, h := range info.Harms() { EventClassifications.With(prometheus.Labels{ "roomId": roomId, - "classification": cls.String(), + "classification": string(h), }).Inc() } } diff --git a/queue/pool.go b/queue/pool.go index ef3b872..a4c7e8e 100644 --- a/queue/pool.go +++ b/queue/pool.go @@ -7,33 +7,28 @@ import ( "log" "time" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/community" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/media" "github.com/matrix-org/policyserv/metrics" "github.com/matrix-org/policyserv/storage" - "github.com/matrix-org/gomatrixserverlib" "github.com/panjf2000/ants/v2" "github.com/prometheus/client_golang/prometheus" typedsf "github.com/t2bot/go-typed-singleflight" ) type PoolResult struct { - // Nil if there was an error. Otherwise, contains filter results. - Vectors confidence.Vectors - - // True when the event is considered spam. False indicates neutrality or not-spam. - // False if there was an error. - IsProbablySpam bool + // Nil if there was an error. + ContentInfo *harms.ContentInfo // The error processing the event, if any. Err error } type sfResult struct { - firstTimeSeen bool - vecs confidence.Vectors - isProbablySpam bool + firstTimeSeen bool + contentInfo *harms.ContentInfo } type PoolConfig struct { @@ -80,7 +75,7 @@ func (p *Pool) Submit(ctx context.Context, event gomatrixserverlib.PDU, mediaDow t := metrics.StartQueueTimer() // Note: waitCh might be nil or unbuffered, so we spawn this in a goroutine later on. - notifyResult := func(vecs confidence.Vectors, isSpam bool, err error) { + notifyResult := func(info *harms.ContentInfo, err error) { if err == nil { t.ObserveDurationWithExemplar(prometheus.Labels{"waitedUntil": "result"}) } else if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { @@ -91,9 +86,8 @@ func (p *Pool) Submit(ctx context.Context, event gomatrixserverlib.PDU, mediaDow if waitCh != nil { res := &PoolResult{ - Vectors: vecs, - IsProbablySpam: isSpam, - Err: err, + ContentInfo: info, + Err: err, } // First, check to see if the channel is likely going to be closed already @@ -116,7 +110,7 @@ func (p *Pool) Submit(ctx context.Context, event gomatrixserverlib.PDU, mediaDow // If the context is cancelled, save CPU and don't bother checking if err := ctx.Err(); err != nil { defer metrics.RecordFailedEventCheck(event.RoomID().String()) - go notifyResult(nil, false, err) + go notifyResult(nil, err) if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { log.Printf("Not checking %s because context was cancelled/timed out", event.EventID()) return @@ -139,7 +133,7 @@ func (p *Pool) Submit(ctx context.Context, event gomatrixserverlib.PDU, mediaDow if err != nil { defer metrics.RecordFailedEventCheck(event.RoomID().String()) } else { - defer metrics.RecordSuccessfulEventCheck(event.RoomID().String(), res.firstTimeSeen, res.vecs) + defer metrics.RecordSuccessfulEventCheck(event.RoomID().String(), res.firstTimeSeen, res.contentInfo) } return res, err }) @@ -149,9 +143,9 @@ func (p *Pool) Submit(ctx context.Context, event gomatrixserverlib.PDU, mediaDow // "should never happen" err = errors.New("nil result") } - go notifyResult(nil, false, err) + go notifyResult(nil, err) } else { - go notifyResult(res.vecs, res.isProbablySpam, err) + go notifyResult(res.contentInfo, err) } } @@ -166,9 +160,8 @@ func (p *Pool) doFilter(ctx context.Context, event gomatrixserverlib.PDU, mediaD } if res != nil { return &sfResult{ - firstTimeSeen: false, - isProbablySpam: res.IsProbablySpam, - vecs: res.ConfidenceVectors, + firstTimeSeen: false, + contentInfo: res.ContentInfo, }, nil } @@ -182,23 +175,19 @@ func (p *Pool) doFilter(ctx context.Context, event gomatrixserverlib.PDU, mediaD } // Run the event through the filters - vecs, err := set.CheckEvent(ctx, event, mediaDownloader) + info, err := set.CheckEvent(ctx, event, mediaDownloader) if err != nil { return nil, err } - isSpam := set.IsSpamResponse(ctx, vecs) - if isSpam { - log.Printf("%s in %s is spam", event.EventID(), event.RoomID().String()) - } else { - log.Printf("%s in %s is neutral", event.EventID(), event.RoomID().String()) - } + isSpam := info.Class() == harms.ContentClassProhibited + log.Printf("%s in %s is %s", event.EventID(), event.RoomID().String(), info.Class().String()) // Persist results err = p.storage.UpsertEventResult(ctx, &storage.StoredEventResult{ - EventId: event.EventID(), - IsProbablySpam: isSpam, - ConfidenceVectors: vecs, + EventId: event.EventID(), + IsProbablySpam: isSpam, + ContentInfo: info, }) if err != nil { return nil, err @@ -206,8 +195,7 @@ func (p *Pool) doFilter(ctx context.Context, event gomatrixserverlib.PDU, mediaD // Finally, return return &sfResult{ - firstTimeSeen: true, - isProbablySpam: isSpam, - vecs: vecs, + firstTimeSeen: true, + contentInfo: info, }, nil } diff --git a/queue/pool_test.go b/queue/pool_test.go index 163036b..d964f15 100644 --- a/queue/pool_test.go +++ b/queue/pool_test.go @@ -6,8 +6,7 @@ import ( "github.com/matrix-org/policyserv/community" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/storage" "github.com/matrix-org/policyserv/test" "github.com/stretchr/testify/assert" @@ -48,9 +47,9 @@ func TestPool(t *testing.T) { // Cache a result to ensure the happy path works res := &storage.StoredEventResult{ - EventId: event.EventID(), - IsProbablySpam: true, - ConfidenceVectors: confidence.Vectors{classification.Spam: 0.5, classification.Mentions: 1.0}, + EventId: event.EventID(), + IsProbablySpam: true, + ContentInfo: harms.ProhibitedContent(harms.SpamGeneral, harms.SpamFlooding), } err = db.UpsertEventResult(context.Background(), res) assert.NoError(t, err) @@ -62,10 +61,10 @@ func TestPool(t *testing.T) { poolResult := <-ch assert.NotNil(t, poolResult) assert.Equal(t, &PoolResult{ - Vectors: res.ConfidenceVectors, - IsProbablySpam: res.IsProbablySpam, - Err: nil, + ContentInfo: res.ContentInfo, + Err: nil, }, poolResult) + test.AssertEqualContentInfo(t, res.ContentInfo, poolResult.ContentInfo) } func TestPoolHandlesErrors(t *testing.T) { @@ -110,8 +109,7 @@ func TestPoolHandlesErrors(t *testing.T) { poolResult := <-ch assert.NotNil(t, poolResult) assert.Equal(t, &PoolResult{ - Vectors: nil, - IsProbablySpam: false, - Err: test.SimulatedError, + ContentInfo: nil, + Err: test.SimulatedError, }, poolResult) } diff --git a/storage/interface.go b/storage/interface.go index 5e436b6..1251657 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -4,11 +4,11 @@ import ( "context" "database/sql/driver" "encoding/json" + "fmt" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/classification" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" ) type StoredRoom struct { @@ -20,9 +20,9 @@ type StoredRoom struct { } type StoredEventResult struct { - EventId string `json:"event_id"` - IsProbablySpam bool `json:"is_probably_spam"` - ConfidenceVectors confidence.Vectors `json:"confidence_vectors"` + EventId string `json:"event_id"` + IsProbablySpam bool `json:"is_probably_spam"` + ContentInfo *harms.ContentInfo `json:"-"` // can't be exported to/imported from JSON } type StoredCommunity struct { @@ -62,15 +62,22 @@ type MatrixTransaction struct { Edus []gomatrixserverlib.EDU } -// StoredClassifications implements the SQL driver interface for scanning/setting values. Note that the -// Value() and Scan() functions are on different receivers - this is because the Value() is not going to -// be on a pointer (see StoredMediaClassification), but Scan() will always be called on a pointer. If Scan() -// was changed to use a value (non-pointer) receiver instead, the value we read from the database would never -// actually leave the function call, confusing the calling code. -type StoredClassifications []classification.Classification +// StoredClassifications implements the SQL driver interface for scanning/setting values. Note that this used to +// be a []classification.Classification type, but was modernized to use harms instead. Legacy data is still possible +// and we need to maintain backwards compatibility, so we go out of our way to reinterpret the simple array into a +// harms.ContentInfo struct. +// Note that difference receivers are used for Value and Scan - this is because Go's pointers are expecting certain +// values depending on whether it's being read or written. +type StoredClassifications struct { + *harms.ContentInfo +} func (c StoredClassifications) Value() (driver.Value, error) { - return json.Marshal(c) + strArray := []string{c.ContentInfo.Class().String()} // put the content class first for easy decoding + for _, h := range c.ContentInfo.Harms() { + strArray = append(strArray, string(h)) + } + return json.Marshal(strArray) } func (c *StoredClassifications) Scan(src interface{}) error { @@ -78,7 +85,37 @@ func (c *StoredClassifications) Scan(src interface{}) error { if !ok { return nil } - return json.Unmarshal(b, &c) + strArray := make([]string, 0) + err := json.Unmarshal(b, &strArray) + if err != nil { + return err + } + if len(strArray) > 0 { + if strArray[0] == harms.ContentClassNeutral.String() { + if len(strArray) != 1 { + return fmt.Errorf("invalid neutral content classification: %v", strArray) + } + c.ContentInfo = harms.NeutralContent() + } else if strArray[0] == harms.ContentClassAllowed.String() { + if len(strArray) != 1 { + return fmt.Errorf("invalid allowed content classification: %v", strArray) + } + c.ContentInfo = harms.AllowedContent() + } else { + // Assume it's prohibited content + if strArray[0] == harms.ContentClassProhibited.String() { + strArray = strArray[1:] // remove the content class if it was specified + } + harmIds := make([]harms.Harm, 0) + for _, h := range strArray { + harmIds = append(harmIds, harms.Harm(h)) + } + c.ContentInfo = harms.ProhibitedContent(harmIds...) + } + } else { + c.ContentInfo = harms.NeutralContent() + } + return nil } type Transaction interface { // mirror of sql.Tx interface for ease of compatibility diff --git a/storage/psql.go b/storage/psql.go index 5dc4473..d94d378 100644 --- a/storage/psql.go +++ b/storage/psql.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "log" + "strings" "time" cache "github.com/Code-Hex/go-generics-cache" @@ -16,7 +17,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/policyserv/config" - "github.com/matrix-org/policyserv/filter/confidence" + "github.com/matrix-org/policyserv/harms" "github.com/matrix-org/policyserv/metrics/dbmetrics" "github.com/ryanuber/go-glob" "golang.org/x/sync/singleflight" @@ -280,11 +281,34 @@ func (s *PostgresStorage) GetEventResult(ctx context.Context, eventId string) (* } if encodedVectors != "" { - if err := json.Unmarshal([]byte(encodedVectors), &eventResult.ConfidenceVectors); err != nil { + // Legacy code populated vectors as a map[string]float64, so for backwards compatibility we retain that. + decodedVectors := make(map[string]float64) + if err := json.Unmarshal([]byte(encodedVectors), &decodedVectors); err != nil { return nil, err } + wasAllowed := false + harmIds := make([]harms.Harm, 0) + for k, _ := range decodedVectors { + // Per UpsertEventResult, we encode the content class if it's allowed as a "vector". Neutral is never encoded, + // and prohibited is implied by the presence of harms (or other legacy vectors). + if k == harms.ContentClassAllowed.String() { + wasAllowed = true + continue + } + // TODO: We should probably do intelligent mapping of legacy vectors to harms. For now, we just namespace them. + if strings.HasPrefix(k, "h:") { + harmIds = append(harmIds, harms.Harm(k[2:])) + } else { + harmIds = append(harmIds, harms.Harm("org.matrix.policyserv.vec."+k)) + } + } + if wasAllowed && len(harmIds) == 0 { + // an event's content info can only be "allowed" if no other harms were found + eventResult.ContentInfo = harms.AllowedContent() + } + eventResult.ContentInfo = harms.ProhibitedContent(harmIds...) } else { - eventResult.ConfidenceVectors = confidence.NewConfidenceVectors() // populate empty + eventResult.ContentInfo = harms.NeutralContent() } return eventResult, nil @@ -294,7 +318,24 @@ func (s *PostgresStorage) UpsertEventResult(ctx context.Context, event *StoredEv t := dbmetrics.StartSelfDatabaseTimer("UpsertEventResult") defer t.ObserveDuration() - encodedVectors, err := json.Marshal(event.ConfidenceVectors) + // Legacy code populated vectors as a map[string]float64, so for backwards compatibility we retain that. + decodedVectors := make(map[string]float64) + for _, h := range event.ContentInfo.Harms() { + // TODO: Like above, we should probably do intelligent mapping of legacy vectors to harms. For now, we just namespace them. + if strings.HasPrefix(string(h), "org.matrix.policyserv.vec.") { + // Retain namespace-free vectors in the database. This should only happen if we're updating an event + // result rather than on insert, but it's still possible. + decodedVectors[string(h)[len("org.matrix.policyserv.vec."):]] = 1.0 + } else { + // Otherwise, prefix "new" harm IDs with `h:` so GetEventResult can find them. + decodedVectors["h:"+string(h)] = 1.0 + } + } + if event.ContentInfo.Class() == harms.ContentClassAllowed { + decodedVectors[harms.ContentClassAllowed.String()] = 1.0 + } + + encodedVectors, err := json.Marshal(decodedVectors) if err != nil { return err } diff --git a/test/content_info.go b/test/content_info.go new file mode 100644 index 0000000..ef91aaf --- /dev/null +++ b/test/content_info.go @@ -0,0 +1,15 @@ +package test + +import ( + "testing" + + "github.com/matrix-org/policyserv/harms" + "github.com/stretchr/testify/assert" +) + +func AssertEqualContentInfo(t *testing.T, expected *harms.ContentInfo, actual *harms.ContentInfo) { + assert.NotNil(t, expected) + assert.NotNil(t, actual) + assert.Equal(t, expected.Class().String(), actual.Class().String()) // compare strings to make debugging easier + assert.Equal(t, expected.Harms(), actual.Harms()) +} diff --git a/test/content_scanner.go b/test/content_scanner.go index 3f516c2..c28a78e 100644 --- a/test/content_scanner.go +++ b/test/content_scanner.go @@ -5,14 +5,14 @@ import ( "testing" "github.com/matrix-org/policyserv/content" - "github.com/matrix-org/policyserv/filter/classification" + "github.com/matrix-org/policyserv/harms" "github.com/stretchr/testify/assert" ) type scanExpectation struct { contentType content.Type content []byte - expected []classification.Classification + expected *harms.ContentInfo err error } @@ -25,11 +25,11 @@ func NewMemoryContentScanner(t *testing.T) *MemoryContentScanner { return &MemoryContentScanner{T: t} } -func (m *MemoryContentScanner) Expect(contentType content.Type, content []byte, retClassifications []classification.Classification, retErr error) { - m.expectations = append(m.expectations, scanExpectation{contentType, content, retClassifications, retErr}) +func (m *MemoryContentScanner) Expect(contentType content.Type, content []byte, retInfo *harms.ContentInfo, retErr error) { + m.expectations = append(m.expectations, scanExpectation{contentType, content, retInfo, retErr}) } -func (m *MemoryContentScanner) Scan(ctx context.Context, contentType content.Type, content []byte) ([]classification.Classification, error) { +func (m *MemoryContentScanner) Scan(ctx context.Context, contentType content.Type, content []byte) (*harms.ContentInfo, error) { assert.NotNil(m.T, ctx, "context is required") for _, exp := range m.expectations {