diff --git a/go.mod b/go.mod index 38425c1..bcc029d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Clarit-AI/Plexium go 1.25.1 require ( - github.com/Clarit-AI/markedup v0.0.0-20260418185353-12e3cc5f0d89 + github.com/Clarit-AI/markedup v0.0.0-20260419063450-0c5745b5a986 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/gobwas/glob v0.2.3 github.com/pelletier/go-toml/v2 v2.1.0 @@ -33,6 +33,7 @@ require ( golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.3.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index becb1da..405bf7c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Clarit-AI/markedup v0.0.0-20260418185353-12e3cc5f0d89 h1:7DxgiIXe17JLwfjaE+YnAmMzzTqneVvjMCn7baLY4G4= -github.com/Clarit-AI/markedup v0.0.0-20260418185353-12e3cc5f0d89/go.mod h1:IMDGULtUy68kQ1dKlDgT7gD9lD7KJZ0dDRDPqf79x9s= +github.com/Clarit-AI/markedup v0.0.0-20260419063450-0c5745b5a986 h1:7wQfT3hNTXKTcAFrJnQMJmOTzxAJ6/HQKUiDDpfbS9o= +github.com/Clarit-AI/markedup v0.0.0-20260419063450-0c5745b5a986/go.mod h1:Dpgx+f+jzufgRjrHkujDruoKOJOh0H7EtHcyRpliXWQ= github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI= github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -72,6 +72,8 @@ golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/convert/ingest.go b/internal/convert/ingest.go index ec4f8f2..ebaec0c 100644 --- a/internal/convert/ingest.go +++ b/internal/convert/ingest.go @@ -257,7 +257,7 @@ func (ing *Ingestor) buildModulePage(readme ReadmeDoc) string { b.WriteString(fmt.Sprintf("last-updated: %s\n", now)) b.WriteString("updated-by: plexium-convert\n") b.WriteString(fmt.Sprintf("source-files: [\"%s\"]\n", readme.Path)) - b.WriteString("confidence: high\n") + b.WriteString("confidence: 0.9\n") b.WriteString("review-status: unreviewed\n") b.WriteString("---\n\n") b.WriteString(readme.Content) @@ -275,7 +275,7 @@ func (ing *Ingestor) buildModuleStubPage(name string, files []string) string { b.WriteString("ownership: managed\n") b.WriteString(fmt.Sprintf("last-updated: %s\n", now)) b.WriteString("updated-by: plexium-convert\n") - b.WriteString("confidence: medium\n") + b.WriteString("confidence: 0.5\n") b.WriteString("review-status: unreviewed\n") b.WriteString("---\n\n") b.WriteString(fmt.Sprintf("# %s\n\n", title)) @@ -315,7 +315,7 @@ func (ing *Ingestor) buildGuidePage(title, content string) string { b.WriteString("ownership: managed\n") b.WriteString(fmt.Sprintf("last-updated: %s\n", now)) b.WriteString("updated-by: plexium-convert\n") - b.WriteString("confidence: medium\n") + b.WriteString("confidence: 0.5\n") b.WriteString("review-status: unreviewed\n") b.WriteString("---\n\n") b.WriteString(content) diff --git a/internal/convert/lint.go b/internal/convert/lint.go index 489f607..732070d 100644 --- a/internal/convert/lint.go +++ b/internal/convert/lint.go @@ -131,7 +131,7 @@ func (cl *ConvertLinter) createStub(moduleName string) PageData { b.WriteString("---\n") b.WriteString(fmt.Sprintf("title: %q\n", title)) b.WriteString("ownership: managed\n") - b.WriteString("confidence: low\n") + b.WriteString("confidence: 0.1\n") b.WriteString("review-status: unreviewed\n") b.WriteString("---\n\n") b.WriteString(fmt.Sprintf("# %s\n\n", title)) diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 15538b4..ef8940a 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -43,13 +43,14 @@ type PageEntry struct { // "I have zero confidence" signal is semantically equivalent to // "unset" for current consumers — if we ever need to distinguish // the two cases we should switch this to *float64 at that point. - EntityType string `json:"entityType,omitempty"` - Entities []EntityRef `json:"entities,omitempty"` - Relationships []RelationshipRef `json:"relationships,omitempty"` - Confidence float64 `json:"confidence,omitempty"` - SemanticHints []string `json:"semanticHints,omitempty"` - LastEnriched string `json:"lastEnriched,omitempty"` - EnrichedBy string `json:"enrichedBy,omitempty"` + EntityType string `json:"entityType,omitempty"` + Entities []EntityRef `json:"entities,omitempty"` + Relationships []RelationshipRef `json:"relationships,omitempty"` + SemanticRelationships []RelationshipRef `json:"semanticRelationships,omitempty"` + Confidence float64 `json:"confidence,omitempty"` + SemanticHints []string `json:"semanticHints,omitempty"` + LastEnriched string `json:"lastEnriched,omitempty"` + EnrichedBy string `json:"enrichedBy,omitempty"` } // EntityRef is a reference to a named entity on a page. It mirrors the @@ -182,13 +183,14 @@ func NewEmptyManifest() *Manifest { // GraphMetadata bundles the knowledge-graph fields set by an enrichment // plugin. It's accepted by ApplyGraphMetadata to update a page in-place. type GraphMetadata struct { - EntityType string - Entities []EntityRef - Relationships []RelationshipRef - Confidence float64 - SemanticHints []string - LastEnriched string - EnrichedBy string + EntityType string + Entities []EntityRef + Relationships []RelationshipRef + SemanticRelationships []RelationshipRef + Confidence float64 + SemanticHints []string + LastEnriched string + EnrichedBy string } // ApplyGraphMetadata overwrites the v2 graph fields on the page entry with @@ -204,6 +206,7 @@ func (m *Manifest) ApplyGraphMetadata(wikiPath string, g GraphMetadata) bool { m.Pages[i].EntityType = g.EntityType m.Pages[i].Entities = g.Entities m.Pages[i].Relationships = g.Relationships + m.Pages[i].SemanticRelationships = g.SemanticRelationships m.Pages[i].Confidence = g.Confidence m.Pages[i].SemanticHints = g.SemanticHints m.Pages[i].LastEnriched = g.LastEnriched @@ -222,6 +225,7 @@ func hasGraphFields(g GraphMetadata) bool { return g.EntityType != "" || len(g.Entities) > 0 || len(g.Relationships) > 0 || + len(g.SemanticRelationships) > 0 || g.Confidence != 0 || len(g.SemanticHints) > 0 || g.LastEnriched != "" || @@ -243,13 +247,14 @@ func (m *Manifest) GraphMetadataForPage(wikiPath string) (GraphMetadata, bool) { } p := m.Pages[i] return GraphMetadata{ - EntityType: p.EntityType, - Entities: p.Entities, - Relationships: p.Relationships, - Confidence: p.Confidence, - SemanticHints: p.SemanticHints, - LastEnriched: p.LastEnriched, - EnrichedBy: p.EnrichedBy, + EntityType: p.EntityType, + Entities: p.Entities, + Relationships: p.Relationships, + SemanticRelationships: p.SemanticRelationships, + Confidence: p.Confidence, + SemanticHints: p.SemanticHints, + LastEnriched: p.LastEnriched, + EnrichedBy: p.EnrichedBy, }, true } return GraphMetadata{}, false @@ -290,6 +295,14 @@ func GraphMetadataSemanticEqual(a, b GraphMetadata) bool { return false } } + if len(a.SemanticRelationships) != len(b.SemanticRelationships) { + return false + } + for i := range a.SemanticRelationships { + if a.SemanticRelationships[i] != b.SemanticRelationships[i] { + return false + } + } if len(a.SemanticHints) != len(b.SemanticHints) { return false } diff --git a/internal/plugins/markedup/config.go b/internal/plugins/markedup/config.go index 8b0b8eb..93aa5a7 100644 --- a/internal/plugins/markedup/config.go +++ b/internal/plugins/markedup/config.go @@ -18,6 +18,7 @@ package markedup import ( "fmt" + "strings" "time" ) @@ -144,9 +145,9 @@ func DefaultConfig() Config { // root of the plugins.markedup block. Anything else is a config error. var allowedTopLevelKeys = map[string]struct{}{ "enabled": {}, - "autoEnrich": {}, - "modelEnrich": {}, - "writeEnrichedFrontmatter": {}, + "autoenrich": {}, + "modelenrich": {}, + "writeenrichedfrontmatter": {}, "embeddings": {}, "reranking": {}, "daemon": {}, @@ -155,7 +156,7 @@ var allowedTopLevelKeys = map[string]struct{}{ // allowedDaemonKeys constrains the nested daemon block (see allowed*Keys // for the rationale). var allowedDaemonKeys = map[string]struct{}{ - "refreshInterval": {}, + "refreshinterval": {}, } // allowedEmbeddingsKeys and allowedRerankingKeys constrain the nested @@ -166,7 +167,7 @@ var allowedEmbeddingsKeys = map[string]struct{}{ "provider": {}, "model": {}, "endpoint": {}, - "apiKeyEnv": {}, + "apikeyenv": {}, "dims": {}, } @@ -175,7 +176,7 @@ var allowedRerankingKeys = map[string]struct{}{ "provider": {}, "model": {}, "endpoint": {}, - "apiKeyEnv": {}, + "apikeyenv": {}, } // ParseConfig reads the plugins.markedup block from the raw map form used @@ -191,6 +192,8 @@ var allowedRerankingKeys = map[string]struct{}{ // - An unknown key, a malformed nested block, or an invalid value // (e.g. non-positive embeddings.dims) returns an error. func ParseConfig(raw map[string]any) (Config, error) { + normalizeKeys(raw) + cfg := DefaultConfig() cfg.Enabled = false // strict: only an explicit `enabled: true` turns it on if len(raw) == 0 { @@ -206,19 +209,19 @@ func ParseConfig(raw map[string]any) (Config, error) { } else if _, present := raw["enabled"]; present { return Config{}, fmt.Errorf("markedup.enabled must be a boolean") } - if v, ok := raw["autoEnrich"].(bool); ok { + if v, ok := raw["autoenrich"].(bool); ok { cfg.AutoEnrich = v - } else if _, present := raw["autoEnrich"]; present { + } else if _, present := raw["autoenrich"]; present { return Config{}, fmt.Errorf("markedup.autoEnrich must be a boolean") } - if v, ok := raw["modelEnrich"].(bool); ok { + if v, ok := raw["modelenrich"].(bool); ok { cfg.ModelEnrich = v - } else if _, present := raw["modelEnrich"]; present { + } else if _, present := raw["modelenrich"]; present { return Config{}, fmt.Errorf("markedup.modelEnrich must be a boolean") } - if v, ok := raw["writeEnrichedFrontmatter"].(bool); ok { + if v, ok := raw["writeenrichedfrontmatter"].(bool); ok { cfg.WriteEnrichedFrontmatter = v - } else if _, present := raw["writeEnrichedFrontmatter"]; present { + } else if _, present := raw["writeenrichedfrontmatter"]; present { return Config{}, fmt.Errorf("markedup.writeEnrichedFrontmatter must be a boolean") } @@ -244,7 +247,7 @@ func ParseConfig(raw map[string]any) (Config, error) { if v, ok := emb["endpoint"].(string); ok { cfg.Embeddings.Endpoint = v } - if v, ok := emb["apiKeyEnv"].(string); ok { + if v, ok := emb["apikeyenv"].(string); ok { cfg.Embeddings.APIKeyEnv = v } if dimsRaw, p := emb["dims"]; p { @@ -278,7 +281,7 @@ func ParseConfig(raw map[string]any) (Config, error) { if v, ok := rr["endpoint"].(string); ok { cfg.Reranking.Endpoint = v } - if v, ok := rr["apiKeyEnv"].(string); ok { + if v, ok := rr["apikeyenv"].(string); ok { cfg.Reranking.APIKeyEnv = v } } @@ -291,9 +294,9 @@ func ParseConfig(raw map[string]any) (Config, error) { if err := rejectUnknownKeys("markedup.daemon", dm, allowedDaemonKeys); err != nil { return Config{}, err } - if v, ok := dm["refreshInterval"].(string); ok { + if v, ok := dm["refreshinterval"].(string); ok { cfg.Daemon.RefreshInterval = v - } else if _, p := dm["refreshInterval"]; p { + } else if _, p := dm["refreshinterval"]; p { return Config{}, fmt.Errorf("markedup.daemon.refreshInterval must be a string (e.g. \"24h\")") } } @@ -304,11 +307,30 @@ func ParseConfig(raw map[string]any) (Config, error) { return cfg, nil } +// normalizeKeys recursively lowercases all string keys in the map. +// Viper lowercases YAML keys when decoding to map[string]any, but +// tests and other callers may pass camelCase keys directly. This +// ensures lookups in ParseConfig always use lowercase keys. +func normalizeKeys(m map[string]any) { + for k, v := range m { + lower := strings.ToLower(k) + if lower != k { + delete(m, k) + m[lower] = v + } + if nested, ok := v.(map[string]any); ok { + normalizeKeys(nested) + } + } +} + // rejectUnknownKeys returns a config error if got contains any key that // isn't in allowed. scope is prepended to the error message for context. +// Keys are compared case-insensitively because Viper lowercases all map +// keys on unmarshal. func rejectUnknownKeys(scope string, got map[string]any, allowed map[string]struct{}) error { for k := range got { - if _, ok := allowed[k]; !ok { + if _, ok := allowed[strings.ToLower(k)]; !ok { return fmt.Errorf("%s: unknown key %q", scope, k) } } diff --git a/internal/plugins/markedup/enricher.go b/internal/plugins/markedup/enricher.go index b8d2a97..0d1de2f 100644 --- a/internal/plugins/markedup/enricher.go +++ b/internal/plugins/markedup/enricher.go @@ -387,16 +387,26 @@ func toGraphMetadata(fm schema.GraphFrontmatter, timestamp string) manifest.Grap }) } + semanticRels := make([]manifest.RelationshipRef, 0, len(fm.SemanticRelationships)) + for _, r := range fm.SemanticRelationships { + semanticRels = append(semanticRels, manifest.RelationshipRef{ + Target: r.Target, + Type: r.Type, + Strength: r.Strength, + }) + } + hints := make([]string, len(fm.SemanticHints)) copy(hints, fm.SemanticHints) return manifest.GraphMetadata{ - EntityType: fm.EntityType, - Entities: entities, - Relationships: rels, - Confidence: fm.Confidence, - SemanticHints: hints, - LastEnriched: timestamp, - EnrichedBy: EnricherVersion, + EntityType: fm.EntityType, + Entities: entities, + Relationships: rels, + SemanticRelationships: semanticRels, + Confidence: fm.Confidence, + SemanticHints: hints, + LastEnriched: timestamp, + EnrichedBy: EnricherVersion, } } diff --git a/internal/plugins/markedup/enricher_modelenrich_test.go b/internal/plugins/markedup/enricher_modelenrich_test.go index e179d1d..1838dcf 100644 --- a/internal/plugins/markedup/enricher_modelenrich_test.go +++ b/internal/plugins/markedup/enricher_modelenrich_test.go @@ -35,7 +35,7 @@ func (f *fakeLLM) ExtractGraph(ctx context.Context, body string) (*enrich.ModelR // what Tier 1 produced. func TestEnricherPlugin_ModelEnrichMergesLLMResult(t *testing.T) { repoRoot, wikiRoot := setupWikiFixture(t, map[string]string{ - "a.md": "# A\n\nThis page is about authentication.\n", + "a.md": "# A\n\nRelates to [[b]].\nThis page is about authentication.\n", }) llm := &fakeLLM{ @@ -89,15 +89,25 @@ func TestEnricherPlugin_ModelEnrichMergesLLMResult(t *testing.T) { if !foundOAuth || !foundJWT { t.Errorf("expected OAuth2 and JWT entities from LLM; got %+v", entry.Entities) } - // LLM-extracted relationship should appear too. - foundRel := false + // Tier 1 wikilink should be in entry.Relationships. + foundB := false for _, r := range entry.Relationships { + if r.Target == "b" { + foundB = true + } + } + if !foundB { + t.Errorf("expected Tier 1 wikilink to survive in Relationships; got %+v", entry.Relationships) + } + // Tier 2 NER edge should be in SemanticRelationships. + foundAuthServer := false + for _, r := range entry.SemanticRelationships { if r.Target == "auth-server" && r.Type == "depends-on" { - foundRel = true + foundAuthServer = true } } - if !foundRel { - t.Errorf("expected auth-server relationship from LLM; got %+v", entry.Relationships) + if !foundAuthServer { + t.Errorf("expected Tier 2 NER edge to appear in SemanticRelationships; got %+v", entry.SemanticRelationships) } if entry.EnrichedBy != EnricherVersion { t.Errorf("expected EnrichedBy=%q, got %q", EnricherVersion, entry.EnrichedBy) diff --git a/validation/contract_test.go b/validation/contract_test.go index 21101fb..1f1142e 100644 --- a/validation/contract_test.go +++ b/validation/contract_test.go @@ -39,13 +39,14 @@ func TestContract_ManifestStructFields(t *testing.T) { // v2 knowledge-graph fields (populated by the MarkedUp enricher // plugin; all JSON-tagged `omitempty` so v1 manifests round-trip // unchanged). - "EntityType": "string", - "Entities": "[]manifest.EntityRef", - "Relationships": "[]manifest.RelationshipRef", - "Confidence": "float64", - "SemanticHints": "[]string", - "LastEnriched": "string", - "EnrichedBy": "string", + "EntityType": "string", + "Entities": "[]manifest.EntityRef", + "Relationships": "[]manifest.RelationshipRef", + "SemanticRelationships": "[]manifest.RelationshipRef", + "Confidence": "float64", + "SemanticHints": "[]string", + "LastEnriched": "string", + "EnrichedBy": "string", } for name, expectedType := range expectedFields {