feat: add conceptors

docs/pages.jl

@@ -14,10 +14,12 @@ pages = [
     ],
     "Examples" => Any[
         "Building a model to add to ReservoirComputing.jl" => "examples/model_es2n.md",
+        "Morphing patterns with conceptors" => "examples/conceptors_morphing.md",
     ],
     "API Documentation" => Any[
         "Layers" => "api/layers.md",
         "Models" => "api/models.md",
+        "Conceptors" => "api/conceptors.md",
         "Utilities" => "api/utils.md",
         "Train" => "api/train.md",
         "Predict" => "api/predict.md",

docs/src/api/conceptors.md

@@ -0,0 +1,58 @@
+# Conceptors
+
+Conceptors after Jaeger (2014), [Jaeger2014conceptors](@cite). See the
+[Morphing patterns with conceptors](@ref) example for an end-to-end walkthrough.
+
+## Conceptor matrices
+
+```@docs
+    conceptor_matrix
+    correlation_matrix
+    conceptor_from_states
+    conceptor_singular_values
+    quota
+```
+
+## Aperture adaptation
+
+```@docs
+    aperture_adapt
+    adapt_singular_value
+    reaperture
+    attenuation
+    optimal_aperture
+```
+
+## Boolean algebra
+
+```@docs
+    conceptor_not
+    conceptor_and
+    conceptor_or
+```
+
+## The `Conceptor` wrapper and its library
+
+```@docs
+    Conceptor
+    has_conceptor
+    get_conceptor
+    store_conceptor!
+    set_active_conceptor
+    active_conceptor
+```
+
+## Loading, generation, and morphing
+
+```@docs
+    load!
+    generate
+    morph_conceptor
+    ridge_map
+```
+
+## Conceptor-filtered training
+
+```@docs
+    store_conceptors!
+```

docs/src/assets/Project.toml

@@ -0,0 +1,41 @@
+[deps]
+CellularAutomata = "878138dc-5b27-11ea-1a71-cb95d38d6b29"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+DataInterpolations = "82cc6244-b520-54b8-b5a6-8a565e85f1d0"
+DelayDiffEq = "bcd4f6db-9728-5f36-b5f7-82caef46ccdb"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244"
+DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+LIBSVM = "b1bec4e5-fd48-53fe-b0cb-9723c09d164b"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
+MLJLinearModels = "6ee0df7b-362f-4a72-a706-9e79364fb692"
+OrdinaryDiffEqAdamsBashforthMoulton = "89bda076-bce5-4f1c-845f-551c83cdda9a"
+OrdinaryDiffEqTsit5 = "b1df2697-797e-41e3-8120-5422d3b24e4a"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+ReservoirComputing = "7c2d2b1e-3dd4-11ea-355a-8f6a8116e294"
+SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+Static = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
+StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+
+[compat]
+CellularAutomata = "0.0.6"
+ConcreteStructs = "0.2"
+DataInterpolations = "6, 7, 8"
+DelayDiffEq = "6"
+Documenter = "1"
+DocumenterCitations = "1"
+DocumenterInterLinks = "1"
+JLD2 = "0.6"
+LIBSVM = "0.8"
+LinearSolve = "3"
+MLJLinearModels = "0.10"
+OrdinaryDiffEqAdamsBashforthMoulton = "2"
+OrdinaryDiffEqTsit5 = "1, 2"
+Plots = "1"
+ReservoirComputing = "0.12.0"
+SciMLBase = "2.51, 3"
+Static = "1"
+StatsBase = "0.34.4"

docs/src/examples/conceptors_morphing.md

@@ -0,0 +1,127 @@
+# Morphing patterns with conceptors
+
+This example reproduces the morphing square from Jaeger (2014), *Controlling
+Recurrent Neural Networks by Conceptors* ([Jaeger2014conceptors](@cite),
+arXiv:1403.3369, Figure 2). A single reservoir is *loaded* with four patterns;
+their [`conceptor`](@ref conceptor_matrix)s are then linearly **morphed** and used
+to drive the reservoir autonomously, generating a continuum of patterns that
+interpolate between, and extrapolate beyond, the four originals.
+
+A conceptor is a positive semidefinite matrix ``C = R (R + \alpha^{-2} I)^{-1}``
+built from the correlation matrix ``R`` of a reservoir's driven states. Inserted
+into the autonomous update ``x(n) = C\,\tanh(W x(n-1) + b)``, it constrains the
+reservoir dynamics to the subspace excited by one loaded pattern, so the reservoir
+re-generates that pattern. A linear combination of conceptors morphs between them.
+
+## The four driving patterns
+
+Following the original demonstration we use two sines of slightly different
+period and two minor variations of a 5-periodic random pattern, all in
+``[-0.9, 0.9]``. The two period-5 vectors are the ones from the published figure.
+
+```@example morphing
+using ReservoirComputing
+using Random
+using Plots
+
+const PERIOD_1 = 8.8342522
+const PERIOD_2 = 9.8342522
+const DRIVER_LEN = 1500
+
+# period-5 patterns, sampled as rp[mod(n,5)+1] (values from Jaeger 2014, Fig. 1B)
+const RAND5 = [0.16, 0.9, -0.9, -0.21, -0.55]
+const PERTURB5 = [0.10, 0.9, -0.9, -0.65, -0.54]
+period5(rp, n) = rp[mod(n, 5) + 1]
+
+n = 1:DRIVER_LEN
+patterns = [
+    :s1 => reshape(sin.(2π .* n ./ PERIOD_1), 1, :),
+    :s2 => reshape(sin.(2π .* n ./ PERIOD_2), 1, :),
+    :r3 => reshape([period5(RAND5, i) for i in n], 1, :),
+    :r4 => reshape([period5(PERTURB5, i) for i in n], 1, :),
+]
+nothing # hide
+```
+
+## Building and loading the reservoir
+
+We build a 100-unit [`ESN`](@ref) with the scalings used in the report
+(spectral radius 1.5, input scaling 1.5, bias scaling 0.2) and wrap it in a
+[`Conceptor`](@ref). [`load!`](@ref) drives the reservoir with each pattern,
+stores a conceptor for it, and recomputes the recurrent weights into an
+input-internalizing matrix so the reservoir can run autonomously.
+
+```@example morphing
+rng = Xoshiro(3)
+
+bias_init(r, dims...) = 0.2f0 .* randn(r, Float32, dims...)
+input_init(r, dims...) = 1.5f0 .* randn(r, Float32, dims...)
+res_init(r, dims...) = rand_sparse(r, dims...; radius = 1.5f0, sparsity = 0.1f0)
+
+esn = ESN(1, 100, 1; use_bias = true, init_bias = bias_init,
+    init_input = input_init, init_reservoir = res_init)
+concept = Conceptor(esn)
+ps = initialparameters(rng, concept)
+st = initialstates(rng, concept)
+
+ps, st = load!(rng, concept, patterns, ps, st; aperture = 4.0, washout = 500)
+nothing # hide
+```
+
+## Morphing across the square
+
+For a grid of mixing coordinates ``a, b \in \{-0.5, \dots, 1.5\}`` we form the
+morphed conceptor ``M = \mu_1 C_{s1} + \mu_2 C_{s2} + \mu_3 C_{r3} + \mu_4 C_{r4}``
+with
+```math
+\mu_1 = (1-a)\,b,\quad \mu_2 = a\,b,\quad \mu_3 = (1-a)(1-b),\quad \mu_4 = a\,(1-b),
+```
+(the coefficients always sum to one), and run the reservoir autonomously under
+``M`` with [`generate`](@ref). All panels share one start state so their phases are
+comparable.
+
+```@example morphing
+morph_weights(a, b) = (; s1 = (1 - a) * b, s2 = a * b,
+    r3 = (1 - a) * (1 - b), r4 = a * (1 - b))
+
+grid = collect(-0.5:0.25:1.5)
+x0 = rand(Xoshiro(4), 100)
+prototypes = Set([(0.0, 1.0), (1.0, 1.0), (0.0, 0.0), (1.0, 0.0)])
+
+plt = plot(layout = (length(grid), length(grid)), size = (1000, 1000),
+    legend = false, framestyle = :box, ticks = false, link = :all)
+
+for (ib, b) in enumerate(grid), (ia, a) in enumerate(grid)
+    M = morph_conceptor(st, morph_weights(a, b))
+    Y, _ = generate(concept, ps, st; conceptor = M, steps = 15,
+        washout = 190, init_state = x0)
+    k = (ib - 1) * length(grid) + ia          # row ib (top = b = -0.5), col ia
+    proto = (a, b) in prototypes
+    plot!(plt, subplot = k, vec(Y); color = :black, linewidth = proto ? 2.5 : 1.2,
+        ylims = (-1, 1), background_color_subplot = proto ? :gray88 : :white)
+end
+plt
+```
+
+The four shaded panels are the loaded prototypes: the two sines sit on the
+``b = 1`` row, the two period-5 patterns on the ``b = 0`` row. The inner block
+(``a, b \in [0, 1]``) interpolates smoothly between them; the outer panels
+extrapolate, producing distorted, higher-frequency variants.
+
+## Single-pattern recall and the conceptor algebra
+
+Loading also lets the reservoir recall a single pattern by name, and the stored
+conceptors support an aperture adaptation ([`aperture_adapt`](@ref)) and a Boolean
+algebra ([`conceptor_and`](@ref), [`conceptor_or`](@ref), [`conceptor_not`](@ref)).
+
+```@example morphing
+# autonomous recall of the first sine
+Yrec, _ = generate(concept, ps, st; conceptor = :s1, steps = 100, washout = 200)
+
+# conceptor algebra: shared vs. combined subspaces of the two sines
+Cs1, Cs2 = get_conceptor(st, :s1), get_conceptor(st, :s2)
+(quota(conceptor_and(Cs1, Cs2)), quota(Cs1), quota(conceptor_or(Cs1, Cs2)))
+```
+
+The triple is increasing — `AND` keeps only the shared directions, `OR` spans the
+union — illustrating the lattice structure of the conceptor algebra.

docs/src/refs.bib

@@ -1,3 +1,12 @@
+@article{Jaeger2014conceptors,
+  title = {Controlling Recurrent Neural Networks by Conceptors},
+  url = {https://arxiv.org/abs/1403.3369},
+  DOI = {10.48550/arXiv.1403.3369},
+  journal = {arXiv preprint arXiv:1403.3369},
+  author = {Jaeger,  Herbert},
+  year = {2014},
+}
+
 @article{Lu2017,
   title = {Reservoir observers: Model-free inference of unmeasured variables in chaotic systems},
   volume = {27},

src/ReservoirComputing.jl

@@ -2,12 +2,13 @@ module ReservoirComputing
 
 using ArrayInterface: ArrayInterface
 using ConcreteStructs: @concrete
-using LinearAlgebra: eigvals, I, qr, Diagonal, diag, mul!, Symmetric, norm
+using LinearAlgebra: eigvals, eigen, I, qr, Diagonal, diag, mul!, Symmetric, norm,
+    svd, svdvals, nullspace, pinv, tr, checksquare, issymmetric
 using LuxCore: AbstractLuxLayer, AbstractLuxContainerLayer, AbstractLuxWrapperLayer,
     setup, apply, replicate
 import LuxCore: initialparameters, initialstates, statelength, outputsize
 using NNlib: tanh_fast
-using Random: Random, AbstractRNG, randperm
+using Random: Random, AbstractRNG, randperm, randn
 using Static: StaticBool, StaticSymbol, True, False, static, known, StaticInteger
 using Reexport: Reexport, @reexport
 using WeightInitializers: WeightInitializers, DeviceAgnostic, PartialFunction, Utils,
@@ -59,6 +60,8 @@ include("models/lifesn.jl")
 include("models/ngrc.jl")
 include("models/rmnesn.jl")
 include("models/rmnresesn.jl")
+#conceptors
+include("conceptors.jl")
 #extensions
 include("extensions/reca.jl")
 
@@ -85,6 +88,14 @@ export polynomial_monomials, chebyshev_monomials, predict, QRSolver, resetcarry!
 export AdditiveEIESN, DeepESN, DelayESN, EIESN, ES2N, ESN, EuSN, HybridESN, InputDelayESN, LIFESN, ResESN, StateDelayESN, SVESM
 export NGRC
 export RMNESN, RMNResESN
+#conceptors
+export Conceptor, correlation_matrix, conceptor_matrix, conceptor_from_states,
+    conceptor_singular_values, quota
+export aperture_adapt, adapt_singular_value, reaperture, attenuation, optimal_aperture
+export conceptor_not, conceptor_and, conceptor_or
+export has_conceptor, get_conceptor, store_conceptor!, store_conceptors!,
+    set_active_conceptor, active_conceptor
+export ridge_map, load!, generate, morph_conceptor
 #ext
 export RECACell, RECA
 export RandomMapping, RandomMaps