From fbb24ad6f4359e703081ef6f47ded334368bcacc Mon Sep 17 00:00:00 2001 From: Evan Almloff Date: Sun, 15 Mar 2026 17:26:23 -0500 Subject: [PATCH 01/49] backprop init --- .../core/examples/train_linear_regression.rs | 54 ++ fusor-ml/core/src/autograd.rs | 198 ++++++ fusor-ml/core/src/composite/where_cond.rs | 15 +- fusor-ml/core/src/compute_graph/backward.rs | 615 ++++++++++++++++++ fusor-ml/core/src/compute_graph/mod.rs | 74 ++- fusor-ml/core/src/compute_graph/resolve.rs | 13 +- fusor-ml/core/src/element_wise.rs | 253 ++++--- fusor-ml/core/src/lib.rs | 2 + fusor-ml/core/src/map_layout.rs | 54 +- fusor-ml/core/src/matmul/mod.rs | 12 +- fusor-ml/core/src/pair_wise.rs | 93 ++- fusor-ml/core/src/resize.rs | 8 +- fusor-ml/core/src/slice_assign.rs | 15 +- fusor-ml/core/src/tensor.rs | 20 + 14 files changed, 1270 insertions(+), 156 deletions(-) create mode 100644 fusor-ml/core/examples/train_linear_regression.rs create mode 100644 fusor-ml/core/src/autograd.rs create mode 100644 fusor-ml/core/src/compute_graph/backward.rs diff --git a/fusor-ml/core/examples/train_linear_regression.rs b/fusor-ml/core/examples/train_linear_regression.rs new file mode 100644 index 000000000..7034018e0 --- /dev/null +++ b/fusor-ml/core/examples/train_linear_regression.rs @@ -0,0 +1,54 @@ +use fusor_core::{Device, Tensor}; + +const LEARNING_RATE: f32 = 0.05; +const EPOCHS: usize = 80; + +#[tokio::main] +async fn main() { + let device = Device::new().await.unwrap(); + + // Learn y = 2x + 1 from a tiny synthetic dataset. + let inputs: Tensor<2, f32> = Tensor::new(&device, &[[0.0], [1.0], [2.0], [3.0], [4.0]]); + let targets: Tensor<2, f32> = Tensor::new(&device, &[[1.0], [3.0], [5.0], [7.0], [9.0]]); + + let mut weight: Tensor<2, f32> = Tensor::new(&device, &[[0.0]]); + let mut bias: Tensor<2, f32> = Tensor::new(&device, &[[0.0]]); + + for epoch in 0..EPOCHS { + let bias_broadcast: Tensor<2, f32> = bias.broadcast_as([inputs.shape()[0], 1]); + let prediction = inputs.mat_mul(&weight) + &bias_broadcast; + let error = &prediction - &targets; + let squared_error = &error * &error; + let loss: Tensor<0, f32> = squared_error.sum::<1>(0).sum::<0>(0) / inputs.shape()[0] as f32; + + let loss_value = loss.to_scalar().await.unwrap(); + let gradients = loss.backward().unwrap(); + let weight_grad = gradients.get(&weight).unwrap(); + let bias_grad = gradients.get(&bias).unwrap(); + + // Apply a simple SGD update. + let next_weight = &weight - &(weight_grad * LEARNING_RATE); + let next_bias = &bias - &(bias_grad * LEARNING_RATE); + + // Recreate the parameter tensors from host values so each SGD step starts a fresh graph. + let next_weight_host = next_weight.as_slice().await.unwrap(); + let next_bias_host = next_bias.as_slice().await.unwrap(); + let weight_value = next_weight_host[[0, 0]]; + let bias_value = next_bias_host[[0, 0]]; + weight = Tensor::new(&device, &[[weight_value]]); + bias = Tensor::new(&device, &[[bias_value]]); + + if epoch % 10 == 0 || epoch + 1 == EPOCHS { + println!( + "epoch {:>2}: loss={:.6} weight={:.4} bias={:.4}", + epoch + 1, + loss_value, + weight_value, + bias_value, + ); + } + } + + let final_prediction = inputs.mat_mul(&weight) + &bias.broadcast_as([inputs.shape()[0], 1]); + println!("final predictions: {:?}", final_prediction.as_slice().await.unwrap()); +} diff --git a/fusor-ml/core/src/autograd.rs b/fusor-ml/core/src/autograd.rs new file mode 100644 index 000000000..e45f35bbc --- /dev/null +++ b/fusor-ml/core/src/autograd.rs @@ -0,0 +1,198 @@ +use rustc_hash::FxHashMap; + +use crate::{ + DataType, FloatDataType, Result, Tensor, + compute_graph::{BackwardRule, NodeIndex}, + tensor::LazyTensorData, +}; + +pub struct Gradients { + gradients: FxHashMap, +} + +pub struct BackwardTarget { + node: NodeIndex, + gradient: LazyTensorData, +} + +impl Gradients { + pub(crate) fn new(gradients: FxHashMap) -> Self { + Self { gradients } + } +} + +impl Gradients { + pub fn get(&self, tensor: &Tensor) -> Option> { + self.gradients + .get(&tensor.key()) + .cloned() + .map(Tensor::from_parts) + } +} + +impl BackwardTarget { + pub fn wrt(tensor: &Tensor, gradient: Tensor) -> Self { + Self { + node: tensor.key(), + gradient: gradient.data().clone(), + } + } +} + +impl Tensor { + pub fn with_backwards(self, backwards: F) -> Self + where + F: Fn(Tensor) -> Result> + Send + Sync + 'static, + { + let backward: BackwardRule = std::sync::Arc::new(move |gradient: LazyTensorData| { + let gradient = Tensor::from_parts(gradient); + let gradients = backwards(gradient)?; + Ok(gradients + .into_iter() + .map(|target| (target.node, target.gradient)) + .collect()) + }); + self.device().compute_graph().set_backward_rule(self.key(), backward); + self + } +} + +impl Tensor { + pub fn backward(&self) -> Result { + if self.shape().iter().product::() != 1 { + return Err(crate::Error::msg( + "backward() requires a single-element tensor; use backward_with() for non-scalars", + )); + } + + let seed = Tensor::splat(self.device(), D::one(), *self.shape()); + self.backward_with(&seed) + } + + pub fn backward_with(&self, seed: &Tensor) -> Result { + if self.shape() != seed.shape() { + return Err(crate::Error::msg(format!( + "gradient seed shape mismatch: expected {:?}, got {:?}", + self.shape(), + seed.shape() + ))); + } + + let gradients = self + .device() + .compute_graph() + .backward(self.key(), seed.data().clone())?; + Ok(Gradients::new(gradients)) + } +} + +#[cfg(test)] +fn assert_close(left: f32, right: f32) { + assert!( + (left - right).abs() < 1e-3, + "expected {right}, got {left}" + ); +} + +#[cfg(test)] +#[tokio::test] +async fn test_backward_squared_sum() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[1.0f32, 2.0, 3.0]); + let loss: Tensor<0, f32> = (&x * &x).sum::<0>(0); + + let gradients = loss.backward().unwrap(); + let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); + + assert_close(dx[[0]], 2.0); + assert_close(dx[[1]], 4.0); + assert_close(dx[[2]], 6.0); +} + +#[cfg(test)] +#[tokio::test] +async fn test_backward_matmul_with_broadcast_bias() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let w = Tensor::new(&device, &[[0.5f32], [1.0], [1.5]]); + let b = Tensor::new(&device, &[[2.0f32]]); + + let y = x.mat_mul(&w) + &b.broadcast_as([2, 1]); + let loss: Tensor<0, f32> = y.sum::<1>(0).sum::<0>(0); + + let gradients = loss.backward().unwrap(); + + let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); + assert_close(dx[[0, 0]], 0.5); + assert_close(dx[[0, 1]], 1.0); + assert_close(dx[[0, 2]], 1.5); + assert_close(dx[[1, 0]], 0.5); + assert_close(dx[[1, 1]], 1.0); + assert_close(dx[[1, 2]], 1.5); + + let dw = gradients.get(&w).unwrap().as_slice().await.unwrap(); + assert_close(dw[[0, 0]], 5.0); + assert_close(dw[[1, 0]], 7.0); + assert_close(dw[[2, 0]], 9.0); + + let db = gradients.get(&b).unwrap().as_slice().await.unwrap(); + assert_close(db[[0, 0]], 2.0); +} + +#[cfg(test)] +#[tokio::test] +async fn test_backward_slice() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let sliced = x.slice([0..2, 1..3]); + let loss: Tensor<0, f32> = sliced.sum::<1>(0).sum::<0>(0); + + let gradients = loss.backward().unwrap(); + let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); + + assert_close(dx[[0, 0]], 0.0); + assert_close(dx[[0, 1]], 1.0); + assert_close(dx[[0, 2]], 1.0); + assert_close(dx[[1, 0]], 0.0); + assert_close(dx[[1, 1]], 1.0); + assert_close(dx[[1, 2]], 1.0); +} + +#[cfg(test)] +#[tokio::test] +async fn test_backward_relu() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[1.0f32, -2.0, 0.0, 4.0]); + let loss: Tensor<0, f32> = x.relu().sum::<0>(0); + + let gradients = loss.backward().unwrap(); + let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); + + assert_close(dx[[0]], 1.0); + assert_close(dx[[1]], 0.0); + assert_close(dx[[2]], 0.0); + assert_close(dx[[3]], 1.0); +} + +#[cfg(test)] +#[tokio::test] +async fn test_with_backwards_override() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[1.0f32, 2.0]); + let captured = x.clone(); + let y = (x.clone() + 1.0).with_backwards(move |grad| { + Ok(vec![BackwardTarget::wrt(&captured, grad * 3.0)]) + }); + let loss: Tensor<0, f32> = y.sum::<0>(0); + + let gradients = loss.backward().unwrap(); + let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); + + assert_close(dx[[0]], 3.0); + assert_close(dx[[1]], 3.0); +} diff --git a/fusor-ml/core/src/composite/where_cond.rs b/fusor-ml/core/src/composite/where_cond.rs index 6195792e8..42c289d10 100644 --- a/fusor-ml/core/src/composite/where_cond.rs +++ b/fusor-ml/core/src/composite/where_cond.rs @@ -1,4 +1,4 @@ -use crate::{DataType, Tensor, compute_graph::NodeIndex, tensor::DataTypeEnum}; +use crate::{BackwardTarget, DataType, Tensor, compute_graph::NodeIndex, tensor::DataTypeEnum}; impl Tensor { pub fn where_cond(self, on_true: &Tensor, on_false: &Tensor) -> Tensor @@ -14,8 +14,17 @@ impl Tensor { self.shape(), ); let data = on_true.data(); - - Tensor::from_parts(data.where_cond(operation)) + let output = Tensor::from_parts(data.where_cond(operation)); + let condition = self.clone(); + let on_true = on_true.clone(); + let on_false = on_false.clone(); + output.with_backwards(move |grad| { + let zeros = Tensor::zeros(grad.device(), *grad.shape()); + Ok(vec![ + BackwardTarget::wrt(&on_true, condition.clone().where_cond(&grad, &zeros)), + BackwardTarget::wrt(&on_false, condition.clone().where_cond(&zeros, &grad)), + ]) + }) } } diff --git a/fusor-ml/core/src/compute_graph/backward.rs b/fusor-ml/core/src/compute_graph/backward.rs new file mode 100644 index 000000000..8065e5d90 --- /dev/null +++ b/fusor-ml/core/src/compute_graph/backward.rs @@ -0,0 +1,615 @@ +use std::ops::Range; + +use rustc_hash::{FxHashMap, FxHashSet}; + +use crate::{ + DataTypeEnum, Layout, MatMulOperation, PairWiseFunction, ReduceFunction, Result, TensorInfo, + composite::where_cond::WhereCondOperation, + map_layout::{MapLayoutKind, MapLayoutOperation}, + mir::operation::Operation, + pair_wise::PairWiseOperation, + reduce::ReduceOperation, + resize::ResizeOperation, + slice_assign::SliceAssignOperation, + tensor::{LazyTensorData, TensorData}, +}; + +use super::{BackwardRule, ComputeGraph, ComputeGraphInner, ComputeGraphNodeVariant, NodeIndex}; + +#[derive(Clone)] +struct NodeSnapshot { + variant: ComputeGraphNodeVariant, + info: TensorInfo, + backward: Option, +} + +impl ComputeGraph { + pub(crate) fn backward( + &self, + target: NodeIndex, + seed: LazyTensorData, + ) -> Result> { + let snapshots = { + let inner = self.inner.read(); + snapshot_subgraph(&inner, target)? + }; + let target_info = snapshots + .get(&target) + .ok_or_else(|| crate::Error::msg("backpropagation target was not found"))?; + + if seed.info().shape() != target_info.info.shape() + || seed.info().datatype() != target_info.info.datatype() + { + return Err(crate::Error::msg(format!( + "gradient seed shape/datatype mismatch: expected {:?} {}, got {:?} {}", + target_info.info.shape(), + target_info.info.datatype(), + seed.info().shape(), + seed.info().datatype() + ))); + } + + let mut gradients = FxHashMap::default(); + gradients.insert(target, seed); + + let mut visited = FxHashSet::default(); + let mut topo = Vec::new(); + build_topological_order(&snapshots, target, &mut visited, &mut topo); + + for node in topo.into_iter().rev() { + let Some(grad) = gradients.get(&node).cloned() else { + continue; + }; + let snapshot = snapshots + .get(&node) + .ok_or_else(|| crate::Error::msg("missing node while backpropagating"))?; + propagate_gradient(&snapshots, node, snapshot, ErasedTensor::from_lazy(grad), &mut gradients)?; + } + + Ok(gradients) + } +} + +fn snapshot_subgraph( + graph: &ComputeGraphInner, + target: NodeIndex, +) -> Result> { + let mut visited = FxHashSet::default(); + let mut order = Vec::new(); + collect_postorder(graph, target, &mut visited, &mut order)?; + + let mut infos = FxHashMap::default(); + let mut snapshots = FxHashMap::default(); + for node in order { + let variant = graph + .nodes + .nodes + .node_weight(node) + .ok_or_else(|| crate::Error::msg(format!("missing node {node:?} in compute graph")))? + .variant + .clone(); + let info = infer_info(&variant, &infos)?; + infos.insert(node, info.clone()); + let backward = graph + .nodes + .nodes + .node_weight(node) + .and_then(|node| node.backward.clone()); + snapshots.insert( + node, + NodeSnapshot { + variant, + info, + backward, + }, + ); + } + Ok(snapshots) +} + +fn collect_postorder( + graph: &ComputeGraphInner, + node: NodeIndex, + visited: &mut FxHashSet, + order: &mut Vec, +) -> Result<()> { + if !visited.insert(node) { + return Ok(()); + } + + let variant = graph + .nodes + .nodes + .node_weight(node) + .ok_or_else(|| crate::Error::msg(format!("missing node {node:?} in compute graph")))? + .variant + .clone(); + + let mut dependencies = Vec::new(); + variant.visit_dependencies(&mut |dependency| { + dependencies.push(dependency); + }); + for dependency in dependencies { + collect_postorder(graph, dependency, visited, order)?; + } + order.push(node); + Ok(()) +} + +fn build_topological_order( + snapshots: &FxHashMap, + node: NodeIndex, + visited: &mut FxHashSet, + order: &mut Vec, +) { + if !visited.insert(node) { + return; + } + + if let Some(snapshot) = snapshots.get(&node) { + let mut dependencies = Vec::new(); + snapshot.variant.visit_dependencies(&mut |dependency| { + dependencies.push(dependency); + }); + for dependency in dependencies { + build_topological_order(snapshots, dependency, visited, order); + } + } + + order.push(node); +} + +fn infer_info( + variant: &ComputeGraphNodeVariant, + infos: &FxHashMap, +) -> Result { + let info = match variant { + ComputeGraphNodeVariant::ElementWise(op) => { + TensorInfo::new(op.shape().into(), op.functions.out_datatype()) + } + ComputeGraphNodeVariant::PairWise(op) => { + TensorInfo::new(op.shape().into(), op.function.datatype) + } + ComputeGraphNodeVariant::Nary(op) => TensorInfo::new(op.shape.clone(), op.output_datatype), + ComputeGraphNodeVariant::SliceAssign(op) => { + let input = infos + .get(&op.input) + .ok_or_else(|| crate::Error::msg("slice_assign input info missing"))?; + TensorInfo::new(op.input_shape.clone(), input.datatype()) + } + ComputeGraphNodeVariant::Resize(op) => { + let input = infos + .get(&op.input) + .ok_or_else(|| crate::Error::msg("resize input info missing"))?; + TensorInfo::new(op.new_shape.clone(), input.datatype()) + } + ComputeGraphNodeVariant::MapLayout(op) => { + let input = infos + .get(&op.input) + .ok_or_else(|| crate::Error::msg("map_layout input info missing"))?; + let layout = op.map_layout(&Layout::contiguous(input.shape())); + TensorInfo::new(layout.shape().into(), input.datatype()) + } + ComputeGraphNodeVariant::Dequantize(op) => { + TensorInfo::new( + op.matrix.shape().to_vec().into_boxed_slice(), + op.post_dequantize.out_datatype(), + ) + } + ComputeGraphNodeVariant::MatMul(op) => { + TensorInfo::new(op.out_shape.clone(), op.post_element_wise.out_datatype()) + } + ComputeGraphNodeVariant::QMatMul(op) => TensorInfo::new(op.out_shape.clone(), op.input_datatype), + ComputeGraphNodeVariant::Tensor(data) => { + TensorInfo::new(data.layout().shape().into(), data.datatype()) + } + ComputeGraphNodeVariant::Reduce(op) => { + let shape = op + .shape + .iter() + .enumerate() + .filter_map(|(index, dim)| (index != op.axis).then_some(*dim)) + .collect(); + TensorInfo::new(shape, op.out_datatype()) + } + ComputeGraphNodeVariant::IndexSelect(op) => { + let input = infos + .get(&op.input) + .ok_or_else(|| crate::Error::msg("index_select input info missing"))?; + TensorInfo::new(op.output_shape(), input.datatype()) + } + ComputeGraphNodeVariant::WhereCond(op) => { + TensorInfo::new(op.shape.clone(), op.output_datatype) + } + ComputeGraphNodeVariant::Custom(op) => { + let layouts = infos + .iter() + .map(|(node, info)| { + ( + *node, + crate::TensorLayoutInfo::new(Layout::contiguous(info.shape()), info.datatype()), + ) + }) + .collect(); + let layout = op.output_layout(&layouts); + TensorInfo::new(layout.shape().into(), layout.datatype()) + } + }; + Ok(info) +} + +fn propagate_gradient( + snapshots: &FxHashMap, + _node: NodeIndex, + snapshot: &NodeSnapshot, + gradient: ErasedTensor, + gradients: &mut FxHashMap, +) -> Result<()> { + if let Some(backward) = &snapshot.backward { + for (dependency, dependency_gradient) in backward(gradient.into_lazy())? { + accumulate_gradient(gradients, dependency, ErasedTensor::from_lazy(dependency_gradient)); + } + return Ok(()); + } + + match &snapshot.variant { + ComputeGraphNodeVariant::Tensor(_) => Ok(()), + ComputeGraphNodeVariant::ElementWise(_) | ComputeGraphNodeVariant::PairWise(_) => { + Err(crate::Error::msg(format!( + "backpropagation does not support op `{}` without an attached backward rule", + variant_name(&snapshot.variant) + ))) + } + ComputeGraphNodeVariant::MatMul(op) => { + let first_info = snapshots + .get(&op.first) + .ok_or_else(|| crate::Error::msg("matmul lhs info missing"))? + .info + .clone(); + let second_info = snapshots + .get(&op.second) + .ok_or_else(|| crate::Error::msg("matmul rhs info missing"))? + .info + .clone(); + let first = ErasedTensor::reference(gradient.device().clone(), first_info, op.first); + let second = ErasedTensor::reference(gradient.device().clone(), second_info, op.second); + accumulate_gradient(gradients, op.first, gradient.mat_mul(&second.transpose_last_two())); + accumulate_gradient(gradients, op.second, first.transpose_last_two().mat_mul(&gradient)); + Ok(()) + } + ComputeGraphNodeVariant::Reduce(op) => { + if op.function.name() != "sum" { + return Err(crate::Error::msg(format!( + "backpropagation does not support reduce op `{}`", + op.function.name() + ))); + } + + let input_shape = op.shape.clone(); + let mut keepdim_shape = input_shape.to_vec(); + keepdim_shape[op.axis] = 1; + let input_grad = gradient.reshape(&keepdim_shape).broadcast_to(&input_shape); + accumulate_gradient(gradients, op.value, input_grad); + Ok(()) + } + ComputeGraphNodeVariant::MapLayout(op) => { + match &op.kind { + MapLayoutKind::Slice(slices) => { + let input_info = snapshots + .get(&op.input) + .ok_or_else(|| crate::Error::msg("slice input info missing"))? + .info + .clone(); + let zeros = ErasedTensor::zeros( + gradient.device().clone(), + input_info.shape(), + input_info.datatype(), + ); + accumulate_gradient(gradients, op.input, zeros.slice_assign(&gradient, slices)); + } + MapLayoutKind::Permute(axes) => { + let mut inverse = vec![0; axes.len()]; + for (new_axis, old_axis) in axes.iter().copied().enumerate() { + inverse[old_axis] = new_axis; + } + accumulate_gradient(gradients, op.input, gradient.permute(&inverse)); + } + MapLayoutKind::Broadcast => { + let input_info = snapshots + .get(&op.input) + .ok_or_else(|| crate::Error::msg("broadcast input info missing"))? + .info + .clone(); + let reduce_axes = broadcast_reduce_axes(input_info.shape(), gradient.shape())?; + let mut reduced = gradient; + for axis in reduce_axes.into_iter().rev() { + reduced = reduced.sum(axis); + } + accumulate_gradient(gradients, op.input, reduced.reshape(input_info.shape())); + } + MapLayoutKind::Custom => { + return Err(crate::Error::msg(format!( + "backpropagation does not support custom layout op `{}`", + op.name() + ))); + } + } + Ok(()) + } + ComputeGraphNodeVariant::Resize(op) => { + let full_fill = op.fill_shape == op.new_shape; + let same_elements = + op.current_shape.iter().product::() == op.new_shape.iter().product::(); + if !full_fill || !same_elements { + return Err(crate::Error::msg(format!( + "backpropagation only supports reshape-style resize ops, found `{}`", + op.name() + ))); + } + accumulate_gradient(gradients, op.input, gradient.reshape(&op.current_shape)); + Ok(()) + } + ComputeGraphNodeVariant::SliceAssign(op) => { + let value_info = snapshots + .get(&op.value) + .ok_or_else(|| crate::Error::msg("slice_assign value info missing"))? + .info + .clone(); + let zero_value = ErasedTensor::zeros( + gradient.device().clone(), + value_info.shape(), + value_info.datatype(), + ); + accumulate_gradient( + gradients, + op.input, + gradient.clone().slice_assign(&zero_value, &op.slices), + ); + accumulate_gradient(gradients, op.value, gradient.slice(&op.slices)); + Ok(()) + } + ComputeGraphNodeVariant::WhereCond(op) => { + let condition_info = snapshots + .get(&op.condition) + .ok_or_else(|| crate::Error::msg("where condition info missing"))? + .info + .clone(); + let condition = ErasedTensor::reference( + gradient.device().clone(), + condition_info, + op.condition, + ); + let zeros = ErasedTensor::zeros( + gradient.device().clone(), + gradient.shape(), + gradient.datatype(), + ); + accumulate_gradient( + gradients, + op.on_true, + condition.where_cond(&gradient, &zeros), + ); + accumulate_gradient( + gradients, + op.on_false, + condition.where_cond(&zeros, &gradient), + ); + Ok(()) + } + ComputeGraphNodeVariant::Dequantize(_) + | ComputeGraphNodeVariant::QMatMul(_) + | ComputeGraphNodeVariant::IndexSelect(_) + | ComputeGraphNodeVariant::Nary(_) + | ComputeGraphNodeVariant::Custom(_) => Err(crate::Error::msg(format!( + "backpropagation does not support op `{}`", + variant_name(&snapshot.variant) + ))), + } +} + +fn variant_name(variant: &ComputeGraphNodeVariant) -> &'static str { + match variant { + ComputeGraphNodeVariant::ElementWise(_) => "element_wise", + ComputeGraphNodeVariant::PairWise(_) => "pair_wise", + ComputeGraphNodeVariant::Nary(_) => "nary", + ComputeGraphNodeVariant::SliceAssign(_) => "slice_assign", + ComputeGraphNodeVariant::Resize(_) => "resize", + ComputeGraphNodeVariant::MapLayout(_) => "map_layout", + ComputeGraphNodeVariant::Dequantize(_) => "dequantize", + ComputeGraphNodeVariant::MatMul(_) => "mat_mul", + ComputeGraphNodeVariant::QMatMul(_) => "q_mat_mul", + ComputeGraphNodeVariant::Tensor(_) => "tensor", + ComputeGraphNodeVariant::Reduce(_) => "reduce", + ComputeGraphNodeVariant::IndexSelect(_) => "index_select", + ComputeGraphNodeVariant::WhereCond(_) => "where_cond", + ComputeGraphNodeVariant::Custom(_) => "custom", + } +} + +fn accumulate_gradient( + gradients: &mut FxHashMap, + node: NodeIndex, + gradient: ErasedTensor, +) { + if let Some(existing) = gradients.get(&node).cloned() { + let combined = ErasedTensor::from_lazy(existing).add(&gradient); + gradients.insert(node, combined.into_lazy()); + } else { + gradients.insert(node, gradient.into_lazy()); + } +} + +fn broadcast_reduce_axes(input_shape: &[usize], output_shape: &[usize]) -> Result> { + let mut reduce_axes = Vec::new(); + let mut input_iter = input_shape.iter().rev().peekable(); + + for (axis, &target_dim) in output_shape.iter().enumerate().rev() { + let reduce = if let Some(&&source_dim) = input_iter.peek() { + if source_dim == target_dim || (source_dim == 1 && target_dim > 1) { + input_iter.next(); + source_dim == 1 && target_dim > 1 + } else { + target_dim > 1 + } + } else { + target_dim > 1 + }; + + if reduce { + reduce_axes.push(axis); + } + } + + if input_iter.next().is_some() { + return Err(crate::Error::msg(format!( + "failed to match broadcast input shape {input_shape:?} to output shape {output_shape:?}" + ))); + } + + Ok(reduce_axes) +} + +#[derive(Clone)] +struct ErasedTensor { + data: LazyTensorData, +} + +impl ErasedTensor { + fn from_lazy(data: LazyTensorData) -> Self { + Self { data } + } + + fn reference(device: crate::Device, info: TensorInfo, key: NodeIndex) -> Self { + Self { + data: LazyTensorData::reference(device, info, key), + } + } + + fn zeros(device: crate::Device, shape: &[usize], datatype: DataTypeEnum) -> Self { + let data = match datatype { + DataTypeEnum::F32 => TensorData::new_splat(&device, shape, 0.0f32), + DataTypeEnum::F16 => TensorData::new_splat(&device, shape, half::f16::ZERO), + DataTypeEnum::U32 => TensorData::new_splat(&device, shape, 0u32), + }; + Self::from_lazy(LazyTensorData::new(data)) + } + + fn into_lazy(self) -> LazyTensorData { + self.data + } + + fn shape(&self) -> &[usize] { + self.data.info().shape() + } + + fn datatype(&self) -> DataTypeEnum { + self.data.info().datatype() + } + + fn device(&self) -> &crate::Device { + self.data.device() + } + + fn key(&self) -> NodeIndex { + self.data.key() + } + + fn map_layout( + &self, + map_layout_fn: impl Fn(&Layout) -> Layout + Send + Sync + 'static, + ) -> Self { + Self::from_lazy(self.data.map_layout(MapLayoutOperation::new( + self.key(), + map_layout_fn, + ))) + } + + fn reshape(&self, new_shape: &[usize]) -> Self { + Self::from_lazy(self.data.resize(ResizeOperation::new( + self.key(), + self.shape().into(), + new_shape.into(), + new_shape.into(), + ))) + } + + fn broadcast_to(&self, target_shape: &[usize]) -> Self { + let target_shape: Box<[usize]> = target_shape.into(); + self.map_layout(move |layout| layout.broadcast_to(&target_shape)) + } + + fn permute(&self, axes: &[usize]) -> Self { + let axes: Box<[usize]> = axes.into(); + self.map_layout(move |layout| layout.permute(&axes)) + } + + fn slice(&self, slices: &[Range]) -> Self { + let slices: Box<[Range]> = slices.into(); + self.map_layout(move |layout| layout.slice(&slices)) + } + + fn slice_assign(&self, value: &Self, slices: &[Range]) -> Self { + Self::from_lazy(self.data.slice_assign(SliceAssignOperation::new( + self.key(), + value.key(), + slices.into(), + self.shape().into(), + ))) + } + + fn where_cond(&self, on_true: &Self, on_false: &Self) -> Self { + let operation = WhereCondOperation::new( + self.key(), + on_true.key(), + on_false.key(), + self.datatype(), + on_true.datatype(), + on_true.shape(), + ); + Self::from_lazy(on_true.data.where_cond(operation)) + } + + fn mat_mul(&self, other: &Self) -> Self { + Self::from_lazy(self.data.mat_mul(MatMulOperation::new( + self.datatype(), + self.key(), + other.key(), + self.shape(), + other.shape(), + None, + ))) + } + + fn sum(&self, axis: usize) -> Self { + Self::from_lazy(self.data.reduce(ReduceOperation::new( + self.key(), + ReduceFunction { + name: Some("sum".to_string()), + operation: "let output = a + b;".to_string(), + initial_value: "0.0".to_string(), + datatype: self.datatype(), + }, + axis, + self.shape(), + ))) + } + + fn add(&self, other: &Self) -> Self { + Self::from_lazy( + self.data + .pair_wise(PairWiseOperation::new( + PairWiseFunction::new("let output = a + b;", self.datatype()), + self.key(), + other.key(), + self.shape(), + )), + ) + } + + fn transpose_last_two(&self) -> Self { + let rank = self.shape().len(); + let mut axes: Vec = (0..rank).collect(); + axes.swap(rank - 1, rank - 2); + self.permute(&axes) + } +} diff --git a/fusor-ml/core/src/compute_graph/mod.rs b/fusor-ml/core/src/compute_graph/mod.rs index d89c6f76b..d8f1923b3 100644 --- a/fusor-ml/core/src/compute_graph/mod.rs +++ b/fusor-ml/core/src/compute_graph/mod.rs @@ -9,6 +9,7 @@ use wgpu::CommandEncoderDescriptor; mod layout_pass; mod queue; +mod backward; mod resolve; mod visualize; @@ -18,9 +19,14 @@ use crate::{ compute_graph::resolve::ResolverResult, dequantize::DequantizeOperation, index_select::IndexSelectOperation, map_layout::MapLayoutOperation, mir::operation::Operation, nary_wise::NaryOperation, quantized::matmul::QMatMulOperation, resize::ResizeOperation, - slice_assign::SliceAssignOperation, tensor::TensorData, visit_tiled::MaybeQData, + slice_assign::SliceAssignOperation, + tensor::{LazyTensorData, TensorData}, + visit_tiled::MaybeQData, }; +pub(crate) type BackwardRule = + Arc crate::Result> + Send + Sync>; + #[derive(Clone)] pub(crate) struct ComputeGraph { inner: Arc>, @@ -106,10 +112,18 @@ impl ComputeGraph { .create_command_encoder(&CommandEncoderDescriptor { label: Some("ComputeGraph Encoder"), }); - let data = self.with_mut(|inner| { - let mut resolver = Resolver::new(inner, key, &mut encoder); - resolver.run(inner) - }); + let (data, removed) = { + let mut inner = self.inner.write(); + let mut removed = Vec::new(); + let mut resolver = Resolver::new(&mut inner, key, &mut encoder); + let data = resolver.run(&mut inner, &mut removed); + #[cfg(feature = "extra_assertions")] + { + inner.verify_integrity() + } + (data, removed) + }; + drop(removed); device.wgpu_queue().submit(Some(encoder.finish())); // Reset the written flag on all buffers device.reset_initialized_buffers(); @@ -137,8 +151,31 @@ impl ComputeGraph { self.with_mut(|inner| inner.add_reference(key)); } + pub(crate) fn set_backward_rule(&self, key: NodeIndex, backward: BackwardRule) { + let replaced = { + let mut inner = self.inner.write(); + let replaced = inner.set_backward_rule(key, backward); + #[cfg(feature = "extra_assertions")] + { + inner.verify_integrity() + } + replaced + }; + drop(replaced); + } + pub(crate) fn remove_reference(&self, key: NodeIndex) { - self.with_mut(|inner| inner.remove_reference(key)); + let removed = { + let mut inner = self.inner.write(); + let mut removed = Vec::new(); + inner.remove_reference(key, &mut removed); + #[cfg(feature = "extra_assertions")] + { + inner.verify_integrity() + } + removed + }; + drop(removed); } } @@ -151,6 +188,7 @@ pub(crate) struct ComputeGraphNode { variant: ComputeGraphNodeVariant, reference_count: u32, cached: Option, + backward: Option, } #[derive(Clone, Debug)] @@ -242,6 +280,7 @@ impl ComputeGraphInner { variant: node, reference_count: 1, cached: None, + backward: None, }); self.add_dependency_edges(node); node @@ -253,6 +292,13 @@ impl ComputeGraphInner { node.reference_count += 1; } + fn set_backward_rule(&mut self, key: NodeIndex, backward: BackwardRule) -> Option { + self.nodes + .nodes + .node_weight_mut(key) + .and_then(|node| node.backward.replace(backward)) + } + fn add_dependency_edges(&mut self, key: NodeIndex) { let mut dependencies = Vec::new(); self.visit_dependencies(key, &mut |dep| { @@ -269,13 +315,13 @@ impl ComputeGraphInner { } } - fn remove_reference(&mut self, key: NodeIndex) { + fn remove_reference(&mut self, key: NodeIndex, removed: &mut Vec) { let node = self.nodes.nodes.node_weight_mut(key).unwrap(); node.reference_count = node.reference_count.saturating_sub(1); - self.check_life(key); + self.check_life(key, removed); } - fn check_life(&mut self, key: NodeIndex) { + fn check_life(&mut self, key: NodeIndex, removed: &mut Vec) { // Check the reference count let ref_count = self.nodes.nodes.node_weight(key).map(|n| n.reference_count); match ref_count { @@ -315,17 +361,19 @@ impl ComputeGraphInner { // If no other nodes depend on this key and it has zero references, it is dead // remove it from the graph - self.remove_key(key); + self.remove_key(key, removed); // Then check if any nodes it depends on are alive for dependency in dependencies { - self.check_life(dependency); + self.check_life(dependency, removed); } } - fn remove_key(&mut self, key: NodeIndex) { + fn remove_key(&mut self, key: NodeIndex, removed: &mut Vec) { // Remove the node from the graph (this also removes all edges) - self.nodes.nodes.remove_node(key); + if let Some(node) = self.nodes.nodes.remove_node(key) { + removed.push(node); + } } pub(crate) fn get_result_or_qmatrix(&self, key: NodeIndex) -> Option { diff --git a/fusor-ml/core/src/compute_graph/resolve.rs b/fusor-ml/core/src/compute_graph/resolve.rs index 6c5547d8c..8de45723a 100644 --- a/fusor-ml/core/src/compute_graph/resolve.rs +++ b/fusor-ml/core/src/compute_graph/resolve.rs @@ -20,7 +20,7 @@ use crate::{ tensor::TensorData, }; -use super::{ComputeGraphInner, ComputeGraphNodeVariant, NodeIndex}; +use super::{ComputeGraphInner, ComputeGraphNode, ComputeGraphNodeVariant, NodeIndex}; pub(crate) struct ResolverResult { pub(crate) data: TensorData, @@ -72,7 +72,11 @@ impl<'a> Resolver<'a> { } } - pub(crate) fn run(&mut self, graph: &mut ComputeGraphInner) -> ResolverResult { + pub(crate) fn run( + &mut self, + graph: &mut ComputeGraphInner, + removed: &mut Vec, + ) -> ResolverResult { let device = graph.device(); let max_subgroup_size = device.max_subgroup_size(); @@ -134,6 +138,7 @@ impl<'a> Resolver<'a> { &inputs, &all_input_values, old_best, + removed, ); pending_operations.clear(); all_input_values.clear(); @@ -184,6 +189,7 @@ impl<'a> Resolver<'a> { &inputs, &all_input_values, old_best, + removed, ); } @@ -1051,6 +1057,7 @@ impl<'a> Resolver<'a> { inputs: &[Vec], all_input_values: &[KernelInputValue], workgroup_shape: workgroup_shape::WorkgroupShape, + removed: &mut Vec, ) { let mut max_dispatch_size = [0; 3]; for ((key, operation), inputs) in queued_operations.iter().zip(inputs) { @@ -1087,7 +1094,7 @@ impl<'a> Resolver<'a> { dependencies.push(dependent_key); }); for dependency in dependencies { - graph.check_life(dependency); + graph.check_life(dependency, removed); } } kernel.set_workgroup_size(workgroup_shape); diff --git a/fusor-ml/core/src/element_wise.rs b/fusor-ml/core/src/element_wise.rs index 6b40cf4d7..d3da68cf0 100644 --- a/fusor-ml/core/src/element_wise.rs +++ b/fusor-ml/core/src/element_wise.rs @@ -5,6 +5,7 @@ use std::{ }; use crate::{ + BackwardTarget, Tensor, compute_graph::NodeIndex, mir::{function::Function, kernel::GenericKernel}, @@ -142,17 +143,61 @@ impl ElementWiseFunction { } } +fn elementwise_with_backward( + input: &Tensor, + function: ElementWiseFunction, + backward: impl Fn(Tensor, &Tensor) -> Tensor + Send + Sync + 'static, +) -> Tensor { + let output = input.element_wise(ElementWiseOperation::new( + input.datatype(), + input.key(), + function, + input.shape().as_slice(), + )); + let input = input.clone(); + output.with_backwards(move |grad| Ok(vec![BackwardTarget::wrt(&input, backward(grad, &input))])) +} + +fn greater_than_const_mask( + input: &Tensor, + value: &str, +) -> Tensor { + input.element_wise(ElementWiseOperation::new( + input.datatype(), + input.key(), + ElementWiseFunction::new( + format!("let output = {}(input > {value});", D::WGSL_TYPE), + D::WGSL_TYPE, + ), + input.shape().as_slice(), + )) +} + +fn less_than_const_mask( + input: &Tensor, + value: &str, +) -> Tensor { + input.element_wise(ElementWiseOperation::new( + input.datatype(), + input.key(), + ElementWiseFunction::new( + format!("let output = {}(input < {value});", D::WGSL_TYPE), + D::WGSL_TYPE, + ), + input.shape().as_slice(), + )) +} + impl Add for Tensor { type Output = Tensor; fn add(self, rhs: T) -> Self::Output { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + &self, ElementWiseFunction::new(format!("let output = input + {rhs};"), T::WGSL_TYPE) .with_name("add_const"), - self.shape().as_slice(), - )) + |grad, _input| grad, + ) } } @@ -419,13 +464,12 @@ impl Sub for Tensor { type Output = Tensor; fn sub(self, rhs: T) -> Self::Output { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + &self, ElementWiseFunction::new(format!("let output = input - {rhs};"), T::WGSL_TYPE) .with_name("subtract_const"), - self.shape().as_slice(), - )) + |grad, _input| grad, + ) } } @@ -456,16 +500,15 @@ macro_rules! impl_sub { type Output = Tensor; fn sub(self, rhs: Tensor) -> Self::Output { - rhs.element_wise(ElementWiseOperation::new( - rhs.datatype(), - rhs.key(), + elementwise_with_backward( + &rhs, ElementWiseFunction::new( format!("let output = {self} - input;"), <$t>::WGSL_TYPE, ) .with_name("subtract_const"), - rhs.shape().as_slice(), - )) + |grad, _input| -grad, + ) } } )* @@ -497,13 +540,12 @@ impl Mul for Tensor { type Output = Tensor; fn mul(self, rhs: T) -> Self::Output { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + &self, ElementWiseFunction::new(format!("let output = input * {rhs};"), T::WGSL_TYPE) .with_name("multiply_const"), - self.shape().as_slice(), - )) + move |grad, _input| grad * rhs, + ) } } @@ -574,13 +616,12 @@ impl Div for Tensor { type Output = Tensor; fn div(self, rhs: T) -> Self::Output { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + &self, ElementWiseFunction::new(format!("let output = input / {rhs};"), T::WGSL_TYPE) .with_name("divide_const"), - self.shape().as_slice(), - )) + move |grad, _input| grad / rhs, + ) } } @@ -611,16 +652,15 @@ macro_rules! impl_div { type Output = Tensor; fn div(self, rhs: Tensor) -> Self::Output { - rhs.element_wise(ElementWiseOperation::new( - rhs.datatype(), - rhs.key(), + elementwise_with_backward( + &rhs, ElementWiseFunction::new( format!("let output = {} / input;", self), <$t>::WGSL_TYPE, ) .with_name("divide_const"), - rhs.shape().as_slice(), - )) + move |grad, input| -((grad * self) / &(input * input)), + ) } } )* @@ -964,12 +1004,11 @@ impl Tensor { } pub fn exp(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = exp(input);", D::WGSL_TYPE).with_name("exp"), - self.shape().as_slice(), - )) + |grad, input| grad * &input.exp(), + ) } } @@ -993,14 +1032,13 @@ async fn test_exp() { assert!((output[[2, 1]] - data[2][1].exp()).abs() < 0.001); } -impl Tensor { +impl Tensor { pub fn exp2(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = exp2(input);", D::WGSL_TYPE).with_name("exp2"), - self.shape().as_slice(), - )) + |grad, input| (grad * &input.exp2()) * D::from_f32(0.6931471805599453), + ) } } @@ -1026,12 +1064,11 @@ async fn test_exp2() { impl Tensor { pub fn log(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = log(input);", D::WGSL_TYPE).with_name("log"), - self.shape().as_slice(), - )) + |grad, input| grad / input, + ) } } @@ -1055,14 +1092,13 @@ async fn test_log() { assert!((output[[2, 1]] - data[2][1].ln()).abs() < 0.001); } -impl Tensor { +impl Tensor { pub fn log2(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = log2(input);", D::WGSL_TYPE).with_name("log2"), - self.shape().as_slice(), - )) + |grad, input| grad / &(input * D::from_f32(0.6931471805599453)), + ) } } @@ -1088,16 +1124,15 @@ async fn test_log2() { impl Tensor { pub fn pow_elementwise(&self, exponent: D) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new( format!("let output = pow(input, {exponent});"), D::WGSL_TYPE, ) .with_name("pow"), - self.shape().as_slice(), - )) + move |grad, input| (grad * exponent) * &input.pow_elementwise(exponent - D::one()), + ) } } @@ -1121,14 +1156,13 @@ async fn test_pow() { assert!((output[[2, 1]] - data[2][1].powi(2)).abs() < 0.001); } -impl Tensor { +impl Tensor { pub fn sqrt(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = sqrt(input);", D::WGSL_TYPE).with_name("sqrt"), - self.shape().as_slice(), - )) + |grad, input| grad / &(input.sqrt() * D::from_f32(2.0)), + ) } } @@ -1154,12 +1188,11 @@ async fn test_sqrt() { impl Tensor { pub fn sin(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = sin(input);", D::WGSL_TYPE).with_name("sin"), - self.shape().as_slice(), - )) + |grad, input| grad * &input.cos(), + ) } } @@ -1185,12 +1218,11 @@ async fn test_sin() { impl Tensor { pub fn cos(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = cos(input);", D::WGSL_TYPE).with_name("cos"), - self.shape().as_slice(), - )) + |grad, input| -(grad * &input.sin()), + ) } } @@ -1410,28 +1442,36 @@ async fn test_cosh() { impl Tensor { pub fn tanh(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new("let output = tanh(input);", D::WGSL_TYPE).with_name("tanh"), - self.shape().as_slice(), - )) + |grad, input| { + let output = input.tanh(); + let ones = Tensor::splat(input.device(), D::one(), *input.shape()); + let squared = &output * &output; + grad * &(ones - squared) + }, + ) } } impl Tensor { /// Calculates tanh with (e^x - e^-x) / (e^x + e^-x) pub fn tanh_exact(&self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + self, ElementWiseFunction::new( "let output = (exp(input) - exp(-input)) / (exp(input) + exp(-input));", D::WGSL_TYPE, ) .with_name("tanh_exact"), - self.shape().as_slice(), - )) + |grad, input| { + let output = input.tanh_exact(); + let ones = Tensor::splat(input.device(), D::one(), *input.shape()); + let squared = &output * &output; + grad * &(ones - squared) + }, + ) } } @@ -1596,12 +1636,11 @@ impl Neg for Tensor { type Output = Tensor; fn neg(self) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + elementwise_with_backward( + &self, ElementWiseFunction::new("let output = -input;", D::WGSL_TYPE).with_name("neg"), - self.shape().as_slice(), - )) + |grad, _input| -grad, + ) } } @@ -1636,13 +1675,13 @@ async fn test_neg() { impl Tensor { pub fn max_elementwise(&self, element: D) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + let element_str = element.to_string(); + elementwise_with_backward( + self, ElementWiseFunction::new(format!("let output = max(input, {element});"), D::WGSL_TYPE) .with_name("max"), - self.shape().as_slice(), - )) + move |grad, input| grad * &greater_than_const_mask(input, &element_str), + ) } } @@ -1669,13 +1708,13 @@ async fn test_max() { impl Tensor { pub fn min_elementwise(&self, element: D) -> Self { - self.element_wise(ElementWiseOperation::new( - self.datatype(), - self.key(), + let element_str = element.to_string(); + elementwise_with_backward( + self, ElementWiseFunction::new(format!("let output = min(input, {element});"), D::WGSL_TYPE) .with_name("max"), - self.shape().as_slice(), - )) + move |grad, input| grad * &less_than_const_mask(input, &element_str), + ) } } @@ -1746,13 +1785,12 @@ impl CastTensor for u32 { impl CastTensor for f32 { fn cast(tensor: &Tensor) -> Tensor { - tensor.element_wise(ElementWiseOperation::new( - tensor.datatype(), - tensor.key(), + elementwise_with_backward( + tensor, ElementWiseFunction::new("let output = f16(input);", DataTypeEnum::F16) .with_name("cast"), - tensor.shape().as_slice(), - )) + |grad, _input| grad.cast(), + ) } } @@ -1781,13 +1819,12 @@ async fn test_f32_to_f16_cast() { impl CastTensor for half::f16 { fn cast(tensor: &Tensor) -> Tensor { - tensor.element_wise(ElementWiseOperation::new( - tensor.datatype(), - tensor.key(), + elementwise_with_backward( + tensor, ElementWiseFunction::new("let output = f32(input);", DataTypeEnum::F32) .with_name("cast"), - tensor.shape().as_slice(), - )) + |grad, _input| grad.cast(), + ) } } diff --git a/fusor-ml/core/src/lib.rs b/fusor-ml/core/src/lib.rs index 7ec5b24a9..9d71b95b4 100644 --- a/fusor-ml/core/src/lib.rs +++ b/fusor-ml/core/src/lib.rs @@ -11,6 +11,7 @@ pub use rank::*; pub use reduce::*; pub use tensor::MappedBuffer; pub use tensor::*; +pub use autograd::{BackwardTarget, Gradients}; // Re-export wasm-compatible Send/Sync traits pub use wgpu::{WasmNotSend, WasmNotSendSync, WasmNotSync}; @@ -21,6 +22,7 @@ pub(crate) use pair_wise::*; pub use resize::ShapeWithOneHole; pub use varbuilder::{ShardedVarBuilder, VarBuilder}; +mod autograd; pub mod cache; mod composite; pub use composite::{ToVec1, ToVec2, ToVec3}; diff --git a/fusor-ml/core/src/map_layout.rs b/fusor-ml/core/src/map_layout.rs index ec75e8a50..0f6346111 100644 --- a/fusor-ml/core/src/map_layout.rs +++ b/fusor-ml/core/src/map_layout.rs @@ -7,10 +7,19 @@ use crate::{ type MapLayout = Arc Layout + Send + Sync>; +#[derive(Clone, Debug)] +pub(crate) enum MapLayoutKind { + Slice(Box<[Range]>), + Permute(Box<[usize]>), + Broadcast, + Custom, +} + #[derive(Clone)] pub(crate) struct MapLayoutOperation { pub(crate) input: NodeIndex, pub(crate) map_layout_fn: MapLayout, + pub(crate) kind: MapLayoutKind, } impl Debug for MapLayoutOperation { @@ -25,10 +34,19 @@ impl MapLayoutOperation { pub fn new( input: NodeIndex, map_layout_fn: impl Fn(&Layout) -> Layout + Send + Sync + 'static, + ) -> Self { + Self::with_kind(input, map_layout_fn, MapLayoutKind::Custom) + } + + pub fn with_kind( + input: NodeIndex, + map_layout_fn: impl Fn(&Layout) -> Layout + Send + Sync + 'static, + kind: MapLayoutKind, ) -> Self { Self { input, map_layout_fn: Arc::new(map_layout_fn), + kind, } } @@ -103,23 +121,33 @@ impl Operation for MapLayoutOperation { impl Tensor { pub fn slice(&self, slices: [Range; R]) -> Tensor { - self.add_map_layout(MapLayoutOperation::new(self.key(), move |layout| { - layout.slice(&slices) - })) + let kind = MapLayoutKind::Slice(slices.clone().into()); + self.add_map_layout(MapLayoutOperation::with_kind( + self.key(), + move |layout| layout.slice(&slices), + kind, + )) } pub fn permute(&self, axes: [usize; R]) -> Tensor { - self.add_map_layout(MapLayoutOperation::new(self.key(), move |layout| { - layout.permute(&axes) - })) + let kind = MapLayoutKind::Permute(axes.to_vec().into_boxed_slice()); + self.add_map_layout(MapLayoutOperation::with_kind( + self.key(), + move |layout| layout.permute(&axes), + kind, + )) } pub fn transpose(&self, first_axis: impl Dim, second_axis: impl Dim) -> Tensor { let first_axis = first_axis.resolve(); let second_axis = second_axis.resolve(); - self.add_map_layout(MapLayoutOperation::new(self.key(), move |layout| { - layout.transpose(first_axis, second_axis) - })) + let mut axes: Vec = (0..R).collect(); + axes.swap(first_axis, second_axis); + self.add_map_layout(MapLayoutOperation::with_kind( + self.key(), + move |layout| layout.transpose(first_axis, second_axis), + MapLayoutKind::Permute(axes.into_boxed_slice()), + )) } pub fn t(&self) -> Tensor { @@ -145,9 +173,11 @@ impl Tensor { ) }; - self.add_map_layout(MapLayoutOperation::new(self.key(), move |layout| { - layout.broadcast_to(&out_shape) - })) + self.add_map_layout(MapLayoutOperation::with_kind( + self.key(), + move |layout| layout.broadcast_to(&out_shape), + MapLayoutKind::Broadcast, + )) } pub(crate) fn broadcast_together( diff --git a/fusor-ml/core/src/matmul/mod.rs b/fusor-ml/core/src/matmul/mod.rs index 638d619f2..ba50500d5 100644 --- a/fusor-ml/core/src/matmul/mod.rs +++ b/fusor-ml/core/src/matmul/mod.rs @@ -2,7 +2,7 @@ use crate::matmul::sgemm_params::gemm_parameters; use crate::matmul::sgemv_params::gemv_parameters; use crate::mir::operation::Operation; use crate::{ - Device, ElementWiseFunctions, Tensor, + BackwardTarget, Device, ElementWiseFunctions, Tensor, compute_graph::NodeIndex, mir::kernel::GenericKernel, tensor::{DataType, DataTypeEnum, TensorData}, @@ -273,7 +273,15 @@ impl Operation for MatMulOperation { impl Tensor { pub fn mat_mul(&self, other: &Self) -> Self { - self.add_mat_mul(other, None) + let output = self.add_mat_mul(other, None); + let lhs = self.clone(); + let rhs = other.clone(); + output.with_backwards(move |grad| { + Ok(vec![ + BackwardTarget::wrt(&lhs, grad.clone().mat_mul(&rhs.t())), + BackwardTarget::wrt(&rhs, lhs.t().mat_mul(&grad)), + ]) + }) } pub fn mat_mul_with_parameters(&self, other: &Self, parameters: MatMulParams) -> Self { diff --git a/fusor-ml/core/src/pair_wise.rs b/fusor-ml/core/src/pair_wise.rs index 131d2ebbb..d87460be1 100644 --- a/fusor-ml/core/src/pair_wise.rs +++ b/fusor-ml/core/src/pair_wise.rs @@ -4,7 +4,7 @@ use std::{ }; use crate::{ - ElementWiseFunction, MaxRank, Tensor, + BackwardTarget, ElementWiseFunction, MaxRank, Tensor, compute_graph::NodeIndex, tensor::{DataType, DataTypeEnum}, }; @@ -87,6 +87,21 @@ impl PairWiseFunction { } } +fn pairwise_with_backward( + lhs: &Tensor, + rhs: &Tensor, + function: PairWiseFunction, + backward: impl Fn(Tensor, &Tensor, &Tensor) -> Vec + + Send + + Sync + + 'static, +) -> Tensor { + let output = lhs.pair_wise(rhs, function); + let lhs = lhs.clone(); + let rhs = rhs.clone(); + output.with_backwards(move |grad| Ok(backward(grad, &lhs, &rhs))) +} + /// Macro to implement pairwise operators (Add, Sub, Mul, Div) for Tensor. /// /// Generates all four combinations of owned/reference implementations: @@ -97,7 +112,7 @@ impl PairWiseFunction { /// /// Also generates a broadcast method `op_()` for tensors of different ranks. macro_rules! impl_pairwise_op { - ($trait:ident, $method:ident, $op_str:literal, $op_name:literal, $broadcast_method:ident, {$op:tt}) => { + ($trait:ident, $method:ident, $op_str:literal, $op_name:literal, $broadcast_method:ident, {$op:tt}, $backward:expr) => { // Owned + Owned: delegates to ref + ref impl $trait> for Tensor { type Output = Tensor; @@ -112,13 +127,15 @@ macro_rules! impl_pairwise_op { type Output = Tensor; fn $method(self, rhs: &Tensor) -> Self::Output { - self.pair_wise( + pairwise_with_backward( + self, rhs, PairWiseFunction::new( concat!("let output = a ", $op_str, " b;"), T::WGSL_TYPE, ) .with_name($op_name), + $backward, ) } } @@ -156,7 +173,18 @@ macro_rules! impl_pairwise_op { }; } -impl_pairwise_op!(Add, add, "+", "add", add_, {+}); +impl_pairwise_op!( + Add, + add, + "+", + "add", + add_, + {+}, + |grad, lhs, rhs| vec![ + BackwardTarget::wrt(lhs, grad.clone()), + BackwardTarget::wrt(rhs, grad), + ] +); #[cfg(test)] #[tokio::test] @@ -312,7 +340,18 @@ async fn test_pair_wise_add_sparse() { assert_eq!(as_slice[[2, 0]], 5. + 5.); } -impl_pairwise_op!(Sub, sub, "-", "sub", sub_, {-}); +impl_pairwise_op!( + Sub, + sub, + "-", + "sub", + sub_, + {-}, + |grad, lhs, rhs| vec![ + BackwardTarget::wrt(lhs, grad.clone()), + BackwardTarget::wrt(rhs, -grad), + ] +); #[cfg(test)] #[tokio::test] @@ -338,7 +377,18 @@ async fn test_pair_wise_sub() { assert_eq!(as_slice[[2, 1]], 6. - 6.); } -impl_pairwise_op!(Mul, mul, "*", "mul", mul_, {*}); +impl_pairwise_op!( + Mul, + mul, + "*", + "mul", + mul_, + {*}, + |grad, lhs, rhs| vec![ + BackwardTarget::wrt(lhs, grad.clone() * rhs), + BackwardTarget::wrt(rhs, grad * lhs), + ] +); #[cfg(test)] #[tokio::test] @@ -364,7 +414,18 @@ async fn test_pair_wise_mul() { assert_eq!(as_slice[[2, 1]], 6. * 6.); } -impl_pairwise_op!(Div, div, "/", "div", div_, {/}); +impl_pairwise_op!( + Div, + div, + "/", + "div", + div_, + {/}, + |grad, lhs, rhs| vec![ + BackwardTarget::wrt(lhs, grad.clone() / rhs), + BackwardTarget::wrt(rhs, -((grad * lhs) / &(rhs * rhs))), + ] +); #[cfg(test)] #[tokio::test] @@ -395,13 +456,15 @@ async fn test_pair_wise_div() { /// Unlike `impl_pairwise_op!` which implements std::ops traits, this macro generates /// regular methods on Tensor for operations that don't have corresponding operators. macro_rules! impl_pairwise_method { - ($method:ident, $wgsl_op:literal, $op_name:literal, $broadcast_method:ident, |$a:ident, $b:ident| $expr:expr) => { + ($method:ident, $wgsl_op:literal, $op_name:literal, $broadcast_method:ident, |$a:ident, $b:ident| $expr:expr, $backward:expr) => { impl Tensor { pub fn $method(&self, other: &Self) -> Self { - self.pair_wise( + pairwise_with_backward( + self, other, PairWiseFunction::new(concat!("let output = ", $wgsl_op, ";"), T::WGSL_TYPE) .with_name($op_name), + $backward, ) } @@ -418,7 +481,17 @@ macro_rules! impl_pairwise_method { }; } -impl_pairwise_method!(pow, "pow(a, b)", "pow", pow_, |a, b| a.pow(&b)); +impl_pairwise_method!( + pow, + "pow(a, b)", + "pow", + pow_, + |a, b| a.pow(&b), + |grad, lhs, rhs| vec![ + BackwardTarget::wrt(lhs, (grad.clone() * rhs) * &lhs.pow(&(rhs.clone() - T::one()))), + BackwardTarget::wrt(rhs, (grad * &lhs.pow(rhs)) * &lhs.log()), + ] +); #[cfg(test)] #[tokio::test] diff --git a/fusor-ml/core/src/resize.rs b/fusor-ml/core/src/resize.rs index f09bc2fad..75c0292c6 100644 --- a/fusor-ml/core/src/resize.rs +++ b/fusor-ml/core/src/resize.rs @@ -1,7 +1,7 @@ use std::fmt::Write; use crate::{ - DataTypeEnum, Layout, SmallerRank, TILE_SIZE, Tensor, TensorData, + BackwardTarget, DataTypeEnum, Layout, SmallerRank, TILE_SIZE, Tensor, TensorData, compute_graph::NodeIndex, map_layout::MapLayoutOperation, mir::{ @@ -251,12 +251,14 @@ impl Tensor { ); let new_shape: Box<[usize]> = new_shape.into(); let input = self.key(); - self.add_resize(ResizeOperation::new( + let output = self.add_resize(ResizeOperation::new( input, (*self.shape()).into(), new_shape.clone(), new_shape.clone(), - )) + )); + let input = self.clone(); + output.with_backwards(move |grad| Ok(vec![BackwardTarget::wrt(&input, grad.reshape(*input.shape()))])) } pub fn flatten_last_n(&self) -> Tensor diff --git a/fusor-ml/core/src/slice_assign.rs b/fusor-ml/core/src/slice_assign.rs index e9b49417e..1ef368c17 100644 --- a/fusor-ml/core/src/slice_assign.rs +++ b/fusor-ml/core/src/slice_assign.rs @@ -1,7 +1,7 @@ use std::{fmt::Write, ops::Range}; use crate::{ - TILE_SIZE, Tensor, TensorData, + BackwardTarget, TILE_SIZE, Tensor, TensorData, compute_graph::{ComputeGraphInner, NodeIndex}, mir::{ inputs::MirValue, @@ -184,7 +184,18 @@ impl Operation for SliceAssignOperation { impl Tensor { pub fn slice_assign(&self, slices: [Range; R], value: &Self) -> Self { - self.add_slice_assign(value, slices) + let output = self.add_slice_assign(value, slices.clone()); + let input = self.clone(); + let value = value.clone(); + output.with_backwards(move |grad| { + Ok(vec![ + BackwardTarget::wrt( + &input, + grad.slice_assign(slices.clone(), &Tensor::zeros(grad.device(), *value.shape())), + ), + BackwardTarget::wrt(&value, grad.slice(slices.clone())), + ]) + }) } } diff --git a/fusor-ml/core/src/tensor.rs b/fusor-ml/core/src/tensor.rs index 41e6b2286..533795fbd 100644 --- a/fusor-ml/core/src/tensor.rs +++ b/fusor-ml/core/src/tensor.rs @@ -38,6 +38,9 @@ pub trait DataType: + AnyBitPattern + Debug + Display + + Send + + Sync + + 'static { const WGSL_TYPE: DataTypeEnum; @@ -235,6 +238,11 @@ impl LazyTensorData { } } + pub(crate) fn reference(device: Device, info: TensorInfo, key: NodeIndex) -> Self { + device.compute_graph().add_reference(key); + Self { device, info, key } + } + pub(crate) fn from_parts(device: Device, info: TensorInfo, key: NodeIndex) -> Self { Self { device, info, key } } @@ -351,6 +359,18 @@ impl LazyTensorData { (result.data, result.total_kernels) } + pub(crate) fn device(&self) -> &Device { + &self.device + } + + pub(crate) fn info(&self) -> &TensorInfo { + &self.info + } + + pub(crate) fn key(&self) -> NodeIndex { + self.key + } + pub fn graphvis(&self) -> Graph { self.device.compute_graph().graphvis(self.key) } From 235d2460868ffedf70f4382c0d35df8c92f27e67 Mon Sep 17 00:00:00 2001 From: Evan Almloff Date: Sun, 15 Mar 2026 17:47:32 -0500 Subject: [PATCH 02/49] transformer example --- .../core/examples/train_linear_regression.rs | 5 +- .../core/examples/train_tiny_transformer.rs | 262 +++++++++++++++++ fusor-ml/core/src/autograd.rs | 264 ++++++++++++++++++ fusor-ml/core/src/compute_graph/mod.rs | 37 ++- 4 files changed, 559 insertions(+), 9 deletions(-) create mode 100644 fusor-ml/core/examples/train_tiny_transformer.rs diff --git a/fusor-ml/core/examples/train_linear_regression.rs b/fusor-ml/core/examples/train_linear_regression.rs index 7034018e0..727660d9b 100644 --- a/fusor-ml/core/examples/train_linear_regression.rs +++ b/fusor-ml/core/examples/train_linear_regression.rs @@ -3,8 +3,8 @@ use fusor_core::{Device, Tensor}; const LEARNING_RATE: f32 = 0.05; const EPOCHS: usize = 80; -#[tokio::main] -async fn main() { +fn main() { + pollster::block_on(async { let device = Device::new().await.unwrap(); // Learn y = 2x + 1 from a tiny synthetic dataset. @@ -51,4 +51,5 @@ async fn main() { let final_prediction = inputs.mat_mul(&weight) + &bias.broadcast_as([inputs.shape()[0], 1]); println!("final predictions: {:?}", final_prediction.as_slice().await.unwrap()); + }); } diff --git a/fusor-ml/core/examples/train_tiny_transformer.rs b/fusor-ml/core/examples/train_tiny_transformer.rs new file mode 100644 index 000000000..3c082032a --- /dev/null +++ b/fusor-ml/core/examples/train_tiny_transformer.rs @@ -0,0 +1,262 @@ +use fusor_core::{Device, Gradients, Tensor, cache::AttentionMask}; +use rand::{Rng, SeedableRng, rngs::StdRng}; + +const VOCAB_SIZE: usize = 6; +const SEQ_LEN: usize = 4; +const BATCH_SIZE: usize = 6; +const MODEL_DIM: usize = 8; +const FF_DIM: usize = 16; +const EPOCHS: usize = 120; +const LEARNING_RATE: f32 = 0.08; +const EPS: f32 = 1e-5; + +#[derive(Clone)] +struct TinyTransformer { + token_projection: Tensor<2, f32>, + position_projection: Tensor<2, f32>, + ln1_weight: Tensor<1, f32>, + ln1_bias: Tensor<1, f32>, + w_q: Tensor<2, f32>, + w_k: Tensor<2, f32>, + w_v: Tensor<2, f32>, + w_o: Tensor<2, f32>, + ln2_weight: Tensor<1, f32>, + ln2_bias: Tensor<1, f32>, + w1: Tensor<2, f32>, + b1: Tensor<1, f32>, + w2: Tensor<2, f32>, + b2: Tensor<1, f32>, + ln_out_weight: Tensor<1, f32>, + ln_out_bias: Tensor<1, f32>, + lm_head: Tensor<2, f32>, +} + +impl TinyTransformer { + fn new(device: &Device) -> Self { + let mut rng = StdRng::seed_from_u64(7); + Self { + token_projection: random_matrix::(device, &mut rng, 0.12), + position_projection: random_matrix::(device, &mut rng, 0.12), + ln1_weight: ones::(device), + ln1_bias: zeros::(device), + w_q: random_matrix::(device, &mut rng, 0.10), + w_k: random_matrix::(device, &mut rng, 0.10), + w_v: random_matrix::(device, &mut rng, 0.10), + w_o: random_matrix::(device, &mut rng, 0.10), + ln2_weight: ones::(device), + ln2_bias: zeros::(device), + w1: random_matrix::(device, &mut rng, 0.10), + b1: zeros::(device), + w2: random_matrix::(device, &mut rng, 0.10), + b2: zeros::(device), + ln_out_weight: ones::(device), + ln_out_bias: zeros::(device), + lm_head: random_matrix::(device, &mut rng, 0.10), + } + } + + fn forward( + &self, + token_inputs: &Tensor<3, f32>, + position_inputs: &Tensor<2, f32>, + causal_mask: &AttentionMask, + ) -> Tensor<3, f32> { + let token_embeddings = + token_inputs.mat_mul(&self.token_projection.broadcast_as([BATCH_SIZE, VOCAB_SIZE, MODEL_DIM])); + let position_embeddings: Tensor<2, f32> = position_inputs.mat_mul(&self.position_projection); + let mut x = token_embeddings.add_(&position_embeddings.broadcast_as([BATCH_SIZE, SEQ_LEN, MODEL_DIM])); + + let attn_input = + x.layer_norm(&self.ln1_weight, Some(&self.ln1_bias), EPS, true); + let q = attn_input.mat_mul(&self.w_q.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); + let k = attn_input.mat_mul(&self.w_k.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); + let v = attn_input.mat_mul(&self.w_v.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); + + let scores = q.mat_mul(&k.transpose(1, 2)) / (MODEL_DIM as f32).sqrt(); + let masked_scores = causal_mask.apply(&scores); + let weights_exp = masked_scores.exp(); + let attention = weights_exp.div_(&weights_exp.sum_keepdim(2)); + let attention_output = attention + .mat_mul(&v) + .mat_mul(&self.w_o.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); + x = x + attention_output; + + let ff_input = x.layer_norm(&self.ln2_weight, Some(&self.ln2_bias), EPS, true); + let ff_hidden = ff_input + .mat_mul(&self.w1.broadcast_as([BATCH_SIZE, MODEL_DIM, FF_DIM])) + .add_(&self.b1) + .relu(); + let ff_output = ff_hidden + .mat_mul(&self.w2.broadcast_as([BATCH_SIZE, FF_DIM, MODEL_DIM])) + .add_(&self.b2); + x = x + ff_output; + + let output = x.layer_norm(&self.ln_out_weight, Some(&self.ln_out_bias), EPS, true); + output.mat_mul(&self.lm_head.broadcast_as([BATCH_SIZE, MODEL_DIM, VOCAB_SIZE])) + } + + async fn step(self, gradients: &Gradients, device: &Device) -> Self { + Self { + token_projection: sgd_step_2d::(&self.token_projection, gradients, device).await, + position_projection: sgd_step_2d::(&self.position_projection, gradients, device).await, + ln1_weight: sgd_step_1d::(&self.ln1_weight, gradients, device).await, + ln1_bias: sgd_step_1d::(&self.ln1_bias, gradients, device).await, + w_q: sgd_step_2d::(&self.w_q, gradients, device).await, + w_k: sgd_step_2d::(&self.w_k, gradients, device).await, + w_v: sgd_step_2d::(&self.w_v, gradients, device).await, + w_o: sgd_step_2d::(&self.w_o, gradients, device).await, + ln2_weight: sgd_step_1d::(&self.ln2_weight, gradients, device).await, + ln2_bias: sgd_step_1d::(&self.ln2_bias, gradients, device).await, + w1: sgd_step_2d::(&self.w1, gradients, device).await, + b1: sgd_step_1d::(&self.b1, gradients, device).await, + w2: sgd_step_2d::(&self.w2, gradients, device).await, + b2: sgd_step_1d::(&self.b2, gradients, device).await, + ln_out_weight: sgd_step_1d::(&self.ln_out_weight, gradients, device).await, + ln_out_bias: sgd_step_1d::(&self.ln_out_bias, gradients, device).await, + lm_head: sgd_step_2d::(&self.lm_head, gradients, device).await, + } + } +} + +#[tokio::main] +async fn main() { + let device = Device::new().await.unwrap(); + + let token_ids = training_sequences(); + let token_inputs: Tensor<3, f32> = Tensor::new(&device, &token_one_hot(&token_ids)); + let targets: Tensor<3, f32> = Tensor::new(&device, &next_token_one_hot(&token_ids)); + let position_inputs: Tensor<2, f32> = Tensor::new(&device, &position_one_hot()); + let causal_mask = AttentionMask::causal(&device, SEQ_LEN); + + let mut model = TinyTransformer::new(&device); + + for epoch in 0..EPOCHS { + let logits = model.forward(&token_inputs, &position_inputs, &causal_mask); + let error = &logits - &targets; + let loss: Tensor<0, f32> = (&error * &error) + .sum::<2>(2) + .sum::<1>(1) + .sum::<0>(0) + / (BATCH_SIZE * SEQ_LEN * VOCAB_SIZE) as f32; + + let loss_value = loss.to_scalar().await.unwrap(); + let gradients = loss.backward().unwrap(); + model = model.step(&gradients, &device).await; + + if epoch % 20 == 0 || epoch + 1 == EPOCHS { + println!("epoch {:>3}: loss={loss_value:.6}", epoch + 1); + } + } + + let logits = model.forward(&token_inputs, &position_inputs, &causal_mask); + let predictions = argmax_last_dim(logits.to_vec3().await.unwrap()); + + println!("training sequences:"); + for sequence in &token_ids { + println!(" {sequence:?}"); + } + println!("predicted next tokens:"); + for prediction in predictions { + println!(" {prediction:?}"); + } +} + +fn training_sequences() -> [[u32; SEQ_LEN]; BATCH_SIZE] { + [ + [0, 1, 2, 3], + [1, 2, 3, 4], + [2, 3, 4, 5], + [3, 4, 5, 0], + [4, 5, 0, 1], + [5, 0, 1, 2], + ] +} + +fn token_one_hot(tokens: &[[u32; SEQ_LEN]; BATCH_SIZE]) -> [[[f32; VOCAB_SIZE]; SEQ_LEN]; BATCH_SIZE] { + std::array::from_fn(|batch| { + std::array::from_fn(|position| { + let token = tokens[batch][position] as usize; + std::array::from_fn(|vocab| if vocab == token { 1.0 } else { 0.0 }) + }) + }) +} + +fn next_token_one_hot( + tokens: &[[u32; SEQ_LEN]; BATCH_SIZE], +) -> [[[f32; VOCAB_SIZE]; SEQ_LEN]; BATCH_SIZE] { + std::array::from_fn(|batch| { + std::array::from_fn(|position| { + let token = ((tokens[batch][position] as usize) + 1) % VOCAB_SIZE; + std::array::from_fn(|vocab| if vocab == token { 1.0 } else { 0.0 }) + }) + }) +} + +fn position_one_hot() -> [[f32; SEQ_LEN]; SEQ_LEN] { + std::array::from_fn(|position| { + std::array::from_fn(|column| if column == position { 1.0 } else { 0.0 }) + }) +} + +fn random_matrix( + device: &Device, + rng: &mut StdRng, + scale: f32, +) -> Tensor<2, f32> { + let data: [[f32; COLS]; ROWS] = std::array::from_fn(|_| { + std::array::from_fn(|_| rng.random_range(-scale..scale)) + }); + Tensor::new(device, &data) +} + +fn ones(device: &Device) -> Tensor<1, f32> { + Tensor::new(device, &[1.0; LEN]) +} + +fn zeros(device: &Device) -> Tensor<1, f32> { + Tensor::new(device, &[0.0; LEN]) +} + +async fn sgd_step_1d( + parameter: &Tensor<1, f32>, + gradients: &Gradients, + device: &Device, +) -> Tensor<1, f32> { + let gradient = gradients.get(parameter).unwrap(); + let next = parameter - &(gradient * LEARNING_RATE); + let host = next.to_vec1().await.unwrap(); + let host: [f32; LEN] = host.try_into().unwrap(); + Tensor::new(device, &host) +} + +async fn sgd_step_2d( + parameter: &Tensor<2, f32>, + gradients: &Gradients, + device: &Device, +) -> Tensor<2, f32> { + let gradient = gradients.get(parameter).unwrap(); + let next = parameter - &(gradient * LEARNING_RATE); + let host = next.to_vec2().await.unwrap(); + let host: [[f32; COLS]; ROWS] = + std::array::from_fn(|row| std::array::from_fn(|col| host[row][col])); + Tensor::new(device, &host) +} + +fn argmax_last_dim(logits: Vec>>) -> Vec> { + logits + .into_iter() + .map(|sequence| { + sequence + .into_iter() + .map(|token_logits| { + token_logits + .iter() + .enumerate() + .max_by(|(_, left), (_, right)| left.total_cmp(right)) + .map(|(index, _)| index) + .unwrap() + }) + .collect() + }) + .collect() +} diff --git a/fusor-ml/core/src/autograd.rs b/fusor-ml/core/src/autograd.rs index e45f35bbc..103af2ba3 100644 --- a/fusor-ml/core/src/autograd.rs +++ b/fusor-ml/core/src/autograd.rs @@ -141,6 +141,34 @@ async fn test_backward_matmul_with_broadcast_bias() { assert_close(db[[0, 0]], 2.0); } +#[cfg(test)] +#[tokio::test] +async fn test_backward_matmul_with_broadcasted_weight_batch() { + let device = crate::Device::test_instance(); + + let x = Tensor::new( + &device, + &[ + [[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]], + [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]], + ], + ); + let w = Tensor::new(&device, &[[0.5f32, 1.0], [1.5, 2.0], [2.5, 3.0]]); + + let y = x.mat_mul(&w.broadcast_as([2, 3, 2])); + let loss: Tensor<0, f32> = y.sum::<2>(2).sum::<1>(1).sum::<0>(0); + + let gradients = loss.backward().unwrap(); + + let dw = gradients.get(&w).unwrap().as_slice().await.unwrap(); + assert_close(dw[[0, 0]], 22.0); + assert_close(dw[[0, 1]], 22.0); + assert_close(dw[[1, 0]], 26.0); + assert_close(dw[[1, 1]], 26.0); + assert_close(dw[[2, 0]], 30.0); + assert_close(dw[[2, 1]], 30.0); +} + #[cfg(test)] #[tokio::test] async fn test_backward_slice() { @@ -196,3 +224,239 @@ async fn test_with_backwards_override() { assert_close(dx[[0]], 3.0); assert_close(dx[[1]], 3.0); } + +#[cfg(test)] +#[tokio::test] +async fn test_backward_after_materializing_loss_scalar() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[1.0f32, -2.0, 3.0]); + let loss: Tensor<0, f32> = ((x.relu() + 1.0) * 2.0).sum::<0>(0); + + let loss_value = loss.to_scalar().await.unwrap(); + assert_close(loss_value, 14.0); + + let gradients = loss.backward().unwrap(); + let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); + + assert_close(dx[[0]], 2.0); + assert_close(dx[[1]], 0.0); + assert_close(dx[[2]], 2.0); +} + +#[cfg(test)] +#[tokio::test] +async fn test_backward_tiny_transformer_parameter_grads_present() { + use crate::cache::AttentionMask; + + const VOCAB: usize = 4; + const SEQ: usize = 3; + const BATCH: usize = 2; + const MODEL: usize = 4; + const FF: usize = 6; + const EPS: f32 = 1e-5; + + let device = crate::Device::test_instance(); + + let token_inputs: Tensor<3, f32> = Tensor::new( + &device, + &[ + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + ], + [ + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], + ], + ); + let targets: Tensor<3, f32> = Tensor::new( + &device, + &[ + [ + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], + [ + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + [1.0, 0.0, 0.0, 0.0], + ], + ], + ); + let position_inputs: Tensor<2, f32> = Tensor::new( + &device, + &[ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ], + ); + let causal_mask = AttentionMask::causal(&device, SEQ); + + let token_projection = Tensor::new( + &device, + &[ + [0.10, -0.02, 0.03, 0.04], + [0.05, 0.06, -0.07, 0.08], + [-0.04, 0.03, 0.02, -0.01], + [0.07, -0.05, 0.06, 0.02], + ], + ); + let position_projection = Tensor::new( + &device, + &[ + [0.01, 0.02, 0.03, 0.04], + [0.04, 0.03, 0.02, 0.01], + [-0.02, 0.01, 0.00, 0.03], + ], + ); + let ln1_weight = Tensor::new(&device, &[1.0, 1.0, 1.0, 1.0]); + let ln1_bias = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); + let w_q = Tensor::new( + &device, + &[ + [0.02, 0.03, 0.01, -0.02], + [0.01, -0.01, 0.04, 0.02], + [0.05, 0.02, -0.03, 0.01], + [-0.02, 0.01, 0.02, 0.03], + ], + ); + let w_k = Tensor::new( + &device, + &[ + [0.01, -0.03, 0.02, 0.04], + [0.02, 0.05, -0.01, 0.03], + [0.03, 0.01, 0.04, -0.02], + [0.00, 0.02, 0.01, 0.05], + ], + ); + let w_v = Tensor::new( + &device, + &[ + [0.04, 0.01, -0.02, 0.03], + [-0.01, 0.03, 0.02, 0.04], + [0.02, 0.05, 0.01, -0.03], + [0.03, -0.02, 0.04, 0.01], + ], + ); + let w_o = Tensor::new( + &device, + &[ + [0.03, 0.02, 0.01, -0.01], + [0.04, -0.02, 0.03, 0.02], + [0.01, 0.05, -0.01, 0.03], + [0.02, 0.01, 0.04, -0.02], + ], + ); + let ln2_weight = Tensor::new(&device, &[1.0, 1.0, 1.0, 1.0]); + let ln2_bias = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); + let w1 = Tensor::new( + &device, + &[ + [0.02, 0.01, 0.03, -0.02, 0.04, 0.01], + [0.01, 0.04, -0.01, 0.02, 0.03, 0.05], + [0.03, -0.02, 0.05, 0.01, -0.01, 0.02], + [0.04, 0.02, 0.01, 0.03, 0.02, -0.02], + ], + ); + let b1 = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]); + let w2 = Tensor::new( + &device, + &[ + [0.01, 0.02, 0.03, 0.04], + [0.02, 0.03, -0.01, 0.01], + [0.04, -0.02, 0.01, 0.03], + [0.03, 0.01, 0.02, -0.01], + [0.01, 0.04, 0.03, 0.02], + [0.02, -0.01, 0.04, 0.03], + ], + ); + let b2 = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); + let ln_out_weight = Tensor::new(&device, &[1.0, 1.0, 1.0, 1.0]); + let ln_out_bias = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); + let lm_head = Tensor::new( + &device, + &[ + [0.02, 0.01, 0.03, 0.04], + [0.03, 0.02, 0.01, -0.01], + [0.04, -0.02, 0.02, 0.03], + [0.01, 0.03, 0.04, 0.02], + ], + ); + + let token_embeddings = + token_inputs.mat_mul(&token_projection.broadcast_as([BATCH, VOCAB, MODEL])); + let position_embeddings: Tensor<2, f32> = position_inputs.mat_mul(&position_projection); + let position_embeddings_broadcast = position_embeddings.broadcast_as([BATCH, SEQ, MODEL]); + let mut x = token_embeddings.add_(&position_embeddings_broadcast); + let embedding_sum = x.clone(); + + let attn_input = x.layer_norm(&ln1_weight, Some(&ln1_bias), EPS, true); + let q = attn_input.mat_mul(&w_q.broadcast_as([BATCH, MODEL, MODEL])); + let k = attn_input.mat_mul(&w_k.broadcast_as([BATCH, MODEL, MODEL])); + let v = attn_input.mat_mul(&w_v.broadcast_as([BATCH, MODEL, MODEL])); + + let scores = q.mat_mul(&k.transpose(1, 2)) / (MODEL as f32).sqrt(); + let masked_scores = causal_mask.apply(&scores); + let weights_exp = masked_scores.exp(); + let attention = weights_exp.div_(&weights_exp.sum_keepdim(2)); + let attention_output = attention + .mat_mul(&v) + .mat_mul(&w_o.broadcast_as([BATCH, MODEL, MODEL])); + x = x + attention_output; + let after_attention = x.clone(); + + let ff_input = x.layer_norm(&ln2_weight, Some(&ln2_bias), EPS, true); + let ff_hidden = ff_input + .mat_mul(&w1.broadcast_as([BATCH, MODEL, FF])) + .add_(&b1) + .relu(); + let ff_output = ff_hidden + .mat_mul(&w2.broadcast_as([BATCH, FF, MODEL])) + .add_(&b2); + x = x + ff_output; + let after_ff = x.clone(); + + let output = x.layer_norm(&ln_out_weight, Some(&ln_out_bias), EPS, true); + let logits = output.mat_mul(&lm_head.broadcast_as([BATCH, MODEL, VOCAB])); + let error = &logits - &targets; + let loss: Tensor<0, f32> = (&error * &error) + .sum::<2>(2) + .sum::<1>(1) + .sum::<0>(0) + / (BATCH * SEQ * VOCAB) as f32; + + let _ = loss.to_scalar().await.unwrap(); + let gradients = loss.backward().unwrap(); + + assert!(gradients.get(&token_embeddings).is_some()); + assert!(gradients.get(&embedding_sum).is_some()); + assert!(gradients.get(&attn_input).is_some()); + assert!(gradients.get(&after_attention).is_some()); + assert!(gradients.get(&ff_input).is_some()); + assert!(gradients.get(&after_ff).is_some()); + assert!(gradients.get(&output).is_some()); + assert!(gradients.get(&logits).is_some()); + assert!(gradients.get(&token_projection).is_some()); + assert!(gradients.get(&position_projection).is_some()); + assert!(gradients.get(&w_q).is_some()); + assert!(gradients.get(&w_k).is_some()); + assert!(gradients.get(&w_v).is_some()); + assert!(gradients.get(&w_o).is_some()); + assert!(gradients.get(&w1).is_some()); + assert!(gradients.get(&w2).is_some()); + assert!(gradients.get(&lm_head).is_some()); + assert!(gradients.get(&ln1_weight).is_some()); + assert!(gradients.get(&ln1_bias).is_some()); + assert!(gradients.get(&ln2_weight).is_some()); + assert!(gradients.get(&ln2_bias).is_some()); + assert!(gradients.get(&b1).is_some()); + assert!(gradients.get(&b2).is_some()); + assert!(gradients.get(&ln_out_weight).is_some()); + assert!(gradients.get(&ln_out_bias).is_some()); +} diff --git a/fusor-ml/core/src/compute_graph/mod.rs b/fusor-ml/core/src/compute_graph/mod.rs index d8f1923b3..43eb900c0 100644 --- a/fusor-ml/core/src/compute_graph/mod.rs +++ b/fusor-ml/core/src/compute_graph/mod.rs @@ -4,6 +4,7 @@ use parking_lot::RwLock; pub(crate) use petgraph::graph::NodeIndex; use petgraph::prelude::StableGraph; use resolve::Resolver; +use rustc_hash::FxHashSet; use tabbycat::Graph; use wgpu::CommandEncoderDescriptor; @@ -344,13 +345,12 @@ impl ComputeGraphInner { .collect(); for dependant in dependents { - // If the dependant still exists and it hasn't been computed yet - // keep this node alive - if let Some(dep_node) = self.nodes.nodes.node_weight(dependant) { - let computed = dep_node.cached.is_some(); - if !computed { - return; - } + // Keep dependencies alive while any downstream dependent is materially + // live, even if intermediate nodes have already been computed. This + // preserves the full ancestry needed for backprop after materializing + // a live output tensor. + if self.has_materially_live_dependant(dependant, &mut FxHashSet::default()) { + return; } } @@ -369,6 +369,29 @@ impl ComputeGraphInner { } } + fn has_materially_live_dependant( + &self, + key: NodeIndex, + visited: &mut FxHashSet, + ) -> bool { + if !visited.insert(key) { + return false; + } + + let Some(node) = self.nodes.nodes.node_weight(key) else { + return false; + }; + + if node.reference_count > 0 || node.cached.is_none() { + return true; + } + + self.nodes + .nodes + .neighbors_directed(key, petgraph::Direction::Outgoing) + .any(|dependant| self.has_materially_live_dependant(dependant, visited)) + } + fn remove_key(&mut self, key: NodeIndex, removed: &mut Vec) { // Remove the node from the graph (this also removes all edges) if let Some(node) = self.nodes.nodes.remove_node(key) { From 0528e0cfa597904f423f39826a7800c81ff584c7 Mon Sep 17 00:00:00 2001 From: Evan Almloff Date: Sun, 15 Mar 2026 17:52:22 -0500 Subject: [PATCH 03/49] faster --- .../core/examples/train_tiny_transformer.rs | 65 +++++++++---------- fusor-ml/core/src/autograd.rs | 19 ++++++ fusor-ml/core/src/tensor.rs | 7 ++ 3 files changed, 57 insertions(+), 34 deletions(-) diff --git a/fusor-ml/core/examples/train_tiny_transformer.rs b/fusor-ml/core/examples/train_tiny_transformer.rs index 3c082032a..3b270a3b9 100644 --- a/fusor-ml/core/examples/train_tiny_transformer.rs +++ b/fusor-ml/core/examples/train_tiny_transformer.rs @@ -9,6 +9,7 @@ const FF_DIM: usize = 16; const EPOCHS: usize = 120; const LEARNING_RATE: f32 = 0.08; const EPS: f32 = 1e-5; +const LOG_EVERY: usize = 20; #[derive(Clone)] struct TinyTransformer { @@ -95,25 +96,25 @@ impl TinyTransformer { output.mat_mul(&self.lm_head.broadcast_as([BATCH_SIZE, MODEL_DIM, VOCAB_SIZE])) } - async fn step(self, gradients: &Gradients, device: &Device) -> Self { + fn step(self, gradients: &Gradients) -> Self { Self { - token_projection: sgd_step_2d::(&self.token_projection, gradients, device).await, - position_projection: sgd_step_2d::(&self.position_projection, gradients, device).await, - ln1_weight: sgd_step_1d::(&self.ln1_weight, gradients, device).await, - ln1_bias: sgd_step_1d::(&self.ln1_bias, gradients, device).await, - w_q: sgd_step_2d::(&self.w_q, gradients, device).await, - w_k: sgd_step_2d::(&self.w_k, gradients, device).await, - w_v: sgd_step_2d::(&self.w_v, gradients, device).await, - w_o: sgd_step_2d::(&self.w_o, gradients, device).await, - ln2_weight: sgd_step_1d::(&self.ln2_weight, gradients, device).await, - ln2_bias: sgd_step_1d::(&self.ln2_bias, gradients, device).await, - w1: sgd_step_2d::(&self.w1, gradients, device).await, - b1: sgd_step_1d::(&self.b1, gradients, device).await, - w2: sgd_step_2d::(&self.w2, gradients, device).await, - b2: sgd_step_1d::(&self.b2, gradients, device).await, - ln_out_weight: sgd_step_1d::(&self.ln_out_weight, gradients, device).await, - ln_out_bias: sgd_step_1d::(&self.ln_out_bias, gradients, device).await, - lm_head: sgd_step_2d::(&self.lm_head, gradients, device).await, + token_projection: sgd_step_2d(&self.token_projection, gradients), + position_projection: sgd_step_2d(&self.position_projection, gradients), + ln1_weight: sgd_step_1d(&self.ln1_weight, gradients), + ln1_bias: sgd_step_1d(&self.ln1_bias, gradients), + w_q: sgd_step_2d(&self.w_q, gradients), + w_k: sgd_step_2d(&self.w_k, gradients), + w_v: sgd_step_2d(&self.w_v, gradients), + w_o: sgd_step_2d(&self.w_o, gradients), + ln2_weight: sgd_step_1d(&self.ln2_weight, gradients), + ln2_bias: sgd_step_1d(&self.ln2_bias, gradients), + w1: sgd_step_2d(&self.w1, gradients), + b1: sgd_step_1d(&self.b1, gradients), + w2: sgd_step_2d(&self.w2, gradients), + b2: sgd_step_1d(&self.b2, gradients), + ln_out_weight: sgd_step_1d(&self.ln_out_weight, gradients), + ln_out_bias: sgd_step_1d(&self.ln_out_bias, gradients), + lm_head: sgd_step_2d(&self.lm_head, gradients), } } } @@ -139,11 +140,16 @@ async fn main() { .sum::<0>(0) / (BATCH_SIZE * SEQ_LEN * VOCAB_SIZE) as f32; - let loss_value = loss.to_scalar().await.unwrap(); let gradients = loss.backward().unwrap(); - model = model.step(&gradients, &device).await; + let should_log = epoch % LOG_EVERY == 0 || epoch + 1 == EPOCHS; + let loss_value = if should_log { + Some(loss.to_scalar().await.unwrap()) + } else { + None + }; + model = model.step(&gradients); - if epoch % 20 == 0 || epoch + 1 == EPOCHS { + if let Some(loss_value) = loss_value { println!("epoch {:>3}: loss={loss_value:.6}", epoch + 1); } } @@ -217,29 +223,20 @@ fn zeros(device: &Device) -> Tensor<1, f32> { Tensor::new(device, &[0.0; LEN]) } -async fn sgd_step_1d( +fn sgd_step_1d( parameter: &Tensor<1, f32>, gradients: &Gradients, - device: &Device, ) -> Tensor<1, f32> { let gradient = gradients.get(parameter).unwrap(); - let next = parameter - &(gradient * LEARNING_RATE); - let host = next.to_vec1().await.unwrap(); - let host: [f32; LEN] = host.try_into().unwrap(); - Tensor::new(device, &host) + (parameter - &(gradient * LEARNING_RATE)).detach() } -async fn sgd_step_2d( +fn sgd_step_2d( parameter: &Tensor<2, f32>, gradients: &Gradients, - device: &Device, ) -> Tensor<2, f32> { let gradient = gradients.get(parameter).unwrap(); - let next = parameter - &(gradient * LEARNING_RATE); - let host = next.to_vec2().await.unwrap(); - let host: [[f32; COLS]; ROWS] = - std::array::from_fn(|row| std::array::from_fn(|col| host[row][col])); - Tensor::new(device, &host) + (parameter - &(gradient * LEARNING_RATE)).detach() } fn argmax_last_dim(logits: Vec>>) -> Vec> { diff --git a/fusor-ml/core/src/autograd.rs b/fusor-ml/core/src/autograd.rs index 103af2ba3..414c70d83 100644 --- a/fusor-ml/core/src/autograd.rs +++ b/fusor-ml/core/src/autograd.rs @@ -244,6 +244,25 @@ async fn test_backward_after_materializing_loss_scalar() { assert_close(dx[[2]], 2.0); } +#[cfg(test)] +#[tokio::test] +async fn test_detach_cuts_history() { + let device = crate::Device::test_instance(); + + let x = Tensor::new(&device, &[1.0f32, 2.0, 3.0]); + let detached = (&x * 2.0).detach(); + let loss: Tensor<0, f32> = detached.sum::<0>(0); + + let gradients = loss.backward().unwrap(); + + assert!(gradients.get(&x).is_none()); + + let d_detached = gradients.get(&detached).unwrap().as_slice().await.unwrap(); + assert_close(d_detached[[0]], 1.0); + assert_close(d_detached[[1]], 1.0); + assert_close(d_detached[[2]], 1.0); +} + #[cfg(test)] #[tokio::test] async fn test_backward_tiny_transformer_parameter_grads_present() { diff --git a/fusor-ml/core/src/tensor.rs b/fusor-ml/core/src/tensor.rs index 533795fbd..bf72db232 100644 --- a/fusor-ml/core/src/tensor.rs +++ b/fusor-ml/core/src/tensor.rs @@ -893,6 +893,13 @@ impl Tensor { } } + /// Resolve the current tensor value on device and return a fresh leaf tensor + /// that no longer carries the original compute graph history. + pub fn detach(&self) -> Self { + let (data, _) = self.data.materialize(); + Self::from_parts(LazyTensorData::new(data)) + } + /// How many kernel calls are needed to fully resolve this tensor pub fn count_kernels_to_resolve(&self) -> usize { let (_, count) = self.data.materialize(); From 8b6839a575bce6e8b1e5f8a37e67dac0d073c761 Mon Sep 17 00:00:00 2001 From: Evan Almloff Date: Sun, 15 Mar 2026 18:30:15 -0500 Subject: [PATCH 04/49] nanochat --- Cargo.lock | 9 + Cargo.toml | 1 + .../core/examples/train_tiny_transformer.rs | 259 ----------------- fusor-ml/gguf/src/lib.rs | 2 + fusor-ml/nanochat/Cargo.toml | 10 + fusor-ml/nanochat/chat.txt | 88 ++++++ fusor-ml/nanochat/src/config.rs | 20 ++ fusor-ml/nanochat/src/data.rs | 207 ++++++++++++++ fusor-ml/nanochat/src/main.rs | 135 +++++++++ fusor-ml/nanochat/src/model.rs | 264 ++++++++++++++++++ 10 files changed, 736 insertions(+), 259 deletions(-) delete mode 100644 fusor-ml/core/examples/train_tiny_transformer.rs create mode 100644 fusor-ml/nanochat/Cargo.toml create mode 100644 fusor-ml/nanochat/chat.txt create mode 100644 fusor-ml/nanochat/src/config.rs create mode 100644 fusor-ml/nanochat/src/data.rs create mode 100644 fusor-ml/nanochat/src/main.rs create mode 100644 fusor-ml/nanochat/src/model.rs diff --git a/Cargo.lock b/Cargo.lock index 906a2da05..3c6940c7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3075,6 +3075,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "fusor-nanochat" +version = "0.1.0" +dependencies = [ + "fusor-core", + "pollster", + "rand 0.9.2", +] + [[package]] name = "fusor-types" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e5c8aa78e..796931d02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ members = [ "fusor-ml/cpu", "fusor-ml/fusor", "fusor-ml/types", + "fusor-ml/nanochat", ] [workspace.dependencies] diff --git a/fusor-ml/core/examples/train_tiny_transformer.rs b/fusor-ml/core/examples/train_tiny_transformer.rs deleted file mode 100644 index 3b270a3b9..000000000 --- a/fusor-ml/core/examples/train_tiny_transformer.rs +++ /dev/null @@ -1,259 +0,0 @@ -use fusor_core::{Device, Gradients, Tensor, cache::AttentionMask}; -use rand::{Rng, SeedableRng, rngs::StdRng}; - -const VOCAB_SIZE: usize = 6; -const SEQ_LEN: usize = 4; -const BATCH_SIZE: usize = 6; -const MODEL_DIM: usize = 8; -const FF_DIM: usize = 16; -const EPOCHS: usize = 120; -const LEARNING_RATE: f32 = 0.08; -const EPS: f32 = 1e-5; -const LOG_EVERY: usize = 20; - -#[derive(Clone)] -struct TinyTransformer { - token_projection: Tensor<2, f32>, - position_projection: Tensor<2, f32>, - ln1_weight: Tensor<1, f32>, - ln1_bias: Tensor<1, f32>, - w_q: Tensor<2, f32>, - w_k: Tensor<2, f32>, - w_v: Tensor<2, f32>, - w_o: Tensor<2, f32>, - ln2_weight: Tensor<1, f32>, - ln2_bias: Tensor<1, f32>, - w1: Tensor<2, f32>, - b1: Tensor<1, f32>, - w2: Tensor<2, f32>, - b2: Tensor<1, f32>, - ln_out_weight: Tensor<1, f32>, - ln_out_bias: Tensor<1, f32>, - lm_head: Tensor<2, f32>, -} - -impl TinyTransformer { - fn new(device: &Device) -> Self { - let mut rng = StdRng::seed_from_u64(7); - Self { - token_projection: random_matrix::(device, &mut rng, 0.12), - position_projection: random_matrix::(device, &mut rng, 0.12), - ln1_weight: ones::(device), - ln1_bias: zeros::(device), - w_q: random_matrix::(device, &mut rng, 0.10), - w_k: random_matrix::(device, &mut rng, 0.10), - w_v: random_matrix::(device, &mut rng, 0.10), - w_o: random_matrix::(device, &mut rng, 0.10), - ln2_weight: ones::(device), - ln2_bias: zeros::(device), - w1: random_matrix::(device, &mut rng, 0.10), - b1: zeros::(device), - w2: random_matrix::(device, &mut rng, 0.10), - b2: zeros::(device), - ln_out_weight: ones::(device), - ln_out_bias: zeros::(device), - lm_head: random_matrix::(device, &mut rng, 0.10), - } - } - - fn forward( - &self, - token_inputs: &Tensor<3, f32>, - position_inputs: &Tensor<2, f32>, - causal_mask: &AttentionMask, - ) -> Tensor<3, f32> { - let token_embeddings = - token_inputs.mat_mul(&self.token_projection.broadcast_as([BATCH_SIZE, VOCAB_SIZE, MODEL_DIM])); - let position_embeddings: Tensor<2, f32> = position_inputs.mat_mul(&self.position_projection); - let mut x = token_embeddings.add_(&position_embeddings.broadcast_as([BATCH_SIZE, SEQ_LEN, MODEL_DIM])); - - let attn_input = - x.layer_norm(&self.ln1_weight, Some(&self.ln1_bias), EPS, true); - let q = attn_input.mat_mul(&self.w_q.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); - let k = attn_input.mat_mul(&self.w_k.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); - let v = attn_input.mat_mul(&self.w_v.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); - - let scores = q.mat_mul(&k.transpose(1, 2)) / (MODEL_DIM as f32).sqrt(); - let masked_scores = causal_mask.apply(&scores); - let weights_exp = masked_scores.exp(); - let attention = weights_exp.div_(&weights_exp.sum_keepdim(2)); - let attention_output = attention - .mat_mul(&v) - .mat_mul(&self.w_o.broadcast_as([BATCH_SIZE, MODEL_DIM, MODEL_DIM])); - x = x + attention_output; - - let ff_input = x.layer_norm(&self.ln2_weight, Some(&self.ln2_bias), EPS, true); - let ff_hidden = ff_input - .mat_mul(&self.w1.broadcast_as([BATCH_SIZE, MODEL_DIM, FF_DIM])) - .add_(&self.b1) - .relu(); - let ff_output = ff_hidden - .mat_mul(&self.w2.broadcast_as([BATCH_SIZE, FF_DIM, MODEL_DIM])) - .add_(&self.b2); - x = x + ff_output; - - let output = x.layer_norm(&self.ln_out_weight, Some(&self.ln_out_bias), EPS, true); - output.mat_mul(&self.lm_head.broadcast_as([BATCH_SIZE, MODEL_DIM, VOCAB_SIZE])) - } - - fn step(self, gradients: &Gradients) -> Self { - Self { - token_projection: sgd_step_2d(&self.token_projection, gradients), - position_projection: sgd_step_2d(&self.position_projection, gradients), - ln1_weight: sgd_step_1d(&self.ln1_weight, gradients), - ln1_bias: sgd_step_1d(&self.ln1_bias, gradients), - w_q: sgd_step_2d(&self.w_q, gradients), - w_k: sgd_step_2d(&self.w_k, gradients), - w_v: sgd_step_2d(&self.w_v, gradients), - w_o: sgd_step_2d(&self.w_o, gradients), - ln2_weight: sgd_step_1d(&self.ln2_weight, gradients), - ln2_bias: sgd_step_1d(&self.ln2_bias, gradients), - w1: sgd_step_2d(&self.w1, gradients), - b1: sgd_step_1d(&self.b1, gradients), - w2: sgd_step_2d(&self.w2, gradients), - b2: sgd_step_1d(&self.b2, gradients), - ln_out_weight: sgd_step_1d(&self.ln_out_weight, gradients), - ln_out_bias: sgd_step_1d(&self.ln_out_bias, gradients), - lm_head: sgd_step_2d(&self.lm_head, gradients), - } - } -} - -#[tokio::main] -async fn main() { - let device = Device::new().await.unwrap(); - - let token_ids = training_sequences(); - let token_inputs: Tensor<3, f32> = Tensor::new(&device, &token_one_hot(&token_ids)); - let targets: Tensor<3, f32> = Tensor::new(&device, &next_token_one_hot(&token_ids)); - let position_inputs: Tensor<2, f32> = Tensor::new(&device, &position_one_hot()); - let causal_mask = AttentionMask::causal(&device, SEQ_LEN); - - let mut model = TinyTransformer::new(&device); - - for epoch in 0..EPOCHS { - let logits = model.forward(&token_inputs, &position_inputs, &causal_mask); - let error = &logits - &targets; - let loss: Tensor<0, f32> = (&error * &error) - .sum::<2>(2) - .sum::<1>(1) - .sum::<0>(0) - / (BATCH_SIZE * SEQ_LEN * VOCAB_SIZE) as f32; - - let gradients = loss.backward().unwrap(); - let should_log = epoch % LOG_EVERY == 0 || epoch + 1 == EPOCHS; - let loss_value = if should_log { - Some(loss.to_scalar().await.unwrap()) - } else { - None - }; - model = model.step(&gradients); - - if let Some(loss_value) = loss_value { - println!("epoch {:>3}: loss={loss_value:.6}", epoch + 1); - } - } - - let logits = model.forward(&token_inputs, &position_inputs, &causal_mask); - let predictions = argmax_last_dim(logits.to_vec3().await.unwrap()); - - println!("training sequences:"); - for sequence in &token_ids { - println!(" {sequence:?}"); - } - println!("predicted next tokens:"); - for prediction in predictions { - println!(" {prediction:?}"); - } -} - -fn training_sequences() -> [[u32; SEQ_LEN]; BATCH_SIZE] { - [ - [0, 1, 2, 3], - [1, 2, 3, 4], - [2, 3, 4, 5], - [3, 4, 5, 0], - [4, 5, 0, 1], - [5, 0, 1, 2], - ] -} - -fn token_one_hot(tokens: &[[u32; SEQ_LEN]; BATCH_SIZE]) -> [[[f32; VOCAB_SIZE]; SEQ_LEN]; BATCH_SIZE] { - std::array::from_fn(|batch| { - std::array::from_fn(|position| { - let token = tokens[batch][position] as usize; - std::array::from_fn(|vocab| if vocab == token { 1.0 } else { 0.0 }) - }) - }) -} - -fn next_token_one_hot( - tokens: &[[u32; SEQ_LEN]; BATCH_SIZE], -) -> [[[f32; VOCAB_SIZE]; SEQ_LEN]; BATCH_SIZE] { - std::array::from_fn(|batch| { - std::array::from_fn(|position| { - let token = ((tokens[batch][position] as usize) + 1) % VOCAB_SIZE; - std::array::from_fn(|vocab| if vocab == token { 1.0 } else { 0.0 }) - }) - }) -} - -fn position_one_hot() -> [[f32; SEQ_LEN]; SEQ_LEN] { - std::array::from_fn(|position| { - std::array::from_fn(|column| if column == position { 1.0 } else { 0.0 }) - }) -} - -fn random_matrix( - device: &Device, - rng: &mut StdRng, - scale: f32, -) -> Tensor<2, f32> { - let data: [[f32; COLS]; ROWS] = std::array::from_fn(|_| { - std::array::from_fn(|_| rng.random_range(-scale..scale)) - }); - Tensor::new(device, &data) -} - -fn ones(device: &Device) -> Tensor<1, f32> { - Tensor::new(device, &[1.0; LEN]) -} - -fn zeros(device: &Device) -> Tensor<1, f32> { - Tensor::new(device, &[0.0; LEN]) -} - -fn sgd_step_1d( - parameter: &Tensor<1, f32>, - gradients: &Gradients, -) -> Tensor<1, f32> { - let gradient = gradients.get(parameter).unwrap(); - (parameter - &(gradient * LEARNING_RATE)).detach() -} - -fn sgd_step_2d( - parameter: &Tensor<2, f32>, - gradients: &Gradients, -) -> Tensor<2, f32> { - let gradient = gradients.get(parameter).unwrap(); - (parameter - &(gradient * LEARNING_RATE)).detach() -} - -fn argmax_last_dim(logits: Vec>>) -> Vec> { - logits - .into_iter() - .map(|sequence| { - sequence - .into_iter() - .map(|token_logits| { - token_logits - .iter() - .enumerate() - .max_by(|(_, left), (_, right)| left.total_cmp(right)) - .map(|(index, _)| index) - .unwrap() - }) - .collect() - }) - .collect() -} diff --git a/fusor-ml/gguf/src/lib.rs b/fusor-ml/gguf/src/lib.rs index 752413303..4bf13b92f 100644 --- a/fusor-ml/gguf/src/lib.rs +++ b/fusor-ml/gguf/src/lib.rs @@ -877,6 +877,7 @@ impl GgufBlock for BlockQ4_0 { } } +#[cfg(not(any(all(target_arch = "aarch64", nightly), target_arch = "x86_64")))] #[inline(always)] fn q4_0_vec_dot_scalar(data: &[u8; 16], y_data: &[i8; 32]) -> i32 { const CENTER: i8 = 8; @@ -1303,6 +1304,7 @@ impl GgufBlock for BlockQ8_0 { } } +#[cfg(not(any(all(target_arch = "aarch64", nightly), target_arch = "x86_64")))] #[inline(always)] fn q8_0_vec_dot_scalar(x_data: &[i8; 32], y_data: &[i8; 32]) -> i32 { let mut sum: i32 = 0; diff --git a/fusor-ml/nanochat/Cargo.toml b/fusor-ml/nanochat/Cargo.toml new file mode 100644 index 000000000..70fd6574c --- /dev/null +++ b/fusor-ml/nanochat/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "fusor-nanochat" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +fusor-core.workspace = true +pollster = "0.4.0" +rand.workspace = true diff --git a/fusor-ml/nanochat/chat.txt b/fusor-ml/nanochat/chat.txt new file mode 100644 index 000000000..6c977aeda --- /dev/null +++ b/fusor-ml/nanochat/chat.txt @@ -0,0 +1,88 @@ +What does Rust help with? Rust helps build fast software with strong memory safety. +Why use attention? Attention lets each token gather the context it needs. +What are embeddings? Embeddings turn discrete symbols into useful dense vectors. +Why does layer norm matter? Layer norm keeps activations stable during training. +How do transformers predict text? Transformers use context to score the next token. +What does gradient descent do? Gradient descent nudges weights toward lower loss. +Why use residual connections? Residual connections help information and gradients flow. +What does softmax produce? Softmax turns scores into smooth probabilities. +How does causal masking work? Causal masking blocks attention to future positions. +What makes tiny models useful? Tiny models are great for learning and debugging ideas. +How do position embeddings help? Position embeddings tell the model where each token is. +What is this demo training? This demo trains a tiny chat model on simple explanations. +What is a tensor? A tensor is a multidimensional array that stores model values. +Why train on examples? Training on examples teaches the model which outputs fit each prompt. +What does a loss measure? A loss measures how far predictions are from the desired targets. +What is a parameter? A parameter is a learned weight that changes during optimization. +What is a model? A model is a function with learned parameters. +Why do we need data? Data gives the model patterns to imitate and compress. +What is a token? A token is a unit the model predicts one step at a time. +Why keep the context window fixed? A fixed context window keeps training batches simple and uniform. +What does the optimizer update? The optimizer updates parameters using gradient information. +Why does the learning rate matter? The learning rate controls how large each parameter update is. +What happens if the learning rate is too high? A learning rate that is too high can make training unstable. +What happens if the learning rate is too low? A learning rate that is too low makes learning very slow. +Why use a validation set? A validation set checks whether the model generalizes beyond training data. +What is overfitting? Overfitting means the model memorizes training examples too narrowly. +What is generalization? Generalization is the ability to handle related prompts not seen exactly before. +Why shuffle batches? Shuffling batches exposes the model to examples in varied order. +What is a batch? A batch is a small set of examples processed together. +Why do batches help? Batches make training more efficient on modern hardware. +What does normalization do? Normalization keeps values in a healthier range for learning. +Why use a feed forward layer? A feed forward layer adds extra nonlinear capacity after attention. +What is nonlinear about relu? Relu keeps positive values and clips negative values to zero. +Why do models need nonlinear layers? Nonlinear layers let models represent more complex functions. +What does self attention compare? Self attention compares each token with the other tokens in context. +Why is context useful? Context helps the model resolve meaning from nearby words. +What is next token prediction? Next token prediction trains the model to continue a sequence. +Why does cross entropy work well? Cross entropy strongly rewards putting probability on the correct token. +What does a probability distribution mean here? It means the model assigns relative likelihood to each next token. +What is a logit? A logit is an unnormalized score before softmax. +Why do we subtract the max in softmax? Subtracting the max keeps exponentials numerically stable. +What is decoding? Decoding turns model scores into concrete generated tokens. +What is greedy decoding? Greedy decoding always picks the highest scoring next token. +Why can greedy decoding help this demo? Greedy decoding makes a weak toy model easier to inspect. +What is temperature in sampling? Temperature changes how sharp or flat the sampling distribution is. +Why use a compact vocabulary? A compact vocabulary avoids wasting probability mass on unused symbols. +What is a chat format? A chat format wraps prompts and replies in role marked text. +Why add system text? System text gives the model a consistent behavior target. +What is an assistant reply? It is the portion of text the model should generate after the prompt. +Why end examples with a stop token? A stop token teaches the model when a reply should end. +What is a prompt? A prompt is the text that conditions the next generated tokens. +Why do prompts matter? Prompts shape the context the model uses to answer. +What does inference mean? Inference is the process of generating outputs from a trained model. +What does training mean? Training is the process of adjusting weights to reduce loss. +Why can tiny datasets still help? Tiny datasets help us debug the training loop and model behavior. +What is the compute graph? The compute graph records how tensors depend on earlier tensors. +Why is backpropagation useful? Backpropagation efficiently computes gradients through the graph. +What is a gradient? A gradient shows how changing a parameter would change the loss. +Why detach updated parameters? Detaching makes the next training step start from fresh leaf tensors. +What does caching save? Caching saves repeated work or repeated transfers. +Why keep optimizer updates on the gpu? Keeping updates on the gpu avoids slow host round trips. +What does a mask do in the loss? A loss mask ignores padded positions that should not count. +Why pad sequences? Padding lets sequences of different lengths share one tensor shape. +What is a hidden dimension? A hidden dimension is the feature width used inside the model. +Why can wider models help? Wider models can store richer representations. +What does depth add? Depth lets the model refine representations over multiple layers. +Why do residual paths help deep models? Residual paths make it easier for gradients to move through depth. +What is memorization in a toy demo? Memorization means the model can reproduce the small training set well. +Why is memorization acceptable here? Memorization is fine here because the goal is to verify the pipeline. +What does a corpus contain? A corpus contains the text examples used for training. +Why use short answers? Short answers make the tiny dataset easier for the model to learn. +What is instruction tuning? Instruction tuning teaches the model to answer user requests in a desired style. +Why does a role prefix help? A role prefix tells the model which speaker is currently talking. +What is a response style? A response style is the tone and structure the model learns to imitate. +Why can repeated patterns help? Repeated patterns make the training signal clearer for small models. +What should this demo reply like? This demo should reply with short clear factual explanations. +What does the assistant know? The assistant knows only the small concepts shown in the chat data. +Why does more data help this crate? More data gives the tiny model more completions to imitate. +What is a good first debugging sign? A steadily falling loss is a good first debugging sign. +What is a better final sign? A better final sign is readable answers to held prompts. +Why compare prompts after training? Comparing prompts after training shows whether generation improved. +What is the purpose of this crate? The purpose of this crate is to train a tiny chat transformer with fusor. +How should the assistant answer? The assistant should answer briefly clearly and helpfully. +What does the crate demonstrate? The crate demonstrates data loading training autograd and generation. +Why is this called nanochat? It is called nanochat because it is a very small chat style model. +What does fusor provide here? Fusor provides tensor ops compute graphs and backpropagation. +Why are examples important? Examples show how the library works in an end to end task. +What is the final goal of the demo? The final goal is a tiny assistant that produces readable toy answers. diff --git a/fusor-ml/nanochat/src/config.rs b/fusor-ml/nanochat/src/config.rs new file mode 100644 index 000000000..e754f6c93 --- /dev/null +++ b/fusor-ml/nanochat/src/config.rs @@ -0,0 +1,20 @@ +pub const CHARSET: &str = "\n !,.:?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; +pub const BOS_TOKEN: u32 = CHARSET.len() as u32; +pub const EOT_TOKEN: u32 = CHARSET.len() as u32 + 1; +pub const VOCAB_SIZE: usize = CHARSET.len() + 2; + +pub const BLOCK_SIZE: usize = 224; +pub const BATCH_SIZE: usize = 8; +pub const N_EMBD: usize = 64; +pub const N_FF: usize = 128; +pub const N_LAYER: usize = 4; + +pub const TRAIN_STEPS: usize = 2200; +pub const LEARNING_RATE: f32 = 0.03; +pub const EPS: f32 = 1e-5; +pub const LOG_EVERY: usize = 100; + +pub const SAMPLE_TOKENS: usize = 80; + +pub const SYSTEM_PROMPT: &str = + "You are nanochat, a tiny helpful assistant trained with fusor."; diff --git a/fusor-ml/nanochat/src/data.rs b/fusor-ml/nanochat/src/data.rs new file mode 100644 index 000000000..b16cf393e --- /dev/null +++ b/fusor-ml/nanochat/src/data.rs @@ -0,0 +1,207 @@ +use std::array::from_fn; + +use rand::{Rng, rngs::StdRng}; + +use crate::config::{BATCH_SIZE, BLOCK_SIZE, BOS_TOKEN, CHARSET, EOT_TOKEN, SYSTEM_PROMPT, VOCAB_SIZE}; + +pub struct Tokenizer; + +pub struct ChatDataset { + examples: Vec, +} + +pub struct ChatExample { + user: String, + assistant: String, + tokens: Vec, + assistant_target_start: usize, +} + +pub struct Batch { + pub windows: [[u32; BLOCK_SIZE + 1]; BATCH_SIZE], + pub mask: [[f32; BLOCK_SIZE]; BATCH_SIZE], + pub valid_tokens: f32, +} + +impl Tokenizer { + pub fn encode_text(&self, text: &str) -> Vec { + text.chars() + .map(|ch| self.encode_char(ch)) + .collect() + } + + pub fn encode_chat_example(&self, user: &str, assistant: &str) -> Vec { + let mut tokens = vec![BOS_TOKEN]; + tokens.extend(self.encode_text("system: ")); + tokens.extend(self.encode_text(SYSTEM_PROMPT)); + tokens.extend(self.encode_text("\nuser: ")); + tokens.extend(self.encode_text(user)); + tokens.extend(self.encode_text("\nassistant: ")); + tokens.extend(self.encode_text(assistant)); + tokens.push(EOT_TOKEN); + tokens + } + + pub fn encode_chat_prompt(&self, user: &str) -> Vec { + let mut tokens = vec![BOS_TOKEN]; + tokens.extend(self.encode_text("system: ")); + tokens.extend(self.encode_text(SYSTEM_PROMPT)); + tokens.extend(self.encode_text("\nuser: ")); + tokens.extend(self.encode_text(user)); + tokens.extend(self.encode_text("\nassistant: ")); + tokens + } + + pub fn decode_text(&self, tokens: &[u32]) -> String { + tokens + .iter() + .copied() + .filter_map(|token| self.decode_token(token)) + .collect() + } + + pub fn decode_assistant_reply(&self, prompt_tokens: usize, tokens: &[u32]) -> String { + let generated: Vec = tokens[prompt_tokens..] + .iter() + .copied() + .take_while(|&token| token != EOT_TOKEN) + .collect(); + self.decode_text(&generated).trim().to_string() + } + + fn encode_char(&self, ch: char) -> u32 { + CHARSET + .chars() + .position(|candidate| candidate == ch) + .unwrap_or_else(|| panic!("character {ch:?} missing from nanochat charset")) as u32 + } + + fn decode_token(&self, token: u32) -> Option { + CHARSET.chars().nth(token as usize) + } +} + +impl ChatDataset { + pub fn from_tsv(text: &str, tokenizer: &Tokenizer) -> Self { + let examples = text + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| { + let (user, assistant) = line + .split_once('\t') + .expect("chat.txt must contain tab-separated user and assistant text"); + let prompt = tokenizer.encode_chat_prompt(user); + let tokens = tokenizer.encode_chat_example(user, assistant); + ChatExample { + user: user.to_string(), + assistant: assistant.to_string(), + tokens, + // The mask applies to positions whose next token belongs to the assistant reply. + assistant_target_start: prompt.len().saturating_sub(1), + } + }) + .collect(); + Self { examples } + } + + pub fn num_docs(&self) -> usize { + self.examples.len() + } + + pub fn num_tokens(&self) -> usize { + self.examples.iter().map(|example| example.tokens.len()).sum() + } + + pub fn sample_batch(&self, rng: &mut StdRng) -> Batch { + let sampled: [([u32; BLOCK_SIZE + 1], [f32; BLOCK_SIZE], f32); BATCH_SIZE] = + from_fn(|_| self.sample_example(rng)); + + Batch { + windows: from_fn(|index| sampled[index].0), + mask: from_fn(|index| sampled[index].1), + valid_tokens: sampled.iter().map(|(_, _, valid)| *valid).sum(), + } + } + + pub fn max_tokens_per_example(&self) -> usize { + self.examples + .iter() + .map(|example| example.tokens.len()) + .max() + .unwrap_or(0) + } + + pub fn examples(&self) -> &[ChatExample] { + &self.examples + } + + fn sample_example(&self, rng: &mut StdRng) -> ([u32; BLOCK_SIZE + 1], [f32; BLOCK_SIZE], f32) { + let example = &self.examples[rng.random_range(0..self.examples.len())]; + let mut window = [EOT_TOKEN; BLOCK_SIZE + 1]; + let mut mask = [0.0; BLOCK_SIZE]; + + assert!( + example.tokens.len() <= BLOCK_SIZE + 1, + "example length {} exceeds block size {}", + example.tokens.len(), + BLOCK_SIZE + 1 + ); + + window[..example.tokens.len()].copy_from_slice(&example.tokens); + let last_valid_input = example.tokens.len().saturating_sub(1); + if last_valid_input > example.assistant_target_start { + mask[example.assistant_target_start..last_valid_input].fill(1.0); + } + let valid = mask.iter().sum(); + (window, mask, valid) + } +} + +impl ChatExample { + pub fn user(&self) -> &str { + &self.user + } + + pub fn assistant(&self) -> &str { + &self.assistant + } +} + +pub fn windows_to_inputs( + windows: &[[u32; BLOCK_SIZE + 1]; BATCH_SIZE], +) -> [[[f32; VOCAB_SIZE]; BLOCK_SIZE]; BATCH_SIZE] { + let tokens = from_fn(|batch| from_fn(|index| windows[batch][index])); + one_hot(&tokens) +} + +pub fn windows_to_targets( + windows: &[[u32; BLOCK_SIZE + 1]; BATCH_SIZE], +) -> [[[f32; VOCAB_SIZE]; BLOCK_SIZE]; BATCH_SIZE] { + let tokens = from_fn(|batch| from_fn(|index| windows[batch][index + 1])); + one_hot(&tokens) +} + +pub fn autoregressive_context(tokens: &[u32]) -> ([u32; BLOCK_SIZE], usize) { + let mut context = [EOT_TOKEN; BLOCK_SIZE]; + let slice = if tokens.len() > BLOCK_SIZE { + &tokens[tokens.len() - BLOCK_SIZE..] + } else { + tokens + }; + context[..slice.len()].copy_from_slice(slice); + (context, slice.len().saturating_sub(1)) +} + +pub fn one_hot(tokens: &[[u32; T]; B]) -> [[[f32; VOCAB_SIZE]; T]; B] { + from_fn(|batch| { + from_fn(|position| { + let token = tokens[batch][position] as usize; + from_fn(|vocab| if vocab == token { 1.0 } else { 0.0 }) + }) + }) +} + +pub fn position_one_hot() -> [[f32; BLOCK_SIZE]; BLOCK_SIZE] { + from_fn(|position| from_fn(|column| if column == position { 1.0 } else { 0.0 })) +} diff --git a/fusor-ml/nanochat/src/main.rs b/fusor-ml/nanochat/src/main.rs new file mode 100644 index 000000000..a81b66e28 --- /dev/null +++ b/fusor-ml/nanochat/src/main.rs @@ -0,0 +1,135 @@ +mod config; +mod data; +mod model; + +use data::{ + ChatDataset, Tokenizer, autoregressive_context, one_hot, position_one_hot, windows_to_inputs, + windows_to_targets, +}; +use fusor_core::{Device, Tensor, cache::AttentionMask}; +use model::NanoChatModel; +use rand::{SeedableRng, rngs::StdRng}; + +use crate::config::{BATCH_SIZE, BLOCK_SIZE, EOT_TOKEN, LOG_EVERY, SAMPLE_TOKENS, TRAIN_STEPS, VOCAB_SIZE}; + +fn main() { + pollster::block_on(async { + let tokenizer = Tokenizer; + let dataset = ChatDataset::from_tsv(include_str!("../chat.txt"), &tokenizer); + assert!( + dataset.num_docs() >= 80, + "embedded chat dataset is unexpectedly small: {} docs", + dataset.num_docs() + ); + assert!( + dataset.max_tokens_per_example() <= BLOCK_SIZE + 1, + "max example length {} exceeds block size {}", + dataset.max_tokens_per_example(), + BLOCK_SIZE + 1 + ); + let mut rng = StdRng::seed_from_u64(1337); + + let device = Device::new().await.unwrap(); + let position_inputs: Tensor<2, f32> = Tensor::new(&device, &position_one_hot()); + let causal_mask = AttentionMask::causal(&device, BLOCK_SIZE); + let mut model = NanoChatModel::new(&device, &mut rng); + + println!( + "docs={} tokens={} vocab={} params={} max_example_tokens={}", + dataset.num_docs(), + dataset.num_tokens(), + VOCAB_SIZE, + model.num_parameters(), + dataset.max_tokens_per_example(), + ); + + for step in 0..TRAIN_STEPS { + let batch = dataset.sample_batch(&mut rng); + let token_inputs: Tensor<3, f32> = + Tensor::new(&device, &windows_to_inputs(&batch.windows)); + let targets: Tensor<3, f32> = + Tensor::new(&device, &windows_to_targets(&batch.windows)); + let mask: Tensor<2, f32> = Tensor::new(&device, &batch.mask); + + let logits = model.forward::(&token_inputs, &position_inputs, &causal_mask); + let loss = masked_cross_entropy::(&logits, &targets, &mask, batch.valid_tokens); + + let gradients = loss.backward().unwrap(); + let should_log = step % LOG_EVERY == 0 || step + 1 == TRAIN_STEPS; + let loss_value = if should_log { + Some(loss.to_scalar().await.unwrap()) + } else { + None + }; + model = model.step(&gradients); + + if let Some(loss_value) = loss_value { + println!("step {:>4} | loss={loss_value:.6}", step + 1); + } + } + + println!("\n--- training prompt eval ---"); + for example in dataset.examples().iter().take(8) { + let reply = generate_reply( + &model, + &device, + &position_inputs, + &causal_mask, + &tokenizer, + example.user(), + ) + .await; + println!("user: {}", example.user()); + println!("expected: {}", example.assistant()); + println!("assistant: {reply}\n"); + } + }); +} + +async fn generate_reply( + model: &NanoChatModel, + device: &Device, + position_inputs: &Tensor<2, f32>, + causal_mask: &AttentionMask, + tokenizer: &Tokenizer, + prompt: &str, +) -> String { + let mut tokens = tokenizer.encode_chat_prompt(prompt); + let prompt_len = tokens.len(); + + for _ in 0..SAMPLE_TOKENS { + let (context, last_index) = autoregressive_context(&tokens); + let batch = [context]; + let token_inputs: Tensor<3, f32> = Tensor::new(device, &one_hot(&batch)); + let logits = model.forward::<1>(&token_inputs, position_inputs, causal_mask); + let logits = logits.to_vec3().await.unwrap(); + let next = sample_from_logits(&logits[0][last_index]); + tokens.push(next); + if next == EOT_TOKEN { + break; + } + } + + tokenizer.decode_assistant_reply(prompt_len, &tokens) +} + +fn sample_from_logits(logits: &[f32]) -> u32 { + logits + .iter() + .enumerate() + .max_by(|(_, left), (_, right)| left.total_cmp(right)) + .map(|(index, _)| index as u32) + .unwrap() +} + +fn masked_cross_entropy( + logits: &Tensor<3, f32>, + targets: &Tensor<3, f32>, + mask: &Tensor<2, f32>, + valid_tokens: f32, +) -> Tensor<0, f32> { + let log_norm = logits.exp().sum_keepdim::<2>(2).log(); + let log_probs = logits.sub_(&log_norm.broadcast_as([B, BLOCK_SIZE, VOCAB_SIZE])); + let token_nll = -((targets * &log_probs).sum::<2>(2) * mask); + token_nll.sum::<1>(1).sum::<0>(0) / valid_tokens.max(1.0) +} diff --git a/fusor-ml/nanochat/src/model.rs b/fusor-ml/nanochat/src/model.rs new file mode 100644 index 000000000..9d9b6b3fb --- /dev/null +++ b/fusor-ml/nanochat/src/model.rs @@ -0,0 +1,264 @@ +use std::array::from_fn; + +use fusor_core::{Device, Gradients, Tensor, cache::AttentionMask}; +use rand::{Rng, rngs::StdRng}; + +use crate::config::{ + BATCH_SIZE, BLOCK_SIZE, EPS, LEARNING_RATE, N_EMBD, N_FF, N_LAYER, VOCAB_SIZE, +}; + +#[derive(Clone)] +pub struct NanoChatModel { + wte: Tensor<2, f32>, + wpe: Tensor<2, f32>, + blocks: [TransformerBlock; N_LAYER], + ln_f_weight: Tensor<1, f32>, + ln_f_bias: Tensor<1, f32>, + lm_head: Tensor<2, f32>, +} + +#[derive(Clone)] +struct TransformerBlock { + ln_1_weight: Tensor<1, f32>, + ln_1_bias: Tensor<1, f32>, + attn: CausalSelfAttention, + ln_2_weight: Tensor<1, f32>, + ln_2_bias: Tensor<1, f32>, + mlp: Mlp, +} + +#[derive(Clone)] +struct CausalSelfAttention { + c_attn_q: Tensor<2, f32>, + c_attn_k: Tensor<2, f32>, + c_attn_v: Tensor<2, f32>, + c_proj: Tensor<2, f32>, +} + +#[derive(Clone)] +struct Mlp { + c_fc: Tensor<2, f32>, + c_fc_bias: Tensor<1, f32>, + c_proj: Tensor<2, f32>, + c_proj_bias: Tensor<1, f32>, +} + +impl NanoChatModel { + pub fn new(device: &Device, rng: &mut StdRng) -> Self { + Self { + wte: random_matrix::(device, rng, 0.08), + wpe: random_matrix::(device, rng, 0.08), + blocks: from_fn(|_| TransformerBlock::new(device, rng)), + ln_f_weight: ones::(device), + ln_f_bias: zeros::(device), + lm_head: random_matrix::(device, rng, 0.08), + } + } + + pub fn forward( + &self, + token_inputs: &Tensor<3, f32>, + position_inputs: &Tensor<2, f32>, + causal_mask: &AttentionMask, + ) -> Tensor<3, f32> { + let token_embeddings = + token_inputs.mat_mul(&self.wte.broadcast_as([B, VOCAB_SIZE, N_EMBD])); + let position_embeddings: Tensor<2, f32> = position_inputs.mat_mul(&self.wpe); + let mut x = token_embeddings.add_(&position_embeddings.broadcast_as([B, BLOCK_SIZE, N_EMBD])); + + for block in &self.blocks { + x = block.forward::(x, causal_mask); + } + + let x = x.layer_norm(&self.ln_f_weight, Some(&self.ln_f_bias), EPS, true); + x.mat_mul(&self.lm_head.broadcast_as([B, N_EMBD, VOCAB_SIZE])) + } + + pub fn step(self, gradients: &Gradients) -> Self { + Self { + wte: sgd_step_2d(&self.wte, gradients), + wpe: sgd_step_2d(&self.wpe, gradients), + blocks: self.blocks.map(|block| block.step(gradients)), + ln_f_weight: sgd_step_1d(&self.ln_f_weight, gradients), + ln_f_bias: sgd_step_1d(&self.ln_f_bias, gradients), + lm_head: sgd_step_2d(&self.lm_head, gradients), + } + } + + pub fn num_parameters(&self) -> usize { + tensor_len(&self.wte) + + tensor_len(&self.wpe) + + self + .blocks + .iter() + .map(TransformerBlock::num_parameters) + .sum::() + + tensor_len(&self.ln_f_weight) + + tensor_len(&self.ln_f_bias) + + tensor_len(&self.lm_head) + } +} + +impl TransformerBlock { + fn new(device: &Device, rng: &mut StdRng) -> Self { + Self { + ln_1_weight: ones::(device), + ln_1_bias: zeros::(device), + attn: CausalSelfAttention::new(device, rng), + ln_2_weight: ones::(device), + ln_2_bias: zeros::(device), + mlp: Mlp::new(device, rng), + } + } + + fn forward( + &self, + x: Tensor<3, f32>, + causal_mask: &AttentionMask, + ) -> Tensor<3, f32> { + let attn_input = x.layer_norm(&self.ln_1_weight, Some(&self.ln_1_bias), EPS, true); + let attn_output = self.attn.forward::(&attn_input, causal_mask); + let x = x + attn_output; + + let mlp_input = x.layer_norm(&self.ln_2_weight, Some(&self.ln_2_bias), EPS, true); + x + self.mlp.forward::(&mlp_input) + } + + fn step(self, gradients: &Gradients) -> Self { + Self { + ln_1_weight: sgd_step_1d(&self.ln_1_weight, gradients), + ln_1_bias: sgd_step_1d(&self.ln_1_bias, gradients), + attn: self.attn.step(gradients), + ln_2_weight: sgd_step_1d(&self.ln_2_weight, gradients), + ln_2_bias: sgd_step_1d(&self.ln_2_bias, gradients), + mlp: self.mlp.step(gradients), + } + } + + fn num_parameters(&self) -> usize { + tensor_len(&self.ln_1_weight) + + tensor_len(&self.ln_1_bias) + + self.attn.num_parameters() + + tensor_len(&self.ln_2_weight) + + tensor_len(&self.ln_2_bias) + + self.mlp.num_parameters() + } +} + +impl CausalSelfAttention { + fn new(device: &Device, rng: &mut StdRng) -> Self { + Self { + c_attn_q: random_matrix::(device, rng, 0.08), + c_attn_k: random_matrix::(device, rng, 0.08), + c_attn_v: random_matrix::(device, rng, 0.08), + c_proj: random_matrix::(device, rng, 0.08), + } + } + + fn forward( + &self, + x: &Tensor<3, f32>, + causal_mask: &AttentionMask, + ) -> Tensor<3, f32> { + let q = x.mat_mul(&self.c_attn_q.broadcast_as([B, N_EMBD, N_EMBD])); + let k = x.mat_mul(&self.c_attn_k.broadcast_as([B, N_EMBD, N_EMBD])); + let v = x.mat_mul(&self.c_attn_v.broadcast_as([B, N_EMBD, N_EMBD])); + + let scores = q.mat_mul(&k.transpose(1, 2)) / (N_EMBD as f32).sqrt(); + let masked = causal_mask.apply(&scores); + let weights_exp = masked.exp(); + let attention = weights_exp.div_(&weights_exp.sum_keepdim(2)); + + attention + .mat_mul(&v) + .mat_mul(&self.c_proj.broadcast_as([B, N_EMBD, N_EMBD])) + } + + fn step(self, gradients: &Gradients) -> Self { + Self { + c_attn_q: sgd_step_2d(&self.c_attn_q, gradients), + c_attn_k: sgd_step_2d(&self.c_attn_k, gradients), + c_attn_v: sgd_step_2d(&self.c_attn_v, gradients), + c_proj: sgd_step_2d(&self.c_proj, gradients), + } + } + + fn num_parameters(&self) -> usize { + tensor_len(&self.c_attn_q) + + tensor_len(&self.c_attn_k) + + tensor_len(&self.c_attn_v) + + tensor_len(&self.c_proj) + } +} + +impl Mlp { + fn new(device: &Device, rng: &mut StdRng) -> Self { + Self { + c_fc: random_matrix::(device, rng, 0.08), + c_fc_bias: zeros::(device), + c_proj: random_matrix::(device, rng, 0.08), + c_proj_bias: zeros::(device), + } + } + + fn forward(&self, x: &Tensor<3, f32>) -> Tensor<3, f32> { + let hidden = x + .mat_mul(&self.c_fc.broadcast_as([B, N_EMBD, N_FF])) + .add_(&self.c_fc_bias) + .relu(); + + hidden + .mat_mul(&self.c_proj.broadcast_as([B, N_FF, N_EMBD])) + .add_(&self.c_proj_bias) + } + + fn step(self, gradients: &Gradients) -> Self { + Self { + c_fc: sgd_step_2d(&self.c_fc, gradients), + c_fc_bias: sgd_step_1d(&self.c_fc_bias, gradients), + c_proj: sgd_step_2d(&self.c_proj, gradients), + c_proj_bias: sgd_step_1d(&self.c_proj_bias, gradients), + } + } + + fn num_parameters(&self) -> usize { + tensor_len(&self.c_fc) + + tensor_len(&self.c_fc_bias) + + tensor_len(&self.c_proj) + + tensor_len(&self.c_proj_bias) + } +} + +fn random_matrix( + device: &Device, + rng: &mut StdRng, + scale: f32, +) -> Tensor<2, f32> { + let data: [[f32; COLS]; ROWS] = from_fn(|_| from_fn(|_| rng.random_range(-scale..scale))); + Tensor::new(device, &data) +} + +fn ones(device: &Device) -> Tensor<1, f32> { + Tensor::new(device, &[1.0; LEN]) +} + +fn zeros(device: &Device) -> Tensor<1, f32> { + Tensor::new(device, &[0.0; LEN]) +} + +fn sgd_step_1d(parameter: &Tensor<1, f32>, gradients: &Gradients) -> Tensor<1, f32> { + let gradient = gradients.get(parameter).unwrap(); + (parameter - &(gradient * LEARNING_RATE)).detach() +} + +fn sgd_step_2d(parameter: &Tensor<2, f32>, gradients: &Gradients) -> Tensor<2, f32> { + let gradient = gradients.get(parameter).unwrap(); + (parameter - &(gradient * LEARNING_RATE)).detach() +} + +fn tensor_len(tensor: &Tensor) -> usize { + tensor.shape().iter().product() +} + +#[allow(dead_code)] +const _: usize = BATCH_SIZE; From 1ba33f645ea0562d9c1097f2c73ed6a0adfa9373 Mon Sep 17 00:00:00 2001 From: Evan Almloff Date: Mon, 16 Mar 2026 19:25:13 -0500 Subject: [PATCH 05/49] wip --- Cargo.lock | 25 +- .../core/examples/train_linear_regression.rs | 92 +- fusor-ml/core/src/autograd.rs | 481 ------ fusor-ml/core/src/composite/where_cond.rs | 14 +- fusor-ml/core/src/compute_graph/backward.rs | 615 -------- fusor-ml/core/src/compute_graph/mod.rs | 44 +- fusor-ml/core/src/element_wise.rs | 9 +- fusor-ml/core/src/gather.rs | 142 ++ fusor-ml/core/src/lib.rs | 2 - fusor-ml/core/src/matmul/mod.rs | 12 +- fusor-ml/core/src/matmul/sgemm.rs | 27 +- fusor-ml/core/src/matmul/sgemm_params.rs | 2 +- fusor-ml/core/src/mir/kernel.rs | 9 + fusor-ml/core/src/pair_wise.rs | 59 +- fusor-ml/core/src/resize.rs | 8 +- fusor-ml/core/src/slice_assign.rs | 15 +- fusor-ml/core/src/tensor.rs | 19 +- fusor-ml/cpu/src/quantized.rs | 448 ++---- fusor-ml/fusor/src/autograd.rs | 1306 +++++++++++++++++ fusor-ml/fusor/src/layers/recurrent.rs | 557 +++++++ fusor-ml/fusor/src/lib.rs | 1 + fusor-ml/gguf/src/lib.rs | 28 + fusor-ml/nanochat/.env | 33 + fusor-ml/nanochat/Cargo.toml | 10 +- fusor-ml/nanochat/chat.txt | 88 -- fusor-ml/nanochat/nanochat-step-00100.gguf | Bin 0 -> 200552 bytes fusor-ml/nanochat/nanochat-step-00200.gguf | Bin 0 -> 565022 bytes fusor-ml/nanochat/nanochat-step-00300.gguf | Bin 0 -> 200552 bytes fusor-ml/nanochat/nanochat-step-00400.gguf | Bin 0 -> 565022 bytes fusor-ml/nanochat/nanochat-step-00500.gguf | Bin 0 -> 200552 bytes fusor-ml/nanochat/nanochat-step-00600.gguf | Bin 0 -> 565022 bytes fusor-ml/nanochat/nanochat-step-00800.gguf | Bin 0 -> 565022 bytes fusor-ml/nanochat/nanochat.gguf | Bin 0 -> 2931259 bytes fusor-ml/nanochat/src/config.rs | 176 ++- fusor-ml/nanochat/src/cpu_model.rs | 758 ++++++++++ fusor-ml/nanochat/src/data.rs | 1302 ++++++++++++++-- fusor-ml/nanochat/src/interactive_model.rs | 452 ++++++ fusor-ml/nanochat/src/main.rs | 881 ++++++++++- fusor-ml/nanochat/src/model.rs | 1072 ++++++++++++-- 39 files changed, 6569 insertions(+), 2118 deletions(-) delete mode 100644 fusor-ml/core/src/autograd.rs delete mode 100644 fusor-ml/core/src/compute_graph/backward.rs create mode 100644 fusor-ml/core/src/gather.rs create mode 100644 fusor-ml/fusor/src/autograd.rs create mode 100644 fusor-ml/fusor/src/layers/recurrent.rs create mode 100644 fusor-ml/nanochat/.env delete mode 100644 fusor-ml/nanochat/chat.txt create mode 100644 fusor-ml/nanochat/nanochat-step-00100.gguf create mode 100644 fusor-ml/nanochat/nanochat-step-00200.gguf create mode 100644 fusor-ml/nanochat/nanochat-step-00300.gguf create mode 100644 fusor-ml/nanochat/nanochat-step-00400.gguf create mode 100644 fusor-ml/nanochat/nanochat-step-00500.gguf create mode 100644 fusor-ml/nanochat/nanochat-step-00600.gguf create mode 100644 fusor-ml/nanochat/nanochat-step-00800.gguf create mode 100644 fusor-ml/nanochat/nanochat.gguf create mode 100644 fusor-ml/nanochat/src/cpu_model.rs create mode 100644 fusor-ml/nanochat/src/interactive_model.rs diff --git a/Cargo.lock b/Cargo.lock index 3c6940c7f..25c1db5d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2467,6 +2467,12 @@ dependencies = [ "zip 0.6.6", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "double-ended-peekable" version = "0.1.0" @@ -3079,9 +3085,17 @@ dependencies = [ name = "fusor-nanochat" version = "0.1.0" dependencies = [ - "fusor-core", + "bytemuck", + "dotenvy", + "flate2", + "fusor", + "fusor-gguf", + "half 2.7.1", + "midly", "pollster", "rand 0.9.2", + "reqwest 0.12.28", + "tar", ] [[package]] @@ -5558,6 +5572,15 @@ dependencies = [ "paste", ] +[[package]] +name = "midly" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "207d755f4cb882d20c4da58d707ca9130a0c9bc5061f657a4f299b8e36362b7a" +dependencies = [ + "rayon", +] + [[package]] name = "miette" version = "5.10.0" diff --git a/fusor-ml/core/examples/train_linear_regression.rs b/fusor-ml/core/examples/train_linear_regression.rs index 727660d9b..f275587d5 100644 --- a/fusor-ml/core/examples/train_linear_regression.rs +++ b/fusor-ml/core/examples/train_linear_regression.rs @@ -5,51 +5,55 @@ const EPOCHS: usize = 80; fn main() { pollster::block_on(async { - let device = Device::new().await.unwrap(); - - // Learn y = 2x + 1 from a tiny synthetic dataset. - let inputs: Tensor<2, f32> = Tensor::new(&device, &[[0.0], [1.0], [2.0], [3.0], [4.0]]); - let targets: Tensor<2, f32> = Tensor::new(&device, &[[1.0], [3.0], [5.0], [7.0], [9.0]]); - - let mut weight: Tensor<2, f32> = Tensor::new(&device, &[[0.0]]); - let mut bias: Tensor<2, f32> = Tensor::new(&device, &[[0.0]]); - - for epoch in 0..EPOCHS { - let bias_broadcast: Tensor<2, f32> = bias.broadcast_as([inputs.shape()[0], 1]); - let prediction = inputs.mat_mul(&weight) + &bias_broadcast; - let error = &prediction - &targets; - let squared_error = &error * &error; - let loss: Tensor<0, f32> = squared_error.sum::<1>(0).sum::<0>(0) / inputs.shape()[0] as f32; - - let loss_value = loss.to_scalar().await.unwrap(); - let gradients = loss.backward().unwrap(); - let weight_grad = gradients.get(&weight).unwrap(); - let bias_grad = gradients.get(&bias).unwrap(); - - // Apply a simple SGD update. - let next_weight = &weight - &(weight_grad * LEARNING_RATE); - let next_bias = &bias - &(bias_grad * LEARNING_RATE); - - // Recreate the parameter tensors from host values so each SGD step starts a fresh graph. - let next_weight_host = next_weight.as_slice().await.unwrap(); - let next_bias_host = next_bias.as_slice().await.unwrap(); - let weight_value = next_weight_host[[0, 0]]; - let bias_value = next_bias_host[[0, 0]]; - weight = Tensor::new(&device, &[[weight_value]]); - bias = Tensor::new(&device, &[[bias_value]]); - - if epoch % 10 == 0 || epoch + 1 == EPOCHS { - println!( - "epoch {:>2}: loss={:.6} weight={:.4} bias={:.4}", - epoch + 1, - loss_value, - weight_value, - bias_value, - ); + let device = Device::new().await.unwrap(); + + // Learn y = 2x + 1 from a tiny synthetic dataset. + let inputs: Tensor<2, f32> = Tensor::new(&device, &[[0.0], [1.0], [2.0], [3.0], [4.0]]); + let targets: Tensor<2, f32> = Tensor::new(&device, &[[1.0], [3.0], [5.0], [7.0], [9.0]]); + + let mut weight: Tensor<2, f32> = Tensor::new(&device, &[[0.0]]); + let mut bias: Tensor<2, f32> = Tensor::new(&device, &[[0.0]]); + + for epoch in 0..EPOCHS { + let bias_broadcast: Tensor<2, f32> = bias.broadcast_as([inputs.shape()[0], 1]); + let prediction = inputs.mat_mul(&weight) + &bias_broadcast; + let error = &prediction - &targets; + let squared_error = &error * &error; + let loss: Tensor<0, f32> = + squared_error.sum::<1>(0).sum::<0>(0) / inputs.shape()[0] as f32; + + let loss_value = loss.to_scalar().await.unwrap(); + let gradients = loss.backward().unwrap(); + let weight_grad = gradients.get(&weight).unwrap(); + let bias_grad = gradients.get(&bias).unwrap(); + + // Apply a simple SGD update. + let next_weight = &weight - &(weight_grad * LEARNING_RATE); + let next_bias = &bias - &(bias_grad * LEARNING_RATE); + + // Recreate the parameter tensors from host values so each SGD step starts a fresh graph. + let next_weight_host = next_weight.as_slice().await.unwrap(); + let next_bias_host = next_bias.as_slice().await.unwrap(); + let weight_value = next_weight_host[[0, 0]]; + let bias_value = next_bias_host[[0, 0]]; + weight = Tensor::new(&device, &[[weight_value]]); + bias = Tensor::new(&device, &[[bias_value]]); + + if epoch % 10 == 0 || epoch + 1 == EPOCHS { + println!( + "epoch {:>2}: loss={:.6} weight={:.4} bias={:.4}", + epoch + 1, + loss_value, + weight_value, + bias_value, + ); + } } - } - let final_prediction = inputs.mat_mul(&weight) + &bias.broadcast_as([inputs.shape()[0], 1]); - println!("final predictions: {:?}", final_prediction.as_slice().await.unwrap()); + let final_prediction = inputs.mat_mul(&weight) + &bias.broadcast_as([inputs.shape()[0], 1]); + println!( + "final predictions: {:?}", + final_prediction.as_slice().await.unwrap() + ); }); } diff --git a/fusor-ml/core/src/autograd.rs b/fusor-ml/core/src/autograd.rs deleted file mode 100644 index 414c70d83..000000000 --- a/fusor-ml/core/src/autograd.rs +++ /dev/null @@ -1,481 +0,0 @@ -use rustc_hash::FxHashMap; - -use crate::{ - DataType, FloatDataType, Result, Tensor, - compute_graph::{BackwardRule, NodeIndex}, - tensor::LazyTensorData, -}; - -pub struct Gradients { - gradients: FxHashMap, -} - -pub struct BackwardTarget { - node: NodeIndex, - gradient: LazyTensorData, -} - -impl Gradients { - pub(crate) fn new(gradients: FxHashMap) -> Self { - Self { gradients } - } -} - -impl Gradients { - pub fn get(&self, tensor: &Tensor) -> Option> { - self.gradients - .get(&tensor.key()) - .cloned() - .map(Tensor::from_parts) - } -} - -impl BackwardTarget { - pub fn wrt(tensor: &Tensor, gradient: Tensor) -> Self { - Self { - node: tensor.key(), - gradient: gradient.data().clone(), - } - } -} - -impl Tensor { - pub fn with_backwards(self, backwards: F) -> Self - where - F: Fn(Tensor) -> Result> + Send + Sync + 'static, - { - let backward: BackwardRule = std::sync::Arc::new(move |gradient: LazyTensorData| { - let gradient = Tensor::from_parts(gradient); - let gradients = backwards(gradient)?; - Ok(gradients - .into_iter() - .map(|target| (target.node, target.gradient)) - .collect()) - }); - self.device().compute_graph().set_backward_rule(self.key(), backward); - self - } -} - -impl Tensor { - pub fn backward(&self) -> Result { - if self.shape().iter().product::() != 1 { - return Err(crate::Error::msg( - "backward() requires a single-element tensor; use backward_with() for non-scalars", - )); - } - - let seed = Tensor::splat(self.device(), D::one(), *self.shape()); - self.backward_with(&seed) - } - - pub fn backward_with(&self, seed: &Tensor) -> Result { - if self.shape() != seed.shape() { - return Err(crate::Error::msg(format!( - "gradient seed shape mismatch: expected {:?}, got {:?}", - self.shape(), - seed.shape() - ))); - } - - let gradients = self - .device() - .compute_graph() - .backward(self.key(), seed.data().clone())?; - Ok(Gradients::new(gradients)) - } -} - -#[cfg(test)] -fn assert_close(left: f32, right: f32) { - assert!( - (left - right).abs() < 1e-3, - "expected {right}, got {left}" - ); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_squared_sum() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[1.0f32, 2.0, 3.0]); - let loss: Tensor<0, f32> = (&x * &x).sum::<0>(0); - - let gradients = loss.backward().unwrap(); - let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); - - assert_close(dx[[0]], 2.0); - assert_close(dx[[1]], 4.0); - assert_close(dx[[2]], 6.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_matmul_with_broadcast_bias() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); - let w = Tensor::new(&device, &[[0.5f32], [1.0], [1.5]]); - let b = Tensor::new(&device, &[[2.0f32]]); - - let y = x.mat_mul(&w) + &b.broadcast_as([2, 1]); - let loss: Tensor<0, f32> = y.sum::<1>(0).sum::<0>(0); - - let gradients = loss.backward().unwrap(); - - let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); - assert_close(dx[[0, 0]], 0.5); - assert_close(dx[[0, 1]], 1.0); - assert_close(dx[[0, 2]], 1.5); - assert_close(dx[[1, 0]], 0.5); - assert_close(dx[[1, 1]], 1.0); - assert_close(dx[[1, 2]], 1.5); - - let dw = gradients.get(&w).unwrap().as_slice().await.unwrap(); - assert_close(dw[[0, 0]], 5.0); - assert_close(dw[[1, 0]], 7.0); - assert_close(dw[[2, 0]], 9.0); - - let db = gradients.get(&b).unwrap().as_slice().await.unwrap(); - assert_close(db[[0, 0]], 2.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_matmul_with_broadcasted_weight_batch() { - let device = crate::Device::test_instance(); - - let x = Tensor::new( - &device, - &[ - [[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]], - [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]], - ], - ); - let w = Tensor::new(&device, &[[0.5f32, 1.0], [1.5, 2.0], [2.5, 3.0]]); - - let y = x.mat_mul(&w.broadcast_as([2, 3, 2])); - let loss: Tensor<0, f32> = y.sum::<2>(2).sum::<1>(1).sum::<0>(0); - - let gradients = loss.backward().unwrap(); - - let dw = gradients.get(&w).unwrap().as_slice().await.unwrap(); - assert_close(dw[[0, 0]], 22.0); - assert_close(dw[[0, 1]], 22.0); - assert_close(dw[[1, 0]], 26.0); - assert_close(dw[[1, 1]], 26.0); - assert_close(dw[[2, 0]], 30.0); - assert_close(dw[[2, 1]], 30.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_slice() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); - let sliced = x.slice([0..2, 1..3]); - let loss: Tensor<0, f32> = sliced.sum::<1>(0).sum::<0>(0); - - let gradients = loss.backward().unwrap(); - let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); - - assert_close(dx[[0, 0]], 0.0); - assert_close(dx[[0, 1]], 1.0); - assert_close(dx[[0, 2]], 1.0); - assert_close(dx[[1, 0]], 0.0); - assert_close(dx[[1, 1]], 1.0); - assert_close(dx[[1, 2]], 1.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_relu() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[1.0f32, -2.0, 0.0, 4.0]); - let loss: Tensor<0, f32> = x.relu().sum::<0>(0); - - let gradients = loss.backward().unwrap(); - let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); - - assert_close(dx[[0]], 1.0); - assert_close(dx[[1]], 0.0); - assert_close(dx[[2]], 0.0); - assert_close(dx[[3]], 1.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_with_backwards_override() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[1.0f32, 2.0]); - let captured = x.clone(); - let y = (x.clone() + 1.0).with_backwards(move |grad| { - Ok(vec![BackwardTarget::wrt(&captured, grad * 3.0)]) - }); - let loss: Tensor<0, f32> = y.sum::<0>(0); - - let gradients = loss.backward().unwrap(); - let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); - - assert_close(dx[[0]], 3.0); - assert_close(dx[[1]], 3.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_after_materializing_loss_scalar() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[1.0f32, -2.0, 3.0]); - let loss: Tensor<0, f32> = ((x.relu() + 1.0) * 2.0).sum::<0>(0); - - let loss_value = loss.to_scalar().await.unwrap(); - assert_close(loss_value, 14.0); - - let gradients = loss.backward().unwrap(); - let dx = gradients.get(&x).unwrap().as_slice().await.unwrap(); - - assert_close(dx[[0]], 2.0); - assert_close(dx[[1]], 0.0); - assert_close(dx[[2]], 2.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_detach_cuts_history() { - let device = crate::Device::test_instance(); - - let x = Tensor::new(&device, &[1.0f32, 2.0, 3.0]); - let detached = (&x * 2.0).detach(); - let loss: Tensor<0, f32> = detached.sum::<0>(0); - - let gradients = loss.backward().unwrap(); - - assert!(gradients.get(&x).is_none()); - - let d_detached = gradients.get(&detached).unwrap().as_slice().await.unwrap(); - assert_close(d_detached[[0]], 1.0); - assert_close(d_detached[[1]], 1.0); - assert_close(d_detached[[2]], 1.0); -} - -#[cfg(test)] -#[tokio::test] -async fn test_backward_tiny_transformer_parameter_grads_present() { - use crate::cache::AttentionMask; - - const VOCAB: usize = 4; - const SEQ: usize = 3; - const BATCH: usize = 2; - const MODEL: usize = 4; - const FF: usize = 6; - const EPS: f32 = 1e-5; - - let device = crate::Device::test_instance(); - - let token_inputs: Tensor<3, f32> = Tensor::new( - &device, - &[ - [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - [ - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - [0.0, 0.0, 0.0, 1.0], - ], - ], - ); - let targets: Tensor<3, f32> = Tensor::new( - &device, - &[ - [ - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - [0.0, 0.0, 0.0, 1.0], - ], - [ - [0.0, 0.0, 1.0, 0.0], - [0.0, 0.0, 0.0, 1.0], - [1.0, 0.0, 0.0, 0.0], - ], - ], - ); - let position_inputs: Tensor<2, f32> = Tensor::new( - &device, - &[ - [1.0, 0.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 0.0, 1.0], - ], - ); - let causal_mask = AttentionMask::causal(&device, SEQ); - - let token_projection = Tensor::new( - &device, - &[ - [0.10, -0.02, 0.03, 0.04], - [0.05, 0.06, -0.07, 0.08], - [-0.04, 0.03, 0.02, -0.01], - [0.07, -0.05, 0.06, 0.02], - ], - ); - let position_projection = Tensor::new( - &device, - &[ - [0.01, 0.02, 0.03, 0.04], - [0.04, 0.03, 0.02, 0.01], - [-0.02, 0.01, 0.00, 0.03], - ], - ); - let ln1_weight = Tensor::new(&device, &[1.0, 1.0, 1.0, 1.0]); - let ln1_bias = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); - let w_q = Tensor::new( - &device, - &[ - [0.02, 0.03, 0.01, -0.02], - [0.01, -0.01, 0.04, 0.02], - [0.05, 0.02, -0.03, 0.01], - [-0.02, 0.01, 0.02, 0.03], - ], - ); - let w_k = Tensor::new( - &device, - &[ - [0.01, -0.03, 0.02, 0.04], - [0.02, 0.05, -0.01, 0.03], - [0.03, 0.01, 0.04, -0.02], - [0.00, 0.02, 0.01, 0.05], - ], - ); - let w_v = Tensor::new( - &device, - &[ - [0.04, 0.01, -0.02, 0.03], - [-0.01, 0.03, 0.02, 0.04], - [0.02, 0.05, 0.01, -0.03], - [0.03, -0.02, 0.04, 0.01], - ], - ); - let w_o = Tensor::new( - &device, - &[ - [0.03, 0.02, 0.01, -0.01], - [0.04, -0.02, 0.03, 0.02], - [0.01, 0.05, -0.01, 0.03], - [0.02, 0.01, 0.04, -0.02], - ], - ); - let ln2_weight = Tensor::new(&device, &[1.0, 1.0, 1.0, 1.0]); - let ln2_bias = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); - let w1 = Tensor::new( - &device, - &[ - [0.02, 0.01, 0.03, -0.02, 0.04, 0.01], - [0.01, 0.04, -0.01, 0.02, 0.03, 0.05], - [0.03, -0.02, 0.05, 0.01, -0.01, 0.02], - [0.04, 0.02, 0.01, 0.03, 0.02, -0.02], - ], - ); - let b1 = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]); - let w2 = Tensor::new( - &device, - &[ - [0.01, 0.02, 0.03, 0.04], - [0.02, 0.03, -0.01, 0.01], - [0.04, -0.02, 0.01, 0.03], - [0.03, 0.01, 0.02, -0.01], - [0.01, 0.04, 0.03, 0.02], - [0.02, -0.01, 0.04, 0.03], - ], - ); - let b2 = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); - let ln_out_weight = Tensor::new(&device, &[1.0, 1.0, 1.0, 1.0]); - let ln_out_bias = Tensor::new(&device, &[0.0, 0.0, 0.0, 0.0]); - let lm_head = Tensor::new( - &device, - &[ - [0.02, 0.01, 0.03, 0.04], - [0.03, 0.02, 0.01, -0.01], - [0.04, -0.02, 0.02, 0.03], - [0.01, 0.03, 0.04, 0.02], - ], - ); - - let token_embeddings = - token_inputs.mat_mul(&token_projection.broadcast_as([BATCH, VOCAB, MODEL])); - let position_embeddings: Tensor<2, f32> = position_inputs.mat_mul(&position_projection); - let position_embeddings_broadcast = position_embeddings.broadcast_as([BATCH, SEQ, MODEL]); - let mut x = token_embeddings.add_(&position_embeddings_broadcast); - let embedding_sum = x.clone(); - - let attn_input = x.layer_norm(&ln1_weight, Some(&ln1_bias), EPS, true); - let q = attn_input.mat_mul(&w_q.broadcast_as([BATCH, MODEL, MODEL])); - let k = attn_input.mat_mul(&w_k.broadcast_as([BATCH, MODEL, MODEL])); - let v = attn_input.mat_mul(&w_v.broadcast_as([BATCH, MODEL, MODEL])); - - let scores = q.mat_mul(&k.transpose(1, 2)) / (MODEL as f32).sqrt(); - let masked_scores = causal_mask.apply(&scores); - let weights_exp = masked_scores.exp(); - let attention = weights_exp.div_(&weights_exp.sum_keepdim(2)); - let attention_output = attention - .mat_mul(&v) - .mat_mul(&w_o.broadcast_as([BATCH, MODEL, MODEL])); - x = x + attention_output; - let after_attention = x.clone(); - - let ff_input = x.layer_norm(&ln2_weight, Some(&ln2_bias), EPS, true); - let ff_hidden = ff_input - .mat_mul(&w1.broadcast_as([BATCH, MODEL, FF])) - .add_(&b1) - .relu(); - let ff_output = ff_hidden - .mat_mul(&w2.broadcast_as([BATCH, FF, MODEL])) - .add_(&b2); - x = x + ff_output; - let after_ff = x.clone(); - - let output = x.layer_norm(&ln_out_weight, Some(&ln_out_bias), EPS, true); - let logits = output.mat_mul(&lm_head.broadcast_as([BATCH, MODEL, VOCAB])); - let error = &logits - &targets; - let loss: Tensor<0, f32> = (&error * &error) - .sum::<2>(2) - .sum::<1>(1) - .sum::<0>(0) - / (BATCH * SEQ * VOCAB) as f32; - - let _ = loss.to_scalar().await.unwrap(); - let gradients = loss.backward().unwrap(); - - assert!(gradients.get(&token_embeddings).is_some()); - assert!(gradients.get(&embedding_sum).is_some()); - assert!(gradients.get(&attn_input).is_some()); - assert!(gradients.get(&after_attention).is_some()); - assert!(gradients.get(&ff_input).is_some()); - assert!(gradients.get(&after_ff).is_some()); - assert!(gradients.get(&output).is_some()); - assert!(gradients.get(&logits).is_some()); - assert!(gradients.get(&token_projection).is_some()); - assert!(gradients.get(&position_projection).is_some()); - assert!(gradients.get(&w_q).is_some()); - assert!(gradients.get(&w_k).is_some()); - assert!(gradients.get(&w_v).is_some()); - assert!(gradients.get(&w_o).is_some()); - assert!(gradients.get(&w1).is_some()); - assert!(gradients.get(&w2).is_some()); - assert!(gradients.get(&lm_head).is_some()); - assert!(gradients.get(&ln1_weight).is_some()); - assert!(gradients.get(&ln1_bias).is_some()); - assert!(gradients.get(&ln2_weight).is_some()); - assert!(gradients.get(&ln2_bias).is_some()); - assert!(gradients.get(&b1).is_some()); - assert!(gradients.get(&b2).is_some()); - assert!(gradients.get(&ln_out_weight).is_some()); - assert!(gradients.get(&ln_out_bias).is_some()); -} diff --git a/fusor-ml/core/src/composite/where_cond.rs b/fusor-ml/core/src/composite/where_cond.rs index 42c289d10..f3b733a3c 100644 --- a/fusor-ml/core/src/composite/where_cond.rs +++ b/fusor-ml/core/src/composite/where_cond.rs @@ -1,4 +1,4 @@ -use crate::{BackwardTarget, DataType, Tensor, compute_graph::NodeIndex, tensor::DataTypeEnum}; +use crate::{DataType, Tensor, compute_graph::NodeIndex, tensor::DataTypeEnum}; impl Tensor { pub fn where_cond(self, on_true: &Tensor, on_false: &Tensor) -> Tensor @@ -14,17 +14,7 @@ impl Tensor { self.shape(), ); let data = on_true.data(); - let output = Tensor::from_parts(data.where_cond(operation)); - let condition = self.clone(); - let on_true = on_true.clone(); - let on_false = on_false.clone(); - output.with_backwards(move |grad| { - let zeros = Tensor::zeros(grad.device(), *grad.shape()); - Ok(vec![ - BackwardTarget::wrt(&on_true, condition.clone().where_cond(&grad, &zeros)), - BackwardTarget::wrt(&on_false, condition.clone().where_cond(&zeros, &grad)), - ]) - }) + Tensor::from_parts(data.where_cond(operation)) } } diff --git a/fusor-ml/core/src/compute_graph/backward.rs b/fusor-ml/core/src/compute_graph/backward.rs deleted file mode 100644 index 8065e5d90..000000000 --- a/fusor-ml/core/src/compute_graph/backward.rs +++ /dev/null @@ -1,615 +0,0 @@ -use std::ops::Range; - -use rustc_hash::{FxHashMap, FxHashSet}; - -use crate::{ - DataTypeEnum, Layout, MatMulOperation, PairWiseFunction, ReduceFunction, Result, TensorInfo, - composite::where_cond::WhereCondOperation, - map_layout::{MapLayoutKind, MapLayoutOperation}, - mir::operation::Operation, - pair_wise::PairWiseOperation, - reduce::ReduceOperation, - resize::ResizeOperation, - slice_assign::SliceAssignOperation, - tensor::{LazyTensorData, TensorData}, -}; - -use super::{BackwardRule, ComputeGraph, ComputeGraphInner, ComputeGraphNodeVariant, NodeIndex}; - -#[derive(Clone)] -struct NodeSnapshot { - variant: ComputeGraphNodeVariant, - info: TensorInfo, - backward: Option, -} - -impl ComputeGraph { - pub(crate) fn backward( - &self, - target: NodeIndex, - seed: LazyTensorData, - ) -> Result> { - let snapshots = { - let inner = self.inner.read(); - snapshot_subgraph(&inner, target)? - }; - let target_info = snapshots - .get(&target) - .ok_or_else(|| crate::Error::msg("backpropagation target was not found"))?; - - if seed.info().shape() != target_info.info.shape() - || seed.info().datatype() != target_info.info.datatype() - { - return Err(crate::Error::msg(format!( - "gradient seed shape/datatype mismatch: expected {:?} {}, got {:?} {}", - target_info.info.shape(), - target_info.info.datatype(), - seed.info().shape(), - seed.info().datatype() - ))); - } - - let mut gradients = FxHashMap::default(); - gradients.insert(target, seed); - - let mut visited = FxHashSet::default(); - let mut topo = Vec::new(); - build_topological_order(&snapshots, target, &mut visited, &mut topo); - - for node in topo.into_iter().rev() { - let Some(grad) = gradients.get(&node).cloned() else { - continue; - }; - let snapshot = snapshots - .get(&node) - .ok_or_else(|| crate::Error::msg("missing node while backpropagating"))?; - propagate_gradient(&snapshots, node, snapshot, ErasedTensor::from_lazy(grad), &mut gradients)?; - } - - Ok(gradients) - } -} - -fn snapshot_subgraph( - graph: &ComputeGraphInner, - target: NodeIndex, -) -> Result> { - let mut visited = FxHashSet::default(); - let mut order = Vec::new(); - collect_postorder(graph, target, &mut visited, &mut order)?; - - let mut infos = FxHashMap::default(); - let mut snapshots = FxHashMap::default(); - for node in order { - let variant = graph - .nodes - .nodes - .node_weight(node) - .ok_or_else(|| crate::Error::msg(format!("missing node {node:?} in compute graph")))? - .variant - .clone(); - let info = infer_info(&variant, &infos)?; - infos.insert(node, info.clone()); - let backward = graph - .nodes - .nodes - .node_weight(node) - .and_then(|node| node.backward.clone()); - snapshots.insert( - node, - NodeSnapshot { - variant, - info, - backward, - }, - ); - } - Ok(snapshots) -} - -fn collect_postorder( - graph: &ComputeGraphInner, - node: NodeIndex, - visited: &mut FxHashSet, - order: &mut Vec, -) -> Result<()> { - if !visited.insert(node) { - return Ok(()); - } - - let variant = graph - .nodes - .nodes - .node_weight(node) - .ok_or_else(|| crate::Error::msg(format!("missing node {node:?} in compute graph")))? - .variant - .clone(); - - let mut dependencies = Vec::new(); - variant.visit_dependencies(&mut |dependency| { - dependencies.push(dependency); - }); - for dependency in dependencies { - collect_postorder(graph, dependency, visited, order)?; - } - order.push(node); - Ok(()) -} - -fn build_topological_order( - snapshots: &FxHashMap, - node: NodeIndex, - visited: &mut FxHashSet, - order: &mut Vec, -) { - if !visited.insert(node) { - return; - } - - if let Some(snapshot) = snapshots.get(&node) { - let mut dependencies = Vec::new(); - snapshot.variant.visit_dependencies(&mut |dependency| { - dependencies.push(dependency); - }); - for dependency in dependencies { - build_topological_order(snapshots, dependency, visited, order); - } - } - - order.push(node); -} - -fn infer_info( - variant: &ComputeGraphNodeVariant, - infos: &FxHashMap, -) -> Result { - let info = match variant { - ComputeGraphNodeVariant::ElementWise(op) => { - TensorInfo::new(op.shape().into(), op.functions.out_datatype()) - } - ComputeGraphNodeVariant::PairWise(op) => { - TensorInfo::new(op.shape().into(), op.function.datatype) - } - ComputeGraphNodeVariant::Nary(op) => TensorInfo::new(op.shape.clone(), op.output_datatype), - ComputeGraphNodeVariant::SliceAssign(op) => { - let input = infos - .get(&op.input) - .ok_or_else(|| crate::Error::msg("slice_assign input info missing"))?; - TensorInfo::new(op.input_shape.clone(), input.datatype()) - } - ComputeGraphNodeVariant::Resize(op) => { - let input = infos - .get(&op.input) - .ok_or_else(|| crate::Error::msg("resize input info missing"))?; - TensorInfo::new(op.new_shape.clone(), input.datatype()) - } - ComputeGraphNodeVariant::MapLayout(op) => { - let input = infos - .get(&op.input) - .ok_or_else(|| crate::Error::msg("map_layout input info missing"))?; - let layout = op.map_layout(&Layout::contiguous(input.shape())); - TensorInfo::new(layout.shape().into(), input.datatype()) - } - ComputeGraphNodeVariant::Dequantize(op) => { - TensorInfo::new( - op.matrix.shape().to_vec().into_boxed_slice(), - op.post_dequantize.out_datatype(), - ) - } - ComputeGraphNodeVariant::MatMul(op) => { - TensorInfo::new(op.out_shape.clone(), op.post_element_wise.out_datatype()) - } - ComputeGraphNodeVariant::QMatMul(op) => TensorInfo::new(op.out_shape.clone(), op.input_datatype), - ComputeGraphNodeVariant::Tensor(data) => { - TensorInfo::new(data.layout().shape().into(), data.datatype()) - } - ComputeGraphNodeVariant::Reduce(op) => { - let shape = op - .shape - .iter() - .enumerate() - .filter_map(|(index, dim)| (index != op.axis).then_some(*dim)) - .collect(); - TensorInfo::new(shape, op.out_datatype()) - } - ComputeGraphNodeVariant::IndexSelect(op) => { - let input = infos - .get(&op.input) - .ok_or_else(|| crate::Error::msg("index_select input info missing"))?; - TensorInfo::new(op.output_shape(), input.datatype()) - } - ComputeGraphNodeVariant::WhereCond(op) => { - TensorInfo::new(op.shape.clone(), op.output_datatype) - } - ComputeGraphNodeVariant::Custom(op) => { - let layouts = infos - .iter() - .map(|(node, info)| { - ( - *node, - crate::TensorLayoutInfo::new(Layout::contiguous(info.shape()), info.datatype()), - ) - }) - .collect(); - let layout = op.output_layout(&layouts); - TensorInfo::new(layout.shape().into(), layout.datatype()) - } - }; - Ok(info) -} - -fn propagate_gradient( - snapshots: &FxHashMap, - _node: NodeIndex, - snapshot: &NodeSnapshot, - gradient: ErasedTensor, - gradients: &mut FxHashMap, -) -> Result<()> { - if let Some(backward) = &snapshot.backward { - for (dependency, dependency_gradient) in backward(gradient.into_lazy())? { - accumulate_gradient(gradients, dependency, ErasedTensor::from_lazy(dependency_gradient)); - } - return Ok(()); - } - - match &snapshot.variant { - ComputeGraphNodeVariant::Tensor(_) => Ok(()), - ComputeGraphNodeVariant::ElementWise(_) | ComputeGraphNodeVariant::PairWise(_) => { - Err(crate::Error::msg(format!( - "backpropagation does not support op `{}` without an attached backward rule", - variant_name(&snapshot.variant) - ))) - } - ComputeGraphNodeVariant::MatMul(op) => { - let first_info = snapshots - .get(&op.first) - .ok_or_else(|| crate::Error::msg("matmul lhs info missing"))? - .info - .clone(); - let second_info = snapshots - .get(&op.second) - .ok_or_else(|| crate::Error::msg("matmul rhs info missing"))? - .info - .clone(); - let first = ErasedTensor::reference(gradient.device().clone(), first_info, op.first); - let second = ErasedTensor::reference(gradient.device().clone(), second_info, op.second); - accumulate_gradient(gradients, op.first, gradient.mat_mul(&second.transpose_last_two())); - accumulate_gradient(gradients, op.second, first.transpose_last_two().mat_mul(&gradient)); - Ok(()) - } - ComputeGraphNodeVariant::Reduce(op) => { - if op.function.name() != "sum" { - return Err(crate::Error::msg(format!( - "backpropagation does not support reduce op `{}`", - op.function.name() - ))); - } - - let input_shape = op.shape.clone(); - let mut keepdim_shape = input_shape.to_vec(); - keepdim_shape[op.axis] = 1; - let input_grad = gradient.reshape(&keepdim_shape).broadcast_to(&input_shape); - accumulate_gradient(gradients, op.value, input_grad); - Ok(()) - } - ComputeGraphNodeVariant::MapLayout(op) => { - match &op.kind { - MapLayoutKind::Slice(slices) => { - let input_info = snapshots - .get(&op.input) - .ok_or_else(|| crate::Error::msg("slice input info missing"))? - .info - .clone(); - let zeros = ErasedTensor::zeros( - gradient.device().clone(), - input_info.shape(), - input_info.datatype(), - ); - accumulate_gradient(gradients, op.input, zeros.slice_assign(&gradient, slices)); - } - MapLayoutKind::Permute(axes) => { - let mut inverse = vec![0; axes.len()]; - for (new_axis, old_axis) in axes.iter().copied().enumerate() { - inverse[old_axis] = new_axis; - } - accumulate_gradient(gradients, op.input, gradient.permute(&inverse)); - } - MapLayoutKind::Broadcast => { - let input_info = snapshots - .get(&op.input) - .ok_or_else(|| crate::Error::msg("broadcast input info missing"))? - .info - .clone(); - let reduce_axes = broadcast_reduce_axes(input_info.shape(), gradient.shape())?; - let mut reduced = gradient; - for axis in reduce_axes.into_iter().rev() { - reduced = reduced.sum(axis); - } - accumulate_gradient(gradients, op.input, reduced.reshape(input_info.shape())); - } - MapLayoutKind::Custom => { - return Err(crate::Error::msg(format!( - "backpropagation does not support custom layout op `{}`", - op.name() - ))); - } - } - Ok(()) - } - ComputeGraphNodeVariant::Resize(op) => { - let full_fill = op.fill_shape == op.new_shape; - let same_elements = - op.current_shape.iter().product::() == op.new_shape.iter().product::(); - if !full_fill || !same_elements { - return Err(crate::Error::msg(format!( - "backpropagation only supports reshape-style resize ops, found `{}`", - op.name() - ))); - } - accumulate_gradient(gradients, op.input, gradient.reshape(&op.current_shape)); - Ok(()) - } - ComputeGraphNodeVariant::SliceAssign(op) => { - let value_info = snapshots - .get(&op.value) - .ok_or_else(|| crate::Error::msg("slice_assign value info missing"))? - .info - .clone(); - let zero_value = ErasedTensor::zeros( - gradient.device().clone(), - value_info.shape(), - value_info.datatype(), - ); - accumulate_gradient( - gradients, - op.input, - gradient.clone().slice_assign(&zero_value, &op.slices), - ); - accumulate_gradient(gradients, op.value, gradient.slice(&op.slices)); - Ok(()) - } - ComputeGraphNodeVariant::WhereCond(op) => { - let condition_info = snapshots - .get(&op.condition) - .ok_or_else(|| crate::Error::msg("where condition info missing"))? - .info - .clone(); - let condition = ErasedTensor::reference( - gradient.device().clone(), - condition_info, - op.condition, - ); - let zeros = ErasedTensor::zeros( - gradient.device().clone(), - gradient.shape(), - gradient.datatype(), - ); - accumulate_gradient( - gradients, - op.on_true, - condition.where_cond(&gradient, &zeros), - ); - accumulate_gradient( - gradients, - op.on_false, - condition.where_cond(&zeros, &gradient), - ); - Ok(()) - } - ComputeGraphNodeVariant::Dequantize(_) - | ComputeGraphNodeVariant::QMatMul(_) - | ComputeGraphNodeVariant::IndexSelect(_) - | ComputeGraphNodeVariant::Nary(_) - | ComputeGraphNodeVariant::Custom(_) => Err(crate::Error::msg(format!( - "backpropagation does not support op `{}`", - variant_name(&snapshot.variant) - ))), - } -} - -fn variant_name(variant: &ComputeGraphNodeVariant) -> &'static str { - match variant { - ComputeGraphNodeVariant::ElementWise(_) => "element_wise", - ComputeGraphNodeVariant::PairWise(_) => "pair_wise", - ComputeGraphNodeVariant::Nary(_) => "nary", - ComputeGraphNodeVariant::SliceAssign(_) => "slice_assign", - ComputeGraphNodeVariant::Resize(_) => "resize", - ComputeGraphNodeVariant::MapLayout(_) => "map_layout", - ComputeGraphNodeVariant::Dequantize(_) => "dequantize", - ComputeGraphNodeVariant::MatMul(_) => "mat_mul", - ComputeGraphNodeVariant::QMatMul(_) => "q_mat_mul", - ComputeGraphNodeVariant::Tensor(_) => "tensor", - ComputeGraphNodeVariant::Reduce(_) => "reduce", - ComputeGraphNodeVariant::IndexSelect(_) => "index_select", - ComputeGraphNodeVariant::WhereCond(_) => "where_cond", - ComputeGraphNodeVariant::Custom(_) => "custom", - } -} - -fn accumulate_gradient( - gradients: &mut FxHashMap, - node: NodeIndex, - gradient: ErasedTensor, -) { - if let Some(existing) = gradients.get(&node).cloned() { - let combined = ErasedTensor::from_lazy(existing).add(&gradient); - gradients.insert(node, combined.into_lazy()); - } else { - gradients.insert(node, gradient.into_lazy()); - } -} - -fn broadcast_reduce_axes(input_shape: &[usize], output_shape: &[usize]) -> Result> { - let mut reduce_axes = Vec::new(); - let mut input_iter = input_shape.iter().rev().peekable(); - - for (axis, &target_dim) in output_shape.iter().enumerate().rev() { - let reduce = if let Some(&&source_dim) = input_iter.peek() { - if source_dim == target_dim || (source_dim == 1 && target_dim > 1) { - input_iter.next(); - source_dim == 1 && target_dim > 1 - } else { - target_dim > 1 - } - } else { - target_dim > 1 - }; - - if reduce { - reduce_axes.push(axis); - } - } - - if input_iter.next().is_some() { - return Err(crate::Error::msg(format!( - "failed to match broadcast input shape {input_shape:?} to output shape {output_shape:?}" - ))); - } - - Ok(reduce_axes) -} - -#[derive(Clone)] -struct ErasedTensor { - data: LazyTensorData, -} - -impl ErasedTensor { - fn from_lazy(data: LazyTensorData) -> Self { - Self { data } - } - - fn reference(device: crate::Device, info: TensorInfo, key: NodeIndex) -> Self { - Self { - data: LazyTensorData::reference(device, info, key), - } - } - - fn zeros(device: crate::Device, shape: &[usize], datatype: DataTypeEnum) -> Self { - let data = match datatype { - DataTypeEnum::F32 => TensorData::new_splat(&device, shape, 0.0f32), - DataTypeEnum::F16 => TensorData::new_splat(&device, shape, half::f16::ZERO), - DataTypeEnum::U32 => TensorData::new_splat(&device, shape, 0u32), - }; - Self::from_lazy(LazyTensorData::new(data)) - } - - fn into_lazy(self) -> LazyTensorData { - self.data - } - - fn shape(&self) -> &[usize] { - self.data.info().shape() - } - - fn datatype(&self) -> DataTypeEnum { - self.data.info().datatype() - } - - fn device(&self) -> &crate::Device { - self.data.device() - } - - fn key(&self) -> NodeIndex { - self.data.key() - } - - fn map_layout( - &self, - map_layout_fn: impl Fn(&Layout) -> Layout + Send + Sync + 'static, - ) -> Self { - Self::from_lazy(self.data.map_layout(MapLayoutOperation::new( - self.key(), - map_layout_fn, - ))) - } - - fn reshape(&self, new_shape: &[usize]) -> Self { - Self::from_lazy(self.data.resize(ResizeOperation::new( - self.key(), - self.shape().into(), - new_shape.into(), - new_shape.into(), - ))) - } - - fn broadcast_to(&self, target_shape: &[usize]) -> Self { - let target_shape: Box<[usize]> = target_shape.into(); - self.map_layout(move |layout| layout.broadcast_to(&target_shape)) - } - - fn permute(&self, axes: &[usize]) -> Self { - let axes: Box<[usize]> = axes.into(); - self.map_layout(move |layout| layout.permute(&axes)) - } - - fn slice(&self, slices: &[Range]) -> Self { - let slices: Box<[Range]> = slices.into(); - self.map_layout(move |layout| layout.slice(&slices)) - } - - fn slice_assign(&self, value: &Self, slices: &[Range]) -> Self { - Self::from_lazy(self.data.slice_assign(SliceAssignOperation::new( - self.key(), - value.key(), - slices.into(), - self.shape().into(), - ))) - } - - fn where_cond(&self, on_true: &Self, on_false: &Self) -> Self { - let operation = WhereCondOperation::new( - self.key(), - on_true.key(), - on_false.key(), - self.datatype(), - on_true.datatype(), - on_true.shape(), - ); - Self::from_lazy(on_true.data.where_cond(operation)) - } - - fn mat_mul(&self, other: &Self) -> Self { - Self::from_lazy(self.data.mat_mul(MatMulOperation::new( - self.datatype(), - self.key(), - other.key(), - self.shape(), - other.shape(), - None, - ))) - } - - fn sum(&self, axis: usize) -> Self { - Self::from_lazy(self.data.reduce(ReduceOperation::new( - self.key(), - ReduceFunction { - name: Some("sum".to_string()), - operation: "let output = a + b;".to_string(), - initial_value: "0.0".to_string(), - datatype: self.datatype(), - }, - axis, - self.shape(), - ))) - } - - fn add(&self, other: &Self) -> Self { - Self::from_lazy( - self.data - .pair_wise(PairWiseOperation::new( - PairWiseFunction::new("let output = a + b;", self.datatype()), - self.key(), - other.key(), - self.shape(), - )), - ) - } - - fn transpose_last_two(&self) -> Self { - let rank = self.shape().len(); - let mut axes: Vec = (0..rank).collect(); - axes.swap(rank - 1, rank - 2); - self.permute(&axes) - } -} diff --git a/fusor-ml/core/src/compute_graph/mod.rs b/fusor-ml/core/src/compute_graph/mod.rs index 43eb900c0..d75ce2f48 100644 --- a/fusor-ml/core/src/compute_graph/mod.rs +++ b/fusor-ml/core/src/compute_graph/mod.rs @@ -10,24 +10,26 @@ use wgpu::CommandEncoderDescriptor; mod layout_pass; mod queue; -mod backward; mod resolve; mod visualize; use crate::{ DataTypeEnum, Device, ElementWiseOperation, MatMulOperation, PairWiseOperation, QMatrix, - ReduceOperation, composite::where_cond::WhereCondOperation, - compute_graph::resolve::ResolverResult, dequantize::DequantizeOperation, - index_select::IndexSelectOperation, map_layout::MapLayoutOperation, mir::operation::Operation, - nary_wise::NaryOperation, quantized::matmul::QMatMulOperation, resize::ResizeOperation, + ReduceOperation, + composite::where_cond::WhereCondOperation, + compute_graph::resolve::ResolverResult, + dequantize::DequantizeOperation, + index_select::IndexSelectOperation, + map_layout::MapLayoutOperation, + mir::operation::Operation, + nary_wise::NaryOperation, + quantized::matmul::QMatMulOperation, + resize::ResizeOperation, slice_assign::SliceAssignOperation, - tensor::{LazyTensorData, TensorData}, + tensor::TensorData, visit_tiled::MaybeQData, }; -pub(crate) type BackwardRule = - Arc crate::Result> + Send + Sync>; - #[derive(Clone)] pub(crate) struct ComputeGraph { inner: Arc>, @@ -152,19 +154,6 @@ impl ComputeGraph { self.with_mut(|inner| inner.add_reference(key)); } - pub(crate) fn set_backward_rule(&self, key: NodeIndex, backward: BackwardRule) { - let replaced = { - let mut inner = self.inner.write(); - let replaced = inner.set_backward_rule(key, backward); - #[cfg(feature = "extra_assertions")] - { - inner.verify_integrity() - } - replaced - }; - drop(replaced); - } - pub(crate) fn remove_reference(&self, key: NodeIndex) { let removed = { let mut inner = self.inner.write(); @@ -189,7 +178,6 @@ pub(crate) struct ComputeGraphNode { variant: ComputeGraphNodeVariant, reference_count: u32, cached: Option, - backward: Option, } #[derive(Clone, Debug)] @@ -281,7 +269,6 @@ impl ComputeGraphInner { variant: node, reference_count: 1, cached: None, - backward: None, }); self.add_dependency_edges(node); node @@ -293,13 +280,6 @@ impl ComputeGraphInner { node.reference_count += 1; } - fn set_backward_rule(&mut self, key: NodeIndex, backward: BackwardRule) -> Option { - self.nodes - .nodes - .node_weight_mut(key) - .and_then(|node| node.backward.replace(backward)) - } - fn add_dependency_edges(&mut self, key: NodeIndex) { let mut dependencies = Vec::new(); self.visit_dependencies(key, &mut |dep| { @@ -382,7 +362,7 @@ impl ComputeGraphInner { return false; }; - if node.reference_count > 0 || node.cached.is_none() { + if node.reference_count > 0 { return true; } diff --git a/fusor-ml/core/src/element_wise.rs b/fusor-ml/core/src/element_wise.rs index d3da68cf0..a17afb5fa 100644 --- a/fusor-ml/core/src/element_wise.rs +++ b/fusor-ml/core/src/element_wise.rs @@ -5,7 +5,6 @@ use std::{ }; use crate::{ - BackwardTarget, Tensor, compute_graph::NodeIndex, mir::{function::Function, kernel::GenericKernel}, @@ -146,16 +145,14 @@ impl ElementWiseFunction { fn elementwise_with_backward( input: &Tensor, function: ElementWiseFunction, - backward: impl Fn(Tensor, &Tensor) -> Tensor + Send + Sync + 'static, + _backward: impl Fn(Tensor, &Tensor) -> Tensor + Send + Sync + 'static, ) -> Tensor { - let output = input.element_wise(ElementWiseOperation::new( + input.element_wise(ElementWiseOperation::new( input.datatype(), input.key(), function, input.shape().as_slice(), - )); - let input = input.clone(); - output.with_backwards(move |grad| Ok(vec![BackwardTarget::wrt(&input, backward(grad, &input))])) + )) } fn greater_than_const_mask( diff --git a/fusor-ml/core/src/gather.rs b/fusor-ml/core/src/gather.rs new file mode 100644 index 000000000..06ef12120 --- /dev/null +++ b/fusor-ml/core/src/gather.rs @@ -0,0 +1,142 @@ +use std::{fmt::Write, sync::Arc}; + +use crate::{ + DataTypeEnum, FloatDataType, Layout, LazyTensorData, Tensor, TensorData, + TensorInfo, + compute_graph::{ComputeGraphInner, NodeIndex}, + mir::{ + inputs::MirValue, + kernel::GenericKernel, + operation::Operation, + workgroup_shape::{Constraint, WorkgroupShape, WorkgroupShapeConstraints}, + }, + visit_tiled::distribute_workgroups, +}; + +const BLOCKSIZE: u32 = 256; + +#[derive(Debug, Clone)] +struct GatherLastOperation { + values: NodeIndex, + indexes: NodeIndex, + rows: usize, + width: usize, + datatype: DataTypeEnum, +} + +impl Operation for GatherLastOperation { + fn workgroup_shape_constraints(&self, _: &crate::Device) -> WorkgroupShapeConstraints { + let mut constraints = WorkgroupShapeConstraints::new(); + constraints.add_constraint(0, Constraint::equals(BLOCKSIZE)); + constraints.add_constraint(1, Constraint::equals(1)); + constraints.add_constraint(2, Constraint::equals(1)); + constraints + } + + fn dispatch_size(&self, _: &WorkgroupShape, _: &[MirValue]) -> [u32; 3] { + distribute_workgroups((self.rows as u32).div_ceil(BLOCKSIZE)) + } + + fn visit_dependencies(&self, f: &mut dyn FnMut(NodeIndex)) { + f(self.values); + f(self.indexes); + } + + fn inputs(&self, nodes: &ComputeGraphInner) -> Vec { + let values = nodes.get_cached_result(self.values).unwrap().clone(); + let indexes = nodes.get_cached_result(self.indexes).unwrap().clone(); + let output = TensorData::new_for_shape(&nodes.device(), &[self.rows], self.datatype); + vec![values.into(), indexes.into(), output.into()] + } + + fn output(&self, _: &ComputeGraphInner, inputs: &[MirValue]) -> MirValue { + inputs[2].clone() + } + + fn build_kernel( + &self, + _: &ComputeGraphInner, + workgroup_shape: &WorkgroupShape, + _: &[MirValue], + kernel: &mut GenericKernel, + ) { + let values = kernel.add_tensor_input(2, false, self.datatype); + let indexes = kernel.add_tensor_input(1, false, DataTypeEnum::U32); + let output = kernel.add_tensor_input(1, true, self.datatype); + let workgroup_local_index = kernel.workgroup_local_index(); + let linearized_workgroup = workgroup_shape.linearized_workgroup_index(kernel); + + writeln!( + kernel, + "let row = ({linearized_workgroup}) * {BLOCKSIZE}u + {workgroup_local_index};" + ) + .unwrap(); + writeln!(kernel, "if row < {} {{", output.shape_binding(0)).unwrap(); + write!(kernel, "let index_offset = ").unwrap(); + indexes.strided_index(kernel, ["row"]); + writeln!(kernel, ";").unwrap(); + writeln!(kernel, "let column = {indexes}[index_offset];").unwrap(); + write!(kernel, "let value_offset = ").unwrap(); + values.strided_index(kernel, ["row", "column"]); + writeln!(kernel, ";").unwrap(); + write!(kernel, "let output_offset = ").unwrap(); + output.strided_index(kernel, ["row"]); + writeln!(kernel, ";").unwrap(); + writeln!(kernel, "{output}[output_offset] = {values}[value_offset];").unwrap(); + writeln!(kernel, "}}").unwrap(); + } + + fn name(&self) -> String { + format!("gather_last_{}x{}", self.rows, self.width) + } + + fn output_layout( + &self, + _: &rustc_hash::FxHashMap, + ) -> crate::TensorLayoutInfo { + crate::TensorLayoutInfo::new(Layout::contiguous(&[self.rows]), self.datatype) + } +} + +impl Tensor<2, D> { + pub fn gather_last(&self, indexes: &Tensor<1, u32>) -> Tensor<1, D> { + assert_eq!( + self.shape()[0], + indexes.shape()[0], + "gather_last expects one index per row" + ); + + let rows = self.shape()[0]; + let width = self.shape()[1]; + let operation = GatherLastOperation { + values: self.key(), + indexes: indexes.key(), + rows, + width, + datatype: self.datatype(), + }; + let device = self.device().clone(); + let key = device.compute_graph().create_custom(Arc::new(operation)); + Tensor::from_parts(LazyTensorData::from_parts( + device, + TensorInfo::new(vec![rows].into_boxed_slice(), self.datatype()), + key, + )) + } +} + +#[cfg(test)] +#[tokio::test] +async fn test_gather_last() { + use crate::Device; + + let device = Device::test_instance(); + let values = Tensor::new(&device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let indexes = Tensor::new(&device, &[2u32, 0]); + let gathered = values.gather_last(&indexes); + + let output = gathered.as_slice().await.unwrap(); + assert_eq!(output[[0]], 3.0); + assert_eq!(output[[1]], 4.0); +} + diff --git a/fusor-ml/core/src/lib.rs b/fusor-ml/core/src/lib.rs index 9d71b95b4..7ec5b24a9 100644 --- a/fusor-ml/core/src/lib.rs +++ b/fusor-ml/core/src/lib.rs @@ -11,7 +11,6 @@ pub use rank::*; pub use reduce::*; pub use tensor::MappedBuffer; pub use tensor::*; -pub use autograd::{BackwardTarget, Gradients}; // Re-export wasm-compatible Send/Sync traits pub use wgpu::{WasmNotSend, WasmNotSendSync, WasmNotSync}; @@ -22,7 +21,6 @@ pub(crate) use pair_wise::*; pub use resize::ShapeWithOneHole; pub use varbuilder::{ShardedVarBuilder, VarBuilder}; -mod autograd; pub mod cache; mod composite; pub use composite::{ToVec1, ToVec2, ToVec3}; diff --git a/fusor-ml/core/src/matmul/mod.rs b/fusor-ml/core/src/matmul/mod.rs index ba50500d5..638d619f2 100644 --- a/fusor-ml/core/src/matmul/mod.rs +++ b/fusor-ml/core/src/matmul/mod.rs @@ -2,7 +2,7 @@ use crate::matmul::sgemm_params::gemm_parameters; use crate::matmul::sgemv_params::gemv_parameters; use crate::mir::operation::Operation; use crate::{ - BackwardTarget, Device, ElementWiseFunctions, Tensor, + Device, ElementWiseFunctions, Tensor, compute_graph::NodeIndex, mir::kernel::GenericKernel, tensor::{DataType, DataTypeEnum, TensorData}, @@ -273,15 +273,7 @@ impl Operation for MatMulOperation { impl Tensor { pub fn mat_mul(&self, other: &Self) -> Self { - let output = self.add_mat_mul(other, None); - let lhs = self.clone(); - let rhs = other.clone(); - output.with_backwards(move |grad| { - Ok(vec![ - BackwardTarget::wrt(&lhs, grad.clone().mat_mul(&rhs.t())), - BackwardTarget::wrt(&rhs, lhs.t().mat_mul(&grad)), - ]) - }) + self.add_mat_mul(other, None) } pub fn mat_mul_with_parameters(&self, other: &Self, parameters: MatMulParams) -> Self { diff --git a/fusor-ml/core/src/matmul/sgemm.rs b/fusor-ml/core/src/matmul/sgemm.rs index 3555287c2..2ae7f6eb3 100644 --- a/fusor-ml/core/src/matmul/sgemm.rs +++ b/fusor-ml/core/src/matmul/sgemm.rs @@ -202,11 +202,11 @@ pub(super) fn build_kernel( ) .unwrap(); - // Allocate thread-local cache for results + // Allocate thread-local cache for results as vec4 accumulators when thread_n is 2-4 + let thread_n_vec_type = maybe_vec_storage_type(thread_n_size, datatype); writeln!( kernel, - "var threadResults: array<{datatype}, {}>;", - thread_m_size * thread_n_size + "var threadResults: array<{thread_n_vec_type}, {thread_m_size}u>;" ) .unwrap(); @@ -415,6 +415,8 @@ pub(super) fn build_kernel( } // Calculate per-thread results for current tile with overlapped prefetch + let stride_a = block_k_size + PADDING; + let stride_b = block_k_size + PADDING; writeln!( kernel, " for (var dotIdx = 0u; dotIdx < {block_k_size}u; dotIdx++) {{" @@ -423,7 +425,6 @@ pub(super) fn build_kernel( writeln!(kernel, " let reg_m_offset = {a_offset}threadRow * {thread_m_size}u * ({block_k_size}u + {PADDING}) + dotIdx;").unwrap(); // Vectorized loads with padding for bank conflict avoidance - let stride_a = block_k_size + PADDING; write!(kernel, " regM = {thread_m_dtype}(").unwrap(); for i in 0..thread_m_size { if i > 0 { @@ -440,7 +441,6 @@ pub(super) fn build_kernel( .unwrap(); // Vectorized load for N register with padding - let stride_b = block_k_size + PADDING; write!(kernel, " regN = {thread_n_dtype}(").unwrap(); for i in 0..thread_n_size { if i > 0 { @@ -454,22 +454,9 @@ pub(super) fn build_kernel( let indexed_reg_m = maybe_vec_storage_index(thread_m_size, "regM", res_idx_m); writeln!( kernel, - " let result_{res_idx_m} = {indexed_reg_m} * regN;" + " threadResults[{res_idx_m}] += {thread_n_dtype}({indexed_reg_m}) * regN;" ) .unwrap(); - for res_idx_n in 0..thread_n_size { - let indexed_result = maybe_vec_storage_index( - thread_m_size, - format_args!("result_{res_idx_m}"), - res_idx_n, - ); - writeln!( - kernel, - " threadResults[{} * {thread_n_size}u + {}] += {indexed_result};", - res_idx_m, res_idx_n - ) - .unwrap(); - } } writeln!(kernel, " }}").unwrap(); @@ -687,7 +674,7 @@ pub(super) fn build_kernel( ) .unwrap(); let result = post_element_wise_functions.iter().fold( - format!("threadResults[(outRow{res_idx_m} - outRowOffset) * {thread_n_size}u + (outCol{res_idx_n} - outColOffset)]"), + format!("threadResults[outRow{res_idx_m} - outRowOffset][outCol{res_idx_n} - outColOffset]"), |acc, f| f.call(vec![acc]), ); writeln!(kernel, "{result};").unwrap(); diff --git a/fusor-ml/core/src/matmul/sgemm_params.rs b/fusor-ml/core/src/matmul/sgemm_params.rs index 3d7654ca6..25a00208d 100644 --- a/fusor-ml/core/src/matmul/sgemm_params.rs +++ b/fusor-ml/core/src/matmul/sgemm_params.rs @@ -214,6 +214,6 @@ pub fn gemm_parameters(m: usize, n: usize, k: usize) -> SgemmParams { } else if sum_dim <= 4608f32 { SgemmParams::new(false, 32u32, 32u32, 16u32, 4u32, 4u32) } else { - SgemmParams::new(false, 64u32, 64u32, 16u32, 4u32, 4u32) + SgemmParams::new(false, 48u32, 32u32, 8u32, 6u32, 4u32) } } diff --git a/fusor-ml/core/src/mir/kernel.rs b/fusor-ml/core/src/mir/kernel.rs index 63fa434b2..4b5b13b43 100644 --- a/fusor-ml/core/src/mir/kernel.rs +++ b/fusor-ml/core/src/mir/kernel.rs @@ -390,6 +390,15 @@ impl GenericKernel { let module = self.kernel.get_or_init(|| { let mut kernel = String::new(); self.kernel(&mut kernel, device).unwrap(); + if let Ok(path) = std::env::var("FUSOR_DUMP_WGSL") { + let dump_path = format!( + "{}.{}.wgsl", + path, + self.name.replace("/", "_").replace(" ", "_") + ); + let _ = std::fs::write(&dump_path, &kernel); + eprintln!("Dumped WGSL to {dump_path} (name={})", self.name); + } device .shader_module_cache() .write() diff --git a/fusor-ml/core/src/pair_wise.rs b/fusor-ml/core/src/pair_wise.rs index d87460be1..2a5dd6077 100644 --- a/fusor-ml/core/src/pair_wise.rs +++ b/fusor-ml/core/src/pair_wise.rs @@ -4,7 +4,7 @@ use std::{ }; use crate::{ - BackwardTarget, ElementWiseFunction, MaxRank, Tensor, + ElementWiseFunction, MaxRank, Tensor, compute_graph::NodeIndex, tensor::{DataType, DataTypeEnum}, }; @@ -87,19 +87,12 @@ impl PairWiseFunction { } } -fn pairwise_with_backward( +fn pairwise_op( lhs: &Tensor, rhs: &Tensor, function: PairWiseFunction, - backward: impl Fn(Tensor, &Tensor, &Tensor) -> Vec - + Send - + Sync - + 'static, ) -> Tensor { - let output = lhs.pair_wise(rhs, function); - let lhs = lhs.clone(); - let rhs = rhs.clone(); - output.with_backwards(move |grad| Ok(backward(grad, &lhs, &rhs))) + lhs.pair_wise(rhs, function) } /// Macro to implement pairwise operators (Add, Sub, Mul, Div) for Tensor. @@ -112,7 +105,7 @@ fn pairwise_with_backward( /// /// Also generates a broadcast method `op_()` for tensors of different ranks. macro_rules! impl_pairwise_op { - ($trait:ident, $method:ident, $op_str:literal, $op_name:literal, $broadcast_method:ident, {$op:tt}, $backward:expr) => { + ($trait:ident, $method:ident, $op_str:literal, $op_name:literal, $broadcast_method:ident, {$op:tt}) => { // Owned + Owned: delegates to ref + ref impl $trait> for Tensor { type Output = Tensor; @@ -127,7 +120,7 @@ macro_rules! impl_pairwise_op { type Output = Tensor; fn $method(self, rhs: &Tensor) -> Self::Output { - pairwise_with_backward( + pairwise_op( self, rhs, PairWiseFunction::new( @@ -135,7 +128,6 @@ macro_rules! impl_pairwise_op { T::WGSL_TYPE, ) .with_name($op_name), - $backward, ) } } @@ -179,11 +171,7 @@ impl_pairwise_op!( "+", "add", add_, - {+}, - |grad, lhs, rhs| vec![ - BackwardTarget::wrt(lhs, grad.clone()), - BackwardTarget::wrt(rhs, grad), - ] + {+} ); #[cfg(test)] @@ -346,11 +334,7 @@ impl_pairwise_op!( "-", "sub", sub_, - {-}, - |grad, lhs, rhs| vec![ - BackwardTarget::wrt(lhs, grad.clone()), - BackwardTarget::wrt(rhs, -grad), - ] + {-} ); #[cfg(test)] @@ -383,11 +367,7 @@ impl_pairwise_op!( "*", "mul", mul_, - {*}, - |grad, lhs, rhs| vec![ - BackwardTarget::wrt(lhs, grad.clone() * rhs), - BackwardTarget::wrt(rhs, grad * lhs), - ] + {*} ); #[cfg(test)] @@ -420,11 +400,7 @@ impl_pairwise_op!( "/", "div", div_, - {/}, - |grad, lhs, rhs| vec![ - BackwardTarget::wrt(lhs, grad.clone() / rhs), - BackwardTarget::wrt(rhs, -((grad * lhs) / &(rhs * rhs))), - ] + {/} ); #[cfg(test)] @@ -456,15 +432,14 @@ async fn test_pair_wise_div() { /// Unlike `impl_pairwise_op!` which implements std::ops traits, this macro generates /// regular methods on Tensor for operations that don't have corresponding operators. macro_rules! impl_pairwise_method { - ($method:ident, $wgsl_op:literal, $op_name:literal, $broadcast_method:ident, |$a:ident, $b:ident| $expr:expr, $backward:expr) => { + ($method:ident, $wgsl_op:literal, $op_name:literal, $broadcast_method:ident, |$a:ident, $b:ident| $expr:expr) => { impl Tensor { pub fn $method(&self, other: &Self) -> Self { - pairwise_with_backward( + pairwise_op( self, other, PairWiseFunction::new(concat!("let output = ", $wgsl_op, ";"), T::WGSL_TYPE) .with_name($op_name), - $backward, ) } @@ -481,17 +456,7 @@ macro_rules! impl_pairwise_method { }; } -impl_pairwise_method!( - pow, - "pow(a, b)", - "pow", - pow_, - |a, b| a.pow(&b), - |grad, lhs, rhs| vec![ - BackwardTarget::wrt(lhs, (grad.clone() * rhs) * &lhs.pow(&(rhs.clone() - T::one()))), - BackwardTarget::wrt(rhs, (grad * &lhs.pow(rhs)) * &lhs.log()), - ] -); +impl_pairwise_method!(pow, "pow(a, b)", "pow", pow_, |a, b| a.pow(&b)); #[cfg(test)] #[tokio::test] diff --git a/fusor-ml/core/src/resize.rs b/fusor-ml/core/src/resize.rs index 75c0292c6..f09bc2fad 100644 --- a/fusor-ml/core/src/resize.rs +++ b/fusor-ml/core/src/resize.rs @@ -1,7 +1,7 @@ use std::fmt::Write; use crate::{ - BackwardTarget, DataTypeEnum, Layout, SmallerRank, TILE_SIZE, Tensor, TensorData, + DataTypeEnum, Layout, SmallerRank, TILE_SIZE, Tensor, TensorData, compute_graph::NodeIndex, map_layout::MapLayoutOperation, mir::{ @@ -251,14 +251,12 @@ impl Tensor { ); let new_shape: Box<[usize]> = new_shape.into(); let input = self.key(); - let output = self.add_resize(ResizeOperation::new( + self.add_resize(ResizeOperation::new( input, (*self.shape()).into(), new_shape.clone(), new_shape.clone(), - )); - let input = self.clone(); - output.with_backwards(move |grad| Ok(vec![BackwardTarget::wrt(&input, grad.reshape(*input.shape()))])) + )) } pub fn flatten_last_n(&self) -> Tensor diff --git a/fusor-ml/core/src/slice_assign.rs b/fusor-ml/core/src/slice_assign.rs index 1ef368c17..e9b49417e 100644 --- a/fusor-ml/core/src/slice_assign.rs +++ b/fusor-ml/core/src/slice_assign.rs @@ -1,7 +1,7 @@ use std::{fmt::Write, ops::Range}; use crate::{ - BackwardTarget, TILE_SIZE, Tensor, TensorData, + TILE_SIZE, Tensor, TensorData, compute_graph::{ComputeGraphInner, NodeIndex}, mir::{ inputs::MirValue, @@ -184,18 +184,7 @@ impl Operation for SliceAssignOperation { impl Tensor { pub fn slice_assign(&self, slices: [Range; R], value: &Self) -> Self { - let output = self.add_slice_assign(value, slices.clone()); - let input = self.clone(); - let value = value.clone(); - output.with_backwards(move |grad| { - Ok(vec![ - BackwardTarget::wrt( - &input, - grad.slice_assign(slices.clone(), &Tensor::zeros(grad.device(), *value.shape())), - ), - BackwardTarget::wrt(&value, grad.slice(slices.clone())), - ]) - }) + self.add_slice_assign(value, slices) } } diff --git a/fusor-ml/core/src/tensor.rs b/fusor-ml/core/src/tensor.rs index bf72db232..37937795d 100644 --- a/fusor-ml/core/src/tensor.rs +++ b/fusor-ml/core/src/tensor.rs @@ -568,6 +568,18 @@ impl Clone for Tensor { } } +impl Tensor { + /// Resolve the current tensor value on device and return a fresh leaf tensor + /// that no longer carries the original compute graph history. + pub fn detach(&self) -> Self { + let (data, _) = self.data.materialize(); + Self { + data: LazyTensorData::new(data), + datatype: PhantomData, + } + } +} + impl fusor_types::FromArray<0, D, (), Device> for Tensor<0, D> { fn from_array(_data: (), device: &Device) -> Self { let iter = std::iter::empty(); @@ -893,13 +905,6 @@ impl Tensor { } } - /// Resolve the current tensor value on device and return a fresh leaf tensor - /// that no longer carries the original compute graph history. - pub fn detach(&self) -> Self { - let (data, _) = self.data.materialize(); - Self::from_parts(LazyTensorData::new(data)) - } - /// How many kernel calls are needed to fully resolve this tensor pub fn count_kernels_to_resolve(&self) -> usize { let (_, count) = self.data.materialize(); diff --git a/fusor-ml/cpu/src/quantized.rs b/fusor-ml/cpu/src/quantized.rs index ec6bb985b..da21013e3 100644 --- a/fusor-ml/cpu/src/quantized.rs +++ b/fusor-ml/cpu/src/quantized.rs @@ -384,7 +384,7 @@ where type Output = (); #[inline(always)] - fn with_simd(self, _simd: S) -> Self::Output { + fn with_simd(self, simd: S) -> Self::Output { let Self { lhs_data, rhs_blocks, @@ -396,16 +396,16 @@ where .. } = self; + // Use f32 dequantize path: dequantize weight blocks to f32 and use SIMD mul_add. + // This avoids quantizing activations (which introduces compounding error across layers). + // Special fast path for m=1 (common inference case): parallelize over output columns if m == 1 { - // m=1 (token generation): memory-bandwidth bound, parallelize over output columns. - // Scale thread count based on work size: each thread needs enough vec_dot calls - // to amortize std::thread::scope's thread creation overhead (~10µs per thread). - let max_threads = crate::parallel::num_threads(); - let total_work = n * blocks_per_weight_row; - let n_threads = (total_work / 16384).min(max_threads).max(1); + let n_threads = crate::parallel::num_threads(); - if n_threads == 1 { - process_row_integer_tiled::( + // For small n or single-threaded, don't parallelize + if n < 64 || n_threads == 1 { + process_row_simd_tiled::( + simd, lhs_data, rhs_blocks, out_data, @@ -413,8 +413,7 @@ where blocks_per_weight_row, ); } else { - // Same CHUNK_SIZE=32 aligned thread distribution as before, - // but each thread quantizes activations once instead of per chunk. + // Parallelize over output column chunks using scoped threads const CHUNK_SIZE: usize = 32; let total_chunks = n.div_ceil(CHUNK_SIZE); let chunks_per_thread = total_chunks.div_ceil(n_threads); @@ -441,67 +440,54 @@ where start_n += this_size; scope.spawn(move || { - // Quantize activations ONCE per thread (not per chunk). - let act_blocks: Vec = (0..blocks_per_weight_row) - .map(|block_idx| { - let start = block_idx * B::BLOCK_SIZE; - B::quantize_activation(&lhs_data[start..start + B::BLOCK_SIZE]) - }) - .collect(); - + // Process each CHUNK_SIZE piece within this thread for (i, out_chunk) in thread_chunk.chunks_mut(CHUNK_SIZE).enumerate() { let chunk_start = thread_start_n + i * CHUNK_SIZE; - let chunk_n = out_chunk.len(); - for (idx, out_elem) in - out_chunk.iter_mut().enumerate().take(chunk_n) - { - let n_out = chunk_start + idx; - let mut sum = 0.0f32; - for (block_idx, act_block) in act_blocks.iter().enumerate() { - sum += rhs_blocks - [n_out * blocks_per_weight_row + block_idx] - .vec_dot(act_block); - } - *out_elem = sum; - } + process_row_simd_range::( + simd, + lhs_data, + rhs_blocks, + out_chunk, + chunk_start, + out_chunk.len(), + blocks_per_weight_row, + ); } }); } }); } - } else { - // Multi-row path (m≥2): use outer-loop unrolled 3×4 tiling. - // Weight blocks are loaded once and reused across 3 LHS rows, - // reducing memory traffic by ~3× compared to row-at-a-time processing. + } else if m >= 4 { let n_threads = crate::parallel::num_threads(); - // Use at most m/3 threads so each thread gets ≥3 rows, - // maximizing benefit of the 3-row kernel. - let effective_threads = (m / 3).min(n_threads).max(1); - - if effective_threads <= 1 { - process_multi_row_tiled::( - lhs_data, - rhs_blocks, - out_data, - m, - k, - n, - blocks_per_weight_row, - ); + if n_threads == 1 { + // Sequential processing + for i in 0..m { + let lhs_row = &lhs_data[i * k..(i + 1) * k]; + let out_row = &mut out_data[i * n..(i + 1) * n]; + process_row_simd_tiled::( + simd, + lhs_row, + rhs_blocks, + out_row, + n, + blocks_per_weight_row, + ); + } } else { - let rows_per_thread = m.div_ceil(effective_threads); + // Process rows in parallel using scoped threads + let rows_per_thread = m.div_ceil(n_threads); std::thread::scope(|scope| { let mut remaining_out = out_data; let mut row_offset = 0; - for thread_id in 0..effective_threads { + for thread_id in 0..n_threads { if remaining_out.is_empty() { break; } - let this_rows = if thread_id == effective_threads - 1 { + let this_rows = if thread_id == n_threads - 1 { m - row_offset } else { rows_per_thread.min(m - row_offset) @@ -514,27 +500,77 @@ where row_offset += this_rows; scope.spawn(move || { - process_multi_row_tiled::( - &lhs_data - [thread_row_offset * k..(thread_row_offset + this_rows) * k], - rhs_blocks, - thread_out, - this_rows, - k, - n, - blocks_per_weight_row, - ); + for i in 0..this_rows { + let global_row = thread_row_offset + i; + let lhs_row = &lhs_data[global_row * k..(global_row + 1) * k]; + let out_row = &mut thread_out[i * n..(i + 1) * n]; + process_row_simd_tiled::( + simd, + lhs_row, + rhs_blocks, + out_row, + n, + blocks_per_weight_row, + ); + } }); } }); } + } else { + // Sequential processing for small matrices (m=2,3) + for i in 0..m { + let lhs_row = &lhs_data[i * k..(i + 1) * k]; + let out_row = &mut out_data[i * n..(i + 1) * n]; + process_row_simd_tiled::( + simd, + lhs_row, + rhs_blocks, + out_row, + n, + blocks_per_weight_row, + ); + } } } } -/// Process a range of output columns for m=1 parallelization +/// Process a range of output columns for m=1 parallelization using integer dot products. +/// Uses NEON intrinsics on aarch64 for efficient i8 x i8 -> i32 computation. #[allow(dead_code)] #[inline(always)] +fn process_row_integer_range( + lhs_row: &[f32], + rhs_blocks: &[B], + out_chunk: &mut [f32], + start_n: usize, + chunk_n: usize, + blocks_per_weight_row: usize, +) where + B::ActivationBlock: Pod, +{ + // Quantize activations once for all output columns + let act_blocks: Vec = (0..blocks_per_weight_row) + .map(|block_idx| { + let start = block_idx * B::BLOCK_SIZE; + let chunk = &lhs_row[start..start + B::BLOCK_SIZE]; + B::quantize_activation(chunk) + }) + .collect(); + + for (i, out_elem) in out_chunk.iter_mut().enumerate().take(chunk_n) { + let n_out = start_n + i; + let mut sum = 0.0f32; + for (block_idx, act_block) in act_blocks.iter().enumerate() { + let weight_block_idx = n_out * blocks_per_weight_row + block_idx; + sum += rhs_blocks[weight_block_idx].vec_dot(act_block); + } + *out_elem = sum; + } +} + +/// Process a range of output columns for m=1 parallelization +#[inline(always)] fn process_row_simd_range( simd: S, lhs_row: &[f32], @@ -555,6 +591,7 @@ fn process_row_simd_range( /// Process a single output row using integer dot products with 4-way tiling. /// Uses NEON intrinsics on aarch64 for efficient i8 x i8 -> i32 computation. +#[allow(dead_code)] #[inline(always)] fn process_row_integer_tiled( lhs_row: &[f32], @@ -605,289 +642,7 @@ fn process_row_integer_tiled( } } -/// Process a range of output columns using pre-quantized activation blocks. -/// Uses 4-way column tiling for instruction-level parallelism. -#[allow(dead_code)] -#[inline(always)] -fn process_range_with_acts( - act_blocks: &[B::ActivationBlock], - rhs_blocks: &[B], - out_chunk: &mut [f32], - start_n: usize, - blocks_per_weight_row: usize, -) where - B::ActivationBlock: Pod, -{ - let chunk_n = out_chunk.len(); - const NR: usize = 4; - let n_tiles = chunk_n / NR; - - for tile in 0..n_tiles { - let local_off = tile * NR; - let col = start_n + local_off; - let mut acc = [0.0f32; NR]; - - for (block_idx, act) in act_blocks.iter().enumerate() { - acc[0] += rhs_blocks[col * blocks_per_weight_row + block_idx].vec_dot(act); - acc[1] += rhs_blocks[(col + 1) * blocks_per_weight_row + block_idx].vec_dot(act); - acc[2] += rhs_blocks[(col + 2) * blocks_per_weight_row + block_idx].vec_dot(act); - acc[3] += rhs_blocks[(col + 3) * blocks_per_weight_row + block_idx].vec_dot(act); - } - - out_chunk[local_off..local_off + NR].copy_from_slice(&acc); - } - - // Handle remainder - for (i, block) in out_chunk - .iter_mut() - .enumerate() - .take(chunk_n) - .skip(n_tiles * NR) - { - let n_out = start_n + i; - let mut sum = 0.0f32; - for (block_idx, act) in act_blocks.iter().enumerate() { - sum += rhs_blocks[n_out * blocks_per_weight_row + block_idx].vec_dot(act); - } - *block = sum; - } -} - -/// Process 3 LHS rows × all N output columns using 3×4 outer-loop unrolled tiling. -/// -/// This is the key optimization from llamafile's matmul approach: by processing -/// 3 rows simultaneously, each weight block is loaded once from memory and reused -/// across all 3 rows. This reduces memory traffic for weights by ~3× compared to -/// processing one row at a time. -/// -/// Layout: lhs_data contains 3 contiguous rows of k elements each. -/// out_data contains 3 contiguous rows of n elements each. -#[inline(always)] -fn process_3rows_integer_tiled( - lhs_data: &[f32], - rhs_blocks: &[B], - out_data: &mut [f32], - k: usize, - n: usize, - blocks_per_weight_row: usize, -) where - B::ActivationBlock: Pod, -{ - // Pre-quantize all 3 rows - let mut act0: Vec = Vec::with_capacity(blocks_per_weight_row); - let mut act1: Vec = Vec::with_capacity(blocks_per_weight_row); - let mut act2: Vec = Vec::with_capacity(blocks_per_weight_row); - - for block_idx in 0..blocks_per_weight_row { - let s = block_idx * B::BLOCK_SIZE; - act0.push(B::quantize_activation(&lhs_data[s..s + B::BLOCK_SIZE])); - act1.push(B::quantize_activation( - &lhs_data[k + s..k + s + B::BLOCK_SIZE], - )); - act2.push(B::quantize_activation( - &lhs_data[2 * k + s..2 * k + s + B::BLOCK_SIZE], - )); - } - - // 3×4 tiled inner loop - const NR: usize = 4; - let n_tiles = n / NR; - - for tile in 0..n_tiles { - let col = tile * NR; - - // 12 explicit accumulators (3 rows × 4 cols) to help register allocation - let (mut a00, mut a01, mut a02, mut a03) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); - let (mut a10, mut a11, mut a12, mut a13) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); - let (mut a20, mut a21, mut a22, mut a23) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); - - for block_idx in 0..blocks_per_weight_row { - // Load 4 weight blocks ONCE (shared across all 3 rows) - let w0 = &rhs_blocks[col * blocks_per_weight_row + block_idx]; - let w1 = &rhs_blocks[(col + 1) * blocks_per_weight_row + block_idx]; - let w2 = &rhs_blocks[(col + 2) * blocks_per_weight_row + block_idx]; - let w3 = &rhs_blocks[(col + 3) * blocks_per_weight_row + block_idx]; - - // Row 0: 4 dot products - let a = &act0[block_idx]; - a00 += w0.vec_dot(a); - a01 += w1.vec_dot(a); - a02 += w2.vec_dot(a); - a03 += w3.vec_dot(a); - - // Row 1: reusing same weight blocks from cache - let a = &act1[block_idx]; - a10 += w0.vec_dot(a); - a11 += w1.vec_dot(a); - a12 += w2.vec_dot(a); - a13 += w3.vec_dot(a); - - // Row 2: reusing same weight blocks from cache - let a = &act2[block_idx]; - a20 += w0.vec_dot(a); - a21 += w1.vec_dot(a); - a22 += w2.vec_dot(a); - a23 += w3.vec_dot(a); - } - - // Store results for all 3 rows - out_data[col] = a00; - out_data[col + 1] = a01; - out_data[col + 2] = a02; - out_data[col + 3] = a03; - out_data[n + col] = a10; - out_data[n + col + 1] = a11; - out_data[n + col + 2] = a12; - out_data[n + col + 3] = a13; - out_data[2 * n + col] = a20; - out_data[2 * n + col + 1] = a21; - out_data[2 * n + col + 2] = a22; - out_data[2 * n + col + 3] = a23; - } - - // Handle remainder columns - for j in (n_tiles * NR)..n { - let (mut s0, mut s1, mut s2) = (0.0f32, 0.0f32, 0.0f32); - for block_idx in 0..blocks_per_weight_row { - let w = &rhs_blocks[j * blocks_per_weight_row + block_idx]; - s0 += w.vec_dot(&act0[block_idx]); - s1 += w.vec_dot(&act1[block_idx]); - s2 += w.vec_dot(&act2[block_idx]); - } - out_data[j] = s0; - out_data[n + j] = s1; - out_data[2 * n + j] = s2; - } -} - -/// Process 2 LHS rows × all N output columns using 2×4 outer-loop unrolled tiling. -/// Same approach as the 3-row version but for the remainder when m % 3 == 2. -#[inline(always)] -fn process_2rows_integer_tiled( - lhs_data: &[f32], - rhs_blocks: &[B], - out_data: &mut [f32], - k: usize, - n: usize, - blocks_per_weight_row: usize, -) where - B::ActivationBlock: Pod, -{ - let mut act0: Vec = Vec::with_capacity(blocks_per_weight_row); - let mut act1: Vec = Vec::with_capacity(blocks_per_weight_row); - - for block_idx in 0..blocks_per_weight_row { - let s = block_idx * B::BLOCK_SIZE; - act0.push(B::quantize_activation(&lhs_data[s..s + B::BLOCK_SIZE])); - act1.push(B::quantize_activation( - &lhs_data[k + s..k + s + B::BLOCK_SIZE], - )); - } - - const NR: usize = 4; - let n_tiles = n / NR; - - for tile in 0..n_tiles { - let col = tile * NR; - - let (mut a00, mut a01, mut a02, mut a03) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); - let (mut a10, mut a11, mut a12, mut a13) = (0.0f32, 0.0f32, 0.0f32, 0.0f32); - - for block_idx in 0..blocks_per_weight_row { - let w0 = &rhs_blocks[col * blocks_per_weight_row + block_idx]; - let w1 = &rhs_blocks[(col + 1) * blocks_per_weight_row + block_idx]; - let w2 = &rhs_blocks[(col + 2) * blocks_per_weight_row + block_idx]; - let w3 = &rhs_blocks[(col + 3) * blocks_per_weight_row + block_idx]; - - let a = &act0[block_idx]; - a00 += w0.vec_dot(a); - a01 += w1.vec_dot(a); - a02 += w2.vec_dot(a); - a03 += w3.vec_dot(a); - - let a = &act1[block_idx]; - a10 += w0.vec_dot(a); - a11 += w1.vec_dot(a); - a12 += w2.vec_dot(a); - a13 += w3.vec_dot(a); - } - - out_data[col] = a00; - out_data[col + 1] = a01; - out_data[col + 2] = a02; - out_data[col + 3] = a03; - out_data[n + col] = a10; - out_data[n + col + 1] = a11; - out_data[n + col + 2] = a12; - out_data[n + col + 3] = a13; - } - - for j in (n_tiles * NR)..n { - let (mut s0, mut s1) = (0.0f32, 0.0f32); - for block_idx in 0..blocks_per_weight_row { - let w = &rhs_blocks[j * blocks_per_weight_row + block_idx]; - s0 += w.vec_dot(&act0[block_idx]); - s1 += w.vec_dot(&act1[block_idx]); - } - out_data[j] = s0; - out_data[n + j] = s1; - } -} - -/// Process m rows using outer-loop unrolled tiling. -/// Groups rows into sets of 3 for maximum weight reuse, with -/// 2-row and 1-row fallbacks for the remainder. -#[inline(always)] -fn process_multi_row_tiled( - lhs_data: &[f32], - rhs_blocks: &[B], - out_data: &mut [f32], - m: usize, - k: usize, - n: usize, - blocks_per_weight_row: usize, -) where - B::ActivationBlock: Pod, -{ - const MR: usize = 3; - let full_groups = m / MR; - let remainder = m % MR; - - for g in 0..full_groups { - let row = g * MR; - process_3rows_integer_tiled::( - &lhs_data[row * k..(row + MR) * k], - rhs_blocks, - &mut out_data[row * n..(row + MR) * n], - k, - n, - blocks_per_weight_row, - ); - } - - let rem_start = full_groups * MR; - match remainder { - 2 => { - process_2rows_integer_tiled::( - &lhs_data[rem_start * k..(rem_start + 2) * k], - rhs_blocks, - &mut out_data[rem_start * n..(rem_start + 2) * n], - k, - n, - blocks_per_weight_row, - ); - } - 1 => { - let lhs_row = &lhs_data[rem_start * k..(rem_start + 1) * k]; - let out_row = &mut out_data[rem_start * n..(rem_start + 1) * n]; - process_row_integer_tiled::(lhs_row, rhs_blocks, out_row, n, blocks_per_weight_row); - } - _ => {} - } -} - /// Process a single output row with SIMD using 4-way tiling for better ILP -#[allow(dead_code)] #[inline(always)] fn process_row_simd_tiled( simd: S, @@ -967,7 +722,6 @@ fn process_row_simd_tiled( } /// Compute a single dot product for one output column -#[allow(dead_code)] #[inline(always)] fn compute_dot_product( simd: S, diff --git a/fusor-ml/fusor/src/autograd.rs b/fusor-ml/fusor/src/autograd.rs new file mode 100644 index 000000000..c122705f6 --- /dev/null +++ b/fusor-ml/fusor/src/autograd.rs @@ -0,0 +1,1306 @@ +use std::{ + any::Any, + collections::{HashMap, HashSet, VecDeque}, + ops::Range, + sync::{Arc, Mutex}, +}; + +use crate::{Device, Error, Result, Tensor as RawTensor, ToVec1, ToVec2, layers::Embedding}; + +type NodeId = usize; +type BackwardRule = + Arc) -> Result> + Send + Sync>; + +#[derive(Clone)] +pub struct Graph { + inner: Arc, +} + +#[derive(Clone)] +pub struct Tensor { + value: RawTensor, + handle: NodeHandle, +} + +pub struct Gradients { + gradients: HashMap>, +} + +pub struct BackwardTarget { + node: NodeId, + gradient: Box, +} + +#[derive(Clone)] +pub struct Parent { + handle: NodeHandle, +} + +#[derive(Clone)] +struct NodeHandle { + graph: Arc, + id: NodeId, +} + +#[derive(Clone)] +struct Node { + parents: Vec, + backward: Option, + requires_grad: bool, +} + +struct GraphInner { + state: Mutex, +} + +struct GraphState { + next_id: NodeId, + nodes: HashMap, +} + +trait AnyTensorValue: Send + Sync { + fn as_any(&self) -> &dyn Any; + fn clone_box(&self) -> Box; + fn add_box(&self, other: &dyn AnyTensorValue) -> Result>; +} + +impl Graph { + pub fn new() -> Self { + Self { + inner: Arc::new(GraphInner { + state: Mutex::new(GraphState { + next_id: 0, + nodes: HashMap::new(), + }), + }), + } + } + + pub fn leaf(&self, value: RawTensor) -> Tensor { + self.tensor_with_grad(value, true) + } + + pub fn constant(&self, value: RawTensor) -> Tensor { + self.tensor_with_grad(value, false) + } + + pub fn tensor(&self, device: &Device, data: T) -> Tensor + where + RawTensor: fusor_types::FromArray, + { + self.leaf(RawTensor::new(device, data)) + } + + pub fn constant_from_data(&self, device: &Device, data: T) -> Tensor + where + RawTensor: fusor_types::FromArray, + { + self.constant(RawTensor::new(device, data)) + } + + fn tensor_with_grad( + &self, + value: RawTensor, + requires_grad: bool, + ) -> Tensor { + let id = self.inner.add_node(Vec::new(), None, requires_grad); + Tensor { + value, + handle: NodeHandle { + graph: self.inner.clone(), + id, + }, + } + } +} + +impl Default for Graph { + fn default() -> Self { + Self::new() + } +} + +impl Tensor { + pub fn from_raw(graph: &Graph, value: RawTensor) -> Self { + graph.leaf(value) + } + + pub fn constant_from_raw(graph: &Graph, value: RawTensor) -> Self { + graph.constant(value) + } + + pub fn new(graph: &Graph, device: &Device, data: T) -> Self + where + RawTensor: fusor_types::FromArray, + { + graph.tensor(device, data) + } + + pub fn from_slice(graph: &Graph, device: &Device, shape: [usize; R], data: &[f32]) -> Self { + graph.leaf(RawTensor::from_slice(device, shape, data)) + } + + pub fn zeros(graph: &Graph, device: &Device, shape: [usize; R]) -> Self { + graph.leaf(RawTensor::zeros(device, shape)) + } + + pub fn splat(graph: &Graph, device: &Device, value: f32, shape: [usize; R]) -> Self { + graph.leaf(RawTensor::splat(device, value, shape)) + } + + pub fn raw(&self) -> &RawTensor { + &self.value + } + + pub fn into_raw(self) -> RawTensor { + self.value + } + + pub fn shape(&self) -> [usize; R] { + self.value.shape() + } + + pub fn device(&self) -> Device { + self.value.device() + } + + pub fn graph(&self) -> Graph { + Graph { + inner: self.handle.graph.clone(), + } + } + + pub fn requires_grad(&self) -> bool { + self.handle.graph.requires_grad(self.handle.id) + } + + pub fn parent(&self) -> Parent { + Parent { + handle: self.handle.clone(), + } + } + + pub fn detach(&self) -> Self { + let requires_grad = self.requires_grad(); + let id = self.handle.graph.add_node(Vec::new(), None, requires_grad); + Self { + value: self.value.to_concrete(), + handle: NodeHandle { + graph: self.handle.graph.clone(), + id, + }, + } + } + + pub fn with_backwards(self, parents: I, backwards: F) -> Self + where + I: IntoIterator, + F: Fn(RawTensor) -> Result> + Send + Sync + 'static, + { + let parent_handles = parents + .into_iter() + .map(|parent| parent.handle) + .collect::>(); + let requires_grad = parent_handles + .iter() + .any(|parent| parent.graph.requires_grad(parent.id)); + let parent_ids = parent_handles + .iter() + .map(|parent| parent.id) + .collect::>(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = gradient + .as_any() + .downcast_ref::>() + .ok_or_else(|| Error::msg("gradient rank mismatch in custom backward"))? + .clone(); + backwards(gradient) + }); + self.handle.graph.replace_node( + self.handle.id, + Node { + parents: parent_ids, + backward: Some(backward), + requires_grad, + }, + ); + self + } + + pub fn backward(&self) -> Result { + let elements = self.shape().iter().product::(); + if elements != 1 { + return Err(Error::msg( + "backward() requires a single-element tensor; use backward_with() for non-scalars", + )); + } + let seed = RawTensor::splat(&self.device(), 1.0, self.shape()); + self.backward_with(seed) + } + + pub fn backward_with(&self, seed: RawTensor) -> Result { + self.handle.graph.backward(self.handle.id, Box::new(seed)) + } + + fn from_op( + &self, + value: RawTensor, + parents: Vec, + backward: Option, + ) -> Tensor { + for parent in &parents { + assert!( + Arc::ptr_eq(&self.handle.graph, &parent.graph), + "cannot mix autograd tensors from different graphs" + ); + } + let requires_grad = parents + .iter() + .any(|parent| parent.graph.requires_grad(parent.id)); + let parent_ids = parents.into_iter().map(|parent| parent.id).collect(); + let id = self + .handle + .graph + .add_node(parent_ids, backward, requires_grad); + Tensor { + value, + handle: NodeHandle { + graph: self.handle.graph.clone(), + id, + }, + } + } + + pub fn add(&self, rhs: &Self) -> Self { + self.binary_op( + rhs, + (self.value.clone() + rhs.value.clone()).to_concrete(), + |grad, _, _| vec![grad.clone().to_concrete(), grad.to_concrete()], + ) + } + + pub fn sub(&self, rhs: &Self) -> Self { + self.binary_op( + rhs, + (self.value.clone() - rhs.value.clone()).to_concrete(), + |grad, _, _| vec![grad.clone().to_concrete(), (-grad).to_concrete()], + ) + } + + pub fn mul(&self, rhs: &Self) -> Self { + self.binary_op( + rhs, + (self.value.clone() * rhs.value.clone()).to_concrete(), + |grad, lhs, rhs| { + vec![ + (grad.clone() * rhs).to_concrete(), + (grad * lhs).to_concrete(), + ] + }, + ) + } + + pub fn div(&self, rhs: &Self) -> Self { + self.binary_op( + rhs, + (self.value.clone() / rhs.value.clone()).to_concrete(), + |grad, lhs, rhs| { + let lhs_grad = (grad.clone() / rhs.clone()).to_concrete(); + let rhs_grad = (-((grad * lhs) / rhs.sqr().to_concrete())).to_concrete(); + vec![lhs_grad, rhs_grad] + }, + ) + } + + pub fn add_scalar(&self, scalar: f32) -> Self { + self.unary_from_value(self.value.add_scalar(scalar), move |grad, _| grad) + } + + pub fn sub_scalar(&self, scalar: f32) -> Self { + self.unary_from_value(self.value.sub_scalar(scalar), move |grad, _| grad) + } + + pub fn mul_scalar(&self, scalar: f32) -> Self { + self.unary_from_value( + self.value.mul_scalar(scalar).to_concrete(), + move |grad, _| grad.mul_scalar(scalar).to_concrete(), + ) + } + + pub fn div_scalar(&self, scalar: f32) -> Self { + self.unary_from_value( + self.value.div_scalar(scalar).to_concrete(), + move |grad, _| grad.div_scalar(scalar).to_concrete(), + ) + } + + pub fn neg(&self) -> Self { + self.unary_from_value((-self.value.clone()).to_concrete(), move |grad, _| { + (-grad).to_concrete() + }) + } + + pub fn sqr(&self) -> Self { + let input = self.value.clone(); + self.unary_from_value(self.value.sqr().to_concrete(), move |grad, _| { + ((grad * input.clone()).to_concrete().mul_scalar(2.0)).to_concrete() + }) + } + + pub fn relu(&self) -> Self { + let output = self.value.relu().to_concrete(); + self.unary_from_value(output.clone(), move |grad, out| { + let zeros = RawTensor::zeros(&out.device(), out.shape()); + let ones = RawTensor::splat(&out.device(), 1.0, out.shape()); + (grad * out.where_cond(&ones, &zeros)).to_concrete() + }) + } + + pub fn tanh(&self) -> Self { + self.unary_from_value(self.value.tanh().to_concrete(), move |grad, out| { + let one_minus_sq = (RawTensor::splat(&out.device(), 1.0, out.shape()) + - out.sqr().to_concrete()) + .to_concrete(); + (grad * one_minus_sq).to_concrete() + }) + } + + pub fn exp(&self) -> Self { + self.unary_from_value(self.value.exp().to_concrete(), move |grad, out| { + (grad * out).to_concrete() + }) + } + + pub fn log(&self) -> Self { + let input = self.value.clone(); + self.unary_from_value(self.value.log().to_concrete(), move |grad, _| { + (grad / input.clone()).to_concrete() + }) + } + + pub fn sqrt(&self) -> Self { + self.unary_from_value(self.value.sqrt().to_concrete(), move |grad, out| { + let denom = out.mul_scalar(2.0).to_concrete(); + (grad / denom).to_concrete() + }) + } + + pub fn reshape(&self, shape: [usize; OUT]) -> Tensor { + let input_shape = self.shape(); + let value = self.value.reshape(shape).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::(&*gradient, "reshape")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.reshape(input_shape).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn transpose(&self, dim0: usize, dim1: usize) -> Self { + let value = self.value.transpose(dim0, dim1).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::(&*gradient, "transpose")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.transpose(dim0, dim1).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn slice(&self, slices: [Range; R]) -> Self { + let input_shape = self.shape(); + let value = self.value.slice(slices.clone()).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::(&*gradient, "slice")?; + let zeros = RawTensor::zeros(&gradient.device(), input_shape); + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(zeros.slice_assign(slices.clone(), &gradient).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn broadcast_as(&self, shape: [usize; OUT]) -> Tensor { + let input_shape = self.shape(); + let value = self.value.broadcast_as(shape).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::(&*gradient, "broadcast_as")?; + let reduced = reduce_broadcast_gradient(gradient, input_shape)?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: reduced, + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + fn unary_from_value( + &self, + value: RawTensor, + backward: impl Fn(RawTensor, RawTensor) -> RawTensor + + Send + + Sync + + 'static, + ) -> Self { + let input = self.handle.clone(); + let output = value.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::(&*gradient, "unary")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(backward(gradient, output.clone()).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + fn binary_op( + &self, + rhs: &Self, + value: RawTensor, + backward: impl Fn( + RawTensor, + RawTensor, + RawTensor, + ) -> Vec> + + Send + + Sync + + 'static, + ) -> Self { + assert!( + Arc::ptr_eq(&self.handle.graph, &rhs.handle.graph), + "cannot mix autograd tensors from different graphs" + ); + let lhs = self.handle.clone(); + let rhs_handle = rhs.handle.clone(); + let lhs_value = self.value.clone(); + let rhs_value = rhs.value.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::(&*gradient, "binary")?; + let gradients = backward(gradient, lhs_value.clone(), rhs_value.clone()); + Ok(vec![ + BackwardTarget { + node: lhs.id, + gradient: Box::new(gradients[0].clone().to_concrete()), + }, + BackwardTarget { + node: rhs_handle.id, + gradient: Box::new(gradients[1].clone().to_concrete()), + }, + ]) + }); + self.from_op( + value, + vec![self.handle.clone(), rhs.handle.clone()], + Some(backward), + ) + } +} + +impl Tensor<1> { + pub fn sum(&self) -> Tensor<0> { + let input_shape = self.shape(); + let value = self.value.sum::<0>(0); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<0>(&*gradient, "sum")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.broadcast_as(input_shape).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn unsqueeze(&self, dim: usize) -> Tensor<2> { + let value = self.value.unsqueeze(dim).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<2>(&*gradient, "unsqueeze")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.squeeze(dim).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } +} + +impl Tensor<2> { + pub fn mat_mul(&self, rhs: &Tensor<2>) -> Tensor<2> { + assert_same_graph(self, rhs); + let value = self.value.mat_mul(&rhs.value); + let lhs = self.handle.clone(); + let rhs_handle = rhs.handle.clone(); + let lhs_value = self.value.clone(); + let rhs_value = rhs.value.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<2>(&*gradient, "mat_mul")?; + Ok(vec![ + BackwardTarget { + node: lhs.id, + gradient: Box::new(gradient.clone().mat_mul(&rhs_value.transpose(0, 1))), + }, + BackwardTarget { + node: rhs_handle.id, + gradient: Box::new(lhs_value.transpose(0, 1).mat_mul(&gradient)), + }, + ]) + }); + self.from_op( + value, + vec![self.handle.clone(), rhs.handle.clone()], + Some(backward), + ) + } + + pub fn squeeze(&self, dim: usize) -> Tensor<1> { + let value = self.value.squeeze(dim).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<1>(&*gradient, "squeeze")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.unsqueeze(dim).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn unsqueeze(&self, dim: usize) -> Tensor<3> { + let value = self.value.unsqueeze(dim).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<3>(&*gradient, "unsqueeze")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.squeeze(dim).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn sum(&self, axis: usize) -> Tensor<1> { + let input_shape = self.shape(); + let value = self.value.sum::<1>(axis).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<1>(&*gradient, "sum")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new( + gradient + .unsqueeze(axis) + .broadcast_as(input_shape) + .to_concrete(), + ), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn sum_keepdim(&self, axis: usize) -> Tensor<2> { + let input_shape = self.shape(); + let value = self.value.sum_keepdim::<1>(axis).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<2>(&*gradient, "sum_keepdim")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.broadcast_as(input_shape).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn gather_last(&self, indices: &RawTensor<1, u32>) -> Tensor<1> { + let shape = self.shape(); + assert_eq!( + shape[0], + indices.shape()[0], + "gather_last expects one index per row" + ); + let width = shape[1]; + let device = self.device(); + let index_values = pollster::block_on(indices.clone().as_slice()) + .unwrap() + .to_vec1(); + let linear_indices = index_values + .iter() + .enumerate() + .map(|(row, &column)| { + assert!( + (column as usize) < width, + "gather_last index {} out of bounds for width {}", + column, + width + ); + (row * width + column as usize) as u32 + }) + .collect::>(); + let linear_indices_tensor = RawTensor::from_slice(&device, [shape[0]], &linear_indices); + let flat = self.value.reshape([shape[0] * width]).to_concrete(); + let value = flat.index_select(0, &linear_indices_tensor).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<1>(&*gradient, "gather_last")?; + let gradient_values = pollster::block_on(gradient.clone().as_slice())?.to_vec1(); + let mut input_gradient = vec![0.0f32; shape[0] * width]; + for (row, &linear_index) in linear_indices.iter().enumerate() { + input_gradient[linear_index as usize] += gradient_values[row]; + } + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(RawTensor::from_slice(&device, shape, &input_gradient)), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn embedding(&self, indices: &RawTensor<2, u32>) -> Tensor<3> { + let value: RawTensor<3, f32> = + Embedding::new_from_tensor(self.value.clone()).forward(indices); + let table = self.handle.clone(); + let table_shape = self.shape(); + let device = self.device(); + let indices = indices.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<3>(&*gradient, "embedding")?; + let index_values = pollster::block_on(indices.clone().as_slice())?.to_vec2(); + let grad_shape = gradient.shape(); + let grad_flat = gradient.reshape([grad_shape[0] * grad_shape[1], grad_shape[2]]); + + let mut rows_by_token = HashMap::>::new(); + for (batch, row) in index_values.iter().enumerate() { + for (position, &token) in row.iter().enumerate() { + let flat_row = (batch * grad_shape[1] + position) as u32; + rows_by_token.entry(token).or_default().push(flat_row); + } + } + + let mut embedding_gradient = RawTensor::zeros(&device, table_shape); + for (token, rows) in rows_by_token { + let row_indices = RawTensor::from_slice(&device, [rows.len()], &rows); + let token_gradient = grad_flat + .index_select(0, &row_indices) + .sum::<1>(0) + .unsqueeze::<2>(0) + .to_concrete(); + embedding_gradient = embedding_gradient.slice_assign( + [token as usize..token as usize + 1, 0..table_shape[1]], + &token_gradient, + ); + } + + Ok(vec![BackwardTarget { + node: table.id, + gradient: Box::new(embedding_gradient), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } +} + +impl Tensor<3> { + pub fn mat_mul(&self, rhs: &Tensor<3>) -> Tensor<3> { + assert_same_graph(self, rhs); + let value = self.value.mat_mul(&rhs.value); + let lhs = self.handle.clone(); + let rhs_handle = rhs.handle.clone(); + let lhs_value = self.value.clone(); + let rhs_value = rhs.value.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<3>(&*gradient, "mat_mul")?; + Ok(vec![ + BackwardTarget { + node: lhs.id, + gradient: Box::new(gradient.clone().mat_mul(&rhs_value.transpose(1, 2))), + }, + BackwardTarget { + node: rhs_handle.id, + gradient: Box::new(lhs_value.transpose(1, 2).mat_mul(&gradient)), + }, + ]) + }); + self.from_op( + value, + vec![self.handle.clone(), rhs.handle.clone()], + Some(backward), + ) + } + + pub fn squeeze(&self, dim: usize) -> Tensor<2> { + let value = self.value.squeeze(dim).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<2>(&*gradient, "squeeze")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.unsqueeze(dim).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn sum(&self, axis: usize) -> Tensor<2> { + let input_shape = self.shape(); + let value = self.value.sum::<2>(axis).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<2>(&*gradient, "sum")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new( + gradient + .unsqueeze(axis) + .broadcast_as(input_shape) + .to_concrete(), + ), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn sum_keepdim(&self, axis: usize) -> Tensor<3> { + let input_shape = self.shape(); + let value = self.value.sum_keepdim::<2>(axis).to_concrete(); + let input = self.handle.clone(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<3>(&*gradient, "sum_keepdim")?; + Ok(vec![BackwardTarget { + node: input.id, + gradient: Box::new(gradient.broadcast_as(input_shape).to_concrete()), + }]) + }); + self.from_op(value, vec![self.handle.clone()], Some(backward)) + } + + pub fn cat(tensors: Vec>, dim: usize) -> Tensor<3> { + assert!(!tensors.is_empty(), "cat requires at least one tensor"); + let graph = tensors[0].handle.graph.clone(); + let raw = tensors + .iter() + .map(|tensor| tensor.value.clone()) + .collect::>(); + let value = RawTensor::cat(raw, dim); + let parents = tensors + .iter() + .map(|tensor| tensor.handle.clone()) + .collect::>(); + let backward_parents = parents.clone(); + let slices = tensors + .iter() + .scan(0usize, |offset, tensor| { + let start = *offset; + let length = tensor.shape()[dim]; + *offset += length; + Some(start..start + length) + }) + .collect::>(); + let backward: BackwardRule = Arc::new(move |gradient| { + let gradient = downcast_tensor::<3>(&*gradient, "cat")?; + let mut targets = Vec::with_capacity(backward_parents.len()); + for (parent, slice) in backward_parents.iter().zip(slices.iter()) { + let grad_slice = match dim { + 0 => gradient.slice([ + slice.clone(), + 0..gradient.shape()[1], + 0..gradient.shape()[2], + ]), + 1 => gradient.slice([ + 0..gradient.shape()[0], + slice.clone(), + 0..gradient.shape()[2], + ]), + 2 => gradient.slice([ + 0..gradient.shape()[0], + 0..gradient.shape()[1], + slice.clone(), + ]), + _ => panic!("invalid cat dim"), + } + .to_concrete(); + targets.push(BackwardTarget { + node: parent.id, + gradient: Box::new(grad_slice), + }); + } + Ok(targets) + }); + let id = graph.add_node( + parents.iter().map(|parent| parent.id).collect(), + Some(backward), + parents + .iter() + .any(|parent| parent.graph.requires_grad(parent.id)), + ); + Tensor { + value, + handle: NodeHandle { graph, id }, + } + } + + pub fn layer_norm(&self, weight: &Tensor<1>, bias: Option<&Tensor<1>>, eps: f32) -> Tensor<3> { + let centered = { + let mean = self.sum_keepdim(2).div_scalar(self.shape()[2] as f32); + self.sub(&mean.broadcast_as(self.shape())) + }; + let variance = centered + .sqr() + .sum_keepdim(2) + .div_scalar(self.shape()[2] as f32); + let std = variance.add_scalar(eps).sqrt(); + let normalized = centered.div(&std.broadcast_as(self.shape())); + let scaled = normalized.mul(&weight.broadcast_as(self.shape())); + if let Some(bias) = bias { + scaled.add(&bias.broadcast_as(self.shape())) + } else { + scaled + } + } +} + +impl Gradients { + pub fn get(&self, tensor: &Tensor) -> Option> { + self.gradients + .get(&tensor.handle.id) + .and_then(|gradient| gradient.as_any().downcast_ref::>()) + .cloned() + } +} + +impl BackwardTarget { + pub fn wrt(tensor: &Tensor, gradient: RawTensor) -> Self { + Self { + node: tensor.handle.id, + gradient: Box::new(gradient), + } + } +} + +impl GraphInner { + fn add_node( + &self, + parents: Vec, + backward: Option, + requires_grad: bool, + ) -> NodeId { + let mut state = self.state.lock().unwrap(); + let id = state.next_id; + state.next_id += 1; + state.nodes.insert( + id, + Node { + parents, + backward, + requires_grad, + }, + ); + id + } + + fn replace_node(&self, id: NodeId, node: Node) { + self.state.lock().unwrap().nodes.insert(id, node); + } + + fn requires_grad(&self, id: NodeId) -> bool { + self.state + .lock() + .unwrap() + .nodes + .get(&id) + .map(|node| node.requires_grad) + .unwrap_or(false) + } + + fn backward(&self, root: NodeId, seed: Box) -> Result { + let nodes = self.reachable_nodes(root); + let mut pending_children = HashMap::::new(); + for (id, node) in &nodes { + pending_children.entry(*id).or_insert(0); + for parent in &node.parents { + *pending_children.entry(*parent).or_insert(0) += 1; + } + } + + let mut gradients = HashMap::>::new(); + gradients.insert(root, seed); + + let mut queue = VecDeque::new(); + queue.push_back(root); + + while let Some(node_id) = queue.pop_front() { + let Some(node) = nodes.get(&node_id) else { + continue; + }; + let Some(backward) = node.backward.as_ref() else { + continue; + }; + let gradient = gradients + .get(&node_id) + .ok_or_else(|| Error::msg(format!("missing gradient for node {node_id}")))? + .clone_box(); + + for target in backward(gradient)? { + let Some(parent_node) = nodes.get(&target.node) else { + continue; + }; + if !parent_node.requires_grad { + continue; + } + accumulate_gradient(&mut gradients, target.node, target.gradient)?; + let remaining = pending_children.get_mut(&target.node).ok_or_else(|| { + Error::msg(format!("missing child count for node {}", target.node)) + })?; + *remaining = remaining.saturating_sub(1); + if *remaining == 0 { + queue.push_back(target.node); + } + } + } + + Ok(Gradients { gradients }) + } + + fn reachable_nodes(&self, root: NodeId) -> HashMap { + let snapshot = self.state.lock().unwrap().nodes.clone(); + let mut reachable = HashMap::new(); + let mut stack = vec![root]; + let mut visited = HashSet::new(); + while let Some(node_id) = stack.pop() { + if !visited.insert(node_id) { + continue; + } + if let Some(node) = snapshot.get(&node_id) { + reachable.insert(node_id, node.clone()); + stack.extend(node.parents.iter().copied()); + } + } + reachable + } +} + +impl AnyTensorValue for RawTensor { + fn as_any(&self) -> &dyn Any { + self + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn add_box(&self, other: &dyn AnyTensorValue) -> Result> { + let other = other + .as_any() + .downcast_ref::>() + .ok_or_else(|| Error::msg("gradient rank mismatch while accumulating"))?; + Ok(Box::new((self.clone() + other.clone()).to_concrete())) + } +} + +fn accumulate_gradient( + gradients: &mut HashMap>, + node: NodeId, + gradient: Box, +) -> Result<()> { + match gradients.get(&node) { + Some(existing) => { + let accumulated = existing.add_box(&*gradient)?; + gradients.insert(node, accumulated); + } + None => { + gradients.insert(node, gradient); + } + } + Ok(()) +} + +fn downcast_tensor( + value: &dyn AnyTensorValue, + context: &str, +) -> Result> { + value + .as_any() + .downcast_ref::>() + .cloned() + .ok_or_else(|| Error::msg(format!("gradient rank mismatch in {context}"))) +} + +fn assert_same_graph(lhs: &Tensor, rhs: &Tensor) { + assert!( + Arc::ptr_eq(&lhs.handle.graph, &rhs.handle.graph), + "cannot mix autograd tensors from different graphs" + ); +} + +fn reduce_broadcast_gradient( + gradient: RawTensor, + input_shape: [usize; IN], +) -> Result> { + match (IN, OUT) { + (1, 1) => Ok(Box::new(reduce_same_rank_broadcast_1( + gradient.reshape([gradient.shape()[0]]).to_concrete(), + [input_shape[0]], + ))), + (2, 2) => Ok(Box::new(reduce_same_rank_broadcast_2( + gradient + .reshape([gradient.shape()[0], gradient.shape()[1]]) + .to_concrete(), + [input_shape[0], input_shape[1]], + ))), + (3, 3) => Ok(Box::new(reduce_same_rank_broadcast_3( + gradient + .reshape([ + gradient.shape()[0], + gradient.shape()[1], + gradient.shape()[2], + ]) + .to_concrete(), + [input_shape[0], input_shape[1], input_shape[2]], + ))), + (1, 2) => { + let reduced = reduce_to_1_from_2( + gradient + .reshape([gradient.shape()[0], gradient.shape()[1]]) + .to_concrete(), + input_shape[0], + ); + Ok(Box::new(reduced)) + } + (1, 3) => { + let reduced = reduce_to_1_from_3( + gradient + .reshape([ + gradient.shape()[0], + gradient.shape()[1], + gradient.shape()[2], + ]) + .to_concrete(), + input_shape[0], + ); + Ok(Box::new(reduced)) + } + (2, 3) => { + let reduced = reduce_to_2_from_3( + gradient + .reshape([ + gradient.shape()[0], + gradient.shape()[1], + gradient.shape()[2], + ]) + .to_concrete(), + [input_shape[0], input_shape[1]], + ); + Ok(Box::new(reduced)) + } + _ => Err(Error::msg( + "unsupported broadcast gradient rank combination", + )), + } +} + +fn reduce_same_rank_broadcast_1( + mut gradient: RawTensor<1, f32>, + input_shape: [usize; 1], +) -> RawTensor<1, f32> { + if input_shape[0] == 1 && gradient.shape()[0] != 1 { + gradient = gradient.sum_keepdim::<0>(0).to_concrete(); + } + gradient.reshape(input_shape).to_concrete() +} + +fn reduce_same_rank_broadcast_2( + mut gradient: RawTensor<2, f32>, + input_shape: [usize; 2], +) -> RawTensor<2, f32> { + let grad_shape = gradient.shape(); + if input_shape[0] == 1 && grad_shape[0] != 1 { + gradient = gradient.sum_keepdim::<1>(0).to_concrete(); + } + if input_shape[1] == 1 && grad_shape[1] != 1 { + gradient = gradient.sum_keepdim::<1>(1).to_concrete(); + } + gradient.reshape(input_shape).to_concrete() +} + +fn reduce_same_rank_broadcast_3( + mut gradient: RawTensor<3, f32>, + input_shape: [usize; 3], +) -> RawTensor<3, f32> { + let grad_shape = gradient.shape(); + if input_shape[0] == 1 && grad_shape[0] != 1 { + gradient = gradient.sum_keepdim::<2>(0).to_concrete(); + } + if input_shape[1] == 1 && grad_shape[1] != 1 { + gradient = gradient.sum_keepdim::<2>(1).to_concrete(); + } + if input_shape[2] == 1 && grad_shape[2] != 1 { + gradient = gradient.sum_keepdim::<2>(2).to_concrete(); + } + gradient.reshape(input_shape).to_concrete() +} + +fn reduce_to_1_from_2(mut gradient: RawTensor<2, f32>, target: usize) -> RawTensor<1, f32> { + if gradient.shape()[0] != 1 { + gradient = gradient.sum_keepdim::<1>(0); + } + if gradient.shape()[1] != target { + gradient = gradient.sum_keepdim::<1>(1); + } + gradient.reshape([target]).to_concrete() +} + +fn reduce_to_1_from_3(mut gradient: RawTensor<3, f32>, target: usize) -> RawTensor<1, f32> { + if gradient.shape()[0] != 1 { + gradient = gradient.sum_keepdim::<2>(0); + } + if gradient.shape()[1] != 1 { + gradient = gradient.sum_keepdim::<2>(1); + } + if gradient.shape()[2] != target { + gradient = gradient.sum_keepdim::<2>(2); + } + gradient.reshape([target]).to_concrete() +} + +fn reduce_to_2_from_3(mut gradient: RawTensor<3, f32>, target: [usize; 2]) -> RawTensor<2, f32> { + if gradient.shape()[0] != 1 { + gradient = gradient.sum_keepdim::<2>(0); + } + if gradient.shape()[1] != target[0] { + gradient = gradient.sum_keepdim::<2>(1); + } + if gradient.shape()[2] != target[1] { + gradient = gradient.sum_keepdim::<2>(2); + } + gradient.reshape(target).to_concrete() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ToVec1, ToVec2}; + + fn assert_close(left: f32, right: f32) { + assert!((left - right).abs() < 1e-3, "expected {right}, got {left}"); + } + + #[tokio::test] + async fn test_backward_squared_sum_cpu() { + let graph = Graph::new(); + let device = Device::cpu(); + + let x: Tensor<1> = Tensor::new(&graph, &device, &[1.0f32, 2.0, 3.0]); + let loss = x.sqr().sum(); + let gradients = loss.backward().unwrap(); + let dx = gradients + .get(&x) + .unwrap() + .as_slice() + .await + .unwrap() + .to_vec1(); + + assert_close(dx[0], 2.0); + assert_close(dx[1], 4.0); + assert_close(dx[2], 6.0); + } + + #[tokio::test] + async fn test_backward_matmul_with_broadcast_bias_cpu() { + let graph = Graph::new(); + let device = Device::cpu(); + + let x: Tensor<2> = Tensor::new(&graph, &device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let w: Tensor<2> = Tensor::new(&graph, &device, &[[0.5f32], [1.0], [1.5]]); + let b: Tensor<1> = Tensor::new(&graph, &device, &[2.0f32]); + + let y = x.mat_mul(&w).add(&b.broadcast_as([2, 1])); + let loss = y.sum(1).sum(); + + let gradients = loss.backward().unwrap(); + let dw = gradients + .get(&w) + .unwrap() + .as_slice() + .await + .unwrap() + .to_vec2(); + let db = gradients + .get(&b) + .unwrap() + .as_slice() + .await + .unwrap() + .to_vec1(); + + assert_close(dw[0][0], 5.0); + assert_close(dw[1][0], 7.0); + assert_close(dw[2][0], 9.0); + assert_close(db[0], 2.0); + } + + #[tokio::test] + async fn test_backward_embedding_cpu() { + let graph = Graph::new(); + let device = Device::cpu(); + + let table: Tensor<2> = + Tensor::new(&graph, &device, &[[1.0f32, 2.0], [3.0, 4.0], [5.0, 6.0]]); + let indices: RawTensor<2, u32> = RawTensor::new(&device, &[[0u32, 2u32]]); + let embedded = table.embedding(&indices); + let loss = embedded.sum(2).sum(1).sum(); + + let gradients = loss.backward().unwrap(); + let dtable = gradients + .get(&table) + .unwrap() + .as_slice() + .await + .unwrap() + .to_vec2(); + + assert_close(dtable[0][0], 1.0); + assert_close(dtable[0][1], 1.0); + assert_close(dtable[1][0], 0.0); + assert_close(dtable[1][1], 0.0); + assert_close(dtable[2][0], 1.0); + assert_close(dtable[2][1], 1.0); + } + + #[tokio::test] + async fn test_backward_gather_last_cpu() { + let graph = Graph::new(); + let device = Device::cpu(); + + let values: Tensor<2> = + Tensor::new(&graph, &device, &[[1.0f32, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let indices: RawTensor<1, u32> = RawTensor::new(&device, &[2u32, 0u32]); + let gathered = values.gather_last(&indices); + let loss = gathered.sum(); + + let gradients = loss.backward().unwrap(); + let dvalues = gradients + .get(&values) + .unwrap() + .as_slice() + .await + .unwrap() + .to_vec2(); + + assert_close(dvalues[0][0], 0.0); + assert_close(dvalues[0][1], 0.0); + assert_close(dvalues[0][2], 1.0); + assert_close(dvalues[1][0], 1.0); + assert_close(dvalues[1][1], 0.0); + assert_close(dvalues[1][2], 0.0); + } +} diff --git a/fusor-ml/fusor/src/layers/recurrent.rs b/fusor-ml/fusor/src/layers/recurrent.rs new file mode 100644 index 000000000..f8a3b5da9 --- /dev/null +++ b/fusor-ml/fusor/src/layers/recurrent.rs @@ -0,0 +1,557 @@ +use fusor_cpu::TensorBacking; + +use crate::{Error, Result, Tensor}; + +#[derive(Clone)] +pub struct RecurrentWeights { + input_proj: Tensor<2, f32>, + state_proj: Tensor<2, f32>, + gate_input_proj: Tensor<2, f32>, + gate_state_proj: Tensor<2, f32>, + out_proj: Tensor<2, f32>, +} + +impl RecurrentWeights { + pub fn new( + input_proj: Tensor<2, f32>, + state_proj: Tensor<2, f32>, + gate_input_proj: Tensor<2, f32>, + gate_state_proj: Tensor<2, f32>, + out_proj: Tensor<2, f32>, + ) -> Self { + Self { + input_proj, + state_proj, + gate_input_proj, + gate_state_proj, + out_proj, + } + } + + pub fn input_proj(&self) -> &Tensor<2, f32> { + &self.input_proj + } + + pub fn state_proj(&self) -> &Tensor<2, f32> { + &self.state_proj + } + + pub fn gate_input_proj(&self) -> &Tensor<2, f32> { + &self.gate_input_proj + } + + pub fn gate_state_proj(&self) -> &Tensor<2, f32> { + &self.gate_state_proj + } + + pub fn out_proj(&self) -> &Tensor<2, f32> { + &self.out_proj + } +} + +pub fn recurrent_forward(x: &Tensor<3, f32, B>, weights: &RecurrentWeights) -> Tensor<3, f32> +where + B: TensorBacking<3, Elem = f32>, +{ + let [batch_size, seq_len, n_embd] = x.shape(); + debug_assert_eq!(weights.input_proj.shape(), [n_embd, n_embd]); + debug_assert_eq!(weights.state_proj.shape(), [n_embd, n_embd]); + debug_assert_eq!(weights.gate_input_proj.shape(), [n_embd, n_embd]); + debug_assert_eq!(weights.gate_state_proj.shape(), [n_embd, n_embd]); + debug_assert_eq!(weights.out_proj.shape(), [n_embd, n_embd]); + + let device = x.device(); + let ones: Tensor<2, f32> = Tensor::splat(&device, 1.0, [batch_size, n_embd]); + let mut state: Tensor<2, f32> = Tensor::splat(&device, 0.0, [batch_size, n_embd]); + let mut outputs = Vec::with_capacity(seq_len); + + for position in 0..seq_len { + let x_t: Tensor<2, f32> = x + .slice([0..batch_size, position..position + 1, 0..n_embd]) + .squeeze(1) + .to_concrete(); + let candidate = (x_t.mat_mul(weights.input_proj()) + state.mat_mul(weights.state_proj())) + .tanh() + .to_concrete(); + let gate_pre = + x_t.mat_mul(weights.gate_input_proj()) + state.mat_mul(weights.gate_state_proj()); + let gate = ((gate_pre.tanh() + &ones) * 0.5).to_concrete(); + let keep = (&ones - &gate).to_concrete(); + state = ((&gate * &candidate) + &(&keep * &state)).to_concrete(); + outputs.push(state.mat_mul(weights.out_proj()).to_concrete()); + } + + Tensor::stack(outputs, 1) +} + +#[derive(Clone, Debug)] +pub struct HostRecurrentWeights { + n_embd: usize, + input_proj: Vec, + state_proj: Vec, + gate_input_proj: Vec, + gate_state_proj: Vec, + out_proj: Vec, +} + +impl HostRecurrentWeights { + pub fn from_nested( + input_proj: Vec>, + state_proj: Vec>, + gate_input_proj: Vec>, + gate_state_proj: Vec>, + out_proj: Vec>, + ) -> Result { + let n_embd = square_matrix_size("input_proj", &input_proj)?; + validate_square_matrix("state_proj", &state_proj, n_embd)?; + validate_square_matrix("gate_input_proj", &gate_input_proj, n_embd)?; + validate_square_matrix("gate_state_proj", &gate_state_proj, n_embd)?; + validate_square_matrix("out_proj", &out_proj, n_embd)?; + + Ok(Self { + n_embd, + input_proj: flatten_matrix(&input_proj), + state_proj: flatten_matrix(&state_proj), + gate_input_proj: flatten_matrix(&gate_input_proj), + gate_state_proj: flatten_matrix(&gate_state_proj), + out_proj: flatten_matrix(&out_proj), + }) + } + + pub fn n_embd(&self) -> usize { + self.n_embd + } +} + +#[derive(Clone, Debug)] +struct HostRecurrentScan { + batch_size: usize, + seq_len: usize, + n_embd: usize, + prev_states: Vec, + candidates: Vec, + gates: Vec, + states: Vec, +} + +#[derive(Clone, Debug)] +pub struct HostRecurrentBackward { + pub grad_input: Vec, + pub grad_input_proj: Vec, + pub grad_state_proj: Vec, + pub grad_gate_input_proj: Vec, + pub grad_gate_state_proj: Vec, + pub grad_out_proj: Vec, +} + +pub fn host_recurrent_forward( + input: &[f32], + batch_size: usize, + seq_len: usize, + weights: &HostRecurrentWeights, +) -> Result> { + Ok(host_recurrent_scan(input, batch_size, seq_len, weights)? + .states_to_outputs(&weights.out_proj)) +} + +pub fn host_recurrent_backward( + input: &[f32], + batch_size: usize, + seq_len: usize, + weights: &HostRecurrentWeights, + grad_output: &[f32], +) -> Result { + let scan = host_recurrent_scan(input, batch_size, seq_len, weights)?; + let n_embd = weights.n_embd; + let step_len = batch_size * n_embd; + let total_len = batch_size * seq_len * n_embd; + if grad_output.len() != total_len { + return Err(Error::msg(format!( + "expected grad_output length {}, got {}", + total_len, + grad_output.len() + ))); + } + + let mut grad_state_next = vec![0.0; step_len]; + let mut grad_input_proj = vec![0.0; n_embd * n_embd]; + let mut grad_state_proj = vec![0.0; n_embd * n_embd]; + let mut grad_gate_input_proj = vec![0.0; n_embd * n_embd]; + let mut grad_gate_state_proj = vec![0.0; n_embd * n_embd]; + let mut grad_out_proj = vec![0.0; n_embd * n_embd]; + let mut grad_input = vec![0.0; total_len]; + + for position in (0..seq_len).rev() { + let grad_output_t = &grad_output[position * step_len..(position + 1) * step_len]; + let state = &scan.states[position * step_len..(position + 1) * step_len]; + let prev_state = &scan.prev_states[position * step_len..(position + 1) * step_len]; + let candidate = &scan.candidates[position * step_len..(position + 1) * step_len]; + let gate = &scan.gates[position * step_len..(position + 1) * step_len]; + let x_t = &input[position * step_len..(position + 1) * step_len]; + + let grad_from_output = + matmul_rhs_transposed(grad_output_t, &weights.out_proj, batch_size, n_embd); + add_outer_product(&mut grad_out_proj, state, grad_output_t, batch_size, n_embd); + + let mut grad_state = vec![0.0; step_len]; + let mut grad_candidate_pre = vec![0.0; step_len]; + let mut grad_gate_pre = vec![0.0; step_len]; + let mut keep = vec![0.0; step_len]; + + for index in 0..step_len { + let grad_state_value = grad_from_output[index] + grad_state_next[index]; + let gate_value = gate[index]; + let candidate_value = candidate[index]; + let prev_state_value = prev_state[index]; + let keep_value = 1.0 - gate_value; + let grad_candidate = grad_state_value * gate_value; + let grad_gate = grad_state_value * (candidate_value - prev_state_value); + let tanh_gate_pre = (gate_value * 2.0) - 1.0; + + grad_state[index] = grad_state_value; + grad_candidate_pre[index] = grad_candidate * (1.0 - candidate_value * candidate_value); + grad_gate_pre[index] = grad_gate * (1.0 - tanh_gate_pre * tanh_gate_pre) * 0.5; + keep[index] = keep_value; + } + + add_outer_product( + &mut grad_input_proj, + x_t, + &grad_candidate_pre, + batch_size, + n_embd, + ); + add_outer_product( + &mut grad_state_proj, + prev_state, + &grad_candidate_pre, + batch_size, + n_embd, + ); + add_outer_product( + &mut grad_gate_input_proj, + x_t, + &grad_gate_pre, + batch_size, + n_embd, + ); + add_outer_product( + &mut grad_gate_state_proj, + prev_state, + &grad_gate_pre, + batch_size, + n_embd, + ); + + let grad_candidate_input = + matmul_rhs_transposed(&grad_candidate_pre, &weights.input_proj, batch_size, n_embd); + let grad_gate_input = + matmul_rhs_transposed(&grad_gate_pre, &weights.gate_input_proj, batch_size, n_embd); + for index in 0..step_len { + grad_input[position * step_len + index] = + grad_candidate_input[index] + grad_gate_input[index]; + } + + let grad_candidate_state = + matmul_rhs_transposed(&grad_candidate_pre, &weights.state_proj, batch_size, n_embd); + let grad_gate_state = + matmul_rhs_transposed(&grad_gate_pre, &weights.gate_state_proj, batch_size, n_embd); + for index in 0..step_len { + grad_state_next[index] = (grad_state[index] * keep[index]) + + grad_candidate_state[index] + + grad_gate_state[index]; + } + } + + Ok(HostRecurrentBackward { + grad_input, + grad_input_proj, + grad_state_proj, + grad_gate_input_proj, + grad_gate_state_proj, + grad_out_proj, + }) +} + +impl HostRecurrentScan { + fn states_to_outputs(&self, out_proj: &[f32]) -> Vec { + let mut outputs = vec![0.0; self.batch_size * self.seq_len * self.n_embd]; + let step_len = self.batch_size * self.n_embd; + for position in 0..self.seq_len { + let state = &self.states[position * step_len..(position + 1) * step_len]; + let output = matmul_row_major(state, out_proj, self.batch_size, self.n_embd); + outputs[position * step_len..(position + 1) * step_len].copy_from_slice(&output); + } + outputs + } +} + +fn host_recurrent_scan( + input: &[f32], + batch_size: usize, + seq_len: usize, + weights: &HostRecurrentWeights, +) -> Result { + let n_embd = weights.n_embd; + let total_len = batch_size * seq_len * n_embd; + if input.len() != total_len { + return Err(Error::msg(format!( + "expected input length {}, got {}", + total_len, + input.len() + ))); + } + + let step_len = batch_size * n_embd; + let mut state = vec![0.0; step_len]; + let mut prev_states = vec![0.0; total_len]; + let mut candidates = vec![0.0; total_len]; + let mut gates = vec![0.0; total_len]; + let mut states = vec![0.0; total_len]; + + for position in 0..seq_len { + let step_offset = position * step_len; + let x_t = &input[step_offset..step_offset + step_len]; + prev_states[step_offset..step_offset + step_len].copy_from_slice(&state); + + let input_candidate = matmul_row_major(x_t, &weights.input_proj, batch_size, n_embd); + let state_candidate = matmul_row_major(&state, &weights.state_proj, batch_size, n_embd); + let input_gate = matmul_row_major(x_t, &weights.gate_input_proj, batch_size, n_embd); + let state_gate = matmul_row_major(&state, &weights.gate_state_proj, batch_size, n_embd); + + for index in 0..step_len { + let candidate = (input_candidate[index] + state_candidate[index]).tanh(); + let gate = ((input_gate[index] + state_gate[index]).tanh() + 1.0) * 0.5; + let next_state = (gate * candidate) + ((1.0 - gate) * state[index]); + candidates[step_offset + index] = candidate; + gates[step_offset + index] = gate; + states[step_offset + index] = next_state; + state[index] = next_state; + } + } + + Ok(HostRecurrentScan { + batch_size, + seq_len, + n_embd, + prev_states, + candidates, + gates, + states, + }) +} + +fn square_matrix_size(name: &str, matrix: &[Vec]) -> Result { + let size = matrix.len(); + validate_square_matrix(name, matrix, size)?; + Ok(size) +} + +fn validate_square_matrix(name: &str, matrix: &[Vec], expected: usize) -> Result<()> { + if matrix.len() != expected { + return Err(Error::msg(format!( + "{name} expected {expected} rows, got {}", + matrix.len() + ))); + } + for (row_index, row) in matrix.iter().enumerate() { + if row.len() != expected { + return Err(Error::msg(format!( + "{name} row {row_index} expected {expected} columns, got {}", + row.len() + ))); + } + } + Ok(()) +} + +fn flatten_matrix(matrix: &[Vec]) -> Vec { + matrix.iter().flat_map(|row| row.iter().copied()).collect() +} + +fn matmul_row_major(lhs: &[f32], rhs: &[f32], rows: usize, cols: usize) -> Vec { + let mut output = vec![0.0; rows * cols]; + for row in 0..rows { + for out_col in 0..cols { + let mut sum = 0.0; + for inner in 0..cols { + sum += lhs[(row * cols) + inner] * rhs[(inner * cols) + out_col]; + } + output[(row * cols) + out_col] = sum; + } + } + output +} + +fn matmul_rhs_transposed(lhs: &[f32], rhs: &[f32], rows: usize, cols: usize) -> Vec { + let mut output = vec![0.0; rows * cols]; + for row in 0..rows { + for out_col in 0..cols { + let mut sum = 0.0; + for inner in 0..cols { + sum += lhs[(row * cols) + inner] * rhs[(out_col * cols) + inner]; + } + output[(row * cols) + out_col] = sum; + } + } + output +} + +fn add_outer_product(accumulator: &mut [f32], lhs: &[f32], rhs: &[f32], rows: usize, cols: usize) { + for row in 0..rows { + let lhs_row = &lhs[row * cols..(row + 1) * cols]; + let rhs_row = &rhs[row * cols..(row + 1) * cols]; + for lhs_col in 0..cols { + for rhs_col in 0..cols { + accumulator[(lhs_col * cols) + rhs_col] += lhs_row[lhs_col] * rhs_row[rhs_col]; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_weights() -> HostRecurrentWeights { + HostRecurrentWeights::from_nested( + vec![vec![0.2, -0.1], vec![0.05, 0.15]], + vec![vec![0.1, 0.03], vec![-0.04, 0.07]], + vec![vec![-0.15, 0.08], vec![0.02, -0.05]], + vec![vec![0.06, -0.02], vec![0.04, 0.09]], + vec![vec![0.3, -0.12], vec![0.11, 0.21]], + ) + .unwrap() + } + + fn loss( + input: &[f32], + batch_size: usize, + seq_len: usize, + weights: &HostRecurrentWeights, + grad_output: &[f32], + ) -> f32 { + host_recurrent_forward(input, batch_size, seq_len, weights) + .unwrap() + .into_iter() + .zip(grad_output.iter().copied()) + .map(|(output, grad)| output * grad) + .sum() + } + + #[tokio::test] + async fn test_recurrent_forward_matches_host_reference_on_cpu() { + let weights = RecurrentWeights::new( + Tensor::Cpu(fusor_cpu::Tensor::from_slice( + [2, 2], + &[0.2, -0.1, 0.05, 0.15], + )), + Tensor::Cpu(fusor_cpu::Tensor::from_slice( + [2, 2], + &[0.1, 0.03, -0.04, 0.07], + )), + Tensor::Cpu(fusor_cpu::Tensor::from_slice( + [2, 2], + &[-0.15, 0.08, 0.02, -0.05], + )), + Tensor::Cpu(fusor_cpu::Tensor::from_slice( + [2, 2], + &[0.06, -0.02, 0.04, 0.09], + )), + Tensor::Cpu(fusor_cpu::Tensor::from_slice( + [2, 2], + &[0.3, -0.12, 0.11, 0.21], + )), + ); + let input: Tensor<3, f32> = Tensor::Cpu(fusor_cpu::Tensor::from_slice( + [1, 2, 2], + &[0.25, -0.4, 0.1, 0.3], + )); + + let output = recurrent_forward(&input, &weights) + .as_slice() + .await + .unwrap(); + let expected = + host_recurrent_forward(&[0.25, -0.4, 0.1, 0.3], 1, 2, &test_weights()).unwrap(); + + for (actual, expected) in output.as_slice().iter().zip(expected.iter()) { + assert!((actual - expected).abs() < 1e-5); + } + } + + #[test] + fn test_host_recurrent_backward_matches_finite_difference() { + let weights = test_weights(); + let input = vec![0.25, -0.4, 0.1, 0.3]; + let grad_output = vec![0.7, -0.25, 0.1, 0.9]; + let grads = host_recurrent_backward(&input, 1, 2, &weights, &grad_output).unwrap(); + let eps = 1e-3; + + for index in 0..input.len() { + let mut plus = input.clone(); + plus[index] += eps; + let mut minus = input.clone(); + minus[index] -= eps; + let numerical = (loss(&plus, 1, 2, &weights, &grad_output) + - loss(&minus, 1, 2, &weights, &grad_output)) + / (2.0 * eps); + assert!((grads.grad_input[index] - numerical).abs() < 2e-3); + } + + for index in 0..weights.input_proj.len() { + let mut plus_weights = weights.clone(); + let mut minus_weights = weights.clone(); + plus_weights.input_proj[index] += eps; + minus_weights.input_proj[index] -= eps; + let numerical = (loss(&input, 1, 2, &plus_weights, &grad_output) + - loss(&input, 1, 2, &minus_weights, &grad_output)) + / (2.0 * eps); + assert!((grads.grad_input_proj[index] - numerical).abs() < 2e-3); + } + + for index in 0..weights.state_proj.len() { + let mut plus_weights = weights.clone(); + let mut minus_weights = weights.clone(); + plus_weights.state_proj[index] += eps; + minus_weights.state_proj[index] -= eps; + let numerical = (loss(&input, 1, 2, &plus_weights, &grad_output) + - loss(&input, 1, 2, &minus_weights, &grad_output)) + / (2.0 * eps); + assert!((grads.grad_state_proj[index] - numerical).abs() < 2e-3); + } + + for index in 0..weights.gate_input_proj.len() { + let mut plus_weights = weights.clone(); + let mut minus_weights = weights.clone(); + plus_weights.gate_input_proj[index] += eps; + minus_weights.gate_input_proj[index] -= eps; + let numerical = (loss(&input, 1, 2, &plus_weights, &grad_output) + - loss(&input, 1, 2, &minus_weights, &grad_output)) + / (2.0 * eps); + assert!((grads.grad_gate_input_proj[index] - numerical).abs() < 2e-3); + } + + for index in 0..weights.gate_state_proj.len() { + let mut plus_weights = weights.clone(); + let mut minus_weights = weights.clone(); + plus_weights.gate_state_proj[index] += eps; + minus_weights.gate_state_proj[index] -= eps; + let numerical = (loss(&input, 1, 2, &plus_weights, &grad_output) + - loss(&input, 1, 2, &minus_weights, &grad_output)) + / (2.0 * eps); + assert!((grads.grad_gate_state_proj[index] - numerical).abs() < 2e-3); + } + + for index in 0..weights.out_proj.len() { + let mut plus_weights = weights.clone(); + let mut minus_weights = weights.clone(); + plus_weights.out_proj[index] += eps; + minus_weights.out_proj[index] -= eps; + let numerical = (loss(&input, 1, 2, &plus_weights, &grad_output) + - loss(&input, 1, 2, &minus_weights, &grad_output)) + / (2.0 * eps); + assert!((grads.grad_out_proj[index] - numerical).abs() < 2e-3); + } + } +} diff --git a/fusor-ml/fusor/src/lib.rs b/fusor-ml/fusor/src/lib.rs index 5e326fdad..11e7cd7e1 100644 --- a/fusor-ml/fusor/src/lib.rs +++ b/fusor-ml/fusor/src/lib.rs @@ -8,6 +8,7 @@ //! - CPU kernel fusion is preserved (expression types stay lazy) //! - GPU laziness is preserved (compute graph batching) +pub mod autograd; pub mod cache; mod composite; mod device; diff --git a/fusor-ml/gguf/src/lib.rs b/fusor-ml/gguf/src/lib.rs index 4bf13b92f..c870fbd9a 100644 --- a/fusor-ml/gguf/src/lib.rs +++ b/fusor-ml/gguf/src/lib.rs @@ -723,6 +723,34 @@ pub struct BlockQ4_0 { impl BlockQ4_0 { pub const WEIGHTS_SIZE: usize = Q4_0_BLOCK_SIZE / 2; pub const BLOCK_SIZE: usize = Q4_0_BLOCK_SIZE; + + /// Quantize 32 f32 values into a Q4_0 block. + /// + /// Each value is mapped to a 4-bit unsigned integer (0..15) centered at 8, + /// so the representable range is [-8, 7] * scale. + pub fn quantize(data: &[f32; Q4_0_BLOCK_SIZE]) -> Self { + let max_abs = data.iter().map(|x| x.abs()).fold(0.0f32, f32::max); + let scale = max_abs / 7.0; // 4-bit signed range is -8..7, symmetric about 0 uses 7 + let inv_scale = if max_abs != 0.0 { 7.0 / max_abs } else { 0.0 }; + + // Quantize to 4-bit values stored as pairs in each byte. + // Low half of block (indices 0..15) go into the low nibble, + // high half (indices 16..31) go into the high nibble. + let mut packed = [0u8; Q4_0_BLOCK_SIZE / 2]; + for i in 0..Q4_0_BLOCK_SIZE / 2 { + let low_val = (data[i] * inv_scale).round().clamp(-8.0, 7.0) as i8; + let high_val = + (data[i + Q4_0_BLOCK_SIZE / 2] * inv_scale).round().clamp(-8.0, 7.0) as i8; + let low_u = (low_val + 8) as u8; // shift to unsigned 0..15 + let high_u = (high_val + 8) as u8; + packed[i] = low_u | (high_u << 4); + } + + Self { + scale: half::f16::from_f32(scale), + data: packed, + } + } } impl GgufBlock for BlockQ4_0 { diff --git a/fusor-ml/nanochat/.env b/fusor-ml/nanochat/.env new file mode 100644 index 000000000..099f7d406 --- /dev/null +++ b/fusor-ml/nanochat/.env @@ -0,0 +1,33 @@ +# Runtime-tunable nanochat MIDI settings. + +NANOCHAT_TRAIN_STEPS=100 +NANOCHAT_WARMUP_STEPS=10 +NANOCHAT_LEARNING_RATE=0.001 +NANOCHAT_MIN_LEARNING_RATE=1e-4 +NANOCHAT_BETA1=0.9 +NANOCHAT_BETA2=0.95 +NANOCHAT_ADAM_EPS=1e-8 +NANOCHAT_WEIGHT_DECAY=0.1 +NANOCHAT_LOG_EVERY=20 +NANOCHAT_EVAL_BATCHES=2 +NANOCHAT_BLOCK_SIZE=32 +NANOCHAT_BATCH_SIZE=16 +NANOCHAT_N_EMBD=256 +NANOCHAT_N_HEAD=4 +NANOCHAT_N_FF=128 +NANOCHAT_N_LAYER=8 +NANOCHAT_CONV_KERNEL_SIZE=3 +NANOCHAT_ATTENTION_PERIOD=2 +NANOCHAT_EPS=1e-5 +NANOCHAT_INIT_SCALE=0.02 +NANOCHAT_SEED=666 +NANOCHAT_SAMPLE_PREFIX_TOKENS=16 +NANOCHAT_SAMPLE_TOKENS=32 +NANOCHAT_SAMPLE_TEMPERATURE=0.7 +NANOCHAT_SAMPLE_TOP_K=8 +NANOCHAT_SAVE_EVERY_STEPS=0 +NANOCHAT_SAVE_FINAL_MODEL=true +NANOCHAT_SAVE_QUANTIZATION=q4_0 +NANOCHAT_DATASET_CACHE_DIR=/tmp/nanochat-midi +NANOCHAT_GGUF_PATH=nanochat.gguf +NANOCHAT_SAMPLE_OUTPUT_PATH=nanochat-sample.mid diff --git a/fusor-ml/nanochat/Cargo.toml b/fusor-ml/nanochat/Cargo.toml index 70fd6574c..aa175ec77 100644 --- a/fusor-ml/nanochat/Cargo.toml +++ b/fusor-ml/nanochat/Cargo.toml @@ -5,6 +5,14 @@ edition = "2024" publish = false [dependencies] -fusor-core.workspace = true +bytemuck = "1.23.2" +dotenvy = "0.15.7" +flate2 = "1.1.2" +fusor.workspace = true +fusor-gguf.workspace = true +half = "2.7.1" +midly = "0.5.3" pollster = "0.4.0" rand.workspace = true +reqwest = "0.12.15" +tar = "0.4.44" diff --git a/fusor-ml/nanochat/chat.txt b/fusor-ml/nanochat/chat.txt deleted file mode 100644 index 6c977aeda..000000000 --- a/fusor-ml/nanochat/chat.txt +++ /dev/null @@ -1,88 +0,0 @@ -What does Rust help with? Rust helps build fast software with strong memory safety. -Why use attention? Attention lets each token gather the context it needs. -What are embeddings? Embeddings turn discrete symbols into useful dense vectors. -Why does layer norm matter? Layer norm keeps activations stable during training. -How do transformers predict text? Transformers use context to score the next token. -What does gradient descent do? Gradient descent nudges weights toward lower loss. -Why use residual connections? Residual connections help information and gradients flow. -What does softmax produce? Softmax turns scores into smooth probabilities. -How does causal masking work? Causal masking blocks attention to future positions. -What makes tiny models useful? Tiny models are great for learning and debugging ideas. -How do position embeddings help? Position embeddings tell the model where each token is. -What is this demo training? This demo trains a tiny chat model on simple explanations. -What is a tensor? A tensor is a multidimensional array that stores model values. -Why train on examples? Training on examples teaches the model which outputs fit each prompt. -What does a loss measure? A loss measures how far predictions are from the desired targets. -What is a parameter? A parameter is a learned weight that changes during optimization. -What is a model? A model is a function with learned parameters. -Why do we need data? Data gives the model patterns to imitate and compress. -What is a token? A token is a unit the model predicts one step at a time. -Why keep the context window fixed? A fixed context window keeps training batches simple and uniform. -What does the optimizer update? The optimizer updates parameters using gradient information. -Why does the learning rate matter? The learning rate controls how large each parameter update is. -What happens if the learning rate is too high? A learning rate that is too high can make training unstable. -What happens if the learning rate is too low? A learning rate that is too low makes learning very slow. -Why use a validation set? A validation set checks whether the model generalizes beyond training data. -What is overfitting? Overfitting means the model memorizes training examples too narrowly. -What is generalization? Generalization is the ability to handle related prompts not seen exactly before. -Why shuffle batches? Shuffling batches exposes the model to examples in varied order. -What is a batch? A batch is a small set of examples processed together. -Why do batches help? Batches make training more efficient on modern hardware. -What does normalization do? Normalization keeps values in a healthier range for learning. -Why use a feed forward layer? A feed forward layer adds extra nonlinear capacity after attention. -What is nonlinear about relu? Relu keeps positive values and clips negative values to zero. -Why do models need nonlinear layers? Nonlinear layers let models represent more complex functions. -What does self attention compare? Self attention compares each token with the other tokens in context. -Why is context useful? Context helps the model resolve meaning from nearby words. -What is next token prediction? Next token prediction trains the model to continue a sequence. -Why does cross entropy work well? Cross entropy strongly rewards putting probability on the correct token. -What does a probability distribution mean here? It means the model assigns relative likelihood to each next token. -What is a logit? A logit is an unnormalized score before softmax. -Why do we subtract the max in softmax? Subtracting the max keeps exponentials numerically stable. -What is decoding? Decoding turns model scores into concrete generated tokens. -What is greedy decoding? Greedy decoding always picks the highest scoring next token. -Why can greedy decoding help this demo? Greedy decoding makes a weak toy model easier to inspect. -What is temperature in sampling? Temperature changes how sharp or flat the sampling distribution is. -Why use a compact vocabulary? A compact vocabulary avoids wasting probability mass on unused symbols. -What is a chat format? A chat format wraps prompts and replies in role marked text. -Why add system text? System text gives the model a consistent behavior target. -What is an assistant reply? It is the portion of text the model should generate after the prompt. -Why end examples with a stop token? A stop token teaches the model when a reply should end. -What is a prompt? A prompt is the text that conditions the next generated tokens. -Why do prompts matter? Prompts shape the context the model uses to answer. -What does inference mean? Inference is the process of generating outputs from a trained model. -What does training mean? Training is the process of adjusting weights to reduce loss. -Why can tiny datasets still help? Tiny datasets help us debug the training loop and model behavior. -What is the compute graph? The compute graph records how tensors depend on earlier tensors. -Why is backpropagation useful? Backpropagation efficiently computes gradients through the graph. -What is a gradient? A gradient shows how changing a parameter would change the loss. -Why detach updated parameters? Detaching makes the next training step start from fresh leaf tensors. -What does caching save? Caching saves repeated work or repeated transfers. -Why keep optimizer updates on the gpu? Keeping updates on the gpu avoids slow host round trips. -What does a mask do in the loss? A loss mask ignores padded positions that should not count. -Why pad sequences? Padding lets sequences of different lengths share one tensor shape. -What is a hidden dimension? A hidden dimension is the feature width used inside the model. -Why can wider models help? Wider models can store richer representations. -What does depth add? Depth lets the model refine representations over multiple layers. -Why do residual paths help deep models? Residual paths make it easier for gradients to move through depth. -What is memorization in a toy demo? Memorization means the model can reproduce the small training set well. -Why is memorization acceptable here? Memorization is fine here because the goal is to verify the pipeline. -What does a corpus contain? A corpus contains the text examples used for training. -Why use short answers? Short answers make the tiny dataset easier for the model to learn. -What is instruction tuning? Instruction tuning teaches the model to answer user requests in a desired style. -Why does a role prefix help? A role prefix tells the model which speaker is currently talking. -What is a response style? A response style is the tone and structure the model learns to imitate. -Why can repeated patterns help? Repeated patterns make the training signal clearer for small models. -What should this demo reply like? This demo should reply with short clear factual explanations. -What does the assistant know? The assistant knows only the small concepts shown in the chat data. -Why does more data help this crate? More data gives the tiny model more completions to imitate. -What is a good first debugging sign? A steadily falling loss is a good first debugging sign. -What is a better final sign? A better final sign is readable answers to held prompts. -Why compare prompts after training? Comparing prompts after training shows whether generation improved. -What is the purpose of this crate? The purpose of this crate is to train a tiny chat transformer with fusor. -How should the assistant answer? The assistant should answer briefly clearly and helpfully. -What does the crate demonstrate? The crate demonstrates data loading training autograd and generation. -Why is this called nanochat? It is called nanochat because it is a very small chat style model. -What does fusor provide here? Fusor provides tensor ops compute graphs and backpropagation. -Why are examples important? Examples show how the library works in an end to end task. -What is the final goal of the demo? The final goal is a tiny assistant that produces readable toy answers. diff --git a/fusor-ml/nanochat/nanochat-step-00100.gguf b/fusor-ml/nanochat/nanochat-step-00100.gguf new file mode 100644 index 0000000000000000000000000000000000000000..21d948e30d19d4c83eb47adb12963ff87f742772 GIT binary patch literal 200552 zcmeEuXIN9s6E8&}35bYDQy?H!0qGzDDk?TCAohY00z|41#GW_y-of%JC@OZPiKrA6 zD@s$b0SbtUC{<8IfxB}~_T=!2<$k!&b3gp^Ji5Q^*`1l4otZ6j$mr2iU8N-?Bnr_# z_^6SEL~l_U%n1(h_UG`;7I669{$Am{W!!G)(Ld;4A5jy2NB}n&)rXn+`2_`<1#!dt zxZzCnkb(X^5;X!I@L{vr-AM$H&4zaHA-=$WEb_Yqi>kq0P1tM<`C=Hz=89`6$YTe~ zbRvY?go~Spu)}bz1KR;ZBepSyTy`*qA!5j3^Kq?{Sd2mJLAaL74pZvHn;HoM3SB;V{>q=x44$W4#%~|>>(Hi zvVAe+h?T@=`@_`3C!8HZ>5jt=#EnD5oWt2XT+0;?iq8(gwH&cVz1jY_HdrhgpY4Tf zL&XZ<1+)Ed^-4Ai+fq1}9Z1Q{7KYzvCKMbBh1j4&Bhuq zqN$6=4#F+0*p|4(Aeu>pi^+wF#fV@B{%j6`T=As(iM0{P4#9*y+1y}Ravmf+xxwsk z@mvIp`2?{!xC2s77^Ns4oAh8Hz^7Tbm)H_~*zBGn`X+3#A0q3;!wh0`F~LD-oRn6) z*&)H0D~B((I5W1n+|Ly8AVf?saUZr9rV+ySp=QTlJfJWRI~2DzVjEyj59bE@vdwVc zU@kinH<}`zaOA#}H-&_WhctxEQW1&RX~>lG;sNvh*<4ByT(&pnN7q9bSFFt-b_4`M z_>hEGvandjY%>k=2`$`F3`GFbH)T6hY=gw(=dgV+Ij$HE6wgUW1e=d#k7S2&D9w4Z z1Mmz)1hXS)Pa@bsm}@BNO~VT?rNwNX*l~QtLERrs4K-zfYz}VE=ONR;f_k$#ox?mB zJ)TA6WTwt+EE&JEf%DjYSQ=lpl_jPggruaDgNzWji3k*r-G|NLQy~z|4{n1PP$NVR z5!P8UFmK!vSu_vh2zEGSMoVc!jug9hC_50hM$y6y(~SrQRkMwZhhXWa(1Vv3djamn z4`ByjmGQ(v8Y5H1O~U!&_`HZ6fpzHMzz*g{;%}UABy1S&vXaH743H;QgAbRz1Pc@h zlPfQ>O!jhd48Sx{aG`7pt;J51?x%=XEuPpK`?F``afGwYFqbeiCdyQN*}m9IP)tK1 zjqQlLAi)&7kOkZW%Y%HKT7<^3sSt#A3>KIh+m{_owO`EUP;(O&!VbcXc*wD+2T1*1 z*bI?&79*WterTOGz+v)q zlN0T}KpPykAEi8RvDQb5Bb^tUM|m7iEFvNqhDoA$z!QfG3bwh9tQdt z7+_$Cfe{AA7!1H*AOIM!hCw(6d<-HmSd2j=21_tlior4rmSeC2gOwPpk`l%4 zn~4;y*Q3@TS5H-qtN<;aS{_-hV9tm-eM--1I z)QFx?x-X)%`<#-XcrRX@%8vX)!4t)5H|gqo~^S{bG1n48^0!5RWZmBh?^dn;42m zk+Dx)dq52T6+F(cRIJ?~uH7t# zl)M#dsVS^jOR2hIEv4UzwUnh+tfjFyB_?)849|<6EwBrSHJp>e8uLQkQ;ITuVi+x(qRe3~}3x7`(@=-?{T2G2XR9jCUu9 z@t!?2yq|`ziE$E*PEwM%KABdZEUw=bcZ6!Q>sTy>V~&g4$DCM0)yM8Rj28~^-W{E+ z*!VOC7(Si%<){L8GocWlLV;ZTluscZb)F!$_P9Yf)gUAc*Eu*~WMqsHHyEQ33UTF{JqU)xC`AAD=;{IP=Rna4r|5@qu@#`hE><4gTHsFXEF^MjT2Re1FUuorhN*-rQMja#k)R>Kjif%`6 zB`*jgD@*(s-OAv~aEeYC#X2JJrwOLdl^-QeAaxy$E*s&FqaYI1$3D}R$tW{6t`QI`sE)r3iTi!aO2trc!-W{xkB#I&%~WDRcVj_-xPYnQFK;cRw>LLTKPt9;QbbW+dOrZwI5#QIK3mW`<2n!#}I}H%~8}=5yzt}9f z-k1+<%wb$gX+aVA;)Ni&l7)F0o2hr@x|kP~m@>LV#Jz&3(QCS z816=~G|1j)A%REbMSQKOBt%ezI7E3<#vTz!jnju_Ufggz2yQSn9zHj$lQeWIMfW^+m`g1HV}r3|CTt5l2DWc7Wz~y0 zln4$yQ08>P9gd!6nUN|OEUJ|qbOCLr}M+o(lLn$hd z+6NHVc6f>+=#i8o4$Gj1S z-kb$gdL6DX`-uV_p(Sy8o#W%D}r=IxJ+$D1EP3W4VkQ^d>9 ze?%s}dht={gV+WPU<%;_P5~1S|DfuB|Nf_g|JlI*Y~X)3@IM>)pAG!~VFP$k$BVof zhmX!yu*>rb<%aP>hz$!Y>Ug(6HROkJc)?!be6&}B=T&Hx|A`nkXaUy;_JjY~D5;_j z@DUTkFH(9f3=av0oyQg}qJ_kA-NoFna3qhegRX^{xvq&WKP)2Ho7mClIs``q2Ab%G zqMadPg1@3~Xp2VdOTu*>W-r&}`RF>Bo9LpO1sEG5MC8R0sc_UQkn0s05{6p%astD- zCc1&>OvMYOzJ_)F8XOW9g!YfixZ)1spQpdGETyJ!mqP_;f(Ms20xh`xU z(Y&}!9);xg2@45D_Yt@cdWV4|Pn$THN?vdTSXOw%0+>IrNvo9G(oI#?hw zhNwU^j8K6H7^4Ca7=Q{yVIV3H2@_Nx5~ip?G|W(ehzvpnqGGP=V2P+$paM~`L`{TJ3`PZ_;()|PRED4e#)Aq(#R(ONiZdz@m7%CW zRED7fQ5lX3M8yRah{_05ASxr#uo0C}s6bR)QGuw8Mg^iW1{H|PSX3Y?<4}R9j7J5c z;)V)DWdbS?m5E4gh{_~XAS#nlfvC8n0#Wfm1)?$q6^M!_DiD>as6bSvp#o8vjtWF& z2GTmBG7}Yu$}ChMDzj06sLYvzNF3Nm$jL?s9nhzhb1dqgD! z6^IIoIQEFjB2*wM$V2QA6%@Yg5f$VL_J~RZDiD>$$ZQamNK_yy$l?bhDoas;s4PPT zqOu$nh{_67ASx?SfvBuP1)_2@5tWF{%}dB~%n_lRmjNJB+5Z4Q#ImCSAadFJ03d?d z`vD-5*{J{!(d;w;h-`K`07UrCRRDOfGFoC13;8>ffv|#E=UMgo_h@eu=Csu0ElufumD@nTMqzH&f5q8QO?^808!4{ z3II{g+YSIx&N~VKQO-LC08!3^;ehSugA%~{^Fcpg|M|cjJm8TK0HXW|D1#R~0pY<9 zo*<=KfF~4z>cAJC15@yZ;=KrfKcF`Z03p(4CyoO`w9)$ofDrNWHR}K&>gC&sIz+yF z2gm>(QoaM^11Blp0Ye8rDNou72(c(n0@Z`JlqbPcaG3HW&M4l@S6RGtpT0Pa+tei#t&NC%n0sme289N<>v z888#zS>+kq00EB-kQw}|JOgYCT&z6f03hIz@h?K)Xyq9|8GNlgBNhwnJIvPMO&jOPHPb|-Z$pU9A&jR_t zAIr1ARKX?7vtUlaE6cM$K5)$PEEq5NW_i{*Km?B@Km-qv3Y@e&3#0-+Ezbg}z*W)P z0)P+?^s)dT!~?x600{9wuL=M{JkXm0fDjM#q5vSoqhc+X9=LDCS|pz(_%C`(01)DV zUJ?KVJk}!pS%M3rR|Eh7kF|)hB{*@#+EaiC9%ld%JkA3mcp$5`1V^q|i^gFIo{ZiN z07URWE04N8od($2zYEl zR$>W0jot_V1UxpK00cY^0cCLOiUc(0mf+XuZ2&;PBN5H7C3rS^7XT3ONJf@z3BHZq z1ONm)lF`gqf_J0$001E#=rsU9hzEKL01)Ew<{zYSOYm^?4ges;1HA$O2=PE~002Tf z&7&Tc%a-rAjG3O2F;}<_&Lh_142BiV?bBn z=P2(F2zXoo?Sh}9tUn;c110?t0zXGNe?W)_O8EmqJW$3T5aNLn{(uk#J9hy^ z@bvnfyI}y}>h(MKfIQ&q2qHtb1ZPJOW)Hl*erFO;0Cz_aW(fSferGaKiIQqzMaApz{N?{epntfCyPZJP1dmtsoqP zA<3*D8lbEVAYxE3J}U?Z?BxIufbS0gQ6_f&RuBZ>d}IYNAOv_o2na`8BrAvjT#y3-06zo(BF#a%u>$w^ z214Nd!N4AzpYH_#k%sev6}Uce@UsHfUrB7c!Sx}Z3y%?XHUMz_2p|fc&j-ff_{)F< z_&rMd07B%$Km>4mXAla!UdIRkqHavoAmS)#2UUnR5AEcvz~!wh0U+8aMF?o#Mz%9f|#KkGgKjNMg|B#5FnE9fDt#8zyyT2 zO@Xek=%a)VAjAbF-~k~HLx>p%|L;5^v6omuM1cG|^gw*MXbW!z5rG^d1O$|PM|BVj zg1|N*29VT9dxFh?u^Bu-th;6-QX+&1l%s~GNTMJZAjAlc4*=jz0K|JBw1N0QoN}!o zKtR$RDiL!ygjzv}Aalg<1#yl5TZ1rx67GNy2b5O>gjhi05+GvYOVCD4Ae)CyQVL#D~hO!*|3E(&_-wxV(_ItRZxun;}3jB9Ih9hcRJE z)!gIEwc79n%6hR{JI)Bz(_L_QrbVulO^4A7d0AuvD)At;>=2=RezIzYsTyzA!cjDP)9jgl~u& zApyh^I$P36ts%0Y6gsp-{KkU#5LHkD9S~v&`E!7XB}$(ILTW_Wb3lkCB+mg7dPELx zO+*$Vbq-$;QJ+M4bad4g@kn$e2J}gvkETjzk=U z_{38ugh&YoAvB2ULDHi+#eby#=?T#j_>gy|H|lo!zs#23vOJoY0hS0aQaA2_2>iCB|D!k|eMfe`r| zIFeErh(oHU(-afy4@lvMo|E!`#z}cRp^h8_3UufTtVz(AFnR(7p;~ep2`!M1(H4`~ z_1f$ zV{%f#NYM5OX!%p5(6$?`ZbT(GngM}`O)cE}LR%D0QH&zs6og0u&B>om~tMVMrQ_Nho3vKH)+G%d6%p(Y;qk+qZpC>usc1#%|{L}4Q?wpgG*Ixa9n z`zT`Qgk&UV!nVk90An)7ft|u$3L1;&fFMjZAgr4-dP*E(KLtGq8x?yWauBFpifD>P z2zn%+Lx>^0lkjqK8b$k1^5g?ur+^HiKm*Q1Y7!ma@D#u(Nx{%riLySl2I9!M2qDsZ z&=DE=CED&2l0w9Q;|zQOd!qURMUl?}^Cd(JP!OIwbcm&UF!3Br3Jo5Jp|yZo9Kd17 zTA)hWSAT?N6D>gma@^?Th8l`ZgN&!-Rtr6GB$|t*`HuvYjT5d)Ds?RCMmh<)Wq@oA zh=xw6sFs|4utv1$f@*XW#g>cCnP{%aYC_9o=s;mZwDpJf=+uuR1)R6=Jfb*-!^0$m zNM`|h!uQ6s*zZ0TzpJ7-W#p6}$^>5dc%(1KTpFAa)WcfXrRF62B&hUziK#1QBoR;a@UA znZ=MWQ=HTO={-mCr9IFaX6h7S+Eejh{s#pi9RQSk5ZC;=)4+^ zp5TN$u``CutLQg~VAD;1)z2@KZ;2WGr!5!1OeuOX5nkBCVlM~`Ftaxc3Jmq~^)`#- zq8%>rHY$h>A2#tn`9nu9MStK-?`j7x5(IlK{zKQz+VCQXgarPT1K?_5ZWb8qWofp6 z$07bmmrb-FSrx1y+v9hHp}iF}x9GBY%Mr3Ky}A9bn_G+i%pAX*Nk}6iPpSa8vq_r7 zZ#-?kP0EAcz9f2>g>k(j!otu=*9#r+|DXY>2$EbUo|ahgUFm>zgXGysXV$+Sm18B@ zo~{JjU#qyR2iY9IH48%;Og8UY2fCs^(%g1g|&Ci@WX>MNQbx-0y~b4GSyMYc44e`7_NKS=)c z;SR>`{rlniiX5xc)SCZY-$qfO?9|ud?}GBODAEK)Z(vFMSIvINchKF+A9VdLn;bY@ z*A~AHJ3dQ)ZvGpyFL*&V*Cq=>Mhm(v{GE`mKK;tux(-D0%B0nIiq>SXL(61W(K=YI ziyqV@cGbgjq)QQM>71`IexixFhsxdfoA{>4&8`*!CPq}gM^{5MFW9=)7X!k0le7Idvv9LWz8;T zOf-nDCt!U#sXoksh`~gUuD2z6C&~7B3`Bbi?ATq&v?`A5fuk+agIpB47Ae!4)BwE$ z;ZByn(2`{x*@5o#7XLMYKRW&DK#~f5LT&NDejn|m6v!d|-1V2Xa9rhQtx$W`uGFY8 zp6oz3PhxM=^*mh{{m>3wBx;&|KbrL-pQP*G?spS*M>H07Qh`K3qr2cARbTN-VSlMV zqtzs{PIf5TeRpkE7evbneORImb=Mt~L|bb7cd}r(r>$Fxa%B4%H7FQMA5!$Cy)}wd-`dF?%RJOv99QMyyyn~+iiq{7db?F2hkC$ zYbiC7$PRSn{$$U8BWlvSUn=*PGg}rw1esfajC-T2e zlR+NL|9Dk3O>~h!w{iNtz+Xq(i95&{pleaIljvG0F{0&}zDc*Bv;N}(?<$HSosK*| zEfs$xQ!Dy;yH1`!yVv-=C#1dZBKZCGDn_G=>nG7pv?~|ztRoexDY9^pql&N8{w4xl z5dF+wCletr8Gh-nfugItPG8CpShV_p&A>~7=jBXQu9jvdX zlAU$Qg1jW7o8(^`x0&c?PCH#}(^s!QE8aDYRD7rS`kMpFby3jxIWqlmm7gez2y~^< zG7`U^;pZYFrH5p?k^Wssema;OEnRw=FLz}tDM!d@>g-NLZi|!{D6+{m#BKJ47vz^t zfse-I_v17XrIG0Ds9jW7^2La(iEjG8x#S8IT`th+lfwPtrqn|e!RRU@!}uR3yjry5 zq)#|`$^I8EeUC>@72QYaq41AB+9;X_`bh2YrB_#)+-^d0?BthLv_SaRe!2S*NvYFD zM-dYBR2!u^GrygW~`qleR@5_Xrh5qf8%Q(?=(IW!w@csMHI7f7K zqC+wz)=m0U{k40lIFnY_>5M=(yT6&R^=C;UbT^@Hp}U%}J1a@K>7Gev`e)DFzK5jH zX}PBds6Uf<^pupk6AAjp`p+chHIpRhhG+5D>-bJl;thw!(2#H*@w*nk$WgOJlTDzajI?+|vzq(kQC)$(KCt7ro({(&_5MAZa#iU)XcV!zZMAs0V4sX9r z3|04*B1VMuDB3Xp_J~*@O1aTTB05TUJ(8zKNUhPQS`^v;BF5zOCk=(J2kLJ0_paVL zi)7qsDW;q0U;DaN11W?KNstH{KV6FdjlVm0CkfG&N9}{Un$rPABujb+(S2Lj+K%q( z#_cx;a<~8FTnQ8SZA!ZJFgXVLYDrhd9|QiXuHFz5T?lujgHxguEZqdCv*_C?Noo?H%G*l9%|{U+nr43qa?TEf1%L?(bl}vCF}pa z(04A9_D{D93;F{3M+bN$x?b$0?%$rzqD5D;o$Oe&pa0YLxK)&R=)~=}d+`CHm_QFY zqD#82gAPk{DDR|Mx=Q}e96a(!&33v*_{|N>bJ55;Y2_~?gPG`UA9_#`Z3F)FdfdU& zq}9?rQ)IPWdFFGG&C)ma^i}9@){XeApCbvmPy7qJuj*pYNEhJGfz07nqC z7Qa|@x})y_EdF``M?`PPblPgu&FgPesFXt*SEt>;Pu~AGlw?v!N}U|)x9L|UQEIA_ zp;Gbn_sdX)=&p=zK6Ksxjl`*A$@!xjJ8g03%4)ZYE$> zEpNJrrF7f;qh-w#rJ#B4Q71s8qj8PU!gf} zM^F7p!xIndRObn&U%V7*%|Fa zrS6vRdS)Hc60O|NUEmb-wdJBI>xqm?#p6dB)~qvvSwrqvT1E_3UUYWefNNhh9}H_R zKFe`s9d7t$;j-l|Ma~DtUW;PW#)z~-VGkl| z7T!NI+Eib-FV1QD$j9?`=M~@Nu|6=;qT;50O4;AcpL1%G)geymhM`Vl=X(oNK6E?k zB+++&(iT?v@Y8AbRd1U7TV@9gT@i7qhS6l*eDJ$K$@ErU^V1t;%UGH^+TT(p46Q7) z8~DZHfnl=Sg>s`z1&&^*>7g&st>D}vD z(Ibhbf*<`B$Q2xJv$MR}P?LZ1iA2XE_r^+hf&Ts5x%mw}REO)0oFHc-$C3(AZ7BOP zq>iN;aNlu?j?S$Fi%Z?#o(vt*FL;&hikS#YX9>Bw>*|dDVRmPVuFR^EmlO)wiKgEMb~EjDzqYpgV~2f7rtA%tYhi_| zPFc~rL+cm!={GHLLd(qlvgao%|B!Rx*b7}#-Tq-&-|m(uap2YS{ARTwN$J^2PCu?% z$*PtKU443}T)KLw$dWav{N_l>)3NeV!d~sRn&T}W%B+_e(sIpKZEVejkH;%xN?GG} zzG!`FUY!#yv{8)G7_5AEq>|I)LD|x>59a1*8^+cRWj(lC;Wz7F$-=AWT0hA~x7*#z zY2~&qG7y$6y`lEtuEv=J=7XE5l1(qAUfk3OxZ-~Fv4?wY*{rb-dka65x;Gv@ZGHC~ zYf4OFziP|P)5;IFUux<-tIx}A$%8bPJ{)(sjCD?^pjSMI^+vWP-a}c&OS*oj?2PYL zwR2Rzo|-GTe?haYD9_G*1k4|rS`zHf^HUNHR$i&uT=Gk zsG8WiytETPgmw1rCJKt%N*_s`m?L|u^hO< zS&vhn_j#qL!x+}MxMF*8RdIHk`ACngTMs^N8QgQZTx(UTM2D8ySUI=uZ@N{#m3f(e zsC^LINa~@g$Au`&q9l3xs_Cl~aCkIG1G?KkdpqL0q0&&0y7I#pOxYUOAh;)&4L* z$No~YT+J)y^l5fV<|hhYTWRX+IE?;~US%{&Q1Bw}-6e@$C$)oE6_qj@n`ZO~&z6f* zEn3ufy2kz)HU0Bs`!{9n9iBBsFp|rXoFm)wV&8r74c|ReC0;PymL!{9bre_+-nfFj z=WDW;^&!^e*Xr3C13r#hZn41dSu^)+i-Fq!qtycja!h{+FC5s?epoK7q@(+IIYE>; z)1-H`{j*{3-qjDwcN?OnW7j6px5P3pOKyBkjd}OhI72JT$%PuD6gplHb>c}E$*N8? z%@!6OR-1U3b-%<;@X)b+^5g3MHb-0X3ndlq)V!>!k^iDytzz-r=(?kP<)bn6ZyM^2 z6m1{LU+$Qo-RiyL(H_>ChJJUbN8Ks43voN2}MlBfO@@U%Wi!(}*(nBTUnoETuUE zUv_t4&#juT`dT?ZVPDnOYcuXI6P~~F)iKq(=V8w0qIna%HATnu<2zkFUS)H*bOC_r7kiqx+0_UH(LP-&A$L)M3)knX(f( z{hBrljNR`%n$V({J2P5Xb=6Pa#I<+#+lzJfoXyN*6+EyrZ+zZ-Ap5MoY+KNXMy=Ls zdZ$z?z8j8dt^T;RQlQP8?Bd<5bni&_4TCa*`?258eO~A|Ylr*Gnk?bh+4omown}By z^uO5svay;{)QdN!CkrpF9w%R~V9@qjFuvV3u0T~&aVYbpk&Cu-Z;eR5J#uxruKv?sk=2}O1c+oG$alF#!b>=G{A9 zoNV8Uo2`$3R6Y0V<$J5DafE{4nuMIE|2^Yg)ytCn zFG9xDSuZ8y^^LELSaPd*=qI^xL$4oV{YW>m9IE#~qfjkwj{LjqFAtOo44(n_o?=(o%l(;?hg=`c^ZiV&FKnwSnf@f{#B^bgCl!UJ*LKUTaZu5pG)I^x zIqcdv%W14JW!`a;HTOpww>ciVDmP5JM{l{;ybtv$XVk679ycs_^;y^}RiQ@aTCnPP zqoxm^R;)ZpH7>oI2M;KB81V z(ZJ~22toEGYq_!6-dG{Stu3xdS%`k0qd%1)!iHucmyY;LP z^zpr8Yx6YVpOTUVWsHc$^wFEQ%Lce1^ryg{Jx zoORo4f9a&*##Zgg?-cV}OT1Lu&*zS-o7tf8K+;iBL-;k3sVgz@?bE9hB?ku?k8M+a z>+)WEWYJoErPL2IoaY(}b6g_jUVWACR@TauuZd6j=vH}nVyxV!Pq}VOys8Jp=t&7x zzYbz87##iiTSj@6H#2`tN5#k8>>`=LRsHnuDGFn3Ru3vwRbV}>Rh?cRojlW3Td?w@ zX16_O(;rtB&UTo4W5C_7V~4IWf5$2;b5I>&`r1kPw$QSp!~ONGSGjNW5-e_LJOK)@!?|bz8pAC-IOi+A}EZ`25&!xbpDNjOKl1DmV2We&6?~ zUHG7%LC(I*TDr#ut37!&W4m?qEXhlyj7?s~X;Zo#dpz*4$#J3eVp*$mFFHJArtdhk zX%&B>aBk%x=UU(aj#?u?kq4KS24~sY*U{bh6#5JEk~z+xpdR&LB!I962{E6<+{vsIa>br zGweR8zS3*m8!8A1y}oS6^{kt1jCVC>MsNRjt0L3vb=v6Y^4YO6#(i84X6gQrOVL`o z?SjzGTVmmj3SnS<17oMwp52V0H#XLmyG?$vC?_MZNKo#T@zlg}zWb(BuecWs=AdsM zR|y#_svH;9?!Q0QK0mg@FfozW*Ux17>#f5#URBj*YU~r5uby0-)IMTG@x1b+-WMwk zt5rRUc6l1*9Vy_h2~7ywu+{e3jp)>r=JOj$XRWanmet?T zX0AUmN0>X~fzQxcD`dAEPGU5DuWVh{obco6Rmp7;YO6~On6X7)8n4&%Z7RI;ZeE(? zzw4vF2Hw6S=&w1dT8Cr1LMo;&Q~vX)`xc%a$Q|7IpMd3$W)z)cAg~}2&_LGxO z1y4Ubd2$ouuu@b*!=f0owKu$Mom&EO*sldHr(Xy=SXKXq~3(O^0N`nwlBsXNSHCRd#dLR_IpniXl5OtR_?a z#nmOf7mp7TOqN=`ecRIs3-`}bdNGWfQIPscgSlc|;i(7ml3toert{LS3Ul_)8l<_K zoz(EHUAwAfAU8vkSu9bRC>Z#|(|Y@&pu3m*4tRfQV#@RNxw6NHz8^Ab$e!g|%xM}q z`)`Nzn|HqU>tJDidZ6XZ&)$KTdN908->C+a-CE6NnjZ9$Zx23yu%Tt#-k@Jvfc=*d%x?`}#@dtAuy_M%`t*&Ei3FnDz{LD3|ZbMF@CzFEoxrXX!%p`_f#CfD#j zOp^hFf}0-2G)s>2(CU6fA}S?&iN0}*V0%uIYkBVRi8C~r%R|k=mu~my7B@z3T!3&$ zPWq(@S*<=+M|jT`6g)q;ccnH{y?%Er+ejvMb(NI5MeJ}h?VbW@$C%;ohfd)SxiWM5 zI~G&k?U7)Ix9#luD~_vcY}dViO-+*~GGCH;>KIFp%gKE`_ ziJ|xI$CL}As+X`%ww*ivU`WlgdPd1hSzqNCd#)XSO_Jcn#=(NUQ5L;sPw2I<)S`6i z#6{kwOyM_W$$1IqCDzU~+Vf#ED{NQWN@JNm5f^R@DQw$wBdV~RAzYAu%JtQ^Ts1?7 ze$CI~q76-=_n))h8?wdSK;o0+q=b)*_TG7d6?z#jW5?~z9l!j&!{cEGEHX;dY^~}a z^!Rb|{G}8|THb==n?faQ3cEc`aQU`u^K_rxKC2&kpP3+c`fjnMfK$P6=GZH<(`EBz z5;b>kH;7(X6&M^MOiYznlezQUD0g4k7M@(!2OC#nQx)Ew~` z|LWoMt(jXsFlK8`>s@wxhP;R4?)*o0Ql1J2*^I6F7^zw_`o4kwlAfQWn1QEb4c~pc zp?1?etl;1>y}h&dmkQ1g**_}K(Kmc|*4D`e%!g-|)C<&nPNmm>OU(-WSL$UuUsK}b zI?v!l{n(asEk4f~iO&0-9|#BBJ=msrXVs!b4cPG>zlb+t2tle5AU4n+|B%b_iw$}{&~9|RO}7&mSehp z&gXB_wHy51bhdf`$MwpHnHq`%;z9*uJk3f2KAb61VQzdYb-TbzFhOBxQ`&-mD{3WL zUQSK3nx#bxe42z#MI+ zYm;WgJ%zhrUoV!bcC%>c-u!6sA*CYWNN;bQfRARotn`@g<$BqKnomg>>2fvH?1Oqh z+Ag8C{()&18wLs6B)I-BWtmfZs*H_WBz!sazA68bCBxImCV+Fg*VM<;6jLR99pcZb zGbe3K$=R$l&!zW~-tUsFQVib83WHgLrAyaTHGD1Dv(f4vL(*(0vW6yS-*^%sBs>!i7%(6p;Qk-jyBX1ri9X|egyx`K~ z$VE?fP99aULS?&q_i?3p6&G!iUEC`Ao!Vnh>=IWMe7j_BX=^&P`HuE)Qz?*%#G$8WBduabFb7H5*b@z|Ifn~y5%Wh`51)|=^Z z!Psh8%l5{guQh8we?D?6YY*?s4fXHB=Sz)0$UW{q?hWIa@3pUHHn#$Y2QXj1T*24! zIkh@=pv?7Ykv%2}pQam?$uesr{r9%c9(J+))+Sn_GaYpogKZg6qmxJ=_! zFKMRvwD^Ih=LJoBFOLqYRT#1&(SOKY&94Uef&EoXT$CK|$}qS0jFo<*y=}7Vj*GR6 zZVXT7ckH?I*zvhhzzSd6s3L)cF0;xZ{y|O-XGZzryE($elU7TZFothhv(e?*B7dd$ zj?Wx6Guyex{8QibGne{~edm$kw>E&iy5ksc%03<|LeOp5MT;DU=1|v+RjiwXKc8t$ zdYNgxB2rOhcF)t}gnlJun^Fgk%I?7o|8!a=)i_|(cQ$X*R9lBs!< zDguu|rPsUn9`HglhWBvFnH=j|Rpzs0n7*@AuROT$_+G}NQ;g(-VS;-x3lB{wYd+?| zS+UchDdT|>^YuXQUGs%)%Nn|^XfM0@W6zS}i*pW}MTZRdw!D!n2hFJSGeBPVQz! z#*T<-J*E(?k=wzzQ?l%|z~;nq?FqIS3wI3EEAk4|DKU)8iyb(O_cXn(UXgiq<-F%F zqu)PX;^MA4wzgl*?R9!5qpJb}_qz$rE_LW>*D-XHIES3oK3sGBcEMp!xbKNj_L%;{ zv?cRTMLwN1=J-1Y4Q7z@^r<~vm*<9W+kNrK(^K&?wxnin37;rX__15TC3sBV-puXy zh4z!S?Hy{XeGbE-k&V?x>5v zWT{h3+^$2f{6Dh{XG`6cU@D$)^zGLtqU`I$k#A0&>k-aNSy&*98+yt`AuBmyk+N?k zBX+Wrj#;AJJ+)_oTT3U&Sx$2Mc%YA~!CRYthEe+so<6Q+jQ7hNVK}5d`u;p;p3~it zmkc}w<-0oO>D=@Wti0A&&E`Bq$$TUKYvPTvNfV~>zumo;f8Z!9TFrBOj(p?#OyQ~; z898Re{YqUvbH|aYISaou9^e0MG{f#Uq1Dl&>3_@@ifhFojQ~b`LFGRlCB|pyTzY;6*Bi7vh6n9jqF{+QJLo zv@5NHv1_Td{-ht*MbvAr+%vJw| z8}}XIMfPP*`+BqRO1-`7hOxUFuX|o!Q|nWj_cAxpd}6!l=@J7WTZOrO@{x&J>caLa zjh$~;O??MS&+Sp_xGKlKC1AKy$>G*vRgC`ndge!luM6oD67MX$?N|KV-Q&#S1b4gp z2aS#n^7vFP&s;KKQpAlyFJVcr+xgS}Bgd{N8kgpI%eVaQ>obv0IDKZ2 zBgoi&z}*puIrI&l}%&Y82Rl6y6YmHwhr|0Bkk(s$aRWL#ZfbTYlaRiNEF6ywOu`Iz2Hz5%cuC#v^AehT}$>Q4(h=S98_X3 ztgmqUzGH%Y*IxT?n^YZ{Y^JHZ?1TKU#CqGR)@p`y%jiG>XY-wG=f0=L+&y_>cH=bH zt=b(|Jle{RN7)${{a_fKe{yNYL#;x$t+vWXKh^GR>{~C~{BncZjFF>zSA3jgqRBjZ z{@rlf!N(K?34(wB9Xf5^_r6Uo!-^h1z4JrsyTRJB;T;UK4}vM1H>>4Em2vkUdYDQYMPyPNmd)8dNauY$b2D8zOGd9n*XS(Sdl~-pD z3wDJ}-rPgkQZ>RyXNt_&e5VW*=Eca#^S9WzOHF;Ra!{*%T&suBqV|W3bbR!JWql27 z4{K#JIL*&{X64U5ttN9%xID8*+2A-wNrULuR~P9o+n+pq3U2^2U`*s4fsLx=4|N~6 zr~Z@I913=OuRO$}pTxoY8FMz5$2TyxCis;HHrN^a3k`h6m-1(H8+7Dwzb~UD``PMy zf1kf{^9zRKId|3dYo+;DUaeZ^@N8f9;+3ia0;dz@!t6b9?Kifmzh|5h#hZi` zdP&ON{u^%_f2kZ888^7c$-x_JoVE`WY`w?0zU*Z0lP}AsEDBkE`RPj6Jr*&is#C69 zmht2Xm)J-5+=AcK;rTyS`<}vC((!-SmVT8-Fk+CJ*!c zS{&;tIo0*J$G|KFS59xO_$}9~Ov`%2yiExe^k&+8zE~;abt-MZs#SZ1{x52*hp&$i zx{u-p>^t%Be%{uZTFmls-0h=wOtrpyVSfJ)x+Br^g+q(-I%Si@1!T(44kl?kv;Q*ry%81 zGJj?I`iZkg)y=$-xwd(vH|dG6YBQr)fc;h4ThU-ATJ`RQY} zGIF-Ru?zqHETN|Cv7oQAJAas-g-KqwAD>2K^jqwiJO5caqiRUl=l;%uw$`;pEv=!K z=JU^&KYpp0-6J|B?2C7B?&w^Gdv>r@&)If@CyU-Zz8$t?*Z3!!TPt!l9G+F)>^^hQ z^0@9ytz6|q{gjd|=k{vxz0dWu(99{Ce#v5#AUfr!#FMYv3O+Ipd{mqzy@1Q;^JsaD zN@%yTZJ)yo+>Z#)c%Ch7eLg;fH=T2U@YCF|#q%bkxTvq=UMoRdk7xUxza&;4KD};hGT&1z z^Pu0QV_NMa5>%N&gKw_`&6RW++rG~jviZfoy(>erym#GM5~28M>imgMt3EI;@6am$ zEIcS&r_ipacMRe1>YA``EQ>)3SL(;Ta=g>~+>t(}>Q-y2T}&pHO)y7qLX7y% za9ohrYsazVaP@A(R=*dNvroqwEXj^&^Q_s{Z^~D7rhZ2EJ*<%2+!49!f^UqLQP|dd zvXb=RPr~xe($Azv-hb83@Rt}OI1n1A7IAaj{^(0b?X6Vx`V1P@s;yOSRO;or%3p)| zUT`JSL14PXLRDxSK25pTd&habkJWii`*)M}zQ}9y4(KpjrwZS$)pj>*9pzmzZ``LI z#$TUr&%M~k*!^K;%(Q>=HuhxJX{b(IuOvM8j=e$OE#j!gfU}dO>u+gj5@w+&pSoos=KtiDKf)0=`?0o-7Wg&x9v^K zJI%NET<^~e&Oi4(IdWEo z&cX)oo99j(YZk_u%6BBG9SEN>qj=ZSAy*hmL8;sMKFVR!Z?ArF{91kD)8rgq!OJGk3Qt5~lCB8+$1#aYkh!V<<^~yz3gzRB179} z8n5Hg+0>8gwxYLR=Et32KR140Ja>yQ$$Pu)f#=JgsU11tbk%xB)7cd+@27uatlj0` zr=pkA+q%Ndz010FJlZO7UO2FSg1^E<`8^)Hr0=kqazcUJ-mk3>Ba56by*yoiaqISx z;VO$lwj2n{b>AT~xk#RQP4HUf>M|{9H-iiteuiQd)6W`@E=qsGZdhU6{DQG} z!SI3wBQ*tj@x`ZX)1CJ9`x2bokoeujamj+rtg5WLSDC|mE_u|}#9k=7@Ik4-Qc`Zd z*I@UDZO)}l_5Ax2DnoRNS)W|=nJ@D0$ttUwD+;YFPpj5{E*@2!Hzun`Z#5@}wrFkU zUiM1WdB^8CO|g+(`Es~i zXrE|3aH^D>vQll;t?PDg)J^5S*yUumw`dq!a@h_QHkO=M`|_s=E)2^|wLgEEeLUlU zA?y6@g!_{Hk0ojk$h!Rg@wY6M5{Jv%`HQ3H-hC)kQQ!A@2XmZYY_?~$R$YVbEw{`Q zpX_6MzMt|<^~mkJt$iF-K3gB1#(JT-NH#;IE~dM&d;f(R=NiK`1^+gE$xDje{X)mF zP~z1aR?ny1rL#}!Wi^Z|9$i#3?c#VVnWKh{W5){Was!5ck$ka_<)~?!Uh<(~r~NRY zs$9-zrAKymx)q-t|HF3gUPimbw6xER>`B9o*55m|=lc$~S8c)zX}9u!NbV0i+pTQy z0KHFd#*}-pl+)hKG|J69I#R!T(b3ei*%8toOf{?CpKGp`66(p787!9@*w!QUn88k# zXPb@ekwW2(sQ21K!_6FCN%VB7c9R+|A;&sWyFziaQ7Lnm$%pASyOwnK_jFT`Q4`*j zIJNuHfIH`AO=rzX{PgsFLt%$=cZmV-9M2XCB>Pxitx4qexZQ2~L92uh1DPdXVk(~G zNA=7W=$OrW>S%S&=oFnG$EQuj!pU8)P9g?5?DZH&OM}O!|Q_BZS z6kFP>>L(8_6Bu@9Rq6G0Qw^}QWgpfR$j&|&KfkuX%WbTyeU!;{g+s$jSWD%m_t(8T zsgGQr(>mx?hKi|^kEnb(dCOnOJYH{CC37#=*onC(t4h%1eQ(6fQj1%fBf2;48KX3; zX{zetIjK$wH{=G&bZ1!<2n`(uPPo&SdC2rqPqVsVrK6-z8>)VgecQ9`-L;RO7t0x( zt!U9dJKoS>pzy=tj*n<}Cu^%bo`;7!>yohk`X!a!JG5?ZR-N0@%7a1DO@55Dy$)_Ud~bF8~^qdE`uM}a$kg_PUth$czjg1PFJuye6stBykqMIzqmfnPC*P!=d+RwX8`5VgiEEb z`3vspO@>XGP2)alE0`p-yMR){THwydBeCe*l|?jb1J#laepoVz)J%g=y*S8#m}rUX z@l73bGnFsF?WK|RGRwoX?H_%x)V@Z;0W%^0xgzN_E+vgKH;PKv0l4Ei7UB_QxAIN1 zY_{TD)Y8Euz0Yxg14>iLK9N>o?bimcR8MKKUBYQ?MQIAc8_V&lZRA4L$@$9YCLQA-Vkr0s2pn5dULgk9iZxHiQYsH*ls4f5fY~J$-rXQ60;i zhTidj5nRG)t9e%SQ{6g7ZO-6?RwP76lglfsjr9=Fh+Iq#>P}&bpfOXRmMLr2G7<4c z5fP(iSGM^nL+Z!Yi{-koM7wvQA+I8D2x*Cl{8tq!OKnP<|_|KPN_AbI2kh<#W zVG1uV3uvz0_}x-3_RSuMeQ@*dmAzdQ5`N#95h7=SmDPr!RhP?jlHG|J;UQE29w1E2C8YcH>k~0>`U&J zHrOtw_WuF6O?VBtKT4Y?8sf+()%3uMP7K3mImw~JzamE|a{6Q%P|Mtw8>3etpV;!q zmk_$gVBxYHM+%UVgVThw0TEpyhaUceM9wz&ZaSH-ozr58ZQsU2r);K^x3;gr`js>y zOj78LKawqyO!VjNVhBx+cA$sBY~bjnYa+*jJuN@fu@fncmX+j)OOIW4?Bd~|9Hxg_ zi;K@s=@(Y6;&PdMJ7$8Ncu>}r7dQvOI5Lro$;g;IRS&rdC^Cr6adl`BHmx~7ll}#f z>gt2!BUoK#U)(nL$fJsh(Z%Jk>})=FR(yEWKs>&%)MxU zY>f`*`5f#5iUG8kez}bngG(SMyd$KiO;JROngu;TYT6hL9_ACigR@sCTf5}1A@Yms zCsW_T0rC-4+cf1V5$sEi% z7cFh7^}wT9V?gotuMGU9v){ydl~x8iUz?^ip%`$O<6hd8!kObLfN*Qty_&LI@H zr??*aMj=qHcT1nK=<-QBkb65MWjN+H=*fH8`-9XV6Qx5fw`d62n z?_~lSvxBSsmk=7;kWJm)Ss;5|bgZhi%m0_Qy!-LGTE5FAFw@p@<}z&JkN&k@;u+gD z*)L+yh=XkvdW1&89hnZk*aMY13e5SYWOfVTmNV&lmwjf?AQEfed(_EbrYc}8^XmkJ zaWd?v1FqQe4Ba`cj*hEbqO0EX65>hD1NphuUnn~rP=4#4gcj%ZGgJc1^?JB*j|bC- zPTfJ>F3VOmT=(6vNd{?;D_GxBZr-}azHKDAo(QcxHI({LgvrCRs^+_8jCVKy0tZts zPAQ*sAsZ_e?i*`r^$5Tt!ju4+CxBigE&T_>BA`X0gVyB!XExu-E?4mRqCQvwE{KrpYG=#0 z;ElE=VNGLanOTSS9|0UDmkJ_Pg6E$2IUwG;d6T8mxbGjvr(F$aP+C9#Sm)^dqa?6W zZhd(0>IXCmud@x~6n!+raQM!MW|gsj821?eOhmXCn~VpQ{~?(wGYsV#9@d5Mbsmnnhn*d^4SVU=I@p9WM_TkQ1QQK^GGw0nHq zM4&4MW#_iVdd9U{@FF?56Y(V}Xv%@1PkvB4Mqn1ZW#F<(1UQdDc8qRT2c%%Gp@Wi) z=ar?2x6)7fg2o<-3C!=!{{o;@4d+!W)wj`f?@Scv47pj|eEB(JYQ+|KUD6l#1P~QK z-lad)K=G6SRU9+oOZ2K8iY}acymRi7>=*!94HN&|1X>qGxf|w}+GA>c^wL>ZzriIT8kIOa&`3wBDf zUtKnSrq>gBQl$jillFg)%*LVYcTM+3ewdRT;R{hQW%6kX%+^3N(?{om>S6e1+GMeI zDT*{!H(+YiG((&EPmzxqy%Jd9i3oNPCp3x8{QUw3+7jDaUfqf0o#{kMZSG?BoY;o7 z9<(a@V1@vl;iZ=1*clobsqlc!-FixR-K8J9#5(FPGI=Zs73zIM?=G~#KWRs0N+g*; zJ&81Z*#R#_`6>DG26eL{xy@_dMrbo^q+beA*{X2p%P-W?CX(ZzH(%I;5+^gYGEQSth*ge55 z+zoEEZzJ?dryrMFCj~j!G3%NlBU^Hl+e2N!D-enT%~s5H(~F#VIIB)IPR+tiaR7~+ zz3nUK()v3gLLaWx$AX6ZMc~1>7)?8mi4iY<>84;F)ZP+4HohRM);cBahbToQw+o%S zl#+7Qfs6HXe(z{lU<$=EPr9j{E6w!C>1&lZupaisb|~v};GKuVlN~{i0sNc6_ag(> zRVmzca~oSuVM$acSwcqdgb^}o708RG*dbF?FHsa)@NmBA&|ca_tV||@Gn{uNc+{4f z#bgi8dw8HUN*C|{WYyoQRj5@drQsJC#sOs?`fFb9t`yF`fz`3ivZo?mePcW5mn<`$ zeRml6rr}dC`j`tHX~=x;K6_$C7`)+VqTkISKP5cbq5_FquF6WF(X6xf!YQLzs^2>-K~Xl{ zbdK(XrNFDXCR>cC^=PxqIDa%-v2w@H(G|0RMiL@K8LsKN~!pv=I`K zyq+_sG8=JJDUly!=%#_VnW|EjyCNE{!h5oPxT3#Cr3K zF$~GBzmSIY2zVp^$b!ZS6SB#OSmWy$C0clw;Frm}OY5Iu(RU+?vD)B4+C@Fvl@0Yf z-98A*qd{m)c#6`kRdAJKgi_?yhV2;iq^a{v=X#_*j+b z^vijlpvz>&MY)CJKgGWM_eXLJNQ3f66n(6J^F!tn%R;vPNE3{JV?In6Pd z_cnGjwC`hf1jeN>OlCC(gPQA>UC=@a{YAmDc##h<>+`%7y4Rsf-XWF^NoG9JYq2Zs zU)aP+OcrmqG=#kSf8e{|i^9Xe_vM>a-)PtNbfNmHL8(u&hmrz(vfXuMlE!WzqJ_P=hK!7fiAZh)Y^hqJtWFM zEy@pK91B9UoI$yv%>p6u6DW;bu|lbTpTCMKBL08mQO9qLOx!~i*Q+%F-~0@gu^JJg z1s%-6YGZ6CjOk&@n6@c5(?n{jkt`M_Q2UtLEpee3cszhf^k{jTHX?+=wfC1uzL)EK zvE8FP0x0xrcsqARC1P4WBuoMbhxDEao`p3DGr z2ff*-#<>;&VVgw3(PInK^Jt#F8bQiakc@1TGQHS8*0_vW*YdEL0n_o;^G)JFhwS*U+OGDDiYvc9YvrRT1ArB z7f#=4&9|d5H;Wxau{m>RIv#IiORP{T2&0R!VW%Aogj@MnVT#PXtXhJI8t}?BN9xjx zo1R&%2svtaEHIKDdInpBrNzCO;W$*E*QP-)i!XFQq`vWU3g&vBgY%XHZJ@PniU|hT{lr|{N+%zLok8#Ub^U+@UlHD|JZ~~rDII-l$ColFhJXz(IozpA8a(JF|dvjkKeYV5jy|whnk`{jrTAwe+I29kvbKjUliDFOK z23Zl@{TEq)6fMyfQ{F;i)K|8i6jzz9KE6ctq>mijs z7~fxaT}hpiDhfrkCdti>Ob+Nm%E!pTC%)$hx)z6I^-31=KvmRXysDKJ(PE)ldEZMvG!jX8SdnSMgk45g#>|D zBkmGNOU`yO99_gVjj66GG3nPG&k!Hyh{^bVu3v|>e}v;0(63nMYCZ|EDZ{ev5iEjO zfg9;ez}s-MaH)CwnPq?H+oh;c@d&XdV%gu>AC+WcFuC<~HcLey*s*Iy@V{-h1)ka2a z+~n5w(Wc^*8&;hFb1JX_siu6~h`H+J{Jbfw2)Vn24Attq_8%kd_dZc1IJA1q0AYE8|CNTaJ>2bN0ZvuQEIEuJaDqX+Hu)I zrkUm))a7mKwP|qVk1NgC#E~dW=1~$8QrMC7V+I4^!<+>h;{QJHbFQgVE<(iU69Zw6 zZF_Unu?|g4!{hIkb3L|G3|YM_FeRe{_ zayid>g^w8K#XrOCXY~BcGACHD5OCKtTJzovJ2RbPJFB7l-;dR0AeO0WcPiO+v3bpW^#7(GbMGHU z!OF}!NLA7{^R^kH{Rp$EdO4eHf(AV+e89=?l~?Zf(54{UkvdJBdY)L-@A4GU14{^L zXi$2^k3$*h#|K?02xvGYMAe~GW3q0&{A|`oF}|T_s$;6k@G0(%G%7+#)qP**=0w*$ zSMu0Hapm4b#ktMny~7U3)Wz95y`2F?kH}VYv!^|^^V%>Z#qJ9=b5*jk2dyuyJb;>; zRrY?e08M4BO#Dl8O9INvEbDvyu7+hQ%kZRX5>j-}e4Hr8n@$(U!3!i*RfxPl;gB$WvUgr=v#v~t8bgA9l~E@ zpY4lk@7_4d4I)Gym)4&u86XE`gwDuU<=77=6#Uj3*af$*P(I~`EEP;^14n2~2>e@g zeFilR3cY(w2~*824j=4i^AFm)5WY;TrK*DXn zv*~$-6x^!OvRRsUYc$>IdP%j-7g@zI&7P4BxPK_06xPv??Zs zlueN29^OWTef89FpUXJQ>DW=@fCt%mHd~{|Y)%&!*2~fJZv}5_KBDhhmKMj!@;Wyy zYzU#AFySmQ7#p<+*iW7QL5d) zD>wktBVo!l){9=E2G2=$CEK`l@5tH}zixvHj@2)ly>!hyA5Y)%g1-wJB&urxn`PGkQ=_iFSL628Y zFkZdoO{G0DFZDkReaL-FiQI)hGIa4%)HF!(-|kfmy~qO04r3NsrgIXlbM+VIF2#{Y zj4nn_i;BSvwo=7}=^27oRkF6J7LxA5+hvX!cLF`7J&9c)F1y_@{A}NPb-sIHTYYDg zlLS3B8OPP|9NcU1iK6{2ab-@-JIO$ewF3NgA}aBXyvl{^{Np-p46X>KS`9f; zXM`NB>G^8${6zFuCrS0d=8MfliYVbO!*CIlV33Tg*rP$5(kJ}nugD(S=~1{+O>jO$ z5bdqWu5dF6C)-eRsY4!6&7_}I)Xh|tN9x;Qh?Hj{!)@NydxJkFR!;hQUDP1SVqw1d ztBF$0sw7&?ooOH_m{r$^t>c1Zr_^MxBLgkiGlc{sNO1FNl1q^S(Eo8hP>il3mc>?% zcq|n47{Lo0g|-?KB_f1c#mn(ugz-_tpW*Vaz905*dnTo%7T$Tw_ z>IX*uwB+w1&(>3g@m3|4aK#2^5*z zY44(E4QL%ZR`C@~*q?FaV1d;HmfM*2^M@t;e7^vUVJ-=q>H)fJJ37vaKQd?9qwOE{ z>(;s%D-sEJ7gBcw@Ni0DRM?Ek5Zf8-^K6tES35hp& zUbLj^H7&Zy)bpjtb;^wwpciN?EEVZ{u+6~TUJm=b(f_pt5g)yk;o8U+sUIqV3g>1~ zoh4ShL*15IK!DbOc6Tiogpw&{Yiq347ZYWC=kMw&U}<1tzIxTV!47oM`c@k&X5#&G z&)iI*)c%Oiu&pP|NVRi<0OzqzuTBMm_v)}XY-tDWk~?+G`> zX)vFPrW-n>RcOQVI@3(46)!csuI~E2Pj#Bcf@l}TY}EoL<%Dl}*;W2gx`S7yVYmL| zqT_|K!uYOlKiW!4{;eqL2iL?)sHeZ{i2HLHkNXxk_ih+4?Oo2QB*9$G_Ye6ZKIv%AMxk9i)1= zcQXAI3Re}Eo?69%0*|JvCibZ*lY3dTZZMxuh>b|1om=B_v%;HF$N6FzEFo3T#mnyE>zuGTyRcDU58H6p-I!A2fy; zraJ|DZ2R^~X#7Ee<*qh);thSsM`!)2b17;_2e*YTi_yhn)-Jgp)f5q%hv8_CVZwYValtTsD9Gr23&&( z%1v>joqmo0r9LJ3p=!>fiH|%4h?v{XJmvC>o;JI1Q1o+7`1oD%cYrG`B7rr;#mCFM zXJi-W(1R~_5i{T2j)9^VvwTnxZ#bZ@9Svkm)4&x!3jGUD#jj2tBXw+kMhogAPq&FF z=D9pT8Jlz_!x&vR{SZ1dLN{JOXQk%GK)2DdN(&^;uSxTK8u>uww(5m`zg$;Q2Jr-% zWjSyZ>T&D7bLeT3@HS$O1v# zlS!P>oSd6Wt0Ox#^2G? zxJl{y-Hi?r{qU(xCM^0Z1xH^*L3~Hk7IzZm4^TY&OeC^JJ6HYWhej_GsbkhNuc45K z8nS1#+jPZSu6zIoyyyqj0fKgiygtX?k@bd2Dxt7is&=G9V5`1cqdo}yN`?Q-+Ijx8 zxK79`br{)tB%G)JAmSg93DM0qk!v6CpaJvOk}ve{CtiVKd`K@Omw*0J-{PscBkXvc zai5BtJKN@e0DqlLiqU{h)2&5!`L}|S&s=(MAfN&J)3ICo|K%K17|9_KzuOdpz0+SC zo0kI z*ymA4+Dih-MalQBq)3Xr4y5XY#%7SnTWXZ=20q2e4MK(K+oCS#@8>C}N?y%^qhUPhrSX;8aVe_h!D?cbH zarD3;ifr==`QWwp^$K`nROeN)*}POL*P~SR?eNa;_d@zNkVk9gD>{zDzXnk3!H}e7 zV=eov0Jq;2obz5}wV7aMe-W~|{?3!H)hZ)^P8MUIWX;elJbHakAzhMC^IGmM_QS=3Y*@TBT}b@})1M!O%# z%K~NpQk}Yz9(mUL&O2E(qEO*~*S=}q8(`bz?cu$3j{^{S-hH(ZLBIcNlJ%j3(o@~h z&1_ho@fF^Z+4*q@BA$01*|(TThmZF?^nc&E`s)xZ6Rba%-vZ#V}g+1&KTwngX40Tp85 zwcqeR;n7A9Y@GbthPlDT1y2U}m zVdTaNJ$I_$t+UyqavoY^r@t)++~TPWl{syzYbDjND*ufRD4%&1sFJNp36yuM4C_A+NMm7S%6a4(bE`P(m|uwS0-z zn`MYHIp9H?5n(sSAe^A~`te^1IMq%Qcxbe%fYA2T5@7C2@`6mPP0Btxlnu&b{`i-6 zCRRVL_=1EA3z~yNOn$y@eDI$ygKMt9&mXpuaE%*SBBq;tC_PF9{yGPX43$XMwQvm#XcOGnD<1 zt_)a6gkYa6*5qbG`Uahw4K^RBZ+av%%2<$)<#edkuIxPrq?xr>Qlu3vcsL1>e zpvwqPo*ixJuVmDmT>J^!8sD*z>(=R{avI)#X00Mw3j&yAs(M{G=f}Q2LieqozGe1X z(#Z}!J6TJ^N+Ydif#o$oUyu8c;MvKRpv&!7p`eD}=7yUTQuRI@P(Az|8*}tU1F(&d z#05sFp47ueSXIEmxoWJq!B|f!m13bRb}sGdwOnLMsaccswY-8?^!ZMA6w5yr1fG0^ zB*e~Qtn;C3?j8^ZaN622o_1Isv%N_TzT9+ziZZ#5_Rc>=E$j3nHa08u_<$z}MVHse zc5|Hf2U5{V59BuBnA6P8uk~0VLY~k2C@l%;@Ou2-di;M8fRYbRj0i^?T;>*PcsVJ2 z{34Ck$<6+okUT#;#eemzA9Z8*ce+M9`PRBka&{H`WkHfht=D)F(%K1Z{X0sXZC%_F_31X1d$wEog0a)1B>!U;gyt>2(cE zM~A;VvjC8%mwU!ml)B)7)1AtD#B=|3sPSLPTVJTiFcGjr+j_2MGefB2fk_$e)>qF@ORLES%3taL&Aa}AKd@WBbtJl zj_&6!a{hzJ(2mrj+d+_roir0wo97MA7dMD0T?e<6V@?YrZt(+{CsHwsuAf~$e1?K3 zl?p}}%5ULCI-^tNSmPIR9y~3_0SFIup-5hbixfj>^rWxixnI$9W{v+O-!g>X_?%#> z`9c5)avi9=$6;T3qUZb`pXYVma=V!TPr(RNv zLKiN6;fK&%g!$Kt`|Jgrz&v78wYY*e;ALA(Wp%e0J?yVQ?0aH7$*MsnXU{jIW{|Me zygS#=4m%_B9r0gW>yEgYo#yJlSB>@5gP;2^RS$&${l1QF@042Bz|ngNlG{*Gtc2nk zEzvs+Ls5<2o{25t02VNNj1fPTDaSd_p&Z{El4_r4ee)iA?)xQ5ITk+_{1b_66Hi|u zFZVFQJnt&0OJ8)^?hg*|QM{FtXVtRLi%b<%TNQyy)BxRR) zLrE+CGaCx{vMNhmJH+~M`lZ8eNYwMGbC~lg?@A)C?(@f%|KW1=4w30D7tKyK?~I`D zTV^{w6@v5 zOhVm)$8g1O>ZNH&Q^Pd|$!N&T2<#T_4sY2+pQStgz-?jVlg1BH2hAYnkRzjUI)oxh z76Gs|zni#q<<)R^&-32nbCVLaUr~r@Ve~9E9!Q6N-I)4-ykbXXU>8&TbI_rpF)z>Q z$>Z9=`UHW<#|op~EsgrnVfxxw=7 zG`jTaI}Wcb&8=6t9s)A{5ZbP}N2Is8ayG`G1-<6|xwZyw;9JqAGd+K)lE?r&yr(Io z7V_69T6pIxz;y+{`a4|5xxL$^OHVC%w6i>3CNFP|2CrrXUm=q1o&FhQ#7 zF$NefT76-WG8=Sy=l z>$0Y*dQbJv)G?xDNdYddRv-k@XOoQI10};k12J~>JaEH9M#@k?Y&ntdwBmq}xX%-q zHGfO~;VyDm;JR@Ek2L@7RkXqV%jmfkp6S}s#)aY*;5 zfCyuNe!?(IX|gNBX0yu?_1I%`U%PQ??Kb_@ab|e2w^v-^5=s6Rb1U&6`w+`jhL2gVXs4%PQ8b=0 zp+uhKo^Tau+42tn;W(ySS}WondgNj7VW~i(PuZM=1Dxkex_R@zQYjYZR6q+OO6sU% zp+2;Qezvq!^q&rJSwts#`9%r79ghmr^PmC6B`^y;@Hw?mS3=e1i+*3X?TzN*uk`i{ z>J!#_>0jqgK(RC~3D&nR#o{hYRd@9D@@N>L>dHBxTpJfL&aF7j?0O)vDC@&dnU$UJZrx^`2@23_x8lD7)^5zv>zS)z+I0d!G`lTnscQwG$(jST?2eLDbC;1s z;YNS`J`X(q)OBWA&}Bo~SQ=Qc2GtTM`-@rdamO8afsc$zcU ziT_Bf-OIBphkJ!;@Qf~O)}WH7gXpJUt8lKVjI5K+?$phr-2+wW3qAedqbb%=USB;> zAZu%`brmS3c7#43OMj9f+L*N-E!TK^*f6}~I-hA|09(_^(27mBL?e9$mJi1|;Us;W ztb0^%anzCSqGY?>=niz~wCVo7F34UKzOQS9`VSTi+6FsLV+?H^Y6%TKbnm6X2EiO| z<0HLAUNHpRO%{|EKbUcE03WC+mJ2_)m{2P!0SRS%11sp$OHs=p$fQ924^wPD+lpqe zL|QN6IPZL!)QlqPPPNq9NUTbNy!5OT8j_Q0kj7%`+A^nXy_ax%AwdBY;vt2z`S;Qt zzhi`CkHyYt1dF!Ta>W01tVqUcY;?^jpq8wEicEV!=gxh?TJGKJBN(-UB`Q$M2AkZ+ zyn3Mp4xwmy8!`WuHtTIcz2e*GSY*nky6Gu-`kWrEE1E-Tr$rAIRotU*sI#MR?W0ut z@Y~+>U6m2LYt0H&Yp=JZAeyqkle%QXAJ*^2)b@`83;XUTs$MM@TkqNMpl8-M{84Bw=n~@=w8PX2Y0A+T8>R{ z*u7pkj-Fd2+Bm!&d@I0CT}^+^Wp5>L+1}J)3m+i374`eJcP6rD%0+>aNHFhpWh0#C zWcBvPtes1apL*a2J?v7h?U17w)2NVBT7D49psm(Qbh~_)iU{%Rnq}b@m>1afW`{H* z&nPoF)|Ak`;GSt`*HQC|!+CgZgoUdB%y@-Rr!7c7QDJ2?5U=i&7!OvldN4P2Huoy9 zkttL==s(k(uA)x~N2>K4dH}>r-f=1D+~@j^0*OjuJzH}3Vzod z!Avj$nWz+#3FgiTqj;d`Dw*fzv=F>Yz)Ca;?l?tuD*V~9`)h8=R9+IC!>%UAWWl$7 zAIg<5xS;Z0_~7`bG$;S_)b$33#q#Jb``>|z$pQ%-UBT{+WLId^^H-i~NK_bG@w#s(`)Asjc#XRU=Q`vLK30iViG^kN2Td`q&8xhI53z@@}%7le4i z$;o0FSZ~V7#nKgxY1S6kIQk*- zjTS6U7XO!M^Q5XJu@Q&7_+qmT^XPL@WYB$F$0u5*>D4R2e*)7hPfWOHy>j-FF*KGQ0)zAty2BW)qx zW0+b7e>5{*4QY+E50vKsamJ1b%??DWSoeowM&DebLcGYY2D;TNu zlrPCQk|slfU;hFkeF3iG>6UX<=)_qcX=MuZ8O8u+aI>6GRj7uDmNHLw(O|gn|9r-p zwm-RqijUiIVj3f#g9pL_DjWtvS|uHhmu!Urd%~zBsW#hg6VEu>rdGWM_zdW_UY)*2 zL!rUW;)Q_Rhx9{6X0|;l;h&mVR*hae;&KR>Y=b~hC(B%Kpj~VaaqlQSD?C>0hAF%P zkf#b$6ZGJJfIWe+x;c`q+l5nZ>NT{h7;@kA_*ACgxA{rg_ZYuXGbV6c-H^U@q3365 z@o*%4p%0{->xSep^wzu4zH}?U(Kt&Q%q&^*PMQ#csVYML6-c)>jcn~ubV;iNpd7}^$V(@N(o%aIi=t+kN|(zW zO)N@dQjR3-A+a!6W_Q?*y%1Ves#RcB=#e6?**OWP@57>iLcmJdn30-mm850l4i~)i z0P8#;aHs7r-<{*wHaUEC59s_Qr*Hd%YWTyEINuJOB2&u8mSb~mn1RA_O0VW>dq5%B zy<&Ul4r|1XG-osEZDEl8lb0T=T($d8cM6iLeDS@n!3&vz=Ae^rrUP@B-wN^aHkYis z0m9WRS*EW2-3ro9Fclcd&5KVrSN_45QMHCpxLPd9IcZ&#b2fjSu@oxv7+$8JXb_m3 zsfX6ql~=`4(&9-p0oL>#8?05~UtYKYmTh-#YbbygDSc}=4&j=>241{XmVLt}&Z?6|| z_4z7jVAdUbrF8(4VQ5BW7`6%^-uhAYhq|csx>|t4BR`b6e8=^Ar+jh%cwGYXQ^4di zwo!v~>Ip40o2@9l#&kftHe5s<&}^+cD*D((sW$8YDF-weO&PCG0-&*l*Zwb~Xfnm( zZ}nM?^M4PkD{_{Y>Qi>qzQP21-~kZSP!*L_c+T|}_Ajhsz)^3@6)HNnU((2YcZsLb zzjeh8=n{^868Nk_viL(RtTrlp$d~=4LKxA1Z2*K0gHqaSDv%#M15-ojWuYtfc)9TM z4}t#QwNg?|95X~~c*GYagnk5oe*{$X;^4?QvNC^^UEQ8l-PI-Ckw#(OUggalN(6%x z9dKGVUjAT(g-OuF(8IDE^3Y3{#@umTt}C3Eaq^TTVAnttL+0mBhMne}ff}U*r5-6y zb@x!tNVgIZT!KaK1_9m%Om^o|w8zyn@fJMLc%+(a#P97N9v#lf-!qu5x5|oRaG>aD z;!ew`OobndcksW>LMRfvGPU7TS?SCdv2RPNkjVtaMTAJ2xYjAeqF?Aeeq@*i8HsDPSb8zWr zbIy;Z&1%Ze2?QFA&FUayDQ%F2*)VxpZ)aF0_B6f8Z9ri@g&nr5@rsOe z*rLDw6^uDG;s^M8LWle!{^TNlSpRUacaLll4{4l?4v6g*Q<`?3&KHhl; zeq^t20wfOw8zAR!vd7*36pr5IwErxt&=HV-A8t`FCs9e*m}`Sp)?+_JdR1LdTI%ks z+zx1w>108fYO4k~56NGbW8#Ky0<0c1Y*0db#Z1TDpUX9?4x+PTGjC_i+ZE8|3>O07 zoC|>cLGaqZdC>wMi;0<{%e za8-I87TdCq2lPfv#7C|C^+c6!tO!}zx<#LT`wGvBijqki@@kP=n@IKdyFWu4aG}K!Nv+I|Dp#lBzJa3{SKX<{ z#T*H+-b4xzA!bR2wp(T+JnUbuhA5$PY}MS2^jWgFNxZdkd?Am9Oy0oKVfIV zRmJzVVVX0CZa8#zcXxM}w19vDf{H!w_yd&?kW#vl?&i?l-QC^YFW=z*5oWEuX76X; z*S@bf3&umU0#u*Wl?Up!wik=opn+-GdCU7DMr> zQ5{n^g;FDrz=%x~aKdoYd}VSr?RtVQygjFt0j-`q(ly8Hiu0-;EbZ?25CZA;KMR6= zOvpIl_f1hi{$~U7$;{)0&X}q?$m@z|UH=r;-xjmP*X+6A_O# zob+8<2@gz{1Y&FpXnD~_ASNx9Xk&sk0$*z*MyzC;(`!uYhPxMiT9`nQfGWHzyH zNdZMIM3^1KN5|!Wk=A_0{_IslxZmnmj<{FTxQ*u6^Oa11w?;~Cp6v(m4OeZo$Uc2M z|MvD@qh4TGu;&FSBJto~9BfSm`u+Nl50Cj{dY0C9OwMgL)*oRh(J||;O4G*Tr1T4z zWA%Z7cX-|AkbmahmL5{`*EfW)0VIZDoXWDNI@vhj4!E%sd4e3>y7pnd&E6M2o-U=$ZZE;4(J3v~rm6bKSfLCmEv3>l;iyRPUstd87`sqYh@}LmY#!^TNX~*8I zApVOU>1EN`6Z~lie0p#@vO7fO50;C?7v?2g>{1ud__^kXyOyny3BD{1yn!(S{NxWG z^`5OP!ijp@p)j^bGL6|7b2YY|drCYSwGeS2&S!C-Aw1ka;dRnQVj@`1`n|o-i$< zc{1jQD+P)P4d6B|NxF8%sXxlmBs!)B=AG_-Sirbjck93W=bJ8xbx;b6?1ut_I5j`V z#RL1;oO}oOlXmGD4J`aBC&um`SxO(x0ar{RSPmb_IAhi$d;1cl{_2f@3E1p0+gVV| zE`)WJF$-&mAq{CsbI1EbvNxUtN^ zL9mUDNgb#nQGW|38l@n1cLI+1Vp9=h*p)w9?487H?;Y8&3_qxW$x(GaqM1_z`|Y@z<6lIs`z8Wx z4a?d|`3N1~ivlg6%x%Wq-g1^~E#|z+Kpu#M5&}LuBmT27GCKD0+)*6^t_d5 z9^NwLjwQ-y)uJhV;)#9HF^X(AlLer3y{vRr6g)<4Mvf-oi%s*$#9hPS_8l! zh)9(pZy2MntdA3I8;Lm8zJ~>wEr$HZfkW@s_@>;<$;cmrd_5u=J_6Rjg04TPiU_p8 z;=KmIvi?EjK=mv=M147V3rV5AqM2rhvzIFRy8V|79XN*=6sU?DNtexuzj;zjzda(L zW`CHH_$;F9jQ8L2Qt2{mB7zgVF-U46WXEb)xcMZilU{bAGB-Lr+2rFaIZtk*4!F~= zz5IcN>u#;C;in%vdf-*+>Rc2)<0_DnZkrh zHTXR4ENm6rlAO+@1Q=j1%GouC0u*^Jq!$aUF)b3d9>AQ!YhSq?qD@(~(>WM?d?Iuo z%bhAyt+PP^r@fHo{#(ditwFbWKe)PJv3f!V*nx~Mj+d2$bqnOzMEZpLc377H-*3l+*UJ7-Drn5;LF)JrY?VY>@EjkM}BIp-Yv&M zMzeKql3hRRLs3ssgjxC()ku#bl%aN=Q^|^3$t1WI*#1>tbokKW(>?b*b;ZDSh;K0;=iL> zush5B?Q`CXz+xoTI|pNWQ`-; z{6{=J+Iqijx&95J&5gZ5hgo>8s!|VDFOm`iu9fu>8h3a+SJe)%x2cRL)*jv9>^Bjc zs!wc;yaXl$2w>#fIBGFvn#j@n(h=vOnY^4eg@5nze<_uHVQ|3g4uPs*HF1$?>=8pt zDd1+CjQnF$k7sQiSW`^O4+|13%;?UpqO0 z3$CFYK&9|qde(2)R|A?zaWEs*d#MxaM_wpSux{rtWl%bk(@Kt#ZwmTw-<^4UTQOdxIp{pYXY6NWQ&ua$trT=^rn+Gd+Fc6Es5J5=>s7hCQXz}U<>6AB@rd+ z^o2X5RZ2p;X?wkI1uq??5H&dsZ_re#)HGS2T;J~le_irO{~k&*`oKxiTI&X2-lq46J}Q)5B%@F)A-MHEYk0vCWY#AztCv!yr^!ZdYI z%i5>7PKgJSOB{_OKQ`gO2_k4+8CXxy zCX|I_FVBudP8*N8WT0r6p}V}}3scQJW1uF%Ipvy!H6@ax_2UD=8xP#9uEf+V#1L${ zF0j;Fj0gjmu!#VTJI^SP4_$H|&aP#JV=xzgQ|=H3 z%+)o`W@-QFHy9Y$0x18XT%EkoT9u)NgddE$_LK=Rv*C@yf*Qy@6ZN@2rT;)>1TPBL z9V@)K8dhx_3**HslidcA=nBTh0cs43j-xfc(XJ#qX!umJq+bj6tkMBc{x=^C_{?8I zl$DHm`RCIfqaj4%+UWZ#nFPlU#212hW?!|M-of#&dHW5<9Hm$K7WfA8{_F?ZYk`%N zJmR8g!)rW~*{P231p^60S7HwyZCIx!$aoknYlwO0maivTb3ClO<|agW1qh9xzZuuZ zE>-Q#y}#BonACrUwyDXwkR*Nux{~R%EmFe`L7CfE=r1u>2wTwFuY{n8?XL% z7LA9R%{;hUNJ1mf&UBBBeYbB$tb2y73E_$oo-<8N5g?2^fV_)y1M|=euMVC-M~D+H zr}eH^dGf4Z< zq%W#TkL#LKOtPpZxDM9=w(;o4qg{wuXJ{9lKSrL(NUvF=mVcP5%r(o|Ytjf5waDRIObTj;8d=^)&13X6&=7tiG6@66^^W^@6XIMmHXp>V=QX%S z?(+l+tJKK*2G?ZZW7@e&CxK#Q>%!;FXCDVZzJ~qTMuOPwvhu&%!dUH`q43SSiMFrj zxXFLirz^!qceJsARQ1)XTP1nnV()l557Bii7i+;sBnT(bJ7w6^0Z za8wyyj!wMK1~{xD#H-c!dw70$`Q!EnbtJGv5`xM<@Rgrk;0AHW`10deQZ8O!WH0jw zJFkgmN}W4`yCek@j*ELn7f7+xmiD}A0P02AZiktpu4}jG)bH)(ei!9n6<3JH_;bZb zyhqH)JO-y8Ti@316ZAqK$f;6AwC1)9(RM0Ob^RR{la&*Pe&!Ou32Z40`0PFrPH`M4 zX&2vR8L@X1Q0V>Tm4b3hwRv}6769AJqrX=#azM z+zIM9(=P2tiQNqS>gssUAlJXEYIDNm=8p@Hb(mZ9lj73-N^mXu>;x4yMycXnRc&ar zkG}AIU_62aIMTBgoaK3BRTu(VCHI*$(T4j#Nz<+_EE6iUWLm0PgRC>S;(*a>@yUP= zpk}7ju7d_PMK|*RatQAp(26P7XPFso?|L8)9h$Wkhy|p9A`v&pm3|I9Z=L_HX3P!6pKESckh3(;4o`u`x;>IIms_J8)%tcY8TnWK> zLMb{=Wn6VMO6YC_>{u%8;|LKCs-PyX8yfGGAlHpJ1>%f|un z6YH2Vx6s&WZ|a|jof>?GwR}aSD!Hxv~m!$ebhF!xH@h}%SAXq4p_T#d-Fwf0VI>QP%kW4DU!jGY&hEM;jq%fn`G z!yYI?vg2^IA`~tQm)C~3jnGUk>v?J?!CNYxBJ)R)UUTl1kqSwib@k219jY1f)Y-N@ zmVG@aAH7G8q4Q6Y#cDdM0z-_ohuLL9oOl=4j;0?N12Gl#2(`DJ98^bjh` ztCHElTPpqB7?nWIK@(dv!w3LDBP}g5TA0+S;I~B?d&bIjzak7x$&=?_lw@i1P1ZTW zWpAJ(W3*cw?i$I-;~C9-wl*;%!sPdHHAk}%4>gSQHvpwT&S$zfnBgo$$Q$81?-eU~!*jT{ zFF9BJ}DhIOcNqg&OZ+RjJ(hhhk!3Sy{ZFybEKzS&xlOq}kA-VK-1*oMY6{ z#f9xi4b}a){x@u#na5dVs9#9LNRODju|IYO$aUZ*?l=qrY|dncDRTk_EU6Xbhs_$_ zNTJ)8SK@fhHaIy7V2kxam>@>{DaOXoa^>~ILQVKOwGdxjhizIJt%N+htDApzN{8Rg zXZpRFSTlHY{pD$NiT9+9p2ZJMj~t|=ulraKv;Xje;kLvzET3$vs88I`5pa>JjRz4Mk1_Ri=p9M_#^_fh zikpyd1|~QCY2NR<@o5A+%LvlGqAS5=p1WVm+@=dth*m+Tb1P<7u9*IYZc~;P4p&4z z%GXUAr_pPIZUD~-79Vez8YPv&ndc*2NjX8;J{O!S-?{qafYWU=f2LYzgy=CL(lNH2 znM8(=rP4s9-x-Pqiqo2(MaP{5ZqrU1w(<~;?k;1FP>Lwx5bC70g3<VWtaf!!_iK^rV->DGczg5&T&yy?~*hU+T%jegy>?K)dUcbbax zy@IGyyb(@Z;0y8mw&EehnCY*hE8(8qR`UF;~K}`Ck znp*=ZyxT%68bgty3*i)dmo_~-P(}JZps-8#P~|(Sil7wS){n6K6$7qRl02LLGMuL0 zM-_`p(h6u5BYM@DDbZLl2H>j7e@O$my=OF&vRiGjPm+zbyd8eT|IVThP-eNAO;Fcv-8Pi*?{+tri4{(NFD;cRO67Gj zoI~7QS~$Q>GWFCm`%c^eCc>+^>=+i{jHBc)y#QX`9Hif>n0!hM7^T&7kXd~uk9NDF zlSs_=fSyl`4Pznt$z!*42kyrwnc>;vW!t~YMmagw|Devy_MSBhb|B!Gjt+T&eca9|475T52_AWuLQ_u9Bh-U$4@>2Wp1&aw(<^l#^^Ds#BYB zUOrWgJ6lZNy1V@iwBU%a?ydo+L! zuY&!{D<_lL*-|rhyKSf>8D1N$NQft)vXmf$@M#%)D&QE4{nIEC==Sjj`m2NrlHu;$WVaP509L&0(?UKT|04R=G z-54P#Ow@~qIrl%cGS_I*a$$BYfo!Z8e{tJ|u6%)r-5I*4NrOJh+MutBvTH8-=hG>U zRC6B$W}Li6SLD6}%4c0Y+{wx8dd%pul{3o?(fMc@HI4S2MqY}cMQWK~5Wt3c;vq*T z(M6{(O}X`$Y$$yJvutf+rwy`jqcu$spQiYVFy~{6i*#iKYT_YjsqY zO2rSRFVh!*HCzmJb`J7uXdNAWvLK_cY}eOZ#(?$ce1~FtbL8i|2b}se^K$zpYhsoC z%!<8E>*8LBWxEnQ#pz+_ys=sJ7zeWoChi19%KD~;=sStt4DX=dB>heO@Ms}YU&p^dP_YvH zEE6!4mU)FOUj~>v);D701F!_W7bEcVCW9&c53G4m8_o+rIJ<~0m<8Zc z+!lvj8m>BaiblBT1PT+I#`NER{cDydnA|*aLL=t`;BE;C@BuMo=vkwdBX;}vag}{- zayecD1ACj`xY4iYQvjDoZP`JBIia}gWJM%J?d6hJR;CX2mj(Dg-TNPl?DX!HH#X=BFh{iW7PMovwe&q)wH1HYa{X z+ig-^HxLVLJu%O58KbS;()C^Y zv#I~N-0^+_D!jX!EeZKyefOkDr?&s1&$c<|`ZN$A0gDK0P2(aV9Du%8ItJe-)&5c< z`k7Smjs!0O*uZHzPayPTNGIa?%& z`L5s&mcpdul-ymW{H;o)W9x)5165){Cy&I(q17Uz1M+tC69B0H%GGDA7uk}ZEI@q! zq1}f>J_O_iOvn_tvky+|hNixQ^)wez*t5LqP^>L>@p?w}8$_-R>1bpUBo^Y zoWeHLh;`I+>*uqA!uv`vbWm)ZnVl0W4zHglY=hSDe2b49Jhf#3{*wqp z>noy%D}rTsZB_m>$1xO588btpjcdylaW%!_=dh4jmf*^k>*V(qlcq41SfMURD5D@L z_At1iDU?m4hu8~{!YB;)2%O_Z|b7V^(h1WcxezFRa98`d^7kX__-E_l2IQT>I5Tm>6J3C3c#+CQZ z$&Ot^5n|&SmPwr^&#tuhPD5L5jRAfO&C0}GX!>xzkM?w!Kb& z0U~9#PE^%*`z0WW^>$|Dw!JO!9=4XZy7y{&>F%1yA(rsbpUJaWn!X>jyG0fQ_Ybj$ z5s|P7uOA*k#(5>7!wlT;RQ=Zx(q9uPFd?JjA3dEBtL2Nrz#0vLb#S}_OZyV$!Y4uWj--U6CKA@V8U4+WFw`m*5-N25Q;gUw&*RUjXYMfm&(+@}a z#ik)vk1bTJCKZ?2UqA$)^D@}4U*jbuhdfSLIyfxo_m6xU&A1=n8E15ET@Sof2g)d> zdX*nD!<=OEv^HpbJ zW$`9Rg+Zq(kk}F%pbJe4na0AMI+J*c5vRr5mz+mT*fwhSZR&d1K*M}4cfhsQf!dzX z8S{D`Ia+)@;|Y)J%IJL}RdGSNSCV0`ctrsw3_tekEVDg<-=+F(+ZQX3E3MszN0vSj zY`aFWP2jpPkaCAgp4+B>M&UTU}kgv_c_on zZN&F)!~?9~Fhj0m@002GfClx6=lt&=_MPpPVrlhF^$Ujp`}a26ux}MSY!@mFaO#I=^e@?YKgcaYDvthzM*jgqB)3(SHR0T$esY=irirb`F2imh&Cy3UJF9=X ztZaL)0PzbE`3FjMJ^H=HW=<9JvC#Y@B{k}66k-s=r*z+1MHWC#{h_f*)2ne$Yp(u< z%P{_++zeGx{XT>LGNm##k#na4jtiK!FMosGwt5l_7lY}f4RxRUJ4w4+et5X#?_VB8 z3jhyXAIwvD@wp`2IfKa<47OcHt@IthpcCJJZ+ImLr?=REq0iUxM4~xhRMADrp@U|C zqsn5CHzl2%vKi0$Am$?+0e+{>4!a}R{0%P z9oJk2p_iTH;D`Oh-LCy>aVrX0mW>Xvnur#@k-So7sqs_l0Js2 zNWD9u@C^3GVkzChZ!CcIM}%!}7uEaN?dn<4mBxBzyS>hIHe7|n{0uB(?<)ue@LLj) z`d6pEdH^SL{94mk0~&r=olx$nG|-B^-mG$O5Cw$$@hM7w{~L8$;%T(;;Ns9RNA15k zbqZN}RY3{PP~wpS{&rZ7@_1`xpfS(t8V;nO^eNEpE|AQz@Z?+s zi*Qto%>b9TbwxqdQ9euU|t9PxEn*?<#{sG?_lQ)fHbknlNO?gG+XujWghhR=bg)WlGhk{}Zh zXFoLH&>eo-y3<;M{V(aezhR3=Ip{T>e)yKflTE2g1mFs*NHBGRsP_5khPmvMiXZln zKh(_4aVt`YPntg^9+CogN76wWU%BLEk`){2IAi7C$W?4vcC)S|1P1Zg8u)^@u-^1j zA)bJ!QF~ks`}pq5?0(QdgB$rpw{(7T^@Z8{r}veXlPO`z*1J>ID1FsuzlTfSe%T6# zgCUMQ-;U`0Uw^`(2AJ`%MtM}jETK6PE1`vAic_*28o%gw2PNK<-}$^GiLPK}ciG9q zD;t-DV6H-d#Q@`{a@McE_P}*ZX@RSw2*AU(As?GX+plLVHsgogIzw zkHesF#W4(<)hn>VJJF?WzTR`-1?W06<%#PVuoI8=h%KLS)%!$LRp=mLOegiQCLhjK zNSf=uArYl6NVLg2c*h)Nw5LCp0C;$j9J8@k8x)%SUv~6wVnIP+b3BltSdL|PtmUmQ z4m~wc)KW>3%Odwg+y}%QAvla*nCG|1HA@E_|H4hN2&T2i1~3X;zvAfyd!d``jrlXa zc_i#>l0JlFy+2`U3I?UfG0+0I!aIoiytV?xK(_^Dw&@5K$Mry=xiIR~g&5F?v9=x` z@DEpU(=Ok*s18IB7(o4{Hz&Q7?Q4U@?u$fa-KFIp(Shs3toqr2>*Q*xxN?vszJUIm z@>-c{VKPV3;2SY!iX#yqw^cW(n8on+k%=2RBE1x+MoUwjt(KU|6C_>8E(jFF0$jnj z`zOC#9{*POFkhogwc>N}psL^ec`Ja#-w2bL zpw@Ve`Kc;y8_J3z2XKI1LCGnX(PTK9UdfnJV3cWv;R{GA0fhzIx^{Y`v);#xB%%U_z9?9L+FKw!Za7p|MaNHP4_kCHkIqHsF<)D` zJQ^wg160N%0?I&o?}WBPR;)G3o*L3_A~`qo{cjNjsET$%rz!ko7L|gS7tXxTBjmy9&W()6XFq3^;fEGGjJhD zjkNC3<^l8A(D)WIGPzeLDb1HT3DRgp zUVLTB(>@|N5~G+QgjWe!@gSfhvJO4`(-(0tOUo;h$M3(L;JBO(J1c(JMvH*iVGoy9 zBO;xd)qfC$RB#Zx%jN(>M5W$rnENUml-oH3%4qYFLOYi^y0dB91tb1jj?Q4hUZp*a z(|ZPCF_SVEECL7=sBGb%?eR+g>zL%X`pO0Djm?*_B(?()rX~_wwfwn2hA>J6KkuDs zZ8W{tFakv8R=rwcBWC;iOPBfLFQ>w{TiE=dZ$+#TO8o*jq{=E`E6HJaQ}L6W3kSr} z?HFJ&NSOsVOFZa3|E-swO@6#djZ0s|?-V{idV0ynnWr4X-X!th39Qq?EXyI8Gerwk z%g#1C#%EXNy{*#Tx1H~rwtPHQM2CR$x-EiKfr@)PRL+r!)t&+N#;^r zUxU5-fK&?;l{U}C8|=8udDy_O0OPa4UJ9`7xmKJBI!=Rx+tb0G&{AoMd7S-A1Jisk zn9w9tRz0#-dfwKQOMr}grL+A3*p)C`40?Soeq1i0+#|6LDJSedHz-H$&Z;|Pn^#+@ zO2KP2fi+h1>4QlpLLB`gBdEqmxSsVQPDG}%XRA;_(vS_#*CFLF-^m8!F6;@P8seCu zyPE{b@VeQk2+$9+gLPUB068&8Fa-tO>|NBuf=cg!(N4#Nw?oH($KQUz{UkI?n@ZSA z%dJ}|@<*2V1%7d~{*Tx&6}raGgM~OzJ_FNR)ajBJaN$tX8yvk>gw3ua<#d~@)dvOQ zt#?#|B+wss3W6152vFA??2N4` z_SiWkcs}TH0w_vMV5o|SBef|0^;XX{uXpT(lO=CSLUJQh)RRr!;cE2uB z=t#dvOgZ3Qf`&TRmBoR=f=$#t_7)L%8~EV1cR(4^HAcFnEX}0~+&i--;Fsc}vkJes zpx&uC0~!ux@+xX=Z7HuRuC*3f=23PS>yp6{O;Up*oaxfy=Kq~V&!L=v~=;f_6s zje!%^$MEB`#KtYt_AN!CHpQ*Y{fe-e8)8ATSXG)+(Q0NX1~3eMHG?*R`ETy=mke$v z8PgBFQzR5RlVie6XSIx7CtWeR1P3)(ZaCfMXzt`&H|e*AwapInZ(j`T=v2xG67%_? z>ev9-9Mtxp_9}5QU6r?~##0PvBP5tlY|h!Qfs_AbA%{*Fr-HaBUVWJQN{Ns_Rz)Ws z;uf{qz_pfX!yvV9Ix0rH^uoCjUm@OsH=NT_mC^HxVx801@?D*kgB+s@qNA;nE_`D+ z3THIv3f-mD^cysWwRasNY}6ETofGxqgb)rJo8p>sA^b@|nYabzY;!R0;0mZ2lu;|A z9!#TJNcYp3EVezXRj`HK4Kpt*8IvB>%hY9D7;Oj1dGQ@UBks1210z%{B7LgX03?=8 zGcIUidu|=maF+5TFT&J%#vmvAsQMk;DQbWCNE>b%MSy&ALw>z)DPNN}n_FVg?W6}zT!K%i0&V({&L3tO2Ssv+=1%L?km+^Vu z3F~BO4J!IUMJMWXEa>t)7^~IZ0dMo@L`oi35K}hqsjYHGQ3fX1?Y?5yMbQ7UKhTod zda22d-kcN73df}^6S*$2&D0p%(pv*aKo@zrh6xmf#!1-G`bWdHTwa_u%hpb@A-q$- zu!wT4k+j1Srprj80}bvn6E}%uF{$nV#}|cDQi%sx)VDfJx@nCZc1a{430)F}pJmdF z%E*4+JNO!r0GbvvM*F-T!EhBzTgAjODaV)#$dEEle*cj6y7MK#=Q^`IZNu5pY(iZJ zS)Ltd1ZxHPAMJ01&;C5Hq{37DKcF07);rmlohPrrIlH{Ay?5fHUin7BTr+^x1eLJR zA#`yxS}KUQ`z*3NrJN+{pqV=E387u8IXLq%#*M<5T8<&>6D6ND&z$mnfUcIvYm0PG zqUP^H=>Z;+A=8PgAOy^k8m;h_dPHYVtM_3>g)N%FT6$5Y! zE*t|!9&x_mX>MWjs_zPfMVdUuJ)onC_0o(o8=zhq;;W_~A=cUpqQ69$mpqT_*0sx$ zP5Z(*cIdx4s{zO;GRRF$AZlCYUa(?E_KNDo1%|#hfgvLg)1rm;EdRgHO{L& zzfaCucP?F&1;che;&1*g*EuO)HKc%sgNG9rO^)(>{wAH!?h1mTRw+*0wQ7|}uPw(O zU~R+Z9Qe#IUg{|wfFg|qye=t(I@r-v0_Puj8VooTQE2h;X64npvf+>SlUGVd+R}~; zR09*_ddY=5M`V=ss*hmQDZ^&WB)T5}G0f`W$zh)3VEnZ0M;6>f%AsTTvH6j-Wk;=} z3EdZ|0{a)v|LME5gf^6VP#3a#y-phhBT=*WKU_egsH3#Hj@&?8ZIF9^<$4h+M3v~x z)wHkXo<`x~Onp9F_{$m7z8<5FSO z(^e01lzMW`i+B@CrxTt{18YjAM3X(`&O>v^cZyByJxam1cH~p@LnEAv%^azCkOt6t zY(Z~l(%nGF^j9NEmb@K9%E}D7Q)O02RUx_Mo8x^8pwZP@GkUR5T{o3vIyZkLg64A) zO0p^!nF;aYr1ii7Te#If!&H6htXjQE1G)Lnl{ag}5T{_V?cUnmjTAMGlcp1(CKSek zD%*^f(XRq#pE;^oO~qimq9&JbH%WS z@w;`dLw`4cd<_Q>c8b)!MQ@n?!NM5@^j7I2ZBFq1t)dq+P)J->1Lv}(mMfZ23*0tN zRboLRwJ9CRuIAW~xT|CE&5fkV6A4i3;g#d}S-j&n$yj;%j{boDUy$-*f6ak})*^DO zo#Fzxy^*~z11Ec`uE#P1_XWstQaw$2WG^;Bamz>irk6sm0Ht>zCB+hfNr4zUzS~{C z@KjIEtG1Oz35zsFMZ#wP)-^zsz2;mM3EwQk6{(Md+KbR_xQtlT*e@2#wfnSaKg)H) zGZbO1M1^CZAe+{l3H(|so|Hy9aW+V$46};4u>qJL1ZJOSZBykI70Q&1m#q?aHk_A2 zsw#+fhk^~=_as80&0m4yvc}gG{IQkq(qQ0+TDzGN_(Qik(^Fzv_*ceE&fa&T2T#>%_;zsFl4}-k~5mWxK!sdtTM( z6FOAo0vugXQQ?$dJVLcn_vC};G#t>i&8j5^du&xhRkICzEn$kc5Y1!V*Wc~AzzP^4}sSQt>IhY$M}@2>JE?zaow z1Qg$*{J~=Q`ae3lCr~^=F%t$-Y8ob1YC^a7a@ ziqgI_#d62<$<{RmV1I}iy~{qlYO$%rF)AW(fQ@O8rDm~x-V|lYtgdicu_E?8o{j#> z`!Rn)XC3pw53Gi%Id`OiCwvrBuYnKlr!T}b3f`YQE4!(TACrw_tr(%l22wa~tGTH$ zHc1xfH;;eV3&-L;NQbApKPLuF}jS>g$iJ zmQ)!qy0b*HEF86rzGaGbUX!wTriHEF)_3Z!|5j6#fs&i;BtQ@LXUV}}S>IZ&hW)wX zVtJ7EdbpBc^W`l9bxlK6M3)f&q%RFZTSfv-*(Iz>F-mwB7{YmnS+uWvgBCzDk-~R# zu&&dE(w)=E>LAN4_MN3)t6=fkfc)nlPCHFEJd)4t4;NvxUluU5S~M{g5@8*f;PrTH zfm9xbF#&F-303kpt+%Ue0NrC?ahKiDnY~>h3z)NJS?6TYBJm!idS#W;`t=v1;Q{Qq z+242dVGg9B3S&0W*73i-p6j5lC-@R9L%%en+5RJg0kQwJ7V~ zV=IqqA5|HeZMVhHGe?vqatUpLd@qHO+-Y@t-12n?J*=()j!~>q$prlz_uZ^!z9A zo78CfkydD<(o$nGMS)zL-;HB&X{dW&>dgjhA$yXWv90wtJ?J|AOQa(4q$=x8hxM?_ z$2uFQqbqQ&3+5&mTlFG{C5g79C+x@g`+jg4^MaW9w0fzRpaq5J{m=sJ>&h2Jtt^wX z#V4h*nG2>2%3!S{c12!I5RWuCsRNS1fmm4Y*s(6H=E&*YRT>-QOy{YyFjBoE$zl4U zK}MeGGdqC#I1b=LqDHUSi1az^XAiY_V#Aq#)V74=lAxf6XXs7A%<}b9802d|1P+=K z_Dp^@>bpyIo;%eS?_?YUIc4x-34!`)BEp;pI)aY`1sp*Qd~0OxQs8qn$YkQ8^#(rj z`{Oa}$b@j~97k>Hz4){ z-+Bpd#pxe8qa*sEY-esPyTm0%ZnEA`8-GG z2Ij+wMV7mqR(;F-(a2~OO8>G&hPgXdV5QEKDHZy`YKj7orbS=W7q)N2I42Te|ecH9qdT&Abk@%Jg2zbr){!$65q4saz?h{zeb)s`x zy5;=&dSZd4P!;B%cMgk7=o`YWI!d1Vo8>PS@PYy~ZKGt@V{xsP7ci{)>-lK}_CUq> zaBroUz7Qb=I&@S>)G^-ntfW+4)p+>mu)M=ck-#pGK*HFvX z^HRv%*!w?_t4@;%V4$q&l~SMIQ#@RSwpRuuMa7ka1Z&;sF(#ZQAB{eVF#sn4&7bep zV{Yi)P-s${Bz1T6mcz7D<69t>yeDl(k-`Pc_Au_dbuv%%AJHhOatjm1uRd#ZJYRyW ztwHjizc+V^PGE~nzj{R0A;GO{S$dxed-(?QkxdFlTD&i!{uN$ zDPF4f6+r+fLm$$6zcv3emz>I!w3pPgSf;Hn87iK$O zbu0zg+}&)NR{^{>1_MhmyI+6Xm|MErO??)zKv;w8Mjo$@JcxFI$m> zOWM+-0eUYu)hSC(lhT&kzci)QWmH{?f>}r65z;-`OyVj94XA({7|YsrgGZpZt(3c{ zyz;m?Y^jDcZ4&HnO=-`3f4~X@NKR@{pvyUFBkvi5qIy4LDopDUu=eHju0oPHDStsR zHPCigYO!{sY62?^GHoj^AjU11F-VPnZ(y8b_0e@G3G3mTYcIS~0K<-8edvLh z{Blx~TApJ~@-cG?ZC-{~PUgm_WRR!P(hYS)KkU!_o^12Xgsgfn*QXA^W(eWD?k;Bf&`S5;C=6}9i3sCO z-yckz|5~=qDtD~B2)avaWLCCH2q=uG-b02!B6mG(K~JLF6|Bh`*=;x+;3}038z!r{ zzI(Yb!iw4q0h3_wb<}B;uoO|>KHjcl$yHQB5`sM|QQXcY?{tLvl?|Y_blum)*X*^M z%96Om?(XjH?(XhR6)Z$hpZUf_r9=@B>F)0C?(XjHhR=WS{)D|| z&6;cWd0xkX2BbAGiiyU#Cj_Q)-FV8I5$aE(+0M*lKq`bEn_)OTiT#Ud3QY16!KH3D zv|qtnYe&_GN|V})vav%bcKa~P7O!@Winn$!Exv~kpH?ju$%&zJGpN|6KbpLzv)X%h z8+xfymNrod`EfDrt>dHV+5-mUj!nX_3+HVF4OaJi*u!qd$kaxl@5Sh4s|@Fpfq_|2 z7&Bb+6Ux@Vt61b*7Oh&UnKQ2C^m4cq3~|pwyUMF=0tQFz%c)RR_o>Jb*;5-#a5~iD zkjpbVv88Op#K>o^hlxz@=)Es8`)oh24V{}Tjm{&DDQ8sa?tkGYN%fTUE}TyRM!C~m zAv$=%L{jcFGr4J=S;TFlF6unB-+2og%iZl#9LcNbi^B=8YTZwCXz>Uyd(ySYea0q7 zQypR$K8nl6yR`>FG+mGpdfl)>@OkNHY39O$%5$a@`H4dibDJgOwgRDo+clleWgm|v zOC-`5d7EtjD))VQxM{3LK(*GaUZ?Q3^at z;5NGq@w%IeE|d3)MY=ra`GNkW46ia*jnV|Igale4K1M)T@jvp(FKa{(GMnCxsbQP!dH ztE}&rl){g)vc~zN+H{Cq42qiD5YjS5Yc)O+JknYdU z4TK`H*Z+NA(5thv=?LK?GSA^)iGZR3KL5gsD2x{O#VZN{{dSEukiLR8!pdm5Ut8Y8 z7olB=euL1SKaJR|hK-^-J6AkQ9)UjYU@^8;Mvo;dRjs}aCPDa7@a8~*?iF9e*D;j) z#s?K7bxCKC+T3g}ER*H%>sI6rYb5m`qW5CsJ6>agde?#TIbeGpDlt!dg|QFFZ)mkV zVT`hqb!n%I+?E{06{XHatqx$&9XkG~4{ynvDCbQs+2E;$(6*4&!>=(HXgYPej3M*` zD3R7`PwUwiCb-mx3vf=0RC~vd4Y-<( zM&L{SNb^}Ocf3RVf#pZ}1?ri$@=il!PgNBH6rTLMW8*Bx*+$Zrn*DhOb^zmJ);spt zI7i7h#xC{oo0@(4zza~vz@f`KS5|5RqojG(d?@mkg(d^poZTJP3Odgj^?>%Z4g>7~ zq;^=YI7panHkxwhzJ{hY`zVgvH2RWJ_;Ljl8%H47I0BT^2{ePqV{Dx>S9ohpJD z+y?eifAjXR0ryKr+;^g7mE@Quah2?q*5yjSzkSZeOop~H4pI}{})`bRQNH*pH z4}zzo;>ng~#kgBGt+^QODHH0*a`Jek3r*%iO%3AiBign}30lm4v|C7-qc+EsIdiSb z94wwVkd2~DSPj&9%?{POz(Gs`ReOzRGa0EnAzxJzEX4rhLoF?A)%JsQgDB2^G3YQ5 zyUUaap=B`M=jHsM+7ZVMR@x1b*G&r{uD#ME3_`3Dm~ChVR?y1k!;$;MAe&~(7vMce z@=Wl6YSHwZkREq<4cAmG{`9hn4*~tMU%{LXU(5oGY(mI`xXNwK8jh#EXt2AX`a}(2 z3KfqKtF?>J6;})aqF>WUxPY*uYR$9l|2s#$jBlqZUAJdP=n|>E@PImnAFlb#UH7!xOI~ zF=lTQeR?V`Y#YqkkgkQ{N~mVYN*J@Tc;E6!OrZwnLZPPoxTGSR`GMME?*md-oF+%f zdWekeRz9;!ODl7CkBnTEBNuM^tKyg2le+x^sYZOk(BCEltK<~MC8gHjo#j*MC*Zh9 zq8)YWN?yiwQvuX)Nm+i$S|e!OlAFCnZoOBro2-xRf?vUJCYm9ODO~~CwtS|Q<$ooq zPQHDfaeaAV!{Jq6OJl{K4{c6~yYKVAM0RTuG}(XQH9oRgCK`4cqnUmd)%pPS4qV8Q zRI$$17S{spuHLltE@{=gJf@coNsvY6H_04jg6vbF`B%KUK z%{!T-qwTC@o3fLTp`!lg(9Cc)XED&*;cY!nye*6WvboC}M301fjM4{g%%5BHAJQx@ zS9N%Ts_W{Zxz8sl^gW8S%zQS|7|`i96(xS{&c}KlWMx~|Bk(&MqFoF;<8t8C*!eN0 zYGK=Pio&xwub%CbE~DNsqgDnKd1Pg`+n9~8tB(1To2KFTeL;AMTTGcP=E@!?0<--A zShuJVR*=xzpe@q%Yd@Q!C)k0AH>Wv`_q}j~EAR|CDnX{(-B@-$L%|I_-MIWlXhQ!45rTEwxo*GG500ZB{P z6TMG(**WAT@_be1KVvDjOm}fdmX@`QUAZ8_x(y)M3#Xy6H|X|PF%{tO7#4q{P<{=4 zWXf=xmSg+E;-;RB#MW>pS_nQGhR&+DQaHs*nXL)M>@-C;E2jPQCq2!R)_}4NvE-x} zs?t>nK(0u8{4wrujQpfkzgrBUF)Zzi$=TB?z{Qd zfqq-2{_l3h3aBgc`P@Gchbj8b{b=!Rbg2|1tCOm^O>$&7N&vS|-5BzyQqdYx-Y;wA za8;y#qCJRdFnM}!IkjdcxIV_}bdJLgR>)iPBVA#hvr*pE>SguX61Q{e-5$$iWSjlV+$4m#NKAt9o?ilt7y5rAh(1d^`R5X{2&? z>u128LHfdHu^4NP$xC^|V_2JdVqj+o;}1;Pa$&g_pOY_zl&n0_jL@_{sY%?>J;EsM zvdm7>xbNWlZUqNnC*$bvNh7nBQR~Kbgb&Y`%w6hYTsR676v>&d1%5$QN)qfnX3L*I zy;Y(RVY$eS58fHne^ebtvw4lC{UybkFGx^Droh2<74(vzh}gnbPJP6x$4fP%hEjd3 z9-T(zWG*8eTWi6p=0;c?3{9f%= zhgp>J5ROS8Vb5zMHe|d~hE)v6QqZxFxC(Qe|s5zHEoHt#sAh&J@wpOi(*} zh|?pvYed_UZGNVsHlud1h3Mznm3T)&qDRpE%TPf#Cl8|gwH+N`oiYRcd-x&$L{ZnK z{837DQ;Q~1WlhGST6vPvLDHdhfU<4j){kgZkK`*iiE2JmRPM-tFF_U$9w(+5M6G|w zH~^S;MZ_!YXS6OLHaGG%Ao>j|{Xrl}*hpYOWE2qww?ENXy zcBhZGXKj<9V-0JzU>d?zl*dM`{kxQ%b#@g!)w*CT1PU~A4<)3ByJy z>jDhS_;!Ol^;|Bp?=QfN#!0F^N4TyzVfU#QRp!U9J0k0nVvxjf!Eg+_K(VO>xY2Eq zdN((uA@8|_^)2o9vq)7UQKlL_f4dM|xXXk{XZauNSJ?oUXG^bPHw}spPQ2ZmG9Pl= zP04R6{S{NlE{c!`4J#Bo&hye96DGJf4t7=vBC4TYR!e!ykhgZJWs9T*9#E6~VfAE) zWwM8gv4-sf)kb?tjZ1?Vcg$pz7ZfqtX0Qsn9#cdc_5S7~jWRVC{SA=^Pfb#87EiU# z$f+Q2kvzUkZY)n}g-}ooA-p|siMQ8Rs_~SdT}Wb!E?w(#^(S&`DZWkB@Z5Bo zGEM>B?BzQ;N}XrF9?I5IC|F#QkpJul-p4E-KbaHBmCxf@{-t%iH*lcvwq`JO*E@;> zy59)Vo(9ZsgM}|=UL9+deil?EG#BipfxV~ABzTI{T=#AmmrvAzl=X*qeaJ^&>?U$L zp{H8*25X);re7jo?1fc1VH6f|^n#;&4mwU_opn*d0?eCs+7ZFn!^SVC7Gi1iu8;~Q z|JMT`tHgCjG&YOZ)+f_YFyWz11sxXgh%u1~gc23>9aM)m04H5|@>*;KONKNWL;1Gz zC*Mfn3CCWm)Vun=ok5){5i4-Q1TU@pH#SvLFWO?In)^+q?kcr)^tkpDZ-JaeoSL63 z(52~rDq8AFU8*mK)3Z&Z5iffbhZYF6;YpzO*O57Y`LSVmAn2IO?amMCP0>deL( z;B==WXG_rK<|17(s*B(F+hBBoYNQaTk)H_ZklF`IV_@`o18&K(I zU{r__>Wi{`|L3_FIiI7)u{eWjfhzKL!lw&D?~?+lQA7)m12h6kwV$lx2HHPjh7R9Z zpvC8$Ym1@=sj8Ky0?Nu{!^w23Hj(62;V}z%tX3iy_d-fe5se`9Lq{YncP@CC5nX~i zb(AY^IXA^C)HxO<#XHZkMScJ=rhOr!kgV5$e2)uHly=_4c=Ct#Tgape+qvxgwSl_< zC7*B)>scjtBSLF0AK*+Jc;cX|y_qhBS?F!hF>M}@7Tfln9MX>CikL`q1DtH#8~36~ z$B(e0dTpDMW}&;HMi6PLBe{Ia7B#_t;5rF*u~51hPFjmBO2c$qVQ>6Ny4{oOfH$fa z-l=EyISe=)5Am=VFO87rL7tD(VM-fc!LpV|=S-2bs5u z*y6k^KtZBeEo553!X+I2>?n`=5TZgK^%_W?4Wn@68E9-1!q@!Bsf(J-6HIcD|wwPmKO0yty&(wvKFdAM4|u#Hf_h?x|B zdnng!ss5>m^-q~Dwn6Zy0>?}YyR&vE{;sSdxy!H`DFo{`agx%9Bb$|`8LzpJa12*j z5CtNS^NHKKie=*aN2^1qBq3)^=MBOR>kZ;L|k7e=;;glm??&Af$ z-%0b2<~$h`bEQa~Fx*rxU%UKSlBY89pEf9dkrH?jm~p+tSpL;AU5F)(9~Sc}dyAD& za=GRED&glL;$0JEZ~DpV0*7H!31SLTtC2m8{dITNG7=FbQFz%LJqDgO$PqU?vW!5v zj2^R+QA|6F3izyN7?*1&BWWgU{Yvy854I5QwY^sRW2`Oq!sfidRG>c+?I)2^!`u_D zTy9QhS)@vBivt4@ti0{Yk0D>JfU8&KhP7PT!!|=c<4mM@ly?U>1QaKqj+gdkD_q)5 zl3(97x#X*hcO?(jZ?qJ4I@+F@YL0MS$6lxle8dgT8Mr2%w}+OIHRECyZ)$T4 zT%wS``cDlk3(H}KjR;wSmym>_CETfyVZJO7k`wIvT z-oXOVkA@I7nrccn#g#DoX4RVe*zuRAZ51O-&2i6fw67LZ%sO6~$bCD{HEu9|pFm$2sPG}E9| zLSGQ`5Yu)|_-|WKfz!L4er?!Txc5h=#|M0)A^pai8$gBHrfe}A2kkCD%o5rx*R+U3R3{Bq{UceJx8<7vqV*PRTdhh>AY^NODvW> z?#^nU2qx=B(95`Mgu@=s11giYBHZpAcHAaf43wHiY*#W5(^;&X0r>ueVs`BlHN5Q~ zOcHHwJMrS-5Ukd-<911FEPTLI_1i`Kc;`%t=_xEeh@0vCWiA1MgC!R_#qv83Hm#Dt zQQolS1U9S|5{sARj7NVrqvghmEP`sE3cz8XTs7N*!n}T{c#(t13qC(GaZxJTVR6WZ znf!-CsuLtEenK1bv_l>7;e~-Fpk7PYB8uc?@~eYvzsgv@5%#;N*}Af@(j@-vloLQJ zFABhhP#_$6rMgk~j9E`=)M{z(pJTSiYBeF%%WCw+1=&3!^o`~5n!V4rUA`ZTjKfV% z)jtR6Oh|X<1plR&Eg#G+`2oM}NpiD-@m7+MVDI9|O;(rejPRN{t47a+NN`LF3J;%{1ROzbzV>BIMl(sl*`SaWt8@N7KT{2L0-zkV#hXPPoL%s~9 z9dD=|73mxK21nZ8&@h5Q+dBJfGcwh@3+>WtV~G;#>3d8w}eFcb>T z57o&C`287vFebZRfOzhxe-Ua-(*qY!8+N|eFTmw4n_edyx$@Q*$0%4FX6Vsf z=d;PC-TnZrBH}WuyB{#&^HEY%OhJ~jpF~0@tFq8q%N=JK^LAV9RUpVW=f8`%QGn7* z;(NnWhs~rS965bNsy$H2du9j?)rF-1Cu!2y+GPA96u>^G)gp!yW2Zsv$n-?Wl{@@) zjaE?v0mgrtf8|@hQ`0mSyKr|9wqN&m6bpm&Jk_&B-+9;Pe8&K~ae6=EGp;xSmhsugv` zhDySC^Bsp)6<-hZ#|juynuFPrOb3df)Gw!QVnvHf!bx>yQ#XJEg)`t8gS0%G}fCyDX;Xl_qf++F?D3jL1o6NQinXUJTE&N5o zT}+<6%B6PCgwQ~uytNf{o#Wh(kjOybN8~v@El?p$eMdyAqw3DNm5yCJWA`9*w-gY? z$Dax_j|1nBKS&@qE6SXZ4YALCXHF56#uNv;u;$(CR_ym!%Er%cn#&qSZ<*ZADzFt;-3Gu(V=;oYm0WYL0f`FCX*h&$t(=xCxy%~${ z8l0d}Y_mZ&x@AY@9&DRgg7R(j2X{z38$4F&)~ZG8pR}NG!_uAh>AC>S?uHP2OP^|| z6x>+gGrOEO6t-g+*QlTk4j%|Z(-eCubQSHP$&oX=x`<2NENrzJ*tHhRG1pH`M)XR74D?u_BDfiVX;+&40bV zlKnTvzl)5cn10&-lYs0#Lzt)E*4NFtF*jKd)M_8gId5;3VbF0n$4Qp7F&!IE=l9aN z3MP5aeVGx489^TfD(c;GA4%%=3{*DTMoFF=>uT z)(g(`>@(lJ_{nkn&Et#0dz9%s@(|q|GN8kxOa<|sQHAFWOS5M$1WNWT{n08T0mA>L7U{nE{i zXFQjvVQ4~2j&@Lb*TP7|SmEXyp$ZyRm6s&DDz=A5`_=Ch-_k4ldr7z0!@ryveOE~| zoZ48xzG_Q4hNxqmC3V>wWxsdS{p7d4P)~@&wbc%Z38owwKJYG!ls<$-x9tAaFSvnC z0W;|wv2E6(u1bss&iBSqW=?P-@|HyKlXuL|tFb27(Y=ntAg{nTt5^6so9p{>Vw@T& z&`S}BK%RYWC-BEO6D5^(c(9exsKTCr0&ae?W>>#cjzM^+ z`FxldS>g6o9=0dppA*FezjicOKbzw)3l=-PV)pwLO!39Du>@@_aTN+N#8=PZB?jnJ z7Gn)xH@3MNID6$T*`8)E<=!lLz$5cphd#4e=v`fifYPv`VF zrhl#UAr4t!vp6{0X9e4(ofTiD{tJY3bckQ+IC!c-ucD^f5?}p`vd;75&;5@Pm+Yq# zOP(sv@}b4=W{)Gvx*gAnscJcFl>!SEIS)P183Dtn$z2LBX)z~q+OcX3|5`$y|5z?~ z$mLDP(Wo^{#+-~${rnUT{eYxDaC$r$HIePU`H~!)-zEFrNFJ{=6fc-K1K}(8Bse0Z ze!s#qkHgT>GOuNFnF&YI{ zQJ;EHPNQ*%0IoV)_Hg=lTMehEm`ek*@2NYin~kMz>oi~cgP;rTmly!aWpUI$aywSv zQGPy`lj!@w!yCZ6xQR<_b2tPjc_&Q37+K6uQWz;V-XBfKD+7cLNYwD`c&n}~emoYJ zfD=V9fDF7?EEKB@#Yueqg_|-IoXo!>yM3hgYE;s4u!t>Op9S<*A3CgT`o{b7_9Xgq zXDE}nmRZ7vxO4f&ym-0S-`m#_1rxEK5%}gN$r^rm-EjNN>7{FP=CIcjnZDShB-^GX zMvyl;X@RP=+o|b&%J-L%a})ZPFa76K^PvOa*QyAG{$T`9w|m#aOMGG9mmgyEr3((j zn5){v*f_(oX%BL9h;aHI5T*_yzaK-`CU@`bJs^{%kPq3&)wz$w68Q=rlN;NkOBujm zu;X}$`>F*063eI5c$nnF%f&xwt7q%>kC@CK=2+hl0NkOy?nhRa$NzqhZL$SvA^rD{ z)8G9G4vL+1l?|Kvxm1Y=m0U~MNLo{|4ftMfQMjGO7Fhg}6szuD{@xOD?L4%5Ktxa5 z+xGiV6~}mK^EpRCJ?W{D!s5BPXBSW1bge6-UHTEokeAS}+U+F_yQI_s+*TQ_r>(XD zr6z>7Ux-=JF!}#65T((fy%!Pf)CpAO;XXCoUDjk@M#bBw4~#PAGT&~mX5s=vR5hRY z4Dr-;_zTyVRkFL<^2beQ&u!0ys;P6^)kj(Pi2YX@93*>NGtK7|2DNW!W6*yQ9en;( zC_W6JT$h#Tu0sULRMZu((#{~cV6p-1s>3sB z{jEyMd64zb#LR)Ak}KA!cSxR`Cl&%GvB)Ku1oYO~?I-3QQds{CEri|B8;Vcab#VWI zn|KL@kqOm#rN+t=QX2}oWFbabt2kM*J$C+lim<)O<$w+bB?<^++Iy~*d3a-j`y3d+ zVnye#$J9J^EZ#{u&A&%Yci0jDCuHl8>RkE9snIj8g(n%Rak`_q_l2Ocnl;Nwh+KEhNO%DKZ z&r*ETn<>IM9XTVOZZ_p~NEG!$7Oh|Q+Tq=%j2*WF^z-Yf3a*@`L7`>^R>*VJ!LC$B z`0gfWiuCE^i&E+FYLGHFU%`UiL9C^E$UP<*a8jo*OmUMMDV~`=vQ!O!cH;qMAxBc+ z{yye?$7=mdS%#alJAzD+B^v&F+$SG1{-Yua{PMk2YAQ(d$%m(%KQ1J!Wh}K<*%%9V z&o2Mzp>oaR%R%h@C20Hi9{mjciHmvO&(*3bJl(tW>E)|^~Br#Q6{&#G+qF2r)?E|sxF8})cPs8>3N98D0YZ2lC9e$@gxkY&h{D0N^ zan(YZPkvmW=1i!y-J&99M;Ug`OPWZfSrtTU$$-TAB;I~f;yoL~?SVV4lF3RiGRNa#g&^wrg&K2W2+ zL*S__imel6iH37SJ5s;@s+L5nS32Dql=^>^Swu7a{QdvH9My}x8ni(P*o{*n+R~<4 ztG>6A8K=A@q-KOe0dKxg1!TI-P@l zVmqZHD$1!Ie9o)+zx2!zw9Ob|>d*>yU*|(@F5^GCXNigFv2(?#jc;G@JqHH|DDR0u3~G!8wq9^KBsqErGVRVpVq5`{>>$$JV!_tA_?1SX~O$@U{*)VR?nuFU`V#Hk04J z&K9O~nY8O_2qT7F0 zVb}Hqa#K`ud^bRnPM>U~$U?ODR|M_J8Y1OeizMv2N0T|_<#>4qs%n)JV3?-&8kV?v z^wKb%Ey;&R8U1f&ZU#3G3T>|0wu)6N@dRLHka(^AqHvLpudbU1NeqkB8XcLh-`L(p zoN+DY)b5;v%HPL*8OMi5Y+CA^m_s|de72#&Oz>LodE!0sh`yB2GvuE!P+Z27q^@4C zEIlET0rMbgbH=ia7p?zHkX*a^ZC@kX9vdD557*mC1M#bpe8G#^K?blN?oJc z#EQobol$Q0WPRvR&&8;VcQ+C|?J3P1*_3622&Vh;ahbM4StgBS!`};5IG+dA4zS1f zfk~+UPW?OgAQ?l+9(oVj^IHv68dUvL-D|vCT??DLKf@r*NO`nuvl=G_e@r^IUd5B1 z`a7f99c5*1TlPLbB*SwH7?r=GtnDBqHQ;~fle8FC3j!ONy6CZ#8=e(m?m@NnNFY{y zVb=_v+oIv;YA>(IeKUYe8!IqxjQqJS6}R%&Zl?hurOWeLmTx2<^Bkucav`HaZr)*| zSHui17we9~1HaQyarCgX1m?8mP|zh%EKv1(*tp0Yl$Yo*dy|$?;mOa;Gn>HLk#B2s z&*l54N@)?GN|Y;$x?(v;)ua0!V#1P>C+5imDlRHBIS61-SaK?(~atV{p_R1fQE-+vft-epsA*Gzfh=za?MF9L)X)59#blV z8$}(~I0Xm9+3hR?-F1R}m5q{3ns_wlg(bUL21hz>MCOc@q!11g^mcLI848;#D5WzL z4wV>{%%R@?X`6+0ID(y?h8rscBT2A2l@uCX&cn*`tb1>!asfFPowSr{BW}_G4=nIU zP7N5oF_Zs?{OpM#2#7LSv7Jr~V*$pFX6 z0TGc>#ewOB#O;w6kt9qq zL%LA&QeiGd1B1(=IsviVljb)W8R2fL4!?dTI$V*z!GaP+5&3QxDr;ivUSp3oPKZr?pABE$LI$jm{%#yJdtJNgQ%Z>mtXx+o!z6yaF$a* zeH^($D{Xe=1X9u+u-<8$&6_dIL&U=^!aHwVQ*{K-unwDCS*zK&dhL)nZu5f9P8=O= znu}I@tl3}tw|HkEz6m0y>4o5=Zc9EL17IvrUx4pj$rmO=@)yt`@y?OV+Bz+E4LC(vR`D>R#3KT4k)> zOM#PHmQ&FOxtn7fgKJ@DxOlKE5!i?tE0R;-VK9M#%%DQL%9p%1pLz$%(sArxFY4%@142oi9AiIoh$0jD)| z?H3o(0T=xZH+e%Bzs$@<&rG4E)=F15iW9-;#XV3ed}^~}@Bex^Xw7EEfKOa}g-L)V zC3Kl)dX<2s))2-6D50<})85sa!0$8>+_DLM-FL)&WV;cR)m!LCAFdX{!jM^fs5y7% z$0BS>RySqLgS}O>95&Y5a1SvHom3w=PjCs~=fud<8CCT1{X!#FoNnQcfnrOV()6g0 zW@Z=mKIr6_0Y$ZPYgG^9s`H&?cb);dgU#Ky7yQ@T8B;YKpS%=1AVH*<6^!2`?TfXA zcd!4KuiwmAo{Np=g-EaT9ySNxq6$_iz#J+nEwHvg=d{F$NjT@?GJ-{u))KA5`n@l^ zq{8g(?ck_g+M16@4Tsg7k>=?rjG46( zI0vJWAaNO`OSZOgJ7}M;9OsVnKTwqDO^oTfP|%-~z!6Aduud!cP4$rGf|D8qm zqrg+$-S>jq46S%X{o8aksOZ@FgyVlso1+^y*vMpd`Z>U^h;yy*mP2EHOv|AWFAkhH zo5>YxBeDx~o*4al&UCH?k_Ah@9w2R8mDqwvUACQ*5Mn`6)d$Ki4NM1!h#oX|6UuYSXHCeMg>@1$sVPCZl?BL6 zcJ#qnxiO8l8XS42aGY}qKPVTt>HTp>K|Yv0Sm?fYvM=ZmgVDBq4~LMwMO>KL4Kr;R zTt;W)LYFLiFn)u*@iyXQX8)wANROu!S#*-Fchf=bCUB3nx~6|$C^E1rEqBw)S9xzm z`YS;JCP&_O&Y~wWnD3vIwNMZA>LH++v!-UA*5jNt!W4HJgKS+OOhXJkjAZVe=90UQ ze&)XIRoqW@sGIt9bR{F}^~sy<2nc3|y|~@rpu&PX(JzSR7*V?oO4c${hY=P{njOVe z+Q#4lYfRgblpRXuHkDOO3(|h_V3jX8L{qs2{_v7*Yt05CuTfw^Gi~RkB!sYdm5il2 zv-X12b^b`5l`{2AI@JTZIKUZmWqf?BhA{#7PG6{b(Y9^8Ra{W>;)A^`h!8P14)F-j zb9^1kr>^|i7FBUwRU<#M(%tuO$)T{n<-f=?<9U72z?P0EWx}r&9?Z+(>M7p4vEh(E zrTS-68-~!-BUHV$atsilp0w?6$w>^$NgQuf)+L$o3DcfYN3J35PguPGjh`M!Np=nV zH&l+eRuaQEEh0`0+y%pHBkA~c)62ONtk#U=9f{rqA=dy!Q-hCys|cqx!1#DD=CMaJa@m29mN5TN_m)?z9yALE}Qkm<1#XpOnpc1y&z2Ji46q@zFxhu5{KH*de ztWtBY?6#A6Oi%Z4g*f$}YJP_?Ms|RDFX}DG*v)x)>0IZJJuCe*z;z31WRnFg)X0c| z%x@nV(_J;D%>pfkPG9U2K1wryIz*S5>u|OB!0(kQUh;8R`d8y$i;l%Gn>+2QlF1&z zH8+g=iof{@>H5HUmht1#!{zEY=X^RGvEi9EOW19jv-VI!Vb8s_tKz$apy`1{`X(a+ zn!^0c)*@EhD4N*g**QMnOYC?Usq<{PGlC%cLKv?He`oI}^nvVU56!~wpGU_gwDHY9 zrzyRTay1EnhL6NEaz2abwg7EUvF1y`;ZJe(ka50&303`qQ3!b?9{>@kDvMB$1GnMx zH&XsOdQ)6qLhPN_<4muQM@7Rw9e+eb?%Y3X?liCc{GFqh$)K5pq@4i9AW?t(v?sa9 zC*dOF8ev2zL;oe~?eGwc_bTjAVNreFFmE%?c56s3ue{#!AE%AwDgbRhg*#3cRL8i9e@b{0BR z{W$MoEP;%i5EhNmUuQHOH{<~$2Ai&)Sy3Hn9v`FX#6#)Vqi{$$J# zn~zrhVGOV=iGiBv#HlQRTdwDKy!<0VzZbhQv2+fi#`LthW=hy2bQew~*p9w^29q_T zG*R?a@b0LE>=PlU4!bn=*@jUe<;zX)_7_hk3#fqD$_8=axF7wzSaZ_vjDc&1_dh}= z_Ol8hx(maO+5H(|&?w;O@QCBS9CtFcnHZsBN0wEIJ8e0kmOeXU%@Pm+n*(b<&xyzy zx;u@RO!$+5A+uR9sP?TQTJyGXtgNG!rBv|9$RKs>SXT>Se}cc9VmeXZ9)^l^NTC( z4>c5;(fGaaoOwU4t$J1P61`5QXS)2ef7^l!WK4$~?3u$YFPWc*cH!OYq)iBflscro zma{95XZ#2-U}rQwY-809jg|aQ>;%%{YELkW>vim=pIb>87c%r1fFziI1%G!glKn)G z{nfF(&sX|7n<>vPYa-g`W<)fxI}ZqrD^J}M46$F{Hxd&>>OFVNuh6j#m2_-SOC2MQ z;NTEeparS=oipf_{P?e=QAu6(7uOROeb&g?H1=K%qCjR5`R#yHIm6J}BRl;jg&>HC!vf*=s*e>DReK zP6RyHYN`~1NdnMu3Ig95!GDb0&eStIl+xsiR~-b`B^-#s{7i<@JM{Y7txTlbJhxc2 zuzGQ9#9Mz{_thk7Gx|uvd%?!w&of%E zLsQ2<_SgSfz|&vA@5LB=^3r+=3oVZJrNnd%cKCR>642H|*W+`~{=P@t4PT=B>;J*I z9C1==RyVe8r;F3<+h7Irq>gp$zI}bK=FL)i7ct4F$zP*}5E5*(V|0)6M#^Qs?2ja}L)+0JVq?55lKQXd$?J9nM9&cCsn_?q zkPrG!VDJa3SaV*B?wj|~vaf^xRF{0o!jp{PcXHx!y{O{%w}dIVidD*WQcz zzwJ^M)CmUN3<3bL#36){X17`W^YG-N=F;4gE}NQ(X-;!st;O(yBn z5>GbWuzi3%bCS)H;%kx+*h%k$+c$gfd0mh$6TB{sjUvoFFECCqFdLA<^AkcqG|?E$j|m#C$$ZE_C!!qvM+@#bjZ6y+h07JP`Cu z@~ufXH`pw$kDiSY_Wy||YgGE(SFS^9l`!68!(S4@X!j6`I8+8J;V8_5k^X-iUT45^ zzx%J4WgoGc`>LmH^|_082n)x*Mi}{o`TX8fX7eroEz5Z8Vd6TMsHE(ymwa`=0YP?S zo>IV9h|~jfA(g86=iexw;C`pik+_Jy8#Dc9HA8Ee-Ba_sAjBT?$gaEtC= zPXvQFv*eTX_=1)f%2bR%w2J9jR7sML;>uiITv>o2M<(6x=AB~*fZt5E3zCL;*`4!%GmYJbm9sZct556z~AJI0(q zEeOI4f03%EPPGVNBQMSI!Zj~``NlIQ$uWPlJbV!ET3h0S89U2|vYjHB8fa@_zh#Np zqo;3orr5Z|sWMzFgjrL6w@7QhIa#;vX)Zu~zWGn)X;bm{p)hL3GEc7yFUS%Sm4ZOjL}~mY%yEo@B*pPf_;6?ZGk1lzgWM^%K%fUgm(5 zDuwq>tUPW*8AEYh_^jnReI;J3Fxyxp9ayO*XfXbOEEwLN^F@1Bkb##@I^1ZXZj_G? zHS6mCP(=aTPJV9pjo0Z1ijRDz2_dJH=i(({>D51y?bW_{Q|eJr0BGL3Q&%GxrSZS8 zm(V6KH(Z17}rsMl8X z$&MKdFX@vC{-}FbgVCFK6i{dL0k@%62!Z@DC~*+ho*wcYGVfp|-L9!ejQk=EpkM$$ z(pyDC@I$8T-a>n*)pw?W&3V7Ey9B-x>Vv^3wih_K09~m?xp1w4`Rd(LpTmdc085)9 zXH4uu{CE87gFH&GBO`#w7nn^l9D+X|IpuSu>c`6{`yDggt0k>&$f~Ikj_@qGM9ITFKR0U5L_%S5Ycm8bBcnm@~WZ(g%>%`9uwsoP4 zOKHnj9f3y6#yd7HFgSiZO`!B8p}IqyJo~$LaR#>q`KM$J=MVAsK4w9PYOXC5ri;BK z&fjOG2b-~aHbba6UG;KH{GIUTXpfO#`%5`9BV$V0XI{kTo)o}SV&yk#(4a2l&Cxm8 z&-V)dZH889a9Zk>setwC>D~W008RGOl|s zci5+gZy)kxY|VzWtPqWz#Q&oO$+^t_vqL^tbiWHT`;2`DtVT&oyrMWhLs%PuCeD0^ zSk8@|Crscvk94FH2~r~aXK`o{A3j%!UA$L@jEY8}?IDirz$SpS z=2kQe2HH&(hAoV(amxV?@zy)M_b2{HY2n<@{{}9(crzd^N`$4^xEy_veibTY;Hl(s z(f2Fy4!fw~QI4d12p-sR_DpEjB_Zykpf0Z{5Dv^Mqe{a}riCWXNdHSzkS#<^5Dm1@ zh7iT6>$6@D{ES6}69rWdGhv{=@=2}PM5#Lm4|Ek&?)^Ag5(EAwA2p1K(47+ga}V!LE=ciExPkQ`yk)LX%stR%!Xeyw zQ5PM9b_Mj%)aS8w%(+vUOC*6iQ(|SI+RgTAPh zC6ESCkf0RF&>*Ggq1kwC(t9?xD439>G>5bDx&gz}g$X)ffTP@UYWvT#)?2vGatftc zV&(|OmUn{hGZyyNJCP{bvcADBC=T<hvyK&`?WIpu%h$&lda%i_XC`j zaQ&W_$(~{i7}af8#!R_h*^NJQTZ+F^&6$Pp9yg}KP#>;r(=U*1cuBnGQdVS*XnrkV z#REM?D&?i}mrc{}zF&jE_fzmgdHSY}9Ex*yO~tuP`Nt3Y#d`DdI)@i~Kmk5pF8HxY zPT}ik^g8k_5w<7xGaPbO8Zh;n~`D$O4<$I+2{UXyQYY z?Rv{^VYii!lmvM{#5UB|U6JmiXW(k8+}98nouco}8aH<$iAOI%Qo8isIn{i-Dq~+w zI`KYGi^xd~0hyC^nkuZ6FbMX!$FSa z%3V3^9#Oum;r513wCp8%Fd3Nogn1$7gDe?FbfQja%1?Qpd&X+lCIpUNjJoKE(I-&C z=?1W5NQ{y~+DRse1i8g$xNs~VSr{n;S`cQyay^r(HX!-)lY{Bl?)%}F4|)&Uk(l8m z!b!`~i8V++hEJ-qq`3xnryavZP>dndPO|CVZtmr@s=*#6qq+TOFl+^?F3a;K0%I%` zj~=G{t0XgX2afBe#L>*|ms+eC)3$>fOfFzSV}NX+c1NLuG-ILTU9Wuua}=m!cxL-5im?iC5uhiS=uyB;sI(M?k;%ii6ff z6txw3yn&fS@+4$E>@WEWNMLDyHK4@e$}I{<_^nt&we?cg9mEa5#~e7L7G(U!!-OZ& zfozNYd!2Ny0^Vm$&)5*QTOu`%tA3*`Qgo966SdK(omAB{-YF1=)LuBw+H~tTI>SQL zMkAzI6eA|-5(&Jb5at4Ne;QUQ(7d|_FDvq83|a|Q>) zq_Od)rSN#&tW&}?ae`WOiq^FiffRvVkD~1u<@Vtfd1z@I_ZK;;lhKiEE;du?8N)HV zo#6ueyV0Qv)sx$0A$`-`^^Z3o2){!smqS~V${avn2b)nObsjK<1vkh|y68tGENakx zRm;M2iB(C@%!;8ex-QF&$EP;C3r`fv2F7a-XEDbl@}JjrA-*@L_;3rGg#< zUQYP6!us;W54(*jCSOIRL_pR#56YB8D;D3jlg%Z%Gyt^d+qu92&?d2;?eNMv23Dhvc?s(L$RMv(BT-z zsK_L14Tl_7&YqfN&NWyMx$i_oAr7XNCWg6Al$bdH0i4H0U|&7We8snLxD|Y=#qh)} z2C)64JX@O)M}1|M2pktxUs{2NWJzvHEG;3)iz^}Xs;c-0YPA~87Td=K@$mpW>OXMK z?uXD=ik1Fa)7bH2Y&~tWn0KGqVO7EtR!Q$IOiPDMKl#~@rbH2P2$-0D?3&OEkX^H3 zG^9x070omb?gA3C8L@YfPVsmU=xS@3mHNVmZ*ClEYUs=9;r54|Opa`xs$Hl`KoJXrkko65bD z$YT$MgsUlhR%gj?QDa0ueXdOM;@D;^GfhR4nkx=6PSqG7=y<)F1w=sYS1YM`C+Zz3 znVP7QLLD9oy1qxZ$_oYSi4!n$L=CvX(XomeV#*yejZiytk8n{1xh}^jMDxt3W>flmILNA!8Mxl8;mK?HawjmlK_id#u?2SQb&l`NS| zcAB_5Bpzh>vsS9SVv~RZEF-zT*IYGXe^NmsUUdTub@Lj2fZUK0);!Zl%V2y!HNf0_ zZ(W+q<15DQo?^?N6|=xDCdQkcIn<^AKGWSVRG$EIQ~RY_6I;UycN@#kq{D_{?wiv? z9#&dxYhW>Hl`X;~>;N=I{=C;nOCHWTGlf-~Ii+l~8bu@1PLs$SOBQ4SAj2)e!#OOy zLh{-vpNslo3PaZ`q0IB+XVb*PY5 z{TF4T`meqDX#@R8Eg!nYZn>13h=0rZ3h+j{j`g*QyT;2f2XPz!Vzo05x>&vA5w2LmJ>CefgY#G7x0E!*C#5x;o25gj2`_2Dl+Hc2=kv(_&Z321N>Tk2C#V;5 zh?MZW033}iW^`<|$UK3av1}ZFa!#ujZ6zNAE#!L+ ztherJs&#-xlx*RK)ihxUd;W)p#R82ULEDl_F$>j6NR20kLJ3P2@al;_b(xL`WY$c! zcyO_({BPTz*Y2(^(_BDJ(ce!r^F4sN-%u`wFBimEM+HaQ!sCT%by4Cn{MSqbNW!i# zai+w8PCfTC)m%IBd2n5WE9X0X5v~HJP1mz`_vjnQv{<~N0Ek=@&>WSPTmLkG9~Vtq znq@|wc8)(SXqY0E4n~t>VBEoU&ufHv2tsSHAy%dp@6g=Tlb~(V)2Pv~gPMivr=su~ zz)W)Ygjzfkt?(UHTsL%|(O5;IGCs8|SW>pFH%Ts+iV9Cf`nt=f`3BT;_hi8okt_2> z9oAL3z7Xk<(=3v|m_FAB%t|#lL8?B@>hgn6CgzGyrM8yn-omf>(=@q0FMCsGCIi#Y zn66g`uGtCP*-E*u=ZA>M>x3}VT8I1o<`Uf;sz3(-2FWtCP_RYOm_V}EjGBI0_S9#v zq1cK&Mfhlw2Opz2fuUZWwL3NV{7^5mUM*h^Ra2Np0P$-ocF@!KsV+zrG|%B`iv^-~ zBFt24bLY3{;@3nuV3f(kHGR{#sKo=j0q2GXV2QUkQcJYesyj|tYt42JDcX@wh|xF6 zTu6;MY0RUWWx&c1NcgDp&cGio6;~BAq1Z6@PfzbRv35R&{H8-^f(r!z z?b9jf#ykk5^>ZGoGt>yh?a+DP5+A*BGmOb(raiE+FGOwg@vU)yzSPYGLpc7Fc{U_1 z8PgSLQF#qJmM(~`q#>KgFZSgw9Nwq2y-P7sQF2&s{dJ!Xm9WQu2H3FMtG1VIP{@8! zDpo77CNBhN$Cb|)HRWk}8=`6qOkJrY5;jTHKaB5*eJg7xX_zd4fzKd{FQEW{Y+l$c#lOGvi4L5oyb3q_&I$sRhbPzyg`mk=laCJtgxgb%hbA=Nx z@LIGE+{tw`dDdJT9?&O)vAC$3!cSmr0d2m1Ae9i3J4|i;&2v--hCW*JDNVK`C-5!c z?$4cBP6e4eUeqWtc-)>`g3p{Ne>J}$j&)JdoNWY0GoyQfK~@_NMjw^rBG;Qy$>BPf z8lBCIOC60iJdh##11Baj=f{nL>qGd~PYbUGEX0SCjRf@1H*Y65AQzW$-Cn>3av0mP z5qG4-A`|Q|dgT2)v@X(~uoc_PtYU5N)XBOHRE{@pSAO7b_Qh5$-ro3yI6&04XBrgXK?HizH!{fo@Th5G1qi_qA?aq9yt;xw_C{#RRp zdh;L56%Giy%Q&U8UFdp2{KF^uPHd}Bl)$8c#K8DpH>YEn6iUiycT@%mEM?JOO)RQ7 z{SE`rF994NT%QE$CEuWqk-T{to7voo)re;KHJ5+Fr zpwrFgf2R!`w?3e;BtdG=bq^PEEg!)D3KuI(ffz~4$A64Ai~tgHq)&zKsEQO>&@_Ey zUnoEX20RZs-n^f_i~kYR|Mrder%T8*NMe(zdL2nU_sdRjCK7P3Cw7D%K}^{pLCyk! zKJ!F1(!`RN!k%gZHmo15DvyS4vj5`(9An z9&q#FAb`%%2B}=_=ty9QPX_c3848$H>F9B+(x^r0FQn(bc)H)-MYn9xAwg%H#^1Ic z%bO4Ep^K#4E1!uf`vR6E7)6STtvZ34>x%_a_Y^`3`|&fN&MV8mX)YCwhyR6rpbH$a zu3kP)*opf| zckkJFA2N~OY}zeaWmvdG*BE-gU&IM&F8OW23oLHqPKz9br8FkF`$2U?pY&hxS)~M( z#H~{#S(zT292x_bb&Z%*o2f}9Sr-+xU^TB}(bG&1)8N_sMM`wb$cir#@I9O@L$<}C z^|ZAPO@>imPKmVd#>0`G_OGB%1afN%L=2E$+k{Eh{{S_=V0!bIv}ab5M55z7xg8Ay z|2|Rq5ZGSB+L;C#jWC6^NAz(0IzT+CXA76t7FpU3la-NWnUW+%%)lN*BFY+p6qUL> zhJYW^5}oU7D`~uRv1a71F1qE|yLtxeKk)8?sx9ThoaoiqDKUqHRo{cv=S+@pvA4F^ zjf5WOur`mwD9D2RzMQ*4Rt!Bloom44%X%m8v}kdpHT()|>T2L)*A1S?%T|$$)mRFNxT`ZSm=4ds${Tp$sSVZe$SI{X*O4rAUK;aAq1n z4NcrCE#WRm5oC zz-FK$E?Y^l?$-O3?Qi$fa=_TW)i~19EKgP{1%ONX&qX^&}93)O3n$8rm+#lgt{HoUoKf5@g4~@_^3f~?G?#? z)%(~blPpsW0Df-!g%Afu8afL(ce!8rHnsbE{V4z-$d0Lb4V}fwz)hOH zYD5LYj{5ay5X)iJy@ptAS@1tS;F?+~7TI~j=MyzO*Lxp+WkWB*B5O0dJ`lM}N!(gn zNeT$3CFCPwcmv^;4pTmU$I2zXqI+*MJWT`_XK4wppN3NbzA0WIvnQDialAQjKrrEJ zMto~CjPo_C%PiVjl z)-2B&n}{>GShC<7AS5Q_vpmQA(X4!E&C}CUN;QfBu+d=j%hl}k4f?6+R0hsi;wgTz zxYb-&-R2)R+Cj~%Vg+)w|HUidL3pWTRvKEpK2aY}jdn%@Jw7?Bdw?UDnJ;_7$0A@t z2oD$e-*A#H;a2?$=$R~Lvi`G1?mziHirDF=C)icVp5J6bOgTf+2PjWu2w?o8kkcg? z%lkoQ(J9jS^FKa-A);!Oti90uFC?^TPlfeF#B{~3S1Bd}8}4a|9NEl31Zc_GcMG9u zOwNcRP~Dd(x{~sQjNm}9aG6Q#qcKMN>PLVE2NbumdyXKQA#R54V&zTYRIeyMO#9vT zYKN;lE86VCP981%31|(Z7AkI7QsY~9b^q9Wq%raV3rFpC4f5ZSPyh?^6@R<)-7LSU z*F~Lt?)3fk15e&9C}IQjT=q-!!}{F;Ymdisd?!Gq?6x{`*^%wn99mJ3_h5RK4vh*8 zf0vByJArwavY(26pse2$89fGhzHzkqsNW*5n&T?oQhlDY-l-1;EUgr4n7KtIp=^I;*J905UV9E8oXUD*E53;XT_P}uR0TyXN6R0*OIduMy8+iZ-4$B@+& zj+eYL>qPRflhU1w#pp4a8P!f)M;7v2s`rMl3cf) zz(ipOj@z< zH^7YrhtX-KD^~pBFU$-gk~)*gMV5I2-zIUw2CrjCIYB2i88>|=S>7cg*%lv(BG4V)1Di!?j9Ml@KU5#{=jB)}_ z6#ZdQjAqST#|8x3z~^WOB_{6Q{r%Lj_6GKkuIj2e;(yCOdLDS?5P9{$3L=`MvphVn z9>YnSn-NU~8mCudI=;rb&Avd%VTV^B4$#wRX-Z)-Lr(aQ`#l<01BkRT*Zc3&CZaGa zcdXHw(6?h)2JNnXR3?W%|5H)h7p3vGZGLHw{!UvmH-A$^S*r?Q8v(@q-i%bl14( z-?eF`2deHUNKV?3#58pTb)CLQUj@AhD&?f}UfT+T#&Ux6_h|q>3b}R5`S6!BatItl zz4(`7%9zR2gIamJ?xzAeQ(Spwp!}wsGQe)NU|eRfV`Au4r=nf`Nds=emcd7;|IZou zTSnj&Uh7GOW<~ZAs&55r#aSt__m^nXc5%jFHG4ik2rUR3u=I+{8{T#Kljka`IKxx@ zBfxQo3`*;_K`Ro4du9Lkfda^{6gObG@ZSJ)trQ~ZS<&7+Zc6N+H&a%dE?j6;9(mIO z%e=6N>30=Xj_-Z7>x>05B8BhueuHYNB9`K#R3&Rc+q~h ztH151%<{*cF!>O$2FnIhl0SdJ*7xb0z1b5O<6O&l zNQ5Rh8IESMPDd0FaIC@4`G)^p{wWR%IOXd@0?47|#csX(6i8or@S$=9z41x;S!q__ zD>3R}5*%!-rK^FN|G*SJ-G>TUzSorZ(5JjD`|fM8PN;|<`9!eQBg03G0KDfQhX`=#`!(@P6sN-N_E-nYd9Ukh8QM`gICZ3Hs}z=kj0EQ7<|(psDF6D;K548os#E zE{SbK4(3nO0CcElU~E^mo`{=%)apl_6BL-iaa?T`PEcPZYh{2SN;d4?*p4hhf#P?y zk>8*Ckl4v+8DBc~YOc>-&h}uZLB(|{fY515Ulu6BY*GFQz0IVQk@;~It&$}g*+x9P zs>XQM@B1?1^CPfTKbE-_8qi|>5=&vj*!tCM!E*+h<#8_`d6+B%t~&_{65n*nGQ}!P zFEG63Jx5Rb8|Vi79%09 z6pCh8?)BZsW~-YB~rDQRVf`Y$VOnvaI z;ZlQ_Z*}g0XX-stbv1abfbt1A<3bpP0(k?uLgRORi_zyIn#gFS_?EfIw-QJ8UOCgC zzgR5k93%b;S6;%;S2Vlkx)fSR;kk3^%voMBpjPcM7=%1`$x`4BNbrIS z%zG+nfnFGmXbV)7SV0wsw`jM%r0^mTzvdbs+6x$cOmKwn8%i>Lu)ec6gqX03() zMG_@56b{qwe_Rvznnnb{DVT@2`Qv+G-7TWP$`j;7yLWe3$J%tGtO4bra`N07Yn6rT zSXP$X{AYi`1K1EP@OFl2p2VuEcu8m3nc^N2$lvko0Y99gZ zJo)c1E!4C1by(G1Urh>b*_gc60mxOl+idJE1}^{FV4;hPbH?>1{4r%ULXlDtSP1-vpmfP(v&<#4wzgUqQ&1t0}&AMpzB zuY?$OWBvLaIkExUyaWKF>fbt{V}0y>l%*qrksgG#8QbCYm#(%;-1Ott3YWYN+;kyB zy*15J)_I*{PpWfimXS>h-cjm&qMf;vNRNdFbpuLwVvM*-bZJmz5yOPbG#$TlLL7;>A&vABW8LI8kld4jd8Cl?$^2Qz4tx%f%%WgmoSpYX02&pi~uqPsK9Tq33pPeyKxp8 zx1h%)wj8a(6h+jp=p)@*jTS?37qBstYgJzcp$G%vOlomTkak4nVM-9~OHq+{J$#eK zz-kykT|)-_Bhaw2oQ#!pkUb6&@8s1k@EFE41vO+^4YzX&;72@30y&$J8^#6l0b`{1 z*d}$d5R4gVt$emD)3Tgp86Ezt{mhutcXvj+U|?NmQVY3A0EVw%lbu#}$w zpHYN$Kdwr)#3`o@_CT*Gl71Ono_^~HQ`%qvuv}SX^d^X`ide+r=k}U zZ20BSw(T~hM1iXE=CCQ;(1q)`j;6dLb|_m!0leZUV5Q+wU;TG!m=n@5tfI@=1K`QN z_hhK3tZT^XVFh^;T2rxcwy$#xEfC2bLN;t<0Va6lnB;MT8jgXyNMfys5^POwOiMJ$ zin>0X<9-bic`W`2ES&b~FD6jek|CY#tXI>s;F=-elxgVHWkI(ae=Jd5tWH)e{UNZL?#Nz)SIJt`>q^(7^os~YiKbdo?a4i`mcE)56lR%ewTXQQU* zF1a=(TzjuD3kx&;=+VAA1-gE7T_d%n3PB2pI~M`u)*3Jd2Y}Iv@D3YS5*YE zvEv3qb>SZq5VG>2Alo~WiIC`2_dD-w_zWr85vml2R-(|Ro79oKk!G9N(X1@lpuG2(~{ud#~mZpLaL zrAriif7XN_c;*0cMvMOff+MMzaJ#(om*JC5RL@!>(%(a(1mo?Fx^!JV>vMp=dAP+c@-+_BOJ8Z}_rM!HA;hgJ3vxesRJ1wO@5CW4Z zndg)%>81C|Jm$b1LTuw(9k}LXUw72RfDpB$0(44EkT-7N@Vdgt!G_;dRh6E^wGIou zoemY@`7)7=D8O~ZD=e4k4GnSCaVX&t_u&yzODg-@z=$gfn)nz>&ef0_IM9Yc*;`K5 zqkTzmzQU3`OToOiOAO!{i}D|&OU=y0>QvV*2o4Xal+^M}Qqk$oQ$TrHnipxnME)um zQ7uAXP?-O0DD9ZU!|&yt(?F*M@kZlZ^H<3ZqDUc3tTiz{@Y7!CIw|wKf`{r8xjzh_ zvokS15nw>MoE&g(R3(?rCY(x)xRy+u|9A~+Aj%p1lhg^Sy%{R`rY#;npyokSS<`gBxsm%8izbWaK;5DwgUr}xqC=$<%>3F z54^B~=b}ZlGpmw(u(kVQo2j^bfo=!bUVh+f^g2;pNFb!u#L$97Ck?FUO8R8j*NpqC zCyA<=09kM3{wduvv~e=28;cLf2yw&u)&?NEskrpiQ%x+N-Wtp)M5~gnOzhx@EcE61 zLNqOo0R;*XS@g(zDj_i%WlrG$|1INWYX)>GAkj=fbnMoQmh&tmBUI##yZ`8o6nr5=TrCh0K+jOq%Y;qg3t%IIC*V%Whl z{i^u`-Bucv2C17=zD>#GtYE5VI&Gsq=n|&g97Y4jiW>)u(RT+3XxF4}PA<%97uaUM zY3&orM15H|BA355fj4L;&Qx{Bs+G-cQ!0KW!du{%#RAE+7wf9JsPj6VIo*KUjo-6K zDjZWY*l2R_&6rrDf4}BUbRFBcYT1}Tp~@Z^xRqlMqLtKS@yuK{J}uh_`lbe%RAM0x zSaONuPEAM128Vw_cIHn5dpujrOSfzICG050^yd2LiYjOXh_3{7b=$?@;_Sm?(xuc4 zYjcg7m?abl^arNai~-rR1P)7YcuV^sEqJ?)*SLfB#tz!}G6Mb;q!7z>PNp?O>vCEo zjH^K~%v=KS4NNDff1atIzFSul$E@zPKBisg07vkQF0Z19=3`4b07S6*4J~TW2~H>( z3)Ma0dGp`s*)g3~k)=rVVmb+_mGK4udD}<@!zUi+AO788C5$6LX4woVp{Nm5XL9xOMGosl zw*a|FSMR%giS4rqW5Z^2hnbe#!e_Pok*^ka>}(WtAnzMMdHSU+x4%28d>*$mF^H8yp=0!hup>(l}UeBKBMyPGib zN|5PMhfti6UAB0ByDn*+xOeUh_T-|0tb){!M*BZ>ub?A>DxrWbXydM9AuY;A&rF7R zhkk!^e}Dam@`Uj8^oaQI1%|PpI$O3zVet*5zRfl~ua8BgBXXFiqZ48dvRs%Y(z<}D z^5BRP3TV6+RLZa@D*P(&Yb*&Qq_Ud)nN+KD_U5Dx39zrD>x(h%&tG?+hXj{2Hrv?H z>dC(7>c^WiLOfbe5-(;n*NFI^spOg6#l22ZlC6~c1u0YJlPSfCyhk7RKNGbUzYdpI079HYY1sANpQ~^{-$)xKxER1iLl?KiGK?M z_43zIYXlBD**xb)LUvnB*q~*(rm1f=Igi8uEHr7g_uFTY<`0+&{4>M|+UkK?CR)}9 zpV8Y1aX_PODqy#Wq7wUDH|K0S6xPnjw(%0 zah5c$bD&>Gti{RFg65GsWAN?`M1DFl0={4C_&(X6H55~y!EOH1{ia~GV$HO`!&mwj zjfn2P2~q$V;wdYz$IFaTgVpF<6J{PwqCk+$h74Lzjn`10O|)SF!atrSve8GfeF+}l zlXAqZ4>;bx5n=adR6^B+1}&IQi(%)`wcm*D>KC%#^@C|}6Jr{eS4S)Bc&%g#yRdIY zn!X4CB~E3rHTJnq?w3;MsWyE6|AwL6JW@RJL7cafy-#%xqyT4IT4XW*$7*G)c1&_tEJ6&XHX}-)^}3y?+~`{if>2yBFs+=s?v- z2SB!NC^U%C^2)+)mgJc|J>KosZ=`z8Jsxo%KZo(`@8AK#Rz~2D=v73tAkDvIdA@60 z)1kQea!al{sAUwP4RUuiFk8cGP%N)TBPLk;&+I+95f$=pikhCogsK#3DhzBz3K}3b z9M|O=1DL5hnyJ4q=SPm@#Tf+a;1>J?;oF=248=Tv8TgP1o8MY)|!Y5N2Qfw6@7O z^Ty&E^O7)bMzWU_r9)f%n{t*Six0ZNNq|~>sGG0+W$;Z zBBfXwfOOo~1$z_{nsAjUBV*vt-ovJ@(bfZRwgvw(tlN$!rYEbh-4=C6FFM z;b^~0_q9F_ zCi_#gofbe@3a%feAWM{}Dp>$K+X$aD^IIcowa)Pg@cNvsFuitR=2(|#I$U8?P~hCx z!(T6oP*^I;x72Sxx|S>~1VLQv2p9${kx}*U=osY?D5e&gJQ&c-&nw3B>;0JRzWkl< z#0wpuj)pG*s&gu-OROz_lK0-6AjE+K`_(#KtCoI?8bOgsY=%8+Q&`*BJncql5xhR%x( zR5hJ(YW)pajM8pZEIXDupBZs&D z`D*L;b6F*OV2v&XilY)`?9k;Jr&#I4UBN`EI(^0a~tn+a95gJ5yk3$TEah@uh z!oSQ|*Mz?pb;uqGI}~YejgHL0Ry(CA`Uwp_!l>Qwr&a=F)w&FzP((Wt)73 zzDA;UKK+QlvMTB2N>b2=rn%o^Y$!?HUy=>v% za>z?XXKhCgKzpsNLu2~U6kSX$v~-2WdSCUio5Xnu=#MA@{yBf-qyQv5_dcyzJvAI2 zcVcbWqES&zd>YPne+%-dXi%dX-|s!bir!Pd-+!;faaO6B>GHegx3#W@mcHL-dVCN^ z_YekI{}(61;b=xY5VojGBZBQ{3DJ;$LJ6h( zC%M85Y$^T?`=0)t;P2P-&0k?vIl1I>W4@p_pYPysTv;4ppIh=O9C@A84Ow3M`8XCX3N$E2%Pyv#>sH`l22!GBTU+!>;|U9q@)kTh%61xTjXRJr zn6XtnED^5-z=W{yb9u+`%=M7=q+_z>2R(K>`to18U3PaGr(TI3lE6T&PW`6TB=JXD z&d^)3>utYXQ_z@%hNE(fpLk-q5sMe-j3D&-@P_39WZK76PO@TsN_beDH}&8{D}i;o zhyWHGfQs1Ac`-bGR1%W2>IoU^YQ|V&;jK-CbZm+mymrT|Cx99`ywl*XyNU?Xgb~Mu zN1CFsTDZ{G0e|c|x)Pr>nt~}%^V z*{w-1H`}8%ta&uLO=_}3_Dh_tG90>NBKkp{-yrRQ9Dc zL-#UI%KTc=cdJBgzk%mUP5W0G5T^@re%&n~qIA3LrDai{hkb$5q)sU{8wrOSY$G}% zs}}1SjLlatQRK-{Ec8Wh%rZJ%3(KEl8w`T}z-l1f-=XVpuq&~@P-@RrOI$8AdoF8! z%)nYAD@bURC>|(vUW{BF=pN|0yt7Dp(gn}la>>UU5l5)<9yqmfvX1`1M7R0YuzdbK zIprPX{tQRnGJk9K&$An(G{Oz7JwnT$ID@7Cdw0A5A)RBg_>lrezY~0`nO{x2e*P4i z3Sk%M5N2}$Myae@=}8k8Lfw_=uQ#{X9oiQwFy_gowLpG8m1=4db0B$b^AiqhRcpY` zXX|ZJJcvk6&&$JN-XpG-Q>|{8uXi8Vi_BmG@?~WOAV$5mS9AFf$P0xP$9Jfk*AIHH z2{kQ)VfJP}Li#eyCD~+9*HqGgl$&MRdsI&0)cWPyl$U-7`+m&Zxl|4!r z_QI#5&Nz^|u1fiTXvE%`1Olpgg&j-WoK*_-zEvGPwjT(D`CK{-3f|>lH;&h;crOnh zKGnY(6Yrnkmt#6hA*cgqfemn~Ij>QgPOEh2^pXXki3DfWUThY};Mx9`^*-wr`oK#y zk`&4|RpO&1(Q{Qo=$pdy{^6gA)@YnzZF2FXrPF8yT%p@i*m)4SIhHi*R3eJTZQrB7 z7-d4hG%44{wN2_hM1WB#TPoVNP7!ah&H|jx*d+zNR`2}8@mk*w`%XeRCHWcnT36;x zSX8&sg&YyF90r-gJZxjOAFoI-Ewg5ZoXHB=;U-T zx&=O$Gq}aA^$b(+9=Wkwy7t#$Nxv-s6iXoUBcwyR|7Ms=R^Z6(jbzP0c^eh$_yJypQ0Q^XHU@ZIO_C3_>KX0KHKoH6ri879>Arn?s)M6Q=bwYl!8ooqZ))c3wLLU z6c$-mE&c$DebmJ}m2(U{rJ7mgA_-DR)t{7WUa#2xqzoDNCa=p3u=M=XlxVvmtZH*t z$iMz+?w|bO5GsgxbpiP?7c+pcCqUuaaE5iX?*||5}5)`Ig8MevUrP7o|ld5V{ zGHn1T?6SbOgmExTooSpLP3~+SMUo)Zv?8-U!CkRR)5vOUU=q3?@L92xi+{NIg@)AB z&A4Ho`zk^;74vWyN@XM>69Ptl&K%gMh71t%wYIp9O~poS^R5tdrwS)phGl&L5&eh( zQl4U-<;2_wlpN-lLjfMHnr9VdVZ+VoD^V-yC2@*oHGmg3CM=R)(N1PbOc@1IZDkWK z+Uk7v$zNv^$Kqe5+dYBBbDx(Z6cCJ5GzjwqS!!6E$D2AjOGA?0!<%ZS*#Tt$&}37w zNdTI6>G2xxO5i6Bc+}x@XwrYcMYny?Aa547%i7Af$F0@p zwLj%lN-y{JPP4>7|LflMM@hozEjddm4+1hI9=&=!9;o00)wp$%@}wPikgNp28OHAP{G*;QCTAHt$lm&^E&`-?CIjT)w`*e zjm^H7`a@e8^iR1Jmck>QuncHrN<4qDzdwMm=&5v7xx;?yCCsgH z3~&FdTLT<-hsr9(>eKWR6fIlWNH$A0*Rv(C&^@vr!4ru;N(3W-0E_h^h-PMF5m`t_ z#+7Be6q0$^XY%i{A;?P}ORy--25=FTp^Ws_n25c#wH^*)!yi_k`Jp zy7mBs7Qw3wiVINPUZRjDFmSmb$t^!Q8#yS}_(2*umahwk%slex)IpGGyt@GGIQuO=MDEZew zkAA`)QWU~oh;!TwNMuLtJh5T#=Ww~?&_Y$uc zUV;s~;Ns)SU+ZL3mr(0g<0duk2d8qc7*W=V);3g$ z3i#+m0@eJWI6QvX-N|gw)A#$#`c%F4;e=0sXLEVj<5=pD2NT}&62l(vet~Gdz`O?b zQYItIqFWz02FKjyh)sh2oOWeUu21god3Ubxbw5rYXG>D^gQ&k7Xzh(BJ?FXB!g==N4;f>B^TxIh9}svCi2_ zTGBVvHw@9F=T_G>#!B_ILSGyZnYIdVcQ4pariN|S-)@!V^Xo=r(vBeGg2)NTv$8?xDt)zQ3+H-D)pG=sW3-HS4M&5z9rIU_KLw{ z$U~+0wym00#Sd4p!s$wOy26rtKu=Z5-o8upOnEIBo&*g zxp$ik(;!^WK|=jM!p|2rs+qP}nHh0Fq z$JnR;(VFXMo=rW^eO(9mXjA3VJX+MKNCES^xBIjFTB4q_WnQhiGqgifdpt{P)sK0dpui*7u+o{k2NxR zl;2HLiVgQ)s?H^xm`?@)7hO9WUK-v)s3KqX`J+ZHeh2(mt}KSLX6w-811>&VUu4?; zg;VWDfx@Mdom86NJV8m%oCw|qJFPivQmcr$uhR;6bWU;}FJ{iJf37U4B`RbYNsMp5 zmxLn6(?$u~v){G40%XB^Zl6>)Io7z%LYvi=RU@7}O^rX-wZsjnXCNA`=>i5@+sN#3 z9SY;+`xfm(%{e&rx=EeaLt*BN&3q85I0XbsdZ>6PR<_F%M?YLQV(!xagy-aji63H- zv~C<}6Yq*zK^Yp1O@$Ltm2JcOk5cm%XTXOqm7iLWe+q=CC-!8>3=U9 zKXEo#JiL$)sAW7DwpIGy|7+ZqmY^;nZ@)gTxzatIAQl;^DriqdNJE26hOvZAPGXzi z*UfnUo3oN8ki%D5nIsN*g`-PbDu%Qla6q@u(outc=IJq^ZWfFDNp(2GJ>|OEcNoaP ztCN2V(iZaxb?%Xjtc6OKwii0a&wj1BLAO1fjApa(in1d{d!Pc1c~RIq4l-}Cv5>dZ zkjdEG_S8^~jxG*O3o1%?s{9U+`wSk%SDrR3FkAg`QlMae^x|_bEakSxlZritd5DoY zASok;sWzydP=q^l*Q#*d*tEsRcW+$+HKmr_ZQeb{j|a{AYHInNB>Y>MrvxjdZ)PeD z@Md9}p_f`ChMoKLwA9n!U_7|%qjWe{F1zTU>#dGHOBDka#Evy2N6)E2{;LJFgE;Hr z1FzqW6MQFvE(Pq-WLAsOXGt<+^qxDAlu+4d>QhjCB|Jb1!Bk9JIl}Y=3AZd!8@VL( z-9N3aeL31I_?SFFUg>SR`Fkyd-~2X7bAd=!M1kkDm_h6)tVQ-T{M4Kc0~Fy3cd-~J zWVsnjBUv8DYlXsa0zFTQARRGz{i+_CL@uDMy<*O!Dc-8&;1jOyD0No&)^I3`dE-?c z%Ui({2?_8c^HY8mEkdhiI6d1}2}%ManfF>WA{efHX9Z-seTd=7LGg$s;g+IT@3-2d=72mxgg9*dVm2|NR`)f%3O<37_i{otj(# zrrP^ce^u1Z#*{UGJNRR5bg36$%zp+F&9qO-BC~|_Zk?}CFFVkO?6|U!wM&?oGBOrB>*rG|H&T3tmZnI0TzH7?Ztq$fJt?49|ix)sY zLrb}8D>ofZ6cWH?>{AnDA6-ih-=96we17-cq_b}s=(MR1>Sv`=d|H<*Sr}g`w~Kn1 zmD@#{Ot!6*`L>C31JT0eL_mv*`NyQrfWbmckwZTm_G+Q&PqjuJ=$1~p+5Ha46XGc_ z^A{=kOPQbZrJPkl@385d`U@x##)Szai;6w_pofr}t-@Bn+xy!weNk$Dxf_PYOOrJC zlwbPbW6?kB#zCNpmWA50MmS2@7cr?&qIEgxdf!X_RyGZ~vbg@wU?VylY~%)`T8Y4< zu9}QE^Jtyw!(e9#rIniKE6ShunwwvTihy|DZxw+km2zd31gM@<^^UAnNrb{VOH@3to*mE0@-NY%l+=VpPdTH9)AK?&Z2l%YfI^@NqR(n>Ii9OD zXjHqPBhgI2E~H=$PZh{%mZhztPX^{Q%@l5w`~P(%Y4z}?m?qSfUS?Je!45u~yo`O( zcJwNMYRRgabu61T^(Y_azQA&1%+5|({^ItW)GPGV+iiX>33;|EM_L`#kd;U*l~9~Q z)V;6LpN59d0r)pT$tg(RKpB6(i{?29Tt)Y^8 zz*vL3fp@uhBGh`RD8IAaTT?(1sVcq;_p$|c#@s$FVFp{Ex7$2^%;{tQD{5i=+K2oA z{0Tv?kQ?3;pg5Ll#d!~)$}2T%1p>j}#uavIOTX?0(fdKCDD*h$>O} zJX_dibp4w^sZ-WQqUwF5tsd#?m{EGVPxMnXwqgL|QAgF!$X>@VqqgZ#X!&H$#h_Zi zBKIP-t9;=RSL|pi=uWTE%}*pO;87Xd>**2V z6>v`(Es^RPeRGe}1Ydhs*e5TJwsqMno+u9>O4E%+OCw90cnXc!Tjrov{^wnmjcz9& zFvm`J;`8Uk5y-;K#12iKNXsSO01r27JN&7|+zP49{mof{g!_>%f)v2wqbTcL*q<3C zsm+;VH*30iU0vttA6bqxRCcWV(A&y<@N?p%6m~p#thD(twUFU%sODd*EGLyS4!>#_ zrtF^tVRXszF5z3zl2@3V1f1iDS(Zb03L{rI>e%$CnBo4@gz>?f(&yqwr|4_@_#SU7 z9+G}bkIP`FxvqD2qRizD@nCM84r7^)@}2}*N`7#2V6`B4S5IV2gTvjONIJwaN>ot0 z!KbLM{jDIfvFD5M6Rc^)})WnFUvqh~zb!B9<_W~Xvy8+~$P>39tG zQ?#&1k^?_uDGqw{N5C-YZ{fA-f2KmFngXwaHuo~kH0K8@ox3N{ALvY+HH%varXnlc z-;d7vy;D!8m*9^qT^`SN$?v`rsm%F0;pHT)a{$R~8-9$wNHbH{rVvCnA zt)f}jV3mQ?=O)1ZLaHd}T;x8U;hj1=kqvz1y^n40a&Cql5P}8{e^8Ku$|CK)x(n3A zj_DsLs3E46Sx0q{e@fEp;HTMpBbzU%&Jg;|&#g+^O9mpKrII%&X{V)MlR^zgGc-Gi z5~X~BwgeU!420Os>71tXYCkudVh@B23|G{9?t1s(RNz|0F>QZm0eY`KQu)i9zQ`NF zFl3f%dH!i3YO5*)rcp3Odhj$P>&gSQQ&fCEuZ|t3B0UULiX2U7ya`Ej!*DK5xt!P# zBfyFZNR6I+Y&)pslV8i>>buYw=kpEEd=o4HON*`bO}7MXaXctravv|ll_Xu1StbMG zqsv-FV}u9r25lH2IaSSY^wJOls%J!dBW}c@H(_{s<*YaVM*FZti+=|j+N#eUk@_Tp z1+H*&_pqgn8@VC>uR*z`&R^yqSw5MOU>FU=G~iG7`VSW{+N1NlKC$mS@zN&L;K+B2 zl9^UK!F6hE8KpqL(9#1brD+?t<&jH>WDp#=J>FJtR69FgImDKr*Ge%Fi-BnyfeMnp z@B9C=_WRUby#E^VA?mu0HO9bwf!y#;k@RbLDE?A{bSEK?NpVW`?mCjOq+i{96YwD6 z;ATLoerS8XX>kd`9U}Z=gHh*FYw0_(ft#VgbAxz(_eeV84kP#pMnIQ-Cx!~RtYhjL zgsGodPp=Ccr7dZ87k0~HF8)3)4y7i(@AUNEN4$h;&{jX;o%~YIpe2uz{5=%k*<5}v zKK88nU(NLSXQ>ke7F?v;i51JRA3!S+KmGC}Bxu5w$NLux=QM1nG$7EJ7u*G0Tv4hS zht)u<^B(Mz0`~uI)2_9ewy*HYB7|#SgHr@K!?frP33j>28<#!~;pYBuXt|ATnBfPx zlfJcYb#LSy!h5(H>SUHZ?aj=xO453LdUde(LotfGwf%b^zQ0@~FB*79REzepEY2vp zcLjJ#BHtd<2|z4?qaW=UH}zf@WEi0Un+E}54|!$SE~@MBo|6Z&Rr_@w`tu4x% z%|5d3IfFk|&*Ty5qqF1-cjQ^<0vh62x8&Z)n1p3_T`LK_Z~O0a<&d2Xj-fv>lvGL} zK{_DUuZQXn74e)4KUFEH>77eYemN`lF6U`;%d1skOK*Hq5M<*>?p~t0+~do5C-;i& znZ=D9OJLF8XTSVzAXXpPj}`p(Ys8<1o1~_LhwERD*I|0DWc@iJRA2sDdz(^}=7P-x z^10SC9NNs%v!dv@D&xHF`E<%xV3BuD)7?0X*-@(_@*t9x&0(hNGg3yu(LnH+;c>^K zF?HxCd%=S}V(Ox};`l^L{4oi|k6U-iNaG?(z_V;GFoMBJe}MmVSMbA!5{fK%HJ$VP z0%MH(^fD#Z>C}BsHv(l12 zvFW_Yfg8D!%Ny!x$_~*SDU8Bx>~;se_7J=f>hfHTSVYg%n)Fp|l;7LTuvetZx=B>QU!WFTdol;3;O-;^FXwJ&evTpFes^M9*h{W@>d zf4Cf$G>6Es!cb#sObl5$$NmkSe7ay96>1!-Cs6;_yup2r?s~q2utORI$G6jXGP?dK z?!b5ODO8K7x`O*l=me`1PZ8*OD{ttT;iR4PV9Q*^bbDKG$$OR4<%VkAe4zxn z`eG~~BHNf$PnCC%`p)a0!JgH#9~9S`34eHxq1N|xtf^DDjG!oUzJR)RjJ0_O%W)dE z(K^WdC8g$AWDxZ}$SWX$Klc<-&&STjaJ6*-xuMj)GsZtfYgaV)o_CZY2$DvwRl;N( z51Enh6xD9Tgd*H}H|^T;cpxKWCB3D#H(P=^Tl!T*Ck61WGzrfeU4VI%jGYSz)5E5) z-eZMG4X5eNTyyE+k@!5YJJ#6Om^e*h@=mRV(4Q{`e6Q3IdjpdsMVqL}|N1Z=oc zw@y`~#&zEMbh*|1=JpT+yua9#n2|M%=}?Q8wLKny63w7z$%ire)2*bbkrDiM@^3t6 zKWb*`bM&OYHr)5PN9?1i@zUaas1>C-gvbL^Y~x1_>bi=Gj2>dn`Yu z5#N%f;%^l)emXk+g!kh!U#_9!(aoJBvp-v`V0klYd>Mtb!lKbj);+N)n&sXPBh9_# z1^jE+<4N9k*v?a^DQ zL4gdTyhV5P->{@#6g)gbuil<>3HPAq#_f zdG#WiXMPQ)`y4`AzAXF6UVOe?e5ptv?%(}@iNk&P6nCvIijbk21XjXX;`6yXF{2X$zNCIx{6Jy<<;Hc~O2 zKlE_A+dpimLn&Ejs^sp(z`OZ#goHMAc!uEk;u4>Ci8k#@DOUz0q{GMK6j@B!j}D4h z&f=E^mI`}b)WG$}Jh|`hh2DGUPHYD_#Hmdx%H`64r$QcW6G9P~x(@8tT}I;9EB|$4 z6+)$tike9`SrtC~e4rASTe$eZDU0Z%v;iFjeDeqFC3SbA2h4H~!pF1@->nJ3taHv3 z*N&?OgF?M<5t?Z~cGZ5#u4~d2)hJR{6~5reLVS|Jb8vDktAD;eZSdnfCb*MDzrY2> zZtL?*^LOa`Z7CzTnUDFJRH1`1iu}bDM}Ej&+UN%NpR7@(ZmN*#SvMZbyOgM2|rE&s(f?Wx@{ez+f86vSs7r?V*ivN{(`+dE}_7J%_L#v)Vqf?lWeo z^97{=Cd=AfEH89=o?V^8q|}63Gxuf{q?XU;@JQb+4f`4)3+~mKt2x_@o{iBmJ5*0s zIWZ~yWDlZxT(Nuv$&8Z4|3zRERoT}vbaCD~J)R$3C-B51F2R^4xv+RaR2G%orU+|+ zyIk%aVyc4TLs1r$N#27Q78>F*61a@6cKuGsr0Y^a9hlA49|bxVTmOX$Q~&8FCr}n- z%1#p=&;7PxXdVwNOo&HdMXI32(n6n=h|o7sgrZcUhAC@a=Z<qHafnDG}b;y|E`wHpa3 zt{sVJG%TlL+(gVmo~$_rvaZrsx%Ohi!X}`lxZOR9g_s(0V#DFE&#HdMYOrI>IA=@K z@6)uAWUrDA$kx-vv}Vnzrg|;LnGF1Rq@Zv?CRd_Smbn@_uj{aZbs+mpw1G30%?S}T zZH7MIKsjwsY*{X$1zFnHFZxFEd^rZhTWSxcL4qUgK)g-;@w?ek8XEM2eYfLB$Fa4v zd!H2-f&N}DtoxI*W~a}tgK#lXrQ>Qmr8PB=155XIK5cv2a`C}@-jC@p&*7wjBbxhw zai+c$Mf=Sw?vZI@G49%ZxYqRr(JX_Ut?7FgPQTnnxUSl#(K6>v`VsXo;)djs2yW)3 zD*(GY#@Gir2_FS7Av_k1!3G7yjgPxfML>>>s!ST(csTqPu{;-9YKnQm*_cR- z5$tBgsORLb@x?ke?GEA6Nk6hLolK=fv;#+NhT@b%$i7B)W~t>N*9xGKkf zWQfh{_B{FDbB*~ZqG301@!&4&I2dLYH#}laQX8Mv#J^5mELdt)q{Y5v1fM**b;79M zj*a)jE5Y?kw3EsL8-s;S=f|PGYTe0IiyHuPtnW}`RmH)e`ii~2+`XfP&%7??d_#ly z!*oH&XRFW(47wiY+O(-kmDNQm2+TIIL;s=KlkQ;Mi7AUjk5HvGGa=)#^ZF793MH!m zfwUP!`$oCvR;!hxqp_wZ%O|Q+@0AHKATNT2=~x!BF6OF?KG{$N@}A+EGFV~J(^d`A zrfKhaBGoI^1mK$bspkGKmpO$MoNe4PUCB|(1|;;!>VowmGz-Y<5|uo)94gFNrY3T% z4#~}V&dGj3dhKC_4Ji~zZ$s__LPMj&r^r5>Tq5;5{wr2HcN)8=(MMkH!Pq_@x{9HT zOj#sR(;dyO(ShK?*ZZu_8E?hx3U@J;5w>(b(Oi|5B1mlFqAF4AH#3SE@=eM57f ze@kD-6XgLbu_u@9m+8io)nGC^P*Sa$6LojM=*tL1b2_xLbE2Wh$n8bTA5$SJSn?T| z!%1JXP2936cgzEH4I~QmYC09?OPug{Tq7A7mE-k9kczu%9sYQjg++yW1ng3FX1vf2 z8+jdrwI-U~)Kiw=xDI3{OeVcyk)|^0q%y^6HiSIK+f5&|W^|M4#-o^;;NMq6N@-jR znq`-8bYQ?c(zaBRr)WDy4wn+k&1a!In>1|}1rG(5+EW!r6WCFVI;OPc;%G*)NuHGO zRSvFI5Z(cCZ^I=*aVAiqvXuk!!|fiEHx%imUnzO+=1Gq950^C7=1L?YFe-{Qw5d^H)iaSil!o7O;>2Cpyh}5+oMnP9okgGzWL{*$@&O0$t*v-GO zMeB48>p4pSYy2E*J5Xc((Rj9^_myY02mfID$dnC1rmK8r>rK>V9&lcx|6$;a>wO(a zB9}p@TY6}3t?QBo^K8o0>8C1aEN+2ElP%1#7Dmxi-cGR;yW|Z+gkI)61v7T#Xmctt z%B`>zGO?MuD{^(`l5FnAgMRtY)$%BRi{<|D*T|CLg z{$5tyc%`pp)o$qhXmiIHo+JnczJ?ko3aqPP3-m(5fDBUBgp;B-Y(AdEV#qY1arx*a zBJ1fpQ=vlMR>ystgoXiDVN*9kQATx(V?F6cDw5*Ex@6=<#k9`18hREmnApj=L@_bO za5b`)7&UxH^~VnGM>pg_ZkC{xo6h(ax9XlXNsVK6Y#sIP-pw1316<<wXy~&s?>5EWusu1@%X7F8akJ?ya z5f5^g^$fI+%I9D0Boxo+>7?AV_O2A*jH!8`^YR)^=2tDl4$(HqZKmjA=>D}utCZM} zX(M*blV-{SvLHNA%ha2Y^nkovlX|GC+JZuE6+Lqot zSRx|HU@zmWpEubf({X0RCr8iSDQcBaBVpNp=i1{)LS*J{I;q8K6J}BRrkI$1c~y)q z<_6SB`jYp{G&^cka>^p;_rs~2Qn|JF!qbp3fIR!NNUa0Pa2&AvrKGpJdHef5fYGaS zNw!>)5roZl`HW=I-;dNGwwp&0Mr}#ZV1F{rdrZ*Kg^@(IzV6B*a(Q=I?|Ro|=Hs#D zm};&(s4OU9dkkNLh^MV(1zZl3R*IeBQ-U1q(RLirf@XtG-W=)t?G9@0N3gEO+w1SW zV1w-E#PdFrId0W$cV5x@k3*1_-2}#YWO{_0S5Tr*XcZh?mYo-_=4_|AJ)?sP%@yq& z)8vnb3-78{5uVh{Uk~RBZ}zdGq>i7R(>&MGo*m5ka~t9K)+VRs3eE-DW+C2LKIu^M zH&d%CHKR&#pZxx57w(ath7)(xRCfd`h*k2jnXvV;dR5w&^0#HFHT^?z6_q@L+IcJmUKRk?7+EqxSrKxwHN>(exVkg`iXv73(pbxzPQXf1_7J->FeXT{>%##pVZeq?0V(`wD7L@v8FM8OL4J< zkI+jHrNDd!zsCAI-@awWt`BxhPE`&oEMC~jJvcM%C^f0HKpXVQNOpv_gPz#8C^hp% zDi5u96VL3j+wvCGa6jQdSrFK%fP1r;w!QDtUqsj+RJw~*cC&oa|3FLaAe%RhiL z-LQ~6<|}poGG#*?lc_rAmw$TdYu!Hlx1NCGv(yA~G)lzJ*wLw|F!U$|t1Qo3yy=FY zAHVmu4uK5rj4XqAkYT)(sf3PK1zp!l?vqyd*%TP7-o(bcWkI-Uvc;JGMpv0 zxs=s70krd9o~;=$on@OJE6_?%fF@@B)bRE8PL&uH96fF*fdPb8Ntq8#c-M3LHogsq zqVZi5$GGGwlfFeWCodRN{ZkDB2{&?54f&ApJdWEm4i5hFf6i5C9*h_(R%FwgcmFQN z)Pa)iz^?EgA`<xywEN%?C!tDAAp}cZ{4WD8LhSz0|A#MFc<%}OXb^5*#W0;NhE^SAp?D}F-3)m- zNX*4ruIV$=e(Y7Ph{i9b;@edWsY7%c%UtqNR$$E36nzyn)J^GX8R7SU4zLRfP4anSWoYpVI zz?`k14dPoa_*XFiKarGBR%Tz25#4Go!VU-hLQ6F9yG+GTl*3FQ0E-D{-d*5vA`SU_ zO;+8AQV?XwrLAJU9y{LU^-6OHmw)wykouS*zoS+GT`w*IonxYn9qDFc_7bkGnUPB0 zIme4>BK{}pvn%VYaY@YA(RyR4;ei0!^U1m`9Z1OzGT98?fM=rA1=mYc^T^OWHdUd^ zDENgJ*iqQC?=zi&pQ!SBE#7>NO^YG2H3=^(tvpVrmdN|Ckq8_{GlnkBAk3~VZhHGt z>6dI`a-J@{(TY=N&9A5hO&{!Xx}S(=`q|8C)x@63R;Qy5S&Ar5?_Gt$KQQp)e(B^T zPl0Thn@lioWYt6PcF%uxi-BUnu3xf8}NC3S$S(*t02+`f-Vk zB`MkO+ez;~(db5(iX93$K?t8JN;U{VKf7ppEn=TAH2C74ykY*+q9a8p&>kc`VEz~! z)BDu1nJ&o^KlzWDs#5|1t0X9hnQObuH!#JsI%{W~BVths_9aq!D@5Bg@Xjt2+GkN? zu1PV(++SG6d{ls2Jb1vHf6TdRP#9fRM8n=R{R~RR66cE3<3CtAgvEo66U1ig2+Bi~ z|LFd6A8=-X#E6^Bs0nTlMN9~SxU8NY?_%Itks$!Q7#(B$Ut;CgET-^8(?1nmxm_>}(yv26WM)5z^EjI=gZ|6}u97?~eEORQUai zf_kf%m2l}wLoyI;lYgmY9s9QN@mB86=pOr}Jju7~k+)l(Jl1!JdR`aeOx4e|&@6qD*&2w90G6;m) z;zppjJS?(%L_j0)kvQ6`yH|G=5H&ZocVlVD({GCJPEAo{zuK}Fc({PM0!SuIKI@mF zdFVkLgSDNMRkRuZ{_I87aFJQcEYBh(cqQ3PpqlzDT2E|)<*6F;H}hb4pV@Lzf?Wt~ zy&9kNAP2M9RTr5%Wi0GFGM*o-oSs{3r={__2UwfJ+7iSLazsOj_}M4u8h8z`&MRby zjvtQwofVm91YKM2?+Fe)9*yTbL?D!Rwgh;Tz6&S zX_JS)A5Ui2^Dn61m8pY;m3$lnzk{JYx-XqwpK!o9Vj|<+u@x2Z(CHr zNX$4Vzu)l!#=Wfz{9!8;SMpMTpf2?r)6*BDZoh&tp8$^WE9hp@{c<{f`zW?0&p2PC zQp0|>n{6U_Y0vg7)BP0QVP7P9QW@ab>%$Thf(U>F z3oy*`k`2`#owIuJky+7=kzt`}|M!hYu*%a`aO(zDvnnm z#=z+CDW{GZ=e0S-32Znm_jxQnPnAw!;zOiCKO(wu=xqk^MdPURy73kE#723eW*m7i zBqnCUWs|F5J8uOdd5MCVTPP0?|v!|Bfbvt>VLRs{ASGf9#ptr6vDwhoD#D9*F~qfW0GvNSOs zsio?#GjSZCJ(N59bWnNprKr(&&fY&|&n4Htz7ZekqL2zVhXfz#Oe1J>RVX%~p9Suz zKQZ-|ifkVpiWvP|n>ElF=X_ZvV=Aox3_EJ`dwVVG0*X6hvFE5qZr2dE&eLHgx2+$% zJ7*gpTJY5M@s85c$xoI!3*9K&tl@9JtE;mP=m+)TpJu30)Qx=Lu=8N-)*=e>E;@gO z>On@08QzwcHPcQJsForg6l_l<4D3>mF?n^iW`KNXHRK@AFa$RZS{sirI((;6nL8Jl z;{v09n4%XS3uH0e)A|{6AJ^RxYF?v7+QQ~JH0yJ6aU3Wh%W(d9n(d25Jyt(ti3vZ$ zOmq7fUQhZ)eJ;D7jPG+3A$aMQ^m%sU(Bd+|Y*`XK+X(tDu3 zZEJ<_;&9d^!T;%awu`^;L)qaT7_X#!)iZ$@LK<`G>;oIDfo41pU+F zqf2R40i4)o7HiJwd|K;aILb%tzPA$`8V7ZE4kJj>RJX}C10mnIvLnW*ylwYKZoh{u z{aNzM3OBi{JFheW$~Z48mMpE9qEdkD10wD#y;Z#5I{MrTZc{%LjtDyNIjP6AFsxU zN@2fnf1meRLHyLukVEzWV}!@t+<^bd!0a>a|Fl zYCP`({weLtx9S$*rvp1WRsvQq4S+YWG|Nc>^St5R0Dl_1(3r7pMyf6FGSbR&{|br- z*+aAyS**X1hJ2bR9DYA#B1*x|;Yx{!2w;)(H|fEhm3oQ;-2N3Bi`aFwCW{sn=gy)4 zG&$M+!SrTyLrBl$Prtm*3}TTJzi@%#ndv&wT_XQ}k!&P7FzS3FXRC_`JQrx@Zvxa#_cX?LGBgCm*C@!I-hl*d!94@?oOve5mp#r_FCgT_4 za8=Q_4Vlk4=V|Q3!gFW67Nig zL7}}VX=QXYuNR71XqgYN)G1eNk9`g&5elCOkB>Zjr_pjuwdhGHhHNCISu5e_JPHZp zK_4l6oneIh?*QD~>HGAIUDfzbnI>2))Z%&jbMv(>R+Xt47ypt6OXiv!#gTuCBwQ8P zwPt5drb$JVBe8r{!1h429mCQdVFm6bDjz&er3eD0h%%Xh|~u2?WtErN8nyGNK(;S_z~E{B3&(C#B0&^EMN1389Tq&l|hr&(b*2vFIMT7Mt>Ex`=T1r2?UjN8MEt(qBUR$-6B zUrBMq-<}-skH1o@Y#iSxL&TL~SVCkQbqP(!(s4G#l$=M1x}Vl>KC6Dh>cZ>y6nS8{ z&u!heE8F|yu+0u{yQ)pq0tGT$E7btR=|flcieBDuwF%YM&NGs>)3 z;IHNCZK`RGJQ=qiOaLhPMYF%-vrmCS#jyO~RS|*NRik{SWLJC z-kptmM;~+X9OD3-k7hFmf8>aN`sM0&L58M!_XB@y`#CnfB9Z?8(5b@D5etJnC*`zX zV3ydKp)#@Wg}|6g;+*1J*x@Qejul>(CLuUfuWM1ys4<}%_~h!{HT@u?ZGO-4%s6F` zQkm_Uiaj)jcn@>_Cz+U<(k;r0k2Yvo7l|Iv8;)8t_WM27vyc5>KGT2}woRAn%`DU? zoN@O&i+rB?N8Hff5q83?HJ$tQoev8&2=cPt$6ctQ5BnVMaXlYM64IU)^`!2p6)ft% z8}4VCyM+jAM3j?KisBUwgM3^$h+0~QuZATAL$u#^AtGicC=wS$FelOZU;pe}eld*Z8K&JeJ9#)q`?plSj*vM(Fw8R-y}yH2+~sl~Ye%2k zI=`!*hOaLgk|modY~m>b1`uFr^aRr4duipUPx^RuA+K`s5C?b&N)(vk8REn0Ohc(^Q<2(mYzpalIU#jWBO58YxxHuk9E2&%5LZ<2RpY0i_J}oYBi*-ebO=Zo}P$a!c z=r|;2Ul4D=g#QnyI4&ag=RG#9w$>U0UtLjWY0vG zm%6+i?Hu2LeNjm~@@V8%h<>o!QSFqqR2w|Jp6qpU=s5p%FF{fZ@#>onUuqrx)AV|Y z(I!IE>e3<(2A|hmYfZS-ZR%iupiWj=my(pz6jWA*6v-8t_f<=mQxSM~i0G+LLW?i^ zpiITPxw|AxYN>vl{kTB{OBL6OC+7(^+_}0RcCvCA5~418{(}8DwYxn$aHa>B zik|&_1)~3!D|;$XpOUT;Mm}R!NVSZ0(XAY#F-r!sZcqa-a$}OdGm#JD$V_c>RYG{@D=NeZc~=lVQ#23Q%lnaj_AfdoRyv+|?)?eh4cOalO|fb4~#jvB}jE zkq<5XGSB(=DtkJ5N&UX{Up_ROPWIH>#Ge>(gG4e;Su7s-rj%Nf2Xn-*Tiz{Al-uL( zQDH=#H2RwgX#)`Hs7|^o7jEw01UKkDEAXD)-)((oy({(8$G?=j-5%#9_@>>h>vXvx zT5A}qt~Ws+InrerJ%DSv#!xiw%`E}{-2m3zsaa2ANNmZB8s!}Xb&Pd?{#xdIgfxaY zYUg8-&KeK26NCk}2qQ!b4NUo?EWHhtMX5EOna=V*I(W9gE=Nu?4N8)<+_49u_%It| zx@_Ue3KFS9_T+EPq%73DQW#e^bj-+|>S445Zl-QlXt`)DDW;`skQf`|RK0g$`B-my zd=GaF2*Ox20%G_Js4}EU^|u#AH>i_VBLBL#t{2(Y>M72W$&sf_Jxhzcj0_cvcS|}_ zti@QBFka7#6JCuD?vP*HnYWHe$JJ4z^;vn(xLH2zURZ)fF1xYGdsNK=r|NXI{&GUv z3G8#V@4Wwz)klJ5QUv9@#TST)4pD{3HV^1Bk=lx1JCq<&`eT*ezP%BOJ99=mEq;>7 zFg>eZzN1c&2NlnsYsmLYf&8F%9cHE0y&sO~pZv@$R`}=OZf1Z+D~i);%mZ-~b0YtT zE%)yK0p|YS=l}n_`=8MwN*&)YQR8m^>2$#=IW^tw+$Bp5?G#*%yxmj|)8(nvXanA* z2_!v1?Zw7UFS;=(N>Fq7=MQXoshOEuoS_|j#BneX$k54z#=n<&mZ$`2I#6|)&->H2 zC8mZ8-y2HQT8g(nFd?t8|E2dyx%U3THEQG_Eecjbu-;Kx(2}ci>?o=r{Wi`4B=zDkliLK6r1x8;kT)jg{xXLJ${qsgE zSR$`Z#Xv|fA{jJsV?1`CuG;B7DLsG7J(IQP zq}@+QJnHj@q%0qvUCC&{I>{-m5}p1+I|?F9U+^)|i0=4AM-!nPS4EUrBo$rpBh z%IGeE!WZTMHuGPYXCa-bsUMcQG*3@HSRW;f5o$Q?E&DSQSu`AJ3T9KEirm(t-@Lof zOd}%Iv|X}>85L~v#!o^Tr3YgK6q=C*$@|OW#S%he$L6g{?qsm^c)$8c>8eR}xpvLq zR=J!n@_JEoRe_2lTJ}gRi!o(I6e`t!(j;4<%`&`5%oMM>Z3)i1HTLgu^|7B#!F?7A z%IDG~heEXS$uAbG@tQ3P+keV{^Y5QNH5@EMsJVg)YYUqnQg(dQs*TIjbkx&w3i;~8 zMS=R=q`T0;8Q~2L=lL&# zh44=)kJem=^`N+Hz)zI#^_Hk(*}AWsE|almhnI+(J??#rZaFlrxvZ&gV}}gS9l5Hs ziFB}_+#*W1)!`_#s6R3$#!7>3JNlYw@TAqi9Vc~Vj11|0Ypg)_*KsFkXN~thd{Ei z`ni}8#*IP}+hXXN%KBuKUJU6O8!_wwS1t-SS>DLxFmV%Tt}@;X9M{O4BzB#8Mkk~I zoPS}72d>}3%V+T>e$jOiaN;}&!)s2X=3Tcrz zS}akd-Tlbj;>8sUn&1-9Q0>2wA}WN~$r64E4RDi&FM$6rh&Ab6rE(VF%(Y*ygNw%_ zN63W%Q-+Z~u|5TjD6Ojw#lwW5u!k6V&tZbzsQ%&OFu*jZ;j+1J3YCWQpP)02cQtk)~GG0GFHtm^$I=m3VF-3v}= zp_?haAU=|Wj7AdcTdV%-IgWfOzj&{t`kH|mDXgRQlAR7_@QE#&sHW!MB|Eo#YjrlG zFn&(|;4QO_`UCogHgw2BTf5GMWFjhzWUjBdf2L7!oUn3>d+ESELq+;XSZwX;C${x7 z(LW^ks{PsAmn0Ym*(EvlR2hU)@)-irE=Z16D1C1eUF->KFq6VCgLu}oMP?erLM_2h zzDQSYzFPpYFUjhOdAz8L^-X7F#M8xhb>LNZI3xtoG0qgjPwK$?5nlQwsOJ4N0+*jt7vQv z)png=YJpzv`6&NTP6|%xb{X1UA&;_eAHKBcS~f(Fu8>zwirnL^DISQtl9}1NG!kIq zzk!402%Ru%>UP&22s4`LkwKv5G6-Y64uf-4&QUadA>elmrJ3#_q$s^6#pPUKt~YiF z2$##{87yu^8E?cx(BEkcbT=83Vws?6I?h|CMRq_6Hps&UI?8E7W^;Y&yTN7sxJZgm zu%)Jk$%r{VfAg7)|F?%TbtiJK+;O;ccV(BBC^!RtaPG9 z_O?lLzvo-S0cD*CY&1H3qO>0=^1W7%C_V<{<|Z;lp0;pP*K>|!rMNNEi^IN;06f=@ zkhOVl#PCQ2dAu<4XX3DInZ0{G|M8%y!hOV4u}8KT5478|vo;f5&4=6-q;Cybi?<(} z&c>0bw1;&gQ|>sqj(#mPk(C$_QQb2ElSb+rE#>Rqb~BB$GH;;lf)?&VVh@j6BF31N z^Ut{dm$0*HsN($^D9w2e-5rPS?(S}+8z})51gxJr@7RG-0)ljRcXxL;NOyO>e1reF zottwxd-h&yeQ!cw$oMIv92Ky8Dm=y)ym%tY`M6RTy8~|J2%#!@SA;Avg8v0+&>Sp? zCr9PNw3$1cVEXh%Rz!inZ}sT0^$4|-lQF0I zG0$wAm0^5_yjf+(Zio2!=h+u_x9pV}lP4t~ISGu0w}!Qk&GrKIA7JBcdEr#55_R+8 z>q9JlLs+-t&_`}lIy}hSVz}1T*ghY`Y*p_zRu*tod`LdQ_0!xpcyJidXRLPSlWkZL zX@a`nMW0@rE%!haXcm`?_^bpm2QA5~L{<$0u6Ua&5`1s6$o@AMeT&vjy0X4WDT7C% z`tRQ+`~^BQ?tek|0Z8smCi@k(>KU-hWMdMHP}ntRg^$dh1A?N4;& zp&|M_qT{rBz7;!b5WTP<$73q*zla)FDz_9xNY{b*OY*22_6S0Wo#_euUNKKoZLX+F z=R0Q4sC-G!o^bGq->mh+k-5Yc@Ll|d;Q}(=qrQhC%8b0|o4!rjbbjFx^2^4ji7hY9 zy#d|Gf1{?N2kxaju62$0@}VBq95CAXP8N-kNCi=0?$1l6tCA8FgKLQ_vnp?~?)vuk3T*B#k_5}W@<|9Nz0>s8l_nG=fvQ;5CW#Ku1 zXH&}a{M4!C`-buRBSOs^XYkatWiyCpQCd05D$2CUzBn$i z)fj+A{F)I1dhkLYBb4mkX)n|tC_9>xnWz(^!Dpp*Ga#(Df4Ml?Q}PSARd}|-gc`&C z20UdYT~#HZf&Bv2*m34IZn-%z-*4m(cFd#+=L6e}z202$3OCxioM(zeZ9*#!u5nQ+ z#Dtkw_k@hrpZ2us2d1#owe|6cDOuXH=i1)Pvv0@TER;v%m=D#!?#7h_s%=L zH$6vKy^nZ9zctD`B)3cS@K3(?{xy^O3F}9>`^SF}E?val8OEW1aG|hu5*&v0hpFDCKgI5gQVk05QIN|8*+|KAV;R z!C!V5k{PyDBH0~+lrsZL95x)Gpx1gKbgahRAJS5PJp(Kzd51zb_f`0;q&ni!zZ*QG zn3;(q3Nd-b{A(Tp$5`B81A43HI`PtGoRfh69xweb)K9v9n{mGPxv2X{uTVE`JK|68 zJnB`mV>AJTv?!9H*<)D<5=VQ7=YtoPelh+$_>0+);(6 zF*o)GF`R0DV2|g90DQHwA62i@j?W1HXRs;cqcCPWa|>$-#2bIk^3FDmP0|F?)uRRk zVub}g>Di}Wp!cg74jCd=vGj`ink6gl&!>24fXcj;BWdP$?C-YmdD>xHf6PKJtT;Cy zD=9gF);z2+T`k~cjxdXX5jiQdqO@VsJ~+7s82IznNpl>V?QIU)8KZYfqM zGU5v29~vbD?@}7Cp~>C9h49a&V=5{&A4bPr%?oWT5E)s(ZbCQ9QOh>11x$J!I%buuO60>Z|? zzUQK3m68a?bp-+*>i`*!1jR=9S$Wle8hKko?%E13Xc8?ald4vm9*g=Tf>T+5?3O%X zuY(rOXeg(KdW^t4lFgq;Ih*in0wix>-1|Y|5olP`Qa^y*N%2kPR~;{z0^7}4PKkvM zog8r{%*c@AUeth9F1U=HY$qjt#asfa7j8KT+-u(@Da+v=OF&oCzeX_wR?do~FF)lO z4};Ge+)T-`VXC1 zcaY{d0B=7>y2xvo&T2yL{#5{qL$iqp3|$<*Efsj;6n-<~Ap&FuqYC@F=NFzd>%3sH zNjxTfl)-@vihYd^$tKbp6GYGe2V`b>kq`y`p@Y!$GkN!?Tnt%vup{SS<=NM%6FQl- zK}56LOXyZQKPP?b%zNq0O9pPWj+DWQpuLK3bJNlT1x1VqSH7VjoH&3`Y# z|EywYk~M}Lx-pmOcYSR>N`kC(Atmvla2+_W?JVF+p#Yh9G8($u=&KpllH``zgcuSK zR^z{F;|)`(QcA4m{R`RSA9~_=L!+@ka+2_R%J$qj&S_N|ls(mQH0N?Ol7 zQNDZoJ@cUgAMvaXDs~Wo5|zO^i{Y;R1t#G?WZ78kh+<_Pcu~j@T>tq<tt-T^?>RVc&^F%C|RsKor0dHee8TlU6= z&O0;2bE58Vg6G7106tWfE&829Z?V4U)p`>6zGG;SDbE(?x83TH*pLlwekS?uy%W%#EN6&@q+#(j$a z4jw3S-n$!4wFZ@w$lvXpRP``L8dT0bB?9=?e)%PTwPWp)6A~2%cA;q8rO&j#QkpQ{ zoPK~gbY%Jw7<^jeD8d_owwpceU*AW(EH%FMmoh5QZ*m#A&6jmRmk4Hmr6iikthdb4 z@wKpAw@b8FTh1qEmXem-OOBCM^(GN;*dwdhX)#4gG34(YPcn=Z#;s)@{>Gdo+t?Y( zV*WzM0}c@QOdLiXPwZDdb0+UD`u>Pz7xUs-*?tMLnPlLf&(eUcK@RW3fB%|I7S06e zKVg;cPjVU!?U4R9NP->zea2t!L*!*Etix{9?McA&59c2)mhlUDy_VWs$5N&#xiOqL zzx!A{5stiY@~=YllG=HD?n$G#N-vD3LFDF7=bS5YE1p5(0JmuDUK&@&x|Q=H$M?DW zWPQ=l5iK0{-=p{zhhm|lOF$p#N1-yWntLdrPef(<^qe{3dA!Egg)NAWvojPQ zc-7OE{QOEtq$KtXkCV-<_T^biL7$2Z{NH2hqz}S+GPL0G*J>fxM@*cza-gs=B*eN| zY7i+ixB{ST_-*NA-b?0vnRCGNNP(+N%k-GqCX;iHHV z108tcMQ>j#Q}tSi6P;!A({WGWPF)LsjK@r7HV9Y`}xxo1l1aD zCtbA!`x3nMp`wZ|-JA4XZ-WhCV>@+=@`}#sBq7i!@(vbOY|MPK4Aa_&eoQ(EKn&Z2 z{2=64Ds5I=!)(k(^u1y8g>+popI?4x$HMe4Rt&5Q3Px$Xn8Lxsq?UVWzKw|9!xK@8 zh!+jV$>nrk%u%94b90Jdh)KzT9{(S7?-HLDE)48nO z^vc8cVZr^4GX=;HsZzD?l)H8M##~BnoihxXVkey`lpH2aa*@Af^KQRIKq*7V_=c`p zj^9%b-FMyw^zzoC4#XN1S(arJ^R3%iF~I>YmUpnCo}ryaW*NQ8;WclV1$`+=Pd(-E z$EUC=^p+0T`?!51eD$9Op59nC^Q-UIB7KLc6vh0lwvZ7*d@A>53E^2aE$KwskuXqO z>@L^GV;a)AKsp8u&`W<<{p^dG@?i}zaz!sM^lhNT_;b>?r7(Ew=iEMX03nn{u`Ef4 z`9XJ?0DSI7g^WR$Q>l+)Pd?#|w)`hjIO4A)UA055$ago}^n(<9TwTl$4t)J^b5WIp z_4$J3;F4a5sA?(YZQNJvr_dpKAL7kpKXD(~Z%D!w3tOa9VjKyTV9i@CNr<2C<9_1! zyGQh3(B}KZTjvjw9WyG0zS6Kl%*)e3Z`1c!6&{NdAygX08weeLVwZ|M*mW>HG$-JA z-{1OkXic|_? z1B%QyVJ~IEJ*~&THRS&aqiy3a6xh%`%x8s&(y*;niHGCtTR`qt4 zOJEuvk)v4QoHwTkM*&EdL`9n`L;PZcHuiPgy+_F1`l(}KSkT99$mbWCtY$Z$aYLvu`vA0MAs&a5Lw;P^(Py9j%58TWKH)=M&>C=r zxfTZIvkMWq+)Fj9rVILC%At|L&uS%Feph2`{EEX7H5eYgD<3AWfBV4e4)I?>s%Bk~ zWV3qQt6(p_R#PF!GxqzSt^qOzrD`N%@!i!zu@uATN zbUPh&kp<_4J7m*pNj251aMIu@0i0C?RIhOEx+r)9D$Jj4Z*z}V!CKXcMo2lSS65x189rq`b)>TpDJ@b(VczIJoW=EGtd$7_l>Y1)#%8>IhvD&n}+ML0o9y)9y zoB`v7tpl@%GEy6-knA&y`I2Q%tnU%O4hCkc=0wDuIPQXh8)rPGS=AxpvFb?>Cr4ZA z(WFrwTbAm5m~qs9RquSS4eUmtJSGw(9(XLN2-*rrb7T)f6YVwhMh@oOQXGt*Y6lVC z@Q;^cdp|daA2(YXJh>oLY8=WLOiFQb{ZSi#%|blBhA?olB#o2LN1${kp_CU^wJDcI z(#WPn&u14UG#jmYqjrGFYg(c^f_E}5=PviP8+%g0K+zC$wd!z5YN^yCXx2?1q<^&- z!T3XRols_oTgy`eQ#z29Y4z!=ub_n9t$bx!yaUE9M2*6z^IO)xSwsh|sRlSoun$u% z^Cjj`Yk|7H)x{gIEGv4kJ z2(!+^Z@)zDvIkmL&IkM{{g6`{$?>yJtnncjcfV{1qi6~d^jKCGCa_nKu!XZzUMPol z7yO}%71(_(0^k!7XBOGkIs-eIWfP&63i+e*UUGx35hUf$(&r1Z8lM4f_v5b!6#Lnf^RnK{x)-sCR#l_y8~+h>~Jwz^D=`dU`6|3)>JhW9E|2oS1k3rZHyz za(&%h3hBjPH`V8RudLTKUN$Klq&wMHd35~XZPm$*o6>j5jIsLFf{#E zl>GrM8&OGBL|lwn&|0d1AtJ~A4|d_8yN`K^nF80V+K1jZsW!5u!$#~b0voIXK4lrV zxQu8FUg9W41vx6+LkYs{Af_e#B)lqlq#M+IGmMd4 zc>DMAP|$vl@$j;xMe#PNC3|YisFLHfrtnI>euqKnMG8M*2MvO3k{sfQ5Wj;cHrx?udv9l{J$p(6?7V};FogA8Mm9uGrP?4kEN zk?h=_-!;PKn)}m??)c5&yKLpt>EN`{!F4++o(c2BI%Dl*yoQOZ4XyLFspyiv)#{2J z{GTq8fch?0S|PWq%Tf>x*&|)yad@xVKG@3hYPR7cr=yr_1NVfF(yJwPBL)`v=HpZW z$d8h0t#`vzcjZJ0w^}8Xb^Lx3FREZR>elIFX|E7vNLR-f^k-P@<`jpBXR3q8l@XH& zh;&XTH_fjeFF}1Zs*qKxqY$e}Rm0OXlHGGKs`s4D37l=Mlvl$j%3ngKoq2u@R7BPgN6+OLL9iI4m=6d%ZDfJ=nv_VrG*TCDd^!iy4F=U z%bS(z{E?+h>Cp$h>(g5Hg^p-MD|8W12CtR=0~)_yOS#504{=j^H3Q5416W1_j#4#O z;`pmK>=G{C`Ac|WGZ~VYXxi2xB)5!va1dniPpBs?DIU|hx0<-p zhKLmT0wfBE8ZC$V-i!xm7b-tW(uOk(-f|}YwOnQtWYm}%uLqb4IHkZ+y=Ui1q z;~Jf3EbJ$KRSyC8>6lvK8E*Ag?J}S>I+i5(-+Z7krH*+ou-$aEd60GadcsGPqmbb7fZXzP`!frP%F8nV;UdSzzQ1du4vTYC`;| z2C{PS$piVCA84A5x`}+cYv}T)JVbgn6`4QFnuME0@%BJJ2P{In973ASzrkG3D_ge} zHYS*!zDR<}#iDT_v`WqtJX-qb7G;g>X7Te&o08NHc=kG7Y<2Lj=n2S3=f-480GOd zXUR96y4olr03}U4?!MLqFLQ&i>m?`E6${#+n5h%Zzkb03n$^ZrKSTi4Qd8%=;_;tF z@_EFl6Z^O-jKuCB?z7Rw)b1Ou#JuIZ_ihY`OsQ^dRrg~l1j_Q^dSltnOs!t9gQ|75mcmgVW{0za zaXZHQaV7X1&0X?M;wksV5e}ijKqvR>M7%c@gP9{!2{d^uj~2GT)+O!UPxoskQdHG8 zN*x^J@}!A}m*`b7LN%<9nHtxUP3_ywzqBY8w_u$MWt3-w86$XSDnqPIwTVrYIh`Pg zo#{w#?qzwWq!XSol`;2Z131bc_rftbD8`4~)wIS$kMJAMY)m{ZJiSAjX=vs9v4wz< zdi~`XeWf9k9ioiOU`CQmb)7}En4tOJGZORuB&zNn;Ki&D*~!zseEp3MQ(!-yEpuo) zX4J;qljU5?z&G?p?0$;2*1BN!uYBPa_rWk>(2AY5)<`Nix|IwPW+irPQ|+k+X-Qa| zD!X*%yf~N<`TSS*3)ZjFs7Zp2b#x0X9AN=YFOcpyvzcQ_zS*W@I!Tr_J!5_NUfmJ; zUU{uwDz8Yh-nj+F9v?QTn+nTXp7;Nw;guEd07Y&P!Sy*I(W=^nob6{RpsevpnRbD` zv(g_e%o}%I&m{F=m=L0`zb-5(8aHWe#YJMg72YpiBo!l0my0@VlBF+*X|f)lRh1xE z(^wX4Plxw``mOzr#LuW26`4L^%AGUr?ouNl>D2)>kvcaUw0_n4j@*TiE$V(*5{SbC zMa6}6&Z7dN7Ks2=6k3y1i8LvX94%69o6~kE+{d(k>dsA?2OZ9#*ev9n)JO8w*k5cp z+^r0P)(V_VVu#*^3bvzrlAXB61)UoA_2RUVsloESVuGw@7StS+xl++NGs~JxTW}Z= z9@(qw9A~2b<`!ic-T1>mVHOPZI9e@0W%KMY&FZ#j3+APCahd%ClB}rb8q_H~v_uzxh4qTqx|)wQyY~p)bnTU< zA0FMv6&dDUv}qv=J=mljmX)U=V$1ookul6!qxKygy6!@R^o@e!CIt!&jc#>Qq|fpn z7B6FaFEir@@z(r80_(zQ=$1p}A)S32|Nrj2R-nUc=T}q!6CDE$8xsQ$;G$zgx_e4LTB*Y-|0qO(yR>lRmVquiTyG7gK=W1^$=JX5j+Q}FYL~SXHTwyY z`2%w&i{_5+f`0Di?Vqe+o!q?Ma$^9;zx4hy?5j4J*_c%S+?M(kih7_{8}^=d`0Hc> zR(9RR1tOVdx%sYhNTEefW9xf$dKz{`<7Bb|!ThJ#UT7{#S+Ex&>}ZtkhrrUo-=29z z6E!T+P>`O%g7*gI;kK=mI!>kb%skVo-l{g1%FwnR8wjd(<;4x1`3P-iI-pB8?w5%n{S zkb2>Ow;$mCq4UY2FV;MwG=iAb3Qgr<#w49%_Zn=#_?Pi*wL^5J9OuWYHRt)GBSgS- zT4tGSsKQkny#s0gX-< zWY-*cVD8Ux;w8B(xREzHhe)9~Qe zmx#BDRyW2MmUzE>BiI#w_lJ&WPs8jhh}yEwtp{<(UoT0hNbzVbM`7maTGJm51GDl@ zOpR%epD==NmZbyfU)A#zmJpp2ZXY2*rSc+6!q0T|B@zLXgu3&zz&Sf!^jo3UuN=-F zI@6W(Chy_c(emn)UB1-Ph8rTB90D-#2~|6pq4~BDv&AVQ1;%z{oh4Ju)K>A2kw*)cPKS2h|Z7dc#L7| zuFH7BNhT`T-03D#?BS9Bmunm_VzILWr9=8$LNt%MllQIOsMvV;*^{t;Ts(WmVyQj4 zhgeKuiaETTF?nJ829?So==__ui)gvu`)pT64pPX+c0&W5dUsTGXd$HwpGb*BeVE)Z zzRlUefgVp8q2mhM?3JwhTWmUuI)NQV7jEJg3WopCj0QVay=u zzg-*f4$j7{-~Wwm{w@3)cmnt^TMdbNq+2%IIfiL~_wQd?Vdx7J-yHsn7R%Zl7w(vP z@0U7h8p|nQmC4PS#F;hRpW#mEy;ty-r zLVPP_UpTG}3o40_<9ZQZehsnz4cRA%I`pcyQGw&KN(#wHHkuu5T5?e$n0s~K^228z z!}*}j`>%kmcsUfn!o+CC>nVAG>FFLz1%}2$jpQ!V^ z6>5zfHp=|7jh@LKI@`6}!*Wp51P3ZnfxofSKs=oHaNVd1gPIMx{@VZG>i4pE^O(X~ zk{`#l!z^N$8X?4D)`#q$-r?OAKDi!ypeYSAHq1Oq+fB`9d~)&DAla2C|>AL6le z^|Bd(FY4@SrzbR+OZV85SB5YvAESoP^gF~w@*(UoQ@dCuJ&>zKZw16WHg?cdaogu` z!LDZy+o~=phY24w=l3L_=`H1<4chY9_wqY9jtbiP^&8?FA zyX@>kUBT$T7Pt~m@IhL1X<`=Lo(ZvA=w|%2)#btq?6iaX(m>jGig60xgG@G%4!?SO zWGI)hkxrZWHuE{P6<|(co}#lvC2&3J3vA(G2k;V)qUMjO-aNX81%fx`|4p_VJ`CYn zpk^!m)H@@jO!ENICHZp5qvaK=AVMM5UAY9-y&;~x_+9z<>fm@_e0?B|4v-(Bs}ePr zgxXm@qA(31DUPftgqDw+TYDsv1D)p-D7O(dyCPWY#uNIH$9C4gua!vqE>D$mApT14`d}HV?1#V}#6Qk0rSpYsy{-%abbbtJW zNbWjz0i;1HA(14<*B3UCy`&wys2~b#&er7xi?dB?JHJK$#w9@)`U5-NGCpKE<-C(V#&s%u3)XqVCx{^R_YIL;b*fj-PGE<>g ziCD(J^Rgo^z>ezEwB^(OOi0EbFmVh?v5y$BRj;Q){u1OqlDVIvkY$C*7$P;2n`F-j zZxNi$s#uR3_{_bWNatTP@?<5wYc3l(O5e>mx`KbJTL$9*zUxF+ZHT;xtwn|qRiT>n z(|C&@&Ggvf*LPbyxmHv%8K7b@MebbhYG62qO3y+3!Hs5>hwOjadAYJBu{llt5;_7k zwpBETVQIFjBvl^>t={W6%|StcdPnU}hdqpAY4FK00xjouBm!^M^FBNF=ymIzpYS_t z<^&w7OM}lL{Z~_E9pd1rSrvGQEfXY&x85OtWvj_?cr2}$S$Hc3$8xeIJ}h(x6OU~D zT$E)(ef!k+9TFF`aa`#2q#b8n&!J=yx8CjONMJ^iJM&5K$ZbeZW_pqBCK*n$%r~6n zEAwIrH!gpK&>se{Gjo6(gAB*W1c`&-q|<0){^WK|`_a+vI1k0XPEn#85X#~f>uce} zM83z~GJ{Ld#iYl=gi9A@MD`X9%T%TK=Mj)+-*8-1gt=?WVsC`@vP0pUBa8zp<3F0| z-OC>}-|~5biUmxfCGwr!v%}gX23YO1w1U#n-F=O!l|Y%{!4~5rHlSq1K@}&ljH=`9 zwP-|o?7x5+%6;YEuGbQDgGpJ1R678X>63XP#dM}rDolN_9-KHN#87!{zN#!s1w$X= zxfxhN@Uw$biOZ{BCMD5omSff_pC!{(u^ zN>!bZ?Z%bMAz4i|+&pD>NGuHW2)kw|S<*sylwecx9bx}vs&HfXu-dHO-7!xz(GTmq z0jtDX2dS?C(}ZM+M3eq+%`pfK|BO_d``pWISHJ{;{S>ftHW+C_`pcS9Nm_@c%G&_~ zmpLCt!l>K~pEZnEIV_fu48=dD*-IJmbUGMO?!tkF%rtcwI+E$OYzL|{$6vem~ zT~I=+XWHl8GXzcjqVvSzcH==VE>tChtn99<5cqoX>Tr2L!r8n}Q6>;z3h9k55WyQ| z<_4y0^R-jti(z>fb&j1#d}&(G z`3P#8@c}q~AxR`qAZsOGbD7P(f%J4_QPpG_6lrQziDKP@1@xA*O4c))77S8vCBy~Zxo^Je<) ztHT(=!LFIn7LrmaX62yuI{e(3#+ZqlM{`bO35HxS;bpfJfX}sIX|(b5mFi+q)SEQ z_uy9dF$PT+W#z|%s-&yCm#1Snnnm*g=Zt(;8?f;+RSend!rBvBwvuty0N~&ttRjZz z@knj+e!q-3znEohs>vuVi{Q^2kc#>-^r;z9^H5hN8u6SXUAl|$BFQj-O!V%24Qg7K z{r%mwq%G?B9bl@X9UN53-^+4JadYD(E*$lo2(-%d9a|K1M5F!#Q^^A9@#-tCa*Wn$Gd7^>r_6lBvO2fk;|2@9Mt z`5Sa5QUOK+Kje*cDhOjkE#ys-x)qvavP?fa3tIjU zC`G|OnP}6V$Z<*%?Fwu>LP7x>-IHqf>O4mhdD_+fH-Mtng1%*MP>h7o{#Vd%YiHdQ z2*(iXS8`>Q4I`D{1jQ`K{Ug0Ex&mto$#BFI2+#bTR8A(lWA3dkjAK_}Cp5Ov25={C zf<(om&zz4P8p2eSsFX7Qp)_btKi(Qdnc&Bc?cD*UM#C~{V_ur9x=ETm8gxh$pPX={ zWKLS!l2<$bi2Jk|p}N1~{?32v0j4z1sr`~d1bru`tL>@SfRTFa18p0q%o-dP%Pq#$ z!@}+=0lzEfe>55I2ID0-AJY)n)ErEXo0ar}-r-!uuK$2}Rf;-|ZG3+Qw={DDhfrXvNMQzY0WtA* z>Ktn0b!2UD2@Y1nnfIWc*fd*+!?U(-%pp!m76;@Ie)r|W2LF0B@tL;CV|&AnQ+--U zhVN{uOE^c#LE$V1?x{N3M1-d?A)c*UIE!GBmtTuJwga|94)r(1 z4C!6`md6=1wsMNR6{Ji^njeR5VpOnBLk%19M_0q9cWvWIm6{nAu|)6Xjvy9_dm5*u z)1obIS%eAtifpQvxH>r9cE&r0$65(YSh~!5cNH(F^#$3Sh?_6ZsOgcYoXNto+&Hgr z3x15blC4U{Znl!TJ|$s00!V+%r9bf>KspkVRVVD(s}Ye}4xbSa2eJ!@D?xpP0-Uto zbA?k+wt;X-k)K>+LA&DGeTiU2yX34eNlr&-tc@Ky@Z;*^xo`p@+|I2+##eK*nJSut zw$NL&P%3iDK*OC56Cx3M?A$gJaBJ+1UAH*<#Zx&CI+@6x*J}vI+0|cM2a;6e)XR4l zYoJN)TDF@ZAGShn7R)GlFVuMYAw(kFh6*v;G>od6aK8bkWHqhsswfk^bJH~1iJF@I z&wS*8Tr0HNr?5RO&w02uNSE`lPJ!*o_f{gAGK+;Rp%7+(EG9 zLf$PwPW&i^y)604U@Qe=R}IJZC!cd~i8u-zS@SBvIzOQ`c?>=Sq-k@MW7zKI+Fgrn z+;Wj!%5#6|?)eW?fnGoHKR@BAE2KoUcO6M*mxtkdpQD?$C$7WvA=)wL#RgzF`Do+8 zA%XskBjjGJU}4m-IOur%nTAx$?5l8Ico5GC_#tPE=x|2YV;PcuVyhUuL5PR1*bWa& z<83dvE1ZuuI|FC+FAIm~p7eUFHEO|e4-7N63VOX~tAcmxWIZ5M)A1wlT6CqtO1IWBMB0pZ@uS)0$X2y^IC>9C%^) zwErt|aHw{*2k{3i5PB3mZgSqUVU=7MIrLoPwxp0VoVAAek$G5`2tA%UOywhiju~kY zooaF+T=Fg)+Jh+kDMh%*rv@Zbv-+A)UZQp!y{3hBH!*X?##*Kj`!n|%0^gqseUB>v zjKb08ovaa~RZqg?Ig(lR7RZXfK&d-ho+ujqDb@cq2+B#L?ITO|YjzV*nowQFp@l#~ z)y==kD~%g0A>?jl-Qw7xBDb}qdtH`lU>Z^ntmoYv#W&0TsQB}8^ zJq@mG|5LwSpC}^auE%NGQ{{9@+4XYtL?~m&UZBcp@1`oV>ozsb>I#=l6G^uF^DIVr zcF<#@)!^h#gCXI1rT8gDZ{Xe)@e&T)%pVD?W!YzQ)}nZb>0$mt9W@0^g&!#?xnci0 zw&hpa*Lr?t0q;?2@=hjcveQfGwurD$?U^3O$nPk5wf+!v#n}L>W_)DNP6i|xt7_Y| zf#tXDe?lgb*IgnY(ANJ%@X*4Ho^bvjFnty>BY6_&Bq3;K`V+1a6{V)y7~1AkR87K= zWLiF5;6-L9e^oa!*s0PK_(O-o;3%KT(u&8tj=4x6VFF8q;wsA`Q#i6)Nj-V2mzyhg ziXrDjog+M=VTL3&(lY8pDFYf4cBJGT`!khF^4d1M5j;(PtI=x%Dx?h~%?Zb1bsFjf zs;~!hE<}PR^BUX)fh~M?o~z6Sit8saUOk=>{0Hvp@sj zJ!}oMGsBy)+d2p&V0Zj;aMQ}_0ADQa@nNQPY)Fy5M;3JSlam*yA5JRX+lOmUr=5sT z`XJ`;I0N~)OPUG!NHSy_TS?wcy;1k!F(F+vuad;VfUr2D;0KPa#I*3pP88e>@-jbh z8lK-1S))Kv@a(~F*E!gN+-qIilXCu(5yl?(4{(PAPfMOg=rFkGo)P19(y6Fk^7MB1 z*9nRg3>q6?{bB+o@5ppHAG9{$6p5l-4taQE6>Xr#fStGr(!NlHgjGEskRM>YV_xig z<$I!MT+K(>;|^U{K40zSK@f}wxfRt z1y!ZL#M9jZ&-FEE%OqLYhG<^t>y+D25sFdCR_xp2u$4nqEYSgQ;> zl(ZsCoiP(>x33mh1wBrOJ6l6R^9d}nC5yAUXln<{oO_UCnZ85`Rt$R`XyNFuQfGGA zf;}SGM}X=0i3B1Eofgk$E@zM@44o+2;H{?A*}gn#ssNP9qg9x9<~%rWDhk|a%~6A7 zgqNsAPhWNzFj!>;|4nhZpI<0-l}Do?=Ue`=U!6sDUen(NUJF|t)o*?bR_WUB_6Cbq z8(g}CT(p}B?a2-fe}j2qekP7hO(EO8>+0bz()iY35QDFAP-&&WMU#Z8g5WtGL!3Y4 zwJvL8(yyU>{wvY(-i)5k37v5cDvZLxezlI`Wr)X%)<7l{T zQazL&j13Di==lb$gLy>sk+=}E_8tMg5(UBk<{AgkASof<-G$4Y%lRpJ6KWw!HfF?@ zJmw&Vya6k#l8BgT35ckemRw)62({>e$-5+Fv#wg!@r4Sui)M9zGxlA)*$KEEop#CS z;UHGKMbMukQK%S8WlCe1HZ`ko&vu7kCxQ&zEi`QvSD)Cq;nI{18Ab4WygER}@xD{4 zCSTqOsDMiH5y^du_Rk$_yJ1I3&72qi$kXdyLoY}V>m+|a7><1J^(7a0&LwU)oo@6f zR1I^Rbw1!jzz_AvsLk!@+0D^7Rz9;ffk!k9bCI!A(qs&hCViRuackyR*FEC!UEUzK zB=xh-BqOBx$-Zow$+3I!E+rJFW5vLw7e*q{L}TVra8skKg(=>Nh&q90ehgya-{}Ko z9&>WGUIA=e%9~G(TP!>L~9b4#wQEIE|S|f2a9*e}8M7 zap(hh+FSPnPDmA}Sr#HU$3o9z<{=8Mk=M;A-$-Ump}*gQ0eH}gK#514bB>NbX~2Mz zvhc+o%hn1Wp-Y;O)v#=N@i*YXoo|Mb7UA}!OW~=tOBpE#Oj>O-#L}#+yQ98WL;0@} zQDil7u_^8X6**8D`or;LfTrmg+s2~DBU1B>SA@;H<1N57*<0*5KBrwbDcmBsgne4rd zUVNWNLpCe{^>2iknkfYQz4DULojB%=u&eHrc=`NqQX4Ff*eBEM0>~8u8B8p6=_2*b z$8?rYo1r1s=H>o_j1a@|^@-zbVSOi{ry_YM64Kw2bJJ9?nrW@$zei1v7N6aXhpq!z z?NN&of&A3?H!+JpV^E6JOk~$-MbCb1#_S6_r1YI;r^|J$m3|f7~VxT7|lafEYzTA zLe1V=@+Qef%rK}7!x>&n&t#>h&oD@m-%EIn^0`w7oL?W9eY0GALO(mGo8@j6$CjI; z1I_lSRz{quI7h0M&>^R|i;tzCR)W3Sjsgj=Fmloxo^-S%{7h`;XMFK2!xAkh^(zlb zs7y#)HNWuHwlymGvZwzhIvhREm+Up>2iVF78=RRV=O*Ao@(-(&Ul7TTQsz-;%feTW z-oKk&7a@sW;+vFNjmnLRot$dFd|@6(c1R7)>w!ed5th&#xOX58OyxC{$mGv%w)-efz zk2!?l@U(Z;HCg>NJ5gRh?`M}8Nk*GSVj{!0EZ9eJe_Ch)-(0-o>d`%{27ZS0gMC54cc_ z7L?^OXj4%f?D@{Oi`>gWHgh}9ozge2uPui&9=`?(g8xt0Ik4y1G+Z~%HCJrgwr!h@ zjV5W6)Vk02tlOlG)v&Q`+qP}n`1<{V=TFQrGkfo~VuP&xR+p*Z*3iN(Y#vv$^*a>9 zVv)msZa}k6aZ3dF#F*c$&9P^25QA8P# zV}H?Cn6yYh8d9R=Mg#MiWVS~8n#zZ5*n3}N{wCpvR_QRII<1PCAjnPseU7%2@LIW1 zOE1{mP_oX0EPCdQSj6Lx1#C~kal-|sy1j}I_I$9Nq|~-00`p0FaFUfn-q^wIw>O2o zUwstM5oE2Y9|s;>RGCh__r+djhu&V`kjPbdlSt!uCU7FP_1 zn!c^qEo}WVVxa0IcW)y|;h+{bz?d1rF4Tt#p=H zj8O7wh?wj)Q)d1;O?^aczoC^+$-e#T9*$h7_idvnh3XIm&Tk!jf6exVcKGFeAB*e# zy&raPKn;=AlNwuVVW|Lk&G z@eb$l=%pd=eyYfqtCQq@ZEApiI4Or1^sm-?9{zTb1`6FwPrSOaF*;RH)`Y=Ko`CzuwzB73KBeDc`po0R$K@baHliAXOMjQ03w~)HNrvkAJb&sA8ZIZ>Co07Uzb`a9u7bw%s zvkXhIeUuFv$!hx%w6-!dIsee@>$_| z{RnR~X2`V*Lx@S$j&FL`PX;k~u9%$02H}hx^1`P#jo4tFa{!rc`o)fk66th`+K?Mp>Y6Xr!h4}C{u5sULxTl!-O$&$|rgz6N} z;So+t<-GZ#mws-#%0T`!;W7hfs#i^6cPPyH*#%U7I75>?cgE=vi=WGFt9p&N$31O5 z>O8UI{K@<|aQipW32dj8o+C{1AE{mH=8ueGR6zgkQOdWTQh(ERme}R-_lp%HHqY!H zi$syUWtR3_9iZ!6&UJ0i=0^3-2L`MG{+5QAsQc6sS_ z4QO$m=hm(hH-63cq(Wyb0vsW%(Di;$8R@`%*Zctp4eL!oS|` z436aX_1Z&g9f(%AR{lJvYv=dRAt;>Y(=J9%vtB~nNf2^N6rrrXD?E!R?MV}dzLQA^ zxjFuSJ*E#`%Y_pQ^oFWO>>3Tf&MLo&j@(!P8jY(dy@$-#7T(9x>zxwT+ zuTQ(0m;k+=vG`-hELw_9e0c4dR`({Abo1J+!J8a1`;`={nqMo3V27QSPZ3e&cp7R{ z{B$ET1|?r$|K1-CPb5Y4^7qIXsjM?}fB!8N}`&5hoQZsfi_x^i&kHA;Vdi(=e#o$Ufvh2qS zZj>N@m*Yfgp~3Fr?@+z)qglb^x(sBFLv(c|+9xaWr|JFk5%w^5eBNH;swnkwmUQ+b z#K(1dp`3G`O8W{)`^!IJD3JEg_yF>}Lx&n~|0it3p12^Jd%*2AKJS+IaBn)~wHngA zg#CT_Th*Cn8wDO}914OJe3$sOMIj4WS{9mR?j`@x{ire2E;e5*HAn%m(_fb)M>L0@v1rdAg2uE`J=Q%eNcHKdKgrzh|)9noK(z#pe zjPua4h_7v4Aq5zl4e9M!!t}o9nWYJ+S|vX8pZW)1d1_UlZSdN)G!ugi++o?FlOL-W z@wNoUeQQaF%rV`Rzo{x#+&fNEFNY5QGY8hwWSM;JdeToj#_Dg4=AN>`yczFk3-g9y zm8D|Hvg)z|nq1yM254Zgr{@8a>OaS0FANEPw~!5^hFT~=SPaGarY zDOXzE#ETACs^E~#Zq8?Xb5zs`EwO;eVdD4B!rZe646kf*oY8}Gg}TfJ-}CyKt;&nC z8)|q8@icz4vf?Xlg9%h}xIwA(t^p$@aN+j2Qx&~>4KrQGuLdEt)SqK`8XDj1$wMfm zoe<>DPWGb`7G4YGYj_x$j38zONsR^5i2Vxc0tlI? z)v05&$ug!oGpmK@oL_=J_^ReAV&<=+V|x1wbHb1TweEuO<%w`}9yGg4kxG%8+z4}U%j4n9KA>f5M)MoOAz!E%BA8^;m#?g`W6 zbk;3Dq@STMkM_@H3Os(Gz%eE#A8(Z+RXW_M*K^jZ8u@P2_39nkg?z8MtpkC1gzKEq zugZYQ+R!iE((n`DA2vusF%fdBoOIrBPi6Xl2*Yd!ucFTX>PT|;X{oIvo~#1s#j{dJ zME=_t>C3Q1^{jk*ZJ{*<+zO!E7zR@tiClc_=Dyk9OEVk-{@P2EGG<0) zHEpA~cTQ1=Su_07no=2m zLQVgS$E^EbfsVuR0ez=vM;sM%d^j55WKuh|PBOiM@B$M3taH=KPT#9zLoo{e+ciF9Uq-ayI(xW<%{Int{uRu#=OXW%9ov9wSUg?#vrl> zWe+vf#8VtDlGE)p4qU;FHa!DWr2F~H(a7N|dpeI>GY$sSg0F{b^J4__aA&&tOs}Ln z0WY9DacS^*G7WRc2Wvm_lVA!_U7}0HM!bNosM^(h=V#s@_t8G0D3fGv*+O5GH=u14pAIbF89a_6DR zLF)bNX*t8(G&pk_ZU>?0f}bFsQ0&*g$WSgYUM(**P@}C&I%49lq!^~rIX6(?T$*EA$Fc<0cQ|3G#g=VYCDtvryiXv9k`5!3Oy3X=Uo zp!+M9Rm2Jk@MQ!BT3KO5c9JfNZ{#iBNC~+~aG-q+C=X-9?SV;8Pq*sx!1llmqLHpi z`#_XZV#fYHIz9(w>g-chFFrI_B90pPiyM^AxQ%Mf(1d0cESA$BC*eRmF>b$OSqb}@ zQ9T4*nnZnp{9vJfoCJ?iwVtvM%u?H5gI38TuZFA6l%RD_U-)o312}yh`6OSMkQ6lG z`iR5ZX~(w{Yz_rEgCuvTbKSh0gyx{SnI!T?f$bNq&XwUKK9&;=AKXD}K=%q3R9~Ct z7&FEJ_Wb^7-8YmOXEH6vojcbM4%0x&Qg)C0?+CMK?~>g!DbP&tm+3W2rv9;^-IkG2l`uSSK~=ZU_s!nJhfnb5W!FWp%g^_>#BW^b@-9mM z_pWglMUX-V=Vl8@iM6`_po;~MUafLhPf=D2A7M8}m6V?@S0QYiuKFqbliF>*zy;#( z2QRD2bZbi*`|$P$`5TH(nO=Sd({|Rf?|eHNo(IF$CMN^zS$*4X*Atj&R5QdP+N87Q z!eD~O_JeAb=3kE7v{ehW1m!!(lC2L-uFA@lnS6n))*>D-L%oJb1@VEDTxI$5?#ncL z+rY5D)#P~(b-out?P#`)Nbh=m(!0Sr(R!@`dK05;l(ID+J~MV&=Rin(7=5+k(LPX) zV&LPdzcrXDWK7yUFBfL%@p-harM)CD^#an!>e!b7Ci2wB^e@DxuLG-|-I4D$48vz) z7z`5E!Bfn7k0`7tUjgG6P_c)8R5Vaz3@N!4o z=-$lI2W?#9y<>9H$O3vLl$L)q&yl#*u{?ZB#nalT;S0UjEa)`jyqe3;Cy&sd~s?FV4yEqKinHni7|C=8U1= zqSq`$-W;{H*S^;{9~lGVenwU)XXiat&>6%OwCf{p51A_x9?zfb;hMj@yoMOnXaFFjad3;4)&{wSq*@_*9*#f@?~p@(xZBmMTqYyqj+k6haU=Oh|G_bX6ON?z2TGB_t6AjNR)<;X)YU92l@y4ARaMY;|S;4qsG-3!PTRO(ohl zSyG4kQE<1%O`Ncv&c{m{ZRfsz8Bm?$d_uO#TfxT8xpU-dvrF)K8`{8e2AcF)&Z1xZ zyB9(BQP({IMedUqENT$8>FuC)`z<-bgANO;R9+i(k9f5O(|(LK61dHscIhw z*ERMNA(Ij(kaPEPalBexd5BjfC*1Py>xi+UVQ!`xGu|Zi0KmQ1%uZ(79(jhHpX-kr75CEpfqJQdx&1#7?C4F9E(+U7Eumm! z)S+s{^bYf&*iyWZ^dMM$O*_&cQMudV&?};C+>0yW%PS(kfi{jaYuO9)NZjEgIr;=p zGQD@CDVgbN6W3W_2GT4x8)J(Ye^#Nj_JO)qR*(X*A@HjxMF#m|w}{EwQ+d-0=HI8X7JJ88 z_#nC68JeIY=ITnM@r&S^QXCfjFVm|@X}}n&T$=8()zZC}e5kMrD@sf%nK(kpV`Y>? z6JcP{sV_vvL;R6uRhGVwaTwW-1cwu^s=XwJu~%Khb8ncqtBVfh!O)TEVz~6s<&gxW zGH5CNpCMvFAl;beA@O6hg)SYcxfIed?k`1^hi+~8P+;FN)RMs>vUxqU{0JW>i9eC} zdY?`LS%hBja=ukN;RcpFSEE|b&kGBXm?YF~K;mg0Wa9IaEz-5w>*UV@ACcv@h++r)P446*5W=&&-Z#~ z7h%$y07)*)<1IK8&^2BXlyP4X#a4(s4*Q_y%ILcsGS7&-fRF`3#%xmjnCCl@)!c_! z7<+?tWsy%~50c7%4vbuc17~yXW3vO~dQiZh3G(bZG$Edl2FNr*(6K0=9F8D1d4(kG zpv5ZzPDAft`v=2$#stg8e4|$xWulUZt_EU#RGLA6WpQ2qNcD-w;iFiaAjj701jbAz zTvupJJ(2wTE@zr%FL<=mCHNGlPXFyFcGO}n(P~tBlL@K1hWABOcG@A9DXa(_^kd3f zat}J*!B@oT4iU==QN!z;?H|jOzM4?O(X>NtKv-~ynXR~+3+tx_I`OE})S?b0nUd?JXc0akiWgdQt7<4TncRn_a!!=2EruG3Xpe=`Zdus#9pqL&iz=lvdA*FuRik3jU0Jdj@KAB^pNl%*5+SK7ls8uy1FvKAV5Y8a<3B zYl+1h7&~c0FfH}18Wip@@X|no>WZ({!`z1(lHX{bFrxp`R<-5LL_s;FAKHZ(r?(aC z@Y;_~LYSEIW)Apfn+>k{Pc?T$uJ8YM7TtleK(x2FdeM2%QP9|Rdo{+&&sW=w6V^~s zo$#v&K1QWp2K~%@6-nhyPPuQ*xhW$5@mq^YbmyU;Nq3)37H-$b7XVW!1vERNIe` z6L!U~0qv8o#+TYQ<0i4pv50!7Vmqq;?Vs1UH&g1)KeFiZFHWG0{cg zUjV;!gkQP3LADPsmqnbes2@M3yCRRA}gPL zX;fM6t^=yM=QX?x+<*HT&OrM|9O}32J#kPUs@>{?`>ml_I9~Dg)=7`jO3lviR0V(> zgOPdiLMAcKqGx@RMnw>0w8Gt}3O&+NTx`?kl@-*SV5eW`Wv!Y ziwRc}Wxs-K&NaRw?7E0TNP5RO3f_EIcqMcBWd_?1BJ=Op#_PA=s z+*7&t$t|VTpN`#)NW)V3SXQ6Z{I-bGKy<%8`pwHFiJFPr`N*0N34B9C2l7AMAM;`~ z)uzGLkDmi5-y^Cvq3}qD_~^`T<3)FSi%T^M{uxy+D3z^Xp~>4pXm~>uki%b3P_TOc&6bnz8|+sa4&>mdu2w2|Gw2UBW6nohelT8#+Wqx*ir*{mYWC z;F`y+oxYb$dsok!&7@lg{&L!Fjjy+zWctBv`_X`SlqT{7H!l`N1&7}H5e`-!6Q2tP zWGE@npiz<>3o0Y=>VkV*K;J~w2{WIS&g#A22s^TlvdGb#S=KBPs|3*pQX#hXS zc;Vd6J*q$p)sYu_QSvWsV4I|ILYWlT2%G5)I!Y**rM!_+^*N0}#q4ap@WS54WBO5! zFVmq~iDS+MS5w&iMT`ny$vDE&1hEo?Mp$n)dWmAjztIK1?O8Zy`b&{%Muq~4yiV1r z`{kKmUp|Z{Yp*x^LR)rk=X!;d6ig&q3+3J1L8jj}yYg>MTOq=CiS6*2=fQKUtYVM( zm>^%4l6hR~6&MUeVJ8Y6UTq~peyGlSLInNu9sO>0)Mgq7sy~cr3UC+zs+;9-MVS~= z=_Bm-mKGE8)L_K~w$0bw@&@ z1ZTt@!c7bHaWIuQjJMtXNX0bBB5@oQof~7|`sIkU-ZpN!0WFZ8Gl^&>!tOjZYC*pp zQOKbB?NeCCwelwkers*|2K^Tm_}U;bbfV%km6CU#;wJ|cqQWL_%SklLb18s;xG}Tw~%7e&&FW-+k3mLqwhi>6#u8qthj8vi10OIo7o=x z)saa!B~d|k;9zsWRmi&D9SmA*qelhpaNAl^B&Vw4A3&J=Uz(b_3*$?Do()VY{lWmu zr|j)`gRdzXX+B+&L8vx{*x!Z(H7Tnc?%xZlJ!$MC09|yttGS4f5Wjbs^6sntr<6K(pgB6-|hE2_9D0Aq%Sys)uXs-j>1?RxSj7Z-_^&NRwOB(|6$uM39x+ ze?FqxS~r}9_vdBJxC*J^B_SNEPeV9mVFw2R*c`kLOh_9}?zkwIiqHEj#r-sc*BXl_ z6sn2g>geaJ7?`967VEV{$c(8Ncw&?F1TQ<xT#0#?`KJT6o(oTBY14Fz z-A9jC7N?pzg6a9~ZQnD=Heq{D2({{Mr{Ded$rhhC;ZF8oA|1%Y{7{k7jk+jB zv-dQc26A`DEGlaw+OG+9gLI3l5q9~K#vMB}sDokoB7}3bYtk`t=dGZ9IK$#gYlxytmP(4HZrl0$77E$G<7CRZ|GcJbbHajKQ&bY$W(3je8T3LaSqX+S89Ph z9Texb#pVcHdwI(yyL#|@YY+?jV}JG<$VA?JW7B&cxSs(3iHvD$(`5ZhF(+a`As%S~ z2y-bS%+GW&$`6X4IUa4#$DKmyKvOk%fLVjyt@Gf^N05aaO)qyJ{%gGZ@xg4Lv4cNUVb>*A{kN$9G0I|=Flo6c#u1X-G(C>hW)9&iognE2o?@6$xBx?hS zD!~dgaOIona*VAFl&2FkXg2-jj3zx@$8&Jot_2wD@=Kq>sQMD;w&m2{+Yly?Dtl23 z_wD5hC&tv2?pt3>>Qgzuqgd-RsyTuOpRP+0u9u&md;$Mfj*7#fsVE1_)lR=;@3tKj zzFyRlMoM9D<|5)#-D%Kb6gR^Y2y#o{mm3G?G0{GCRHnE!zWNltKTwds0(Nijlw-6G zy0CRmi?mJeVR%c$B~6hiDmwDK)pyqf}}&Q+TVbb9PAsFfDP`E`*D>wq9E z;69EClzI2hVg^qNg>>3!9x~et6|4(iwMG9V@;CYfVBLrl889gf39SsX?J7)p4!EJx zvMV=>7s?99J$Nu+QPdoaoDzUf*}uH74hisn;ti|3zNZSve8vh*35pLsZHO*mF8jBK zMVw4Q$ib~#O|rrFPeeAgFh1bAx4rmnp!UjG%jDXIu=x@(5-GrMI?_lAtv2;oQsdq} z`c%j|iLFa!T(;LunM3r_3_OJ2A-#>+0C$07|09Gfn_YITfq8lvF zx)?}(_)7MHJ1z4%1wFrO8xD}Qve?CRlbW(RM_louLXMbCef@X&LvLxb5}*1mW~@gvaFbH0#I9kJ6+;wFI+y`HqyrQrfiDm|q-Oztkpf_Y@ttO*`KY;UMyNeYdJqfS2M? z?eKx@)V&i6(+d-FyC``j?bU2QrGgW9e$nB-(n*BpkDuAE{VfJJJ~0K7)Jv9rFWWz* zOlMZ74~53%9O7S~fW9^L31j$u^4lIR(Skdxd!nn^*LBuoA@Kb2f=6Q8zs^-A+o=R+ zxr2_ErEK$TL$5AQ2U>DP%sayp)My4uH28oTz|FAnFSo!JV|A_|1?h{nIhg!hs(a&RTTSH?hWrJBPSf1sqg6SC z%6D*=sqtI$`AY7oxcu&5-oNN3ZzA*-7wA$|1bB(l1t$$h>3E+fk?TQh6$6fKQ}e8W z4M*(rbVG<&uJlhggr#j zll&!ceubg&O9Y3ik;!JN5Ad!crl9MuL^AU4LU$R`KutN3VS2;hMeGLR!zYTrIMUYv zR3H_BxBE=YYdm*-mGWT@g~=AB?+dB7@5fvfjHu)84hSDGe{4Ug>pi8y8B7oQZk>`q ztx*O@#@_z4g0f@t^IZ4KFlX87P-*$0eeVio-lHgOa;2C>gBs-e<2GB!|7 z{ULAelp?}Fc^7|3c>aat9_uh`h5 zvQKkNLj;b8Lf$cQ5`i^Zd_=|{6v?+ehg_dT@pCu5NiymFG4_TgQ~qu;7>$G?IKt~B z=UFe9H3waRyrssMI;;PFGCO_IBZcVwHH~!t^`$O|E@)T0UIjyS*s+H{JcCte;aU13m^eTx%*co zDIOcaJN+th+9FT=+r-IHscz1fOAo&qeceT@T(Iax+T=ch2H{7WubZKMLmWknq8qdt zG0YeI--~?P{+)%i&B+FXIldz>k4Bte+{qB(aAs5AiD3Za_stTIOj{rAwkPZ#w1R6GG!gk>vPIh+@8m4-PdHl;=%FJXwNysM?aEH%O zGJOKkQ9NIGn^{O;meNki)Xft~ep*NThPGy7MKo>!+xXgeusy!XIiQ!O1b>+8mJ_Sr zycyoZ#V*!PGvvFJL9C{;)dF%engg71HmEw%fo)l2cA&=qbHWxvSWg5JZzcbdKpO@k z?KEwXf<7DYJU&7wgH^BH6|IIUnL>tKq%+evukqR(FanBW;nLfkz1k?jUbw)hacqs` zGX_0{jR(0FneBg1mjTy@e&AuE2UFNd} z_YlX1k?QMHG7J=jmPczvY2Vfz;55uW&O>S%KY#`gQ4F{FcgxKuAde;w)0b;yY4Sz= zo@4w;@YI;gAXEaF1cCO_cNr%_N!mV=W`w8NluFQw0TNAKo0a#gCVFyGp=q=dw>DAIXPtO; z6DN;Tf$~R`1*SnQNogbc07{+W4fo*VkFb^C9?6r1tTLACNds`~v|To$ilZ_0iX8hk zBi#EQH)^z(gIqA5caQfP#~PvrR|+KU0-Kl6ta1f?+zd#Akkg!$>k;do4kM@#_tkrQbLgv<$9#&b+)xzUkb7 zo%9kZt|~aCoz(|p4=!RZpMYg+T6B)-WYkDx<8D`?TAwfJ;3vJgE#Ug-Y z(WF!S94r3nxiaAqE z+cvv+(eEWMe=66v$0vcEH6z#^)yFux`@qPEhFPEXnKI!7QT1zCaWX>eQ>A+ej=g9v z%(@w(6*v{lG1Dt7qkJWIT%_1!L$|8s*-kMacG6QNMer3B&%(W$NRVQ}p4x>NO|;KR1f*G>mXj$ai1~B)irNzhqP#)W^o?k%+Cw{)@Eh7CAaCZ zw#^NMfXZX|NyS6>?T4~PtWU=IJJDzum$w)?!R06uc|+mf0cNcj7YDJ-JCR{WM~wSv z4_9P?HDqc@MtJrjoh}YY)f+{KTA4iiG~YXMSg|l z4G>IL7N5~?!S)+hR7U3xL-`5F?&F=%g>9Hs-|L}x%omTWw%?A@FqH(mS=et= zp|K(1ND}516S9k9wpJ6yVl8P_ zY6xj1xF|*S%Y^uGNsjR4g)&(D_R9(D?b5$Q5rauO>{?7Z75!jQWI zvW5>b4n${g2S+}br|YOf%p{b%S#9zsb}b@52?Xm;HvrAw@^W6BzTy5@RV@AuA##Ho z2KJdBQL6o^W}qL|L6qc%S~y z-98d*C>0lPmS(_A;|%RZUh|BZl9fDF zmCFz}fGXlU+U9F}gZX*0KIRtDh~e=Yk=v{o%V$`=G|#gBcnxgEMaq!PE66As^;!mewqD%2-}@CaStiv-jGrKH*YRK8R^X-8VHaj zGU`*wNTK!b-jP3g;K+X>CAAH=(SZc2&}25FW$spB8gJe%NV{7m9I~KxcW=WJGbH9a z;FkJYYrRmck;oDb1i`-2(I!H(`Bw2mGENW~x>ox9)1^zdbO#j<%mSL^Iwf#oi>@(v zK4=@CFV5XHL#@&WPju97h`n4)LtxSk|h6P2ymDD!gT; z^Rj;&jvrR{Ubi{ z{0xvTS|uXCFQx|JJ;XVY?kUF)4!*&@YO^RElqD%7Pf_p&KSx&bsZT%D`effQ@m*GT zD#ExYk2OolaV-rf_)emD%mKzE+%uGHF(c$~XQ&qIBgHRu#ea2eCx<(r%86|$iaFNj?kK>XaWv{NGW-23)OkngBT<0Z+~*QWO+f3?hyd+v)*!u(M6K1miMF2DUK14*}i;{v~CsKB$QI zQ81yr`^`j+m!KdA*-bI5lq1aT$X)$PZ03FyoxA{99@L#%f>y)DYMKgCkub3edvVQ{ zGM(dh;^`*rTL*3yEW@&pawbo!I9?buVO2xtE)lg$hEQZ|Myv`d~1wSx>4K{s>nV= zk44D0IvdU=0gh0Tc2r@~Lk&LJ-{CprFCjGJX%AZY?7!z`=8MKDJ&9mIT2DvAXf9d1 zid10xx_`J3rf0Fp!u&m2xU2bwnT*W_bX--V)6KuW-4Tb?$;tj~Gg#}MyzIY^v>2z9 zK1}wMkpYYKdrFK4VPu`!KcJ|1p}CjZTn~LFkD6*#v9LONvuxSGp-~9D{@Zw^Ffp&e zsrGx`+v=kqTh1P-I-T1IQ8z&x{h&bmxP_YKZ(jSI7(T1Z@l!3x8qXFxe!Z72v>__M zxE3JYy|Zy2l{fT43(mI=OXuWSEBrYsP6cd-{0gFVfmrv9pu-Gdk3zl47o(Q0Mx!vz zwUEU0=p~Gvw`(wv?j2R6D2#leXE+>D!&Y9X1T%iNE{sJH_3_7=nA)Mp9dIkiI-~Lg z58Tcb`F|HIKQ=4N1hu=E9yr90GcOhF^He`pgm`N!umEY(Me;-}>R3X_nzwE}wPSRR z#r-=h$SeaLpR-&3vZnz|*I)6j_ETZsSZh$LuHmbsL&J5`i9FIhbkXOP_UkN2q-K2> zY47+7i%*kg9oavQ=Kg_-G{tjrQnwE2%g4R({Lr>dD1?fmvmx!iE3UD1YdpS+PQy`-6sG#BG zJf_*v5B#|!I%Fd;3e>T)k{)TY{WOD6X+MK7J#?$i=t6Jn4mE1MTLSH%m)EU`_PZUB5{Rc${XbaaBf z9Q6xhguV0K*4f7~!=dn4oa4c1;^T4-&V3HGckx51Kz9x- zfofWQh+8lQ44cnoIL%lkfSF~g$~z@w@`U7d;H$6dB3=BkHG;dtmc_})QW$rz$po1?@$cm zy{R95*k$xPD}<-7=m*|?1(%bAKAh25CVwC0z#ZUjGWMtK0@d*x;<%bgs6(;|TMgVc z4LV|Lc;9NoP|IE3oC=Ud^Fgy^t`X=4StBL!Us!p;V=i=}<_}EOZ~v^9zOjf#EdPLa zqv`ENZ)_NlN6V_!JpUNolqSpZERe5{dcnI){O;Ewy6ugs8V7I0N#!HtHB~48{RfR_ zYI?y1!<{7w&qs&{C}dE}BVAEY;6N-DI?gmvI?!+Ku4q;G-cH$AVwz2tjqzXEddHn` z@9Pg0hdf);LQNWAO|6vEi2L;Td$zqI5geluQRRxwSyms$Fl?}Whh3F;jTrVb`qpe_ zYwlh+V5SuOTEIEc^s^MooS6}#7s5v~b|nXGwHw#y&`y#^GA#Zs@8i)kN>c?aIzx*$ z;@K#CI)Z{sAhB9YMR9pIWv?r{eJ5;Fn11hDD&#Mq^W+*ypOKRMoe_+(eE&~(B2!D< z)h|@(+pAiimT_ z8ER3Gsp^F@KE$P>&7`RMM8cD|Mre^kKVy)lOEw&h{tTP!a9JS`dnU zcNADmK^d=|vsWPN5f{37`M!r&k*9lVr*cEY#2c!Pv)1tkKKYGAzr#IGGR_FIG`96i zlSi(zzJI|l6G2Z>g^?J01R&UNUU7^1Cr&6I&+D_sQ@jI19;^>}^p&ukbLtZ^#=Q=N z@UfaFBhTyWw@HX_fkE1;z&^;gs2)9@QHJ?)GmdZO5YfcD>gtU19P)iuko?`9&MlNo zH%N`U^^=3ab2&IhTM-AyB!4E%$5dWsy0jNY3xT4(c|Ur!<2T^>2A%1U+mzwB4#6AD zR9D|dB+d6ac@zIrXg!N>@Yz~64to0~el;|6IJSmx4!+mBD*qX)3}I$sr!nNrKKxP= z9`bP;ZY(|$@%0k;KQMtRwv^rJv!1Rf$^kp2L|`Sb`;5yEh`E<(TRohip~-?qkvFA1 z>^K&%bx>!SVXR%&JITpPOl^Y2fmbW!sVb!Z?8kdnuCGQ}V(y&68OX@eiKx~O($v*M zsdAtghOLs*HD!=wjKDcUv2=IdY~2gaC=2{Gmhh9eN)dKdpzY-^do6y6NF^R)toVaS zDaR%z^x7+g%YCM)I|kjblCZuc3fJXG#v-IZx3n=4{mrxCN9%6G33C61<*gj?!*l;u z3}ohscqa#l=Uj$z$Fd8>kI~=S#6jpc?dVH?y-t}wsw!pJ51761G6NMM=vJ-sj!#r6 z9nC_;&dme*(h9?mY5$mCnnp;qK<)L2l@MalTvyQ>&fpe=f@I->X-S zBfMSy``#^HJ38+=^%?8xLR0ZSv(%J%gP+)jrDVM}Q+Yd%EnHG&oq=eGnta5U_p@lu zw?rCfy(R_Pu*fJuukZ)-!`(l1YE=^QC0%j(TDYz4uDj{1!1Y<|a&IU6JlQ99QYukF z#Y*lL+1HBRt?Nu$3xB*rCj)fI;ag=@sTw&83*%cDw+_R3j0J0g)lJx?_*#QQK?(~% z(JsqDb-_NHrKCqv>~SCQ+V?rAsQSu=eiKV}MtI?90pof~`Yg%dEVu_T)=XZ}mgoDJ zez|8^`pzUoOfau`7acUcfB7*K@wYPy)(YDs8I(=YA`|&ckH9>95c7&T{*@sQP(Keq zo{6)Q9%DtZUjF|eu!=9+-l<~EF~~9mt0SB0qY$r&kGYoxFkJ`qN;R5sJ6OCtvkQo; z3_3n$@W?4c)s}gPKb0R2{aI#U-8G~|QBOxL+^q2Btu$GaBEx^dGJ2-WX9(sv3Hb6l zf{@neDz(c{y`I02gE4tLkI6y!#l`yMy%-lzIfwpR(CjSydipb{AoxudrM|O_@=1(S z-OfR4-vpwL7(^wI&4$MOe5u=;GI!VR1YxFsB97;sB3?O;H@FV+rFel5?YiB+AHeo=SNRR6hM=?akEI>$*9HYQd5(_nB6R z;LkIN4T%E&7C)bJlkA>&ghpuC_qo1`aZ#vu^x=@z-)$(&1fD$R0E-j_WZF^QWi?ek zCtk`<jx>#EI8CE-kQBq~_cF<8)3^>?+Uus?-e{vTmy!4}o`y?sECIe>KM z^b92(l2THFN`oRGAu5WXqGFuy*eFucH9d1sT98g@De3N#6qPdI&-J{3=l2r!xz5@v z_PsuLUd+-t>~ZxoO%ExBZ2R{yKJzSq@jccmqQZfZ@hL*OaeG|%(dv8u^l-=5$B3~X z#)~NnvcT_lrrwt`L3Y;SpSJh|tWrqP%*5#T9P!>e;Te5m<&U ze5^=qZ49{uYf6XD#w_2f}#gYnd>Mm(s zPB|n`Uf8QiERbIdBKFxHR8;3+>PgubMvt0IyeB6JJYyz^Y6ry?z~Vi4q*pebSpB2TTPyB1TF|n%9+<9 zfL>`b;fx{TOj1<&*-if2%sN4B;dGWhOIXH>3eC`}&)h)BLG-A9MuUZaCx`AAa!rzR zp#3nxSo`5?1;NWruB)#PT~uQ(1?<$_Qk^`1s4joFAM>?YmdjAZX2|-At!>ATfgm)% z5GQiYOv~!+^=U-x*6yUm+&Ue*Z02yXpo>IEDH^%148(&wmFcz|hMp*F&|8_p#N4=f z6+M%W7jBjMK1@-Le6bHRzGAH{EhBcjDy=wd+h5?^;AcYaH4br$$AO9@b8SYGO#up2kZkwD~d;ti8V-zN!zN~;;WyR<2Gt#l7| zJ{LM+)x4~++fe{8DddQ*GM-%)eyElD^Y=3D`S&wk8VR6xxIP#adZ%Av59V}bmoO+0 zf=Nio4LqY)avfu+lxg@(^PWG6`Ep!|R%zNaws=k=RcWN5lXERcxYUq~gsx;pD!T)&lvIjF#M!M$1Rj zZXzxoxt3Da1KW{!4VG!~+{-R}cAvHNdvfuGx=+fh>|7TiA=JcG1FN4f4*rppT2AN5 zh{&l-v?__~cx1-JvOL)%=X7u;SS@l{`s;9q0kb#YS!5@3AM zVkpAs!!20JoB(-nXLmm}@^Q=y-jid%Xd1?J8)%|R#=`@xV06z{+eKi{|k||nW zdAu?>8^v05{_VE$dji*QiH5$1DpJ70$%Cce5nN!HRL)xO{!&}Dk*Xch(0@hRh8X^3 zI-!OOpr;&p_PPDO!k-&ADszdo{e3+vm-m_T<%%K-wjf{R>q9pJl9kATo@V`3C*dR0 z>e9DkC8 z?C|)p)j%uu&NvqsdeQJPxDSKE=A*eCF#Tu2-HY6}yT%H?&XDUv3z_CkXr=n|*BVa1 z_z?-5w<=5DyjM@Z{YaEu>Gms7acfl&gDv}{sAjtwAPHoF&tUW@7g@=Ulb3Qew!-SD z9wS^E*4dMFK0}GA{8B7{-k+d=pSryYD!xgrz^S2kIO5SmZL>Bwk>Jo(c+qxx7!-cqMG#Xb2X1FvnNjyc)LZ@7;mlFz(sM z;kK*7Lxd+-$23cB{#Dh}e_mw$sebm#w@M}S{H!(59TZ-NBLq-~ThDR3?-q9Wge(XE ztgbV<0i|k3-@wil0DQ9O`CzrWW>9-d)%$)kP*C9NG2}SG_;G^s?Y{ztPj>-&tA-v} zGV#%QRX+cEqwT)wd8D7hn{@+_W8pp1XH?`FT8 zZEFj>es1a6*}Cr6V;K}0xW)G1`O)D-nYea9B!yt3)ZOcn&ZFFa4yl^2+-5saG-NN* z2pjm2CzTmv4wQ5J8xKE%1=(kPc9QIC18Vd?>I~m)<-ax&)~OY23iAR`x&`3@?_=W% zYjES)StyI&IdVOJ)0d94VICWNWu}0_zzgF9e_xg8fDY-U#bWpK1U>$e8dz=4cf3P! zH5whxUIc{efu8CH-*ofXUEJ(bvIo%i55JODlu`R+ z&9_Iu(%hqO>@C$*W_JR;N4}&Na7q@xjG3DCuu=E{-lfR9$aBHIfkhk^U-LO1L{+fQ z$}J1<Ag`c_Tnlfw4tOjZ?V_Vva-<5&OHgC7?t6-&R z(ST&v%0+u4F%PTH%e)P_Sr=c*<%WxbLkp}4%3}kpQ*n2J+}u)RTXj;c=mEb;)j+GR zi*EVusu7Ctg0FIV?fd0Zf|Uq!#6XzOs*JM7_o3Sdl;0Myt2`3g1L>8|4a)vIRq5d0B$ukmBq`F zH;sQ0t_5lOcfMmBig5sUuVFQD&^bxM+@xOHwXk38@~ESep-{=Svz=5o^o(R2Ebzs! zU7g;Ct2cJhX1=gvM@O6*;j-mFk{p1Lzn!0rgsS-7GRh>}A_zAFA9d`- zIF6@_s}dhLjei_w0k{UQ0RG4?HnpAe#9I7|z4L;T$q@+xD& z*vhlDWWG1G8i0YR%~|-hA3k2-iEJJYp~u*x2&h+LbyUw6S#9BE{h$SiywWhYF=g)y zR&}?QZ{27UJ<%j>HUBcVQYHH|1l^St;A(Ts$J*FiHS_+384^s-v1rww>JYiRD_X>V z>I9qeHQi0XTrzr(f<%e4^5Z^ogD?-5lJf8DX{n4af26I~Q!OrC1Cp0Vbjm+|Gg}WP z$ytB3(IBX5ko|k(FCK%M?Qx%%4gdEH_8m*X0=zHueHl)YqxDD6^d_O|x#IlX=*Ct~ z5&y9(9>j_N;VULpBhB+H~n;D-nmLU!4sbI zR?;3^sz>Jc@t|n}a+hzxb@}4lWzVT9>+rvN9BtYi+giWtQ~(u;Gr)*fz?Om98}Hs){d`9WkA+cO-NXS0_4LrMVIVRS6EiR_0X~T`zi^>h#<% zVdKv{RxuMV-S+Mz-P0)mxOweo$E8}$^O`DcQWvn|PNnRGs;sgDYNCpj{#BC-JOE^j zPFr}D$L}auz5BLqCs^K*&<#$-^}LhWyd*4<_GwZOApQ187MZ;}pZ6G`!k+ots436w ziuT28fCRC%&M9mN9canue>+gbBcz9tbBH)=rs!27aHgCP@=2r8bClTlN_rV!OEXnK zOjiTaOI8a4t0(M-Mz&On6qE>O#_F_eb5#qj0c`9}V!u8ZI?rbcJ|2wXh%qd@t)4PB zpi%n}%u*Ot;yS`L&)==Jf|4aVd|YPF1V0d91vv+=g-d@$HcA@%dwZYxic*@ZO?%zs5hUs1AZ%<<|@;)GaB|# zF<;pd1~=9hRm~sBSf9?4cTpW$FcjcA>;{_tQd*|A_%0E)W4<&QEYHqLNTjO?=no;Q z&e)dL0EC!=YAuls;Aw8_VCXmM8rQCDTf~Xr=2IyRKHUxH*4hD~BMKp&gx~iq4>o^~ z43RnpE*_XZczwZb-sptl8yBx@mB5?>WoZR1P&w{<`Q+I$*1K97FoD`{GJ0DdUlB&e zclx9`&N%&Y<1(9**p!^*CJ5JF_-_93K~cs!XMx?UX|tMy@K-=ynAL$_+~y_+t4e30 z-9-NJ=Zv70em$^XSOqf?ysDN6m}B+J3yRNNt6W#jzSRDxh@N;9#OS|lsf$vYGGF8- zUjoQoW1820iNgM<9v}Z2@5|4S(zQ>XN>gYs*S9rXnKn&@y7gg;OIXx^wsSmO zS{;M}Q@**q!MArJZ#nl|^n8`h_?~ZYOE3ADvdp=({85m28DO9$@0ex}pNpH`jr zGoj#DX|sYf*{(aHy8PV!gNd|ZzaOlA+$!ytt>xKuHsq!Oj)3Ol zeg)R6n7zcj79TrFRFVN0-XzwZ>ZCP%!ec}?5kP?G1@dCjsd-N_5S$m)(Os4C*lPK#2*KrSJ*X&1O0j0 zJ6Dp5_5b0Yqyt)rrC|u0G<1yaq3J0pQ_=C9pkBIM7E&;*d^xdZD=`_c-4_l&@)~!{ zrWW{~wrX-961xE+r^LylTvWEID|In3z{Px0YuIXV+v5u*GTXD*&#OPvro(ILZXAT) zXQfYM+w=l~3jwX{je88Br`0(tb3mj9i9Q8#blK(e2CZX^JPkL25c3qcXweY^19=m} zQx_8nj%PA8Lp#BWL5J-poj0jKys4tywni zT0%RJXk%%%?d8h&>a#cW4*8n_ud4bv&6vh~AyChXURz&1&BbeD75J^b#V#NF(cBaC z-m@HYWCask?&4_dIMcC$cX4d4R;jdZ&z8X;gsZ#V1G}1XUQ1iFQ zFDnv$Ssk`HGIv})Il_+r0#H4fc&THD@yktcZYTmQ%~mdx)R^~`lKtmCwA0Zx?mYu= z=MP%u_Dh4;`L%WWTwKu^&2oQ@-e2JR=-6uFVSlYH7SLoK0XvP8Z$1`bauNm9T3p>{ zro4>K8U>yzHMJxeoyP(%ZQ;1fEFPcgk9UQpQS@j2`S9Q_QNsv}urLPX=K`-hWZXHW z$NU_5Zf+QHJ@2uR?gGlJ@)jr4<(NGeA>Hxb7z>Mj$KLiNb}Y}o+3>*jJ3;4oV@I|a z_g8lSscyod6yE>;yhvj!IsvUsfF3a(W#}W>6wMamEC^_ z2cm$|PK9&&`L^$;6Q5u{>7n*m(+1R)LIf?oWV`+?2P;@aff&Voy9{u(`g;36uZMfW z#_l1A&kv&;2Zofw&2ik`hvYa9*`HU~Y-Z$UWIjD9p@&A0B<=ftg$;t4moqO2tuKF- z1R7P=FctYnub=3l#$If=Ly+J;(FbQwo=daJvC|)VRmBeM{Sq>2)aCVm_H-r!nr0oB z2nw;m5O{ykQwnB&b39~+17<&>|7Iwke-Y#um;&y`EoAZ~l!uFPv2$4WSG^gcrNRL| ztQJC3b3>|U1MJdsVR%QsQg3)#zVe&XM{S$b|1DV%FgBXdIX_`MRskLze&fZz{p6>g zHoSEt7W>ZGfzQH#ZwMy!$*@sV)YG}+4yf>|LQv}Q2Wd%MZ4%*M>?Yse(TD?aV7Mh0 zzI{AjWNKi`c5hhQ7mi64^engRw7EGDQg=L;FbT8E5(oEU-l*A6i)`nK8cx9sc8&54 zvcjR#&6oB2c{+SR)g#bY^7X^1(=(#?AyS!2rfzU`5pBJJ_R?Ck^H1R4V}JGvCGDvhQHlUB02TNx(NiS6?o&^sAzm z2=I4qjiDJAw0_L;@a|3tQ5(CwP$&2=u&?_H8zJ{&7xV>&t$l0V#gEW>()^;gB%ly5 z8-4Rp;6jF>zz1;IAE5dR173?IUS%F_`)Oc%5asf>uKfK8_O12(o5)J_HkEj0=M?PE zIw5cBUM$8{V)9TiValrYR`9Lfx;i7PI@uU`NBSoQIG6vOe^=b)Jz+io6F$c?RQXq% zFGaZICeKpS%HGX4mJPT}W8dpJ>z+CXpy%^tSl!#5 zBe%7UoKVUldh!tE`%R|cS3;w0xuQf^KI73iAOie0WVEY2&xtd3r+!HiR)4q(&c5I& zrLz6AN_toM>@+Vgkg1}dcK^Ll-q*gImPJmmDf-#00#gF zO?4ebN-pSQmE=+mYtqgGec`g|Le<@_5Kr;+RcnE~b%sEqP^NlQMGKWyGU2cSOAV_xG#CMgTM|9YW~EwFZXvh^2ai~cFWtR^f_$^(QiiY zEU5I(^E;lz0On2VY*!uzRzE_!ML0fnBEE*_D7$iW+;ah?b(d0a9{Lh5z1qA9COFT_ z!Y>7c_$t@-(Sp6^<2m7i}cyqfB|UUn?lOV#ln49LDjkqD&M7YGdASbiYfdW=7 zD!aYeO-kLy{XOEPtS?R7D^$!TyeV;bZ)wAuvBi!C8io6YU+^wF|J=V(9Dl4mG`@kH z6q|B6Sa^-}+>^ljNEFx^mm#*k4v+jhgSk|ioue%E;0(C^^(S_&eI92gfejvqA)m^& z%dmU!MBIC^{zEn(_f>AZIH*(f8|9q6;Qo|Zbqq+oU(WIzu@*4}vITff7X9v9SODTH zc9$2_232Yz{MUZL9>Gl}A6!1#({T2wbQ(G%gJ8W|7{*t``8&#D==N}=B?nMZ$UhnK z;+o{bnW;a!*S-9&DcrTE;Gej7NhRiiI3F=N;5w_j`rT(Q=Lyto>#m@%l^{ZPKf}0{ z@$<&19j=pJ23xR1Th1qf@gs&rZD>OJR7&Xn8ItA?Q^L!~t1?Y<54ky{frD15rk2wg zH8*ul?=8?aGw1aubKY+dRu9co3w2->XnF)r7=Jt+b$;TE{uMR z`W5(Yr*geDHTo&bqY4nV^FY#&4ZktA=@Y;ZS^?;N_ z6DaF8!^RcaMDVC%3OdwpEme9(*4Qa*_092N?*2I~98GlVIieB*&@WthD!p?Gyj<*utk1Hh%wjb@d;z!HO;T64@ z+xsXN<{I?c*>hGtlh>L}TEB&mJv=S~d-MDUx8He*kS|@40l~lr(9o;U?9!{rnsX-( zp^KKf8R182YLZWAIOrJ^hMiD$1Ddt0k?Sqo-nk1L8*tV!mDlEGg26hQ7xZ$j*$9CM z1`i;6?nfooSwLw3-1;W-Z8mc$tV90>yOqPNe$ds^zFN{|06};eXWZ@c5_j{c_B{82 z?Ec7THjKA5{ut?tY2h~^-$-CghOW$tU`Os9^uHezbbSLhuNhUAXbabUs92Fd@tw2_ z)AkLkDR`%_or7NKWxvF8zGadSYt}maI8T}~Nv!^s%mnb54;Q>b=g-Q(z8G6;fC{o+ zCahH5M#oO8(;fMN6$ZC}PKVm>O`L@T^O3I9iOzC@NWEdMlh$lx~k4)CrIY|7WiP2K{eG= z(QCKMuKo@$oCL~gb@F09@U0USq*L!AM0XEC`B&XG*f;a zzQY>iEkJk!{zk{kUy+EAIX{}+mX|7b16chb!>!ho!!Ga2^sf2+XYk|ogA6v&a)F3- z5RUwmn)?gz;7Lc=8{#_ktiS5-nY}ZV_Gq9en%=K)IgQV?BLpeD1$fWAx6qS~ks+MG zJl^v`NI2JY#O5)I9~7^6CkxIlHHiZAE6z#_T>!_B$=XKkZI-u5xJB)4LUvhod8}WS zB*d2wEF<}h#qZQs$M6{$4V4mhOj3I1$}AKDroprZ?KNUp+n38p07DpD|ZlT01=j&G+JMYLAId4}Jgl-CJ zZNs|dwD*EUBoC`<#AL+Q*EiO2PDN4`ZlA|mf*k~o7U($x;M6g;b(7vfuKXy&>2t2h5WCQYT6Y0#?<0ggTT_0Aw) z*+HTIN!BCMKKe^72o(#U$)X@(=ShhtA6PgoBh~qfV7&iXzbzVCX4Nv+?NvVN1Ju2F zchPCwnSFTnC(C25@7#aC7jCu^EO?iElFMrnIwYO}Q6R6Al;+jZ&n*dZN6)ul&bgbO z;uQUI;jf}fbFX!Y{4LxJ+Gs zs1$K{JQ(?B{h|BM7%Gvl85*oEANs6awZG?A9~mfh-C(C*^hE`*LcoJU>nhc0 z?t~f5P%bo~KGyp~>2-qj;SI@bRiSFpVC-mi>l|E>AFzcka=)}ui zOl|w+Ht7o#s3=Bb47f19XQJx%w-THx zZ38Ue(nS4}6M7{@1ue{`gOQmV?~VdL#q{pCp;k7w?Ccjr+*UHkE_rta75fu4)l1CJ zw}4T6fSz!D`0(jag_$Y`DMC|_j%@4iEK}_s0qv)yh8EIm+2R1F`j2bimSRo7#=`M) z`+uf6v+G7`IyaXt@36wA`^XxOKt=*MexR;z#T&1c>809r`rwAJZ~Ta=L}9SWXt8Ht z^D!>oa#Oj4o==!|z>e^*THj2=zjV5~a1!>{JP%cu*>JY^FdE$L6?%^DHEU5Tf85|C z)jX{x4K-dGj_RH0(rR?i1%cf0{zcsgVCpoRrNC!O*D6@@b9(0IpS90U%VX;&&tM+{ zJc)7uJIb( zfyVvNr&nqW+!~_M>V#y#eYMg6xr(5|0FbQx3*C&nQErm_CTeY z=KHI29+!+OIQu^WH{l;n6QfUZR7m{$0^Thjnhb9PWUA`8PX||qO{2KRwgHfRkodw} zQ$zR|-+o>VqvOle7C_6*y}xfdg@U;pN(2{GT+7!K^ErqW%SS2}x2IM$2wYqyGPo%x z4f2H#n)Ry~`CWqM^ck0(fFa|XCGH%xJ@sS%`z`4`%Q3t1#ir>okw1gyzRC4=_P-dvA;~d2DxEAs7hK&j0c+rOEr(zp7TBZbEiOB^vp9 zHs_6sIJsNhoYcV21sU;RK#9*6`B#3D)7OOV`fv5AKN* z^3v}re?0<0EnBY3=1bf1RRR-GxAZ}9-@KNrJ)ulxbGT_Ty!LWqg|6u@;4iz3-gfsu zZfbp+?>a11`3(OLP1!J?1=bF5IM2BR#{_hqi-eBhs9T;t02QeTvB8c*^t>)sE9?hKwR;J2!G{+$P6gI9lL zTWr8q{_fu{h)6_-?WnC?d329W$$qKYfSbVV+5VU#&`3t{0nLdO7ab0sxQWo~Ru)n% zaY?C!d;=rl-LI_4)31r?{JvafZ7umWt%%&3Q_lq`aHSEl1f=E)o? zEbW-crvVAf4eFSd+UTCl@(KL5sbSNZ@>kq6&}wAHwjAaj9HywTpW}Xwu-qAWwSG`{ z5yH&B}5pO;6SPTSZ7-&|04GY!e38_>shf}OI+i!afuMQ$?})3Ews*m?!@z;5Ka4j|QV|qA_vef+9RU}wa~(=V z`W^Qu%bB@l{^J|a!gl2ZisUHSYpGQD_gp*dt+$-sLPP5y8DUl#k@%oOJujoSrq=sMDQ1*7{aqWsvA&M#wnt z_DoTteX;(d3m|@4F(Iqy>qiSjsJ2Ycx(CY=tKeDR3y_A^2hOrLalX=}%qY#i}-1B!C|olNLP^FzW`Ju;_lJugqKe=TT6+Xew89 z#ok{X!ht1+)10aTUO)pJ-Dkn2?(&M_8T@XdrzZJpewClf!|W!iIav-6-r@TboA4?z9#y&NpIkw$*n|Az8+J9QcZpQzy9xC7GBrt6kd z$~{1JT}OW|3!&eR?$#TpVl)0O;zZc-I{wXlK7;wD`YVICfe+%T{WcZTAXltj)K9u! z>Z94CW4Ak37cfhk>_6K0U@b7P1|@T1Jf>wpX)M)uP_XZ{J794$$Psr#_6xE`>{!Ef z*tTB&HNum}GA)lh4%j8TOw;G|+4o|0&~fMP#k1X2#8m-GCxj?G|E_@_jwqb|#OG+= zy5H6t<>T5VLD9(aNqIM12bjFK5&s%hj;-At{XD~Ryi(DeP~mWg&w^ad*)8?Mt5+3h z{46i$lDOx*MBQ%;^2s9Pd=ag_Yl!%>aTCV$aNb<>2XrN-k1jhCINba$IOVHuaqY#6 z70Ja=xyIJFdCrP{YJLIG{Rr(_bql3e^7`^@9`)FG4L`T?UzoWSq}PUSfAQ@(J6A_v z^NF#BS$;m|)D%IUtiCO~dB?uky=Fr69Mg}-+nofoELrW5`a==bBe|AyVm7SS);}{9 zK%LpoXLEEkH#q{1a=i~3DfiB8d*m`=TI_pQw7<4JUueH@x?5d!tdsEBWbX&y(yvo# za_*!q$g%P?TI1CErGoFoKi`~Szp8!JmGbO1egFuyA?XytpSu^bkQS7gfcD$Jw@kZn z0Z^esd#m%eWpNFFYoQ4Zdbt*F`sPnR+h8HyCd+R;g#tSn-M)PWk-u2%0j|%UPHOsb zwB$?OdmkP&71gi|gM+J5tGR!E9{2129#%>8yWWx0BSTs=72C4r}j2O(1?7I zOi*^JpSQvF!zzCC0N$gkG7@F46h=k;0PkN_NABm+Mr;{ErG(>aE^J@^WRw<1jk;J4 zd-l4&8<+!NyG`FJ|6KcPoqs#>-XiGKU#RdC88F{dUN9~p{pW*>5ErImrp$r++>B3? zad10m!9I9$QMZ3ctJ;fgIk(54rP>C7KfwROd7zUfzFidg-7NKf?u*;|6i_G9VfN+s zq8r!9VZhA^?8$7mHR6rlTm}yO*Ee1=R|^aFyx9G*FYdgKftLnwjP!zb(b5MTbJ`M2 zn4;i>=a(17g4Yv5#17uiztMg78K{)0a4PDSgZCH_uAomyq#W2wOKv#l0!ODVzP)n& z8=o1#whg0c9Ho5uM3DB}zMau5^SaK_e!u1{tMVrDV&hsF3yVpvP}8>&VG(~Q7sXmr zV$z_tt4`3BfkeKD>J)~4JCOdLzW2X5Scf{hnx@vWdmB1x2fGVe-{+K;F`lK9a^Jp> zq`V|VyW(YV{ZJ+S7u^?6px49;U?4P|#DiL5{D3&&`M7?u92yHGPbueBf_CL1kS8(w zG7%{S=s`?4G8X5S{FteSStRkp!|{RC8%R^gjIkxBN&bXh#%)Wh!tt{0GzY{z_yz0) z#Rz|$9xhBJ=i=fq$MB1o9=IMp0#d~765rw3QfZR+()pS3kSooHa*<@1{yyU$bOwKw zRtx1a?i>lA4X0z!VvL8RN3?Acj50%1g_@yzcn9hu#HWnE(#fc|GAHpEtQGDLE*QUq zAC!U1$)c~|yU@jWZM3cI6xs~!iaCKcLxKpLtPFY;{~ceB-$&B$j(8^SxwJM;O!h4% zicJICA;OEli@pCW5??H1g};jgvHdciv9|aU@tblnPmdsx*iQ6+7z)-E&yQEZJL7pV zw{f--CHQZcIgEmcH0mdQM~cofgDXTL#22w>{6(}2d=;Z7QHWYYvE$+BOoRm98>h(M zgeioZVp(u22tFhZ9gMhy1F%jq4^Xe9IdRvJ{)jKwa<~^9gMNV;&a%m*c={D_{Y4|Rj8lEhXfUuV9!yDsI zp=I#;l3x%#(m66!xFj?c+bN@h{)N$`W0PZqZ_>PFZ^sa+^$aHSP#PsNMXAbsMoL0I zL}oIcGrtS{p}u{^0;Q1|*z@UwGLh)v6bN0#pfX7K=gjADFgp2=+l`|Xmw04`XlN9dnzLnL%}R1f5PBn*AW5e zNQqFo9Gnlo6yG4{D>s5#L>(t$=|%XP$WfYN;uqo^&J5LqUZ#3Wvf?x(9g{6Go+oFc z4Jk@U6)GF{6+W9DCh3is#Bo!0iT>258KT&D*;w>3Tt7UUK25T~X>z)v#_8TtQ~#}G zVlj>>wiqm>Oo}Wmj}*Y0BRtR?nFYv3yd0VUx0RBSJDPS}_yy)Ueh~Ga@Dmw7tOnj7 zu^e?W!wQ)KAE)9mhBB5?252g3n_4POM8RS&rXS$o=qE@ekw(ltxjgg$@iFf|DS1?- z^or;$NgToyX)awLCx__4$m8!z0XPxdJBi29Yd8^Ear8E#7Q=y3kS&9!GcaO#h;?Fe zx^oKM-kF5(9_cFge&Fzo=RaX(bIIX$hY)qND;b#b;z7T z+aP_g$&3bEtjse^6T@4kh?zphkS3rEJf13)RtzDrRw*WEB;txBm&_jeGVT=|A>k>j zNGp)!#6Q9>;P2wEqHAMDc$M*g@mcsKOeeM!C53z;o|J9?|0FV&UdepJh+`DcQ8Wie zCKX4|X4<6ZQ_bl2X_-+C^!HFFC5~A_?uOPGpGeJ=SQ;1;N9%!RnO5}Y&{g`=Opd6h zj9MxW6hqV`>(hCm77CWZ0TC$$&_KpL##^Q(W0-N8d5Inm{a^+_W|`AGg6ZM3$MkV1 zl=6a6nz@#u%8;gk#7?R)B+6KWolFg<&+zY1uajyhy7cqVciLNsKxWcXpekApb33jE znkAiN`anq(H%2yThSo}Bg}x;X(7z@`rOszgGEFH4R5&C=Tcy8-V*c|Z|DurN^&uWI zf_D(QK|4!3mZU{_#yHPxOiyJLK+g1Ph(RVoOSE_N4q|!CC%PihpZ=P0g?x=sm-HXg zF&zf=KnfHuhylH2PBGs@S&$p?Yq|{O8{Lf_1Z9vENPLhW@j5i0QVZQ?lrkpis*n=o z%uIj^81@4889@vY29>5r(}`(fCX2xlBgn)QRV)m99rqjK4WAOxho3-uql>Vu*k&q< zK7~6#C?yZ0>o6e_9ES#fuVh*=+~gp17P(OBH8Lzy4}A>zIk^!nf;OR?raF?YBerBi z>B=%Vx@gAB6k%qRL^kri^hYLt`cX8F%7JryM<)}IQ!RQMxZ@O)|V7jTy2IH9s zn)nH|NBc^jWuQ?wbhPwKVhd!C3SzuL6p)w6jzR&n3L;1|ITn+8kD|bo$~4Zf#M`C6 z!=%$Osd9)rOmPMu?FgNnF`W39luo9>btLp5ZH88Q4zioZN@I_qF}#>|LWYb{=^yEK zjGII-U4rrmE|z%-sY%fnXAzxaeq%(@6q!Gn29PpzBk3ASm}y84&9owa&V;DOWCQY6 zL_f2b5=nYSc~84Q(h~`#ykhV&!x?$d0Ih;FOXoRu55gwTr|e{Kk%ge+q-dHyU7vnF zok}`LOh#-{NlexZDaer2BfUs}{Z<0u#qh(PV>TtQlBOum)CR_BS#Rn~#(xk_1|TOB ze?P&I0`Tr|2WBbrJ4p;XNWX_mMM`16Cf!TCz|5jmQ0`%hQzU2-Obq!M1tbYW!W6s2 zVT>&5Khy!r4dI@AEgel~rt1k^M@Hj&k@v)Dkd6#Qy_wdBF2+3*egW4P7m;p*@1as8 z^QeJ%@x(SEPUyP?rz}Dw73+@FM%V@|LotS5b`FaDbjZwn5Nh>4Ea3)ygn1I)7{C_bes85(X@m%=-FzZoDQo>>+^aGqs z+()1dIN0Dc7V}ea65m_dB1%34}OPj_E;Jp!I zcqgnQHU-N-TfKV^hsi$1Iz~3Z!|*I}ark;_8<9lpNqtE<$(&|39>)G{+Hukjt%JNp zL}PvMO%yNKPpNlMQi_83Yhh7JkN701iYh}5g*TA|NH1iDX(przoGw!$T`E6$j5xE(nm}i)fjBe6+ng=?H9FKj5`;i`jUqR6&Ia6$qSLC$uE2M^4L#%>K zoQywQ3XzR+#@&%smn%=N$F-;0BGfT62s!j|e5mX*A|AJnn#4_G0?}c}P{|HtN}2=0 z10l8Q%!CiTMO z=zA$J2~P0ExLtIbA zz?&&~$iMW@NCTW5x}L^IdMacCDN=l>$E59O3|uJWj}0Z2%f68?hFf87;%S&g-dA`$ zdV@NJVLz-U8}VPUZ{yn$g_y6naYVPQ1|ka2hM9_-U?d{nzy9l*0Jk(67p{r-8 z(tQ{#ndeC7l4a?|^x2Fo#(ydHkUuULSu3ZAZ@`J9B*W9N+Q>@0I$jScFV%{CO0%Jt z;D2H|Q#~=dQoL|2G)~kNK8t!N`GJ&8K8a7oUMAfWG?3NJkj5&@vLTdY*vax(MPzw$ z4zh#x33HAHqy0in(=3zyq%Sks)1^p(GL7ufpyi z0XbDlJ$#v0`%uwiM{c7h$vI}qFyyIX^ifJY?HBo+uT|)l?^H3fdU^31>zp;mA-`h6$|)2~u_8H?jX{w;&nJ0%GuRYmvvOarjAk9Oe(I4jzZDMpwyYipOGV@I`bC z-W|`0DnKe9c7&xU9sCs}2kJgj0TV0rN-h{5Mpr^wWHjUdW11w!FpkhnlAe&lq0#hk zW;pa8G)8kv(W7yZQy78J5@VN+r~5<1=)V3}pR)o-?GNn&ht#3pI(k zMGB!7rx-(N31*Di1V75B>$nv{vED?C*sslaLmgTI{Y-2DW`;J#oS;f2(#l^ z*Oc2+01omH@XG=N@j#) z2C<253h~2s(07sH8G9MIwA93Yq$$QrW{@TZHKwmKQ&4M^GbB02FwHDQ8s8$3pV5_~ ziJ~DLGwP9tY9pEfbr;2g_(Ob5w-X*_e&bD~o}x9Qb)>sf(nWIVuVsZqbQo7jhdm%N z>aZ&Ai2X~+gxjZxQ5;fCMcb0Ah+AofsY6f#-4Ja=J1=q+N~5AtM#)k%h}KIBCEG9q zB}*9-iLx?7OaZ0^eUFSLvQRQY(lE|#H26NeHRaU?#v39~}noBn?komF%jNw;~xlX`3Tv2swc(-EL2aD_U zHsBBG9bXdUfRu1GtxQmLR51TB*$6doL)@R$Ah6dw;+i8K;noU&!I@Giu9C?KAN5e~ zT6IchNUCLQb1%b}(o)eK;Y9X!tp?smJLG!kaPZL@D&8Sy&~`jg?k3biYvezq1T@mU zScb5=bw=zXQ51H!bPf{w^XYOa)L(9dcKKSuD&k=9&}#1Mn>jjR7v|_p{gyGz+^9Nf zJ1t<93J$Pt)4j%Y{j~v<5M8M?w@zd~OgnC_4qo;rYZdd3$KUkk!MBE!{-traBCSE* zXf9qp%(E5cZ(ZckD|3%;fy=LwYB7Jv`8t7==wF;BDS`V$Y z?-{<$ddxzs=UiZYP!=0!jU8+Ut4brI1&t=d$SS&0pFmI3KWGahWKF}j)xPu&xomW$ z02fk^X^>Jq#VivTiPLkdDn0S?>=pD*@)g{J*3_Tn#+j4UKp@$$1x4Ckt8Wyc6$x7O zoetF|TFcbmNIzwhrT7OXSIBi(r2}Ww3itr6raMV2{fmn1?73|9wdRv{BuY7CRu3M* zQ~g!R+wgpPmMk`p&~)ujcJ4%x(^Lq)!@C0A11AV)KBoWQbC0oVT3+KhUXj}~c*NZ4 zTR>uPK{LU5QN9XipemW?Y~S4ToSeKJ9+fLeRnaMk9g%@6Y8!qQxFubdW_x_zVg6iL zf{!8B>@ndMkf6sxRlO+O1a_J0;eUDqZ)1Oj+!AmM7#TAFRLlJwEN|^q3&O^Ud$_x5 zaXLI)+$bow!kc`x!C(GlwGoWP0qc0tQtFHNzIYy22 z#v3JkPs}O_HN?iwRDG)9%UxjJO+23wL!Xk3QJ0IiMY{5Z0@!7iF^k}2@*;3ncX9npOnRDOxiIXCI+_t{R)nVL=g`ukO}N*fYM%cE zm+OzUiViP+Bb>*LLhh6zS(B~8!d>+bF|5w^E(rE?{p9}iPBNcI-3udF#@&{w&^pN_ zBh3vY5vcNYrhp`-S59gf;<={edEgYhlKM+~XhaDExCijDI2C<@?I}PceX(~W{S(Bq zy7Zv7o%T|G;y;vPRuq09PN0^aV17}vwFjXHI>Y+YKS=G$D%Eo|nXa?m2GgxBfG$Y~ zkCWek&WM-VA)8>KlV~9vC0!F9@#VOspqNweu#ruk#gZL;5cAq+K^t6ae~5mEkND=U z-{nr|lx-{g<*kj@!qadS)0BVb{*cexn>eWVwQwbEf#^m)xu{gqT@9^D8YcZAeHA{V z9`Y-8wv6US|qJb{KdME0M`(86P_|v zT9VTFeg)s4B1z+zDi!6ZBFDK}qIKv8T;!U^eH7lwgHd7h7t;qsu`EB1Ul_jvnS8or z$%VKWv`si5%W{7?hvE5*-C=fpeCHY{0M(SoXH)KKS|%AL(JgsB-HV2 z6t>Ep(E-p3Wux-)YIqs`0}WV}^?DiFO}#R`V?yw<^Mm8@a#f1&lhd-lX!gA|qqatr+P z%v*4gyh8wB1z7~IE+1_ltQIwNun`my+rgb&JROiaNY23rh`<+zU%jtEAu>}_#fJjYAsKFAye&<7o&@4Dp9m(^Dw@fbd&iDjg2|n$k0ao zmo9obnI|&^a#Aq)`^ium0xxSn;eVc;-VPi=C9LY9Q}!9TNqkX9)7)h7qPP%nsFIK$ zAkUicZLw%ZtUMZi7Tco%q?n@?6qsKy)8EjRsILLFy;Xy4RyX9F#n!prD_6PI;14hX zO7bXfM7+WGvvcr&(o|_y@UGa?)g}0ftBU3Ym+hi`rQK5d2Mk@MlLW>YyCbVonU`=}MJ zQY1j``Mat+QF|$g)C{La+lBJzpm!*$$(@$JdJg97AhZ2*wI(FmpjIw^%3UE|er)10 zYo~u5dFC1}{41{uM{0}cEzfz@Y3tk;dqYF#Hi0{O5=@W}!JpO?ZHOx$37aKt+3*kC zD0`?fQ_e6GL2K^@`O)gB@*Xrfb1M2T{;l=|OhWmVO-04|r}7Wd9MuAw9Q9rQ%BLaW z3#9n*1ov-zM>vI))bis_R#&qit6KZg+PDN=75tyytN*0Gw8mPb?=^j#buBbF^KCej zR>NZE5?_qZkn&A!OV=pbBwjC{TUmdvMk z!@&q`65U0Ynvk3>zC#&p?##Vl7RcC0D_b{c40%Gkq*ceoqtB96YR8p7%mcWlevG`O z>&f!fTS*zS5-FS$g&p{H=0M!iEQ>$*`e|LW%2-wC$FNTyWPQ|iGmrJb+!-CO)uzV^ z9b#(p9+Fqx88-ByG}nU4+3;W3uhpbWlqYzd*)+GkUOTdra)Ml@!%T5e(9BCc;dAO) zc1JG{?+w?q_TeaFHcrE%aGX+w{9_I@Cz!?b`E(?GALxu#e88&dA4JcPdzpQgozgq1 zd(1JWjjqsMTc$A(ccUw4#nqP87O&Q-hUW%{nBT08v_!Zh*IYtSBf{T+iP@!+gePl7lo$pJ>$8HwXQoqd6kGgV`v&&$ZvRLHJ>$ z@+-+hzIEzWIh({9bVb5fb0zL#dcEoT2~sCi)t}&% zgMEWlwaNBd;dc5#eX{|QYT_ut>202QX#R9sAhSAoo}CaXk1OJCVkcVO?*Mnr*4nI4 zccB%jn^Q{Zga_aso>yiuv#OSZ2N;>bSFGQDq_+t=bGDPlzAR^|QODSfJ84U-VPJHy zG+d(AB8F=y>ENrU)lYgE7R(5xAv=x2_^o;Y|FlGWAXJZj)vMB7W>KY%a+Y2Uk1%U! z`C=l}nrZ{&ROX=Jv{hmk+=y1C`(OrnD*P=i({9q~Bu;(6RWavsJ-{@NkxR8gWIZlp zPx< ztMFaW39fhIT=}J(PY~t0co?|M*OnvA;mR&E+uX+#ok`Yl^P=`8_qAE0_(1D7>jW8O zcE|NeHG0whlv@{kl>^((Ir}x`4+!XJ%RW12#qicteLE)7_C_asd#$`ZNUwx z95+n>+SZ6*PnE0N=7R__ECd#>4q8zbzfkGUGr2v?@~N2`S3e zGOoD$`Rc3RoqgbXtq6$ahZ^g>({KUl5&nmt=6!`F<%!XZ+=}`;cOYNNObR}9R*NoH zWEHnZSeD|(t@J5sQ={4Z2Du$UV|S5}9b zwecESgly+={7hqktCiM<>{n_e4RI~>&c(Wr%6In&tXe#vl~bmhpRH`Zb$Cf|hy5r1 ztQYZf=0i(c`b>Q4>tyXCfZjDHhE2~f?MT*Jyv_W_YUP@vmGey}=Zz`eRPx&R5z|~9 zqj*A%_38F=T#dv|xHj=6oV31#l0$Q7d8?7DQl1#GAYCCXQR?_j?P++LK8)mOPU{gl zV-0dv*LwvMvu`;(9uZBk)|yZBT4D^HirRq>vR4R#EInU(YtQoZA?n_=<|Gbm)c+-& zcoFDM9@XTh$)%+re=(w^RVB~|v{07`-;ybep@ zBI-jj3%4W<{2g#4eL_F7uChGMPx`UjqHO2}nFbr#QxY26>w{vpySCC?bu<@Ul%B$T z;C#{$R*C;9v_pG^L+FJ(S{js{!mmKi_<)=wJ`ra-HnGZMK8izIg~i-B7bm=iZ8J+b zXM@8a&nhhV+hG>CAfJ&Ipe1}m?+sfga1h*qUTI~{5KnVkdGsIX z21Bd;n}_gnR*A z@XiBsrEjj$$=kTa%zxV^S8;Vl>5>NOaXTFA#m1n$bPQFt{p(pG{+&}hyCgg<59NPH z%Y~o(MkK}zc4vDx#7-4=qaLWbcmnx7`-JZDZfPnSf`+HYGY03Ld`(tSQ*;MbMI?3! z(oor=rKE9YJ^C$TA=#TottohNMy4al7^r+TGu4a6AaGn7X6?*sN%L#P!hUluof>t= zhsbzP+8AqG&`yN5vpv`V_)t&n0WOjIj+-JK3q5yK^38CSm*!oM2REHZSL@K?&GSCrTm3eeLxLN26l3_*WFhr)jmJnawI|-Wk zXG{H^3R|};_?vKP%p_HmqESDgF}TN10Y$(VdQ+*whj17bmm8q+{IA4(W(U3Hssds| zt|u``8rW(0eb2}mBa#dEaPq z1J3p~bp3{M#E)RFtGm<+T`N9Sy5l(~d_&v#cc?61#^aNky4i!cxE!wr{8C}QsQnG< zhHkr6d6FFAND(e_2f;zu2Q`Izg`d)Uu|G^E2pvO`gn!udlI{b6cFDXlkmIq6xKoBL$r`b+*6}pg^Pkbj$0?oYl(Fe~16qK3^ zpIyVjAFw)_FJDF9g^FmRG)`>8etu2tCOwCJr7UnCMM=>-0KH@t#un?2p1H?!5$><> zt~|GJfB4Q0(J}FkG}`Onr-Dk{8~8&QEq~!ZO54%DTxq@=?6YXG=N_EvT`xaN%4@HH z_6T=Sw3v!c#%~94&?ZS}gfs-z0Wa*I9QW-ZP>TB&J5wI#`eb_#h9%Z=ACcbjkI{JV zC~hlk%}%CA;w3a1Uh-~ReT9yXTEGagbTXN4AUkRK;2L`0tm9Zn&gr`3O^7si`7c_V zSF~fa${4yMTsq@td|5EetVK43f^@g^-TLCY0*0y7JKDREE1!`~juVeGnl#~yVQ2oO zQY!ac9k1mGGd;^aG17DGPPSVcir@0nz42I+M%W(cDc(EeiS&^#%lVZ)cFV)_XULY| z3Uvb<$cKx6p0DrMS1&&8ivRq;Jtrnq}yu+=uix^Y1_f{iShP+i$ioon*9MC-dgGn{zPbosGY*q)RzWoe*JZy^pZ21i=+5AM zYl&W;)!T0Cg;hg8OpR7 zaaT*#E7B`WF?fORT5>4=8c|17Vjakyu!|lc71Ri|x3-2BU@T@N9b}ogDJDIKtBvmz6JpZ0o6KbT}r|#FOYb1b@gH zuCDzr=US#+o9v8&{}?}3^}vk+A-z%flsd)n0C!h9nRDLYh^PN4VMZ&N0N9J3ibab4s{wJ)CnP!ypjMZjQr}c|gdOhJ z_^lajb?3jSy+f)Q4%Ul(q})~~7}1XI=59Q#NMF;4cPhmh89Ip6^Vba)(QX-Sa38%- z?nf*#m1X&|sr*X6jr`QST4`&j@152!`&xJy=TUY3ig}78;H}mW=TYlTZc{h~_Gj+W zPdZI}mEA!|^X8N0W?ps<_BqMfl=q&+!W(Bl+ClHC7V&G|ZTdE7u3TWUw?lDFs`x#Q**HC{?IHyy&ZL1)^ z&RWCm0B>A2!^2g+QjyU@m)HVK+7s4XN zR=JYI#;}mR zc~-~(L`t{#khtBwCGvlAU3kC_;1sqYw;&NkArlnloj*8* zrdi_yt!YiI09h(O1r*2>-d(0_=kYpEImwmUD>_Cp;7kKMYlC;u^{574%aKAs zr3qISJpenUV8)!7d)gz1NjAp!@%&C!tbS-Tlk%F`#F5k+M+HJ?nsb}`tMZWVm3v;8 zr0&tyfiFS?sU-d99WVDYd&HlU&Y?yL@qBh{>G0!lG2}pR{mV!<{cA!mS3dKl&1N1K zCxk}lj1*P}9>`z0$zlPx&0PYX3bynVbTt&Gqis?{__zF%-ypOJ-{7Zu6#G+LH1`epegG)4^57hA+!ZfaBfgkmPlu8NZw_Ecjj3 zxbf0^=|`v?T_cS_z0v0I4|rUyllGS-dLAZig`+&nxi{HS!P>!!-eG}Q{*KUrKZ;u@ zd&oPzzqJTEahh>mnrbf)Y{9+5Q>>oGa9;zyONj!RA+D!Bk@%EFxG9;gRu0!t@&tD! z6*mO&6Zim5k-lmy{Y9$dU)DZ(w{c0HMvpi|Ujlf<2r&)Rj4K6Ccm|3K<<+92&NbIdzN6wQYh(Is@2zK%fFqpRvm zJScQopO`U-X&Qf}-7}}8RSgf*d(hA1leLzqQrqzu^QuPidAcQB+RBg9=m>J3=2gd& zx#^8E#?$%AI>w22re!j_hx`F(J~A8P%Ji|?TknmJtYUK;MzaWDzZckEthVYvne*qTj=9h(DM~;_wDq zK#ejd&tzhJhV?o5PjiF5y+pYXrZ#fiszPQ=!t3p3rQ{6!T zlO_r$#0~MIVAuhqkX+u=nQ!B*E#5Yr${>^v{mDn6T)Bq~gaqv;U&qXr)N||LHlv83p_usaANnU#0*hdFMR*WtpDBN{5td%nB(=ve(8Se!(wi