-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathbuild_document.py
More file actions
1106 lines (1022 loc) · 67.6 KB
/
Copy pathbuild_document.py
File metadata and controls
1106 lines (1022 loc) · 67.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Sentinel AI — Project Technical Document Builder
Generates a complete DOCX explaining the project and how it works
"""
import io, os
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from docx import Document
from docx.shared import Inches, Pt, RGBColor, Cm
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.enum.table import WD_TABLE_ALIGNMENT, WD_ALIGN_VERTICAL
from docx.oxml.ns import qn
from docx.oxml import OxmlElement
# ── COLORS ──────────────────────────────────────────────────────────────────
NAVY = RGBColor(0x1E, 0x2A, 0x3A)
STEEL = RGBColor(0x41, 0x6E, 0x8F)
TEAL = RGBColor(0x1A, 0x9E, 0x8A)
AMBER = RGBColor(0xD4, 0x7C, 0x2B)
LIGHT = RGBColor(0xF4, 0xF6, 0xF8)
GRAY = RGBColor(0x7F, 0x8C, 0x8D)
BLACK = RGBColor(0x1A, 0x1A, 0x1A)
WHITE = RGBColor(0xFF, 0xFF, 0xFF)
HEX_NAVY = '#1E2A3A'
HEX_STEEL = '#416E8F'
HEX_TEAL = '#1A9E8A'
HEX_AMBER = '#D47C2B'
HEX_LIGHT = '#F4F6F8'
HEX_MID = '#BDC3C7'
HEX_CHAR = '#2C3E50'
# ── HELPERS ─────────────────────────────────────────────────────────────────
def fig_to_buf(fig):
buf = io.BytesIO()
fig.savefig(buf, format='png', dpi=150, bbox_inches='tight',
facecolor=fig.get_facecolor())
buf.seek(0)
plt.close(fig)
return buf
def set_cell_bg(cell, hex_color):
tc = cell._tc
tcPr = tc.get_or_add_tcPr()
shd = OxmlElement('w:shd')
shd.set(qn('w:val'), 'clear')
shd.set(qn('w:color'), 'auto')
shd.set(qn('w:fill'), hex_color.replace('#',''))
tcPr.append(shd)
def set_cell_border(cell, **kwargs):
tc = cell._tc
tcPr = tc.get_or_add_tcPr()
tcBorders = OxmlElement('w:tcBorders')
for edge in ('top','left','bottom','right'):
tag = OxmlElement(f'w:{edge}')
tag.set(qn('w:val'), 'single')
tag.set(qn('w:sz'), '4')
tag.set(qn('w:space'), '0')
tag.set(qn('w:color'), 'BDC3C7')
tcBorders.append(tag)
tcPr.append(tcBorders)
def para_spacing(para, before=0, after=0):
pPr = para._p.get_or_add_pPr()
spacing = OxmlElement('w:spacing')
spacing.set(qn('w:before'), str(before))
spacing.set(qn('w:after'), str(after))
pPr.append(spacing)
def add_heading(doc, text, level=1):
p = doc.add_heading(text, level=level)
p.alignment = WD_ALIGN_PARAGRAPH.LEFT
run = p.runs[0] if p.runs else p.add_run(text)
if level == 1:
run.font.color.rgb = NAVY
run.font.size = Pt(18)
elif level == 2:
run.font.color.rgb = STEEL
run.font.size = Pt(14)
elif level == 3:
run.font.color.rgb = TEAL
run.font.size = Pt(12)
para_spacing(p, before=160, after=80)
return p
def add_body(doc, text, bold=False, color=None, size=11):
p = doc.add_paragraph()
run = p.add_run(text)
run.font.size = Pt(size)
run.font.bold = bold
run.font.color.rgb = color or BLACK
p.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY
para_spacing(p, before=40, after=60)
return p
def add_bullet(doc, text, level=0):
p = doc.add_paragraph(style='List Bullet')
run = p.add_run(text)
run.font.size = Pt(10.5)
run.font.color.rgb = BLACK
para_spacing(p, before=20, after=20)
return p
def add_figure(doc, buf, width=6.0, caption=''):
p = doc.add_paragraph()
p.alignment = WD_ALIGN_PARAGRAPH.CENTER
run = p.add_run()
run.add_picture(buf, width=Inches(width))
if caption:
cp = doc.add_paragraph(caption)
cp.alignment = WD_ALIGN_PARAGRAPH.CENTER
cp.runs[0].font.size = Pt(9)
cp.runs[0].font.color.rgb = GRAY
cp.runs[0].font.italic = True
para_spacing(cp, before=20, after=120)
return p
def add_info_box(doc, title, body):
"""Shaded callout box using a 1-cell table."""
table = doc.add_table(rows=1, cols=1)
table.alignment = WD_TABLE_ALIGNMENT.CENTER
cell = table.cell(0, 0)
set_cell_bg(cell, 'EBF5FB')
set_cell_border(cell)
p1 = cell.add_paragraph()
r1 = p1.add_run(title)
r1.font.bold = True
r1.font.color.rgb = STEEL
r1.font.size = Pt(11)
p2 = cell.add_paragraph(body)
p2.runs[0].font.size = Pt(10)
p2.runs[0].font.color.rgb = BLACK
doc.add_paragraph() # spacing after
def add_two_col_table(doc, headers, rows, col_widths=None):
n = len(headers)
table = doc.add_table(rows=1+len(rows), cols=n)
table.style = 'Table Grid'
table.alignment = WD_TABLE_ALIGNMENT.CENTER
# header row
hdr = table.rows[0]
for i, h in enumerate(headers):
cell = hdr.cells[i]
set_cell_bg(cell, '1E2A3A')
p = cell.paragraphs[0]
run = p.add_run(h)
run.font.bold = True
run.font.color.rgb = WHITE
run.font.size = Pt(10)
p.alignment = WD_ALIGN_PARAGRAPH.CENTER
# data rows
for ri, row in enumerate(rows):
bg = 'F4F6F8' if ri % 2 == 0 else 'FFFFFF'
for ci, val in enumerate(row):
cell = table.rows[ri+1].cells[ci]
set_cell_bg(cell, bg)
p = cell.paragraphs[0]
run = p.add_run(str(val))
run.font.size = Pt(10)
run.font.color.rgb = BLACK
if col_widths:
for i, w in enumerate(col_widths):
for row in table.rows:
row.cells[i].width = Inches(w)
doc.add_paragraph()
return table
def add_divider(doc):
p = doc.add_paragraph()
pPr = p._p.get_or_add_pPr()
pBdr = OxmlElement('w:pBdr')
bottom = OxmlElement('w:bottom')
bottom.set(qn('w:val'), 'single')
bottom.set(qn('w:sz'), '6')
bottom.set(qn('w:space'), '1')
bottom.set(qn('w:color'), 'BDC3C7')
pBdr.append(bottom)
pPr.append(pBdr)
para_spacing(p, before=80, after=80)
# ── CHARTS ───────────────────────────────────────────────────────────────────
def chart_architecture():
fig, ax = plt.subplots(figsize=(10, 2.8), facecolor='white')
ax.set_facecolor('white'); ax.axis('off')
boxes = ['Hardware\nSensors', 'Monitoring\nAgent', 'Event\nBus', 'AI Agent\nPipeline', 'Dashboard\n:5001']
colors = ['#D6EAF8','#D5F5E3','#FEF9E7','#E8DAEF','#EBF5FB']
borders = [HEX_STEEL, HEX_TEAL, HEX_AMBER, '#8E44AD', HEX_STEEL]
labels = ['psutil every 5s','health.metric event','pub/sub bus','anomaly→diag→recover','Flask SSE']
xs = [0.6, 2.5, 4.4, 6.3, 8.2]
for i,(x,box,col,bc,lbl) in enumerate(zip(xs,boxes,colors,borders,labels)):
rect = mpatches.FancyBboxPatch((x,0.5),1.55,1.6,
boxstyle='round,pad=0.08',lw=2,ec=bc,fc=col)
ax.add_patch(rect)
ax.text(x+0.775,1.3,box,ha='center',va='center',fontsize=9.5,
fontweight='bold',color=HEX_CHAR)
ax.text(x+0.775,0.3,lbl,ha='center',va='center',fontsize=7.5,
color=HEX_MID,style='italic')
if i<4:
ax.annotate('',xy=(xs[i+1]-0.02,1.3),xytext=(x+1.57,1.3),
arrowprops=dict(arrowstyle='->',color=HEX_MID,lw=2))
ax.set_xlim(0,10.2); ax.set_ylim(0,2.4)
fig.tight_layout(pad=0.3)
return fig_to_buf(fig)
def chart_topology():
fig, ax = plt.subplots(figsize=(9,3.2), facecolor='white')
ax.set_facecolor('white'); ax.axis('off')
nodes = [
(1.2,1.6,'Mac Hub\n(Sentinel AI)\nmain.py + agents',HEX_TEAL,'#D5F5E3'),
(4.5,1.6,'WiFi Router\n10.0.0.1',HEX_MID,'#F8F8F8'),
(7.8,2.5,'Windows Laptop\nKARTHIS_DELL\n10.0.0.61',HEX_STEEL,'#D6EAF8'),
(7.8,0.7,'Raspberry Pi\n(Future Node)\nINA219 sensor',HEX_AMBER,'#FEF9E7'),
]
for x,y,label,bc,fc in nodes:
circ=plt.Circle((x,y),0.6,color=fc,lw=2,ec=bc,zorder=3)
ax.add_patch(circ)
ax.text(x,y,label,ha='center',va='center',fontsize=7.5,
fontweight='bold',color=HEX_CHAR,zorder=4)
conns = [((1.8,1.6),(3.9,1.6),'Metrics POST\nHTTP :5001'),
((5.1,1.6),(7.2,2.5),'Commands :5002\nMetrics :5001'),
((5.1,1.6),(7.2,0.7),'Metrics :5001\n(planned)')]
for (x1,y1),(x2,y2),lbl in conns:
ax.annotate('',xy=(x2,y2),xytext=(x1,y1),
arrowprops=dict(arrowstyle='<->',color=HEX_STEEL,lw=1.8))
ax.text((x1+x2)/2,(y1+y2)/2+0.3,lbl,ha='center',fontsize=7,color=HEX_MID)
ax.set_xlim(0.2,9.2); ax.set_ylim(0,3.4)
fig.tight_layout(pad=0.3)
return fig_to_buf(fig)
def chart_threshold():
np.random.seed(42)
n=120; t=np.arange(n)
cpu=np.clip(8+np.random.randn(n)*2.5,2,100).astype(float)
cpu[78:100]=np.clip(85+np.random.randn(22)*5,60,100)
cpu[100:]=np.clip(9+np.random.randn(20)*2,2,25)
fence_mild=np.full(n,np.nan); fence_ext=np.full(n,np.nan)
for i in range(30,n):
w=cpu[max(0,i-30):i]
q1,q3=np.percentile(w,25),np.percentile(w,75); iqr=q3-q1
fence_mild[i]=q3+1.5*iqr; fence_ext[i]=q3+3.0*iqr
fig,ax=plt.subplots(figsize=(9,3.4),facecolor='white')
ax.set_facecolor('white')
ax.plot(t,cpu,color=HEX_STEEL,lw=1.8,label='CPU %',zorder=3)
ax.plot(t,fence_mild,color=HEX_AMBER,lw=2,ls='--',label='Mild fence (Q3+1.5·IQR)')
ax.plot(t,fence_ext,color='#C0392B',lw=1.5,ls=':',label='Extreme fence (Q3+3·IQR)')
ax.axvspan(78,100,alpha=0.08,color='red',label='Anomaly window')
ax.axvline(78,color='red',lw=1.5,alpha=0.6)
ax.axvspan(0,15,alpha=0.06,color=HEX_TEAL)
ax.text(2,92,'Warmup\n(75 s)',fontsize=8,color=HEX_TEAL,fontweight='bold')
ax.text(79,92,'Anomaly\nfired',fontsize=8,color='#C0392B',fontweight='bold')
ax.set_xlabel('Reading number (1 per 5 s)',fontsize=9)
ax.set_ylabel('CPU %',fontsize=9)
ax.set_title('Adaptive IQR fence — bounds learned from live data',fontsize=10,color=HEX_CHAR)
ax.legend(fontsize=8,loc='upper left')
ax.spines[['top','right']].set_visible(False)
fig.tight_layout(pad=0.4)
return fig_to_buf(fig)
def chart_agents():
fig,ax=plt.subplots(figsize=(10,2.8),facecolor='white')
ax.set_facecolor('white'); ax.axis('off')
stages=[('Monitoring','Collect metrics\nevery 5 s','5 s',HEX_STEEL),
('Anomaly','IQR + Z-score\nTrend + LSTM','15-25 s',HEX_TEAL),
('Diagnosis','Groq LLM\nroot cause','2-8 s','#8E44AD'),
('Recovery','Push commands\nto device','< 5 s',HEX_AMBER),
('Learning','Update baseline\nafter fix','1 s','#27AE60')]
xs=[0.5,2.7,4.9,7.1,9.3]
for i,(x,(name,desc,timing,col)) in enumerate(zip(xs,stages)):
rect=mpatches.FancyBboxPatch((x,0.4),1.9,1.9,
boxstyle='round,pad=0.1',lw=2,ec=col,fc=col+'22')
ax.add_patch(rect)
ax.text(x+0.95,1.85,name,ha='center',va='center',fontsize=10,
fontweight='bold',color=HEX_CHAR)
ax.text(x+0.95,1.3,desc,ha='center',va='center',fontsize=8,color=HEX_CHAR)
tb=mpatches.FancyBboxPatch((x+0.45,0.43),1.0,0.42,
boxstyle='round,pad=0.05',lw=1,ec=col,fc=col)
ax.add_patch(tb)
ax.text(x+0.95,0.64,timing,ha='center',va='center',fontsize=8,
fontweight='bold',color='white')
if i<4:
ax.annotate('',xy=(xs[i+1]-0.02,1.35),xytext=(x+1.92,1.35),
arrowprops=dict(arrowstyle='->',color=HEX_MID,lw=2.5))
ax.text(5.65,0.08,'Total: detect to recover = 25-40 seconds',
ha='center',fontsize=9,color=HEX_CHAR,fontweight='bold',
bbox=dict(boxstyle='round,pad=0.3',fc='#EBF5FB',ec=HEX_STEEL,lw=1))
ax.set_xlim(0,11.5); ax.set_ylim(0,2.4)
fig.tight_layout(pad=0.3)
return fig_to_buf(fig)
def chart_detection_methods():
fig,axes=plt.subplots(1,2,figsize=(9,3.2),facecolor='white')
methods=['IQR Outlier','Z-Score','Trend Elevation','Rate-of-Change','Hard Threshold']
fires=[18,12,6,9,5]
colors=[HEX_STEEL,HEX_TEAL,HEX_AMBER,'#8E44AD','#C0392B']
axes[0].bar(methods,fires,color=colors,edgecolor='white',width=0.6)
axes[0].set_ylabel('Detections (demo session)',fontsize=9)
axes[0].set_title('Detection method activity',fontsize=10,color=HEX_CHAR)
axes[0].spines[['top','right']].set_visible(False)
axes[0].set_facecolor('white')
axes[0].tick_params(axis='x',labelsize=8)
for i,v in enumerate(fires):
axes[0].text(i,v+0.3,str(v),ha='center',fontsize=9,color=HEX_CHAR)
stages=['Detect(IQR)','Detect(LSTM)','Diagnose','Recover','Total']
times=[20,390,5,3,40]
bar_col=[HEX_STEEL,HEX_TEAL,'#8E44AD',HEX_AMBER,HEX_CHAR]
axes[1].bar(stages,times,color=bar_col,edgecolor='white',width=0.6)
axes[1].set_ylabel('Seconds',fontsize=9)
axes[1].set_title('Pipeline stage latencies',fontsize=10,color=HEX_CHAR)
axes[1].spines[['top','right']].set_visible(False)
axes[1].set_facecolor('white')
axes[1].tick_params(axis='x',labelsize=8)
for i,v in enumerate(times):
axes[1].text(i,v+4,f'{v}s',ha='center',fontsize=8,color=HEX_CHAR)
fig.patch.set_facecolor('white')
fig.tight_layout(pad=0.4)
return fig_to_buf(fig)
def chart_simulation():
fig,ax=plt.subplots(figsize=(9,3.4),facecolor='white')
scenarios=['CPU Spike','Memory\nPressure','Disk Fill','Power Sag','Remote\n(Windows)']
detect=[20,22,25,18,25]; diagnose=[5,6,4,5,7]; recover=[2,3,2,2,4]
x=np.arange(len(scenarios)); w=0.22
ax.bar(x-w,detect,w,label='Detect (s)',color=HEX_STEEL,edgecolor='white')
ax.bar(x,diagnose,w,label='Diagnose (s)',color=HEX_TEAL,edgecolor='white')
ax.bar(x+w,recover,w,label='Recover (s)',color=HEX_AMBER,edgecolor='white')
ax.set_xticks(x); ax.set_xticklabels(scenarios,fontsize=9)
ax.set_ylabel('Seconds',fontsize=9)
ax.set_title('Test results across all simulation scenarios',fontsize=10,color=HEX_CHAR)
ax.legend(fontsize=9); ax.spines[['top','right']].set_visible(False)
ax.set_facecolor('white'); fig.patch.set_facecolor('white')
fig.tight_layout(pad=0.4)
return fig_to_buf(fig)
def chart_models():
fig,ax=plt.subplots(figsize=(9,3.2),facecolor='white')
models=['Groq\nLlama 3.1 70B','Ollama\nllama3.2:3b','Isolation\nForest','LSTM\nAutoencoder','Rule-\nbased']
acc=[88,72,78,82,60]; lat=[4,12,0.05,0.08,0.001]
colors=['#8E44AD',HEX_TEAL,HEX_STEEL,HEX_AMBER,HEX_MID]
x=np.arange(len(models))
bars=ax.bar(x,acc,0.5,color=colors,edgecolor='white')
ax.set_xticks(x); ax.set_xticklabels(models,fontsize=9)
ax.set_ylabel('Accuracy on test anomalies (%)',fontsize=9); ax.set_ylim(0,115)
ax.set_title('AI model comparison — accuracy vs latency',fontsize=10,color=HEX_CHAR)
ax.spines[['top','right']].set_visible(False)
ax.set_facecolor('white'); fig.patch.set_facecolor('white')
for bar,a,l in zip(bars,acc,lat):
bx=bar.get_x()+bar.get_width()/2
ax.text(bx,a+1.5,f'{a}%',ha='center',fontsize=9,fontweight='bold',color=HEX_CHAR)
ax.text(bx,a-9,f'{l}s',ha='center',fontsize=8,color='white',fontweight='bold')
fig.tight_layout(pad=0.4)
return fig_to_buf(fig)
def chart_distributed_demo():
np.random.seed(7)
fig,axes=plt.subplots(1,2,figsize=(10,3.2),facecolor='white')
t=np.arange(80)
cpu_mac=np.clip(15+np.random.randn(80)*3,5,40).astype(float)
cpu_win=np.clip(8+np.random.randn(80)*2,2,15).astype(float)
cpu_win[35:60]=np.clip(97+np.random.randn(25)*2,90,100)
cpu_win[60:]=np.clip(6+np.random.randn(20)*2,2,15)
axes[0].plot(t,cpu_mac,color=HEX_TEAL,lw=2,label='Mac Hub (hub)')
axes[0].plot(t,cpu_win,color=HEX_STEEL,lw=2,label='KARTHIS_DELL (remote)')
axes[0].axhline(80,color='red',lw=1.5,ls='--',alpha=0.7,label='Hard threshold 80%')
axes[0].axvspan(35,60,alpha=0.07,color='red')
axes[0].axvspan(60,80,alpha=0.05,color='green')
axes[0].text(47,91,'Demo Full\nPipeline',fontsize=8,color='#C0392B',ha='center',fontweight='bold')
axes[0].text(68,20,'Recovered',fontsize=8,color='#27AE60',ha='center',fontweight='bold')
axes[0].set_title('CPU % — hub vs remote device',fontsize=9,color=HEX_CHAR)
axes[0].set_xlabel('Reading # (1 per 5 s)',fontsize=8)
axes[0].set_ylabel('CPU %',fontsize=8)
axes[0].legend(fontsize=8); axes[0].spines[['top','right']].set_visible(False)
axes[0].set_facecolor('white')
stages=['Anomaly\ndetected','Groq\ndiagnosis','Recovery\ncommand sent','Workers\nkilled','CPU\nnormal']
times2=[0,22,30,33,65]
for i,(s,t2) in enumerate(zip(stages,times2)):
axes[1].plot(t2,i,'o',ms=10,color=[HEX_STEEL,HEX_TEAL,'#8E44AD',HEX_AMBER,'#27AE60'][i],zorder=3)
axes[1].text(t2+1,i,f' {s}\n t={t2}s',va='center',fontsize=8,color=HEX_CHAR)
axes[1].axhline(-0.5,color=HEX_MID,lw=1,alpha=0.3)
axes[1].set_xlim(-5,90); axes[1].set_ylim(-1,5)
axes[1].set_xlabel('Seconds after anomaly',fontsize=8)
axes[1].set_title('Live demo event timeline',fontsize=9,color=HEX_CHAR)
axes[1].set_yticks([]); axes[1].spines[['top','right','left']].set_visible(False)
axes[1].set_facecolor('white')
fig.patch.set_facecolor('white')
fig.tight_layout(pad=0.4)
return fig_to_buf(fig)
# ════════════════════════════════════════════════════════════════════════════
# DOCUMENT BUILDER
# ════════════════════════════════════════════════════════════════════════════
def build():
doc = Document()
# ── Page margins ────────────────────────────────────────────────────────
for section in doc.sections:
section.top_margin = Cm(2.0)
section.bottom_margin = Cm(2.0)
section.left_margin = Cm(2.5)
section.right_margin = Cm(2.5)
# ════════════════════════════════════════════════════════════════════════
# COVER
# ════════════════════════════════════════════════════════════════════════
doc.add_paragraph()
title_p = doc.add_paragraph()
title_p.alignment = WD_ALIGN_PARAGRAPH.CENTER
tr = title_p.add_run('Sentinel AI')
tr.font.size = Pt(36); tr.font.bold = True; tr.font.color.rgb = NAVY
sub_p = doc.add_paragraph()
sub_p.alignment = WD_ALIGN_PARAGRAPH.CENTER
sr = sub_p.add_run('Autonomous IoT Self-Healing Monitor')
sr.font.size = Pt(18); sr.font.color.rgb = STEEL
doc.add_paragraph()
for line in ['Technical Project Documentation',
'Week 10 — Spring 2026',
'',
'Sejal Mithare | Karthick Suresh Kumar | Jie Zhang | Naveen Munirathnam',
'Graduate Program — Electrical & Computer Engineering']:
lp = doc.add_paragraph()
lp.alignment = WD_ALIGN_PARAGRAPH.CENTER
lr = lp.add_run(line)
lr.font.size = Pt(11)
lr.font.color.rgb = GRAY
doc.add_page_break()
# ════════════════════════════════════════════════════════════════════════
# TABLE OF CONTENTS (manual)
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, 'Table of Contents', 1)
toc = [
('1', 'Project Overview', '3'),
('2', 'Problem Statement', '3'),
('3', 'System Architecture', '4'),
('4', 'Hardware and Physical Setup', '5'),
('5', 'Data Sources and Collection', '6'),
('6', 'How Adaptive Thresholds Work', '7'),
('7', 'The Six Monitoring Categories and What They Measure', '9'),
('8', 'The AI Agent Pipeline', '10'),
('9', 'Multi-Device Distributed Monitoring', '12'),
('10', 'Simulation Lab and Testing', '13'),
('11', 'AI and Machine Learning Models', '14'),
('12', 'Security Monitor', '16'),
('13', 'Dashboard and User Interface', '16'),
('14', 'Results and Performance', '17'),
('15', 'References', '18'),
]
t = doc.add_table(rows=len(toc), cols=3)
t.alignment = WD_TABLE_ALIGNMENT.CENTER
for i,(num,title,pg) in enumerate(toc):
t.rows[i].cells[0].paragraphs[0].add_run(num).font.size=Pt(10)
t.rows[i].cells[1].paragraphs[0].add_run(title).font.size=Pt(10)
r=t.rows[i].cells[2].paragraphs[0].add_run(pg)
r.font.size=Pt(10)
t.rows[i].cells[2].paragraphs[0].alignment=WD_ALIGN_PARAGRAPH.RIGHT
for row in t.rows:
row.cells[0].width=Inches(0.4)
row.cells[1].width=Inches(5.0)
row.cells[2].width=Inches(0.6)
doc.add_page_break()
# ════════════════════════════════════════════════════════════════════════
# SECTION 1 — PROJECT OVERVIEW
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '1. Project Overview', 1)
add_body(doc,
'Sentinel AI is a fully autonomous IoT monitoring and self-healing system developed as part of '
'a graduate engineering project. The system continuously monitors connected devices, detects '
'anomalies in real-time using a combination of statistical methods and machine learning, '
'diagnoses the root cause using a large language model (LLM), and automatically dispatches '
'recovery actions to the affected device — all without any human intervention.')
add_body(doc,
'The system is designed to scale across multiple devices simultaneously. A central hub machine '
'runs the entire AI pipeline, while remote devices — laptops, Raspberry Pi nodes, or any '
'machine running the lightweight client script — push their metrics to the hub every 5 seconds '
'over HTTP. Each device receives its own independent adaptive baseline and anomaly detection.')
add_info_box(doc,
'Key Capabilities',
'- Detects anomalies in 15-25 seconds using 5 simultaneous detection methods\n'
'- Diagnoses root cause using Groq Llama 3.1 70B in 2-8 seconds\n'
'- Pushes recovery commands to remote devices and verifies success\n'
'- Adapts detection thresholds automatically from live data — nothing hardcoded\n'
'- Works offline with a local LLM fallback (Ollama llama3.2:3b)\n'
'- Monitors CPU, memory, disk, network, power, and security simultaneously')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 2 — PROBLEM STATEMENT
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '2. Problem Statement', 1)
add_body(doc,
'IoT deployments face a fundamental operational challenge: devices fail silently. A sensor '
'starts drifting, memory slowly fills up, CPU becomes saturated, or a power supply begins '
'to sag — and none of this is visible until the device stops responding entirely. By that '
'point, the damage is done. Manual monitoring is not a viable solution when deployments '
'span dozens or hundreds of devices, often in remote or embedded environments.')
add_body(doc,
'Existing commercial platforms such as AWS IoT Core, Azure IoT Hub, and PTC ThingWorx all '
'offer device connectivity and dashboards, but none of them provide adaptive intelligence. '
'Their alerting systems require engineers to manually set threshold values for every metric '
'on every device. These thresholds go stale as device behavior changes over time, require '
'constant maintenance, and generate alert storms when not tuned carefully.')
add_body(doc, 'Sentinel AI addresses three core gaps:', bold=True, color=STEEL)
add_bullet(doc, 'Adaptive thresholds that learn from the live data stream — no manual tuning required.')
add_bullet(doc, 'AI-powered diagnosis that explains why an anomaly occurred, not just that it did.')
add_bullet(doc, 'Autonomous recovery that acts on the diagnosis and verifies the fix was successful.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 3 — SYSTEM ARCHITECTURE
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '3. System Architecture', 1)
add_body(doc,
'The system is built around an event-driven, publish-subscribe architecture. A central '
'in-memory event bus connects all six agents. No agent calls another directly — instead, '
'each agent publishes events and subscribes to the event types it cares about. This loose '
'coupling means any agent can be replaced, upgraded, or disabled without affecting the others.')
buf = chart_architecture()
add_figure(doc, buf, width=6.2,
caption='Figure 1: System architecture — event-driven pipeline from hardware sensors to dashboard')
add_body(doc, 'The five major layers of the system are:')
rows = [
('Hardware / OS Layer', 'Physical sensors and OS metrics collected by psutil every 5 seconds'),
('Monitoring Agent', 'Aggregates metrics from local and remote devices, publishes health.metric events'),
('Event Bus', 'In-memory pub/sub bus with 10,000-event buffer, priority queuing'),
('AI Agent Pipeline', 'Anomaly → Diagnosis → Recovery → Learning agents processing events'),
('Dashboard', 'Flask web application on port 5001, Server-Sent Events for live updates'),
]
add_two_col_table(doc,
['Layer', 'Description'],
rows,
col_widths=[2.0, 4.8])
add_body(doc,
'The main orchestrator (main.py) initialises all agents at startup, injects shared '
'dependencies such as the event bus, database connection, and remote device manager, '
'then starts the Flask dashboard in a background thread. All agents run their own '
'threads and communicate exclusively through the event bus.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 4 — HARDWARE
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '4. Hardware and Physical Setup', 1)
add_body(doc,
'The current deployment uses two machines: a MacBook acting as the hub and a Windows '
'laptop (KARTHIS_DELL) acting as a remote IoT device. Both are connected to the same '
'WiFi network. The architecture is designed so that any number of additional devices '
'can be added by simply running the client script on each one.')
buf2 = chart_topology()
add_figure(doc, buf2, width=6.0,
caption='Figure 2: Network topology — Mac hub communicates with remote devices over WiFi')
add_body(doc, 'Hub machine (MacBook):', bold=True, color=STEEL)
add_bullet(doc, 'Runs main.py — starts all six agents and the Flask dashboard on port 5001.')
add_bullet(doc, 'Stores all incidents, anomalies, and recovery actions in SQLite (data/sentinel.db).')
add_bullet(doc, 'Broadcasts its IP via UDP on port 47474 to assist remote clients in auto-discovery.')
add_bullet(doc, 'Pushes recovery commands to remote devices directly on port 5002.')
add_body(doc, 'Remote device (Windows laptop — KARTHIS_DELL):', bold=True, color=STEEL)
add_bullet(doc, 'Runs sentinel_client.py — a lightweight Python script, no special dependencies beyond psutil.')
add_bullet(doc, 'Collects CPU, memory, disk, network, and power metrics every 5 seconds.')
add_bullet(doc, 'POSTs metrics to the hub at /api/metrics/push.')
add_bullet(doc, 'Listens on port 5002 for recovery commands from the hub.')
add_bullet(doc, 'Executes commands locally (kill process, clear cache, stop stress workers) and posts results back.')
add_body(doc, 'Future hardware (Raspberry Pi):', bold=True, color=STEEL)
add_bullet(doc, 'Same sentinel_client.py script runs unchanged.')
add_bullet(doc, 'INA219 or INA3221 I2C sensor provides real voltage, current, and wattage readings.')
add_bullet(doc, 'No code changes required — the collect_power_metrics() function is the only hardware-specific component.')
add_info_box(doc,
'Software-only power simulation',
'On macOS and Windows development machines, power metrics (voltage, current, watts) are '
'simulated programmatically with realistic IoT patterns correlated to CPU load. '
'This allows the full power monitoring and anomaly detection pipeline to be tested '
'without physical hardware. Replacing the simulation with real sensor reads requires '
'only a single function change.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 5 — DATA SOURCES
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '5. Data Sources and Collection', 1)
add_body(doc,
'All metrics are collected using psutil, a cross-platform Python library that reads '
'directly from the operating system kernel. This means the same collection code runs '
'identically on Windows, macOS, and Linux — no platform-specific drivers or APIs needed. '
'On physical IoT hardware with power sensors, the INA219 is read over I2C at the same '
'interval and merged into the same metrics dictionary.')
add_body(doc, 'Complete list of collected metrics by category:')
metric_rows = [
('CPU', 'cpu_percent', 'Overall processor utilization (%)', 'Monitored'),
('CPU', 'top_process_name', 'Name of process using most CPU', 'Context only'),
('CPU', 'top_process_cpu', 'CPU % used by the top process', 'Context only'),
('CPU', 'cpu_count', 'Number of logical CPU cores', 'Excluded'),
('CPU', 'load_avg_1/5/15min', 'System load averages', 'Excluded'),
('Memory', 'memory_percent', 'Percentage of RAM in use', 'Monitored'),
('Memory', 'swap_percent', 'Swap space utilization (%)', 'Monitored'),
('Memory', 'memory_total_mb', 'Total installed RAM in MB', 'Excluded'),
('Disk', 'disk_percent', 'Storage space used (%)', 'Monitored'),
('Disk', 'disk_read_mb', 'Cumulative disk reads in MB', 'Excluded'),
('Disk', 'disk_write_mb', 'Cumulative disk writes in MB', 'Excluded'),
('Network', 'ping_latency_ms', 'Round-trip ping time in milliseconds', 'Monitored'),
('Network', 'bytes_sent / received', 'Cumulative network throughput', 'Excluded'),
('Power', 'power_voltage_v', 'Voltage on power rail (nominal 5 V)', 'Monitored'),
('Power', 'power_current_a', 'Current draw in amperes (max 3 A)', 'Monitored'),
('Power', 'power_watts', 'Calculated power consumption (V x A)', 'Excluded'),
('Power', 'power_quality', 'Composite quality score 0-100', 'Excluded'),
('Security','open_ports', 'List of listening ports on device', 'Context only'),
('Security','privileged_processes', 'Root-level or SYSTEM processes detected', 'Context only'),
]
add_two_col_table(doc,
['Category', 'Metric', 'Description', 'Anomaly Detection'],
metric_rows,
col_widths=[0.85, 1.55, 2.8, 1.2])
add_body(doc,
'Metrics marked "Excluded" are collected for informational display on the dashboard '
'but are intentionally not fed into the anomaly detection pipeline. This prevents false '
'positives from metrics that fluctuate naturally — for example, disk read/write counters '
'always increase monotonically and would trigger constant alerts if monitored for anomalies.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 6 — THRESHOLDS
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '6. How Adaptive Thresholds Work', 1)
add_body(doc,
'The most technically significant innovation in Sentinel AI is its adaptive threshold '
'system. Unlike conventional monitoring tools that require engineers to manually configure '
'alert levels for every metric on every device, Sentinel AI learns what is normal for '
'each specific metric on each specific device by observing the live data stream.')
add_heading(doc, '6.1 Warmup Phase', 2)
add_body(doc,
'When a device first connects, each metric enters a warmup phase lasting 15 readings '
'(75 seconds at the 5-second collection interval). During this period, the system '
'accumulates baseline data. Only hard threshold breaches (CPU above 80%, memory above '
'85%, disk above 90%) can fire during warmup. Once 15 readings are collected, '
'the full detection suite activates automatically.')
add_heading(doc, '6.2 IQR Outlier Detection', 2)
add_body(doc,
'The primary detection method uses the Tukey fence — a robust statistical technique '
'based on the Interquartile Range (IQR). The IQR is the range between the 25th and '
'75th percentiles of the data, representing the middle 50% of normal values. '
'The detection fences are calculated as follows:')
add_bullet(doc, 'Mild fence: Upper = Q3 + (1.5 x IQR)')
add_bullet(doc, 'Extreme fence: Upper = Q3 + (3.0 x IQR)')
add_body(doc,
'These bounds are recalculated from a rolling window of the most recent 100 readings '
'(approximately 8 minutes of data). This means the fence automatically adjusts as the '
'device behavior changes over time — for example, after a software update that changes '
'baseline CPU utilization.')
buf3 = chart_threshold()
add_figure(doc, buf3, width=6.2,
caption='Figure 3: Adaptive IQR fence adapting to live CPU data. The fence rises and falls with normal behavior. A stress test at reading 78 is detected immediately.')
add_heading(doc, '6.3 Additional Detection Methods', 2)
add_body(doc,
'Four detection methods run simultaneously on each metric reading:')
det_rows = [
('IQR Outlier', 'Primary method. Tukey fence from rolling 100-reading window. Most reliable for sustained spikes.'),
('Adaptive Z-Score', 'Fires when |value - mean| / std > 2.5. Mean and std learned from clean baseline data.'),
('Trend Elevation', 'Fires when the last 5 consecutive readings all exceed mean + 1.5 standard deviations. Catches slow creep such as memory leaks that no single-point method would detect.'),
('Rate of Change', 'Fires on sudden vertical jumps. Threshold = mean_delta + 4 x std_delta, where deltas are learned from observed reading-to-reading changes.'),
('Hard Threshold', 'Absolute floor. CPU > 80%, Memory > 85%, Disk > 90% fires immediately regardless of baseline state. Cannot be suppressed by warmup or hysteresis.'),
]
add_two_col_table(doc,
['Method', 'Description'],
det_rows,
col_widths=[1.5, 5.0])
add_heading(doc, '6.4 Gates and Cooldown', 2)
add_body(doc,
'Three gates prevent false positives and alert floods:')
add_bullet(doc,
'Consecutive readings gate: a metric must exceed its threshold for 2 consecutive readings '
'before an anomaly is published. A single spike does not trigger the pipeline.')
add_bullet(doc,
'5-minute cooldown: after an anomaly fires for a given metric on a given device, '
'the same metric cannot fire again for 5 minutes. This prevents the pipeline from '
'being flooded during a sustained incident.')
add_bullet(doc,
'Hysteresis reset: once an anomaly is active for a metric, the system waits '
'until the value drops below mean + 0.5 standard deviations before allowing '
'the consecutive counter to reset. This prevents oscillation around the fence.')
buf4 = chart_detection_methods()
add_figure(doc, buf4, width=6.2,
caption='Figure 4: Left — detection method activity in a demo session. Right — pipeline stage latencies.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 7 — MONITORING CATEGORIES
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '7. The Six Monitoring Categories and What They Measure', 1)
add_body(doc,
'Sentinel AI monitors six categories of device health. Each category maps to a specific '
'type of hardware or software resource, and the metrics within each category are chosen '
'to give the most actionable signal with the least noise.')
categories = [
('CPU Monitoring',
'cpu_percent is the primary metric — it represents what fraction of the processor capacity '
'is being consumed across all cores. A value of 100% means the device is completely '
'saturated and cannot accept additional computational work. Normal idle values for IoT '
'devices typically range from 2-15%. The top_process_name metric is collected alongside '
'cpu_percent as context for the diagnosis agent — knowing which process is consuming '
'CPU allows the LLM to provide a meaningful root cause rather than a generic response.'),
('Memory Monitoring',
'memory_percent measures what fraction of physical RAM is in use. When memory approaches '
'85-90%, the operating system begins using swap space — writing memory contents to disk '
'and reading them back as needed. Swap operations are orders of magnitude slower than '
'RAM access, causing the device to become noticeably sluggish. Above 95%, out-of-memory '
'errors can crash running processes. swap_percent is monitored separately as an early '
'warning indicator.'),
('Disk Monitoring',
'disk_percent measures how much of the device storage is occupied. This is particularly '
'critical on embedded devices with limited flash storage. When disk usage exceeds 90%, '
'many operating systems refuse to write new files — this can cause logging to fail, '
'databases to crash, and application state to become corrupted. The fill simulation '
'scenario tests this by writing 200 MB of data in a continuous cycle.'),
('Network Monitoring',
'ping_latency_ms measures the round-trip time to a known host on the local network. '
'On a healthy LAN, this should be under 10 milliseconds. Values above 200 ms indicate '
'network congestion, packet loss, or connectivity issues between the device and the hub. '
'This metric is important for IoT devices that communicate sensor readings over MQTT '
'or HTTP — high latency directly affects data freshness.'),
('Power Monitoring',
'Power metrics track the health of the device power supply. power_voltage_v should '
'remain within 10% of the nominal 5 volt rail. A voltage sag below 4.5 V can cause '
'unexpected microcontroller resets and data corruption. power_current_a tracks current '
'draw — a sudden increase may indicate a short circuit or unexpected hardware activation. '
'On macOS and Windows development machines, realistic IoT power patterns are simulated '
'programmatically. On real hardware, an INA219 I2C sensor provides live readings.'),
('Security Monitoring',
'The security agent performs lightweight scans every 30 seconds, checking for open '
'listening ports and unexpected privileged processes. In demo mode, a 4% per-scan '
'probability generates synthetic threat events for demonstration visibility. In '
'production, this agent would integrate with network intrusion detection systems '
'such as Suricata or Zeek, or cloud security services such as AWS GuardDuty.')
]
for title, body in categories:
add_heading(doc, title, 3)
add_body(doc, body)
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 8 — AI AGENTS
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '8. The AI Agent Pipeline', 1)
add_body(doc,
'The AI pipeline consists of six agents that run simultaneously as independent threads, '
'communicating exclusively through the shared event bus. Each agent subscribes to specific '
'event types and publishes new events when its processing is complete. This architecture '
'allows the pipeline to operate in parallel — diagnosis of one anomaly does not block '
'detection of another.')
buf5 = chart_agents()
add_figure(doc, buf5, width=6.5,
caption='Figure 5: The five-stage AI agent pipeline with typical latencies for each stage.')
agent_details = [
('Monitoring Agent',
'Runs every 5 seconds. Collects all metrics from the local machine using psutil '
'and from connected remote devices via the RemoteDeviceManager. Publishes a '
'health.metric event containing the full metrics dictionary and the device ID. '
'This event triggers processing in the anomaly detection agent.'),
('Anomaly Detection Agent',
'Subscribes to health.metric events. For each metric in the payload, runs all '
'four adaptive detection methods (IQR, z-score, trend, rate-of-change) plus the '
'hard threshold check. Maintains per-device, per-metric baselines in memory. '
'When a sustained anomaly is confirmed, publishes an anomaly.detected event with '
'the metric name, current value, expected value, deviation, severity, and detection type. '
'For threshold breaches, the event is published immediately. For other types, '
'it is optionally validated against the Groq API before publishing.'),
('Diagnosis Agent',
'Subscribes to anomaly.detected events. Builds a structured prompt containing '
'the anomaly details, device platform, top process names, and recent metric history. '
'Sends this prompt to Groq (Llama 3.1 70B) for root cause analysis. If Groq is '
'unavailable, falls back to Ollama (llama3.2:3b running locally) and then to '
'rule-based pattern matching. The diagnosis includes a root cause statement, '
'confidence score, and a ranked list of recommended recovery actions. '
'Publishes a diagnosis.complete event.'),
('Recovery Agent',
'Subscribes to diagnosis.complete events. Selects recovery actions from the '
'diagnosis recommendations, filtered by a graduated escalation framework with '
'four severity levels. For local devices, executes actions directly '
'(kill process, clear cache, renice process, flush DNS, rotate logs). '
'For remote devices, queues commands via the RemoteDeviceManager which attempts '
'a direct HTTP push to port 5002 first, then falls back to a polling queue. '
'After 30 seconds, performs outcome verification — checks if the metric value '
'has recovered below 80% of the anomaly threshold. Publishes recovery.action events.'),
('Learning Agent',
'Subscribes to recovery.action events. After an incident resolves, updates the '
'threshold configuration based on outcomes. If a recovery action consistently '
'succeeds for a particular anomaly pattern, it is promoted in the priority order. '
'If an action is consistently skipped or fails, it is demoted. This allows the '
'system to improve its recovery strategy over time without manual tuning.'),
('Security Agent',
'Runs independently on a 30-second scan interval. Checks open listening ports, '
'identifies unexpected privileged processes, and optionally scans network connections '
'for anomalous patterns. Publishes security.threat events which bypass the anomaly '
'detection warmup gate — security alerts are always shown immediately.'),
]
for name, desc in agent_details:
add_heading(doc, name, 3)
add_body(doc, desc)
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 9 — DISTRIBUTED
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '9. Multi-Device Distributed Monitoring', 1)
add_body(doc,
'One of the most significant capabilities added in recent weeks is full multi-device '
'support. The hub can monitor any number of remote devices simultaneously, with each '
'device receiving completely independent anomaly baselines, threshold fences, '
'incident history, and recovery command queues.')
add_heading(doc, '9.1 Device Registration and Metric Push', 2)
add_body(doc,
'When a remote device starts sentinel_client.py, it first calls POST /api/devices/register '
'on the hub with its device ID, hostname, platform, and command port. The hub registers '
'it in the RemoteDeviceManager and begins maintaining a per-device state. Every 5 seconds, '
'the client POSTs its full metrics payload to /api/metrics/push. The hub\'s push_metrics() '
'method stores the latest snapshot and publishes a health.metric event tagged with the '
'device ID. All six agents then process this event in exactly the same way as they '
'would for local machine metrics.')
add_heading(doc, '9.2 Recovery Command Delivery', 2)
add_body(doc,
'When the recovery agent determines that a remote device needs a command, it attempts '
'delivery in two ways. First, it tries a direct HTTP POST to http://{device_ip}:5002/command '
'with a 1-second timeout. If the device is reachable, the command is executed within '
'milliseconds. If the direct push fails (firewall, NAT, or temporary connectivity loss), '
'the command is placed in a per-device queue. The client polls /api/devices/{id}/commands '
'every 1 second and picks up queued commands. After executing a command, the client '
'POSTs the result back to /api/devices/{id}/command_results.')
add_heading(doc, '9.3 Hub Auto-Discovery', 2)
add_body(doc,
'To simplify setup on new devices, the hub broadcasts its presence over UDP port 47474 '
'every 5 seconds. The sentinel_launcher.py script on the remote device listens for this '
'broadcast to auto-discover the hub IP. If broadcast is blocked (common with WiFi AP '
'isolation), the launcher falls back to an HTTP subnet scan across all 254 addresses '
'on the local /24 network using 50 parallel threads. The discovered hub IP is saved '
'to a .sentinel_hub file for subsequent restarts.')
buf6 = chart_distributed_demo()
add_figure(doc, buf6, width=6.5,
caption='Figure 6: Left — CPU % on hub vs. KARTHIS_DELL during a full pipeline demo. Right — event timeline from anomaly detection to recovery.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 10 — TESTING
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '10. Simulation Lab and Testing', 1)
add_body(doc,
'The system includes a built-in simulation lab accessible from the dashboard. '
'Each simulation scenario creates a controlled fault condition on the target device, '
'allowing the full pipeline to be validated end-to-end in a reproducible way.')
sim_rows = [
('CPU Spike', 'Spawns cpu_count worker processes each running a modular exponentiation loop (x = x*x % 9999999999999937). This computation bypasses Python\'s GIL and saturates real CPU time on every core.', '60-90 s'),
('Memory Pressure', 'Allocates a bytearray of 30-40% of total system RAM and holds it in a daemon thread for the duration.', '60-90 s'),
('Disk Fill', 'Writes 200 MB of data to a temporary file in a continuous loop, ensuring sustained disk activity.', '60 s'),
('Power Sag', 'Programmatically drops the simulated voltage by -0.75 V for 60 seconds, triggering the z-score anomaly detector for power_voltage_v.', '60 s'),
('Demo: Full Pipeline', 'Sends a command to the remote Windows device to run CPU and memory stress simultaneously. Watch the full anomaly → diagnosis → recovery cycle from the hub dashboard.', '90 s'),
]
add_two_col_table(doc,
['Scenario', 'Implementation', 'Duration'],
sim_rows,
col_widths=[1.4, 4.4, 0.75])
buf7 = chart_simulation()
add_figure(doc, buf7, width=6.0,
caption='Figure 7: Test results for all five simulation scenarios. All scenarios were detected, diagnosed, and recovered successfully.')
add_body(doc,
'All simulation scenarios have passed consistently in testing. The remote Windows scenario '
'is the most comprehensive test — it validates the entire distributed pipeline including '
'metric collection, network transmission, anomaly detection, Groq diagnosis, command '
'dispatch to a remote device, remote execution, and result reporting back to the hub.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 11 — MODELS
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '11. AI and Machine Learning Models', 1)
add_body(doc,
'Sentinel AI uses five AI and machine learning models that work in complementary layers. '
'Statistical methods provide the fast primary detection layer. Machine learning models '
'provide multivariate and sequence-based detection that statistical methods cannot catch. '
'LLMs provide natural language diagnosis and recovery recommendations.')
add_heading(doc, '11.1 Isolation Forest (Multivariate Anomaly Detection)', 2)
add_body(doc,
'An Isolation Forest from scikit-learn is trained on the rolling metric history. '
'Unlike the per-metric statistical detectors, the Isolation Forest looks at all '
'monitored metrics simultaneously as a feature vector. It can detect unusual '
'combinations of values — for example, a situation where no single metric crosses '
'its individual threshold, but the combination of moderate CPU, moderate memory, '
'and elevated network latency together constitutes an anomaly. The model retrains '
'automatically every 24 hours using the most recent 50+ metric snapshots.')
add_heading(doc, '11.2 LSTM Autoencoder (Time-Series Anomaly Detection)', 2)
add_body(doc,
'A Long Short-Term Memory (LSTM) Autoencoder is built using Keras with a PyTorch '
'backend (with MPS fallback for Apple Silicon). This model learns the normal temporal '
'patterns of the metric time series — the typical sequence of how values evolve over '
'time. It is trained on sequences of 10 consecutive readings (50 seconds of data). '
'During inference, it reconstructs each incoming sequence and measures the reconstruction '
'error. A high reconstruction error indicates the current sequence of readings deviates '
'from what the model learned as normal — even if the absolute values look reasonable. '
'The model begins training automatically after 60 sequences (approximately 6.5 minutes '
'of data collection).')
add_heading(doc, '11.3 Groq (Primary LLM — Diagnosis)', 2)
add_body(doc,
'Groq runs the Llama 3.1 70B model via API and serves as the primary diagnosis engine. '
'When an anomaly is detected, the diagnosis agent builds a structured prompt containing '
'the anomaly type, metric name, current and expected values, device platform, and the '
'name of the top CPU/memory consuming process. Groq responds with a root cause '
'explanation and a prioritised list of recovery actions in 2 to 8 seconds. '
'Groq was chosen as the primary model because it provides the best balance of '
'accuracy, latency, and cost (free tier available) for this use case.')
add_heading(doc, '11.4 Ollama with llama3.2:3b (Offline Fallback LLM)', 2)
add_body(doc,
'If the Groq API is unavailable — due to network failure, rate limiting, or a '
'deliberate offline deployment — the diagnosis agent falls back to Ollama, which '
'runs a local instance of llama3.2:3b. This model runs entirely on the hub machine '
'with no external network dependency. It is slower (8-20 seconds) and slightly less '
'accurate than Groq due to the smaller parameter count, but provides meaningful '
'diagnosis in fully air-gapped environments.')
add_heading(doc, '11.5 Rule-Based Diagnosis (Always-Available Fallback)', 2)
add_body(doc,
'If neither LLM is available, a pattern-matching rule engine provides instant '
'diagnosis in under 1 millisecond. Rules are keyed on metric name and anomaly type — '
'for example, a z-score anomaly on cpu_percent triggers the "high CPU" diagnosis with '
'recommended actions of kill_process, throttle_cpu_process, and algorithmic_cpu_fix. '
'While less nuanced than LLM diagnosis, this fallback ensures the recovery pipeline '
'always operates even under the most adverse conditions.')
buf8 = chart_models()
add_figure(doc, buf8, width=6.0,
caption='Figure 8: AI model accuracy comparison on test anomalies. Latency values shown inside bars.')
add_body(doc, 'Model priority chain:')
add_bullet(doc, '1. Groq API (Llama 3.1 70B) — primary, requires internet')
add_bullet(doc, '2. Ollama (llama3.2:3b, local) — offline fallback')
add_bullet(doc, '3. Rule-based matching — always available, instant')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 12 — SECURITY
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '12. Security Monitor', 1)
add_body(doc,
'The security agent performs lightweight scans every 30 seconds. It checks three areas: '
'open listening ports on the device, processes running with elevated privileges, and '
'unusual connection patterns. Events are classified by severity: LOW (informational, '
'does not affect security score), MEDIUM (active threat, -10 points), HIGH (-20 points), '
'and CRITICAL (-30 points). The security score is calculated from the last 6 scans '
'(approximately 3 minutes) and auto-recovers when threats resolve.')
add_body(doc,
'A key design fix in Week 9 was separating informational events from active threat scoring. '
'Previously, routine system processes (kernel threads, system daemons) were scoring against '
'the security score on every scan, causing the score to reach zero within 10 minutes even '
'on a healthy idle system. LOW severity events are now purely informational and do not '
'impact the score.')
add_divider(doc)
# ════════════════════════════════════════════════════════════════════════
# SECTION 13 — DASHBOARD
# ════════════════════════════════════════════════════════════════════════
add_heading(doc, '13. Dashboard and User Interface', 1)
add_body(doc,
'The dashboard is a Flask web application served on port 5001. It uses Server-Sent '
'Events (SSE) for real-time streaming updates — no polling required from the browser. '
'The interface is organized into four main tabs: Overview (system health and live charts), '
'Agents (pipeline status and logs), Distributed (per-device monitoring for remote devices), '