Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ RUN apk add --no-cache ca-certificates sqlite-libs && \
mkdir -p /data && chown oberwatch:oberwatch /data
COPY --from=builder /app/oberwatch /usr/local/bin/oberwatch

WORKDIR /data
EXPOSE 8080
VOLUME ["/data"]

Expand Down
3 changes: 3 additions & 0 deletions dashboard/svelte/src/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,12 @@ export interface BudgetUpdateRequest {

export interface GlobalBudget {
period: string;
period_starts_at: string;
period_resets_at: string;
limit_usd: number;
spent_usd: number;
remaining_usd: number;
percentage_used: number;
}

export interface BudgetsResponse {
Expand Down
29 changes: 27 additions & 2 deletions dashboard/svelte/src/routes/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@
AgentsResponse,
Alert,
AlertsResponse,
BudgetsResponse,
CostBreakdown,
CostsResponse,
GlobalBudget,
HealthResponse
} from '$lib/types';

Expand All @@ -34,6 +36,7 @@
let labels = $state<string[]>([]);
let values = $state<number[]>([]);
let recentAlerts = $state<Alert[]>([]);
let globalBudget = $state<GlobalBudget | null>(null);

const lineDatasets = $derived<ChartDataset<'line', number[]>[]>([
{
Expand Down Expand Up @@ -70,11 +73,12 @@

try {
const from = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
const [costs, agentsRes, health, alertsRes] = await Promise.all([
const [costs, agentsRes, health, alertsRes, budgetsRes] = await Promise.all([
fetchJSON<CostsResponse>(`/costs?group_by=hour&from=${encodeURIComponent(from)}`),
fetchJSON<AgentsResponse>('/agents'),
fetchJSON<HealthResponse>('/health'),
fetchJSON<AlertsResponse>(`/alerts?from=${encodeURIComponent(from)}`)
fetchJSON<AlertsResponse>(`/alerts?from=${encodeURIComponent(from)}`),
fetchJSON<BudgetsResponse>('/budgets')
]);

const hourly = costs.breakdown as HourlyCostBreakdown[];
Expand All @@ -87,6 +91,7 @@
uptimeSeconds = health.uptime_seconds;
emergencyStopActive = health.emergency_stop ?? false;
recentAlerts = alertsRes.alerts.slice(0, 5);
globalBudget = budgetsRes.global.limit_usd > 0 ? budgetsRes.global : null;
} catch (err) {
errorMessage = err instanceof Error ? err.message : 'Failed to load overview data.';
} finally {
Expand Down Expand Up @@ -191,6 +196,26 @@
<KPICard title="Uptime" value={formatUptime(uptimeSeconds)} subtitle="Proxy process uptime" />
</div>

{#if globalBudget}
<div class="rounded-lg border border-border-default bg-surface p-4">
<div class="flex items-center justify-between gap-4">
<div>
<p class="text-xs font-medium uppercase tracking-wider text-text-secondary">Global Budget</p>
<p class="mt-1 text-2xl font-semibold text-text-primary">
{formatUSD(globalBudget.spent_usd)} <span class="text-sm font-normal text-text-secondary">/ {formatUSD(globalBudget.limit_usd)}</span>
</p>
<p class="mt-0.5 text-xs text-text-secondary capitalize">{globalBudget.period} · {globalBudget.percentage_used.toFixed(1)}% used</p>
</div>
<div class="h-2 w-32 overflow-hidden rounded-full bg-bg-elevated">
<div
class={`h-full rounded-full transition-all ${globalBudget.percentage_used >= 90 ? 'bg-danger' : globalBudget.percentage_used >= 75 ? 'bg-warning' : 'bg-accent'}`}
style={`width: ${Math.min(globalBudget.percentage_used, 100)}%`}
></div>
</div>
</div>
</div>
{/if}

{#if loading}
<section class="flex h-[320px] items-center justify-center rounded-lg border border-border-default bg-surface p-4 text-sm text-text-secondary">
Loading overview data...
Expand Down
26 changes: 8 additions & 18 deletions internal/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@ type Server struct {
version string
storageBackend string
pricing []config.PricingEntry
globalPeriod config.BudgetPeriod
globalLimitUSD float64
broker *broker
}

Expand Down Expand Up @@ -73,8 +71,6 @@ func New(cfg config.Config, budgetManager *budget.BudgetManager, store storage.S
"anthropic": providerStatus(cfg.Upstream.Anthropic.BaseURL),
"ollama": providerStatus(cfg.Upstream.Ollama.BaseURL),
},
globalPeriod: cfg.Gate.GlobalBudget.Period,
globalLimitUSD: cfg.Gate.GlobalBudget.LimitUSD,
broker: &broker{
clients: make(map[chan sseEvent]struct{}),
},
Expand Down Expand Up @@ -256,28 +252,22 @@ func (s *Server) handleBudgets(w http.ResponseWriter, r *http.Request) {
}

items := make([]map[string]any, 0, len(records))
globalSpent := 0.0
for _, record := range records {
view := s.budget.GetBudget(record.Name)
globalSpent += view.SpentUSD
items = append(items, encodeBudgetRecord(record, view))
}

globalRemaining := 0.0
if s.globalLimitUSD > 0 {
globalRemaining = s.globalLimitUSD - globalSpent
if globalRemaining < 0 {
globalRemaining = 0
}
}

gv := s.budget.GetGlobalBudget()
writeJSON(w, http.StatusOK, map[string]any{
"budgets": items,
"global": map[string]any{
"period": s.globalPeriod,
"limit_usd": s.globalLimitUSD,
"spent_usd": globalSpent,
"remaining_usd": globalRemaining,
"period": string(gv.Period),
"period_starts_at": gv.PeriodStartsAt,
"period_resets_at": gv.PeriodResetsAt,
"limit_usd": gv.LimitUSD,
"spent_usd": gv.SpentUSD,
"remaining_usd": gv.RemainingUSD,
"percentage_used": gv.PercentageUsed,
},
})
}
Expand Down
101 changes: 101 additions & 0 deletions internal/budget/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,28 @@ type Dispatcher interface {
Dispatch(context.Context, alert.Alert)
}

// globalBudgetState tracks cross-agent spending within a single period.
//
//nolint:govet // field order kept for semantic clarity: times grouped together.
type globalBudgetState struct {
periodStartsAt time.Time
periodResetsAt time.Time
spentUSD float64
}

// GlobalBudgetView is an API-friendly snapshot of global budget state.
//
//nolint:govet // field order kept for API clarity.
type GlobalBudgetView struct {
Period config.BudgetPeriod
PeriodStartsAt time.Time
PeriodResetsAt time.Time
LimitUSD float64
SpentUSD float64
RemainingUSD float64
PercentageUsed float64
}

// BudgetManager tracks agent spend and applies budget enforcement rules.
//
//nolint:revive,govet // Name required by spec; field grouping aids maintainability.
Expand All @@ -147,6 +169,10 @@ type BudgetManager struct {
flushInterval time.Duration
flushStop chan struct{}
flushWG sync.WaitGroup

globalLimitUSD float64
globalPeriod config.BudgetPeriod
globalState globalBudgetState
}

// NewManager creates a budget manager from gate configuration.
Expand Down Expand Up @@ -261,6 +287,12 @@ func newManager(
clock = realClock{}
}

now := clock.Now().UTC()
globalPeriod := gate.GlobalBudget.Period
if globalPeriod == "" {
globalPeriod = config.BudgetPeriodMonthly
}

manager := &BudgetManager{
clock: clock,
logger: logger,
Expand All @@ -279,6 +311,12 @@ func newManager(
downgradeChain: append([]string(nil), gate.DefaultDowngradeChain...),
alertThresholdsPct: append([]float64(nil), gate.AlertThresholdsPct...),
},
globalLimitUSD: gate.GlobalBudget.LimitUSD,
globalPeriod: globalPeriod,
globalState: globalBudgetState{
periodStartsAt: now,
periodResetsAt: nextPeriodReset(now, globalPeriod),
},
}

for _, entry := range gate.Agents {
Expand Down Expand Up @@ -373,6 +411,21 @@ func (m *BudgetManager) CheckBudgetDetailed(agent string, estimatedCostUSD float
}
}

// Check global budget before per-agent.
m.maybeResetGlobalPeriodLocked(now)
if m.globalLimitUSD > 0 && m.globalState.spentUSD+estimatedCostUSD > m.globalLimitUSD {
return Decision{
Action: ActionReject,
Code: "global_budget_exceeded",
Message: fmt.Sprintf("Global budget limit of $%.2f exceeded (spent: $%.2f)", m.globalLimitUSD, m.globalState.spentUSD),
Agent: normalizedAgent,
Period: m.globalPeriod,
LimitUSD: m.globalLimitUSD,
SpentUSD: m.globalState.spentUSD,
Over: true,
}
}

if m.registerRequestAndDetectRunawayLocked(state, now) {
state.killed = true
state.disableReason = disableReasonRunaway
Expand Down Expand Up @@ -462,6 +515,9 @@ func (m *BudgetManager) RecordSpend(agent string, costUSD float64) {
state.lastSeenAt = now
m.maybeResetPeriodLocked(state, policy, now)

m.maybeResetGlobalPeriodLocked(now)
m.globalState.spentUSD += costUSD

before := percentageUsed(policy.limitUSD, state.spentUSD)
state.spentUSD += costUSD
state.dirty = true
Expand Down Expand Up @@ -503,6 +559,39 @@ func (m *BudgetManager) RecordSpend(agent string, costUSD float64) {
}
}

// GetGlobalBudget returns the current global budget view.
func (m *BudgetManager) GetGlobalBudget() GlobalBudgetView {
now := m.clock.Now().UTC()

m.mu.Lock()
defer m.mu.Unlock()

m.maybeResetGlobalPeriodLocked(now)

spent := m.globalState.spentUSD
limit := m.globalLimitUSD
remaining := 0.0
if limit > 0 {
remaining = limit - spent
if remaining < 0 {
remaining = 0
}
}
pct := 0.0
if limit > 0 {
pct = percentageUsed(limit, spent)
}
return GlobalBudgetView{
Period: m.globalPeriod,
PeriodStartsAt: m.globalState.periodStartsAt,
PeriodResetsAt: m.globalState.periodResetsAt,
LimitUSD: limit,
SpentUSD: spent,
RemainingUSD: remaining,
PercentageUsed: pct,
}
}

// Snapshot returns current budget state for an agent.
func (m *BudgetManager) Snapshot(agent string) Snapshot {
normalizedAgent := normalizeAgent(agent)
Expand Down Expand Up @@ -712,6 +801,18 @@ func (m *BudgetManager) stateForAgentLocked(agent string, policy agentPolicy, no
return state, true
}

func (m *BudgetManager) maybeResetGlobalPeriodLocked(now time.Time) {
if m.globalLimitUSD <= 0 {
return
}
if now.Before(m.globalState.periodResetsAt) {
return
}
m.globalState.spentUSD = 0
m.globalState.periodStartsAt = now
m.globalState.periodResetsAt = nextPeriodReset(now, m.globalPeriod)
}

func (m *BudgetManager) maybeResetPeriodLocked(state *agentState, policy agentPolicy, now time.Time) {
if now.Before(state.periodResetsAt) {
return
Expand Down
Loading
Loading