Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions crates/database/src/backups.rs
Original file line number Diff line number Diff line change
Expand Up @@ -806,6 +806,12 @@ pub struct BackupRun {
pub snapshot_id: Option<String>,
#[diesel(deserialize_as = jiff_diesel::Timestamp, serialize_as = jiff_diesel::Timestamp)]
pub reported_at: Timestamp,
/// S3 traffic tallied by bestool's proxy: `raw` counts the full HTTP message
/// (incl. SigV4 chunk framing), `payload` the decoded object data.
pub s3_sent_raw_bytes: Option<i64>,
pub s3_sent_payload_bytes: Option<i64>,
pub s3_received_raw_bytes: Option<i64>,
pub s3_received_payload_bytes: Option<i64>,
}

#[derive(Debug, Clone, Insertable)]
Expand All @@ -824,6 +830,10 @@ pub struct NewBackupRun {
pub error: Option<String>,
pub bytes_uploaded: Option<i64>,
pub snapshot_id: Option<String>,
pub s3_sent_raw_bytes: Option<i64>,
pub s3_sent_payload_bytes: Option<i64>,
pub s3_received_raw_bytes: Option<i64>,
pub s3_received_payload_bytes: Option<i64>,
}

impl BackupRun {
Expand Down
4 changes: 4 additions & 0 deletions crates/database/src/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ diesel::table! {
bytes_uploaded -> Nullable<Int8>,
snapshot_id -> Nullable<Text>,
reported_at -> Timestamptz,
s3_sent_raw_bytes -> Nullable<Int8>,
s3_sent_payload_bytes -> Nullable<Int8>,
s3_received_raw_bytes -> Nullable<Int8>,
s3_received_payload_bytes -> Nullable<Int8>,
}
}

Expand Down
4 changes: 4 additions & 0 deletions crates/database/tests/backup_detection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ async fn insert_backup_success_aged(
error: None,
bytes_uploaded: Some(42),
snapshot_id: Some("kopia-snap".into()),
s3_sent_raw_bytes: None,
s3_sent_payload_bytes: None,
s3_received_raw_bytes: None,
s3_received_payload_bytes: None,
},
)
.await
Expand Down
4 changes: 4 additions & 0 deletions crates/database/tests/backups.rs
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,10 @@ fn new_run(
error: None,
bytes_uploaded: Some(42),
snapshot_id: Some("kopia-snap-1".into()),
s3_sent_raw_bytes: None,
s3_sent_payload_bytes: None,
s3_received_raw_bytes: None,
s3_received_payload_bytes: None,
}
}

Expand Down
29 changes: 29 additions & 0 deletions crates/public-server/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -1470,6 +1470,35 @@
"format": "uuid",
"description": "The run-uuid bestool minted at run start (becomes `backup_runs.id`)."
},
"s3_received_payload_bytes": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"s3_received_raw_bytes": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"s3_sent_payload_bytes": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"s3_sent_raw_bytes": {
"type": [
"integer",
"null"
],
"format": "int64",
"description": "S3 traffic the proxy tallied during the run: `raw` is the full HTTP\nmessage (incl. SigV4 chunk framing), `payload` the decoded object data.\nReported on both success and failure; absent from older clients."
},
"snapshot_id": {
"type": [
"string",
Expand Down
11 changes: 11 additions & 0 deletions crates/public-server/src/backup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,13 @@ pub struct ReportArgs {
pub error: Option<String>,
pub bytes_uploaded: Option<i64>,
pub snapshot_id: Option<String>,
/// S3 traffic the proxy tallied during the run: `raw` is the full HTTP
/// message (incl. SigV4 chunk framing), `payload` the decoded object data.
/// Reported on both success and failure; absent from older clients.
pub s3_sent_raw_bytes: Option<i64>,
pub s3_sent_payload_bytes: Option<i64>,
pub s3_received_raw_bytes: Option<i64>,
pub s3_received_payload_bytes: Option<i64>,
}

#[utoipa::path(
Expand Down Expand Up @@ -512,6 +519,10 @@ async fn report(
error: rep.error,
bytes_uploaded: rep.bytes_uploaded,
snapshot_id: rep.snapshot_id,
s3_sent_raw_bytes: rep.s3_sent_raw_bytes,
s3_sent_payload_bytes: rep.s3_sent_payload_bytes,
s3_received_raw_bytes: rep.s3_received_raw_bytes,
s3_received_payload_bytes: rep.s3_received_payload_bytes,
},
)
.await?;
Expand Down
8 changes: 8 additions & 0 deletions crates/public-server/tests/backup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,10 @@ async fn report_writes_run_with_context_attribution_and_204() {
"group_id": bogus_group,
"bytes_uploaded": 4096,
"snapshot_id": "k0pia123",
"s3_sent_raw_bytes": 5000,
"s3_sent_payload_bytes": 4096,
"s3_received_raw_bytes": 320,
"s3_received_payload_bytes": 128,
}))
.await;
resp.assert_status(http::StatusCode::NO_CONTENT);
Expand All @@ -311,6 +315,10 @@ async fn report_writes_run_with_context_attribution_and_204() {
assert_eq!(run.device_id, device_id);
assert_eq!(run.server_id, Some(server));
assert_eq!(run.bytes_uploaded, Some(4096));
assert_eq!(run.s3_sent_raw_bytes, Some(5000));
assert_eq!(run.s3_sent_payload_bytes, Some(4096));
assert_eq!(run.s3_received_raw_bytes, Some(320));
assert_eq!(run.s3_received_payload_bytes, Some(128));
},
)
.await;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ALTER TABLE backup_runs
DROP COLUMN s3_sent_raw_bytes,
DROP COLUMN s3_sent_payload_bytes,
DROP COLUMN s3_received_raw_bytes,
DROP COLUMN s3_received_payload_bytes;
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
-- S3 traffic tallied by bestool's proxy during a run: raw counts the full HTTP
-- message (incl. SigV4 chunk framing), payload counts the decoded object data.
-- Nullable: older clients omit them; both backup and restore runs may report.
ALTER TABLE backup_runs
ADD COLUMN s3_sent_raw_bytes BIGINT,
ADD COLUMN s3_sent_payload_bytes BIGINT,
ADD COLUMN s3_received_raw_bytes BIGINT,
ADD COLUMN s3_received_payload_bytes BIGINT;
46 changes: 45 additions & 1 deletion private-web/e2e/backups.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -327,10 +327,54 @@ test.describe("backups ready: stats + backup-now", () => {

// Error detail is hidden until the row is expanded.
await expect(page.getByText(/disk quota exceeded/i)).toBeHidden();
await runs.getByRole("button", { name: /show error/i }).click();
await runs.getByRole("button", { name: /show details/i }).click();
await expect(page.getByText(/disk quota exceeded/i)).toBeVisible();
});

test("run with S3 traffic but no upload size shows ~payload and expandable traffic detail", async ({
page,
sql,
}) => {
const group = await seedServerGroup(sql, { name: "s3-group" });
const device = await seedDevice(sql);
const server = await seedServer(sql, {
name: "s3-srv",
groupId: group.id,
deviceId: device.id,
});
await seedServerGroupBackupConfig(sql, {
groupId: group.id,
status: "ready",
intervalSeconds: 3600,
});
// No explicit upload size, but the proxy tallied S3 traffic → the Uploaded
// column falls back to the payload-sent figure, marked approximate.
await seedBackupRun(sql, {
deviceId: device.id,
groupId: group.id,
serverId: server.id,
outcome: "success",
bytesUploaded: null,
s3SentPayloadBytes: 2048, // 2.0 KiB → the Uploaded approximation
s3SentRawBytes: 3072, // 3.0 KiB
s3ReceivedPayloadBytes: 512, // 512 B
s3ReceivedRawBytes: 1024, // 1.0 KiB
});

await page.goto(`/groups/${group.id}/backups`);
const runs = page.getByRole("table").last();
await expect(runs.getByText("s3-srv")).toBeVisible();
// Uploaded falls back to the payload-sent figure, prefixed "~".
await expect(runs.getByText("~2.0 KiB")).toBeVisible();

// The S3 traffic breakdown is hidden until the row is expanded.
await expect(page.getByText(/s3 traffic/i)).toBeHidden();
await runs.getByRole("button", { name: /show details/i }).click();
await expect(page.getByText(/s3 traffic/i)).toBeVisible();
await expect(page.getByText(/2\.0 KiB payload \/ 3\.0 KiB raw/i)).toBeVisible();
await expect(page.getByText(/512 B payload \/ 1\.0 KiB raw/i)).toBeVisible();
});

test("backup-now writes a request row; cancel deletes it", async ({
page,
sql,
Expand Down
13 changes: 11 additions & 2 deletions private-web/e2e/seed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -411,13 +411,18 @@ export async function seedBackupRun(
error?: string | null;
bytesUploaded?: number | null;
snapshotId?: string | null;
s3SentRawBytes?: number | null;
s3SentPayloadBytes?: number | null;
s3ReceivedRawBytes?: number | null;
s3ReceivedPayloadBytes?: number | null;
},
): Promise<{ id: string }> {
const id = randomUUID();
await sql.query(
`INSERT INTO backup_runs
(id, device_id, group_id, server_id, type, purpose, outcome, error, bytes_uploaded, snapshot_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
(id, device_id, group_id, server_id, type, purpose, outcome, error, bytes_uploaded, snapshot_id,
s3_sent_raw_bytes, s3_sent_payload_bytes, s3_received_raw_bytes, s3_received_payload_bytes)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
[
id,
opts.deviceId,
Expand All @@ -429,6 +434,10 @@ export async function seedBackupRun(
opts.error ?? null,
opts.bytesUploaded ?? null,
opts.snapshotId ?? null,
opts.s3SentRawBytes ?? null,
opts.s3SentPayloadBytes ?? null,
opts.s3ReceivedRawBytes ?? null,
opts.s3ReceivedPayloadBytes ?? null,
],
);
return { id };
Expand Down
29 changes: 29 additions & 0 deletions private-web/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -5508,6 +5508,35 @@
"type": "string",
"format": "date-time"
},
"s3_received_payload_bytes": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"s3_received_raw_bytes": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"s3_sent_payload_bytes": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"s3_sent_raw_bytes": {
"type": [
"integer",
"null"
],
"format": "int64",
"description": "S3 traffic tallied by bestool's proxy: `raw` counts the full HTTP message\n(incl. SigV4 chunk framing), `payload` the decoded object data."
},
"server_id": {
"type": [
"string",
Expand Down
12 changes: 12 additions & 0 deletions private-web/src/api-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2279,6 +2279,18 @@ export interface components {
purpose: components["schemas"]["BackupPurpose"];
/** Format: date-time */
reported_at: string;
/** Format: int64 */
s3_received_payload_bytes?: number | null;
/** Format: int64 */
s3_received_raw_bytes?: number | null;
/** Format: int64 */
s3_sent_payload_bytes?: number | null;
/**
* Format: int64
* @description S3 traffic tallied by bestool's proxy: `raw` counts the full HTTP message
* (incl. SigV4 chunk framing), `payload` the decoded object data.
*/
s3_sent_raw_bytes?: number | null;
/** Format: uuid */
server_id?: string | null;
snapshot_id?: string | null;
Expand Down
Loading