From 66ee5057cd1761f6c1d004ff6a5949b945c2f588 Mon Sep 17 00:00:00 2001 From: Ivan Pushkin Date: Sat, 13 Jun 2020 14:24:48 +0200 Subject: [PATCH 1/6] rename errors --- client/client_names.go | 2 +- client/clients.go | 2 +- client/clients_over_time.go | 2 +- client/domains.go | 2 +- client/forward_destinations.go | 2 +- client/ftl_client.go | 22 +++++++++++----------- client/query_types.go | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/client/client_names.go b/client/client_names.go index 1429329..7d095ed 100644 --- a/client/client_names.go +++ b/client/client_names.go @@ -33,7 +33,7 @@ func (client *FTLClient) GetClientNames() (*[]Client, error) { var clients []Client for { name, err := readString(conn) - if err == EOF { + if err == errEndOfInput { break } if err != nil { diff --git a/client/clients.go b/client/clients.go index 3527439..f8cf34e 100644 --- a/client/clients.go +++ b/client/clients.go @@ -49,7 +49,7 @@ func topClientsFor(command string, client *FTLClient) (*Entries, error) { for { _, err := readString(conn) - if err == EOF { + if err == errEndOfInput { break } if err != nil { diff --git a/client/clients_over_time.go b/client/clients_over_time.go index 4930ca7..678d6ee 100644 --- a/client/clients_over_time.go +++ b/client/clients_over_time.go @@ -19,7 +19,7 @@ import ( "net" ) -// GetQueriesOverTime retrieves amount of queries grouped by client +// GetClientsOverTime retrieves amount of queries grouped by client // for the last 24 hours aggregated over 10 minute intervals // from response of `>ClientsoverTime` command // Warning: API might be not public diff --git a/client/domains.go b/client/domains.go index 163dfde..d1af454 100644 --- a/client/domains.go +++ b/client/domains.go @@ -48,7 +48,7 @@ func topQueriesFor(command string, client *FTLClient) (*Entries, error) { for { domainName, err := readString(conn) - if err == EOF { + if err == errEndOfInput { break } if err != nil { diff --git a/client/forward_destinations.go b/client/forward_destinations.go index 053fc1b..74769cc 100644 --- a/client/forward_destinations.go +++ b/client/forward_destinations.go @@ -33,7 +33,7 @@ func (client *FTLClient) GetForwardDestinations() (*[]UpstreamDestination, error var destinations []UpstreamDestination for { name, err := readString(conn) - if err == EOF { + if err == errEndOfInput { break } if err != nil { diff --git a/client/ftl_client.go b/client/ftl_client.go index d8b72f5..4882649 100644 --- a/client/ftl_client.go +++ b/client/ftl_client.go @@ -31,8 +31,8 @@ const ( formatEOF uint8 = 0xc1 // 193 ) -var EOF = errors.New("EOF") -var invalidFormat = errors.New("unexpected format") +var errEndOfInput = errors.New("end of the input") +var errInvalidFormat = errors.New("unexpected format") // FTLClient for Pi-holes's FTL daemon. Contains address to a unix socket type FTLClient struct { @@ -66,18 +66,18 @@ func readString(conn *net.UnixConn) (string, error) { var format uint8 if err := binary.Read(conn, binary.BigEndian, &format); err != nil { if err == io.EOF { - return "", EOF + return "", errEndOfInput } return "", err } if format == formatEOF { - return "", EOF + return "", errEndOfInput } if format != formatString { - return "", invalidFormat + return "", errInvalidFormat } var length uint32 @@ -98,18 +98,18 @@ func readFloat32(conn *net.UnixConn) (float32, error) { var format uint8 if err := binary.Read(conn, binary.BigEndian, &format); err != nil { if err == io.EOF { - return 0.0, EOF + return 0.0, errEndOfInput } return 0.0, err } if format == formatEOF { - return 0.0, EOF + return 0.0, errEndOfInput } if format != formatFloat32 { - return 0.0, invalidFormat + return 0.0, errInvalidFormat } var value float32 @@ -124,18 +124,18 @@ func readUint32(conn *net.UnixConn) (uint32, error) { var format uint8 if err := binary.Read(conn, binary.BigEndian, &format); err != nil { if err == io.EOF { - return 0, EOF + return 0, errEndOfInput } return 0, err } if format == formatEOF { - return 0, EOF + return 0, errEndOfInput } if format != formatUint32 { - return 0, invalidFormat + return 0, errInvalidFormat } var value uint32 diff --git a/client/query_types.go b/client/query_types.go index dc48627..68d52c9 100644 --- a/client/query_types.go +++ b/client/query_types.go @@ -33,7 +33,7 @@ func (client *FTLClient) GetQueryTypes() (*map[string]float32, error) { queryTypes := make(map[string]float32) for { name, err := readString(conn) - if err == EOF { + if err == errEndOfInput { break } if err != nil { From d759142863f4fa8fc02fd154a5f29ad67333e7c7 Mon Sep 17 00:00:00 2001 From: Ivan Pushkin Date: Sat, 13 Jun 2020 22:41:04 +0200 Subject: [PATCH 2/6] update model --- .github/workflows/go.yml | 4 +- client/clients.go | 23 ++++---- client/clients_over_time.go | 8 +-- client/db_stats.go | 10 +++- client/domains.go | 22 ++++---- client/model.go | 96 ++++++++++++++++++---------------- client/queries_over_time.go | 39 ++++++++++---- client/stats.go | 20 +++---- collector/ad_domains.go | 6 +-- collector/clients.go | 8 +-- collector/clients_over_time.go | 2 +- collector/db_stats.go | 4 +- collector/domains.go | 6 +-- collector/queries_over_time.go | 8 +-- 14 files changed, 147 insertions(+), 109 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f6feb75..afb6c3d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -28,5 +28,5 @@ jobs: - name: Build run: go build -v . -# - name: Test -# run: go test -v . + - name: Test + run: go test -v ./... diff --git a/client/clients.go b/client/clients.go index f8cf34e..25942a2 100644 --- a/client/clients.go +++ b/client/clients.go @@ -20,17 +20,17 @@ import ( // GetTopClients retrieves the list of clients together with amount of queries // made by each client from response of `>top-clients` command -func (client *FTLClient) GetTopClients() (*Entries, error) { +func (client *FTLClient) GetTopClients() (*TopEntries, error) { return topClientsFor(">top-clients", client) } // GetTopBlockedClients retrieves the list of clients together with amount of blocked // queries made by each client from response of `>top-clients` command -func (client *FTLClient) GetTopBlockedClients() (*Entries, error) { +func (client *FTLClient) GetTopBlockedClients() (*TopEntries, error) { return topClientsFor(">top-clients blocked", client) } -func topClientsFor(command string, client *FTLClient) (*Entries, error) { +func topClientsFor(command string, client *FTLClient) (*TopEntries, error) { conn, err := net.DialUnix("unix", nil, client.addr) if err != nil { return nil, err @@ -41,12 +41,15 @@ func topClientsFor(command string, client *FTLClient) (*Entries, error) { return nil, err } - var result Entries - - if err := binary.Read(conn, binary.BigEndian, &result.Total); err != nil { + var total ftlUInt32 + if err := binary.Read(conn, binary.BigEndian, &total); err != nil { return nil, err } + result := TopEntries{ + Total: int(total.Value), + } + for { _, err := readString(conn) if err == errEndOfInput { @@ -66,10 +69,10 @@ func topClientsFor(command string, client *FTLClient) (*Entries, error) { return nil, err } - result.List = append(result.List, struct { - Entry string - Count uint32 - }{Entry: address, Count: count}) + result.Entries = append(result.Entries, struct { + Label string + Count int + }{Label: address, Count: int(count)}) } return &result, nil diff --git a/client/clients_over_time.go b/client/clients_over_time.go index 678d6ee..8e2ba0d 100644 --- a/client/clients_over_time.go +++ b/client/clients_over_time.go @@ -43,7 +43,7 @@ func (client *FTLClient) GetClientsOverTime() (*[]TimestampClients, error) { break } - var clients []Int32Block + var clients []int var timestamp uint32 err = binary.Read(conn, binary.BigEndian, ×tamp) @@ -52,7 +52,7 @@ func (client *FTLClient) GetClientsOverTime() (*[]TimestampClients, error) { } for { - var clientQueryCount Int32Block + var clientQueryCount ftlInt32 err := binary.Read(conn, binary.BigEndian, &clientQueryCount) if err != nil { return nil, err @@ -62,11 +62,11 @@ func (client *FTLClient) GetClientsOverTime() (*[]TimestampClients, error) { break } - clients = append(clients, clientQueryCount) + clients = append(clients, int(clientQueryCount.Value)) } timestamps = append(timestamps, TimestampClients{ - Timestamp: timestamp, + Timestamp: int(timestamp), Count: clients, }) } diff --git a/client/db_stats.go b/client/db_stats.go index 353fd62..91a9e74 100644 --- a/client/db_stats.go +++ b/client/db_stats.go @@ -30,10 +30,16 @@ func (client *FTLClient) GetDBStats() (*DBStats, error) { return nil, err } - var stats DBStats + var stats struct { + Rows ftlUInt32 + Size ftlUInt64 + } if err := binary.Read(conn, binary.BigEndian, &stats); err != nil { return nil, err } - return &stats, nil + return &DBStats{ + RowsCount: int(stats.Rows.Value), + FileSize: int(stats.Size.Value), + }, nil } diff --git a/client/domains.go b/client/domains.go index d1af454..88cbdd8 100644 --- a/client/domains.go +++ b/client/domains.go @@ -20,17 +20,17 @@ import ( // GetTopDomains retrieves the list of domains together with amount of queries // made for each domain from response of `>top-domains` command -func (client *FTLClient) GetTopDomains() (*Entries, error) { +func (client *FTLClient) GetTopDomains() (*TopEntries, error) { return topQueriesFor(">top-domains", client) } // GetTopAds retrieves the list of ad domains together with amount of queries // made for each domain from response of `>top-ads` command -func (client *FTLClient) GetTopAds() (*Entries, error) { +func (client *FTLClient) GetTopAds() (*TopEntries, error) { return topQueriesFor(">top-ads", client) } -func topQueriesFor(command string, client *FTLClient) (*Entries, error) { +func topQueriesFor(command string, client *FTLClient) (*TopEntries, error) { conn, err := net.DialUnix("unix", nil, client.addr) if err != nil { return nil, err @@ -41,11 +41,15 @@ func topQueriesFor(command string, client *FTLClient) (*Entries, error) { return nil, err } - var result Entries - if err := binary.Read(conn, binary.BigEndian, &result.Total); err != nil { + var total ftlUInt32 + if err := binary.Read(conn, binary.BigEndian, &total); err != nil { return nil, err } + result := TopEntries{ + Total: int(total.Value), + } + for { domainName, err := readString(conn) if err == errEndOfInput { @@ -60,10 +64,10 @@ func topQueriesFor(command string, client *FTLClient) (*Entries, error) { return nil, err } - result.List = append(result.List, struct { - Entry string - Count uint32 - }{Entry: domainName, Count: domainCount}) + result.Entries = append(result.Entries, struct { + Label string + Count int + }{Label: domainName, Count: int(domainCount)}) } return &result, nil diff --git a/client/model.go b/client/model.go index 7559df0..6679243 100644 --- a/client/model.go +++ b/client/model.go @@ -27,71 +27,79 @@ type Stats struct { Status int } -// DBStats represents the response of `>db-stats` command +// DBStats represents the response of `>db-stats` command. +// It contains amount of rows in database and current file size of the database type DBStats struct { - Rows UInt32Block - Size UInt64Block + RowsCount int + FileSize int } -type UInt32Block struct { - _ uint8 - Value uint32 -} - -type Int32Block struct { - _ uint8 - Value int32 +// TopEntries represents the response of `>top-clients` and `>top-domains` commands. +// It contains a total amount of entries and a list of entries label and count +type TopEntries struct { + Total int + Entries []struct { + Label string + Count int + } } -type UInt64Block struct { - _ uint8 - Value uint64 +// UpstreamDestination represents the response `>forward-dest` command. +// It contains a name, address and percentage of total requests +type UpstreamDestination struct { + Name string + Address string + Percentage float32 } -type UInt8Block struct { - _ uint8 - Value uint8 +// TimestampClients represents the response `>ClientsoverTime` command. +// It contains a timestamp and a list of amount of requests made by each client. +// Order of requests counts represents clients from `>client-names` command +type TimestampClients struct { + Timestamp int + Count []int } -type Float32Block struct { - _ uint8 - Value float32 +// Client represents the response `>client-names` command. +// It contains a name and address of the client +type Client struct { + Name string + Address string } -type Entries struct { - Total UInt32Block - List []struct { - Entry string - Count uint32 - } +// QueriesOverTime represents the response `>overTime` command. +// It contains list of amounts of forwarded and blocked requests grouped by 10 minute intervals +type QueriesOverTime struct { + Forwarded []timestampCount + Blocked []timestampCount } -type UpstreamDestination struct { - Name string - Address string - Percentage float32 +type timestampCount struct { + Timestamp int + Count int } -type TimestampCount struct { - Timestamp UInt32Block - Count UInt32Block +type ftlUInt32 struct { + _ uint8 + Value uint32 } -type TimestampClients struct { - Timestamp uint32 - Count []Int32Block +type ftlUInt64 struct { + _ uint8 + Value uint64 } -type ClientsOverTime struct { - List []TimestampClients +type ftlInt32 struct { + _ uint8 + Value int32 } -type Client struct { - Name string - Address string +type ftlInt8 struct { + _ uint8 + Value uint8 } -type OverTime struct { - Forwarded []TimestampCount - Blocked []TimestampCount +type ftlFloat32 struct { + _ uint8 + Value float32 } diff --git a/client/queries_over_time.go b/client/queries_over_time.go index e8a77bc..328cb09 100644 --- a/client/queries_over_time.go +++ b/client/queries_over_time.go @@ -21,7 +21,7 @@ import ( // GetQueriesOverTime retrieves amount of allowed and blocked queries // for the last 24 hours aggregated over 10 minute intervals // from response of `>overTime` command -func (client *FTLClient) GetQueriesOverTime() (*OverTime, error) { +func (client *FTLClient) GetQueriesOverTime() (*QueriesOverTime, error) { conn, err := net.DialUnix("unix", nil, client.addr) if err != nil { return nil, err @@ -32,6 +32,8 @@ func (client *FTLClient) GetQueriesOverTime() (*OverTime, error) { return nil, err } + var result QueriesOverTime + var lines struct { _ uint8 Lines uint16 @@ -40,24 +42,39 @@ func (client *FTLClient) GetQueriesOverTime() (*OverTime, error) { return nil, err } - forwarded := make([]TimestampCount, lines.Lines) - - if err := binary.Read(conn, binary.BigEndian, &forwarded); err != nil { + response := make([]struct { + Timestamp ftlUInt32 + Count ftlUInt32 + }, lines.Lines) + if err := binary.Read(conn, binary.BigEndian, &response); err != nil { return nil, err } + for _, r := range response { + result.Forwarded = append(result.Forwarded, struct { + Timestamp int + Count int + }{Timestamp: int(r.Timestamp.Value), Count: int(r.Count.Value)}) + } + if err := binary.Read(conn, binary.BigEndian, &lines); err != nil { return nil, err } - blocked := make([]TimestampCount, lines.Lines) - - if err := binary.Read(conn, binary.BigEndian, &blocked); err != nil { + response = make([]struct { + Timestamp ftlUInt32 + Count ftlUInt32 + }, lines.Lines) + if err := binary.Read(conn, binary.BigEndian, &response); err != nil { return nil, err } - return &OverTime{ - Forwarded: forwarded, - Blocked: blocked, - }, nil + for _, r := range response { + result.Blocked = append(result.Blocked, struct { + Timestamp int + Count int + }{Timestamp: int(r.Timestamp.Value), Count: int(r.Count.Value)}) + } + + return &result, nil } diff --git a/client/stats.go b/client/stats.go index a74797e..246ba32 100644 --- a/client/stats.go +++ b/client/stats.go @@ -31,16 +31,16 @@ func (client *FTLClient) GetStats() (*Stats, error) { } var stats struct { - DomainsBeingBlocked UInt32Block - DnsQueriesToday UInt32Block - AdsBlockedToday UInt32Block - AdsPercentageToday Float32Block - UniqueDomains UInt32Block - QueriesForwarded UInt32Block - QueriesCached UInt32Block - ClientsEverSeen UInt32Block - UniqueClients UInt32Block - Status UInt8Block + DomainsBeingBlocked ftlUInt32 + DnsQueriesToday ftlUInt32 + AdsBlockedToday ftlUInt32 + AdsPercentageToday ftlFloat32 + UniqueDomains ftlUInt32 + QueriesForwarded ftlUInt32 + QueriesCached ftlUInt32 + ClientsEverSeen ftlUInt32 + UniqueClients ftlUInt32 + Status ftlInt8 } if err := binary.Read(conn, binary.BigEndian, &stats); err != nil { return nil, err diff --git a/collector/ad_domains.go b/collector/ad_domains.go index dec4165..d97edf3 100644 --- a/collector/ad_domains.go +++ b/collector/ad_domains.go @@ -49,10 +49,10 @@ func (c *adDomainCollector) update(client *client.FTLClient, ch chan<- prometheu return err } - ch <- prometheus.MustNewConstMetric(c.totalAdDomainsToday, prometheus.GaugeValue, float64(queries.Total.Value)) + ch <- prometheus.MustNewConstMetric(c.totalAdDomainsToday, prometheus.GaugeValue, float64(queries.Total)) - for _, hits := range queries.List { - ch <- prometheus.MustNewConstMetric(c.topAdDomainsToday, prometheus.GaugeValue, float64(hits.Count), hits.Entry) + for _, hits := range queries.Entries { + ch <- prometheus.MustNewConstMetric(c.topAdDomainsToday, prometheus.GaugeValue, float64(hits.Count), hits.Label) } return nil diff --git a/collector/clients.go b/collector/clients.go index 2294f49..99481c9 100644 --- a/collector/clients.go +++ b/collector/clients.go @@ -49,8 +49,8 @@ func (c *clientCollector) update(client *client.FTLClient, ch chan<- prometheus. return err } - for _, hits := range clients.List { - ch <- prometheus.MustNewConstMetric(c.topClientsToday, prometheus.GaugeValue, float64(hits.Count), hits.Entry) + for _, hits := range clients.Entries { + ch <- prometheus.MustNewConstMetric(c.topClientsToday, prometheus.GaugeValue, float64(hits.Count), hits.Label) } blockedClients, err := client.GetTopBlockedClients() @@ -58,8 +58,8 @@ func (c *clientCollector) update(client *client.FTLClient, ch chan<- prometheus. return err } - for _, hits := range blockedClients.List { - ch <- prometheus.MustNewConstMetric(c.topBlockedClientsToday, prometheus.GaugeValue, float64(hits.Count), hits.Entry) + for _, hits := range blockedClients.Entries { + ch <- prometheus.MustNewConstMetric(c.topBlockedClientsToday, prometheus.GaugeValue, float64(hits.Count), hits.Label) } return nil diff --git a/collector/clients_over_time.go b/collector/clients_over_time.go index 3a604fc..0bf15dd 100644 --- a/collector/clients_over_time.go +++ b/collector/clients_over_time.go @@ -64,7 +64,7 @@ func (c *clientsOverTimeCollector) update(client *client.FTLClient, ch chan<- pr ch <- prometheus.MustNewConstMetric( c.clients, prometheus.GaugeValue, - float64(count.Value), + float64(count), address, ) } diff --git a/collector/db_stats.go b/collector/db_stats.go index c74431c..37bd980 100644 --- a/collector/db_stats.go +++ b/collector/db_stats.go @@ -51,8 +51,8 @@ func (c *dbStatsCollector) update(client *client.FTLClient, ch chan<- prometheus return err } - ch <- prometheus.MustNewConstMetric(c.queriesInDatabase, prometheus.CounterValue, float64(dbStats.Rows.Value)) - ch <- prometheus.MustNewConstMetric(c.databaseFileSize, prometheus.CounterValue, float64(dbStats.Size.Value)) + ch <- prometheus.MustNewConstMetric(c.queriesInDatabase, prometheus.CounterValue, float64(dbStats.RowsCount)) + ch <- prometheus.MustNewConstMetric(c.databaseFileSize, prometheus.CounterValue, float64(dbStats.FileSize)) return nil } diff --git a/collector/domains.go b/collector/domains.go index 6593389..a46c26c 100644 --- a/collector/domains.go +++ b/collector/domains.go @@ -49,10 +49,10 @@ func (c *domainCollector) update(client *client.FTLClient, ch chan<- prometheus. return err } - ch <- prometheus.MustNewConstMetric(c.totalDomainsToday, prometheus.GaugeValue, float64(queries.Total.Value)) + ch <- prometheus.MustNewConstMetric(c.totalDomainsToday, prometheus.GaugeValue, float64(queries.Total)) - for _, hits := range queries.List { - ch <- prometheus.MustNewConstMetric(c.topDomainsToday, prometheus.GaugeValue, float64(hits.Count), hits.Entry) + for _, hits := range queries.Entries { + ch <- prometheus.MustNewConstMetric(c.topDomainsToday, prometheus.GaugeValue, float64(hits.Count), hits.Label) } return nil diff --git a/collector/queries_over_time.go b/collector/queries_over_time.go index a6db411..747bc09 100644 --- a/collector/queries_over_time.go +++ b/collector/queries_over_time.go @@ -51,19 +51,19 @@ func (c *queriesOverTimeCollector) update(client *client.FTLClient, ch chan<- pr } sort.SliceStable(queriesOverTime.Forwarded, func(i, j int) bool { - return queriesOverTime.Forwarded[i].Timestamp.Value > queriesOverTime.Forwarded[j].Timestamp.Value + return queriesOverTime.Forwarded[i].Timestamp > queriesOverTime.Forwarded[j].Timestamp }) lastForwardedOverTime := queriesOverTime.Forwarded[:1] for _, hits := range lastForwardedOverTime { - ch <- prometheus.MustNewConstMetric(c.queriesForwarded, prometheus.GaugeValue, float64(hits.Count.Value)) + ch <- prometheus.MustNewConstMetric(c.queriesForwarded, prometheus.GaugeValue, float64(hits.Count)) } sort.SliceStable(queriesOverTime.Blocked, func(i, j int) bool { - return queriesOverTime.Blocked[i].Timestamp.Value > queriesOverTime.Blocked[j].Timestamp.Value + return queriesOverTime.Blocked[i].Timestamp > queriesOverTime.Blocked[j].Timestamp }) lastBlockedOverTime := queriesOverTime.Blocked[:1] for _, hits := range lastBlockedOverTime { - ch <- prometheus.MustNewConstMetric(c.queriesBlocked, prometheus.GaugeValue, float64(hits.Count.Value)) + ch <- prometheus.MustNewConstMetric(c.queriesBlocked, prometheus.GaugeValue, float64(hits.Count)) } return nil From 19ba1ba6bf8850b3c8d6a5c28989ce34a268d3e4 Mon Sep 17 00:00:00 2001 From: Ivan Pushkin Date: Sat, 13 Jun 2020 22:49:08 +0200 Subject: [PATCH 3/6] replace uint32 with just int32 --- client/clients.go | 4 ++-- client/db_stats.go | 2 +- client/domains.go | 4 ++-- client/ftl_client.go | 8 ++++---- client/model.go | 5 ----- client/queries_over_time.go | 8 ++++---- client/stats.go | 16 ++++++++-------- 7 files changed, 21 insertions(+), 26 deletions(-) diff --git a/client/clients.go b/client/clients.go index 25942a2..a8dea22 100644 --- a/client/clients.go +++ b/client/clients.go @@ -41,7 +41,7 @@ func topClientsFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - var total ftlUInt32 + var total ftlInt32 if err := binary.Read(conn, binary.BigEndian, &total); err != nil { return nil, err } @@ -64,7 +64,7 @@ func topClientsFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - count, err := readUint32(conn) + count, err := readInt32(conn) if err != nil { return nil, err } diff --git a/client/db_stats.go b/client/db_stats.go index 91a9e74..483345b 100644 --- a/client/db_stats.go +++ b/client/db_stats.go @@ -31,7 +31,7 @@ func (client *FTLClient) GetDBStats() (*DBStats, error) { } var stats struct { - Rows ftlUInt32 + Rows ftlInt32 Size ftlUInt64 } if err := binary.Read(conn, binary.BigEndian, &stats); err != nil { diff --git a/client/domains.go b/client/domains.go index 88cbdd8..1adccac 100644 --- a/client/domains.go +++ b/client/domains.go @@ -41,7 +41,7 @@ func topQueriesFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - var total ftlUInt32 + var total ftlInt32 if err := binary.Read(conn, binary.BigEndian, &total); err != nil { return nil, err } @@ -59,7 +59,7 @@ func topQueriesFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - domainCount, err := readUint32(conn) + domainCount, err := readInt32(conn) if err != nil { return nil, err } diff --git a/client/ftl_client.go b/client/ftl_client.go index 4882649..f830f4e 100644 --- a/client/ftl_client.go +++ b/client/ftl_client.go @@ -22,7 +22,7 @@ import ( ) const ( - formatUint32 uint8 = 0xd2 // 210 + formatInt32 uint8 = 0xd2 // 210 formatFloat32 uint8 = 0xca // 202 formatUInt8 uint8 = 0xcc // 204 formatString uint8 = 0xdb // 219 @@ -120,7 +120,7 @@ func readFloat32(conn *net.UnixConn) (float32, error) { return value, nil } -func readUint32(conn *net.UnixConn) (uint32, error) { +func readInt32(conn *net.UnixConn) (int, error) { var format uint8 if err := binary.Read(conn, binary.BigEndian, &format); err != nil { if err == io.EOF { @@ -134,7 +134,7 @@ func readUint32(conn *net.UnixConn) (uint32, error) { return 0, errEndOfInput } - if format != formatUint32 { + if format != formatInt32 { return 0, errInvalidFormat } @@ -143,7 +143,7 @@ func readUint32(conn *net.UnixConn) (uint32, error) { return 0, err } - return value, nil + return int(value), nil } func sendCommand(conn *net.UnixConn, command string) error { diff --git a/client/model.go b/client/model.go index 6679243..8b1bbee 100644 --- a/client/model.go +++ b/client/model.go @@ -79,11 +79,6 @@ type timestampCount struct { Count int } -type ftlUInt32 struct { - _ uint8 - Value uint32 -} - type ftlUInt64 struct { _ uint8 Value uint64 diff --git a/client/queries_over_time.go b/client/queries_over_time.go index 328cb09..19313b7 100644 --- a/client/queries_over_time.go +++ b/client/queries_over_time.go @@ -43,8 +43,8 @@ func (client *FTLClient) GetQueriesOverTime() (*QueriesOverTime, error) { } response := make([]struct { - Timestamp ftlUInt32 - Count ftlUInt32 + Timestamp ftlInt32 + Count ftlInt32 }, lines.Lines) if err := binary.Read(conn, binary.BigEndian, &response); err != nil { return nil, err @@ -62,8 +62,8 @@ func (client *FTLClient) GetQueriesOverTime() (*QueriesOverTime, error) { } response = make([]struct { - Timestamp ftlUInt32 - Count ftlUInt32 + Timestamp ftlInt32 + Count ftlInt32 }, lines.Lines) if err := binary.Read(conn, binary.BigEndian, &response); err != nil { return nil, err diff --git a/client/stats.go b/client/stats.go index 246ba32..4255dc5 100644 --- a/client/stats.go +++ b/client/stats.go @@ -31,15 +31,15 @@ func (client *FTLClient) GetStats() (*Stats, error) { } var stats struct { - DomainsBeingBlocked ftlUInt32 - DnsQueriesToday ftlUInt32 - AdsBlockedToday ftlUInt32 + DomainsBeingBlocked ftlInt32 + DnsQueriesToday ftlInt32 + AdsBlockedToday ftlInt32 AdsPercentageToday ftlFloat32 - UniqueDomains ftlUInt32 - QueriesForwarded ftlUInt32 - QueriesCached ftlUInt32 - ClientsEverSeen ftlUInt32 - UniqueClients ftlUInt32 + UniqueDomains ftlInt32 + QueriesForwarded ftlInt32 + QueriesCached ftlInt32 + ClientsEverSeen ftlInt32 + UniqueClients ftlInt32 Status ftlInt8 } if err := binary.Read(conn, binary.BigEndian, &stats); err != nil { From 38303ee7c2b5d37fdc74d3a7b1811a296229f330 Mon Sep 17 00:00:00 2001 From: Ivan Pushkin Date: Sat, 13 Jun 2020 23:30:11 +0200 Subject: [PATCH 4/6] use readInt* fuctions --- client/clients.go | 12 ++++-------- client/clients_over_time.go | 24 ++++++++---------------- client/db_stats.go | 15 ++++++++------- client/domains.go | 12 ++++-------- client/ftl_client.go | 29 ++++++++++++++++++++++++++++- client/model.go | 15 ++++++--------- client/queries_over_time.go | 16 ++++++++-------- collector/collector.go | 5 +++-- 8 files changed, 69 insertions(+), 59 deletions(-) diff --git a/client/clients.go b/client/clients.go index a8dea22..1ef4610 100644 --- a/client/clients.go +++ b/client/clients.go @@ -14,7 +14,6 @@ package client import ( - "encoding/binary" "net" ) @@ -41,13 +40,13 @@ func topClientsFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - var total ftlInt32 - if err := binary.Read(conn, binary.BigEndian, &total); err != nil { + total, err := readInt32(conn) + if err != nil { return nil, err } result := TopEntries{ - Total: int(total.Value), + Total: total, } for { @@ -69,10 +68,7 @@ func topClientsFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - result.Entries = append(result.Entries, struct { - Label string - Count int - }{Label: address, Count: int(count)}) + result.Entries = append(result.Entries, entry{Label: address, Count: count}) } return &result, nil diff --git a/client/clients_over_time.go b/client/clients_over_time.go index 8e2ba0d..75003c9 100644 --- a/client/clients_over_time.go +++ b/client/clients_over_time.go @@ -14,8 +14,6 @@ package client import ( - "encoding/binary" - "io" "net" ) @@ -36,37 +34,31 @@ func (client *FTLClient) GetClientsOverTime() (*[]TimestampClients, error) { var timestamps []TimestampClients for { - var format uint8 - err := binary.Read(conn, binary.BigEndian, &format) - - if err == io.EOF || format == formatEOF { + timestamp, err := readInt32(conn) + if err == errEndOfInput { break } - - var clients []int - - var timestamp uint32 - err = binary.Read(conn, binary.BigEndian, ×tamp) if err != nil { return nil, err } + var clients []int + for { - var clientQueryCount ftlInt32 - err := binary.Read(conn, binary.BigEndian, &clientQueryCount) + clientQueryCount, err := readInt32(conn) if err != nil { return nil, err } - if clientQueryCount.Value == -1 { + if clientQueryCount == -1 { break } - clients = append(clients, int(clientQueryCount.Value)) + clients = append(clients, clientQueryCount) } timestamps = append(timestamps, TimestampClients{ - Timestamp: int(timestamp), + Timestamp: timestamp, Count: clients, }) } diff --git a/client/db_stats.go b/client/db_stats.go index 483345b..df6a266 100644 --- a/client/db_stats.go +++ b/client/db_stats.go @@ -14,7 +14,6 @@ package client import ( - "encoding/binary" "net" ) @@ -30,16 +29,18 @@ func (client *FTLClient) GetDBStats() (*DBStats, error) { return nil, err } - var stats struct { - Rows ftlInt32 - Size ftlUInt64 + rows, err := readInt32(conn) + if err != nil { + return nil, err } - if err := binary.Read(conn, binary.BigEndian, &stats); err != nil { + + size, err := readInt64(conn) + if err != nil { return nil, err } return &DBStats{ - RowsCount: int(stats.Rows.Value), - FileSize: int(stats.Size.Value), + RowsCount: rows, + FileSize: size, }, nil } diff --git a/client/domains.go b/client/domains.go index 1adccac..fbf4c10 100644 --- a/client/domains.go +++ b/client/domains.go @@ -14,7 +14,6 @@ package client import ( - "encoding/binary" "net" ) @@ -41,13 +40,13 @@ func topQueriesFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - var total ftlInt32 - if err := binary.Read(conn, binary.BigEndian, &total); err != nil { + total, err := readInt32(conn) + if err != nil { return nil, err } result := TopEntries{ - Total: int(total.Value), + Total: int(total), } for { @@ -64,10 +63,7 @@ func topQueriesFor(command string, client *FTLClient) (*TopEntries, error) { return nil, err } - result.Entries = append(result.Entries, struct { - Label string - Count int - }{Label: domainName, Count: int(domainCount)}) + result.Entries = append(result.Entries, entry{Label: domainName, Count: domainCount}) } return &result, nil diff --git a/client/ftl_client.go b/client/ftl_client.go index f830f4e..d282d7b 100644 --- a/client/ftl_client.go +++ b/client/ftl_client.go @@ -23,6 +23,7 @@ import ( const ( formatInt32 uint8 = 0xd2 // 210 + formatInt64 uint8 = 0xd3 // 211 formatFloat32 uint8 = 0xca // 202 formatUInt8 uint8 = 0xcc // 204 formatString uint8 = 0xdb // 219 @@ -138,7 +139,33 @@ func readInt32(conn *net.UnixConn) (int, error) { return 0, errInvalidFormat } - var value uint32 + var value int32 + if err := binary.Read(conn, binary.BigEndian, &value); err != nil { + return 0, err + } + + return int(value), nil +} + +func readInt64(conn *net.UnixConn) (int, error) { + var format uint8 + if err := binary.Read(conn, binary.BigEndian, &format); err != nil { + if err == io.EOF { + return 0, errEndOfInput + } + + return 0, err + } + + if format == formatEOF { + return 0, errEndOfInput + } + + if format != formatInt64 { + return 0, errInvalidFormat + } + + var value int64 if err := binary.Read(conn, binary.BigEndian, &value); err != nil { return 0, err } diff --git a/client/model.go b/client/model.go index 8b1bbee..cc463db 100644 --- a/client/model.go +++ b/client/model.go @@ -38,10 +38,12 @@ type DBStats struct { // It contains a total amount of entries and a list of entries label and count type TopEntries struct { Total int - Entries []struct { - Label string - Count int - } + Entries []entry +} + +type entry struct { + Label string + Count int } // UpstreamDestination represents the response `>forward-dest` command. @@ -79,11 +81,6 @@ type timestampCount struct { Count int } -type ftlUInt64 struct { - _ uint8 - Value uint64 -} - type ftlInt32 struct { _ uint8 Value int32 diff --git a/client/queries_over_time.go b/client/queries_over_time.go index 19313b7..f27a29b 100644 --- a/client/queries_over_time.go +++ b/client/queries_over_time.go @@ -51,10 +51,10 @@ func (client *FTLClient) GetQueriesOverTime() (*QueriesOverTime, error) { } for _, r := range response { - result.Forwarded = append(result.Forwarded, struct { - Timestamp int - Count int - }{Timestamp: int(r.Timestamp.Value), Count: int(r.Count.Value)}) + result.Forwarded = append(result.Forwarded, timestampCount{ + Timestamp: int(r.Timestamp.Value), + Count: int(r.Count.Value), + }) } if err := binary.Read(conn, binary.BigEndian, &lines); err != nil { @@ -70,10 +70,10 @@ func (client *FTLClient) GetQueriesOverTime() (*QueriesOverTime, error) { } for _, r := range response { - result.Blocked = append(result.Blocked, struct { - Timestamp int - Count int - }{Timestamp: int(r.Timestamp.Value), Count: int(r.Count.Value)}) + result.Blocked = append(result.Blocked, timestampCount{ + Timestamp: int(r.Timestamp.Value), + Count: int(r.Count.Value), + }) } return &result, nil diff --git a/collector/collector.go b/collector/collector.go index a89e35a..177a986 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -92,14 +92,14 @@ func NewExporter(socket string) (*Exporter, error) { } } - client, err := client.NewClient(socket) + ftlClient, err := client.NewClient(socket) if err != nil { return nil, err } return &Exporter{ collectors: collectors, - client: client, + client: ftlClient, }, nil } @@ -123,6 +123,7 @@ func execute(name string, c Collector, client *client.FTLClient, ch chan<- prome success := float64(1) if err != nil { + log.Println(name, err) success = 0 } ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, duration.Seconds(), name) From 4e3b226e902fd4177dfbc87ae727df0539222457 Mon Sep 17 00:00:00 2001 From: Ivan Pushkin Date: Sun, 14 Jun 2020 13:29:35 +0200 Subject: [PATCH 5/6] add doc --- collector/collector.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/collector/collector.go b/collector/collector.go index 177a986..8721030 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -46,6 +46,7 @@ var ( ) ) +// registerCollector registers and enables collectors based on flags provided func registerCollector(collector string, isDefaultEnabled bool, factory func() (Collector, error)) { var helpDefaultState string if isDefaultEnabled { @@ -116,6 +117,7 @@ func (collector Exporter) Collect(ch chan<- prometheus.Metric) { } } +// execute runs the collector's update function, sets duration and success metrics for collector func execute(name string, c Collector, client *client.FTLClient, ch chan<- prometheus.Metric) { begin := time.Now() err := c.update(client, ch) @@ -123,7 +125,6 @@ func execute(name string, c Collector, client *client.FTLClient, ch chan<- prome success := float64(1) if err != nil { - log.Println(name, err) success = 0 } ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, duration.Seconds(), name) From 4583c1bcd4ddba9b115cce3eb30116241eb1a6f9 Mon Sep 17 00:00:00 2001 From: Ivan Pushkin Date: Sun, 14 Jun 2020 14:39:30 +0200 Subject: [PATCH 6/6] use read* functions --- client/ftl_client.go | 26 ++++++++++++++++++ client/queries_over_time.go | 55 +++++++++++++++++++------------------ 2 files changed, 54 insertions(+), 27 deletions(-) diff --git a/client/ftl_client.go b/client/ftl_client.go index d282d7b..9cfd578 100644 --- a/client/ftl_client.go +++ b/client/ftl_client.go @@ -173,6 +173,32 @@ func readInt64(conn *net.UnixConn) (int, error) { return int(value), nil } +func readMapCount(conn *net.UnixConn) (int, error) { + var format uint8 + if err := binary.Read(conn, binary.BigEndian, &format); err != nil { + if err == io.EOF { + return 0, errEndOfInput + } + + return 0, err + } + + if format == formatEOF { + return 0, errEndOfInput + } + + if format != formatMap16 { + return 0, errInvalidFormat + } + + var value int16 + if err := binary.Read(conn, binary.BigEndian, &value); err != nil { + return 0, err + } + + return int(value), nil +} + func sendCommand(conn *net.UnixConn, command string) error { if _, err := conn.Write([]byte(command)); err != nil { return err diff --git a/client/queries_over_time.go b/client/queries_over_time.go index f27a29b..efa41e7 100644 --- a/client/queries_over_time.go +++ b/client/queries_over_time.go @@ -14,7 +14,6 @@ package client import ( - "encoding/binary" "net" ) @@ -34,45 +33,47 @@ func (client *FTLClient) GetQueriesOverTime() (*QueriesOverTime, error) { var result QueriesOverTime - var lines struct { - _ uint8 - Lines uint16 - } - if err := binary.Read(conn, binary.BigEndian, &lines); err != nil { + lines, err := readMapCount(conn) + if err != nil { return nil, err } - response := make([]struct { - Timestamp ftlInt32 - Count ftlInt32 - }, lines.Lines) - if err := binary.Read(conn, binary.BigEndian, &response); err != nil { - return nil, err - } + for i := 0; i < lines; i++ { + timestamp, err := readInt32(conn) + if err != nil { + return nil, err + } + + count, err := readInt32(conn) + if err != nil { + return nil, err + } - for _, r := range response { result.Forwarded = append(result.Forwarded, timestampCount{ - Timestamp: int(r.Timestamp.Value), - Count: int(r.Count.Value), + Timestamp: timestamp, + Count: count, }) } - if err := binary.Read(conn, binary.BigEndian, &lines); err != nil { + lines, err = readMapCount(conn) + if err != nil { return nil, err } - response = make([]struct { - Timestamp ftlInt32 - Count ftlInt32 - }, lines.Lines) - if err := binary.Read(conn, binary.BigEndian, &response); err != nil { - return nil, err - } + for i := 0; i < lines; i++ { + timestamp, err := readInt32(conn) + if err != nil { + return nil, err + } + + count, err := readInt32(conn) + if err != nil { + return nil, err + } - for _, r := range response { result.Blocked = append(result.Blocked, timestampCount{ - Timestamp: int(r.Timestamp.Value), - Count: int(r.Count.Value), + Timestamp: timestamp, + Count: count, }) }