8000 issues/453 introduced metrics etldplusone_total and etldplusone_top by kabenin · Pull Request #465 · dmachard/DNS-collector · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

issues/453 introduced metrics etldplusone_total and etldplusone_top #465

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Nov 21, 2023
Merged

issues/453 introduced metrics etldplusone_total and etldplusone_top #465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
39ddafb
issues/453 introduced metrics etldplusone_total and etldplusone_top
kabenin Nov 13, 2023
642b845
Merge branch 'upstream' into fix-for-453
kabenin Nov 17, 2023
3dd3283
Addressed some linting issues
kabenin Nov 17, 2023
0671a46
ran gofmt -s
kabenin Nov 17, 2023
4bfed2a
Fixed linting findings
kabenin Nov 17, 2023
87e5ac9
Addressed linting issues
kabenin Nov 17, 2023
dafb27c
Addressed linting issues
kabenin Nov 17, 2023
80bbdfa
Merge branch 'upstream' into fix-for-453
kabenin Nov 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 95 additions & 55 deletions loggers/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,30 +93,32 @@ type PrometheusCountersSet struct {
prom *Prometheus

// Counters
requesters map[string]int // Requests number made by a specific requestor
domains map[string]int // Requests number made to find out about a specific domain
nxdomains map[string]int // Requests number ended up in NXDOMAIN
sfdomains map[string]int // Requests number ended up in SERVFAIL
tlds map[string]int // Requests number for a specific TLD
suspicious map[string]int // Requests number for a specific name that looked suspicious
evicted map[string]int // Requests number for a specific name that timed out

epsCounters EpsCounters
topRequesters *topmap.TopMap
topEvicted *topmap.TopMap
topSfDomains *topmap.TopMap
topDomains *topmap.TopMap
topNxDomains *topmap.TopMap
topTlds *topmap.TopMap
topSuspicious *topmap.TopMap
requesters map[string]int // Requests number made by a specific requestor
domains map[string]int // Requests number made to find out about a specific domain
nxdomains map[string]int // Requests number ended up in NXDOMAIN
sfdomains map[string]int // Requests number ended up in SERVFAIL
tlds map[string]int // Requests number for a specific TLD
etldplusone map[string]int // Requests number for a specific eTLD+1
suspicious map[string]int // Requests number for a specific name that looked suspicious
evicted map[string]int // Requests number for a specific name that timed out

epsCounters EpsCounters
topRequesters *topmap.TopMap
topEvicted *topmap.TopMap
topSfDomains *topmap.TopMap
topDomains *topmap.TopMap
topNxDomains *topmap.TopMap
topTlds *topmap.TopMap
topETLDPlusOne *topmap.TopMap
topSuspicious *topmap.TopMap

labels prometheus.Labels // Do we really need to keep that map outside of registration?
sync.Mutex // Each PrometheusCountersSet locks independently
}

// PromCounterCatalogueContainer is the implementation of PrometheusCountersCatalogue interface
// That maps a single label into other Containers or CounterSet
// The 'chain' of nested Containers keep track of label_names requested by the config
// The 'chain' of nested Containers keep track of labelNames requested by the config
// to figure out whether nested Container should be created, or, if all labels but the last one
// already considered at the upper levels, it is time to create individual CounterSet
type PromCounterCatalogueContainer struct {
Expand All @@ -132,8 +134,8 @@ type PromCounterCatalogueContainer struct {

// This is the unique set of label-value pairs for this catalogue element.
// The topmost Catalog has it empty, when it creates a new entry it provides the pair of
// label_names[0]->selector(message) to the constructor. Lower levels get these pair
// collected. Ultimately, when all label names in label_names is exausted, Catalogue creates
// labelNames[0]->selector(message) to the constructor. Lower levels get these pair
// collected. Ultimately, when all label names in labelNames is exausted, Catalogue creates
// an instance of newPrometheusCounterSet and provides it with labels map to properly wrap
// in Prometheus registry.
// The goal is to separate label/values pairs construction and individual counters collection
Expand Down Expand Up @@ -178,21 +180,23 @@ type Prometheus struct {
counters *PromCounterCatalogueContainer

// All metrics use these descriptions when regestering
gaugeTopDomains *prometheus.Desc
gaugeTopNxDomains *prometheus.Desc
gaugeTopSfDomains *prometheus.Desc
gaugeTopRequesters *prometheus.Desc
gaugeTopTlds *prometheus.Desc
gaugeTopSuspicious *prometheus.Desc
gaugeTopEvicted *prometheus.Desc

counterDomains *prometheus.Desc
counterDomainsNx *prometheus.Desc
counterDomainsSf *prometheus.Desc
counterRequesters *prometheus.Desc
counterTlds *prometheus.Desc
counterSuspicious *prometheus.Desc
counterEvicted *prometheus.Desc
gaugeTopDomains *prometheus.Desc
gaugeTopNxDomains *prometheus.Desc
gaugeTopSfDomains *prometheus.Desc
gaugeTopRequesters *prometheus.Desc
gaugeTopTlds *prometheus.Desc
gaugeTopETldsPlusOne *prometheus.Desc
gaugeTopSuspicious *prometheus.Desc
gaugeTopEvicted *prometheus.Desc

counterDomains *prometheus.Desc
counterDomainsNx *prometheus.Desc
counterDomainsSf *prometheus.Desc
counterRequesters *prometheus.Desc
counterTlds *prometheus.Desc
counterETldPlusOne *prometheus.Desc
counterSuspicious *prometheus.Desc
counterEvicted *prometheus.Desc

gaugeEps *prometheus.Desc
gaugeEpsMax *prometheus.Desc
Expand Down Expand Up @@ -229,15 +233,16 @@ type Prometheus struct {

func newPrometheusCounterSet(p *Prometheus, labels prometheus.Labels) *PrometheusCountersSet {
pcs := &PrometheusCountersSet{
prom: p,
labels: labels,
requesters: make(map[string]int),
domains: make(map[string]int),
nxdomains: make(map[string]int),
sfdomains: make(map[string]int),
tlds: make(map[string]int),
suspicious: make(map[string]int),
evicted: make(map[string]int),
prom: p,
labels: labels,
requesters: make(map[string]int),
domains: make(map[string]int),
nxdomains: make(map[string]int),
sfdomains: make(map[string]int),
tlds: make(map[string]int),
etldplusone: make(map[string]int),
suspicious: make(map[string]int),
evicted: make(map[string]int),

epsCounters: EpsCounters{
TotalRcodes: make(map[string]float64),
Expand All @@ -246,13 +251,14 @@ func newPrometheusCounterSet(p *Prometheus, labels prometheus.Labels) *Prometheu
TotalIPProtocol: make(map[string]float64),
},

topRequesters: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topEvicted: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topSfDomains: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topDomains: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topNxDomains: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topTlds: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topSuspicious: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topRequesters: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topEvicted: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topSfDomains: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topDomains: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topNxDomains: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topTlds: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topETLDPlusOne: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
topSuspicious: topmap.NewTopMap(p.config.Loggers.Prometheus.TopN),
}

prometheus.WrapRegistererWith(labels, p.promRegistry).MustRegister(pcs)
Expand All @@ -274,6 +280,7 @@ func (c *PrometheusCountersSet) Describe(ch chan<- *prometheus.Desc) {
ch <- c.prom.gaugeTopSfDomains
ch <- c.prom.gaugeTopRequesters
ch <- c.prom.gaugeTopTlds
ch <- c.prom.gaugeTopETldsPlusOne
ch <- c.prom.gaugeTopSuspicious
ch <- c.prom.gaugeTopEvicted

Expand All @@ -283,6 +290,7 @@ func (c *PrometheusCountersSet) Describe(ch chan<- *prometheus.Desc) {
ch <- c.prom.counterDomainsSf
ch <- c.prom.counterRequesters
ch <- c.prom.counterTlds
ch <- c.prom.counterETldPlusOne
ch <- c.prom.counterSuspicious
ch <- c.prom.counterEvicted

Expand Down Expand Up @@ -338,7 +346,6 @@ func (c *PrometheusCountersSet) Record(dm dnsutils.DNSMessage) {
} else {
c.sfdomains[dm.DNS.Qname] += 1
}

c.topSfDomains.Record(dm.DNS.Qname, c.sfdomains[dm.DNS.Qname])

case dnsutils.DNSRcodeNXDomain:
Expand Down Expand Up @@ -370,6 +377,18 @@ func (c *PrometheusCountersSet) Record(dm dnsutils.DNSMessage) {
}
}

// count TLD+1 if it is set
if dm.PublicSuffix != nil {
if dm.PublicSuffix.QnameEffectiveTLDPlusOne != "-" {
if _, exists := c.tlds[dm.PublicSuffix.QnameEffectiveTLDPlusOne]; !exists {
c.etldplusone[dm.PublicSuffix.QnameEffectiveTLDPlusOne] = 1
} else {
c.etldplusone[dm.PublicSuffix.QnameEffectiveTLDPlusOne] += 1
}
c.topETLDPlusOne.Record(dm.PublicSuffix.QnameEffectiveTLDPlusOne, c.etldplusone[dm.PublicSuffix.QnameEffectiveTLDPlusOne])
}
}

// suspicious domains
if dm.Suspicious != nil {
if dm.Suspicious.Score > 0.0 {
Expand Down Expand Up @@ -485,6 +504,10 @@ func (c *PrometheusCountersSet) Collect(ch chan<- prometheus.Metric) {
float64(len(c.tlds)),
)

ch <- prometheus.MustNewConstMetric(c.prom.counterETldPlusOne, prometheus.CounterValue,
float64(len(c.etldplusone)),
)

// Count number of unique suspicious names
ch <- prometheus.MustNewConstMetric(c.prom.counterSuspicious, prometheus.CounterValue,
float64(len(c.suspicious)),
Expand Down Expand Up @@ -519,6 +542,11 @@ func (c *PrometheusCountersSet) Collect(ch chan<- prometheus.Metric) {
float64(r.Hit), r.Name)
}

for _, r := range c.topETLDPlusOne.Get() {
ch <- prometheus.MustNewConstMetric(c.prom.gaugeTopETldsPlusOne, prometheus.GaugeValue,
float64(r.Hit), r.Name)
}

for _, r := range c.topSuspicious.Get() {
ch <- prometheus.MustNewConstMetric(c.prom.gaugeTopSuspicious, prometheus.GaugeValue,
float64(r.Hit), r.Name)
Expand Down Expand Up @@ -617,7 +645,7 @@ func (c *PrometheusCountersSet) ComputeEventsPerSecond() {

func NewPromCounterCatalogueContainer(p *Prometheus, selLabels []string, l map[string]string) *PromCounterCatalogueContainer {
if len(selLabels) == 0 {
panic("Cannot create a new PromCounterCatalogueContainer with empty list of sel_labels")
panic("Cannot create a new PromCounterCatalogueContainer with empty list of selLabels")
}
sel, ok := catalogueSelectors[selLabels[0]]
if !ok {
Expand Down Expand Up @@ -707,15 +735,15 @@ func (c *PromCounterCatalogueContainer) GetCountersSet(dm *dnsutils.DNSMessage)

// This function checks the configuration, to determine which label dimentions were requested
// by configuration, and returns correct implementation of Catalogue.
func CreateSystemCatalogue(prom *Prometheus) ([]string, *PromCounterCatalogueContainer) {
lbls := prom.config.Loggers.Prometheus.LabelsList
func CreateSystemCatalogue(o *Prometheus) ([]string, *PromCounterCatalogueContainer) {
lbls := o.config.Loggers.Prometheus.LabelsList

// Default configuration is label with stream_id, to keep us backward compatible
if len(lbls) == 0 {
lbls = []string{"stream_id"}
}
return lbls, NewPromCounterCatalogueContainer(
prom,
o,
lbls,
make(map[string]string),
)
Expand Down Expand Up @@ -823,6 +851,12 @@ func (c *Prometheus) InitProm() {
"Number of hit per tld - topN",
[]string{"suffix"}, nil,
)
// etldplusone_top_total
c.gaugeTopETldsPlusOne = prometheus.NewDesc(
fmt.Sprintf("%s_top_etldplusone", promPrefix),
"Number of hit per eTLD+1 - topN",
[]string{"suffix"}, nil,
)

c.gaugeTopSuspicious = prometheus.NewDesc(
fmt.Sprintf("%s_top_suspicious", promPrefix),
Expand Down Expand Up @@ -879,6 +913,12 @@ func (c *Prometheus) InitProm() {
nil, nil,
)

c.counterETldPlusOne = prometheus.NewDesc(
fmt.Sprintf("%s_etldplusone_total", promPrefix),
"The total number of tld per stream identity",
nil, nil,
)

c.counterSuspicious = prometheus.NewDesc(
fmt.Sprintf("%s_suspicious_total", promPrefix),
"The total number of suspicious domain per stream identity",
Expand Down
29 changes: 28 additions & 1 deletion loggers/prometheus_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func getMetricsTestCase(config *dnsutils.Config, labels map[string]string) func(
nxRecord.NetworkInfo.Family = IPv4
nxRecord.DNS.Length = 123

// nx_record.PublicSuffix = &dnsutils.TransformPublicSuffix{
// nxRecord.PublicSuffix = &dnsutils.TransformPublicSuffix{
// QnamePublicSuffix: "faketld1",
// }
g.Record(nxRecord)
Expand Down Expand Up @@ -234,6 +234,33 @@ func TestPrometheus_ConfirmDifferentResolvers(t *testing.T) {
ensureMetricValue(t, mf, "dnscollector_bytes_total", map[string]string{"resolver": "10.10.10.10"}, 999)
}

func TestPrometheus_etldplusone(t *testing.T) {
config := dnsutils.GetFakeConfig()
config.Loggers.Prometheus.LabelsList = []string{"stream_id"}
g := NewPrometheus(config, logger.New(false), "test")

noErrorRecord := dnsutils.GetFakeDNSMessage()
noErrorRecord.DNS.Type = dnsutils.DNSQuery
noErrorRecord.PublicSuffix = &dnsutils.TransformPublicSuffix{
QnamePublicSuffix: "co.uk",
QnameEffectiveTLDPlusOne: "domain.co.uk",
}
noErrorRecord.DNS.Flags.AA = true
noErrorRecord.DNSTap.Latency = 0.05
noErrorRecord.NetworkInfo.Protocol = UDP
noErrorRecord.NetworkInfo.Family = IPv4
noErrorRecord.DNS.Length = 123

g.Record(noErrorRecord)
// The next would be a different TLD+1
noErrorRecord.PublicSuffix.QnameEffectiveTLDPlusOne = "anotherdomain.co.uk"
g.Record(noErrorRecord)

mf := getMetrics(g, t)
ensureMetricValue(t, mf, "dnscollector_etldplusone_total", map[string]string{"stream_id": "collector"}, 2)
ensureMetricValue(t, mf, "dnscollector_etldplusone_top", map[string]string{"stream_id": "collector", "suffix": "anotherdomain.co.uk"}, 1)
}

func ensureMetricValue(t *testing.T, mf map[string]*dto.MetricFamily, name string, labels map[string]string, value float64) bool {
m, found := mf[name]
if !found {
Expand Down
2 changes: 1 addition & 1 deletion transformers/normalize_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func TestNormalize_AddTldPlusOne(t *testing.T) {

psl.GetEffectiveTldPlusOne(&dm)
if dm.PublicSuffix.QnameEffectiveTLDPlusOne != tc.want {
t.Errorf("Bad TLD, got: %s, expected: com", dm.PublicSuffix.QnameEffectiveTLDPlusOne)
t.Errorf("Bad TLD, got: %s, expected: %s", dm.PublicSuffix.QnameEffectiveTLDPlusOne, tc.want)

}
})
Expand Down
0