From 42291cd5479b84f6230d6bd685038efde2d105f0 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Tue, 5 Sep 2023 15:13:35 +0300 Subject: [PATCH] Pull request 1983: 5720-wildcard-ignored-domains Updates #5720. Squashed commit of the following: commit e8093c990f15e2efc496f1a04f87360825e34e96 Merge: df5413eef 28fefaff1 Author: Stanislav Chzhen Date: Tue Sep 5 15:06:33 2023 +0300 Merge branch 'master' into 5720-wildcard-ignored-domains commit df5413eefeac2c7e34eb725db9e2908b5b2d08cb Author: Stanislav Chzhen Date: Tue Sep 5 14:49:05 2023 +0300 confmigrate: imp docs commit 1644d99b730cc7f22c9d75b8e990149d3ce5b32a Merge: 9542ee161 1e4517898 Author: Stanislav Chzhen Date: Tue Sep 5 14:23:42 2023 +0300 Merge branch 'master' into 5720-wildcard-ignored-domains commit 9542ee1616c1dd4bdb6ec9a2af79a2af3858a7e3 Author: Stanislav Chzhen Date: Tue Sep 5 12:48:48 2023 +0300 all: upd chlog commit 183f84a7f73c7bd33669bd108076f60514ca101e Author: Stanislav Chzhen Date: Fri Sep 1 17:11:31 2023 +0300 all: imp chlog commit a704325352a577a9b6652f011b82180ec3a6e095 Author: Stanislav Chzhen Date: Thu Aug 31 18:59:52 2023 +0300 all: imp code commit fe99c3b883500850399b1feb72c914ab878b3107 Merge: 7f11e9460 0182b9ec1 Author: Stanislav Chzhen Date: Thu Aug 31 18:43:09 2023 +0300 Merge branch 'master' into 5720-wildcard-ignored-domains commit 7f11e94609027ed821a125d27a1ffde03f37334a Author: Stanislav Chzhen Date: Wed Aug 30 19:57:51 2023 +0300 aghnet: add tests commit f10f9190ce1064a5d03155e8b6bba61db977897b Author: Stanislav Chzhen Date: Wed Aug 30 18:32:07 2023 +0300 all: add conf migration commit a53c14df129765366966c5230dd53aa29bdd25c5 Author: Stanislav Chzhen Date: Wed Aug 30 13:37:30 2023 +0300 all: add ignore engine --- CHANGELOG.md | 9 +- client/src/__locales/en.json | 4 +- internal/aghnet/ignore.go | 56 ++++++++ internal/aghnet/ignore_test.go | 46 +++++++ internal/confmigrate/migrations_test.go | 83 ++++++++++++ internal/confmigrate/migrator.go | 3 +- internal/confmigrate/migrator_test.go | 4 + .../TestMigrateConfig_Migrate/v27/input.yml | 123 ++++++++++++++++++ .../TestMigrateConfig_Migrate/v27/output.yml | 123 ++++++++++++++++++ internal/confmigrate/v27.go | 77 +++++++++++ internal/dnsforward/access.go | 1 + internal/home/config.go | 3 - internal/home/dns.go | 8 +- internal/querylog/http.go | 7 +- internal/querylog/qlog.go | 1 - internal/querylog/qlog_test.go | 31 ++++- internal/querylog/querylog.go | 6 +- internal/stats/http.go | 7 +- internal/stats/http_test.go | 23 ---- internal/stats/stats.go | 14 +- internal/stats/stats_test.go | 8 +- internal/stats/unit.go | 4 +- 22 files changed, 577 insertions(+), 64 deletions(-) create mode 100644 internal/aghnet/ignore.go create mode 100644 internal/aghnet/ignore_test.go create mode 100644 internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml create mode 100644 internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml create mode 100644 internal/confmigrate/v27.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a554fe6..4ae24312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ NOTE: Add new changes BELOW THIS COMMENT. ### Added +- AdBlock-style syntax support for ignored domains in logs and statistics + ([#5720]). - [`Strict-Transport-Security`][hsts] header in the HTTP API and DNS-over-HTTPS responses when HTTPS is forced ([#2998]). See [RFC 6979][rfc6797]. - UI for the schedule of the service-blocking pause ([#951]). @@ -50,7 +52,11 @@ NOTE: Add new changes BELOW THIS COMMENT. #### Configuration Changes -In this release, the schema version has changed from 24 to 26. +In this release, the schema version has changed from 24 to 27. + +- Ignore rules blocking `.` in `querylog.…` and `stats.…` have been migrated to + AdBlock syntax (`|.^`). To rollback this change, restore the rules and + change the `schema_version` back to `26`. - Filtering-related settings have been moved from `dns` section of the YAML configuration file to the new section `filtering`: @@ -156,6 +162,7 @@ In this release, the schema version has changed from 24 to 26. [#1453]: https://github.com/AdguardTeam/AdGuardHome/issues/1453 [#2998]: https://github.com/AdguardTeam/AdGuardHome/issues/2998 [#3701]: https://github.com/AdguardTeam/AdGuardHome/issues/3701 +[#5720]: https://github.com/AdguardTeam/AdGuardHome/issues/5720 [#5793]: https://github.com/AdguardTeam/AdGuardHome/issues/5793 [#5948]: https://github.com/AdguardTeam/AdGuardHome/issues/5948 [#6020]: https://github.com/AdguardTeam/AdGuardHome/issues/6020 diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index d4673e5e..d7ea54a2 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -537,8 +537,8 @@ "statistics_enable": "Enable statistics", "ignore_domains": "Ignored domains (separated by newline)", "ignore_domains_title": "Ignored domains", - "ignore_domains_desc_stats": "Queries for these domains are not written to the statistics", - "ignore_domains_desc_query": "Queries for these domains are not written to the query log", + "ignore_domains_desc_stats": "Queries matching these rules are not written to the statistics", + "ignore_domains_desc_query": "Queries matching these rules are not written to the query log", "interval_hours": "{{count}} hour", "interval_hours_plural": "{{count}} hours", "filters_configuration": "Filters configuration", diff --git a/internal/aghnet/ignore.go b/internal/aghnet/ignore.go new file mode 100644 index 00000000..69146800 --- /dev/null +++ b/internal/aghnet/ignore.go @@ -0,0 +1,56 @@ +package aghnet + +import ( + "strings" + + "github.com/AdguardTeam/urlfilter" + "github.com/AdguardTeam/urlfilter/filterlist" + "golang.org/x/exp/slices" +) + +// IgnoreEngine contains the list of rules for ignoring hostnames and matches +// them. +// +// TODO(s.chzhen): Move all urlfilter stuff to aghfilter. +type IgnoreEngine struct { + // engine is the filtering engine that can match rules for ignoring + // hostnames. + engine *urlfilter.DNSEngine + + // ignored is the list of rules for ignoring hostnames. + ignored []string +} + +// NewIgnoreEngine creates a new instance of the IgnoreEngine and stores the +// list of rules for ignoring hostnames. +func NewIgnoreEngine(ignored []string) (e *IgnoreEngine, err error) { + ruleList := &filterlist.StringRuleList{ + RulesText: strings.ToLower(strings.Join(ignored, "\n")), + IgnoreCosmetic: true, + } + ruleStorage, err := filterlist.NewRuleStorage([]filterlist.RuleList{ruleList}) + if err != nil { + return nil, err + } + + return &IgnoreEngine{ + engine: urlfilter.NewDNSEngine(ruleStorage), + ignored: ignored, + }, nil +} + +// Has returns true if IgnoreEngine matches the host. +func (e *IgnoreEngine) Has(host string) (ignore bool) { + if e == nil { + return false + } + + _, ignore = e.engine.Match(host) + + return ignore +} + +// Values returns a copy of list of rules for ignoring hostnames. +func (e *IgnoreEngine) Values() (ignored []string) { + return slices.Clone(e.ignored) +} diff --git a/internal/aghnet/ignore_test.go b/internal/aghnet/ignore_test.go new file mode 100644 index 00000000..0cf936d3 --- /dev/null +++ b/internal/aghnet/ignore_test.go @@ -0,0 +1,46 @@ +package aghnet_test + +import ( + "testing" + + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" + "github.com/stretchr/testify/require" +) + +func TestIgnoreEngine_Has(t *testing.T) { + hostnames := []string{ + "*.example.com", + "example.com", + "|.^", + } + + engine, err := aghnet.NewIgnoreEngine(hostnames) + require.NotNil(t, engine) + require.NoError(t, err) + + testCases := []struct { + name string + host string + ignore bool + }{{ + name: "basic", + host: "example.com", + ignore: true, + }, { + name: "root", + host: ".", + ignore: true, + }, { + name: "wildcard", + host: "www.example.com", + ignore: true, + }, { + name: "not_ignored", + host: "something.com", + ignore: false, + }} + + for _, tc := range testCases { + require.Equal(t, tc.ignore, engine.Has(tc.host)) + } +} diff --git a/internal/confmigrate/migrations_test.go b/internal/confmigrate/migrations_test.go index 2fdc57a9..5e9ea657 100644 --- a/internal/confmigrate/migrations_test.go +++ b/internal/confmigrate/migrations_test.go @@ -1563,3 +1563,86 @@ func TestUpgradeSchema25to26(t *testing.T) { }) } } + +func TestUpgradeSchema26to27(t *testing.T) { + const newSchemaVer = 27 + + testCases := []struct { + in yobj + want yobj + name string + }{{ + name: "empty", + in: yobj{}, + want: yobj{ + "schema_version": newSchemaVer, + }, + }, { + name: "single_dot", + in: yobj{ + "querylog": yobj{ + "ignored": yarr{ + ".", + }, + }, + "statistics": yobj{ + "ignored": yarr{ + ".", + }, + }, + }, + want: yobj{ + "querylog": yobj{ + "ignored": yarr{ + "|.^", + }, + }, + "statistics": yobj{ + "ignored": yarr{ + "|.^", + }, + }, + "schema_version": newSchemaVer, + }, + }, { + name: "mixed", + in: yobj{ + "querylog": yobj{ + "ignored": yarr{ + ".", + "example.com", + }, + }, + "statistics": yobj{ + "ignored": yarr{ + ".", + "example.org", + }, + }, + }, + want: yobj{ + "querylog": yobj{ + "ignored": yarr{ + "|.^", + "example.com", + }, + }, + "statistics": yobj{ + "ignored": yarr{ + "|.^", + "example.org", + }, + }, + "schema_version": newSchemaVer, + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := migrateTo27(tc.in) + require.NoError(t, err) + + assert.Equal(t, tc.want, tc.in) + }) + } +} diff --git a/internal/confmigrate/migrator.go b/internal/confmigrate/migrator.go index d8488929..636db690 100644 --- a/internal/confmigrate/migrator.go +++ b/internal/confmigrate/migrator.go @@ -10,7 +10,7 @@ import ( ) // LastSchemaVersion is the most recent schema version. -const LastSchemaVersion uint = 26 +const LastSchemaVersion uint = 27 // Config is a the configuration for initializing a [Migrator]. type Config struct { @@ -122,6 +122,7 @@ func (m *Migrator) upgradeConfigSchema(current, target uint, diskConf yobj) (err 23: migrateTo24, 24: migrateTo25, 25: migrateTo26, + 26: migrateTo27, } for i, migrate := range upgrades[current:target] { diff --git a/internal/confmigrate/migrator_test.go b/internal/confmigrate/migrator_test.go index 6188e7db..5cc8f3fb 100644 --- a/internal/confmigrate/migrator_test.go +++ b/internal/confmigrate/migrator_test.go @@ -181,6 +181,10 @@ func TestMigrateConfig_Migrate(t *testing.T) { yamlEqFunc: require.YAMLEq, name: "v26", targetVersion: 26, + }, { + yamlEqFunc: require.YAMLEq, + name: "v27", + targetVersion: 27, }} for _, tc := range testCases { diff --git a/internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml b/internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml new file mode 100644 index 00000000..6472343d --- /dev/null +++ b/internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/input.yml @@ -0,0 +1,123 @@ +http: + address: 127.0.0.1:3000 + session_ttl: 3h + pprof: + enabled: true + port: 6060 +users: +- name: testuser + password: testpassword +dns: + bind_hosts: + - 127.0.0.1 + port: 53 + parental_sensitivity: 0 + upstream_dns: + - tls://1.1.1.1 + - tls://1.0.0.1 + - quic://8.8.8.8:784 + bootstrap_dns: + - 8.8.8.8:53 + edns_client_subnet: + enabled: true + use_custom: false + custom_ip: "" +filtering: + filtering_enabled: true + parental_enabled: false + safebrowsing_enabled: false + safe_search: + enabled: false + bing: true + duckduckgo: true + google: true + pixabay: true + yandex: true + youtube: true + protection_enabled: true + blocked_services: + schedule: + time_zone: Local + ids: + - 500px + blocked_response_ttl: 10 +filters: +- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt + name: "" + enabled: true +- url: https://adaway.org/hosts.txt + name: AdAway + enabled: false +- url: https://hosts-file.net/ad_servers.txt + name: hpHosts - Ad and Tracking servers only + enabled: false +- url: http://www.malwaredomainlist.com/hostslist/hosts.txt + name: MalwareDomainList.com Hosts List + enabled: false +clients: + persistent: + - name: localhost + ids: + - 127.0.0.1 + - aa:aa:aa:aa:aa:aa + use_global_settings: true + use_global_blocked_services: true + filtering_enabled: false + parental_enabled: false + safebrowsing_enabled: false + safe_search: + enabled: true + bing: true + duckduckgo: true + google: true + pixabay: true + yandex: true + youtube: true + blocked_services: + schedule: + time_zone: Local + ids: + - 500px + runtime_sources: + whois: true + arp: true + rdns: true + dhcp: true + hosts: true +dhcp: + enabled: false + interface_name: vboxnet0 + local_domain_name: local + dhcpv4: + gateway_ip: 192.168.0.1 + subnet_mask: 255.255.255.0 + range_start: 192.168.0.10 + range_end: 192.168.0.250 + lease_duration: 1234 + icmp_timeout_msec: 10 +schema_version: 26 +user_rules: [] +querylog: + enabled: true + file_enabled: true + interval: 720h + size_memory: 1000 + ignored: + - '.' +statistics: + enabled: true + interval: 240h + ignored: + - '.' +os: + group: '' + rlimit_nofile: 123 + user: '' +log: + file: "" + max_backups: 0 + max_size: 100 + max_age: 3 + compress: true + local_time: false + verbose: true diff --git a/internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml b/internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml new file mode 100644 index 00000000..e25b02c7 --- /dev/null +++ b/internal/confmigrate/testdata/TestMigrateConfig_Migrate/v27/output.yml @@ -0,0 +1,123 @@ +http: + address: 127.0.0.1:3000 + session_ttl: 3h + pprof: + enabled: true + port: 6060 +users: +- name: testuser + password: testpassword +dns: + bind_hosts: + - 127.0.0.1 + port: 53 + parental_sensitivity: 0 + upstream_dns: + - tls://1.1.1.1 + - tls://1.0.0.1 + - quic://8.8.8.8:784 + bootstrap_dns: + - 8.8.8.8:53 + edns_client_subnet: + enabled: true + use_custom: false + custom_ip: "" +filtering: + filtering_enabled: true + parental_enabled: false + safebrowsing_enabled: false + safe_search: + enabled: false + bing: true + duckduckgo: true + google: true + pixabay: true + yandex: true + youtube: true + protection_enabled: true + blocked_services: + schedule: + time_zone: Local + ids: + - 500px + blocked_response_ttl: 10 +filters: +- url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt + name: "" + enabled: true +- url: https://adaway.org/hosts.txt + name: AdAway + enabled: false +- url: https://hosts-file.net/ad_servers.txt + name: hpHosts - Ad and Tracking servers only + enabled: false +- url: http://www.malwaredomainlist.com/hostslist/hosts.txt + name: MalwareDomainList.com Hosts List + enabled: false +clients: + persistent: + - name: localhost + ids: + - 127.0.0.1 + - aa:aa:aa:aa:aa:aa + use_global_settings: true + use_global_blocked_services: true + filtering_enabled: false + parental_enabled: false + safebrowsing_enabled: false + safe_search: + enabled: true + bing: true + duckduckgo: true + google: true + pixabay: true + yandex: true + youtube: true + blocked_services: + schedule: + time_zone: Local + ids: + - 500px + runtime_sources: + whois: true + arp: true + rdns: true + dhcp: true + hosts: true +dhcp: + enabled: false + interface_name: vboxnet0 + local_domain_name: local + dhcpv4: + gateway_ip: 192.168.0.1 + subnet_mask: 255.255.255.0 + range_start: 192.168.0.10 + range_end: 192.168.0.250 + lease_duration: 1234 + icmp_timeout_msec: 10 +schema_version: 27 +user_rules: [] +querylog: + enabled: true + file_enabled: true + interval: 720h + size_memory: 1000 + ignored: + - '|.^' +statistics: + enabled: true + interval: 240h + ignored: + - '|.^' +os: + group: '' + rlimit_nofile: 123 + user: '' +log: + file: "" + max_backups: 0 + max_size: 100 + max_age: 3 + compress: true + local_time: false + verbose: true diff --git a/internal/confmigrate/v27.go b/internal/confmigrate/v27.go new file mode 100644 index 00000000..ddba8671 --- /dev/null +++ b/internal/confmigrate/v27.go @@ -0,0 +1,77 @@ +package confmigrate + +// migrateTo27 performs the following changes: +// +// # BEFORE: +// 'querylog': +// 'ignored': +// - '.' +// - # … +// # … +// 'statistics': +// 'ignored': +// - '.' +// - # … +// # … +// # … +// +// # AFTER: +// 'querylog': +// 'ignored': +// - '|.^' +// - # … +// # … +// 'statistics': +// 'ignored': +// - '|.^' +// - # … +// # … +// # … +func migrateTo27(diskConf yobj) (err error) { + diskConf["schema_version"] = 27 + + keys := []string{"querylog", "statistics"} + for _, k := range keys { + err = replaceDot(diskConf, k) + if err != nil { + return err + } + } + + return nil +} + +// replaceDot replaces rules blocking root domain "." with AdBlock style syntax +// "|.^". +func replaceDot(diskConf yobj, key string) (err error) { + var obj yobj + var ok bool + obj, ok, err = fieldVal[yobj](diskConf, key) + if err != nil { + return err + } else if !ok { + return nil + } + + var ignored yarr + ignored, ok, err = fieldVal[yarr](obj, "ignored") + if err != nil { + return err + } else if !ok { + return nil + } + + for i, hostVal := range ignored { + var host string + host, ok = hostVal.(string) + if !ok { + continue + } + + if host == "." { + ignored[i] = "|.^" + } + } + + return nil +} diff --git a/internal/dnsforward/access.go b/internal/dnsforward/access.go index ba9300c2..2c0d0dc4 100644 --- a/internal/dnsforward/access.go +++ b/internal/dnsforward/access.go @@ -28,6 +28,7 @@ type accessManager struct { allowedClientIDs *stringutil.Set blockedClientIDs *stringutil.Set + // TODO(s.chzhen): Use [aghnet.IgnoreEngine]. blockedHostsEng *urlfilter.DNSEngine // TODO(a.garipov): Create a type for a set of IP networks. diff --git a/internal/home/config.go b/internal/home/config.go index 233b4189..0be84753 100644 --- a/internal/home/config.go +++ b/internal/home/config.go @@ -22,7 +22,6 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/timeutil" "github.com/google/renameio/v2/maybe" - "golang.org/x/exp/slices" yaml "gopkg.in/yaml.v3" ) @@ -594,7 +593,6 @@ func (c *configuration) write() (err error) { config.Stats.Interval = timeutil.Duration{Duration: statsConf.Limit} config.Stats.Enabled = statsConf.Enabled config.Stats.Ignored = statsConf.Ignored.Values() - slices.Sort(config.Stats.Ignored) } if Context.queryLog != nil { @@ -606,7 +604,6 @@ func (c *configuration) write() (err error) { config.QueryLog.Interval = timeutil.Duration{Duration: dc.RotationIvl} config.QueryLog.MemSize = dc.MemSize config.QueryLog.Ignored = dc.Ignored.Values() - slices.Sort(config.Stats.Ignored) } if Context.filters != nil { diff --git a/internal/home/dns.go b/internal/home/dns.go index 309c0111..59f3a71e 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -59,12 +59,12 @@ func initDNS() (err error) { ShouldCountClient: Context.clients.shouldCountClient, } - set, err := aghnet.NewDomainNameSet(config.Stats.Ignored) + engine, err := aghnet.NewIgnoreEngine(config.Stats.Ignored) if err != nil { return fmt.Errorf("statistics: ignored list: %w", err) } - statsConf.Ignored = set + statsConf.Ignored = engine Context.stats, err = stats.New(statsConf) if err != nil { return fmt.Errorf("init stats: %w", err) @@ -83,12 +83,12 @@ func initDNS() (err error) { FileEnabled: config.QueryLog.FileEnabled, } - set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored) + engine, err = aghnet.NewIgnoreEngine(config.QueryLog.Ignored) if err != nil { return fmt.Errorf("querylog: ignored list: %w", err) } - conf.Ignored = set + conf.Ignored = engine Context.queryLog, err = querylog.New(conf) if err != nil { return fmt.Errorf("init querylog: %w", err) diff --git a/internal/querylog/http.go b/internal/querylog/http.go index 7a5c24a9..3e71af57 100644 --- a/internal/querylog/http.go +++ b/internal/querylog/http.go @@ -17,7 +17,6 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/timeutil" - "golang.org/x/exp/slices" "golang.org/x/net/idna" ) @@ -141,8 +140,6 @@ func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Reques } }() - slices.Sort(resp.Ignored) - aghhttp.WriteJSONResponseOK(w, r, resp) } @@ -224,7 +221,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques return } - set, err := aghnet.NewDomainNameSet(newConf.Ignored) + engine, err := aghnet.NewIgnoreEngine(newConf.Ignored) if err != nil { aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err) @@ -258,7 +255,7 @@ func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Reques conf := *l.conf - conf.Ignored = set + conf.Ignored = engine conf.RotationIvl = ivl conf.Enabled = newConf.Enabled == aghalg.NBTrue diff --git a/internal/querylog/qlog.go b/internal/querylog/qlog.go index be9f1fee..67363510 100644 --- a/internal/querylog/qlog.go +++ b/internal/querylog/qlog.go @@ -127,7 +127,6 @@ func (l *queryLog) WriteDiskConfig(c *Config) { defer l.confMu.RUnlock() *c = *l.conf - c.Ignored = l.conf.Ignored.Clone() } // Clear memory buffer and remove log files diff --git a/internal/querylog/qlog_test.go b/internal/querylog/qlog_test.go index db616ba8..0b2a476b 100644 --- a/internal/querylog/qlog_test.go +++ b/internal/querylog/qlog_test.go @@ -5,8 +5,8 @@ import ( "net" "testing" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/filtering" - "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/testutil" "github.com/AdguardTeam/golibs/timeutil" "github.com/miekg/dns" @@ -256,10 +256,21 @@ func TestQueryLogFileDisabled(t *testing.T) { func TestQueryLogShouldLog(t *testing.T) { const ( - ignored1 = "ignor.ed" - ignored2 = "ignored.to" + ignored1 = "ignor.ed" + ignored2 = "ignored.to" + ignoredWildcard = "*.ignored.com" + ignoredRoot = "|.^" ) - set := stringutil.NewSet(ignored1, ignored2) + + ignored := []string{ + ignored1, + ignored2, + ignoredWildcard, + ignoredRoot, + } + + engine, err := aghnet.NewIgnoreEngine(ignored) + require.NoError(t, err) findClient := func(ids []string) (c *Client, err error) { log := ids[0] == "no_log" @@ -268,7 +279,7 @@ func TestQueryLogShouldLog(t *testing.T) { } l, err := newQueryLog(Config{ - Ignored: set, + Ignored: engine, Enabled: true, RotationIvl: timeutil.Day, MemSize: 100, @@ -297,6 +308,16 @@ func TestQueryLogShouldLog(t *testing.T) { host: ignored2, ids: []string{"whatever"}, wantLog: false, + }, { + name: "no_log_ignored_wildcard", + host: "www.ignored.com", + ids: []string{"whatever"}, + wantLog: false, + }, { + name: "no_log_ignored_root", + host: ".", + ids: []string{"whatever"}, + wantLog: false, }, { name: "no_log_client_ignore", host: "example.com", diff --git a/internal/querylog/querylog.go b/internal/querylog/querylog.go index 603b6cf6..7a09f40f 100644 --- a/internal/querylog/querylog.go +++ b/internal/querylog/querylog.go @@ -11,7 +11,6 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/stringutil" "github.com/miekg/dns" ) @@ -36,8 +35,9 @@ type QueryLog interface { // // Do not alter any fields of this structure after using it. type Config struct { - // Ignored is the list of host names, which should not be written to log. - Ignored *stringutil.Set + // Ignored contains the list of host names, which should not be written to + // log, and matches them. + Ignored *aghnet.IgnoreEngine // Anonymizer processes the IP addresses to anonymize those if needed. Anonymizer *aghnet.IPMut diff --git a/internal/stats/http.go b/internal/stats/http.go index 3eee5bbb..faec0d14 100644 --- a/internal/stats/http.go +++ b/internal/stats/http.go @@ -12,7 +12,6 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/timeutil" - "golang.org/x/exp/slices" ) // topAddrs is an alias for the types of the TopFoo fields of statsResponse. @@ -140,8 +139,6 @@ func (s *StatsCtx) handleGetStatsConfig(w http.ResponseWriter, r *http.Request) } }() - slices.Sort(resp.Ignored) - aghhttp.WriteJSONResponseOK(w, r, resp) } @@ -184,7 +181,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request) return } - set, err := aghnet.NewDomainNameSet(reqData.Ignored) + engine, err := aghnet.NewIgnoreEngine(reqData.Ignored) if err != nil { aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err) @@ -210,7 +207,7 @@ func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request) s.confMu.Lock() defer s.confMu.Unlock() - s.ignored = set + s.ignored = engine s.limit = ivl s.enabled = reqData.Enabled == aghalg.NBTrue } diff --git a/internal/stats/http_test.go b/internal/stats/http_test.go index 69e75d6b..7e358a03 100644 --- a/internal/stats/http_test.go +++ b/internal/stats/http_test.go @@ -75,29 +75,6 @@ func TestHandleStatsConfig(t *testing.T) { }, wantCode: http.StatusOK, wantErr: "", - }, { - name: "ignored_duplicate", - body: getConfigResp{ - Enabled: aghalg.NBTrue, - Interval: float64(minIvl.Milliseconds()), - Ignored: []string{ - "ignor.ed", - "ignor.ed", - }, - }, - wantCode: http.StatusUnprocessableEntity, - wantErr: "ignored: duplicate hostname \"ignor.ed\" at index 1\n", - }, { - name: "ignored_empty", - body: getConfigResp{ - Enabled: aghalg.NBTrue, - Interval: float64(minIvl.Milliseconds()), - Ignored: []string{ - "", - }, - }, - wantCode: http.StatusUnprocessableEntity, - wantErr: "ignored: at index 0: hostname is empty\n", }, { name: "enabled_is_null", body: getConfigResp{ diff --git a/internal/stats/stats.go b/internal/stats/stats.go index 4090f07b..e9533ca2 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -12,9 +12,9 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" - "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/timeutil" "go.etcd.io/bbolt" ) @@ -58,8 +58,9 @@ type Config struct { // endpoints. HTTPRegister aghhttp.RegisterFunc - // Ignored is the list of host names, which should not be counted. - Ignored *stringutil.Set + // Ignored contains the list of host names, which should not be counted, + // and matches them. + Ignored *aghnet.IgnoreEngine // Filename is the name of the database file. Filename string @@ -117,8 +118,9 @@ type StatsCtx struct { // confMu protects ignored, limit, and enabled. confMu *sync.RWMutex - // ignored is the list of host names, which should not be counted. - ignored *stringutil.Set + // ignored contains the list of host names, which should not be counted, + // and matches them. + ignored *aghnet.IgnoreEngine // shouldCountClient returns client's ignore setting. shouldCountClient func([]string) bool @@ -289,7 +291,7 @@ func (s *StatsCtx) WriteDiskConfig(dc *Config) { s.confMu.RLock() defer s.confMu.RUnlock() - dc.Ignored = s.ignored.Clone() + dc.Ignored = s.ignored dc.Limit = s.limit dc.Enabled = s.enabled } diff --git a/internal/stats/stats_test.go b/internal/stats/stats_test.go index bcfdc643..004e9513 100644 --- a/internal/stats/stats_test.go +++ b/internal/stats/stats_test.go @@ -11,9 +11,9 @@ import ( "testing" "time" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/stats" "github.com/AdguardTeam/golibs/netutil" - "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/golibs/testutil" "github.com/AdguardTeam/golibs/timeutil" "github.com/miekg/dns" @@ -215,13 +215,15 @@ func TestShouldCount(t *testing.T) { ignored1 = "ignor.ed" ignored2 = "ignored.to" ) - set := stringutil.NewSet(ignored1, ignored2) + ignored := []string{ignored1, ignored2} + engine, err := aghnet.NewIgnoreEngine(ignored) + require.NoError(t, err) s, err := stats.New(stats.Config{ Enabled: true, Filename: filepath.Join(t.TempDir(), "stats.db"), Limit: timeutil.Day, - Ignored: set, + Ignored: engine, ShouldCountClient: func(ids []string) (a bool) { return ids[0] != "no_count" }, diff --git a/internal/stats/unit.go b/internal/stats/unit.go index b9969683..31fc3984 100644 --- a/internal/stats/unit.go +++ b/internal/stats/unit.go @@ -7,9 +7,9 @@ import ( "fmt" "time" + "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" - "github.com/AdguardTeam/golibs/stringutil" "go.etcd.io/bbolt" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -370,7 +370,7 @@ type pairsGetter func(u *unitDB) (pairs []countPair) // topsCollector collects statistics about highest values from the given *unitDB // slice using pg to retrieve data. -func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 { +func topsCollector(units []*unitDB, max int, ignored *aghnet.IgnoreEngine, pg pairsGetter) []map[string]uint64 { m := map[string]uint64{} for _, u := range units { for _, cp := range pg(u) {