From 4f50519b9f11dd03f7f8e61985f4567665885f9a Mon Sep 17 00:00:00 2001 From: Eugene Burkov Date: Fri, 26 Nov 2021 18:25:43 +0300 Subject: [PATCH] Pull request: 3846 filter lists ids Merge in DNS/adguard-home from 3846-list-ids to master Closes #3846. Squashed commit of the following: commit 02a12fc27bc5d3cf1a17fd43c6f05e2c389bd71d Author: Ildar Kamalov Date: Fri Nov 26 16:58:13 2021 +0300 client: fix name commit 6220570e6e9c968f0d3fa9d02c12099ce66aaaad Author: Ildar Kamalov Date: Fri Nov 26 16:46:54 2021 +0300 client: handle special filter ids commit dcdeb2d7f4500aab6ce5ffe642bdaacf291f5951 Author: Eugene Burkov Date: Fri Nov 26 15:52:06 2021 +0300 all: mv constants, imp config commit 8ceb4a2b351e595929d8b2af564c6d0267afa230 Author: Eugene Burkov Date: Fri Nov 26 15:04:36 2021 +0300 all: fix custom list id, log changes commit acb8b456e7f41a556da34cf10647eecee058beec Author: Eugene Burkov Date: Thu Nov 25 20:04:37 2021 +0300 all: rm global ctx, add const flt ids --- client/src/__locales/en.json | 4 +- client/src/helpers/constants.js | 10 +- client/src/helpers/helpers.js | 37 ++++-- internal/aghnet/hostscontainer.go | 17 ++- internal/aghnet/hostscontainer_test.go | 16 ++- internal/dnsforward/dnsforward_test.go | 2 +- internal/filtering/blocked.go | 2 +- internal/filtering/dnsrewrite_test.go | 2 +- internal/filtering/filtering.go | 151 ++++++++++++------------ internal/filtering/filtering_test.go | 57 +++++---- internal/filtering/rewrites_test.go | 8 +- internal/filtering/safebrowsing.go | 10 +- internal/filtering/safebrowsing_test.go | 14 +-- internal/filtering/safesearch.go | 10 +- internal/home/filter.go | 1 + internal/home/home.go | 1 + openapi/CHANGELOG.md | 15 +++ 17 files changed, 211 insertions(+), 146 deletions(-) diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index d63552f2..9caf8845 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -625,5 +625,7 @@ "filter_allowlist": "WARNING: This action also will exclude the rule \"{{disallowed_rule}}\" from the list of allowed clients.", "last_rule_in_allowlist": "Cannot disallow this client because excluding the rule \"{{disallowed_rule}}\" will DISABLE \"Allowed clients\" list.", "experimental": "Experimental", - "use_saved_key": "Use the previously saved key" + "use_saved_key": "Use the previously saved key", + "parental_control": "Parental control", + "safe_browsing": "Safe browsing" } diff --git a/client/src/helpers/constants.js b/client/src/helpers/constants.js index 0109e030..ee0fd779 100644 --- a/client/src/helpers/constants.js +++ b/client/src/helpers/constants.js @@ -528,8 +528,14 @@ export const DETAILED_DATE_FORMAT_OPTIONS = { month: 'long', }; -export const CUSTOM_FILTERING_RULES_ID = 0; -export const SYSTEM_HOSTS_FILTER_ID = -1; +export const SPECIAL_FILTER_ID = { + CUSTOM_FILTERING_RULES: 0, + SYSTEM_HOSTS: -1, + BLOCKED_SERVICES: -2, + PARENTAL: -3, + SAFE_BROWSING: -4, + SAFE_SEARCH: -5, +}; export const BLOCK_ACTIONS = { BLOCK: 'block', diff --git a/client/src/helpers/helpers.js b/client/src/helpers/helpers.js index f5271106..4c9803ae 100644 --- a/client/src/helpers/helpers.js +++ b/client/src/helpers/helpers.js @@ -13,7 +13,6 @@ import { ADDRESS_TYPES, CHECK_TIMEOUT, COMMENT_LINE_DEFAULT_TOKEN, - CUSTOM_FILTERING_RULES_ID, DEFAULT_DATE_FORMAT_OPTIONS, DEFAULT_LANGUAGE, DEFAULT_TIME_FORMAT, @@ -26,7 +25,7 @@ import { STANDARD_DNS_PORT, STANDARD_HTTPS_PORT, STANDARD_WEB_PORT, - SYSTEM_HOSTS_FILTER_ID, + SPECIAL_FILTER_ID, } from './constants'; /** @@ -774,6 +773,30 @@ export const sortIp = (a, b) => { } }; + +/** + * @param {number} filterId + * @returns {string} + */ +export const getSpecialFilterName = (filterId) => { + switch (filterId) { + case SPECIAL_FILTER_ID.CUSTOM_FILTERING_RULES: + return i18n.t('custom_filter_rules'); + case SPECIAL_FILTER_ID.SYSTEM_HOSTS: + return i18n.t('system_host_files'); + case SPECIAL_FILTER_ID.BLOCKED_SERVICES: + return i18n.t('blocked_services'); + case SPECIAL_FILTER_ID.PARENTAL: + return i18n.t('parental_control'); + case SPECIAL_FILTER_ID.SAFE_BROWSING: + return i18n.t('safe_browsing'); + case SPECIAL_FILTER_ID.SAFE_SEARCH: + return i18n.t('safe_search'); + default: + return i18n.t('unknown_filter', { filterId }); + } +}; + /** * @param {array} filters * @param {array} whitelistFilters @@ -785,15 +808,11 @@ export const getFilterName = ( filters, whitelistFilters, filterId, - customFilterTranslationKey = 'custom_filter_rules', resolveFilterName = (filter) => (filter ? filter.name : i18n.t('unknown_filter', { filterId })), ) => { - if (filterId === CUSTOM_FILTERING_RULES_ID) { - return i18n.t(customFilterTranslationKey); - } - - if (filterId === SYSTEM_HOSTS_FILTER_ID) { - return i18n.t('system_host_files'); + const specialFilterIds = Object.values(SPECIAL_FILTER_ID); + if (specialFilterIds.includes(filterId)) { + return getSpecialFilterName(filterId); } const matchIdPredicate = (filter) => filter.id === filterId; diff --git a/internal/aghnet/hostscontainer.go b/internal/aghnet/hostscontainer.go index d6ca93f4..1e1808bc 100644 --- a/internal/aghnet/hostscontainer.go +++ b/internal/aghnet/hostscontainer.go @@ -102,6 +102,9 @@ type HostsContainer struct { // embedded to implement MatchRequest and Translate for *HostsContainer. requestMatcher + // listID is the identifier for the list of generated rules. + listID int + // done is the channel to sign closing the container. done chan struct{} @@ -124,9 +127,11 @@ type HostsContainer struct { const ErrNoHostsPaths errors.Error = "no valid paths to hosts files provided" // NewHostsContainer creates a container of hosts, that watches the paths with -// w. paths shouldn't be empty and each of paths should locate either a file or -// a directory in fsys. fsys and w must be non-nil. +// w. listID is used as an identifier of the underlying rules list. paths +// shouldn't be empty and each of paths should locate either a file or a +// directory in fsys. fsys and w must be non-nil. func NewHostsContainer( + listID int, fsys fs.FS, w aghos.FSWatcher, paths ...string, @@ -149,6 +154,7 @@ func NewHostsContainer( requestMatcher: requestMatcher{ stateLock: &sync.RWMutex{}, }, + listID: listID, done: make(chan struct{}, 1), updates: make(chan *netutil.IPMap, 1), fsys: fsys, @@ -507,10 +513,9 @@ func (hp *hostsParser) sendUpd(ch chan *netutil.IPMap) { } // newStrg creates a new rules storage from parsed data. -func (hp *hostsParser) newStrg() (s *filterlist.RuleStorage, err error) { +func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) { return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{ - // TODO(e.burkov): Make configurable. - ID: -1, + ID: id, RulesText: hp.rulesBuilder.String(), IgnoreCosmetic: true, }}) @@ -538,7 +543,7 @@ func (hc *HostsContainer) refresh() (err error) { hc.last = hp.table.ShallowClone() var rulesStrg *filterlist.RuleStorage - if rulesStrg, err = hp.newStrg(); err != nil { + if rulesStrg, err = hp.newStrg(hc.listID); err != nil { return fmt.Errorf("initializing rules storage: %w", err) } diff --git a/internal/aghnet/hostscontainer_test.go b/internal/aghnet/hostscontainer_test.go index 5686a11b..9d2b6a10 100644 --- a/internal/aghnet/hostscontainer_test.go +++ b/internal/aghnet/hostscontainer_test.go @@ -73,7 +73,7 @@ func TestNewHostsContainer(t *testing.T) { return eventsCh } - hc, err := NewHostsContainer(testFS, &aghtest.FSWatcher{ + hc, err := NewHostsContainer(0, testFS, &aghtest.FSWatcher{ OnEvents: onEvents, OnAdd: onAdd, OnClose: func() (err error) { panic("not implemented") }, @@ -98,7 +98,7 @@ func TestNewHostsContainer(t *testing.T) { t.Run("nil_fs", func(t *testing.T) { require.Panics(t, func() { - _, _ = NewHostsContainer(nil, &aghtest.FSWatcher{ + _, _ = NewHostsContainer(0, nil, &aghtest.FSWatcher{ // Those shouldn't panic. OnEvents: func() (e <-chan struct{}) { return nil }, OnAdd: func(name string) (err error) { return nil }, @@ -109,7 +109,7 @@ func TestNewHostsContainer(t *testing.T) { t.Run("nil_watcher", func(t *testing.T) { require.Panics(t, func() { - _, _ = NewHostsContainer(testFS, nil, p) + _, _ = NewHostsContainer(0, testFS, nil, p) }) }) @@ -122,7 +122,7 @@ func TestNewHostsContainer(t *testing.T) { OnClose: func() (err error) { panic("not implemented") }, } - hc, err := NewHostsContainer(testFS, errWatcher, p) + hc, err := NewHostsContainer(0, testFS, errWatcher, p) require.ErrorIs(t, err, errOnAdd) assert.Nil(t, hc) @@ -164,7 +164,7 @@ func TestHostsContainer_Refresh(t *testing.T) { OnClose: func() (err error) { panic("not implemented") }, } - hc, err := NewHostsContainer(testFS, w, dirname) + hc, err := NewHostsContainer(0, testFS, w, dirname) require.NoError(t, err) checkRefresh := func(t *testing.T, wantHosts *stringutil.Set) { @@ -291,6 +291,8 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) { } func TestHostsContainer(t *testing.T) { + const listID = 1234 + testdata := os.DirFS("./testdata") nRewrites := func(t *testing.T, res *urlfilter.DNSResult, n int) (rws []*rules.DNSRewrite) { @@ -300,6 +302,8 @@ func TestHostsContainer(t *testing.T) { assert.Len(t, rewrites, n) for _, rewrite := range rewrites { + require.Equal(t, listID, rewrite.FilterListID) + rw := rewrite.DNSRewrite require.NotNil(t, rw) @@ -382,7 +386,7 @@ func TestHostsContainer(t *testing.T) { OnClose: func() (err error) { panic("not implemented") }, } - hc, err := NewHostsContainer(testdata, &stubWatcher, "etc_hosts") + hc, err := NewHostsContainer(listID, testdata, &stubWatcher, "etc_hosts") require.NoError(t, err) for _, tc := range testCases { diff --git a/internal/dnsforward/dnsforward_test.go b/internal/dnsforward/dnsforward_test.go index 6f0ccc70..512eb87a 100644 --- a/internal/dnsforward/dnsforward_test.go +++ b/internal/dnsforward/dnsforward_test.go @@ -1078,7 +1078,7 @@ func TestPTRResponseFromHosts(t *testing.T) { } var eventsCalledCounter uint32 - hc, err := aghnet.NewHostsContainer(testFS, &aghtest.FSWatcher{ + hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{ OnEvents: func() (e <-chan struct{}) { assert.Equal(t, uint32(1), atomic.AddUint32(&eventsCalledCounter, 1)) diff --git a/internal/filtering/blocked.go b/internal/filtering/blocked.go index df50f739..a11b72c7 100644 --- a/internal/filtering/blocked.go +++ b/internal/filtering/blocked.go @@ -239,7 +239,7 @@ func initBlockedServices() { for _, s := range serviceRulesArray { netRules := []*rules.NetworkRule{} for _, text := range s.rules { - rule, err := rules.NewNetworkRule(text, 0) + rule, err := rules.NewNetworkRule(text, BlockedSvcsListID) if err != nil { log.Error("rules.NewNetworkRule: %s rule: %s", err, text) continue diff --git a/internal/filtering/dnsrewrite_test.go b/internal/filtering/dnsrewrite_test.go index c471aa5f..877b46a8 100644 --- a/internal/filtering/dnsrewrite_test.go +++ b/internal/filtering/dnsrewrite_test.go @@ -49,7 +49,7 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) { |1.2.3.5.in-addr.arpa^$dnsrewrite=NOERROR;PTR;new-ptr-with-dot. ` - f := newForTest(nil, []Filter{{ID: 0, Data: []byte(text)}}) + f := newForTest(t, nil, []Filter{{ID: 0, Data: []byte(text)}}) setts := &Settings{ FilteringEnabled: true, } diff --git a/internal/filtering/filtering.go b/internal/filtering/filtering.go index 156b08e1..ab0c427a 100644 --- a/internal/filtering/filtering.go +++ b/internal/filtering/filtering.go @@ -4,6 +4,7 @@ package filtering import ( "context" "fmt" + "io/fs" "net" "net/http" "os" @@ -16,6 +17,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/golibs/cache" + "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/stringutil" "github.com/AdguardTeam/urlfilter" @@ -24,6 +26,18 @@ import ( "github.com/miekg/dns" ) +// The IDs of built-in filter lists. +// +// Keep in sync with client/src/helpers/contants.js. +const ( + CustomListID = -iota + SysHostsListID + BlockedSvcsListID + ParentalListID + SafeBrowsingListID + SafeSearchListID +) + // ServiceEntry - blocked service array element type ServiceEntry struct { Name string @@ -125,6 +139,10 @@ type DNSFilter struct { parentalUpstream upstream.Upstream safeBrowsingUpstream upstream.Upstream + safebrowsingCache cache.Cache + parentalCache cache.Cache + safeSearchCache cache.Cache + Config // for direct access by library users, even a = assignment // confLock protects Config. confLock sync.RWMutex @@ -340,14 +358,6 @@ func (d *DNSFilter) reset() { } } -type dnsFilterContext struct { - safebrowsingCache cache.Cache - parentalCache cache.Cache - safeSearchCache cache.Cache -} - -var gctx dnsFilterContext - // ResultRule contains information about applied rules. type ResultRule struct { // Text is the text of the rule. @@ -598,71 +608,70 @@ func matchBlockedServicesRules( // Adding rule and matching against the rules // -// fileExists returns true if file exists. -func fileExists(fn string) bool { - _, err := os.Stat(fn) - return err == nil -} - -func createFilteringEngine(filters []Filter) (*filterlist.RuleStorage, *urlfilter.DNSEngine, error) { - listArray := []filterlist.RuleList{} +func newRuleStorage(filters []Filter) (rs *filterlist.RuleStorage, err error) { + lists := make([]filterlist.RuleList, 0, len(filters)) for _, f := range filters { - var list filterlist.RuleList - - if f.ID == 0 { - list = &filterlist.StringRuleList{ - ID: 0, + switch id := int(f.ID); { + case len(f.Data) != 0: + lists = append(lists, &filterlist.StringRuleList{ + ID: id, RulesText: string(f.Data), IgnoreCosmetic: true, - } - } else if !fileExists(f.FilePath) { - list = &filterlist.StringRuleList{ - ID: int(f.ID), - IgnoreCosmetic: true, - } - } else if runtime.GOOS == "windows" { - // On Windows we don't pass a file to urlfilter because - // it's difficult to update this file while it's being - // used. - data, err := os.ReadFile(f.FilePath) - if err != nil { - return nil, nil, fmt.Errorf("reading filter content: %w", err) + }) + case f.FilePath == "": + continue + case runtime.GOOS == "windows": + // On Windows we don't pass a file to urlfilter because it's + // difficult to update this file while it's being used. + var data []byte + data, err = os.ReadFile(f.FilePath) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return nil, fmt.Errorf("reading filter content: %w", err) } - list = &filterlist.StringRuleList{ - ID: int(f.ID), + lists = append(lists, &filterlist.StringRuleList{ + ID: id, RulesText: string(data), IgnoreCosmetic: true, + }) + default: + var list *filterlist.FileRuleList + list, err = filterlist.NewFileRuleList(id, f.FilePath, true) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return nil, fmt.Errorf("creating file rule list with %q: %w", f.FilePath, err) } - } else { - var err error - list, err = filterlist.NewFileRuleList(int(f.ID), f.FilePath, true) - if err != nil { - return nil, nil, fmt.Errorf("filterlist.NewFileRuleList(): %s: %w", f.FilePath, err) - } + + lists = append(lists, list) } - listArray = append(listArray, list) } - rulesStorage, err := filterlist.NewRuleStorage(listArray) + rs, err = filterlist.NewRuleStorage(lists) if err != nil { - return nil, nil, fmt.Errorf("filterlist.NewRuleStorage(): %w", err) + return nil, fmt.Errorf("creating rule stroage: %w", err) } - filteringEngine := urlfilter.NewDNSEngine(rulesStorage) - return rulesStorage, filteringEngine, nil + + return rs, nil } // Initialize urlfilter objects. func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { - rulesStorage, filteringEngine, err := createFilteringEngine(blockFilters) + rulesStorage, err := newRuleStorage(blockFilters) if err != nil { return err } - rulesStorageAllow, filteringEngineAllow, err := createFilteringEngine(allowFilters) + + rulesStorageAllow, err := newRuleStorage(allowFilters) if err != nil { return err } + filteringEngine := urlfilter.NewDNSEngine(rulesStorage) + filteringEngineAllow := urlfilter.NewDNSEngine(rulesStorageAllow) + func() { d.engineLock.Lock() defer d.engineLock.Unlock() @@ -855,43 +864,37 @@ func makeResult(matchedRules []rules.Rule, reason Reason) (res Result) { } } -// InitModule manually initializes blocked services map. +// InitModule manually initializes blocked services map using blockedSvcListID +// as list ID for the rules. func InitModule() { initBlockedServices() } // New creates properly initialized DNS Filter that is ready to be used. -func New(c *Config, blockFilters []Filter) *DNSFilter { - var resolver Resolver = net.DefaultResolver +func New(c *Config, blockFilters []Filter) (d *DNSFilter) { + d = &DNSFilter{ + resolver: net.DefaultResolver, + } if c != nil { - cacheConf := cache.Config{ + + d.safebrowsingCache = cache.New(cache.Config{ EnableLRU: true, - } - - if gctx.safebrowsingCache == nil { - cacheConf.MaxSize = c.SafeBrowsingCacheSize - gctx.safebrowsingCache = cache.New(cacheConf) - } - - if gctx.safeSearchCache == nil { - cacheConf.MaxSize = c.SafeSearchCacheSize - gctx.safeSearchCache = cache.New(cacheConf) - } - - if gctx.parentalCache == nil { - cacheConf.MaxSize = c.ParentalCacheSize - gctx.parentalCache = cache.New(cacheConf) - } + MaxSize: c.SafeBrowsingCacheSize, + }) + d.safeSearchCache = cache.New(cache.Config{ + EnableLRU: true, + MaxSize: c.SafeSearchCacheSize, + }) + d.parentalCache = cache.New(cache.Config{ + EnableLRU: true, + MaxSize: c.ParentalCacheSize, + }) if c.CustomResolver != nil { - resolver = c.CustomResolver + d.resolver = c.CustomResolver } } - d := &DNSFilter{ - resolver: resolver, - } - d.hostCheckers = []hostChecker{{ check: d.matchSysHosts, name: "hosts container", diff --git a/internal/filtering/filtering_test.go b/internal/filtering/filtering_test.go index b389b386..79c4d040 100644 --- a/internal/filtering/filtering_test.go +++ b/internal/filtering/filtering_test.go @@ -27,11 +27,11 @@ var setts = Settings{ // Helpers. -func purgeCaches() { +func purgeCaches(d *DNSFilter) { for _, c := range []cache.Cache{ - gctx.safebrowsingCache, - gctx.parentalCache, - gctx.safeSearchCache, + d.safebrowsingCache, + d.parentalCache, + d.safeSearchCache, } { if c != nil { c.Clear() @@ -39,7 +39,7 @@ func purgeCaches() { } } -func newForTest(c *Config, filters []Filter) *DNSFilter { +func newForTest(t testing.TB, c *Config, filters []Filter) *DNSFilter { setts = Settings{ ProtectionEnabled: true, FilteringEnabled: true, @@ -54,7 +54,8 @@ func newForTest(c *Config, filters []Filter) *DNSFilter { setts.ParentalEnabled = c.ParentalEnabled } d := New(c, filters) - purgeCaches() + purgeCaches(d) + return d } @@ -105,7 +106,7 @@ func TestEtcHostsMatching(t *testing.T) { filters := []Filter{{ ID: 0, Data: []byte(text), }} - d := newForTest(nil, filters) + d := newForTest(t, nil, filters) t.Cleanup(d.Close) d.checkMatchIP(t, "google.com", addr, dns.TypeA) @@ -170,7 +171,7 @@ func TestSafeBrowsing(t *testing.T) { aghtest.ReplaceLogWriter(t, logOutput) aghtest.ReplaceLogLevel(t, log.DEBUG) - d := newForTest(&Config{SafeBrowsingEnabled: true}, nil) + d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil) t.Cleanup(d.Close) const matching = "wmconvirus.narod.ru" d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{ @@ -193,7 +194,7 @@ func TestSafeBrowsing(t *testing.T) { } func TestParallelSB(t *testing.T) { - d := newForTest(&Config{SafeBrowsingEnabled: true}, nil) + d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil) t.Cleanup(d.Close) const matching = "wmconvirus.narod.ru" d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{ @@ -217,7 +218,7 @@ func TestParallelSB(t *testing.T) { // Safe Search. func TestSafeSearch(t *testing.T) { - d := newForTest(&Config{SafeSearchEnabled: true}, nil) + d := newForTest(t, &Config{SafeSearchEnabled: true}, nil) t.Cleanup(d.Close) val, ok := d.SafeSearchDomain("www.google.com") require.True(t, ok) @@ -226,7 +227,9 @@ func TestSafeSearch(t *testing.T) { } func TestCheckHostSafeSearchYandex(t *testing.T) { - d := newForTest(&Config{SafeSearchEnabled: true}, nil) + d := newForTest(t, &Config{ + SafeSearchEnabled: true, + }, nil) t.Cleanup(d.Close) yandexIP := net.IPv4(213, 180, 193, 56) @@ -249,13 +252,14 @@ func TestCheckHostSafeSearchYandex(t *testing.T) { require.Len(t, res.Rules, 1) assert.Equal(t, yandexIP, res.Rules[0].IP) + assert.EqualValues(t, SafeSearchListID, res.Rules[0].FilterListID) }) } } func TestCheckHostSafeSearchGoogle(t *testing.T) { resolver := &aghtest.TestResolver{} - d := newForTest(&Config{ + d := newForTest(t, &Config{ SafeSearchEnabled: true, CustomResolver: resolver, }, nil) @@ -282,12 +286,13 @@ func TestCheckHostSafeSearchGoogle(t *testing.T) { require.Len(t, res.Rules, 1) assert.Equal(t, ip, res.Rules[0].IP) + assert.EqualValues(t, SafeSearchListID, res.Rules[0].FilterListID) }) } } func TestSafeSearchCacheYandex(t *testing.T) { - d := newForTest(nil, nil) + d := newForTest(t, nil, nil) t.Cleanup(d.Close) const domain = "yandex.ru" @@ -301,7 +306,7 @@ func TestSafeSearchCacheYandex(t *testing.T) { yandexIP := net.IPv4(213, 180, 193, 56) - d = newForTest(&Config{SafeSearchEnabled: true}, nil) + d = newForTest(t, &Config{SafeSearchEnabled: true}, nil) t.Cleanup(d.Close) res, err = d.CheckHost(domain, dns.TypeA, &setts) @@ -312,7 +317,7 @@ func TestSafeSearchCacheYandex(t *testing.T) { assert.Equal(t, res.Rules[0].IP, yandexIP) // Check cache. - cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain) + cachedValue, isFound := getCachedResult(d.safeSearchCache, domain) require.True(t, isFound) require.Len(t, cachedValue.Rules, 1) @@ -321,7 +326,7 @@ func TestSafeSearchCacheYandex(t *testing.T) { func TestSafeSearchCacheGoogle(t *testing.T) { resolver := &aghtest.TestResolver{} - d := newForTest(&Config{ + d := newForTest(t, &Config{ CustomResolver: resolver, }, nil) t.Cleanup(d.Close) @@ -334,7 +339,7 @@ func TestSafeSearchCacheGoogle(t *testing.T) { require.Empty(t, res.Rules) - d = newForTest(&Config{SafeSearchEnabled: true}, nil) + d = newForTest(t, &Config{SafeSearchEnabled: true}, nil) t.Cleanup(d.Close) d.resolver = resolver @@ -361,7 +366,7 @@ func TestSafeSearchCacheGoogle(t *testing.T) { assert.True(t, res.Rules[0].IP.Equal(ip)) // Check cache. - cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain) + cachedValue, isFound := getCachedResult(d.safeSearchCache, domain) require.True(t, isFound) require.Len(t, cachedValue.Rules, 1) @@ -375,7 +380,7 @@ func TestParentalControl(t *testing.T) { aghtest.ReplaceLogWriter(t, logOutput) aghtest.ReplaceLogLevel(t, log.DEBUG) - d := newForTest(&Config{ParentalEnabled: true}, nil) + d := newForTest(t, &Config{ParentalEnabled: true}, nil) t.Cleanup(d.Close) const matching = "pornhub.com" d.SetParentalUpstream(&aghtest.TestBlockUpstream{ @@ -679,7 +684,7 @@ func TestMatching(t *testing.T) { for _, tc := range testCases { t.Run(fmt.Sprintf("%s-%s", tc.name, tc.host), func(t *testing.T) { filters := []Filter{{ID: 0, Data: []byte(tc.rules)}} - d := newForTest(nil, filters) + d := newForTest(t, nil, filters) t.Cleanup(d.Close) res, err := d.CheckHost(tc.host, tc.wantDNSType, &setts) @@ -705,7 +710,7 @@ func TestWhitelist(t *testing.T) { whiteFilters := []Filter{{ ID: 0, Data: []byte(whiteRules), }} - d := newForTest(nil, filters) + d := newForTest(t, nil, filters) err := d.SetFilters(filters, whiteFilters, false) require.NoError(t, err) @@ -750,7 +755,7 @@ func applyClientSettings(setts *Settings) { } func TestClientSettings(t *testing.T) { - d := newForTest( + d := newForTest(t, &Config{ ParentalEnabled: true, SafeBrowsingEnabled: false, @@ -829,7 +834,7 @@ func TestClientSettings(t *testing.T) { // Benchmarks. func BenchmarkSafeBrowsing(b *testing.B) { - d := newForTest(&Config{SafeBrowsingEnabled: true}, nil) + d := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil) b.Cleanup(d.Close) blocked := "wmconvirus.narod.ru" d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{ @@ -845,7 +850,7 @@ func BenchmarkSafeBrowsing(b *testing.B) { } func BenchmarkSafeBrowsingParallel(b *testing.B) { - d := newForTest(&Config{SafeBrowsingEnabled: true}, nil) + d := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil) b.Cleanup(d.Close) blocked := "wmconvirus.narod.ru" d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{ @@ -863,7 +868,7 @@ func BenchmarkSafeBrowsingParallel(b *testing.B) { } func BenchmarkSafeSearch(b *testing.B) { - d := newForTest(&Config{SafeSearchEnabled: true}, nil) + d := newForTest(b, &Config{SafeSearchEnabled: true}, nil) b.Cleanup(d.Close) for n := 0; n < b.N; n++ { val, ok := d.SafeSearchDomain("www.google.com") @@ -874,7 +879,7 @@ func BenchmarkSafeSearch(b *testing.B) { } func BenchmarkSafeSearchParallel(b *testing.B) { - d := newForTest(&Config{SafeSearchEnabled: true}, nil) + d := newForTest(b, &Config{SafeSearchEnabled: true}, nil) b.Cleanup(d.Close) b.RunParallel(func(pb *testing.PB) { for pb.Next() { diff --git a/internal/filtering/rewrites_test.go b/internal/filtering/rewrites_test.go index 551a7faf..f02582c1 100644 --- a/internal/filtering/rewrites_test.go +++ b/internal/filtering/rewrites_test.go @@ -12,7 +12,7 @@ import ( // TODO(e.burkov): All the tests in this file may and should me merged together. func TestRewrites(t *testing.T) { - d := newForTest(nil, nil) + d := newForTest(t, nil, nil) t.Cleanup(d.Close) d.Rewrites = []RewriteEntry{{ @@ -163,7 +163,7 @@ func TestRewrites(t *testing.T) { } func TestRewritesLevels(t *testing.T) { - d := newForTest(nil, nil) + d := newForTest(t, nil, nil) t.Cleanup(d.Close) // Exact host, wildcard L2, wildcard L3. d.Rewrites = []RewriteEntry{{ @@ -209,7 +209,7 @@ func TestRewritesLevels(t *testing.T) { } func TestRewritesExceptionCNAME(t *testing.T) { - d := newForTest(nil, nil) + d := newForTest(t, nil, nil) t.Cleanup(d.Close) // Wildcard and exception for a sub-domain. d.Rewrites = []RewriteEntry{{ @@ -257,7 +257,7 @@ func TestRewritesExceptionCNAME(t *testing.T) { } func TestRewritesExceptionIP(t *testing.T) { - d := newForTest(nil, nil) + d := newForTest(t, nil, nil) t.Cleanup(d.Close) // Exception for AAAA record. d.Rewrites = []RewriteEntry{{ diff --git a/internal/filtering/safebrowsing.go b/internal/filtering/safebrowsing.go index ec626315..495a9067 100644 --- a/internal/filtering/safebrowsing.go +++ b/internal/filtering/safebrowsing.go @@ -318,7 +318,7 @@ func (d *DNSFilter) checkSafeBrowsing( sctx := &sbCtx{ host: host, svc: "SafeBrowsing", - cache: gctx.safebrowsingCache, + cache: d.safebrowsingCache, cacheTime: d.Config.CacheTime, } @@ -326,7 +326,8 @@ func (d *DNSFilter) checkSafeBrowsing( IsFiltered: true, Reason: FilteredSafeBrowsing, Rules: []*ResultRule{{ - Text: "adguard-malware-shavar", + Text: "adguard-malware-shavar", + FilterListID: SafeBrowsingListID, }}, } @@ -351,7 +352,7 @@ func (d *DNSFilter) checkParental( sctx := &sbCtx{ host: host, svc: "Parental", - cache: gctx.parentalCache, + cache: d.parentalCache, cacheTime: d.Config.CacheTime, } @@ -359,7 +360,8 @@ func (d *DNSFilter) checkParental( IsFiltered: true, Reason: FilteredParental, Rules: []*ResultRule{{ - Text: "parental CATEGORY_BLACKLISTED", + Text: "parental CATEGORY_BLACKLISTED", + FilterListID: ParentalListID, }}, } diff --git a/internal/filtering/safebrowsing_test.go b/internal/filtering/safebrowsing_test.go index c88576f1..2dec3668 100644 --- a/internal/filtering/safebrowsing_test.go +++ b/internal/filtering/safebrowsing_test.go @@ -108,7 +108,7 @@ func TestSafeBrowsingCache(t *testing.T) { } func TestSBPC_checkErrorUpstream(t *testing.T) { - d := newForTest(&Config{SafeBrowsingEnabled: true}, nil) + d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil) t.Cleanup(d.Close) ups := &aghtest.TestErrUpstream{} @@ -130,7 +130,7 @@ func TestSBPC_checkErrorUpstream(t *testing.T) { } func TestSBPC(t *testing.T) { - d := newForTest(&Config{SafeBrowsingEnabled: true}, nil) + d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil) t.Cleanup(d.Close) const hostname = "example.org" @@ -147,22 +147,22 @@ func TestSBPC(t *testing.T) { name string block bool }{{ - testCache: gctx.safebrowsingCache, + testCache: d.safebrowsingCache, testFunc: d.checkSafeBrowsing, name: "sb_no_block", block: false, }, { - testCache: gctx.safebrowsingCache, + testCache: d.safebrowsingCache, testFunc: d.checkSafeBrowsing, name: "sb_block", block: true, }, { - testCache: gctx.parentalCache, + testCache: d.parentalCache, testFunc: d.checkParental, name: "pc_no_block", block: false, }, { - testCache: gctx.parentalCache, + testCache: d.parentalCache, testFunc: d.checkParental, name: "pc_block", block: true, @@ -217,6 +217,6 @@ func TestSBPC(t *testing.T) { assert.Equal(t, 1, ups.RequestsCount()) }) - purgeCaches() + purgeCaches(d) } } diff --git a/internal/filtering/safesearch.go b/internal/filtering/safesearch.go index ff89b950..c67ac735 100644 --- a/internal/filtering/safesearch.go +++ b/internal/filtering/safesearch.go @@ -84,7 +84,7 @@ func (d *DNSFilter) checkSafeSearch( } // Check cache. Return cached result if it was found - cachedValue, isFound := getCachedResult(gctx.safeSearchCache, host) + cachedValue, isFound := getCachedResult(d.safeSearchCache, host) if isFound { // atomic.AddUint64(&gctx.stats.Safesearch.CacheHits, 1) log.Tracef("SafeSearch: found in cache: %s", host) @@ -99,12 +99,14 @@ func (d *DNSFilter) checkSafeSearch( res = Result{ IsFiltered: true, Reason: FilteredSafeSearch, - Rules: []*ResultRule{{}}, + Rules: []*ResultRule{{ + FilterListID: SafeSearchListID, + }}, } if ip := net.ParseIP(safeHost); ip != nil { res.Rules[0].IP = ip - valLen := d.setCacheResult(gctx.safeSearchCache, host, res) + valLen := d.setCacheResult(d.safeSearchCache, host, res) log.Debug("SafeSearch: stored in cache: %s (%d bytes)", host, valLen) return res, nil @@ -123,7 +125,7 @@ func (d *DNSFilter) checkSafeSearch( res.Rules[0].IP = ip - l := d.setCacheResult(gctx.safeSearchCache, host, res) + l := d.setCacheResult(d.safeSearchCache, host, res) log.Debug("SafeSearch: stored in cache: %s (%d bytes)", host, l) return res, nil diff --git a/internal/home/filter.go b/internal/home/filter.go index 74f85356..e7688f3f 100644 --- a/internal/home/filter.go +++ b/internal/home/filter.go @@ -711,6 +711,7 @@ func enableFilters(async bool) { func enableFiltersLocked(async bool) { filters := []filtering.Filter{{ + ID: filtering.CustomListID, Data: []byte(strings.Join(config.UserRules, "\n")), }} diff --git a/internal/home/home.go b/internal/home/home.go index 1148ed6a..ec103b1f 100644 --- a/internal/home/home.go +++ b/internal/home/home.go @@ -244,6 +244,7 @@ func setupHostsContainer() (err error) { } Context.etcHosts, err = aghnet.NewHostsContainer( + filtering.SysHostsListID, aghos.RootDirFS(), Context.hostsWatcher, aghnet.DefaultHostsPaths()..., diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index ef450844..7fceb418 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -4,6 +4,21 @@ ## v0.107: API changes +### New constant values for `filter_list_id` field in `ResultRule` + +* Value of `0` is now used for custom filtering rules list. + +* Value of `-1` is now used for rules generated from the operating system hosts + files. + +* Value of `-2` is now used for blocked services' rules. + +* Value of `-3` is now used for rules generated by parental control web service. + +* Value of `-4` is now used for rules generated by safe browsing web service. + +* Value of `-5` is now used for rules generated by safe search web service. + ### New possible value of `"name"` field in `QueryLogItemClient` * The value of `"name"` field in `GET /control/querylog` method is never empty: