From 47dfa44cf6764120cf737a1cec86b26b0cb99eb0 Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Tue, 5 Nov 2024 12:25:39 +0300 Subject: [PATCH] Pull request 2297: AG-20945-filter-storage Squashed commit of the following: commit 2611fd57815910c035621879094ab954389f31b4 Author: Ainar Garipov Date: Fri Nov 1 16:29:06 2024 +0300 dnsforward: imp test commit 5efcfda937da2ce229ae1a3c57d7a4de35ee7caf Author: Ainar Garipov Date: Fri Nov 1 15:54:18 2024 +0300 rulelist: imp docs, tests commit 7a759c46997ed69e931a0a7c4f25820c52660a3f Author: Ainar Garipov Date: Fri Nov 1 14:36:08 2024 +0300 all: add filtering storage; upd golibs --- go.mod | 2 +- go.sum | 4 +- internal/aghhttp/aghhttp.go | 6 - internal/dnsforward/dnsforward_test.go | 2 + internal/filtering/filter_test.go | 4 +- internal/filtering/http.go | 16 +-- internal/filtering/rulelist/engine.go | 79 ++++++------ internal/filtering/rulelist/engine_test.go | 4 + internal/filtering/rulelist/filter.go | 2 +- internal/filtering/rulelist/filter_test.go | 7 +- internal/filtering/rulelist/rulelist.go | 7 ++ internal/filtering/rulelist/rulelist_test.go | 21 ++-- internal/filtering/rulelist/storage.go | 112 ++++++++++++++++++ internal/filtering/rulelist/storage_test.go | 49 ++++++++ internal/filtering/rulelist/textengine.go | 6 +- .../filtering/rulelist/textengine_test.go | 2 + internal/home/control.go | 5 +- internal/home/dns.go | 3 +- internal/home/home.go | 8 +- internal/home/mobileconfig.go | 4 +- internal/home/service.go | 4 +- internal/home/web.go | 6 +- internal/next/websvc/dns_test.go | 3 +- internal/next/websvc/http_test.go | 3 +- internal/next/websvc/settings_test.go | 3 +- internal/next/websvc/system_test.go | 3 +- internal/next/websvc/websvc_test.go | 3 +- 27 files changed, 279 insertions(+), 89 deletions(-) create mode 100644 internal/filtering/rulelist/storage.go create mode 100644 internal/filtering/rulelist/storage_test.go diff --git a/go.mod b/go.mod index 35a1fbf0..89556346 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.2 require ( // TODO(a.garipov): Update when v0.73.3 is released. github.com/AdguardTeam/dnsproxy v0.73.3-0.20241004151328-c7c7b977a2a3 - github.com/AdguardTeam/golibs v0.30.0 + github.com/AdguardTeam/golibs v0.30.1 github.com/AdguardTeam/urlfilter v0.20.0 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.3.0 diff --git a/go.sum b/go.sum index 0b14fc83..492565f1 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/AdguardTeam/dnsproxy v0.73.3-0.20241004151328-c7c7b977a2a3 h1:IGXwBjdKDzUm007QzZyxSllMnkbdXe7K79x7JWcBW/E= github.com/AdguardTeam/dnsproxy v0.73.3-0.20241004151328-c7c7b977a2a3/go.mod h1:356iHROxo+SOdBVifp1MXEh6qHyydtzGCcsQMfx+ZVs= -github.com/AdguardTeam/golibs v0.30.0 h1:3pTdW1B9GZgqARrA5BvmYlAaEG1zAHI/ReikCDxrhiE= -github.com/AdguardTeam/golibs v0.30.0/go.mod h1:vjw1OVZG6BYyoqGRY88U4LCJLOMfhBFhU0UJBdaSAuQ= +github.com/AdguardTeam/golibs v0.30.1 h1:/yv7dq2h7WXw/jTDxkE3FP9zHerRT+i03PZRHJX4fPU= +github.com/AdguardTeam/golibs v0.30.1/go.mod h1:FkwcNQEJoGsgDGXcalrVa/4gWbE68KsmE2guXWtBQUE= github.com/AdguardTeam/urlfilter v0.20.0 h1:X32qiuVCVd8WDYCEsbdZKfXMzwdVqrdulamtUi4rmzs= github.com/AdguardTeam/urlfilter v0.20.0/go.mod h1:gjrywLTxfJh6JOkwi9SU+frhP7kVVEZ5exFGkR99qpk= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= diff --git a/internal/aghhttp/aghhttp.go b/internal/aghhttp/aghhttp.go index 48694a8e..3717b655 100644 --- a/internal/aghhttp/aghhttp.go +++ b/internal/aghhttp/aghhttp.go @@ -14,12 +14,6 @@ import ( "github.com/AdguardTeam/golibs/logutil/slogutil" ) -// HTTP scheme constants. -const ( - SchemeHTTP = "http" - SchemeHTTPS = "https" -) - // RegisterFunc is the function that sets the handler to handle the URL for the // method. // diff --git a/internal/dnsforward/dnsforward_test.go b/internal/dnsforward/dnsforward_test.go index de9ef09c..548c79da 100644 --- a/internal/dnsforward/dnsforward_test.go +++ b/internal/dnsforward/dnsforward_test.go @@ -592,6 +592,8 @@ func TestSafeSearch(t *testing.T) { r, _, errExch := client.Exchange(req, addr) if assert.NoError(c, errExch) { once.Do(func() { reply = r }) + } else { + t.Logf("got error: %v", errExch) } }, testTimeout*10, testTimeout) diff --git a/internal/filtering/filter_test.go b/internal/filtering/filter_test.go index 229b7a9b..8cfcdef9 100644 --- a/internal/filtering/filter_test.go +++ b/internal/filtering/filter_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/AdguardTeam/golibs/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,7 +33,7 @@ func serveHTTPLocally(t *testing.T, h http.Handler) (urlStr string) { require.IsType(t, (*net.TCPAddr)(nil), addr) return (&url.URL{ - Scheme: aghhttp.SchemeHTTP, + Scheme: urlutil.SchemeHTTP, Host: addr.String(), }).String() } diff --git a/internal/filtering/http.go b/internal/filtering/http.go index 08832b40..60b0d1d5 100644 --- a/internal/filtering/http.go +++ b/internal/filtering/http.go @@ -17,6 +17,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/miekg/dns" ) @@ -41,19 +42,14 @@ func (d *DNSFilter) validateFilterURL(urlStr string) (err error) { u, err := url.ParseRequestURI(urlStr) if err != nil { - // Don't wrap the error since it's informative enough as is. + // Don't wrap the error, because it's informative enough as is. return err } - if s := u.Scheme; s != aghhttp.SchemeHTTP && s != aghhttp.SchemeHTTPS { - return &url.Error{ - Op: "Check scheme", - URL: urlStr, - Err: fmt.Errorf("only %v allowed", []string{ - aghhttp.SchemeHTTP, - aghhttp.SchemeHTTPS, - }), - } + err = urlutil.ValidateHTTPURL(u) + if err != nil { + // Don't wrap the error, because it's informative enough as is. + return err } return nil diff --git a/internal/filtering/rulelist/engine.go b/internal/filtering/rulelist/engine.go index 65e488ce..27d45843 100644 --- a/internal/filtering/rulelist/engine.go +++ b/internal/filtering/rulelist/engine.go @@ -3,11 +3,12 @@ package rulelist import ( "context" "fmt" + "log/slog" "net/http" "sync" "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/logutil/slogutil" "github.com/AdguardTeam/urlfilter" "github.com/AdguardTeam/urlfilter/filterlist" "github.com/c2h5oh/datasize" @@ -18,6 +19,9 @@ import ( // // TODO(a.garipov): Merge with [TextEngine] in some way? type Engine struct { + // logger is used to log the operation of the engine and its refreshes. + logger *slog.Logger + // mu protects engine and storage. // // TODO(a.garipov): See if anything else should be protected. @@ -29,8 +33,7 @@ type Engine struct { // storage is the filtering-rule storage. It is saved here to close it. storage *filterlist.RuleStorage - // name is the human-readable name of the engine, like "allowed", "blocked", - // or "custom". + // name is the human-readable name of the engine. name string // filters is the data about rule filters in this engine. @@ -40,12 +43,15 @@ type Engine struct { // EngineConfig is the configuration for rule-list filtering engines created by // combining refreshable filters. type EngineConfig struct { - // Name is the human-readable name of this engine, like "allowed", - // "blocked", or "custom". + // Logger is used to log the operation of the engine. It must not be nil. + Logger *slog.Logger + + // name is the human-readable name of the engine; see [EngineNameAllow] and + // similar constants. Name string // Filters is the data about rule lists in this engine. There must be no - // other references to the elements of this slice. + // other references to the items of this slice. Each item must not be nil. Filters []*Filter } @@ -53,6 +59,7 @@ type EngineConfig struct { // refreshed, so a refresh should be performed before use. func NewEngine(c *EngineConfig) (e *Engine) { return &Engine{ + logger: c.Logger, mu: &sync.RWMutex{}, name: c.Name, filters: c.Filters, @@ -85,7 +92,7 @@ func (e *Engine) FilterRequest( } // currentEngine returns the current filtering engine. -func (e *Engine) currentEngine() (enging *urlfilter.DNSEngine) { +func (e *Engine) currentEngine() (engine *urlfilter.DNSEngine) { e.mu.RLock() defer e.mu.RUnlock() @@ -96,7 +103,7 @@ func (e *Engine) currentEngine() (enging *urlfilter.DNSEngine) { // parseBuf, cli, cacheDir, and maxSize are used for updates of rule-list // filters; see [Filter.Refresh]. // -// TODO(a.garipov): Unexport and test in an internal test or through enigne +// TODO(a.garipov): Unexport and test in an internal test or through engine // tests. func (e *Engine) Refresh( ctx context.Context, @@ -115,20 +122,20 @@ func (e *Engine) Refresh( } if len(filtersToRefresh) == 0 { - log.Info("filtering: updating engine %q: no rule-list filters", e.name) + e.logger.InfoContext(ctx, "updating: no rule-list filters") return nil } engRefr := &engineRefresh{ - httpCli: cli, - cacheDir: cacheDir, - engineName: e.name, - parseBuf: parseBuf, - maxSize: maxSize, + logger: e.logger, + httpCli: cli, + cacheDir: cacheDir, + parseBuf: parseBuf, + maxSize: maxSize, } - ruleLists, errs := engRefr.process(ctx, e.filters) + ruleLists, errs := engRefr.process(ctx, filtersToRefresh) if isOneTimeoutError(errs) { // Don't wrap the error since it's informative enough as is. return err @@ -141,14 +148,14 @@ func (e *Engine) Refresh( return errors.Join(errs...) } - e.resetStorage(storage) + e.resetStorage(ctx, storage) return errors.Join(errs...) } // resetStorage sets e.storage and e.engine and closes the previous storage. // Errors from closing the previous storage are logged. -func (e *Engine) resetStorage(storage *filterlist.RuleStorage) { +func (e *Engine) resetStorage(ctx context.Context, storage *filterlist.RuleStorage) { e.mu.Lock() defer e.mu.Unlock() @@ -161,7 +168,7 @@ func (e *Engine) resetStorage(storage *filterlist.RuleStorage) { err := prevStorage.Close() if err != nil { - log.Error("filtering: engine %q: closing old storage: %s", e.name, err) + e.logger.WarnContext(ctx, "closing old storage", slogutil.KeyError, err) } } @@ -179,11 +186,11 @@ func isOneTimeoutError(errs []error) (ok bool) { // engineRefresh represents a single ongoing engine refresh. type engineRefresh struct { - httpCli *http.Client - cacheDir string - engineName string - parseBuf []byte - maxSize datasize.ByteSize + logger *slog.Logger + httpCli *http.Client + cacheDir string + parseBuf []byte + maxSize datasize.ByteSize } // process runs updates of all given rule-list filters. All errors are logged @@ -216,12 +223,12 @@ func (r *engineRefresh) process( errs = append(errs, err) // Also log immediately, since the update can take a lot of time. - log.Error( - "filtering: updating engine %q: rule list %s from url %q: %s\n", - r.engineName, - f.uid, - f.url, - err, + r.logger.ErrorContext( + ctx, + "updating rule list", + "uid", f.uid, + "url", f.url, + slogutil.KeyError, err, ) } @@ -237,17 +244,17 @@ func (r *engineRefresh) processFilter(ctx context.Context, f *Filter) (err error } if prevChecksum == parseRes.Checksum { - log.Info("filtering: engine %q: filter %q: no change", r.engineName, f.uid) + r.logger.InfoContext(ctx, "no change in filter", "uid", f.uid) return nil } - log.Info( - "filtering: updated engine %q: filter %q: %d bytes, %d rules", - r.engineName, - f.uid, - parseRes.BytesWritten, - parseRes.RulesCount, + r.logger.InfoContext( + ctx, + "filter updated", + "uid", f.uid, + "bytes", parseRes.BytesWritten, + "rules", parseRes.RulesCount, ) return nil diff --git a/internal/filtering/rulelist/engine_test.go b/internal/filtering/rulelist/engine_test.go index 9eeda15b..c3b0d82d 100644 --- a/internal/filtering/rulelist/engine_test.go +++ b/internal/filtering/rulelist/engine_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" + "github.com/AdguardTeam/golibs/logutil/slogutil" "github.com/AdguardTeam/golibs/testutil" "github.com/AdguardTeam/urlfilter" "github.com/miekg/dns" @@ -13,6 +14,8 @@ import ( ) func TestEngine_Refresh(t *testing.T) { + t.Parallel() + cacheDir := t.TempDir() fileURL, srvURL := newFilterLocations(t, cacheDir, testRuleTextBlocked, testRuleTextBlocked2) @@ -21,6 +24,7 @@ func TestEngine_Refresh(t *testing.T) { httpFlt := newFilter(t, srvURL, "HTTP Filter") eng := rulelist.NewEngine(&rulelist.EngineConfig{ + Logger: slogutil.NewDiscardLogger(), Name: "Engine", Filters: []*rulelist.Filter{fileFlt, httpFlt}, }) diff --git a/internal/filtering/rulelist/filter.go b/internal/filtering/rulelist/filter.go index a29897de..c438748e 100644 --- a/internal/filtering/rulelist/filter.go +++ b/internal/filtering/rulelist/filter.go @@ -105,7 +105,7 @@ func NewFilter(c *FilterConfig) (f *Filter, err error) { // buffer used to parse information from the data. cli and maxSize are only // used when f is a URL-based list. // -// TODO(a.garipov): Unexport and test in an internal test or through enigne +// TODO(a.garipov): Unexport and test in an internal test or through engine // tests. // // TODO(a.garipov): Consider not returning parseRes. diff --git a/internal/filtering/rulelist/filter_test.go b/internal/filtering/rulelist/filter_test.go index 21709583..008edcea 100644 --- a/internal/filtering/rulelist/filter_test.go +++ b/internal/filtering/rulelist/filter_test.go @@ -8,12 +8,15 @@ import ( "testing" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/AdguardTeam/golibs/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestFilter_Refresh(t *testing.T) { + t.Parallel() + cacheDir := t.TempDir() uid := rulelist.MustNewUID() @@ -37,7 +40,7 @@ func TestFilter_Refresh(t *testing.T) { }, { name: "file", url: &url.URL{ - Scheme: "file", + Scheme: urlutil.SchemeFile, Path: fileURL.Path, }, wantNewErrMsg: "", @@ -49,6 +52,8 @@ func TestFilter_Refresh(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f, err := rulelist.NewFilter(&rulelist.FilterConfig{ URL: tc.url, Name: tc.name, diff --git a/internal/filtering/rulelist/rulelist.go b/internal/filtering/rulelist/rulelist.go index 021254c7..d8a82375 100644 --- a/internal/filtering/rulelist/rulelist.go +++ b/internal/filtering/rulelist/rulelist.go @@ -71,3 +71,10 @@ var _ fmt.Stringer = UID{} func (id UID) String() (s string) { return uuid.UUID(id).String() } + +// Common engine names. +const ( + EngineNameAllow = "allow" + EngineNameBlock = "block" + EngineNameCustom = "custom" +) diff --git a/internal/filtering/rulelist/rulelist_test.go b/internal/filtering/rulelist/rulelist_test.go index 78731f33..85a3d362 100644 --- a/internal/filtering/rulelist/rulelist_test.go +++ b/internal/filtering/rulelist/rulelist_test.go @@ -6,20 +6,16 @@ import ( "net/http/httptest" "net/url" "os" - "path/filepath" "sync/atomic" "testing" "time" "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/AdguardTeam/golibs/testutil" "github.com/stretchr/testify/require" ) -func TestMain(m *testing.M) { - testutil.DiscardLogOutput(m) -} - // testTimeout is the common timeout for tests. const testTimeout = 1 * time.Second @@ -31,6 +27,7 @@ const testTitle = "Test Title" // Common rule texts for tests. const ( + testRuleTextAllowed = "||allowed.example^\n" testRuleTextBadTab = "||bad-tab-and-comment.example^\t# A comment.\n" testRuleTextBlocked = "||blocked.example^\n" testRuleTextBlocked2 = "||blocked-2.example^\n" @@ -79,8 +76,16 @@ func newFilterLocations( fileData string, httpData string, ) (fileURL, srvURL *url.URL) { - filePath := filepath.Join(cacheDir, "initial.txt") - err := os.WriteFile(filePath, []byte(fileData), 0o644) + t.Helper() + + f, err := os.CreateTemp(cacheDir, "") + require.NoError(t, err) + + err = f.Close() + require.NoError(t, err) + + filePath := f.Name() + err = os.WriteFile(filePath, []byte(fileData), 0o644) require.NoError(t, err) testutil.CleanupAndRequireSuccess(t, func() (err error) { @@ -88,7 +93,7 @@ func newFilterLocations( }) fileURL = &url.URL{ - Scheme: "file", + Scheme: urlutil.SchemeFile, Path: filePath, } diff --git a/internal/filtering/rulelist/storage.go b/internal/filtering/rulelist/storage.go new file mode 100644 index 00000000..3281032d --- /dev/null +++ b/internal/filtering/rulelist/storage.go @@ -0,0 +1,112 @@ +package rulelist + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "sync" + + "github.com/AdguardTeam/golibs/errors" + "github.com/c2h5oh/datasize" +) + +// Storage contains the main filtering engines, including the allowlist, the +// blocklist, and the user's custom filtering rules. +type Storage struct { + // refreshMu makes sure that only one update takes place at a time. + refreshMu *sync.Mutex + + allow *Engine + block *Engine + custom *TextEngine + httpCli *http.Client + cacheDir string + parseBuf []byte + maxSize datasize.ByteSize +} + +// StorageConfig is the configuration for the filtering-engine storage. +type StorageConfig struct { + // Logger is used to log the operation of the storage. It must not be nil. + Logger *slog.Logger + + // HTTPClient is the HTTP client used to perform updates of rule lists. + // It must not be nil. + HTTPClient *http.Client + + // CacheDir is the path to the directory used to cache rule-list files. + // It must be set. + CacheDir string + + // AllowFilters are the filtering-rule lists used to exclude domain names + // from the filtering. Each item must not be nil. + AllowFilters []*Filter + + // BlockFilters are the filtering-rule lists used to block domain names. + // Each item must not be nil. + BlockFilters []*Filter + + // CustomRules contains custom rules of the user. They have priority over + // both allow- and blacklist rules. + CustomRules []string + + // MaxRuleListTextSize is the maximum size of a rule-list file. It must be + // greater than zero. + MaxRuleListTextSize datasize.ByteSize +} + +// NewStorage creates a new filtering-engine storage. The engines are not +// refreshed, so a refresh should be performed before use. +func NewStorage(c *StorageConfig) (s *Storage, err error) { + custom, err := NewTextEngine(&TextEngineConfig{ + Name: EngineNameCustom, + Rules: c.CustomRules, + ID: URLFilterIDCustom, + }) + if err != nil { + return nil, fmt.Errorf("creating custom engine: %w", err) + } + + return &Storage{ + refreshMu: &sync.Mutex{}, + allow: NewEngine(&EngineConfig{ + Logger: c.Logger.With("engine", EngineNameAllow), + Name: EngineNameAllow, + Filters: c.AllowFilters, + }), + block: NewEngine(&EngineConfig{ + Logger: c.Logger.With("engine", EngineNameBlock), + Name: EngineNameBlock, + Filters: c.BlockFilters, + }), + custom: custom, + httpCli: c.HTTPClient, + cacheDir: c.CacheDir, + parseBuf: make([]byte, DefaultRuleBufSize), + maxSize: c.MaxRuleListTextSize, + }, nil +} + +// Close closes the underlying rule-list engines. +func (s *Storage) Close() (err error) { + // Don't wrap the errors since they are informative enough as is. + return errors.Join( + s.allow.Close(), + s.block.Close(), + ) +} + +// Refresh updates all engines in s. +// +// TODO(a.garipov): Refresh allow and block separately? +func (s *Storage) Refresh(ctx context.Context) (err error) { + s.refreshMu.Lock() + defer s.refreshMu.Unlock() + + // Don't wrap the errors since they are informative enough as is. + return errors.Join( + s.allow.Refresh(ctx, s.parseBuf, s.httpCli, s.cacheDir, s.maxSize), + s.block.Refresh(ctx, s.parseBuf, s.httpCli, s.cacheDir, s.maxSize), + ) +} diff --git a/internal/filtering/rulelist/storage_test.go b/internal/filtering/rulelist/storage_test.go new file mode 100644 index 00000000..ac7e6ae3 --- /dev/null +++ b/internal/filtering/rulelist/storage_test.go @@ -0,0 +1,49 @@ +package rulelist_test + +import ( + "net/http" + "testing" + + "github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist" + "github.com/AdguardTeam/golibs/logutil/slogutil" + "github.com/AdguardTeam/golibs/testutil" + "github.com/c2h5oh/datasize" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStorage_Refresh(t *testing.T) { + t.Parallel() + + cacheDir := t.TempDir() + + allowedFileURL, _ := newFilterLocations(t, cacheDir, testRuleTextAllowed, "") + allowedFlt := newFilter(t, allowedFileURL, "Allowed 1") + + blockedFileURL, _ := newFilterLocations(t, cacheDir, testRuleTextBlocked, "") + blockedFlt := newFilter(t, blockedFileURL, "Blocked 1") + + strg, err := rulelist.NewStorage(&rulelist.StorageConfig{ + Logger: slogutil.NewDiscardLogger(), + HTTPClient: &http.Client{ + Timeout: testTimeout, + }, + CacheDir: cacheDir, + AllowFilters: []*rulelist.Filter{ + allowedFlt, + }, + BlockFilters: []*rulelist.Filter{ + blockedFlt, + }, + CustomRules: []string{ + testRuleTextBlocked2, + }, + MaxRuleListTextSize: 1 * datasize.KB, + }) + require.NoError(t, err) + testutil.CleanupAndRequireSuccess(t, strg.Close) + + ctx := testutil.ContextWithTimeout(t, testTimeout) + err = strg.Refresh(ctx) + assert.NoError(t, err) +} diff --git a/internal/filtering/rulelist/textengine.go b/internal/filtering/rulelist/textengine.go index 4b5e8ce8..d04a8f5d 100644 --- a/internal/filtering/rulelist/textengine.go +++ b/internal/filtering/rulelist/textengine.go @@ -20,15 +20,15 @@ type TextEngine struct { // storage is the filtering-rule storage. It is saved here to close it. storage *filterlist.RuleStorage - // name is the human-readable name of the engine, like "custom". + // name is the human-readable name of the engine. name string } // TextEngineConfig is the configuration for a rule-list filtering engine // created from a filtering rule text. type TextEngineConfig struct { - // Name is the human-readable name of this engine, like "allowed", - // "blocked", or "custom". + // name is the human-readable name of the engine; see [EngineNameAllow] and + // similar constants. Name string // Rules is the text of the filtering rules for this engine. diff --git a/internal/filtering/rulelist/textengine_test.go b/internal/filtering/rulelist/textengine_test.go index 129d01c7..6b80074b 100644 --- a/internal/filtering/rulelist/textengine_test.go +++ b/internal/filtering/rulelist/textengine_test.go @@ -12,6 +12,8 @@ import ( ) func TestNewTextEngine(t *testing.T) { + t.Parallel() + eng, err := rulelist.NewTextEngine(&rulelist.TextEngineConfig{ Name: "RulesEngine", Rules: []string{ diff --git a/internal/home/control.go b/internal/home/control.go index ec9192fd..61028bd4 100644 --- a/internal/home/control.go +++ b/internal/home/control.go @@ -16,6 +16,7 @@ import ( "github.com/AdguardTeam/golibs/httphdr" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/NYTimes/gziphandler" ) @@ -376,7 +377,7 @@ func handleHTTPSRedirect(w http.ResponseWriter, r *http.Request) (proceed bool) // // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. originURL := &url.URL{ - Scheme: aghhttp.SchemeHTTP, + Scheme: urlutil.SchemeHTTP, Host: r.Host, } @@ -395,7 +396,7 @@ func httpsURL(u *url.URL, host string, portHTTPS uint16) (redirectURL *url.URL) } return &url.URL{ - Scheme: aghhttp.SchemeHTTPS, + Scheme: urlutil.SchemeHTTPS, Host: hostPort, Path: u.Path, RawQuery: u.RawQuery, diff --git a/internal/home/dns.go b/internal/home/dns.go index f0c6d4de..6fcc09ca 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -23,6 +23,7 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/logutil/slogutil" "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/ameshkov/dnscrypt/v2" yaml "gopkg.in/yaml.v3" ) @@ -371,7 +372,7 @@ func getDNSEncryption() (de dnsEncryption) { } de.https = (&url.URL{ - Scheme: "https", + Scheme: urlutil.SchemeHTTPS, Host: addr, Path: "/dns-query", }).String() diff --git a/internal/home/home.go b/internal/home/home.go index ed96dee1..4c2771e4 100644 --- a/internal/home/home.go +++ b/internal/home/home.go @@ -21,7 +21,6 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/aghalg" - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/AdGuardHome/internal/aghtls" @@ -42,6 +41,7 @@ import ( "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/logutil/slogutil" "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/AdguardTeam/golibs/osutil" ) @@ -605,7 +605,7 @@ func run(opts options, clientBuildFS fs.FS, done chan struct{}) { fatalOnError(errors.Annotate(err, "getting executable path: %w")) u := &url.URL{ - Scheme: "https", + Scheme: urlutil.SchemeHTTPS, // TODO(a.garipov): Make configurable. Host: "static.adtidy.org", Path: path.Join("adguardhome", version.Channel(), "version.json"), @@ -936,12 +936,12 @@ func printHTTPAddresses(proto string) { } port := config.HTTPConfig.Address.Port() - if proto == aghhttp.SchemeHTTPS { + if proto == urlutil.SchemeHTTPS { port = tlsConf.PortHTTPS } // TODO(e.burkov): Inspect and perhaps merge with the previous condition. - if proto == aghhttp.SchemeHTTPS && tlsConf.ServerName != "" { + if proto == urlutil.SchemeHTTPS && tlsConf.ServerName != "" { printWebAddrs(proto, tlsConf.ServerName, tlsConf.PortHTTPS) return diff --git a/internal/home/mobileconfig.go b/internal/home/mobileconfig.go index 8d9deb56..f3c82278 100644 --- a/internal/home/mobileconfig.go +++ b/internal/home/mobileconfig.go @@ -8,11 +8,11 @@ import ( "net/url" "path" - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/httphdr" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/google/uuid" "howett.net/plist" ) @@ -84,7 +84,7 @@ func encodeMobileConfig(d *dnsSettings, clientID string) ([]byte, error) { case dnsProtoHTTPS: dspName = fmt.Sprintf("%s DoH", d.ServerName) u := &url.URL{ - Scheme: aghhttp.SchemeHTTPS, + Scheme: urlutil.SchemeHTTPS, Host: d.ServerName, Path: path.Join("/dns-query", clientID), } diff --git a/internal/home/service.go b/internal/home/service.go index 27d59ad3..95ceafc6 100644 --- a/internal/home/service.go +++ b/internal/home/service.go @@ -10,11 +10,11 @@ import ( "syscall" "time" - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghos" "github.com/AdguardTeam/AdGuardHome/internal/version" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/kardianos/service" ) @@ -336,7 +336,7 @@ AdGuard Home is successfully installed and will automatically start on boot. There are a few more things that must be configured before you can use it. Click on the link below and follow the Installation Wizard steps to finish setup. AdGuard Home is now available at the following addresses:`) - printHTTPAddresses(aghhttp.SchemeHTTP) + printHTTPAddresses(urlutil.SchemeHTTP) } } diff --git a/internal/home/web.go b/internal/home/web.go index 37f5a5dc..099f9aeb 100644 --- a/internal/home/web.go +++ b/internal/home/web.go @@ -11,13 +11,13 @@ import ( "sync" "time" - "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/aghnet" "github.com/AdguardTeam/AdGuardHome/internal/updater" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" "github.com/AdguardTeam/golibs/netutil" "github.com/AdguardTeam/golibs/netutil/httputil" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/NYTimes/gziphandler" "github.com/quic-go/quic-go/http3" "golang.org/x/net/http2" @@ -192,7 +192,7 @@ func (web *webAPI) start() { // this loop is used as an ability to change listening host and/or port for !web.httpsServer.inShutdown { - printHTTPAddresses(aghhttp.SchemeHTTP) + printHTTPAddresses(urlutil.SchemeHTTP) errs := make(chan error, 2) // Use an h2c handler to support unencrypted HTTP/2, e.g. for proxies. @@ -286,7 +286,7 @@ func (web *webAPI) tlsServerLoop() { WriteTimeout: web.conf.WriteTimeout, } - printHTTPAddresses(aghhttp.SchemeHTTPS) + printHTTPAddresses(urlutil.SchemeHTTPS) if web.conf.serveHTTP3 { go web.mustStartHTTP3(addr) diff --git a/internal/next/websvc/dns_test.go b/internal/next/websvc/dns_test.go index 5f6e3d44..d7e58eb2 100644 --- a/internal/next/websvc/dns_test.go +++ b/internal/next/websvc/dns_test.go @@ -15,6 +15,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/next/agh" "github.com/AdguardTeam/AdGuardHome/internal/next/dnssvc" "github.com/AdguardTeam/AdGuardHome/internal/next/websvc" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -49,7 +50,7 @@ func TestService_HandlePatchSettingsDNS(t *testing.T) { _, addr := newTestServer(t, confMgr) u := &url.URL{ - Scheme: "http", + Scheme: urlutil.SchemeHTTP, Host: addr.String(), Path: websvc.PathV1SettingsDNS, } diff --git a/internal/next/websvc/http_test.go b/internal/next/websvc/http_test.go index 3168ec03..99569b62 100644 --- a/internal/next/websvc/http_test.go +++ b/internal/next/websvc/http_test.go @@ -13,6 +13,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/next/agh" "github.com/AdguardTeam/AdGuardHome/internal/next/websvc" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,7 +46,7 @@ func TestService_HandlePatchSettingsHTTP(t *testing.T) { _, addr := newTestServer(t, confMgr) u := &url.URL{ - Scheme: "http", + Scheme: urlutil.SchemeHTTP, Host: addr.String(), Path: websvc.PathV1SettingsHTTP, } diff --git a/internal/next/websvc/settings_test.go b/internal/next/websvc/settings_test.go index f07d8b3e..e30002de 100644 --- a/internal/next/websvc/settings_test.go +++ b/internal/next/websvc/settings_test.go @@ -13,6 +13,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/next/agh" "github.com/AdguardTeam/AdGuardHome/internal/next/dnssvc" "github.com/AdguardTeam/AdGuardHome/internal/next/websvc" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -69,7 +70,7 @@ func TestService_HandleGetSettingsAll(t *testing.T) { _, addr := newTestServer(t, confMgr) u := &url.URL{ - Scheme: "http", + Scheme: urlutil.SchemeHTTP, Host: addr.String(), Path: websvc.PathV1SettingsAll, } diff --git a/internal/next/websvc/system_test.go b/internal/next/websvc/system_test.go index acbdcba2..44f9e529 100644 --- a/internal/next/websvc/system_test.go +++ b/internal/next/websvc/system_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/AdguardTeam/AdGuardHome/internal/next/websvc" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,7 +18,7 @@ func TestService_handleGetV1SystemInfo(t *testing.T) { confMgr := newConfigManager() _, addr := newTestServer(t, confMgr) u := &url.URL{ - Scheme: "http", + Scheme: urlutil.SchemeHTTP, Host: addr.String(), Path: websvc.PathV1SystemInfo, } diff --git a/internal/next/websvc/websvc_test.go b/internal/next/websvc/websvc_test.go index cb4c6bc9..f9196aac 100644 --- a/internal/next/websvc/websvc_test.go +++ b/internal/next/websvc/websvc_test.go @@ -15,6 +15,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/next/agh" "github.com/AdguardTeam/AdGuardHome/internal/next/dnssvc" "github.com/AdguardTeam/AdGuardHome/internal/next/websvc" + "github.com/AdguardTeam/golibs/netutil/urlutil" "github.com/AdguardTeam/golibs/testutil" "github.com/AdguardTeam/golibs/testutil/fakefs" "github.com/stretchr/testify/assert" @@ -181,7 +182,7 @@ func TestService_Start_getHealthCheck(t *testing.T) { confMgr := newConfigManager() _, addr := newTestServer(t, confMgr) u := &url.URL{ - Scheme: "http", + Scheme: urlutil.SchemeHTTP, Host: addr.String(), Path: websvc.PathHealthCheck, }