mirrors/AdGuardHome
1
0
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-11-21 12:35:33 +03:00
Code Issues Projects Releases Packages Wiki Activity

Pull request 2094: AG-27796 upd golibs

Browse source 
Squashed commit of the following:

commit a205c1302e3979d1c4270b11d253b6bc0d292216
Merge: de289ff4f 214175eb4
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Dec 7 16:36:53 2023 +0300

    Merge branch 'master' into AG-27796-upd-golibs

commit de289ff4f3199bc2dffb029a9804cabe86b3b886
Merge: b2322093c a0ec0b2b5
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Dec 6 12:12:35 2023 +0300

    Merge branch 'master' into AG-27796-upd-golibs

commit b2322093cea0ecdf34be66b56a9ab0fd7b32c7b9
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Dec 5 19:20:30 2023 +0300

    filtering: imp cognit

commit 563aa45824a2cc9d63d2c394f6a60f053e5d6d3b
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Dec 4 17:02:56 2023 +0300

    all: imp code

commit 064a00bce4340caa4cea052fa8234cedb8dcea01
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Nov 28 18:41:07 2023 +0300

    all: upd golibs
This commit is contained in:
Eugene Burkov 2023-12-07 16:48:55 +03:00
parent 214175eb41
commit ce868268bc
18 changed files with 168 additions and 682 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
internal/aghchan/aghchan.go
View file

@ -1,33 +0,0 @@
// Package aghchan contains channel utilities.
package aghchan
import (
"fmt"
"time"
)
// Receive returns an error if it cannot receive a value form c before timeout
// runs out.
func Receive[T any](c <-chan T, timeout time.Duration) (v T, ok bool, err error) {
var zero T
timeoutCh := time.After(timeout)
select {
case <-timeoutCh:
// TODO(a.garipov): Consider implementing [errors.Aser] for
// os.ErrTimeout.
return zero, false, fmt.Errorf("did not receive after %s", timeout)
case v, ok = <-c:
return v, ok, nil
}
}
// MustReceive panics if it cannot receive a value form c before timeout runs
// out.
func MustReceive[T any](c <-chan T, timeout time.Duration) (v T, ok bool) {
v, ok, err := Receive(c, timeout)
if err != nil {
panic(err)
}
return v, ok
}

192
internal/aghnet/hostscontainer.go
View file

@ -1,65 +1,23 @@
package aghnet
import (
"context"
"fmt"
"io"
"io/fs"
"net/netip"
"path"
"strings"
"sync/atomic"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
// DefaultHostsPaths returns the slice of paths default for the operating system
// to files and directories which are containing the hosts database. The result
// is intended to be used within fs.FS so the initial slash is omitted.
func DefaultHostsPaths() (paths []string) {
return defaultHostsPaths()
}
// MatchAddr returns the records for the IP address.
func (hc *HostsContainer) MatchAddr(ip netip.Addr) (recs []*hostsfile.Record) {
cur := hc.current.Load()
if cur == nil {
return nil
}
return cur.addrs[ip]
}
// MatchName returns the records for the hostname.
func (hc *HostsContainer) MatchName(name string) (recs []*hostsfile.Record) {
cur := hc.current.Load()
if cur != nil {
recs = cur.names[name]
}
return recs
}
// hostsContainerPrefix is a prefix for logging and wrapping errors in
// HostsContainer's methods.
const hostsContainerPrefix = "hosts container"
// Hosts is a map of IP addresses to the records, as it primarily stored in the
// [HostsContainer]. It should not be accessed for writing since it may be read
// concurrently, users should clone it before modifying.
//
// The order of records for each address is preserved from original files, but
// the order of the addresses, being a map key, is not.
//
// TODO(e.burkov): Probably, this should be a sorted slice of records.
type Hosts map[netip.Addr][]*hostsfile.Record
// HostsContainer stores the relevant hosts database provided by the OS and
// processes both A/AAAA and PTR DNS requests for those.
type HostsContainer struct {
@ -67,10 +25,10 @@ type HostsContainer struct {
done chan struct{}
// updates is the channel for receiving updated hosts.
updates chan Hosts
updates chan *hostsfile.DefaultStorage
// current is the last set of hosts parsed.
current atomic.Pointer[hostsIndex]
current atomic.Pointer[hostsfile.DefaultStorage]
// fsys is the working file system to read hosts files from.
fsys fs.FS
@ -111,7 +69,7 @@ func NewHostsContainer(
hc = &HostsContainer{
done: make(chan struct{}, 1),
updates: make(chan Hosts, 1),
updates: make(chan *hostsfile.DefaultStorage, 1),
fsys: fsys,
watcher: w,
patterns: patterns,
@ -152,11 +110,25 @@ func (hc *HostsContainer) Close() (err error) {
return err
}
// Upd returns the channel into which the updates are sent.
func (hc *HostsContainer) Upd() (updates <-chan Hosts) {
// Upd returns the channel into which the updates are sent. The updates
// themselves must not be modified.
func (hc *HostsContainer) Upd() (updates <-chan *hostsfile.DefaultStorage) {
return hc.updates
}
// type check
var _ hostsfile.Storage = (*HostsContainer)(nil)
// ByAddr implements the [hostsfile.Storage] interface for *HostsContainer.
func (hc *HostsContainer) ByAddr(addr netip.Addr) (names []string) {
return hc.current.Load().ByAddr(addr)
}
// ByName implements the [hostsfile.Storage] interface for *HostsContainer.
func (hc *HostsContainer) ByName(name string) (addrs []netip.Addr) {
return hc.current.Load().ByName(name)
}
// pathsToPatterns converts paths into patterns compatible with fs.Glob.
func pathsToPatterns(fsys fs.FS, paths []string) (patterns []string, err error) {
for i, p := range paths {
@ -167,7 +139,7 @@ func pathsToPatterns(fsys fs.FS, paths []string) (patterns []string, err error)
continue
}
// Don't put a filename here since it's already added by fs.Stat.
// Don't put a filename here since it's already added by [fs.Stat].
return nil, fmt.Errorf("path at index %d: %w", i, err)
}
@ -209,7 +181,7 @@ func (hc *HostsContainer) handleEvents() {
}
// sendUpd tries to send the parsed data to the ch.
func (hc *HostsContainer) sendUpd(recs Hosts) {
func (hc *HostsContainer) sendUpd(recs *hostsfile.DefaultStorage) {
log.Debug("%s: sending upd", hostsContainerPrefix)
ch := hc.updates
@ -226,67 +198,6 @@ func (hc *HostsContainer) sendUpd(recs Hosts) {
}
}
// hostsIndex is a [hostsfile.Set] to enumerate all the records.
type hostsIndex struct {
// addrs maps IP addresses to the records.
addrs Hosts
// names maps hostnames to the records.
names map[string][]*hostsfile.Record
}
// walk is a file walking function for hostsIndex.
func (idx *hostsIndex) walk(r io.Reader) (patterns []string, cont bool, err error) {
return nil, true, hostsfile.Parse(idx, r, nil)
}
// type check
var _ hostsfile.Set = (*hostsIndex)(nil)
// Add implements the [hostsfile.Set] interface for *hostsIndex.
func (idx *hostsIndex) Add(rec *hostsfile.Record) {
idx.addrs[rec.Addr] = append(idx.addrs[rec.Addr], rec)
for _, name := range rec.Names {
idx.names[name] = append(idx.names[name], rec)
}
}
// type check
var _ hostsfile.HandleSet = (*hostsIndex)(nil)
// HandleInvalid implements the [hostsfile.HandleSet] interface for *hostsIndex.
func (idx *hostsIndex) HandleInvalid(src string, _ []byte, err error) {
lineErr := &hostsfile.LineError{}
if !errors.As(err, &lineErr) {
// Must not happen if idx passed to [hostsfile.Parse].
return
} else if errors.Is(lineErr, hostsfile.ErrEmptyLine) {
// Ignore empty lines.
return
}
log.Info("%s: warning: parsing %q: %s", hostsContainerPrefix, src, lineErr)
}
// equalRecs is an equality function for [*hostsfile.Record].
func equalRecs(a, b *hostsfile.Record) (ok bool) {
return a.Addr == b.Addr && a.Source == b.Source && slices.Equal(a.Names, b.Names)
}
// equalRecSlices is an equality function for slices of [*hostsfile.Record].
func equalRecSlices(a, b []*hostsfile.Record) (ok bool) { return slices.EqualFunc(a, b, equalRecs) }
// Equal returns true if indexes are equal.
func (idx *hostsIndex) Equal(other *hostsIndex) (ok bool) {
if idx == nil {
return other == nil
} else if other == nil {
return false
}
return maps.EqualFunc(idx.addrs, other.addrs, equalRecSlices)
}
// refresh gets the data from specified files and propagates the updates if
// needed.
//
@ -294,63 +205,22 @@ func (idx *hostsIndex) Equal(other *hostsIndex) (ok bool) {
func (hc *HostsContainer) refresh() (err error) {
log.Debug("%s: refreshing", hostsContainerPrefix)
var addrLen, nameLen int
last := hc.current.Load()
if last != nil {
addrLen, nameLen = len(last.addrs), len(last.names)
}
idx := &hostsIndex{
addrs: make(Hosts, addrLen),
names: make(map[string][]*hostsfile.Record, nameLen),
}
_, err = aghos.FileWalker(idx.walk).Walk(hc.fsys, hc.patterns...)
// The error is always nil here since no readers passed.
strg, _ := hostsfile.NewDefaultStorage()
_, err = aghos.FileWalker(func(r io.Reader) (patterns []string, cont bool, err error) {
// Don't wrap the error since it's already informative enough as is.
return nil, true, hostsfile.Parse(strg, r, nil)
}).Walk(hc.fsys, hc.patterns...)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
// TODO(e.burkov): Serialize updates using time.
if !last.Equal(idx) {
hc.current.Store(idx)
hc.sendUpd(idx.addrs)
// TODO(e.burkov): Serialize updates using [time.Time].
if !hc.current.Load().Equal(strg) {
hc.current.Store(strg)
hc.sendUpd(strg)
}
return nil
}
// type check
var _ upstream.Resolver = (*HostsContainer)(nil)
// LookupNetIP implements the [upstream.Resolver] interface for *HostsContainer.
func (hc *HostsContainer) LookupNetIP(
ctx context.Context,
network string,
hostname string,
) (addrs []netip.Addr, err error) {
// TODO(e.burkov): Think of extracting this logic to a golibs function if
// needed anywhere else.
var isDesiredProto func(ip netip.Addr) (ok bool)
switch network {
case "ip4":
isDesiredProto = (netip.Addr).Is4
case "ip6":
isDesiredProto = (netip.Addr).Is6
case "ip":
isDesiredProto = func(ip netip.Addr) (ok bool) { return true }
default:
return nil, fmt.Errorf("unsupported network: %q", network)
}
idx := hc.current.Load()
recs := idx.names[strings.ToLower(hostname)]
addrs = make([]netip.Addr, 0, len(recs))
for _, rec := range recs {
if isDesiredProto(rec.Addr) {
addrs = append(addrs, rec.Addr)
}
}
return slices.Clip(addrs), nil
}

17
internal/aghnet/hostscontainer_linux.go
View file

@ -1,17 +0,0 @@
//go:build linux
package aghnet
import (
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
)
func defaultHostsPaths() (paths []string) {
paths = []string{"etc/hosts"}
if aghos.IsOpenWrt() {
paths = append(paths, "tmp/hosts")
}
return paths
}

7
internal/aghnet/hostscontainer_others.go
View file

@ -1,7 +0,0 @@
//go:build !(windows || linux)
package aghnet
func defaultHostsPaths() (paths []string) {
return []string{"etc/hosts"}
}

318
internal/aghnet/hostscontainer_test.go
View file

@ -3,13 +3,11 @@ package aghnet_test
import (
"net/netip"
"path"
"path/filepath"
"sync/atomic"
"testing"
"testing/fstest"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghchan"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/errors"
@ -20,139 +18,6 @@ import (
"github.com/stretchr/testify/require"
)
// nl is a newline character.
const nl = "\n"
// Variables mirroring the etc_hosts file from testdata.
var (
addr1000 = netip.MustParseAddr("1.0.0.0")
addr1001 = netip.MustParseAddr("1.0.0.1")
addr1002 = netip.MustParseAddr("1.0.0.2")
addr1003 = netip.MustParseAddr("1.0.0.3")
addr1004 = netip.MustParseAddr("1.0.0.4")
addr1357 = netip.MustParseAddr("1.3.5.7")
addr4216 = netip.MustParseAddr("4.2.1.6")
addr7531 = netip.MustParseAddr("7.5.3.1")
addr0 = netip.MustParseAddr("::")
addr1 = netip.MustParseAddr("::1")
addr2 = netip.MustParseAddr("::2")
addr3 = netip.MustParseAddr("::3")
addr4 = netip.MustParseAddr("::4")
addr42 = netip.MustParseAddr("::42")
addr13 = netip.MustParseAddr("::13")
addr31 = netip.MustParseAddr("::31")
hostsSrc = "./" + filepath.Join("./testdata", "etc_hosts")
testHosts = map[netip.Addr][]*hostsfile.Record{
addr1000: {{
Addr: addr1000,
Source: hostsSrc,
Names: []string{"hello", "hello.world"},
}, {
Addr: addr1000,
Source: hostsSrc,
Names: []string{"hello.world.again"},
}, {
Addr: addr1000,
Source: hostsSrc,
Names: []string{"hello.world"},
}},
addr1001: {{
Addr: addr1001,
Source: hostsSrc,
Names: []string{"simplehost"},
}, {
Addr: addr1001,
Source: hostsSrc,
Names: []string{"simplehost"},
}},
addr1002: {{
Addr: addr1002,
Source: hostsSrc,
Names: []string{"a.whole", "lot.of", "aliases", "for.testing"},
}},
addr1003: {{
Addr: addr1003,
Source: hostsSrc,
Names: []string{"*"},
}},
addr1004: {{
Addr: addr1004,
Source: hostsSrc,
Names: []string{"*.com"},
}},
addr1357: {{
Addr: addr1357,
Source: hostsSrc,
Names: []string{"domain4", "domain4.alias"},
}},
addr7531: {{
Addr: addr7531,
Source: hostsSrc,
Names: []string{"domain4.alias", "domain4"},
}},
addr4216: {{
Addr: addr4216,
Source: hostsSrc,
Names: []string{"domain", "domain.alias"},
}},
addr0: {{
Addr: addr0,
Source: hostsSrc,
Names: []string{"hello", "hello.world"},
}, {
Addr: addr0,
Source: hostsSrc,
Names: []string{"hello.world.again"},
}, {
Addr: addr0,
Source: hostsSrc,
Names: []string{"hello.world"},
}},
addr1: {{
Addr: addr1,
Source: hostsSrc,
Names: []string{"simplehost"},
}, {
Addr: addr1,
Source: hostsSrc,
Names: []string{"simplehost"},
}},
addr2: {{
Addr: addr2,
Source: hostsSrc,
Names: []string{"a.whole", "lot.of", "aliases", "for.testing"},
}},
addr3: {{
Addr: addr3,
Source: hostsSrc,
Names: []string{"*"},
}},
addr4: {{
Addr: addr4,
Source: hostsSrc,
Names: []string{"*.com"},
}},
addr42: {{
Addr: addr42,
Source: hostsSrc,
Names: []string{"domain.alias", "domain"},
}},
addr13: {{
Addr: addr13,
Source: hostsSrc,
Names: []string{"domain6", "domain6.alias"},
}},
addr31: {{
Addr: addr31,
Source: hostsSrc,
Names: []string{"domain6.alias", "domain6"},
}},
}
)
func TestNewHostsContainer(t *testing.T) {
const dirname = "dir"
const filename = "file1"
@ -267,7 +132,21 @@ func TestHostsContainer_refresh(t *testing.T) {
anotherIPStr := "1.2.3.4"
anotherIP := netip.MustParseAddr(anotherIPStr)
testFS := fstest.MapFS{"dir/file1": &fstest.MapFile{Data: []byte(ipStr + ` hostname` + nl)}}
r1 := &hostsfile.Record{
Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}
r2 := &hostsfile.Record{
Addr: anotherIP,
Source: "file2",
Names: []string{"alias"},
}
r1Data, _ := r1.MarshalText()
r2Data, _ := r2.MarshalText()
testFS := fstest.MapFS{"dir/file1": &fstest.MapFile{Data: r1Data}}
// event is a convenient alias for an empty struct{} to emit test events.
type event = struct{}
@ -289,172 +168,47 @@ func TestHostsContainer_refresh(t *testing.T) {
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
checkRefresh := func(t *testing.T, want aghnet.Hosts) {
t.Helper()
upd, ok := aghchan.MustReceive(hc.Upd(), 1*time.Second)
require.True(t, ok)
assert.Equal(t, want, upd)
}
strg, _ := hostsfile.NewDefaultStorage()
strg.Add(r1)
t.Run("initial_refresh", func(t *testing.T) {
checkRefresh(t, aghnet.Hosts{
ip: {{
Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}},
})
upd, ok := testutil.RequireReceive(t, hc.Upd(), 1*time.Second)
require.True(t, ok)
assert.True(t, strg.Equal(upd))
})
strg.Add(r2)
t.Run("second_refresh", func(t *testing.T) {
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(anotherIPStr + ` alias` + nl)}
testFS["dir/file2"] = &fstest.MapFile{Data: r2Data}
eventsCh <- event{}
checkRefresh(t, aghnet.Hosts{
ip: {{
Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}},
anotherIP: {{
Addr: anotherIP,
Source: "file2",
Names: []string{"alias"},
}},
})
upd, ok := testutil.RequireReceive(t, hc.Upd(), 1*time.Second)
require.True(t, ok)
assert.True(t, strg.Equal(upd))
})
t.Run("double_refresh", func(t *testing.T) {
// Make a change once.
testFS["dir/file1"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)}
testFS["dir/file1"] = &fstest.MapFile{Data: []byte(ipStr + " alias\n")}
eventsCh <- event{}
// Require the changes are written.
require.Eventually(t, func() bool {
ips := hc.MatchName("hostname")
current, ok := testutil.RequireReceive(t, hc.Upd(), 1*time.Second)
require.True(t, ok)
return len(ips) == 0
}, 5*time.Second, time.Second/2)
require.Empty(t, current.ByName("hostname"))
// Make a change again.
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` hostname` + nl)}
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + " hostname\n")}
eventsCh <- event{}
// Require the changes are written.
require.Eventually(t, func() bool {
ips := hc.MatchName("hostname")
current, ok = testutil.RequireReceive(t, hc.Upd(), 1*time.Second)
require.True(t, ok)
return len(ips) > 0
}, 5*time.Second, time.Second/2)
assert.Len(t, hc.Upd(), 1)
require.NotEmpty(t, current.ByName("hostname"))
})
}
func TestHostsContainer_MatchName(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
stubWatcher := aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil },
}
testCases := []struct {
req string
name string
want []*hostsfile.Record
}{{
req: "simplehost",
name: "simple",
want: append(testHosts[addr1001], testHosts[addr1]...),
}, {
req: "hello.world",
name: "hello_alias",
want: []*hostsfile.Record{
testHosts[addr1000][0],
testHosts[addr1000][2],
testHosts[addr0][0],
testHosts[addr0][2],
},
}, {
req: "hello.world.again",
name: "other_line_alias",
want: []*hostsfile.Record{
testHosts[addr1000][1],
testHosts[addr0][1],
},
}, {
req: "say.hello",
name: "hello_subdomain",
want: nil,
}, {
req: "say.hello.world",
name: "hello_alias_subdomain",
want: nil,
}, {
req: "for.testing",
name: "lots_of_aliases",
want: append(testHosts[addr1002], testHosts[addr2]...),
}, {
req: "nonexistent.example",
name: "non-existing",
want: nil,
}, {
req: "domain",
name: "issue_4216_4_6",
want: append(testHosts[addr4216], testHosts[addr42]...),
}, {
req: "domain4",
name: "issue_4216_4",
want: append(testHosts[addr1357], testHosts[addr7531]...),
}, {
req: "domain6",
name: "issue_4216_6",
want: append(testHosts[addr13], testHosts[addr31]...),
}}
hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
recs := hc.MatchName(tc.req)
assert.Equal(t, tc.want, recs)
})
}
}
func TestHostsContainer_MatchAddr(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
stubWatcher := aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil },
}
hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
testCases := []struct {
req netip.Addr
name string
want []*hostsfile.Record
}{{
req: netip.AddrFrom4([4]byte{1, 0, 0, 1}),
name: "reverse",
want: testHosts[addr1001],
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
recs := hc.MatchAddr(tc.req)
assert.Equal(t, tc.want, recs)
})
}
}

32
internal/aghnet/hostscontainer_windows.go
View file

@ -1,32 +0,0 @@
//go:build windows
package aghnet
import (
"os"
"path"
"path/filepath"
"strings"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/sys/windows"
)
func defaultHostsPaths() (paths []string) {
sysDir, err := windows.GetSystemDirectory()
if err != nil {
log.Error("aghnet: getting system directory: %s", err)
return []string{}
}
// Split all the elements of the path to join them afterwards. This is
// needed to make the Windows-specific path string returned by
// windows.GetSystemDirectory to be compatible with fs.FS.
pathElems := strings.Split(sysDir, string(os.PathSeparator))
if len(pathElems) > 0 && pathElems[0] == filepath.VolumeName(sysDir) {
pathElems = pathElems[1:]
}
return []string{path.Join(append(pathElems, "drivers/etc/hosts")...)}
}

5
internal/aghnet/net_test.go
View file

@ -1,11 +1,9 @@
package aghnet_test
import (
"io/fs"
"net"
"net/netip"
"net/url"
"os"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
@ -18,9 +16,6 @@ func TestMain(m *testing.M) {
testutil.DiscardLogOutput(m)
}
// testdata is the filesystem containing data for testing the package.
var testdata fs.FS = os.DirFS("./testdata")
func TestParseAddrPort(t *testing.T) {
const defaultPort = 1

38
internal/aghnet/testdata/etc_hosts vendored
View file

@ -1,38 +0,0 @@
#
# Test /etc/hosts file
#
1.0.0.1 simplehost
1.0.0.0 hello hello.world
# See https://github.com/AdguardTeam/AdGuardHome/issues/3846.
1.0.0.2 a.whole lot.of aliases for.testing
# See https://github.com/AdguardTeam/AdGuardHome/issues/3946.
1.0.0.3 *
1.0.0.4 *.com
# See https://github.com/AdguardTeam/AdGuardHome/issues/4079.
1.0.0.0 hello.world.again
# Duplicates of a main host and an alias.
1.0.0.1 simplehost
1.0.0.0 hello.world
# Same for IPv6.
::1 simplehost
:: hello hello.world
::2 a.whole lot.of aliases for.testing
::3 *
::4 *.com
:: hello.world.again
::1 simplehost
:: hello.world
# See https://github.com/AdguardTeam/AdGuardHome/issues/4216.
4.2.1.6 domain domain.alias
::42 domain.alias domain
1.3.5.7 domain4 domain4.alias
7.5.3.1 domain4.alias domain4
::13 domain6 domain6.alias
::31 domain6.alias domain6

1
internal/aghnet/testdata/ifaces vendored
View file

@ -1 +0,0 @@
iface sample_name inet static

5
internal/aghnet/testdata/include-subsources vendored
View file

@ -1,5 +0,0 @@
# The "testdata" part is added here because the test is actually run from the
# parent directory. Real interface files usually contain only absolute paths.
source ./testdata/ifaces
source ./testdata/*

11
internal/dnsforward/dnsforward.go
View file

@ -142,7 +142,7 @@ type Server struct {
// PTR resolving.
sysResolvers SystemResolvers
// etcHosts contains the data from the system's hosts files.
// etcHosts contains the current data from the system's hosts files.
etcHosts upstream.Resolver
// bootstrap is the resolver for upstreams' hostnames.
@ -239,6 +239,11 @@ func NewServer(p DNSCreateParams) (s *Server, err error) {
p.Anonymizer = aghnet.NewIPMut(nil)
}
var etcHosts upstream.Resolver
if p.EtcHosts != nil {
etcHosts = upstream.NewHostsResolver(p.EtcHosts)
}
s = &Server{
dnsFilter: p.DNSFilter,
dhcpServer: p.DHCPServer,
@ -247,6 +252,7 @@ func NewServer(p DNSCreateParams) (s *Server, err error) {
privateNets: p.PrivateNets,
// TODO(e.burkov): Use some case-insensitive string comparison.
localDomainSuffix: strings.ToLower(localDomainSuffix),
etcHosts: etcHosts,
recDetector: newRecursionDetector(recursionTTL, cachedRecurrentReqNum),
clientIDCache: cache.New(cache.Config{
EnableLRU: true,
@ -257,9 +263,6 @@ func NewServer(p DNSCreateParams) (s *Server, err error) {
ServePlainDNS: true,
},
}
if p.EtcHosts != nil {
s.etcHosts = p.EtcHosts
}
s.sysResolvers, err = sysresolv.NewSystemResolvers(nil, defaultPlainDNSPort)
if err != nil {

3
internal/dnsforward/http_test.go
View file

@ -20,6 +20,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/httphdr"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/testutil"
@ -526,7 +527,7 @@ func TestServer_HandleTestUpstreamDNS(t *testing.T) {
},
ServePlainDNS: true,
}, nil)
srv.etcHosts = hc
srv.etcHosts = upstream.NewHostsResolver(hc)
startDeferStop(t, srv)
testCases := []struct {

37
internal/filtering/dnsrewrite.go
View file

@ -1,7 +1,6 @@
package filtering
import (
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
@ -95,39 +94,3 @@ func (d *DNSFilter) processDNSResultRewrites(
return res
}
// appendRewriteResultFromHost appends the rewrite result from rec to vals and
// resRules.
func appendRewriteResultFromHost(
vals []rules.RRValue,
resRules []*ResultRule,
rec *hostsfile.Record,
qtype uint16,
) (updatedVals []rules.RRValue, updatedRules []*ResultRule) {
switch qtype {
case dns.TypeA:
if !rec.Addr.Is4() {
return vals, resRules
}
vals = append(vals, rec.Addr)
case dns.TypeAAAA:
if !rec.Addr.Is6() {
return vals, resRules
}
vals = append(vals, rec.Addr)
case dns.TypePTR:
for _, name := range rec.Names {
vals = append(vals, name)
}
}
recText, _ := rec.MarshalText()
resRules = append(resRules, &ResultRule{
FilterListID: SysHostsListID,
Text: string(recText),
})
return vals, resRules
}

19
internal/filtering/dnsrewrite_test.go
View file

@ -220,15 +220,19 @@ func TestDNSFilter_CheckHost_hostsContainer(t *testing.T) {
addrv4 := netip.MustParseAddr("1.2.3.4")
addrv6 := netip.MustParseAddr("::1")
addrMapped := netip.MustParseAddr("::ffff:1.2.3.4")
addrv4Dup := netip.MustParseAddr("4.3.2.1")
data := fmt.Sprintf(
""+
"%s v4.host.example\n"+
"%s v6.host.example\n"+
"%s mapped.host.example\n",
"%[1]s v4.host.example\n"+
"%[2]s v6.host.example\n"+
"%[3]s mapped.host.example\n"+
"%[4]s v4.host.with-dup\n"+
"%[4]s v4.host.with-dup\n",
addrv4,
addrv6,
addrMapped,
addrv4Dup,
)
files := fstest.MapFS{
@ -343,6 +347,15 @@ func TestDNSFilter_CheckHost_hostsContainer(t *testing.T) {
dtyp: dns.TypeCNAME,
wantRules: nil,
wantResps: nil,
}, {
name: "v4_dup",
host: "v4.host.with-dup",
dtyp: dns.TypeA,
wantRules: []*ResultRule{{
Text: "4.3.2.1 v4.host.with-dup",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{addrv4Dup},
}}
for _, tc := range testCases {

100
internal/filtering/filtering.go
View file

@ -18,7 +18,6 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
@ -100,7 +99,7 @@ type Config struct {
// system configuration files (e.g. /etc/hosts).
//
// TODO(e.burkov): Move it to dnsforward entirely.
EtcHosts *aghnet.HostsContainer `yaml:"-"`
EtcHosts hostsfile.Storage `yaml:"-"`
// Called when the configuration is changed by HTTP request
ConfigModified func() `yaml:"-"`
@ -482,15 +481,6 @@ func (d *DNSFilter) SetProtectionEnabled(status bool) {
d.conf.ProtectionEnabled = status
}
// EtcHostsRecords returns the hosts records for the hostname.
func (d *DNSFilter) EtcHostsRecords(hostname string) (recs []*hostsfile.Record) {
if d.conf.EtcHosts != nil {
return d.conf.EtcHosts.MatchName(hostname)
}
return recs
}
// SetBlockingMode sets blocking mode properties.
func (d *DNSFilter) SetBlockingMode(mode BlockingMode, bIPv4, bIPv6 netip.Addr) {
d.confMu.Lock()
@ -637,39 +627,10 @@ func (d *DNSFilter) matchSysHosts(
) (res Result, err error) {
// TODO(e.burkov): Where else is this checked?
if !setts.FilteringEnabled || d.conf.EtcHosts == nil {
return res, nil
}
var recs []*hostsfile.Record
switch qtype {
case dns.TypeA, dns.TypeAAAA:
recs = d.conf.EtcHosts.MatchName(host)
case dns.TypePTR:
var ip net.IP
ip, err = netutil.IPFromReversedAddr(host)
if err != nil {
log.Debug("filtering: failed to parse PTR record %q: %s", host, err)
return res, nil
}
addr, _ := netip.AddrFromSlice(ip)
recs = d.conf.EtcHosts.MatchAddr(addr)
default:
log.Debug("filtering: unsupported query type %s", dns.Type(qtype))
}
var vals []rules.RRValue
var resRules []*ResultRule
resRulesLen := 0
for _, rec := range recs {
vals, resRules = appendRewriteResultFromHost(vals, resRules, rec, qtype)
if len(resRules) > resRulesLen {
resRulesLen = len(resRules)
log.Debug("filtering: matched %s in %q", host, rec.Source)
}
return Result{}, nil
}
vals, rs := hostsRewrites(qtype, host, d.conf.EtcHosts)
if len(vals) > 0 {
res.DNSRewriteResult = &DNSRewriteResult{
Response: DNSRewriteResultResponse{
@ -677,13 +638,64 @@ func (d *DNSFilter) matchSysHosts(
},
RCode: dns.RcodeSuccess,
}
res.Rules = resRules
res.Reason = RewrittenRule
res.Rules = rs
res.Reason = RewrittenAutoHosts
}
return res, nil
}
// hostsRewrites returns values and rules matched by qt and host within hs.
func hostsRewrites(
qtype uint16,
host string,
hs hostsfile.Storage,
) (vals []rules.RRValue, rs []*ResultRule) {
var isValidProto func(netip.Addr) (ok bool)
switch qtype {
case dns.TypeA:
isValidProto = netip.Addr.Is4
case dns.TypeAAAA:
isValidProto = netip.Addr.Is6
case dns.TypePTR:
// TODO(e.burkov): Add some [netip]-aware alternative to [netutil].
ip, err := netutil.IPFromReversedAddr(host)
if err != nil {
log.Debug("filtering: failed to parse PTR record %q: %s", host, err)
return nil, nil
}
addr, _ := netip.AddrFromSlice(ip)
for _, name := range hs.ByAddr(addr) {
vals = append(vals, name)
rs = append(rs, &ResultRule{
Text: fmt.Sprintf("%s %s", addr, name),
FilterListID: SysHostsListID,
})
}
return vals, rs
default:
log.Debug("filtering: unsupported qtype %d", qtype)
return nil, nil
}
for _, addr := range hs.ByName(host) {
if isValidProto(addr) {
vals = append(vals, addr)
rs = append(rs, &ResultRule{
Text: fmt.Sprintf("%s %s", addr, host),
FilterListID: SysHostsListID,
})
}
}
return vals, rs
}
// processRewrites performs filtering based on the legacy rewrite records.
//
// Firstly, it finds CNAME rewrites for host. If the CNAME is the same as host,

17
internal/home/clients.go
View file

@ -20,6 +20,7 @@ import (
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"golang.org/x/exp/maps"
@ -139,6 +140,9 @@ func (clients *clientsContainer) Init(
return nil
}
// handleHostsUpdates receives the updates from the hosts container and adds
// them to the clients container. It's used to be called in a separate
// goroutine.
func (clients *clientsContainer) handleHostsUpdates() {
for upd := range clients.etcHosts.Upd() {
clients.addFromHostsFile(upd)
@ -870,21 +874,24 @@ func (clients *clientsContainer) rmHostsBySrc(src client.Source) {
// addFromHostsFile fills the client-hostname pairing index from the system's
// hosts files.
func (clients *clientsContainer) addFromHostsFile(hosts aghnet.Hosts) {
func (clients *clientsContainer) addFromHostsFile(hosts *hostsfile.DefaultStorage) {
clients.lock.Lock()
defer clients.lock.Unlock()
clients.rmHostsBySrc(client.SourceHostsFile)
n := 0
for addr, rec := range hosts {
hosts.RangeNames(func(addr netip.Addr, names []string) (cont bool) {
// Only the first name of the first record is considered a canonical
// hostname for the IP address.
//
// TODO(e.burkov): Consider using all the names from all the records.
clients.addHostLocked(addr, rec[0].Names[0], client.SourceHostsFile)
n++
}
if clients.addHostLocked(addr, names[0], client.SourceHostsFile) {
n++
}
return true
})
log.Debug("clients: added %d client aliases from system hosts file", n)
}

13
internal/home/home.go
View file

@ -35,8 +35,10 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/version"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/osutil"
"github.com/AdguardTeam/golibs/stringutil"
"golang.org/x/exp/slices"
)
@ -231,11 +233,12 @@ func setupHostsContainer() (err error) {
return fmt.Errorf("initing hosts watcher: %w", err)
}
Context.etcHosts, err = aghnet.NewHostsContainer(
aghos.RootDirFS(),
hostsWatcher,
aghnet.DefaultHostsPaths()...,
)
paths, err := hostsfile.DefaultHostsPaths()
if err != nil {
return fmt.Errorf("getting default system hosts paths: %w", err)
}
Context.etcHosts, err = aghnet.NewHostsContainer(osutil.RootDirFS(), hostsWatcher, paths...)
if err != nil {
closeErr := hostsWatcher.Close()
if errors.Is(err, aghnet.ErrNoHostsPaths) {

2
scripts/make/go-lint.sh
View file

@ -206,7 +206,6 @@ run_linter gocognit --over='11'\
run_linter gocognit --over='10'\
./internal/aghalg/\
./internal/aghchan/\
./internal/aghhttp/\
./internal/aghrenameio/\
./internal/aghtest/\
@ -244,7 +243,6 @@ run_linter nilness ./...
# TODO(a.garipov): Enable for all.
run_linter fieldalignment \
./internal/aghalg/\
./internal/aghchan/\
./internal/aghhttp/\
./internal/aghos/\
./internal/aghrenameio/\