Pull request 1858: AG-22594-imp-whois

Merge in DNS/adguard-home from AG-22594-imp-whois to master

Squashed commit of the following:

commit 093feed53291d02469fb1bd8d99472597ebd5015
Merge: 956d20dc4 ca313521d
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jun 21 12:42:40 2023 +0300

    Merge branch 'master' into AG-22594-imp-whois

commit 956d20dc473dcec90895b6f618fc56e96e9ff833
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jun 20 18:30:48 2023 +0300

    whois: imp code more

commit c771fd9c5e4d90e76d079a0d25ab097ab5652a42
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jun 20 15:05:45 2023 +0300

    whois: imp code

commit 21900fd468e10d9aee22149a6312b8596ff39810
Merge: 8dbe132c0 371261b2c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jun 20 11:34:06 2023 +0300

    Merge branch 'master' into AG-22594-imp-whois

commit 8dbe132c08d3ad4a63b0d4bdb9d00a5bc25971f4
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jun 20 11:33:26 2023 +0300

    whois: imp code more

commit f5e761a260237579c67cbd48f01ea90499bff6b0
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Jun 19 16:04:35 2023 +0300

    whois: imp code

commit 2780f7e16aacddad8736f83b77ef9bfa1271f8b1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 16 17:33:47 2023 +0300

    all: imp code

commit 1fc67016068b745a46b3d0d341ab14f9f5bdc9aa
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 16 17:29:19 2023 +0300

    whois: imp tests

commit 204761870764fb10feea20065d79dee8c321e70b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 16 11:55:37 2023 +0300

    all: upd deps

commit ded4f59498c5c544277b9c8e249567626547680e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jun 14 20:43:32 2023 +0300

    all: imp tests

commit 0eed9834ff9dd94d0788ce69d0bb0521fa725410
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jun 14 19:31:49 2023 +0300

    all: imp code

commit 9f867587c8ad87363b8c8b061ead536c1ec59c5d
Merge: 504e9484d 681c604c2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jun 13 14:20:44 2023 +0300

    Merge branch 'master' into AG-22594-imp-whois

commit 504e9484dd84ab9d7c84a3f8399993d6422d3b67
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jun 13 14:18:06 2023 +0300

    all: imp cache

commit c492abe41ace7ad76fcd4e297c22b910a90fec30
Merge: db36adb9c 826b314f1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 9 16:06:12 2023 +0300

    Merge branch 'master' into AG-22594-imp-whois

commit db36adb9c14ce92b3971db0d87ec313d5bcd787e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 9 15:53:33 2023 +0300

    all: add todo

commit 5cf192de9f93cd0d8521a3a6b4ded7f2bc5e0031
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jun 8 14:59:26 2023 +0300

    all: imp docs

commit 021aa3eb5b9476a93b4af5fc90cc9ccf014ca152
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Jun 5 18:35:25 2023 +0300

    all: imp naming

commit 4626c3a7fa3f2543501806c9fa1a19531547f394
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 2 17:41:00 2023 +0300

    all: imp tests

commit 1afcc9605ca176e4c7f76a03a2c996cf7d6bde13
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 2 12:44:32 2023 +0300

    all: imp docs

commit cdd0544ff1a63faed5ced3dae6bfb3b783e45428
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jun 1 17:21:37 2023 +0300

    all: add docs

... and 2 more commits
This commit is contained in:
Stanislav Chzhen 2023-06-21 12:53:53 +03:00
parent ca313521dc
commit 06d465b0d1
13 changed files with 658 additions and 500 deletions

14
go.mod
View file

@ -4,10 +4,11 @@ go 1.19
require (
github.com/AdguardTeam/dnsproxy v0.50.2
github.com/AdguardTeam/golibs v0.13.2
github.com/AdguardTeam/golibs v0.13.3
github.com/AdguardTeam/urlfilter v0.16.1
github.com/NYTimes/gziphandler v1.1.1
github.com/ameshkov/dnscrypt/v2 v2.2.7
github.com/bluele/gcache v0.0.2
github.com/digineo/go-ipset/v2 v2.2.1
github.com/dimfeld/httptreemux/v5 v5.5.0
github.com/fsnotify/fsnotify v1.6.0
@ -27,13 +28,13 @@ require (
github.com/mdlayher/raw v0.1.0
github.com/miekg/dns v1.1.54
github.com/quic-go/quic-go v0.35.1
github.com/stretchr/testify v1.8.2
github.com/stretchr/testify v1.8.4
github.com/ti-mo/netfilter v0.5.0
go.etcd.io/bbolt v1.3.7
golang.org/x/crypto v0.9.0
golang.org/x/crypto v0.10.0
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
golang.org/x/net v0.10.0
golang.org/x/sys v0.8.0
golang.org/x/net v0.11.0
golang.org/x/sys v0.9.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v3 v3.0.1
howett.net/plist v1.0.0
@ -44,7 +45,6 @@ require (
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect
github.com/ameshkov/dnsstamps v1.0.3 // indirect
github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/golang/mock v1.6.0 // indirect
@ -61,6 +61,6 @@ require (
github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/text v0.10.0 // indirect
golang.org/x/tools v0.9.3 // indirect
)

28
go.sum
View file

@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.2 h1:p1471SsMZ6SMo7T51Olw4aNluahvMwSLMorwx
github.com/AdguardTeam/dnsproxy v0.50.2/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
github.com/AdguardTeam/golibs v0.13.2 h1:BPASsyQKmb+b8VnvsNOHp7bKfcZl9Z+Z2UhPjOiupSc=
github.com/AdguardTeam/golibs v0.13.2/go.mod h1:7ylQLv2Lqsc3UW3jHoITynYk6Y1tYtgEMkR09ppfsN8=
github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=
@ -113,17 +113,13 @@ github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5
github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA=
github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/ti-mo/netfilter v0.2.0/go.mod h1:8GbBGsY/8fxtyIdfwy29JiluNcPK4K7wIT+x42ipqUU=
github.com/ti-mo/netfilter v0.5.0 h1:MZmsUw5bFRecOb0AeyjOPxTHg4UxYzyEs0Ek/6Lxoy8=
github.com/ti-mo/netfilter v0.5.0/go.mod h1:nt+8B9hx/QpqHr7Hazq+2qMCCA8u2OTkyc/7+U9ARz8=
@ -138,8 +134,8 @@ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@ -156,8 +152,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
@ -181,16 +177,16 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=

View file

@ -7,6 +7,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/golibs/stringutil"
)
@ -127,14 +128,13 @@ func (cs clientSource) MarshalText() (text []byte, err error) {
// RuntimeClient is a client information about which has been obtained using the
// source described in the Source field.
type RuntimeClient struct {
WHOISInfo *RuntimeClientWHOISInfo
Host string
Source clientSource
}
// WHOIS is the filtered WHOIS data of a client.
WHOIS *whois.Info
// RuntimeClientWHOISInfo is the filtered WHOIS data for a runtime client.
type RuntimeClientWHOISInfo struct {
City string `json:"city,omitempty"`
Country string `json:"country,omitempty"`
Orgname string `json:"orgname,omitempty"`
// Host is the host name of a client.
Host string
// Source is the source from which the information about the client has
// been obtained.
Source clientSource
}

View file

@ -14,6 +14,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/errors"
@ -307,18 +308,6 @@ func (clients *clientsContainer) clientSource(ip netip.Addr) (src clientSource)
return rc.Source
}
func toQueryLogWHOIS(wi *RuntimeClientWHOISInfo) (cw *querylog.ClientWHOIS) {
if wi == nil {
return &querylog.ClientWHOIS{}
}
return &querylog.ClientWHOIS{
City: wi.City,
Country: wi.Country,
Orgname: wi.Orgname,
}
}
// findMultiple is a wrapper around Find to make it a valid client finder for
// the query log. c is never nil; if no information about the client is found,
// it returns an artificial client record by only setting the blocking-related
@ -352,7 +341,7 @@ func (clients *clientsContainer) clientOrArtificial(
defer func() {
c.Disallowed, c.DisallowedRule = clients.dnsServer.IsBlockedClient(ip, id)
if c.WHOIS == nil {
c.WHOIS = &querylog.ClientWHOIS{}
c.WHOIS = &whois.Info{}
}
}()
@ -369,7 +358,7 @@ func (clients *clientsContainer) clientOrArtificial(
if ok {
return &querylog.Client{
Name: rc.Host,
WHOIS: toQueryLogWHOIS(rc.WHOISInfo),
WHOIS: rc.WHOIS,
}, false
}
@ -701,7 +690,7 @@ func (clients *clientsContainer) Update(prev, c *Client) (err error) {
}
// setWHOISInfo sets the WHOIS information for a client.
func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWHOISInfo) {
func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *whois.Info) {
clients.lock.Lock()
defer clients.lock.Unlock()
@ -713,7 +702,7 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWH
rc, ok := clients.ipToRC[ip]
if ok {
rc.WHOISInfo = wi
rc.WHOIS = wi
log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi)
return
@ -725,7 +714,7 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWH
Source: ClientSourceWHOIS,
}
rc.WHOISInfo = wi
rc.WHOIS = wi
clients.ipToRC[ip] = rc
@ -762,9 +751,9 @@ func (clients *clientsContainer) addHostLocked(
rc.Source = src
} else {
rc = &RuntimeClient{
Host: host,
Source: src,
WHOISInfo: &RuntimeClientWHOISInfo{},
Host: host,
Source: src,
WHOIS: &whois.Info{},
}
clients.ipToRC[ip] = rc

View file

@ -9,7 +9,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -199,7 +199,7 @@ func TestClients(t *testing.T) {
func TestClientsWHOIS(t *testing.T) {
clients := newClientsContainer()
whois := &RuntimeClientWHOISInfo{
whois := &whois.Info{
Country: "AU",
Orgname: "Example Org",
}
@ -210,7 +210,7 @@ func TestClientsWHOIS(t *testing.T) {
rc := clients.ipToRC[ip]
require.NotNil(t, rc)
assert.Equal(t, rc.WHOISInfo, whois)
assert.Equal(t, rc.WHOIS, whois)
})
t.Run("existing_auto-client", func(t *testing.T) {
@ -222,7 +222,7 @@ func TestClientsWHOIS(t *testing.T) {
rc := clients.ipToRC[ip]
require.NotNil(t, rc)
assert.Equal(t, rc.WHOISInfo, whois)
assert.Equal(t, rc.WHOIS, whois)
})
t.Run("can't_set_manually-added", func(t *testing.T) {

View file

@ -9,6 +9,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
)
// clientJSON is a common structure used by several handlers to deal with
@ -28,7 +29,8 @@ type clientJSON struct {
// the allowlist.
DisallowedRule *string `json:"disallowed_rule,omitempty"`
WHOISInfo *RuntimeClientWHOISInfo `json:"whois_info,omitempty"`
// WHOIS is the filtered WHOIS data of a client.
WHOIS *whois.Info `json:"whois_info,omitempty"`
SafeSearchConf *filtering.SafeSearchConfig `json:"safe_search"`
Name string `json:"name"`
@ -51,7 +53,7 @@ type clientJSON struct {
}
type runtimeClientJSON struct {
WHOISInfo *RuntimeClientWHOISInfo `json:"whois_info"`
WHOIS *whois.Info `json:"whois_info"`
IP netip.Addr `json:"ip"`
Name string `json:"name"`
@ -78,7 +80,7 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, r *http
for ip, rc := range clients.ipToRC {
cj := runtimeClientJSON{
WHOISInfo: rc.WHOISInfo,
WHOIS: rc.WHOIS,
Name: rc.Host,
Source: rc.Source,
@ -344,16 +346,16 @@ func (clients *clientsContainer) findRuntime(ip netip.Addr, idStr string) (cj *c
IDs: []string{idStr},
Disallowed: &disallowed,
DisallowedRule: &rule,
WHOISInfo: &RuntimeClientWHOISInfo{},
WHOIS: &whois.Info{},
}
return cj
}
cj = &clientJSON{
Name: rc.Host,
IDs: []string{idStr},
WHOISInfo: rc.WHOISInfo,
Name: rc.Host,
IDs: []string{idStr},
WHOIS: rc.WHOIS,
}
disallowed, rule := clients.dnsServer.IsBlockedClient(ip, idStr)

View file

@ -8,6 +8,7 @@ import (
"net/url"
"os"
"path/filepath"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@ -17,6 +18,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
@ -25,7 +27,7 @@ import (
yaml "gopkg.in/yaml.v3"
)
// Default ports.
// Default listening ports.
const (
defaultPortDNS = 53
defaultPortHTTP = 80
@ -169,13 +171,72 @@ func initDNSServer(
Context.rdns = NewRDNS(Context.dnsServer, &Context.clients, config.DNS.UsePrivateRDNS)
}
if config.Clients.Sources.WHOIS {
Context.whois = initWHOIS(&Context.clients)
}
initWHOIS()
return nil
}
// initWHOIS initializes the WHOIS.
//
// TODO(s.chzhen): Consider making configurable.
func initWHOIS() {
const (
// defaultQueueSize is the size of queue of IPs for WHOIS processing.
defaultQueueSize = 255
// defaultTimeout is the timeout for WHOIS requests.
defaultTimeout = 5 * time.Second
// defaultCacheSize is the maximum size of the cache. If it's zero,
// cache size is unlimited.
defaultCacheSize = 10_000
// defaultMaxConnReadSize is an upper limit in bytes for reading from
// net.Conn.
defaultMaxConnReadSize = 64 * 1024
// defaultMaxRedirects is the maximum redirects count.
defaultMaxRedirects = 5
// defaultMaxInfoLen is the maximum length of whois.Info fields.
defaultMaxInfoLen = 250
// defaultIPTTL is the Time to Live duration for cached IP addresses.
defaultIPTTL = 1 * time.Hour
)
Context.whoisCh = make(chan netip.Addr, defaultQueueSize)
var w whois.Interface
if config.Clients.Sources.WHOIS {
w = whois.New(&whois.Config{
DialContext: customDialContext,
ServerAddr: whois.DefaultServer,
Port: whois.DefaultPort,
Timeout: defaultTimeout,
CacheSize: defaultCacheSize,
MaxConnReadSize: defaultMaxConnReadSize,
MaxRedirects: defaultMaxRedirects,
MaxInfoLen: defaultMaxInfoLen,
CacheTTL: defaultIPTTL,
})
} else {
w = whois.Empty{}
}
go func() {
defer log.OnPanic("whois")
for ip := range Context.whoisCh {
info, changed := w.Process(context.Background(), ip)
if info != nil && changed {
Context.clients.setWHOISInfo(ip, info)
}
}
}()
}
// parseSubnetSet parses a slice of subnets. If the slice is empty, it returns
// a subnet set that matches all locally served networks, see
// [netutil.IsLocallyServed].
@ -218,9 +279,7 @@ func onDNSRequest(pctx *proxy.DNSContext) {
Context.rdns.Begin(ip)
}
if srcs.WHOIS && !netutil.IsSpecialPurposeAddr(ip) {
Context.whois.Begin(ip)
}
Context.whoisCh <- ip
}
func ipsToTCPAddrs(ips []netip.Addr, port int) (tcpAddrs []*net.TCPAddr) {
@ -463,9 +522,7 @@ func startDNSServer() error {
Context.rdns.Begin(ip)
}
if srcs.WHOIS && !netutil.IsSpecialPurposeAddr(ip) {
Context.whois.Begin(ip)
}
Context.whoisCh <- ip
}
return nil

View file

@ -57,7 +57,6 @@ type homeContext struct {
queryLog querylog.QueryLog // query log module
dnsServer *dnsforward.Server // DNS module
rdns *RDNS // rDNS module
whois *WHOIS // WHOIS module
dhcpServer dhcpd.Interface // DHCP module
auth *Auth // HTTP authentication module
filters *filtering.DNSFilter // DNS filtering module
@ -84,6 +83,9 @@ type homeContext struct {
client *http.Client
appSignalChannel chan os.Signal // Channel for receiving OS signals by the console app
// whoisCh is the channel for receiving IPs for WHOIS processing.
whoisCh chan netip.Addr
// tlsCipherIDs are the ID of the cipher suites that AdGuard Home must use.
tlsCipherIDs []uint16

View file

@ -1,259 +0,0 @@
package home
import (
"context"
"encoding/binary"
"fmt"
"io"
"net"
"net/netip"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
)
const (
defaultServer = "whois.arin.net"
defaultPort = "43"
maxValueLength = 250
whoisTTL = 1 * 60 * 60 // 1 hour
)
// WHOIS - module context
type WHOIS struct {
clients *clientsContainer
ipChan chan netip.Addr
// dialContext specifies the dial function for creating unencrypted TCP
// connections.
dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error)
// Contains IP addresses of clients
// An active IP address is resolved once again after it expires.
// If IP address couldn't be resolved, it stays here for some time to prevent further attempts to resolve the same IP.
ipAddrs cache.Cache
// TODO(a.garipov): Rewrite to use time.Duration. Like, seriously, why?
timeoutMsec uint
}
// initWHOIS creates the WHOIS module context.
func initWHOIS(clients *clientsContainer) *WHOIS {
w := WHOIS{
timeoutMsec: 5000,
clients: clients,
ipAddrs: cache.New(cache.Config{
EnableLRU: true,
MaxCount: 10000,
}),
dialContext: customDialContext,
ipChan: make(chan netip.Addr, 255),
}
go w.workerLoop()
return &w
}
// If the value is too large - cut it and append "..."
func trimValue(s string) string {
if len(s) <= maxValueLength {
return s
}
return s[:maxValueLength-3] + "..."
}
// isWHOISComment returns true if the string is empty or is a WHOIS comment.
func isWHOISComment(s string) (ok bool) {
return len(s) == 0 || s[0] == '#' || s[0] == '%'
}
// strmap is an alias for convenience.
type strmap = map[string]string
// whoisParse parses a subset of plain-text data from the WHOIS response into
// a string map.
func whoisParse(data string) (m strmap) {
m = strmap{}
var orgname string
lines := strings.Split(data, "\n")
for _, l := range lines {
if isWHOISComment(l) {
continue
}
kv := strings.SplitN(l, ":", 2)
if len(kv) != 2 {
continue
}
k := strings.ToLower(strings.TrimSpace(kv[0]))
v := strings.TrimSpace(kv[1])
if v == "" {
continue
}
switch k {
case "orgname", "org-name":
k = "orgname"
v = trimValue(v)
orgname = v
case "city", "country":
v = trimValue(v)
case "descr", "netname":
k = "orgname"
v = stringutil.Coalesce(orgname, v)
orgname = v
case "whois":
k = "whois"
case "referralserver":
k = "whois"
v = strings.TrimPrefix(v, "whois://")
default:
continue
}
m[k] = v
}
return m
}
// MaxConnReadSize is an upper limit in bytes for reading from net.Conn.
const MaxConnReadSize = 64 * 1024
// Send request to a server and receive the response
func (w *WHOIS) query(ctx context.Context, target, serverAddr string) (data string, err error) {
addr, _, _ := net.SplitHostPort(serverAddr)
if addr == "whois.arin.net" {
target = "n + " + target
}
conn, err := w.dialContext(ctx, "tcp", serverAddr)
if err != nil {
return "", err
}
defer func() { err = errors.WithDeferred(err, conn.Close()) }()
r, err := aghio.LimitReader(conn, MaxConnReadSize)
if err != nil {
return "", err
}
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(w.timeoutMsec) * time.Millisecond))
_, err = conn.Write([]byte(target + "\r\n"))
if err != nil {
return "", err
}
// This use of ReadAll is now safe, because we limited the conn Reader.
var whoisData []byte
whoisData, err = io.ReadAll(r)
if err != nil {
return "", err
}
return string(whoisData), nil
}
// Query WHOIS servers (handle redirects)
func (w *WHOIS) queryAll(ctx context.Context, target string) (string, error) {
server := net.JoinHostPort(defaultServer, defaultPort)
const maxRedirects = 5
for i := 0; i != maxRedirects; i++ {
resp, err := w.query(ctx, target, server)
if err != nil {
return "", err
}
log.Debug("whois: received response (%d bytes) from %s IP:%s", len(resp), server, target)
m := whoisParse(resp)
redir, ok := m["whois"]
if !ok {
return resp, nil
}
redir = strings.ToLower(redir)
_, _, err = net.SplitHostPort(redir)
if err != nil {
server = net.JoinHostPort(redir, defaultPort)
} else {
server = redir
}
log.Debug("whois: redirected to %s IP:%s", redir, target)
}
return "", fmt.Errorf("whois: redirect loop")
}
// Request WHOIS information
func (w *WHOIS) process(ctx context.Context, ip netip.Addr) (wi *RuntimeClientWHOISInfo) {
resp, err := w.queryAll(ctx, ip.String())
if err != nil {
log.Debug("whois: error: %s IP:%s", err, ip)
return nil
}
log.Debug("whois: IP:%s response: %d bytes", ip, len(resp))
m := whoisParse(resp)
wi = &RuntimeClientWHOISInfo{
City: m["city"],
Country: m["country"],
Orgname: m["orgname"],
}
// Don't return an empty struct so that the frontend doesn't get
// confused.
if *wi == (RuntimeClientWHOISInfo{}) {
return nil
}
return wi
}
// Begin - begin requesting WHOIS info
func (w *WHOIS) Begin(ip netip.Addr) {
ipBytes := ip.AsSlice()
now := uint64(time.Now().Unix())
expire := w.ipAddrs.Get(ipBytes)
if len(expire) != 0 {
exp := binary.BigEndian.Uint64(expire)
if exp > now {
return
}
}
expire = make([]byte, 8)
binary.BigEndian.PutUint64(expire, now+whoisTTL)
_ = w.ipAddrs.Set(ipBytes, expire)
log.Debug("whois: adding %s", ip)
select {
case w.ipChan <- ip:
default:
log.Debug("whois: queue is full")
}
}
// workerLoop processes the IP addresses it got from the channel and associates
// the retrieving WHOIS info with a client.
func (w *WHOIS) workerLoop() {
for ip := range w.ipChan {
info := w.process(context.Background(), ip)
if info == nil {
continue
}
w.clients.setWHOISInfo(ip, info)
}
}

View file

@ -1,152 +0,0 @@
package home
import (
"context"
"io"
"net"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeConn is a mock implementation of net.Conn to simplify testing.
//
// TODO(e.burkov): Search for other places in code where it may be used. Move
// into aghtest then.
type fakeConn struct {
// Conn is embedded here simply to make *fakeConn a net.Conn without
// actually implementing all methods.
net.Conn
data []byte
}
// Write implements net.Conn interface for *fakeConn. It always returns 0 and a
// nil error without mutating the slice.
func (c *fakeConn) Write(_ []byte) (n int, err error) {
return 0, nil
}
// Read implements net.Conn interface for *fakeConn. It puts the content of
// c.data field into b up to the b's capacity.
func (c *fakeConn) Read(b []byte) (n int, err error) {
return copy(b, c.data), io.EOF
}
// Close implements net.Conn interface for *fakeConn. It always returns nil.
func (c *fakeConn) Close() (err error) {
return nil
}
// SetReadDeadline implements net.Conn interface for *fakeConn. It always
// returns nil.
func (c *fakeConn) SetReadDeadline(_ time.Time) (err error) {
return nil
}
// fakeDial is a mock implementation of customDialContext to simplify testing.
func (c *fakeConn) fakeDial(ctx context.Context, network, addr string) (conn net.Conn, err error) {
return c, nil
}
func TestWHOIS(t *testing.T) {
const (
nl = "\n"
data = `OrgName: FakeOrg LLC` + nl +
`City: Nonreal` + nl +
`Country: Imagiland` + nl
)
fc := &fakeConn{
data: []byte(data),
}
w := WHOIS{
timeoutMsec: 5000,
dialContext: fc.fakeDial,
}
resp, err := w.queryAll(context.Background(), "1.2.3.4")
assert.NoError(t, err)
m := whoisParse(resp)
require.NotEmpty(t, m)
assert.Equal(t, "FakeOrg LLC", m["orgname"])
assert.Equal(t, "Imagiland", m["country"])
assert.Equal(t, "Nonreal", m["city"])
}
func TestWHOISParse(t *testing.T) {
const (
city = "Nonreal"
country = "Imagiland"
orgname = "FakeOrgLLC"
whois = "whois.example.net"
)
testCases := []struct {
want strmap
name string
in string
}{{
want: strmap{},
name: "empty",
in: ``,
}, {
want: strmap{},
name: "comments",
in: "%\n#",
}, {
want: strmap{},
name: "no_colon",
in: "city",
}, {
want: strmap{},
name: "no_value",
in: "city:",
}, {
want: strmap{"city": city},
name: "city",
in: `city: ` + city,
}, {
want: strmap{"country": country},
name: "country",
in: `country: ` + country,
}, {
want: strmap{"orgname": orgname},
name: "orgname",
in: `orgname: ` + orgname,
}, {
want: strmap{"orgname": orgname},
name: "orgname_hyphen",
in: `org-name: ` + orgname,
}, {
want: strmap{"orgname": orgname},
name: "orgname_descr",
in: `descr: ` + orgname,
}, {
want: strmap{"orgname": orgname},
name: "orgname_netname",
in: `netname: ` + orgname,
}, {
want: strmap{"whois": whois},
name: "whois",
in: `whois: ` + whois,
}, {
want: strmap{"whois": whois},
name: "referralserver",
in: `referralserver: whois://` + whois,
}, {
want: strmap{},
name: "other",
in: `other: value`,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := whoisParse(tc.in)
assert.Equal(t, tc.want, got)
})
}
}

View file

@ -1,23 +1,15 @@
package querylog
import "github.com/AdguardTeam/AdGuardHome/internal/whois"
// Client is the information required by the query log to match against clients
// during searches.
type Client struct {
WHOIS *ClientWHOIS `json:"whois,omitempty"`
Name string `json:"name"`
DisallowedRule string `json:"disallowed_rule"`
Disallowed bool `json:"disallowed"`
IgnoreQueryLog bool `json:"-"`
}
// ClientWHOIS is the filtered WHOIS data for the client.
//
// TODO(a.garipov): Merge with home.RuntimeClientWHOISInfo after the
// refactoring is done.
type ClientWHOIS struct {
City string `json:"city,omitempty"`
Country string `json:"country,omitempty"`
Orgname string `json:"orgname,omitempty"`
WHOIS *whois.Info `json:"whois,omitempty"`
Name string `json:"name"`
DisallowedRule string `json:"disallowed_rule"`
Disallowed bool `json:"disallowed"`
IgnoreQueryLog bool `json:"-"`
}
// clientCacheKey is the key by which a cached client information is found.

376
internal/whois/whois.go Normal file
View file

@ -0,0 +1,376 @@
// Package whois provides WHOIS functionality.
package whois
import (
"bytes"
"context"
"fmt"
"io"
"net"
"net/netip"
"strconv"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/bluele/gcache"
)
const (
// DefaultServer is the default WHOIS server.
DefaultServer = "whois.arin.net"
// DefaultPort is the default port for WHOIS requests.
DefaultPort = 43
)
// Interface provides WHOIS functionality.
type Interface interface {
// Process makes WHOIS request and returns WHOIS information or nil.
// changed indicates that Info was updated since last request.
Process(ctx context.Context, ip netip.Addr) (info *Info, changed bool)
}
// Empty is an empty [Interface] implementation which does nothing.
type Empty struct{}
// type check
var _ Interface = (*Empty)(nil)
// Process implements the [Interface] interface for Empty.
func (Empty) Process(_ context.Context, _ netip.Addr) (info *Info, changed bool) {
return nil, false
}
// Config is the configuration structure for Default.
type Config struct {
// DialContext specifies the dial function for creating unencrypted TCP
// connections.
DialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error)
// ServerAddr is the address of the WHOIS server.
ServerAddr string
// Timeout is the timeout for WHOIS requests.
Timeout time.Duration
// CacheTTL is the Time to Live duration for cached IP addresses.
CacheTTL time.Duration
// MaxConnReadSize is an upper limit in bytes for reading from net.Conn.
MaxConnReadSize int64
// MaxRedirects is the maximum redirects count.
MaxRedirects int
// MaxInfoLen is the maximum length of Info fields returned by Process.
MaxInfoLen int
// CacheSize is the maximum size of the cache. It must be greater than
// zero.
CacheSize int
// Port is the port for WHOIS requests.
Port uint16
}
// Default is the default WHOIS information processor.
type Default struct {
// cache is the cache containing IP addresses of clients. An active IP
// address is resolved once again after it expires. If IP address couldn't
// be resolved, it stays here for some time to prevent further attempts to
// resolve the same IP.
cache gcache.Cache
// dialContext connects to a remote server resolving hostname using our own
// DNS server and unecrypted TCP connection.
dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error)
// serverAddr is the address of the WHOIS server.
serverAddr string
// portStr is the port for WHOIS requests.
portStr string
// timeout is the timeout for WHOIS requests.
timeout time.Duration
// cacheTTL is the Time to Live duration for cached IP addresses.
cacheTTL time.Duration
// maxConnReadSize is an upper limit in bytes for reading from net.Conn.
maxConnReadSize int64
// maxRedirects is the maximum redirects count.
maxRedirects int
// maxInfoLen is the maximum length of Info fields returned by Process.
maxInfoLen int
}
// New returns a new default WHOIS information processor. conf must not be
// nil.
func New(conf *Config) (w *Default) {
return &Default{
serverAddr: conf.ServerAddr,
dialContext: conf.DialContext,
timeout: conf.Timeout,
cache: gcache.New(conf.CacheSize).LRU().Build(),
maxConnReadSize: conf.MaxConnReadSize,
maxRedirects: conf.MaxRedirects,
portStr: strconv.Itoa(int(conf.Port)),
maxInfoLen: conf.MaxInfoLen,
cacheTTL: conf.CacheTTL,
}
}
// trimValue trims s and replaces the last 3 characters of the cut with "..."
// to fit into max. max must be greater than 3.
func trimValue(s string, max int) string {
if len(s) <= max {
return s
}
return s[:max-3] + "..."
}
// isWHOISComment returns true if the data is empty or is a WHOIS comment.
func isWHOISComment(data []byte) (ok bool) {
return len(data) == 0 || data[0] == '#' || data[0] == '%'
}
// whoisParse parses a subset of plain-text data from the WHOIS response into a
// string map. It trims values of the returned map to maxLen.
func whoisParse(data []byte, maxLen int) (info map[string]string) {
info = map[string]string{}
var orgname string
lines := bytes.Split(data, []byte("\n"))
for _, l := range lines {
if isWHOISComment(l) {
continue
}
before, after, found := bytes.Cut(l, []byte(":"))
if !found {
continue
}
key := strings.ToLower(string(before))
val := strings.TrimSpace(string(after))
if val == "" {
continue
}
switch key {
case "orgname", "org-name":
key = "orgname"
val = trimValue(val, maxLen)
orgname = val
case "city", "country":
val = trimValue(val, maxLen)
case "descr", "netname":
key = "orgname"
val = stringutil.Coalesce(orgname, val)
orgname = val
case "whois":
key = "whois"
case "referralserver":
key = "whois"
val = strings.TrimPrefix(val, "whois://")
default:
continue
}
info[key] = val
}
return info
}
// query sends request to a server and returns the response or error.
func (w *Default) query(ctx context.Context, target, serverAddr string) (data []byte, err error) {
addr, _, _ := net.SplitHostPort(serverAddr)
if addr == DefaultServer {
// Display type flags for query.
//
// See https://www.arin.net/resources/registry/whois/rws/api/#nicname-whois-queries.
target = "n + " + target
}
conn, err := w.dialContext(ctx, "tcp", serverAddr)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
defer func() { err = errors.WithDeferred(err, conn.Close()) }()
r, err := aghio.LimitReader(conn, w.maxConnReadSize)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
_ = conn.SetReadDeadline(time.Now().Add(w.timeout))
_, err = io.WriteString(conn, target+"\r\n")
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
// This use of ReadAll is now safe, because we limited the conn Reader.
data, err = io.ReadAll(r)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
return data, nil
}
// queryAll queries WHOIS server and handles redirects.
func (w *Default) queryAll(ctx context.Context, target string) (info map[string]string, err error) {
server := net.JoinHostPort(w.serverAddr, w.portStr)
var data []byte
for i := 0; i < w.maxRedirects; i++ {
data, err = w.query(ctx, target, server)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
log.Debug("whois: received response (%d bytes) from %q about %q", len(data), server, target)
info = whoisParse(data, w.maxInfoLen)
redir, ok := info["whois"]
if !ok {
return info, nil
}
redir = strings.ToLower(redir)
_, _, err = net.SplitHostPort(redir)
if err != nil {
server = net.JoinHostPort(redir, w.portStr)
} else {
server = redir
}
log.Debug("whois: redirected to %q about %q", redir, target)
}
return nil, fmt.Errorf("whois: redirect loop")
}
// type check
var _ Interface = (*Default)(nil)
// Process makes WHOIS request and returns WHOIS information or nil. changed
// indicates that Info was updated since last request.
func (w *Default) Process(ctx context.Context, ip netip.Addr) (wi *Info, changed bool) {
if netutil.IsSpecialPurposeAddr(ip) {
return nil, false
}
wi, expired := w.findInCache(ip)
if wi != nil && !expired {
// Don't return an empty struct so that the frontend doesn't get
// confused.
if (*wi == Info{}) {
return nil, false
}
return wi, false
}
var info Info
defer func() {
item := toCacheItem(info, w.cacheTTL)
err := w.cache.Set(ip, item)
if err != nil {
log.Debug("whois: cache: adding item %q: %s", ip, err)
}
}()
kv, err := w.queryAll(ctx, ip.String())
if err != nil {
log.Debug("whois: quering about %q: %s", ip, err)
return nil, true
}
info = Info{
City: kv["city"],
Country: kv["country"],
Orgname: kv["orgname"],
}
// Don't return an empty struct so that the frontend doesn't get confused.
if (info == Info{}) {
return nil, true
}
return &info, wi == nil || info != *wi
}
// findInCache finds Info in the cache. expired indicates that Info is valid.
func (w *Default) findInCache(ip netip.Addr) (wi *Info, expired bool) {
val, err := w.cache.Get(ip)
if err != nil {
if !errors.Is(err, gcache.KeyNotFoundError) {
log.Debug("whois: cache: retrieving info about %q: %s", ip, err)
}
return nil, false
}
item, ok := val.(*cacheItem)
if !ok {
log.Debug("whois: cache: %q bad type %T", ip, val)
return nil, false
}
return fromCacheItem(item)
}
// Info is the filtered WHOIS data for a runtime client.
type Info struct {
City string `json:"city,omitempty"`
Country string `json:"country,omitempty"`
Orgname string `json:"orgname,omitempty"`
}
// cacheItem represents an item that we will store in the cache.
type cacheItem struct {
// expiry is the time when cacheItem will expire.
expiry time.Time
// info is the WHOIS data for a runtime client.
info *Info
}
// toCacheItem creates a cached item from a WHOIS info and Time to Live
// duration.
func toCacheItem(info Info, ttl time.Duration) (item *cacheItem) {
return &cacheItem{
expiry: time.Now().Add(ttl),
info: &info,
}
}
// fromCacheItem creates a WHOIS info from the cached item. expired indicates
// that WHOIS info is valid. item must not be nil.
func fromCacheItem(item *cacheItem) (info *Info, expired bool) {
if time.Now().After(item.expiry) {
return item.info, true
}
return item.info, false
}

View file

@ -0,0 +1,155 @@
package whois_test
import (
"context"
"io"
"net"
"net/netip"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/golibs/testutil/fakenet"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDefault_Process(t *testing.T) {
const (
nl = "\n"
city = "Nonreal"
country = "Imagiland"
orgname = "FakeOrgLLC"
referralserver = "whois.example.net"
)
ip := netip.MustParseAddr("1.2.3.4")
testCases := []struct {
want *whois.Info
name string
data string
}{{
want: nil,
name: "empty",
data: "",
}, {
want: nil,
name: "comments",
data: "%\n#",
}, {
want: nil,
name: "no_colon",
data: "city",
}, {
want: nil,
name: "no_value",
data: "city:",
}, {
want: &whois.Info{
City: city,
},
name: "city",
data: "city: " + city,
}, {
want: &whois.Info{
Country: country,
},
name: "country",
data: "country: " + country,
}, {
want: &whois.Info{
Orgname: orgname,
},
name: "orgname",
data: "orgname: " + orgname,
}, {
want: &whois.Info{
Orgname: orgname,
},
name: "orgname_hyphen",
data: "org-name: " + orgname,
}, {
want: &whois.Info{
Orgname: orgname,
},
name: "orgname_descr",
data: "descr: " + orgname,
}, {
want: &whois.Info{
Orgname: orgname,
},
name: "orgname_netname",
data: "netname: " + orgname,
}, {
want: &whois.Info{
City: city,
Country: country,
Orgname: orgname,
},
name: "full",
data: "OrgName: " + orgname + nl + "City: " + city + nl + "Country: " + country,
}, {
want: nil,
name: "whois",
data: "whois: " + referralserver,
}, {
want: nil,
name: "referralserver",
data: "referralserver: whois://" + referralserver,
}, {
want: nil,
name: "other",
data: "other: value",
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
hit := 0
fakeConn := &fakenet.Conn{
OnRead: func(b []byte) (n int, err error) {
hit++
return copy(b, tc.data), io.EOF
},
OnWrite: func(b []byte) (n int, err error) {
return len(b), nil
},
OnClose: func() (err error) {
return nil
},
OnSetReadDeadline: func(t time.Time) (err error) {
return nil
},
}
w := whois.New(&whois.Config{
Timeout: 5 * time.Second,
DialContext: func(_ context.Context, _, addr string) (_ net.Conn, _ error) {
hit = 0
return fakeConn, nil
},
MaxConnReadSize: 1024,
MaxRedirects: 3,
MaxInfoLen: 250,
CacheSize: 100,
CacheTTL: time.Hour,
})
got, changed := w.Process(context.Background(), ip)
require.True(t, changed)
assert.Equal(t, tc.want, got)
assert.Equal(t, 1, hit)
// From cache.
got, changed = w.Process(context.Background(), ip)
require.False(t, changed)
assert.Equal(t, tc.want, got)
assert.Equal(t, 1, hit)
})
}
}