From 36e273dfd5f7b62b70b201830b562095207a4dac Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 15 May 2019 14:24:30 +0300 Subject: [PATCH 1/7] * use urlfilter --- go.mod | 10 +++------- go.sum | 31 +++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 6ba77195..bb650707 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,9 @@ go 1.12 require ( github.com/AdguardTeam/dnsproxy v0.12.0 github.com/AdguardTeam/golibs v0.1.3 + github.com/AdguardTeam/urlfilter v0.3.0 github.com/NYTimes/gziphandler v1.1.1 - github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 - github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-test/deep v1.0.1 github.com/gobuffalo/packr v1.19.0 github.com/joomcode/errorx v0.1.0 @@ -16,13 +15,10 @@ require ( github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 github.com/miekg/dns v1.1.1 - github.com/shirou/gopsutil v2.18.10+incompatible - github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 github.com/stretchr/testify v1.3.0 - go.uber.org/goleak v0.10.0 - golang.org/x/net v0.0.0-20190119204137-ed066c81e75e - golang.org/x/sys v0.0.0-20190122071731-054c452bb702 + golang.org/x/net v0.0.0-20190313220215-9f648a60d977 + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 gopkg.in/yaml.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index 299f2732..5b625f87 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,12 @@ github.com/AdguardTeam/dnsproxy v0.12.0/go.mod h1:lcZM2QPwcWGEL3pz8RYy06nQdbjj4p github.com/AdguardTeam/golibs v0.1.2/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg= github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko= +github.com/AdguardTeam/urlfilter v0.3.0 h1:WNd3uZEYWwxylUuA8QS6V5DqHNsVFw3ZD/E2rd5HGpo= +github.com/AdguardTeam/urlfilter v0.3.0/go.mod h1:9xfZ6R2vB8LlT8G9LxtbNhDsbr/xybUOSwmJvpXhl/c= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= -github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= +github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw= @@ -15,6 +17,9 @@ github.com/ameshkov/dnscrypt v1.0.6 h1:55wfnNF8c4E3JXDNlwPl2Pbs7UPPIh+kI6KK3THqY github.com/ameshkov/dnscrypt v1.0.6/go.mod h1:ZvT9LaNaJfDNXKIbkYFf24HUgHuQR6MNT6nwVvN4jMQ= github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug= github.com/ameshkov/dnsstamps v1.0.1/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A= +github.com/ameshkov/goproxy v0.0.0-20190328085534-e9f6fabc24d4/go.mod h1:tKA6C/1BQYejT7L6ZX0klDrqloYenYETv3BCk8xCbrQ= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY= @@ -22,8 +27,8 @@ github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0/go.mod h1:8c4/i2Vlov github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4= @@ -32,6 +37,9 @@ github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264 h1:roWyi0eEdiFreSq github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI= github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU= +github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -55,8 +63,9 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= -github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= +github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM= @@ -70,15 +79,18 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= -go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190122013713-64072686203f h1:u1CmMhe3a44hy8VIgpInORnI01UVaUYheqR7x9BxT3c= golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190119204137-ed066c81e75e h1:MDa3fSUp6MdYHouVmCCNz/zaH2a6CRcxY3VhT/K3C5Q= golang.org/x/net v0.0.0-20190119204137-ed066c81e75e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= @@ -87,6 +99,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190122071731-054c452bb702 h1:Lk4tbZFnlyPgV+sLgTw5yGfzrlOn9kx4vSombi2FFlY= golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+lg4zao9W4HIDzlFbMYgSgtvNVHh00MEHvbGpQ= @@ -95,3 +109,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From 9644f79a03894aab9b37a7a6813c0f51aea39d3e Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 15 May 2019 15:08:15 +0300 Subject: [PATCH 2/7] * dnsforward: use separate ServerConfig object --- dnsforward/dnsforward.go | 44 +++++++++++++++++------------------ dnsforward/dnsforward_test.go | 22 +++++++++--------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index 07f65084..a1afd7a9 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -44,7 +44,7 @@ type Server struct { once sync.Once sync.RWMutex - ServerConfig + conf ServerConfig } // NewServer creates a new instance of the dnsforward.Server @@ -123,7 +123,7 @@ func (s *Server) Start(config *ServerConfig) error { // startInternal starts without locking func (s *Server) startInternal(config *ServerConfig) error { if config != nil { - s.ServerConfig = *config + s.conf = *config } if s.dnsFilter != nil || s.dnsProxy != nil { @@ -158,21 +158,21 @@ func (s *Server) startInternal(config *ServerConfig) error { }) proxyConfig := proxy.Config{ - UDPListenAddr: s.UDPListenAddr, - TCPListenAddr: s.TCPListenAddr, - Ratelimit: s.Ratelimit, - RatelimitWhitelist: s.RatelimitWhitelist, - RefuseAny: s.RefuseAny, + UDPListenAddr: s.conf.UDPListenAddr, + TCPListenAddr: s.conf.TCPListenAddr, + Ratelimit: s.conf.Ratelimit, + RatelimitWhitelist: s.conf.RatelimitWhitelist, + RefuseAny: s.conf.RefuseAny, CacheEnabled: true, - Upstreams: s.Upstreams, - DomainsReservedUpstreams: s.DomainsReservedUpstreams, + Upstreams: s.conf.Upstreams, + DomainsReservedUpstreams: s.conf.DomainsReservedUpstreams, Handler: s.handleDNSRequest, - AllServers: s.AllServers, + AllServers: s.conf.AllServers, } - if s.TLSListenAddr != nil && s.CertificateChain != "" && s.PrivateKey != "" { - proxyConfig.TLSListenAddr = s.TLSListenAddr - keypair, err := tls.X509KeyPair([]byte(s.CertificateChain), []byte(s.PrivateKey)) + if s.conf.TLSListenAddr != nil && s.conf.CertificateChain != "" && s.conf.PrivateKey != "" { + proxyConfig.TLSListenAddr = s.conf.TLSListenAddr + keypair, err := tls.X509KeyPair([]byte(s.conf.CertificateChain), []byte(s.conf.PrivateKey)) if err != nil { return errorx.Decorate(err, "Failed to parse TLS keypair") } @@ -202,10 +202,10 @@ func (s *Server) startInternal(config *ServerConfig) error { // Initializes the DNS filter func (s *Server) initDNSFilter() error { log.Tracef("Creating dnsfilter") - s.dnsFilter = dnsfilter.New(&s.Config) + s.dnsFilter = dnsfilter.New(&s.conf.Config) // add rules only if they are enabled - if s.FilteringEnabled { - err := s.dnsFilter.AddRules(s.Filters) + if s.conf.FilteringEnabled { + err := s.dnsFilter.AddRules(s.conf.Filters) if err != nil { return errorx.Decorate(err, "could not initialize dnsfilter") } @@ -336,11 +336,11 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error { msg := d.Req // don't log ANY request if refuseAny is enabled - if len(msg.Question) >= 1 && msg.Question[0].Qtype == dns.TypeANY && s.RefuseAny { + if len(msg.Question) >= 1 && msg.Question[0].Qtype == dns.TypeANY && s.conf.RefuseAny { shouldLog = false } - if s.QueryLogEnabled && shouldLog { + if s.conf.QueryLogEnabled && shouldLog { elapsed := time.Since(start) upstreamAddr := "" if d.Upstream != nil { @@ -361,7 +361,7 @@ func (s *Server) filterDNSRequest(d *proxy.DNSContext) (*dnsfilter.Result, error host := strings.TrimSuffix(msg.Question[0].Name, ".") s.RLock() - protectionEnabled := s.ProtectionEnabled + protectionEnabled := s.conf.ProtectionEnabled dnsFilter := s.dnsFilter s.RUnlock() @@ -402,7 +402,7 @@ func (s *Server) genDNSFilterMessage(d *proxy.DNSContext, result *dnsfilter.Resu return s.genARecord(m, result.IP) } - if s.BlockingMode == "null_ip" { + if s.conf.BlockingMode == "null_ip" { return s.genARecord(m, net.IPv4zero) } @@ -420,7 +420,7 @@ func (s *Server) genServerFailure(request *dns.Msg) *dns.Msg { func (s *Server) genARecord(request *dns.Msg, ip net.IP) *dns.Msg { resp := dns.Msg{} resp.SetReply(request) - answer, err := dns.NewRR(fmt.Sprintf("%s %d A %s", request.Question[0].Name, s.BlockedResponseTTL, ip.String())) + answer, err := dns.NewRR(fmt.Sprintf("%s %d A %s", request.Question[0].Name, s.conf.BlockedResponseTTL, ip.String())) if err != nil { log.Printf("Couldn't generate A record for replacement host '%s': %s", ip.String(), err) return s.genServerFailure(request) @@ -489,7 +489,7 @@ func (s *Server) genSOA(request *dns.Msg) []dns.RR { Hdr: dns.RR_Header{ Name: zone, Rrtype: dns.TypeSOA, - Ttl: s.BlockedResponseTTL, + Ttl: s.conf.BlockedResponseTTL, Class: dns.ClassINET, }, Mbox: "hostmaster.", // zone will be appended later if it's not empty or "." diff --git a/dnsforward/dnsforward_test.go b/dnsforward/dnsforward_test.go index 0c1325c7..1a38e96b 100644 --- a/dnsforward/dnsforward_test.go +++ b/dnsforward/dnsforward_test.go @@ -86,7 +86,7 @@ func TestDotServer(t *testing.T) { s := createTestServer(t) defer removeDataDir(t) - s.TLSConfig = TLSConfig{ + s.conf.TLSConfig = TLSConfig{ TLSListenAddr: &net.TCPAddr{Port: 0}, CertificateChain: string(certPem), PrivateKey: string(keyPem), @@ -149,7 +149,7 @@ func TestServerRace(t *testing.T) { func TestSafeSearch(t *testing.T) { s := createTestServer(t) - s.SafeSearchEnabled = true + s.conf.SafeSearchEnabled = true defer removeDataDir(t) err := s.Start(nil) if err != nil { @@ -295,7 +295,7 @@ func TestBlockedRequest(t *testing.T) { func TestNullBlockedRequest(t *testing.T) { s := createTestServer(t) - s.FilteringConfig.BlockingMode = "null_ip" + s.conf.FilteringConfig.BlockingMode = "null_ip" defer removeDataDir(t) err := s.Start(nil) if err != nil { @@ -451,14 +451,14 @@ func TestBlockedBySafeBrowsing(t *testing.T) { func createTestServer(t *testing.T) *Server { s := NewServer(createDataDir(t)) - s.UDPListenAddr = &net.UDPAddr{Port: 0} - s.TCPListenAddr = &net.TCPAddr{Port: 0} + s.conf.UDPListenAddr = &net.UDPAddr{Port: 0} + s.conf.TCPListenAddr = &net.TCPAddr{Port: 0} - s.QueryLogEnabled = true - s.FilteringConfig.FilteringEnabled = true - s.FilteringConfig.ProtectionEnabled = true - s.FilteringConfig.SafeBrowsingEnabled = true - s.Filters = make([]dnsfilter.Filter, 0) + s.conf.QueryLogEnabled = true + s.conf.FilteringConfig.FilteringEnabled = true + s.conf.FilteringConfig.ProtectionEnabled = true + s.conf.FilteringConfig.SafeBrowsingEnabled = true + s.conf.Filters = make([]dnsfilter.Filter, 0) rules := []string{ "||nxdomain.example.org^", @@ -466,7 +466,7 @@ func createTestServer(t *testing.T) *Server { "127.0.0.1 host.example.org", } filter := dnsfilter.Filter{ID: 1, Rules: rules} - s.Filters = append(s.Filters, filter) + s.conf.Filters = append(s.conf.Filters, filter) return s } From bd68bf2e2515013af8ffd87c4a5faeadd0a9db22 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 15 May 2019 15:32:42 +0300 Subject: [PATCH 3/7] * dns: rename dnsfilter.Filter.Rule -> dnsfilter.Filter.Data --- dns.go | 8 ++++---- dnsfilter/dnsfilter.go | 4 ++-- filter.go | 26 ++++++++++++-------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/dns.go b/dns.go index c37f9770..001876d0 100644 --- a/dns.go +++ b/dns.go @@ -34,13 +34,13 @@ func generateServerConfig() dnsforward.ServerConfig { filters := []dnsfilter.Filter{} userFilter := userFilter() filters = append(filters, dnsfilter.Filter{ - ID: userFilter.ID, - Rules: userFilter.Rules, + ID: userFilter.ID, + Data: userFilter.Data, }) for _, filter := range config.Filters { filters = append(filters, dnsfilter.Filter{ - ID: filter.ID, - Rules: filter.Rules, + ID: filter.ID, + Data: filter.Data, }) } diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index b780c9f2..0e7338ef 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -122,8 +122,8 @@ type Dnsfilter struct { // Filter represents a filter list type Filter struct { - ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase - Rules []string `json:"-" yaml:"-"` // not in yaml or json + ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase + Data []byte `json:"-" yaml:"-"` // List of rules divided by '\n' } //go:generate stringer -type=Reason diff --git a/filter.go b/filter.go index 1a4671b2..592a3c31 100644 --- a/filter.go +++ b/filter.go @@ -35,13 +35,12 @@ type filter struct { // Creates a helper object for working with the user rules func userFilter() filter { - return filter{ + f := filter{ // User filter always has constant ID=0 Enabled: true, - Filter: dnsfilter.Filter{ - Rules: config.UserRules, - }, } + f.Filter.Data = []byte(strings.Join(config.UserRules, "\n")) + return f } // Enable or disable a filter @@ -242,7 +241,7 @@ func refreshFiltersIfNecessary(force bool) int { log.Info("Updated filter #%d. Rules: %d -> %d", f.ID, f.RulesCount, uf.RulesCount) f.Name = uf.Name - f.Rules = uf.Rules + f.Data = uf.Data f.RulesCount = uf.RulesCount f.checksum = uf.checksum updateCount++ @@ -261,7 +260,7 @@ func refreshFiltersIfNecessary(force bool) int { } // A helper function that parses filter contents and returns a number of rules and a filter name (if there's any) -func parseFilterContents(contents []byte) (int, string, []string) { +func parseFilterContents(contents []byte) (int, string) { lines := strings.Split(string(contents), "\n") rulesCount := 0 name := "" @@ -286,7 +285,7 @@ func parseFilterContents(contents []byte) (int, string, []string) { } } - return rulesCount, name, lines + return rulesCount, name } // Perform upgrade on a filter @@ -327,13 +326,13 @@ func (filter *filter) update() (bool, error) { } // Extract filter name and count number of rules - rulesCount, filterName, rules := parseFilterContents(body) + rulesCount, filterName := parseFilterContents(body) log.Printf("Filter %d has been updated: %d bytes, %d rules", filter.ID, len(body), rulesCount) if filterName != "" { filter.Name = filterName } filter.RulesCount = rulesCount - filter.Rules = rules + filter.Data = body filter.checksum = checksum return true, nil @@ -343,9 +342,8 @@ func (filter *filter) update() (bool, error) { func (filter *filter) save() error { filterFilePath := filter.Path() log.Printf("Saving filter %d contents to: %s", filter.ID, filterFilePath) - body := []byte(strings.Join(filter.Rules, "\n")) - err := file.SafeWrite(filterFilePath, body) + err := file.SafeWrite(filterFilePath, filter.Data) // update LastUpdated field after saving the file filter.LastUpdated = filter.LastTimeUpdated() @@ -368,10 +366,10 @@ func (filter *filter) load() error { } log.Tracef("File %s, id %d, length %d", filterFilePath, filter.ID, len(filterFileContents)) - rulesCount, _, rules := parseFilterContents(filterFileContents) + rulesCount, _ := parseFilterContents(filterFileContents) filter.RulesCount = rulesCount - filter.Rules = rules + filter.Data = filterFileContents filter.checksum = crc32.ChecksumIEEE(filterFileContents) filter.LastUpdated = filter.LastTimeUpdated() @@ -380,7 +378,7 @@ func (filter *filter) load() error { // Clear filter rules func (filter *filter) unload() { - filter.Rules = []string{} + filter.Data = nil filter.RulesCount = 0 } From 3396d6801990b45c0c48a8508d9b95587e30352d Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 15 May 2019 15:48:05 +0300 Subject: [PATCH 4/7] * dnsfilter: remove code for filtering rules --- dnsfilter/dnsfilter.go | 471 ------------------------------------ dnsfilter/helpers.go | 40 --- dnsfilter/rule_to_regexp.go | 91 ------- 3 files changed, 602 deletions(-) delete mode 100644 dnsfilter/rule_to_regexp.go diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index 0e7338ef..80b59fc5 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -11,9 +11,7 @@ import ( "io/ioutil" "net" "net/http" - "regexp" "strings" - "sync" "sync/atomic" "time" @@ -60,33 +58,6 @@ type privateConfig struct { safeBrowsingServer string // access via methods } -type rule struct { - text string // text without @@ decorators or $ options - shortcut string // for speeding up lookup - originalText string // original text for reporting back to applications - ip net.IP // IP address (for the case when we're matching a hosts file) - - // options - options []string // optional options after $ - - // parsed options - apps []string - isWhitelist bool - isImportant bool - - // user-supplied data - listID int64 - - // suffix matching - isSuffix bool - suffix string - - // compiled regexp - compiled *regexp.Regexp - - sync.RWMutex -} - // LookupStats store stats collected during safebrowsing or parental checks type LookupStats struct { Requests uint64 // number of HTTP requests that were sent @@ -104,14 +75,6 @@ type Stats struct { // Dnsfilter holds added rules and performs hostname matches against the rules type Dnsfilter struct { - storage map[string]bool // rule storage, not used for matching, just for filtering out duplicates - storageMutex sync.RWMutex - - // rules are checked against these lists in the order defined here - important *rulesTable // more important than whitelist and is checked first - whiteList *rulesTable // more important than blacklist - blackList *rulesTable - // HTTP lookups for safebrowsing and parental client http.Client // handle for http client -- single instance as recommended by docs transport *http.Transport // handle for http transport used by http client @@ -242,308 +205,6 @@ func (d *Dnsfilter) CheckHost(host string) (Result, error) { return Result{}, nil } -// -// rules table -// - -type rulesTable struct { - rulesByHost map[string]*rule - rulesByShortcut map[string][]*rule - rulesLeftovers []*rule - sync.RWMutex -} - -func newRulesTable() *rulesTable { - return &rulesTable{ - rulesByHost: make(map[string]*rule), - rulesByShortcut: make(map[string][]*rule), - rulesLeftovers: make([]*rule, 0), - } -} - -func (r *rulesTable) Add(rule *rule) { - r.Lock() - if rule.ip != nil { - // Hosts syntax - r.rulesByHost[rule.text] = rule - } else if len(rule.shortcut) == shortcutLength && enableFastLookup { - // Adblock syntax with a shortcut - r.rulesByShortcut[rule.shortcut] = append(r.rulesByShortcut[rule.shortcut], rule) - } else { - // Adblock syntax -- too short to have a shortcut - r.rulesLeftovers = append(r.rulesLeftovers, rule) - } - r.Unlock() -} - -func (r *rulesTable) matchByHost(host string) (Result, error) { - // First: examine the hosts-syntax rules - res, err := r.searchByHost(host) - if err != nil { - return res, err - } - if res.Reason.Matched() { - return res, nil - } - - // Second: examine the adblock-syntax rules with shortcuts - res, err = r.searchShortcuts(host) - if err != nil { - return res, err - } - if res.Reason.Matched() { - return res, nil - } - - // Third: examine the others - res, err = r.searchLeftovers(host) - if err != nil { - return res, err - } - if res.Reason.Matched() { - return res, nil - } - - return Result{}, nil -} - -func (r *rulesTable) searchByHost(host string) (Result, error) { - rule, ok := r.rulesByHost[host] - - if ok { - return rule.match(host) - } - - return Result{}, nil -} - -func (r *rulesTable) searchShortcuts(host string) (Result, error) { - // check in shortcuts first - for i := 0; i < len(host); i++ { - shortcut := host[i:] - if len(shortcut) > shortcutLength { - shortcut = shortcut[:shortcutLength] - } - if len(shortcut) != shortcutLength { - continue - } - rules, ok := r.rulesByShortcut[shortcut] - if !ok { - continue - } - for _, rule := range rules { - res, err := rule.match(host) - // error? stop search - if err != nil { - return res, err - } - // matched? stop search - if res.Reason.Matched() { - return res, err - } - // continue otherwise - } - } - return Result{}, nil -} - -func (r *rulesTable) searchLeftovers(host string) (Result, error) { - for _, rule := range r.rulesLeftovers { - res, err := rule.match(host) - // error? stop search - if err != nil { - return res, err - } - // matched? stop search - if res.Reason.Matched() { - return res, err - } - // continue otherwise - } - return Result{}, nil -} - -func findOptionIndex(text string) int { - for i, r := range text { - // ignore non-$ - if r != '$' { - continue - } - // ignore `\$` - if i > 0 && text[i-1] == '\\' { - continue - } - // ignore `$/` - if i > len(text) && text[i+1] == '/' { - continue - } - return i + 1 - } - return -1 -} - -func (rule *rule) extractOptions() error { - optIndex := findOptionIndex(rule.text) - if optIndex == 0 { // starts with $ - return ErrInvalidSyntax - } - if optIndex == len(rule.text) { // ends with $ - return ErrInvalidSyntax - } - if optIndex < 0 { - return nil - } - - optionsStr := rule.text[optIndex:] - rule.text = rule.text[:optIndex-1] // remove options from text - - begin := 0 - i := 0 - for i = 0; i < len(optionsStr); i++ { - switch optionsStr[i] { - case ',': - if i > 0 { - // it might be escaped, if so, ignore - if optionsStr[i-1] == '\\' { - break // from switch, not for loop - } - } - rule.options = append(rule.options, optionsStr[begin:i]) - begin = i + 1 - } - } - if begin != i { - // there's still an option remaining - rule.options = append(rule.options, optionsStr[begin:]) - } - - return nil -} - -func (rule *rule) parseOptions() error { - err := rule.extractOptions() - if err != nil { - return err - } - - for _, option := range rule.options { - switch { - case option == "important": - rule.isImportant = true - case strings.HasPrefix(option, "app="): - option = strings.TrimPrefix(option, "app=") - rule.apps = strings.Split(option, "|") - default: - return ErrInvalidSyntax - } - } - - return nil -} - -func (rule *rule) extractShortcut() { - // regex rules have no shortcuts - if rule.text[0] == '/' && rule.text[len(rule.text)-1] == '/' { - return - } - - fields := strings.FieldsFunc(rule.text, func(r rune) bool { - switch r { - case '*', '^', '|': - return true - } - return false - }) - longestField := "" - for _, field := range fields { - if len(field) > len(longestField) { - longestField = field - } - } - if len(longestField) > shortcutLength { - longestField = longestField[:shortcutLength] - } - rule.shortcut = strings.ToLower(longestField) -} - -func (rule *rule) compile() error { - rule.RLock() - isCompiled := rule.isSuffix || rule.compiled != nil - rule.RUnlock() - if isCompiled { - return nil - } - - isSuffix, suffix := getSuffix(rule.text) - if isSuffix { - rule.Lock() - rule.isSuffix = isSuffix - rule.suffix = suffix - rule.Unlock() - return nil - } - - expr, err := ruleToRegexp(rule.text) - if err != nil { - return err - } - - compiled, err := regexp.Compile(expr) - if err != nil { - return err - } - - rule.Lock() - rule.compiled = compiled - rule.Unlock() - - return nil -} - -// Checks if the rule matches the specified host and returns a corresponding Result object -func (rule *rule) match(host string) (Result, error) { - res := Result{} - - if rule.ip != nil && rule.text == host { - // This is a hosts-syntax rule -- just check that the hostname matches and return the result - return Result{ - IsFiltered: true, - Reason: FilteredBlackList, - Rule: rule.originalText, - IP: rule.ip, - FilterID: rule.listID, - }, nil - } - - err := rule.compile() - if err != nil { - return res, err - } - rule.RLock() - matched := false - if rule.isSuffix { - if host == rule.suffix { - matched = true - } else if strings.HasSuffix(host, "."+rule.suffix) { - matched = true - } - } else { - matched = rule.compiled.MatchString(host) - } - rule.RUnlock() - if matched { - res.Reason = FilteredBlackList - res.IsFiltered = true - res.FilterID = rule.listID - res.Rule = rule.originalText - if rule.isWhitelist { - res.Reason = NotFilteredWhiteList - res.IsFiltered = false - } - } - return res, nil -} - func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) { isFound = false // not found yet @@ -840,133 +501,11 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc // AddRules is a convinience function to add an array of filters in one call func (d *Dnsfilter) AddRules(filters []Filter) error { - for _, f := range filters { - for _, rule := range f.Rules { - err := d.AddRule(rule, f.ID) - if err == ErrAlreadyExists || err == ErrInvalidSyntax { - continue - } - if err != nil { - log.Printf("Cannot add rule %s: %s", rule, err) - // Just ignore invalid rules - continue - } - } - } return nil } -// AddRule adds a rule, checking if it is a valid rule first and if it wasn't added already -func (d *Dnsfilter) AddRule(input string, filterListID int64) error { - input = strings.TrimSpace(input) - d.storageMutex.RLock() - _, exists := d.storage[input] - d.storageMutex.RUnlock() - if exists { - // already added - return ErrAlreadyExists - } - - if !isValidRule(input) { - return ErrInvalidSyntax - } - - // First, check if this is a hosts-syntax rule - if d.parseEtcHosts(input, filterListID) { - // This is a valid hosts-syntax rule, no need for further parsing - return nil - } - - // Start parsing the rule - r := rule{ - text: input, // will be modified - originalText: input, - listID: filterListID, - } - - // Mark rule as whitelist if it starts with @@ - if strings.HasPrefix(r.text, "@@") { - r.isWhitelist = true - r.text = r.text[2:] - } - - err := r.parseOptions() - if err != nil { - return err - } - - r.extractShortcut() - - if !enableDelayedCompilation { - err := r.compile() - if err != nil { - return err - } - } - - destination := d.blackList - if r.isImportant { - destination = d.important - } else if r.isWhitelist { - destination = d.whiteList - } - - d.storageMutex.Lock() - d.storage[input] = true - d.storageMutex.Unlock() - destination.Add(&r) - return nil -} - -// Parses the hosts-syntax rules. Returns false if the input string is not of hosts-syntax. -func (d *Dnsfilter) parseEtcHosts(input string, filterListID int64) bool { - // Strip the trailing comment - ruleText := input - if pos := strings.IndexByte(ruleText, '#'); pos != -1 { - ruleText = ruleText[0:pos] - } - fields := strings.Fields(ruleText) - if len(fields) < 2 { - return false - } - addr := net.ParseIP(fields[0]) - if addr == nil { - return false - } - - d.storageMutex.Lock() - d.storage[input] = true - d.storageMutex.Unlock() - - for _, host := range fields[1:] { - r := rule{ - text: host, - originalText: input, - listID: filterListID, - ip: addr, - } - d.blackList.Add(&r) - } - return true -} - // matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups func (d *Dnsfilter) matchHost(host string) (Result, error) { - lists := []*rulesTable{ - d.important, - d.whiteList, - d.blackList, - } - - for _, table := range lists { - res, err := table.matchByHost(host) - if err != nil { - return res, err - } - if res.Reason.Matched() { - return res, nil - } - } return Result{}, nil } @@ -1061,11 +600,6 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp func New(c *Config) *Dnsfilter { d := new(Dnsfilter) - d.storage = make(map[string]bool) - d.important = newRulesTable() - d.whiteList = newRulesTable() - d.blackList = newRulesTable() - // Customize the Transport to have larger connection pool, // We are not (re)using http.DefaultTransport because of race conditions found by tests d.transport = &http.Transport{ @@ -1141,8 +675,3 @@ func (d *Dnsfilter) SafeSearchDomain(host string) (string, bool) { func (d *Dnsfilter) GetStats() Stats { return stats } - -// Count returns number of rules added to filter -func (d *Dnsfilter) Count() int { - return len(d.storage) -} diff --git a/dnsfilter/helpers.go b/dnsfilter/helpers.go index 68d4ba26..2d60c47c 100644 --- a/dnsfilter/helpers.go +++ b/dnsfilter/helpers.go @@ -1,49 +1,9 @@ package dnsfilter import ( - "strings" "sync/atomic" ) -func isValidRule(rule string) bool { - if len(rule) < 4 { - return false - } - if rule[0] == '!' { - return false - } - if rule[0] == '#' { - return false - } - if strings.HasPrefix(rule, "[Adblock") { - return false - } - - // Filter out all sorts of cosmetic rules: - // https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#cosmetic-rules - masks := []string{ - "##", - "#@#", - "#?#", - "#@?#", - "#$#", - "#@$#", - "#?$#", - "#@?$#", - "$$", - "$@$", - "#%#", - "#@%#", - } - for _, mask := range masks { - if strings.Contains(rule, mask) { - return false - } - } - - return true -} - func updateMax(valuePtr *int64, maxPtr *int64) { for { current := atomic.LoadInt64(valuePtr) diff --git a/dnsfilter/rule_to_regexp.go b/dnsfilter/rule_to_regexp.go deleted file mode 100644 index 41d55e30..00000000 --- a/dnsfilter/rule_to_regexp.go +++ /dev/null @@ -1,91 +0,0 @@ -package dnsfilter - -import ( - "strings" -) - -func ruleToRegexp(rule string) (string, error) { - const hostStart = `(?:^|\.)` - const hostEnd = `$` - - // empty or short rule -- do nothing - if !isValidRule(rule) { - return "", ErrInvalidSyntax - } - - // if starts with / and ends with /, it's already a regexp, just strip the slashes - if rule[0] == '/' && rule[len(rule)-1] == '/' { - return rule[1 : len(rule)-1], nil - } - - var sb strings.Builder - - if rule[0] == '|' && rule[1] == '|' { - sb.WriteString(hostStart) - rule = rule[2:] - } - - for i, r := range rule { - switch { - case r == '?' || r == '.' || r == '+' || r == '[' || r == ']' || r == '(' || r == ')' || r == '{' || r == '}' || r == '#' || r == '\\' || r == '$': - sb.WriteRune('\\') - sb.WriteRune(r) - case r == '|' && i == 0: - // | at start and it's not || at start - sb.WriteRune('^') - case r == '|' && i == len(rule)-1: - // | at end - sb.WriteRune('$') - case r == '|' && i != 0 && i != len(rule)-1: - sb.WriteString(`\|`) - case r == '*': - sb.WriteString(`.*`) - case r == '^': - sb.WriteString(hostEnd) - default: - sb.WriteRune(r) - } - } - - return sb.String(), nil -} - -// handle suffix rule ||example.com^ -- either entire string is example.com or *.example.com -func getSuffix(rule string) (bool, string) { - // if starts with / and ends with /, it's already a regexp - // TODO: if a regexp is simple `/abracadabra$/`, then simplify it maybe? - if rule[0] == '/' && rule[len(rule)-1] == '/' { - return false, "" - } - - // must start with || - if rule[0] != '|' || rule[1] != '|' { - return false, "" - } - rule = rule[2:] - - // suffix rule must end with ^ or | - lastChar := rule[len(rule)-1] - if lastChar != '^' && lastChar != '|' { - return false, "" - } - // last char was checked, eat it - rule = rule[:len(rule)-1] - - // it might also end with ^| - if rule[len(rule)-1] == '^' { - rule = rule[:len(rule)-1] - } - - // check that it doesn't have any special characters inside - for _, r := range rule { - switch r { - case '|': - return false, "" - case '*': - return false, "" - } - } - - return true, rule -} From 829415da5b714b83d07c28728f3ba12e85b6692c Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Wed, 15 May 2019 16:46:11 +0300 Subject: [PATCH 5/7] * dnsfilter: use urlfilter package + new config setting 'filtering_temp_filename' * remove AddRules(), modify New() --- dnsfilter/dnsfilter.go | 81 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go index 80b59fc5..221f6383 100644 --- a/dnsfilter/dnsfilter.go +++ b/dnsfilter/dnsfilter.go @@ -17,6 +17,7 @@ import ( "github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter" "github.com/bluele/gcache" "golang.org/x/net/publicsuffix" ) @@ -46,11 +47,12 @@ const enableDelayedCompilation = true // flag for debugging, must be true in pro // Config allows you to configure DNS filtering with New() or just change variables directly. type Config struct { - ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17 - ParentalEnabled bool `yaml:"parental_enabled"` - SafeSearchEnabled bool `yaml:"safesearch_enabled"` - SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"` - ResolverAddress string // DNS server address + FilteringTempFilename string `yaml:"filtering_temp_filename"` // temporary file for storing unused filtering rules + ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17 + ParentalEnabled bool `yaml:"parental_enabled"` + SafeSearchEnabled bool `yaml:"safesearch_enabled"` + SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"` + ResolverAddress string // DNS server address } type privateConfig struct { @@ -75,6 +77,9 @@ type Stats struct { // Dnsfilter holds added rules and performs hostname matches against the rules type Dnsfilter struct { + rulesStorage *urlfilter.RulesStorage + filteringEngine *urlfilter.DNSEngine + // HTTP lookups for safebrowsing and parental client http.Client // handle for http client -- single instance as recommended by docs transport *http.Transport // handle for http transport used by http client @@ -499,13 +504,59 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc // Adding rule and matching against the rules // -// AddRules is a convinience function to add an array of filters in one call -func (d *Dnsfilter) AddRules(filters []Filter) error { +// Initialize urlfilter objects +func (d *Dnsfilter) initFiltering(filters map[int]string) error { + var err error + d.rulesStorage, err = urlfilter.NewRuleStorage(d.FilteringTempFilename) + if err != nil { + return err + } + + d.filteringEngine = urlfilter.NewDNSEngine(filters, d.rulesStorage) return nil } // matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups func (d *Dnsfilter) matchHost(host string) (Result, error) { + if d.filteringEngine == nil { + return Result{}, nil + } + + rules, ok := d.filteringEngine.Match(host) + if !ok { + return Result{}, nil + } + + for _, rule := range rules { + + log.Tracef("Found rule for host '%s': '%s' list_id: %d", + host, rule.Text(), rule.GetFilterListID()) + + res := Result{} + res.Reason = FilteredBlackList + res.IsFiltered = true + res.FilterID = int64(rule.GetFilterListID()) + res.Rule = rule.Text() + + if netRule, ok := rule.(*urlfilter.NetworkRule); ok { + + if netRule.Whitelist { + res.Reason = NotFilteredWhiteList + res.IsFiltered = false + } + return res, nil + + } else if hostRule, ok := rule.(*urlfilter.HostRule); ok { + + res.IP = hostRule.IP + return res, nil + + } else { + log.Tracef("Rule type is unsupported: '%s' list_id: %d", + rule.Text(), rule.GetFilterListID()) + } + } + return Result{}, nil } @@ -597,7 +648,7 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp } // New creates properly initialized DNS Filter that is ready to be used -func New(c *Config) *Dnsfilter { +func New(c *Config, filters map[int]string) *Dnsfilter { d := new(Dnsfilter) // Customize the Transport to have larger connection pool, @@ -624,6 +675,15 @@ func New(c *Config) *Dnsfilter { d.Config = *c } + if filters != nil { + err := d.initFiltering(filters) + if err != nil { + log.Error("Can't initialize filtering subsystem: %s", err) + d.Destroy() + return nil + } + } + return d } @@ -633,6 +693,11 @@ func (d *Dnsfilter) Destroy() { if d != nil && d.transport != nil { d.transport.CloseIdleConnections() } + + if d.rulesStorage != nil { + d.rulesStorage.Close() + d.rulesStorage = nil + } } // From 5ec747b30b0907d9dd393fb9b3a17574ce8c2b94 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Thu, 16 May 2019 14:03:25 +0300 Subject: [PATCH 6/7] * dnsfilter: adapt tests to new interface --- dnsfilter/dnsfilter_test.go | 801 ++++-------------------------------- 1 file changed, 74 insertions(+), 727 deletions(-) diff --git a/dnsfilter/dnsfilter_test.go b/dnsfilter/dnsfilter_test.go index 85e0c4bf..04861e8c 100644 --- a/dnsfilter/dnsfilter_test.go +++ b/dnsfilter/dnsfilter_test.go @@ -1,278 +1,51 @@ package dnsfilter import ( - "archive/zip" - "bytes" - "io/ioutil" + "fmt" "net" "net/http" "net/http/httptest" "path" - "runtime/pprof" - "strings" + "runtime" "testing" "time" - - "bufio" - "fmt" - "os" - "runtime" - - "github.com/AdguardTeam/golibs/log" - "github.com/shirou/gopsutil/process" - "go.uber.org/goleak" ) -// first in file because it must be run first -func TestLotsOfRulesMemoryUsage(t *testing.T) { - start := getRSS() - log.Tracef("RSS before loading rules - %d kB\n", start/1024) - dumpMemProfile("tests/" + _Func() + "1.pprof") +// HELPERS +// SAFE BROWSING +// SAFE SEARCH +// PARENTAL +// FILTERING +// BENCHMARKS - d := NewForTest() - defer d.Destroy() - err := loadTestRules(d) - if err != nil { - t.Error(err) - } +// HELPERS - afterLoad := getRSS() - log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024) - dumpMemProfile("tests/" + _Func() + "2.pprof") - - tests := []struct { - host string - match bool - }{ - {"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com", false}, - {"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net", true}, +func purgeCaches() { + if safebrowsingCache != nil { + safebrowsingCache.Purge() } - for _, testcase := range tests { - ret, err := d.CheckHost(testcase.host) - if err != nil { - t.Errorf("Error while matching host %s: %s", testcase.host, err) - } - if !ret.IsFiltered && ret.IsFiltered != testcase.match { - t.Errorf("Expected hostname %s to not match", testcase.host) - } - if ret.IsFiltered && ret.IsFiltered != testcase.match { - t.Errorf("Expected hostname %s to match", testcase.host) - } - } - afterMatch := getRSS() - log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024) - dumpMemProfile("tests/" + _Func() + "3.pprof") -} - -func getRSS() uint64 { - proc, err := process.NewProcess(int32(os.Getpid())) - if err != nil { - panic(err) - } - minfo, err := proc.MemoryInfo() - if err != nil { - panic(err) - } - return minfo.RSS -} - -func dumpMemProfile(name string) { - runtime.GC() - f, err := os.Create(name) - if err != nil { - panic(err) - } - defer f.Close() - runtime.GC() // update the stats before writing them - err = pprof.WriteHeapProfile(f) - if err != nil { - panic(err) + if parentalCache != nil { + parentalCache.Purge() } } -const topHostsFilename = "tests/top-1m.csv" - -func fetchTopHostsFromNet() { - log.Tracef("Fetching top hosts from network") - resp, err := http.Get("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip") - if err != nil { - panic(err) - } - defer resp.Body.Close() - - log.Tracef("Reading zipfile body") - zipfile, err := ioutil.ReadAll(resp.Body) - if err != nil { - panic(err) - } - - log.Tracef("Opening zipfile") - r, err := zip.NewReader(bytes.NewReader(zipfile), int64(len(zipfile))) - if err != nil { - panic(err) - } - - if len(r.File) != 1 { - panic(fmt.Errorf("zipfile must have only one entry: %+v", r)) - } - f := r.File[0] - log.Tracef("Unpacking file %s from zipfile", f.Name) - rc, err := f.Open() - if err != nil { - panic(err) - } - log.Tracef("Reading file %s contents", f.Name) - body, err := ioutil.ReadAll(rc) - if err != nil { - panic(err) - } - rc.Close() - - log.Tracef("Writing file %s contents to disk", f.Name) - err = ioutil.WriteFile(topHostsFilename+".tmp", body, 0644) - if err != nil { - panic(err) - } - err = os.Rename(topHostsFilename+".tmp", topHostsFilename) - if err != nil { - panic(err) - } +func _Func() string { + pc := make([]uintptr, 10) // at least 1 entry needed + runtime.Callers(2, pc) + f := runtime.FuncForPC(pc[0]) + return path.Base(f.Name()) } -func getTopHosts() { - // if file doesn't exist, fetch it - if _, err := os.Stat(topHostsFilename); os.IsNotExist(err) { - // file does not exist, fetch it - fetchTopHostsFromNet() - } +func NewForTest() *Dnsfilter { + d := New(nil, nil) + purgeCaches() + return d } -func TestLotsOfRulesLotsOfHostsMemoryUsage(t *testing.T) { - start := getRSS() - log.Tracef("RSS before loading rules - %d kB\n", start/1024) - dumpMemProfile("tests/" + _Func() + "1.pprof") - - d := NewForTest() - defer d.Destroy() - mustLoadTestRules(d) - log.Tracef("Have %d rules", d.Count()) - - afterLoad := getRSS() - log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024) - dumpMemProfile("tests/" + _Func() + "2.pprof") - - getTopHosts() - hostnames, err := os.Open(topHostsFilename) - if err != nil { - t.Fatal(err) - } - defer hostnames.Close() - afterHosts := getRSS() - log.Tracef("RSS after loading hosts - %d kB (%d kB diff)\n", afterHosts/1024, (afterHosts-afterLoad)/1024) - dumpMemProfile("tests/" + _Func() + "2.pprof") - - { - scanner := bufio.NewScanner(hostnames) - for scanner.Scan() { - line := scanner.Text() - records := strings.Split(line, ",") - ret, err := d.CheckHost(records[1] + "." + records[1]) - if err != nil { - t.Error(err) - } - if ret.Reason.Matched() { - // log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason) - } - } - } - - afterMatch := getRSS() - log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024) - dumpMemProfile("tests/" + _Func() + "3.pprof") -} - -func TestRuleToRegexp(t *testing.T) { - tests := []struct { - rule string - result string - err error - }{ - {"/doubleclick/", "doubleclick", nil}, - {"/", "", ErrInvalidSyntax}, - {`|double*?.+[]|(){}#$\|`, `^double.*\?\.\+\[\]\|\(\)\{\}\#\$\\$`, nil}, - {`||doubleclick.net^`, `(?:^|\.)doubleclick\.net$`, nil}, - } - for _, testcase := range tests { - converted, err := ruleToRegexp(testcase.rule) - if err != testcase.err { - t.Error("Errors do not match, got ", err, " expected ", testcase.err) - } - if converted != testcase.result { - t.Error("Results do not match, got ", converted, " expected ", testcase.result) - } - } -} - -func TestSuffixRule(t *testing.T) { - for _, testcase := range []struct { - rule string - isSuffix bool - suffix string - }{ - {`||doubleclick.net^`, true, `doubleclick.net`}, // entire string or subdomain match - {`||doubleclick.net|`, true, `doubleclick.net`}, // entire string or subdomain match - {`|doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net - {`*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net - {`doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net - {`|*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net - {`||*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net - {`||*doubleclick.net|`, false, ``}, // TODO: ends with doubleclick.net - {`||*doublec*lick.net^`, false, ``}, // has a wildcard inside, has to be regexp - {`||*doublec|lick.net^`, false, ``}, // has a special symbol inside, has to be regexp - {`/abracadabra/`, false, ``}, // regexp, not anchored - {`/abracadabra$/`, false, ``}, // TODO: simplify simple suffix regexes - } { - isSuffix, suffix := getSuffix(testcase.rule) - if testcase.isSuffix != isSuffix { - t.Errorf("Results do not match for \"%s\": got %v expected %v", testcase.rule, isSuffix, testcase.isSuffix) - continue - } - if testcase.isSuffix && testcase.suffix != suffix { - t.Errorf("Result suffix does not match for \"%s\": got \"%s\" expected \"%s\"", testcase.rule, suffix, testcase.suffix) - continue - } - // log.Tracef("\"%s\": %v: %s", testcase.rule, isSuffix, suffix) - } -} - -// -// helper functions -// -func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) { - t.Helper() - err := d.AddRule(rule, 0) - if err == nil { - // nothing to report - return - } - if err == ErrInvalidSyntax { - t.Errorf("This rule has invalid syntax: %s", rule) - } - if err != nil { - t.Errorf("Error while adding rule %s: %s", rule, err) - } -} - -func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) { - t.Helper() - err := d.AddRule(rule, 0) - if err == ErrInvalidSyntax || err == ErrAlreadyExists { - return - } - if err != nil { - t.Errorf("Error while adding rule %s: %s", rule, err) - } - t.Errorf("Adding this rule should have failed: %s", rule) +func NewForTestFilters(filters map[int]string) *Dnsfilter { + d := New(nil, filters) + purgeCaches() + return d } func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) { @@ -311,232 +84,21 @@ func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) { } } -func loadTestRules(d *Dnsfilter) error { - filterFileName := "tests/dns.txt" - file, err := os.Open(filterFileName) - if err != nil { - return err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - rule := scanner.Text() - err = d.AddRule(rule, 0) - if err == ErrInvalidSyntax || err == ErrAlreadyExists { - continue - } - if err != nil { - return err - } - } - - err = scanner.Err() - return err -} - -func mustLoadTestRules(d *Dnsfilter) { - err := loadTestRules(d) - if err != nil { - panic(err) - } -} - -func NewForTest() *Dnsfilter { - d := New(nil) - purgeCaches() - return d -} - -// -// tests -// -func TestSanityCheck(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "||doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatch(t, "www.doubleclick.net") - d.checkMatchEmpty(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") - d.checkMatchEmpty(t, "wmconvirus.narod.ru") - d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384") -} - func TestEtcHostsMatching(t *testing.T) { - d := NewForTest() - defer d.Destroy() - addr := "216.239.38.120" text := fmt.Sprintf(" %s google.com www.google.com # enforce google's safesearch ", addr) + filters := make(map[int]string) + filters[0] = text + d := NewForTestFilters(filters) + defer d.Destroy() - d.checkAddRule(t, text) d.checkMatchIP(t, "google.com", addr) d.checkMatchIP(t, "www.google.com", addr) d.checkMatchEmpty(t, "subdomain.google.com") d.checkMatchEmpty(t, "example.org") } -func TestSuffixMatching1(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "||doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatch(t, "www.doubleclick.net") - d.checkMatchEmpty(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") -} - -func TestSuffixMatching2(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "|doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatchEmpty(t, "www.doubleclick.net") - d.checkMatchEmpty(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") -} - -func TestSuffixMatching3(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatch(t, "www.doubleclick.net") - d.checkMatch(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") -} - -func TestSuffixMatching4(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "*doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatch(t, "www.doubleclick.net") - d.checkMatch(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") -} - -func TestSuffixMatching5(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "|*doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatch(t, "www.doubleclick.net") - d.checkMatch(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") -} - -func TestSuffixMatching6(t *testing.T) { - d := NewForTest() - defer d.Destroy() - - d.checkAddRule(t, "||*doubleclick.net^") - d.checkMatch(t, "doubleclick.net") - d.checkMatch(t, "www.doubleclick.net") - d.checkMatch(t, "nodoubleclick.net") - d.checkMatchEmpty(t, "doubleclick.net.ru") -} - -func TestCount(t *testing.T) { - d := NewForTest() - defer d.Destroy() - err := loadTestRules(d) - if err != nil { - t.Fatal(err) - } - count := d.Count() - expected := 12747 - if count != expected { - t.Fatalf("Number of rules parsed should be %d, but it is %d\n", expected, count) - } -} - -func TestDnsFilterBlocking(t *testing.T) { - d := NewForTest() - defer d.Destroy() - d.checkAddRule(t, "||example.org^") - - d.checkMatch(t, "example.org") - d.checkMatch(t, "test.example.org") - d.checkMatch(t, "test.test.example.org") - d.checkMatchEmpty(t, "testexample.org") - d.checkMatchEmpty(t, "onemoreexample.org") -} - -func TestDnsFilterWhitelist(t *testing.T) { - d := NewForTest() - defer d.Destroy() - d.checkAddRule(t, "||example.org^") - d.checkAddRule(t, "@@||test.example.org") - - d.checkMatch(t, "example.org") - d.checkMatchEmpty(t, "test.example.org") - d.checkMatchEmpty(t, "test.test.example.org") - - d.checkAddRule(t, "||googleadapis.l.google.com^|") - d.checkMatch(t, "googleadapis.l.google.com") - d.checkMatch(t, "test.googleadapis.l.google.com") - - d.checkAddRule(t, "@@||googleadapis.l.google.com|") - d.checkMatchEmpty(t, "googleadapis.l.google.com") - d.checkMatchEmpty(t, "test.googleadapis.l.google.com") - -} - -func TestDnsFilterImportant(t *testing.T) { - d := NewForTest() - defer d.Destroy() - d.checkAddRule(t, "@@||example.org^") - d.checkAddRule(t, "||test.example.org^$important") - - d.checkMatchEmpty(t, "example.org") - d.checkMatch(t, "test.example.org") - d.checkMatch(t, "test.test.example.org") - d.checkMatchEmpty(t, "testexample.org") - d.checkMatchEmpty(t, "onemoreexample.org") -} - -func TestDnsFilterRegexrule(t *testing.T) { - d := NewForTest() - defer d.Destroy() - d.checkAddRule(t, "/example\\.org/") - d.checkAddRule(t, "@@||test.example.org^") - - d.checkMatch(t, "example.org") - d.checkMatchEmpty(t, "test.example.org") - d.checkMatchEmpty(t, "test.test.example.org") - d.checkMatch(t, "testexample.org") - d.checkMatch(t, "onemoreexample.org") -} - -func TestDomainMask(t *testing.T) { - d := NewForTest() - defer d.Destroy() - d.checkAddRule(t, "test*.example.org^") - d.checkAddRule(t, "exam*.com") - - d.checkMatch(t, "test.example.org") - d.checkMatch(t, "test2.example.org") - d.checkMatch(t, "example.com") - d.checkMatch(t, "exampleeee.com") - - d.checkMatchEmpty(t, "example.org") - d.checkMatchEmpty(t, "testexample.org") - d.checkMatchEmpty(t, "example.co.uk") -} - -func TestAddRuleFail(t *testing.T) { - d := NewForTest() - defer d.Destroy() - d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384") -} +// SAFE BROWSING func TestSafeBrowsing(t *testing.T) { testCases := []string{ @@ -608,6 +170,25 @@ func TestSafeBrowsingCustomServerFail(t *testing.T) { d.checkMatchEmpty(t, "wmconvirus.narod.ru") } +// SAFE SEARCH + +func TestSafeSearch(t *testing.T) { + d := NewForTest() + defer d.Destroy() + _, ok := d.SafeSearchDomain("www.google.com") + if ok { + t.Errorf("Expected safesearch to error when disabled") + } + d.SafeSearchEnabled = true + val, ok := d.SafeSearchDomain("www.google.com") + if !ok { + t.Errorf("Expected safesearch to find result for www.google.com") + } + if val != "forcesafesearch.google.com" { + t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com") + } +} + func TestCheckHostSafeSearchYandex(t *testing.T) { d := NewForTest() defer d.Destroy() @@ -757,6 +338,8 @@ func TestSafeSearchCacheGoogle(t *testing.T) { } } +// PARENTAL + func TestParentalControl(t *testing.T) { d := NewForTest() defer d.Destroy() @@ -785,63 +368,50 @@ func TestParentalControl(t *testing.T) { d.checkMatchEmpty(t, "api.jquery.com") } -func TestSafeSearch(t *testing.T) { - d := NewForTest() - defer d.Destroy() - _, ok := d.SafeSearchDomain("www.google.com") - if ok { - t.Errorf("Expected safesearch to error when disabled") - } - d.SafeSearchEnabled = true - val, ok := d.SafeSearchDomain("www.google.com") - if !ok { - t.Errorf("Expected safesearch to find result for www.google.com") - } - if val != "forcesafesearch.google.com" { - t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com") - } -} +// FILTERING -// -// parametrized testing -// -var blockingRules = []string{"||example.org^"} -var whitelistRules = []string{"||example.org^", "@@||test.example.org"} -var importantRules = []string{"@@||example.org^", "||test.example.org^$important"} -var regexRules = []string{"/example\\.org/", "@@||test.example.org^"} -var maskRules = []string{"test*.example.org^", "exam*.com"} +var blockingRules = "||example.org^\n" +var whitelistRules = "||example.org^\n@@||test.example.org\n" +var importantRules = "@@||example.org^\n||test.example.org^$important\n" +var regexRules = "/example\\.org/\n@@||test.example.org^\n" +var maskRules = "test*.example.org^\nexam*.com\n" var tests = []struct { testname string - rules []string + rules string hostname string isFiltered bool reason Reason }{ - {"sanity", []string{"||doubleclick.net^"}, "www.doubleclick.net", true, FilteredBlackList}, - {"sanity", []string{"||doubleclick.net^"}, "nodoubleclick.net", false, NotFilteredNotFound}, - {"sanity", []string{"||doubleclick.net^"}, "doubleclick.net.ru", false, NotFilteredNotFound}, - {"sanity", []string{"||doubleclick.net^"}, "wmconvirus.narod.ru", false, NotFilteredNotFound}, + {"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList}, + {"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound}, + {"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound}, + {"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound}, + {"blocking", blockingRules, "example.org", true, FilteredBlackList}, {"blocking", blockingRules, "test.example.org", true, FilteredBlackList}, {"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList}, {"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound}, {"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound}, + {"whitelist", whitelistRules, "example.org", true, FilteredBlackList}, {"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList}, {"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList}, {"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound}, {"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound}, + {"important", importantRules, "example.org", false, NotFilteredWhiteList}, {"important", importantRules, "test.example.org", true, FilteredBlackList}, {"important", importantRules, "test.test.example.org", true, FilteredBlackList}, {"important", importantRules, "testexample.org", false, NotFilteredNotFound}, {"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound}, + {"regex", regexRules, "example.org", true, FilteredBlackList}, {"regex", regexRules, "test.example.org", false, NotFilteredWhiteList}, {"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList}, {"regex", regexRules, "testexample.org", true, FilteredBlackList}, {"regex", regexRules, "onemoreexample.org", true, FilteredBlackList}, + {"mask", maskRules, "test.example.org", true, FilteredBlackList}, {"mask", maskRules, "test2.example.org", true, FilteredBlackList}, {"mask", maskRules, "example.com", true, FilteredBlackList}, @@ -855,14 +425,11 @@ var tests = []struct { func TestMatching(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) { - d := NewForTest() + filters := make(map[int]string) + filters[0] = test.rules + d := NewForTestFilters(filters) defer d.Destroy() - for _, rule := range test.rules { - err := d.AddRule(rule, 0) - if err != nil { - t.Fatal(err) - } - } + ret, err := d.CheckHost(test.hostname) if err != nil { t.Errorf("Error while matching host %s: %s", test.hostname, err) @@ -877,204 +444,7 @@ func TestMatching(t *testing.T) { } } -// -// benchmarks -// -func BenchmarkAddRule(b *testing.B) { - d := NewForTest() - defer d.Destroy() - for n := 0; n < b.N; n++ { - rule := "||doubleclick.net^" - err := d.AddRule(rule, 0) - switch err { - case nil: - case ErrAlreadyExists: // ignore rules which were already added - case ErrInvalidSyntax: // ignore invalid syntax - default: - b.Fatalf("Error while adding rule %s: %s", rule, err) - } - } -} - -func BenchmarkAddRuleParallel(b *testing.B) { - d := NewForTest() - defer d.Destroy() - rule := "||doubleclick.net^" - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - var err error - for pb.Next() { - err = d.AddRule(rule, 0) - } - switch err { - case nil: - case ErrAlreadyExists: // ignore rules which were already added - case ErrInvalidSyntax: // ignore invalid syntax - default: - b.Fatalf("Error while adding rule %s: %s", rule, err) - } - }) -} - -func BenchmarkLotsOfRulesNoMatch(b *testing.B) { - d := NewForTest() - defer d.Destroy() - err := loadTestRules(d) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - for n := 0; n < b.N; n++ { - hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com" - ret, err := d.CheckHost(hostname) - if err != nil { - b.Errorf("Error while matching host %s: %s", hostname, err) - } - if ret.IsFiltered { - b.Errorf("Expected hostname %s to not match", hostname) - } - } -} - -func BenchmarkLotsOfRulesNoMatchParallel(b *testing.B) { - d := NewForTest() - defer d.Destroy() - err := loadTestRules(d) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com" - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - ret, err := d.CheckHost(hostname) - if err != nil { - b.Errorf("Error while matching host %s: %s", hostname, err) - } - if ret.IsFiltered { - b.Errorf("Expected hostname %s to not match", hostname) - } - } - }) -} - -func BenchmarkLotsOfRulesMatch(b *testing.B) { - d := NewForTest() - defer d.Destroy() - err := loadTestRules(d) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - for n := 0; n < b.N; n++ { - const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net" - ret, err := d.CheckHost(hostname) - if err != nil { - b.Errorf("Error while matching host %s: %s", hostname, err) - } - if !ret.IsFiltered { - b.Errorf("Expected hostname %s to match", hostname) - } - } -} - -func BenchmarkLotsOfRulesMatchParallel(b *testing.B) { - d := NewForTest() - defer d.Destroy() - err := loadTestRules(d) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net" - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - ret, err := d.CheckHost(hostname) - if err != nil { - b.Errorf("Error while matching host %s: %s", hostname, err) - } - if !ret.IsFiltered { - b.Errorf("Expected hostname %s to match", hostname) - } - } - }) -} - -func BenchmarkLotsOfRulesLotsOfHosts(b *testing.B) { - d := NewForTest() - defer d.Destroy() - mustLoadTestRules(d) - - getTopHosts() - hostnames, err := os.Open(topHostsFilename) - if err != nil { - b.Fatal(err) - } - defer hostnames.Close() - - scanner := bufio.NewScanner(hostnames) - b.ResetTimer() - for n := 0; n < b.N; n++ { - havedata := scanner.Scan() - if !havedata { - _, _ = hostnames.Seek(0, 0) - scanner = bufio.NewScanner(hostnames) - havedata = scanner.Scan() - } - if !havedata { - b.Fatal(scanner.Err()) - } - line := scanner.Text() - records := strings.Split(line, ",") - ret, err := d.CheckHost(records[1] + "." + records[1]) - if err != nil { - b.Error(err) - } - if ret.Reason.Matched() { - // log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason) - } - } -} - -func BenchmarkLotsOfRulesLotsOfHostsParallel(b *testing.B) { - d := NewForTest() - defer d.Destroy() - mustLoadTestRules(d) - - getTopHosts() - - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - hostnames, err := os.Open(topHostsFilename) - if err != nil { - b.Fatal(err) - } - defer hostnames.Close() - scanner := bufio.NewScanner(hostnames) - for pb.Next() { - havedata := scanner.Scan() - if !havedata { - _, _ = hostnames.Seek(0, 0) - scanner = bufio.NewScanner(hostnames) - havedata = scanner.Scan() - } - if !havedata { - b.Fatal(scanner.Err()) - } - line := scanner.Text() - records := strings.Split(line, ",") - ret, err := d.CheckHost(records[1] + "." + records[1]) - if err != nil { - b.Error(err) - } - if ret.Reason.Matched() { - // log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason) - } - } - }) -} +// BENCHMARKS func BenchmarkSafeBrowsing(b *testing.B) { d := NewForTest() @@ -1141,26 +511,3 @@ func BenchmarkSafeSearchParallel(b *testing.B) { } }) } - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) -} - -// -// helper functions for debugging and testing -// -func purgeCaches() { - if safebrowsingCache != nil { - safebrowsingCache.Purge() - } - if parentalCache != nil { - parentalCache.Purge() - } -} - -func _Func() string { - pc := make([]uintptr, 10) // at least 1 entry needed - runtime.Callers(2, pc) - f := runtime.FuncForPC(pc[0]) - return path.Base(f.Name()) -} From 096a95998749b673bc9be638bc9c8f6f0d13be41 Mon Sep 17 00:00:00 2001 From: Simon Zolin Date: Thu, 16 May 2019 17:32:15 +0300 Subject: [PATCH 7/7] * dnsforward: use new dnsfilter interface --- dnsforward/dnsforward.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dnsforward/dnsforward.go b/dnsforward/dnsforward.go index a1afd7a9..51c6d3f6 100644 --- a/dnsforward/dnsforward.go +++ b/dnsforward/dnsforward.go @@ -202,14 +202,20 @@ func (s *Server) startInternal(config *ServerConfig) error { // Initializes the DNS filter func (s *Server) initDNSFilter() error { log.Tracef("Creating dnsfilter") - s.dnsFilter = dnsfilter.New(&s.conf.Config) - // add rules only if they are enabled + + var filters map[int]string + filters = nil if s.conf.FilteringEnabled { - err := s.dnsFilter.AddRules(s.conf.Filters) - if err != nil { - return errorx.Decorate(err, "could not initialize dnsfilter") + filters = make(map[int]string) + for _, f := range s.conf.Filters { + filters[int(f.ID)] = string(f.Data) } } + + s.dnsFilter = dnsfilter.New(&s.conf.Config, filters) + if s.dnsFilter == nil { + return fmt.Errorf("could not initialize dnsfilter") + } return nil }