mirrors/AdGuardHome
1
0
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-11-23 21:45:46 +03:00
Code Issues Projects Releases Packages Wiki Activity

Merge: dnsfilter: use urlfilter package #714

Browse source 
* commit '096a95998749b673bc9be638bc9c8f6f0d13be41':
  * dnsforward: use new dnsfilter interface
  * dnsfilter: adapt tests to new interface
  * dnsfilter: use urlfilter package
  * dnsfilter: remove code for filtering rules
  * dns: rename dnsfilter.Filter.Rule -> dnsfilter.Filter.Data
  * dnsforward: use separate ServerConfig object
  * use urlfilter
This commit is contained in:
Simon Zolin 2019-05-20 11:00:45 +03:00
commit e3ee7a0c3e
10 changed files with 223 additions and 1398 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
dns.go
View file

@ -34,13 +34,13 @@ func generateServerConfig() dnsforward.ServerConfig {
filters := []dnsfilter.Filter{}
userFilter := userFilter()
filters = append(filters, dnsfilter.Filter{
ID: userFilter.ID,
Rules: userFilter.Rules,
ID: userFilter.ID,
Data: userFilter.Data,
})
for _, filter := range config.Filters {
filters = append(filters, dnsfilter.Filter{
ID: filter.ID,
Rules: filter.Rules,
ID: filter.ID,
Data: filter.Data,
})
}

536
dnsfilter/dnsfilter.go
View file

@ -11,14 +11,13 @@ import (
"io/ioutil"
"net"
"net/http"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/urlfilter"
"github.com/bluele/gcache"
"golang.org/x/net/publicsuffix"
)
@ -48,11 +47,12 @@ const enableDelayedCompilation = true // flag for debugging, must be true in pro
// Config allows you to configure DNS filtering with New() or just change variables directly.
type Config struct {
ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17
ParentalEnabled bool `yaml:"parental_enabled"`
SafeSearchEnabled bool `yaml:"safesearch_enabled"`
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
ResolverAddress string // DNS server address
FilteringTempFilename string `yaml:"filtering_temp_filename"` // temporary file for storing unused filtering rules
ParentalSensitivity int `yaml:"parental_sensitivity"` // must be either 3, 10, 13 or 17
ParentalEnabled bool `yaml:"parental_enabled"`
SafeSearchEnabled bool `yaml:"safesearch_enabled"`
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
ResolverAddress string // DNS server address
}
type privateConfig struct {
@ -60,33 +60,6 @@ type privateConfig struct {
safeBrowsingServer string // access via methods
}
type rule struct {
text string // text without @@ decorators or $ options
shortcut string // for speeding up lookup
originalText string // original text for reporting back to applications
ip net.IP // IP address (for the case when we're matching a hosts file)
// options
options []string // optional options after $
// parsed options
apps []string
isWhitelist bool
isImportant bool
// user-supplied data
listID int64
// suffix matching
isSuffix bool
suffix string
// compiled regexp
compiled *regexp.Regexp
sync.RWMutex
}
// LookupStats store stats collected during safebrowsing or parental checks
type LookupStats struct {
Requests uint64 // number of HTTP requests that were sent
@ -104,13 +77,8 @@ type Stats struct {
// Dnsfilter holds added rules and performs hostname matches against the rules
type Dnsfilter struct {
storage map[string]bool // rule storage, not used for matching, just for filtering out duplicates
storageMutex sync.RWMutex
// rules are checked against these lists in the order defined here
important *rulesTable // more important than whitelist and is checked first
whiteList *rulesTable // more important than blacklist
blackList *rulesTable
rulesStorage *urlfilter.RulesStorage
filteringEngine *urlfilter.DNSEngine
// HTTP lookups for safebrowsing and parental
client http.Client // handle for http client -- single instance as recommended by docs
@ -122,8 +90,8 @@ type Dnsfilter struct {
// Filter represents a filter list
type Filter struct {
ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase
Rules []string `json:"-" yaml:"-"` // not in yaml or json
ID int64 `json:"id"` // auto-assigned when filter is added (see nextFilterID), json by default keeps ID uppercase but we need lowercase
Data []byte `json:"-" yaml:"-"` // List of rules divided by '\n'
}
//go:generate stringer -type=Reason
@ -242,308 +210,6 @@ func (d *Dnsfilter) CheckHost(host string) (Result, error) {
return Result{}, nil
}
//
// rules table
//
type rulesTable struct {
rulesByHost map[string]*rule
rulesByShortcut map[string][]*rule
rulesLeftovers []*rule
sync.RWMutex
}
func newRulesTable() *rulesTable {
return &rulesTable{
rulesByHost: make(map[string]*rule),
rulesByShortcut: make(map[string][]*rule),
rulesLeftovers: make([]*rule, 0),
}
}
func (r *rulesTable) Add(rule *rule) {
r.Lock()
if rule.ip != nil {
// Hosts syntax
r.rulesByHost[rule.text] = rule
} else if len(rule.shortcut) == shortcutLength && enableFastLookup {
// Adblock syntax with a shortcut
r.rulesByShortcut[rule.shortcut] = append(r.rulesByShortcut[rule.shortcut], rule)
} else {
// Adblock syntax -- too short to have a shortcut
r.rulesLeftovers = append(r.rulesLeftovers, rule)
}
r.Unlock()
}
func (r *rulesTable) matchByHost(host string) (Result, error) {
// First: examine the hosts-syntax rules
res, err := r.searchByHost(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
return res, nil
}
// Second: examine the adblock-syntax rules with shortcuts
res, err = r.searchShortcuts(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
return res, nil
}
// Third: examine the others
res, err = r.searchLeftovers(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
return res, nil
}
return Result{}, nil
}
func (r *rulesTable) searchByHost(host string) (Result, error) {
rule, ok := r.rulesByHost[host]
if ok {
return rule.match(host)
}
return Result{}, nil
}
func (r *rulesTable) searchShortcuts(host string) (Result, error) {
// check in shortcuts first
for i := 0; i < len(host); i++ {
shortcut := host[i:]
if len(shortcut) > shortcutLength {
shortcut = shortcut[:shortcutLength]
}
if len(shortcut) != shortcutLength {
continue
}
rules, ok := r.rulesByShortcut[shortcut]
if !ok {
continue
}
for _, rule := range rules {
res, err := rule.match(host)
// error? stop search
if err != nil {
return res, err
}
// matched? stop search
if res.Reason.Matched() {
return res, err
}
// continue otherwise
}
}
return Result{}, nil
}
func (r *rulesTable) searchLeftovers(host string) (Result, error) {
for _, rule := range r.rulesLeftovers {
res, err := rule.match(host)
// error? stop search
if err != nil {
return res, err
}
// matched? stop search
if res.Reason.Matched() {
return res, err
}
// continue otherwise
}
return Result{}, nil
}
func findOptionIndex(text string) int {
for i, r := range text {
// ignore non-$
if r != '$' {
continue
}
// ignore `\$`
if i > 0 && text[i-1] == '\\' {
continue
}
// ignore `$/`
if i > len(text) && text[i+1] == '/' {
continue
}
return i + 1
}
return -1
}
func (rule *rule) extractOptions() error {
optIndex := findOptionIndex(rule.text)
if optIndex == 0 { // starts with $
return ErrInvalidSyntax
}
if optIndex == len(rule.text) { // ends with $
return ErrInvalidSyntax
}
if optIndex < 0 {
return nil
}
optionsStr := rule.text[optIndex:]
rule.text = rule.text[:optIndex-1] // remove options from text
begin := 0
i := 0
for i = 0; i < len(optionsStr); i++ {
switch optionsStr[i] {
case ',':
if i > 0 {
// it might be escaped, if so, ignore
if optionsStr[i-1] == '\\' {
break // from switch, not for loop
}
}
rule.options = append(rule.options, optionsStr[begin:i])
begin = i + 1
}
}
if begin != i {
// there's still an option remaining
rule.options = append(rule.options, optionsStr[begin:])
}
return nil
}
func (rule *rule) parseOptions() error {
err := rule.extractOptions()
if err != nil {
return err
}
for _, option := range rule.options {
switch {
case option == "important":
rule.isImportant = true
case strings.HasPrefix(option, "app="):
option = strings.TrimPrefix(option, "app=")
rule.apps = strings.Split(option, "|")
default:
return ErrInvalidSyntax
}
}
return nil
}
func (rule *rule) extractShortcut() {
// regex rules have no shortcuts
if rule.text[0] == '/' && rule.text[len(rule.text)-1] == '/' {
return
}
fields := strings.FieldsFunc(rule.text, func(r rune) bool {
switch r {
case '*', '^', '|':
return true
}
return false
})
longestField := ""
for _, field := range fields {
if len(field) > len(longestField) {
longestField = field
}
}
if len(longestField) > shortcutLength {
longestField = longestField[:shortcutLength]
}
rule.shortcut = strings.ToLower(longestField)
}
func (rule *rule) compile() error {
rule.RLock()
isCompiled := rule.isSuffix || rule.compiled != nil
rule.RUnlock()
if isCompiled {
return nil
}
isSuffix, suffix := getSuffix(rule.text)
if isSuffix {
rule.Lock()
rule.isSuffix = isSuffix
rule.suffix = suffix
rule.Unlock()
return nil
}
expr, err := ruleToRegexp(rule.text)
if err != nil {
return err
}
compiled, err := regexp.Compile(expr)
if err != nil {
return err
}
rule.Lock()
rule.compiled = compiled
rule.Unlock()
return nil
}
// Checks if the rule matches the specified host and returns a corresponding Result object
func (rule *rule) match(host string) (Result, error) {
res := Result{}
if rule.ip != nil && rule.text == host {
// This is a hosts-syntax rule -- just check that the hostname matches and return the result
return Result{
IsFiltered: true,
Reason: FilteredBlackList,
Rule: rule.originalText,
IP: rule.ip,
FilterID: rule.listID,
}, nil
}
err := rule.compile()
if err != nil {
return res, err
}
rule.RLock()
matched := false
if rule.isSuffix {
if host == rule.suffix {
matched = true
} else if strings.HasSuffix(host, "."+rule.suffix) {
matched = true
}
} else {
matched = rule.compiled.MatchString(host)
}
rule.RUnlock()
if matched {
res.Reason = FilteredBlackList
res.IsFiltered = true
res.FilterID = rule.listID
res.Rule = rule.originalText
if rule.isWhitelist {
res.Reason = NotFilteredWhiteList
res.IsFiltered = false
}
}
return res, nil
}
func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) {
isFound = false // not found yet
@ -838,135 +504,59 @@ func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gc
// Adding rule and matching against the rules
//
// AddRules is a convinience function to add an array of filters in one call
func (d *Dnsfilter) AddRules(filters []Filter) error {
for _, f := range filters {
for _, rule := range f.Rules {
err := d.AddRule(rule, f.ID)
if err == ErrAlreadyExists || err == ErrInvalidSyntax {
continue
}
if err != nil {
log.Printf("Cannot add rule %s: %s", rule, err)
// Just ignore invalid rules
continue
}
}
}
return nil
}
// AddRule adds a rule, checking if it is a valid rule first and if it wasn't added already
func (d *Dnsfilter) AddRule(input string, filterListID int64) error {
input = strings.TrimSpace(input)
d.storageMutex.RLock()
_, exists := d.storage[input]
d.storageMutex.RUnlock()
if exists {
// already added
return ErrAlreadyExists
}
if !isValidRule(input) {
return ErrInvalidSyntax
}
// First, check if this is a hosts-syntax rule
if d.parseEtcHosts(input, filterListID) {
// This is a valid hosts-syntax rule, no need for further parsing
return nil
}
// Start parsing the rule
r := rule{
text: input, // will be modified
originalText: input,
listID: filterListID,
}
// Mark rule as whitelist if it starts with @@
if strings.HasPrefix(r.text, "@@") {
r.isWhitelist = true
r.text = r.text[2:]
}
err := r.parseOptions()
// Initialize urlfilter objects
func (d *Dnsfilter) initFiltering(filters map[int]string) error {
var err error
d.rulesStorage, err = urlfilter.NewRuleStorage(d.FilteringTempFilename)
if err != nil {
return err
}
r.extractShortcut()
if !enableDelayedCompilation {
err := r.compile()
if err != nil {
return err
}
}
destination := d.blackList
if r.isImportant {
destination = d.important
} else if r.isWhitelist {
destination = d.whiteList
}
d.storageMutex.Lock()
d.storage[input] = true
d.storageMutex.Unlock()
destination.Add(&r)
d.filteringEngine = urlfilter.NewDNSEngine(filters, d.rulesStorage)
return nil
}
// Parses the hosts-syntax rules. Returns false if the input string is not of hosts-syntax.
func (d *Dnsfilter) parseEtcHosts(input string, filterListID int64) bool {
// Strip the trailing comment
ruleText := input
if pos := strings.IndexByte(ruleText, '#'); pos != -1 {
ruleText = ruleText[0:pos]
}
fields := strings.Fields(ruleText)
if len(fields) < 2 {
return false
}
addr := net.ParseIP(fields[0])
if addr == nil {
return false
}
d.storageMutex.Lock()
d.storage[input] = true
d.storageMutex.Unlock()
for _, host := range fields[1:] {
r := rule{
text: host,
originalText: input,
listID: filterListID,
ip: addr,
}
d.blackList.Add(&r)
}
return true
}
// matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups
func (d *Dnsfilter) matchHost(host string) (Result, error) {
lists := []*rulesTable{
d.important,
d.whiteList,
d.blackList,
if d.filteringEngine == nil {
return Result{}, nil
}
for _, table := range lists {
res, err := table.matchByHost(host)
if err != nil {
return res, err
}
if res.Reason.Matched() {
rules, ok := d.filteringEngine.Match(host)
if !ok {
return Result{}, nil
}
for _, rule := range rules {
log.Tracef("Found rule for host '%s': '%s' list_id: %d",
host, rule.Text(), rule.GetFilterListID())
res := Result{}
res.Reason = FilteredBlackList
res.IsFiltered = true
res.FilterID = int64(rule.GetFilterListID())
res.Rule = rule.Text()
if netRule, ok := rule.(*urlfilter.NetworkRule); ok {
if netRule.Whitelist {
res.Reason = NotFilteredWhiteList
res.IsFiltered = false
}
return res, nil
} else if hostRule, ok := rule.(*urlfilter.HostRule); ok {
res.IP = hostRule.IP
return res, nil
} else {
log.Tracef("Rule type is unsupported: '%s' list_id: %d",
rule.Text(), rule.GetFilterListID())
}
}
return Result{}, nil
}
@ -1058,14 +648,9 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp
}
// New creates properly initialized DNS Filter that is ready to be used
func New(c *Config) *Dnsfilter {
func New(c *Config, filters map[int]string) *Dnsfilter {
d := new(Dnsfilter)
d.storage = make(map[string]bool)
d.important = newRulesTable()
d.whiteList = newRulesTable()
d.blackList = newRulesTable()
// Customize the Transport to have larger connection pool,
// We are not (re)using http.DefaultTransport because of race conditions found by tests
d.transport = &http.Transport{
@ -1090,6 +675,15 @@ func New(c *Config) *Dnsfilter {
d.Config = *c
}
if filters != nil {
err := d.initFiltering(filters)
if err != nil {
log.Error("Can't initialize filtering subsystem: %s", err)
d.Destroy()
return nil
}
}
return d
}
@ -1099,6 +693,11 @@ func (d *Dnsfilter) Destroy() {
if d != nil && d.transport != nil {
d.transport.CloseIdleConnections()
}
if d.rulesStorage != nil {
d.rulesStorage.Close()
d.rulesStorage = nil
}
}
//
@ -1141,8 +740,3 @@ func (d *Dnsfilter) SafeSearchDomain(host string) (string, bool) {
func (d *Dnsfilter) GetStats() Stats {
return stats
}
// Count returns number of rules added to filter
func (d *Dnsfilter) Count() int {
return len(d.storage)
}

801
dnsfilter/dnsfilter_test.go
View file

@ -1,278 +1,51 @@
package dnsfilter
import (
"archive/zip"
"bytes"
"io/ioutil"
"fmt"
"net"
"net/http"
"net/http/httptest"
"path"
"runtime/pprof"
"strings"
"runtime"
"testing"
"time"
"bufio"
"fmt"
"os"
"runtime"
"github.com/AdguardTeam/golibs/log"
"github.com/shirou/gopsutil/process"
"go.uber.org/goleak"
)
// first in file because it must be run first
func TestLotsOfRulesMemoryUsage(t *testing.T) {
start := getRSS()
log.Tracef("RSS before loading rules - %d kB\n", start/1024)
dumpMemProfile("tests/" + _Func() + "1.pprof")
// HELPERS
// SAFE BROWSING
// SAFE SEARCH
// PARENTAL
// FILTERING
// BENCHMARKS
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
t.Error(err)
}
// HELPERS
afterLoad := getRSS()
log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
dumpMemProfile("tests/" + _Func() + "2.pprof")
tests := []struct {
host string
match bool
}{
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com", false},
{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net", true},
func purgeCaches() {
if safebrowsingCache != nil {
safebrowsingCache.Purge()
}
for _, testcase := range tests {
ret, err := d.CheckHost(testcase.host)
if err != nil {
t.Errorf("Error while matching host %s: %s", testcase.host, err)
}
if !ret.IsFiltered && ret.IsFiltered != testcase.match {
t.Errorf("Expected hostname %s to not match", testcase.host)
}
if ret.IsFiltered && ret.IsFiltered != testcase.match {
t.Errorf("Expected hostname %s to match", testcase.host)
}
}
afterMatch := getRSS()
log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
dumpMemProfile("tests/" + _Func() + "3.pprof")
}
func getRSS() uint64 {
proc, err := process.NewProcess(int32(os.Getpid()))
if err != nil {
panic(err)
}
minfo, err := proc.MemoryInfo()
if err != nil {
panic(err)
}
return minfo.RSS
}
func dumpMemProfile(name string) {
runtime.GC()
f, err := os.Create(name)
if err != nil {
panic(err)
}
defer f.Close()
runtime.GC() // update the stats before writing them
err = pprof.WriteHeapProfile(f)
if err != nil {
panic(err)
if parentalCache != nil {
parentalCache.Purge()
}
}
const topHostsFilename = "tests/top-1m.csv"
func fetchTopHostsFromNet() {
log.Tracef("Fetching top hosts from network")
resp, err := http.Get("http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip")
if err != nil {
panic(err)
}
defer resp.Body.Close()
log.Tracef("Reading zipfile body")
zipfile, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
log.Tracef("Opening zipfile")
r, err := zip.NewReader(bytes.NewReader(zipfile), int64(len(zipfile)))
if err != nil {
panic(err)
}
if len(r.File) != 1 {
panic(fmt.Errorf("zipfile must have only one entry: %+v", r))
}
f := r.File[0]
log.Tracef("Unpacking file %s from zipfile", f.Name)
rc, err := f.Open()
if err != nil {
panic(err)
}
log.Tracef("Reading file %s contents", f.Name)
body, err := ioutil.ReadAll(rc)
if err != nil {
panic(err)
}
rc.Close()
log.Tracef("Writing file %s contents to disk", f.Name)
err = ioutil.WriteFile(topHostsFilename+".tmp", body, 0644)
if err != nil {
panic(err)
}
err = os.Rename(topHostsFilename+".tmp", topHostsFilename)
if err != nil {
panic(err)
}
func _Func() string {
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])
return path.Base(f.Name())
}
func getTopHosts() {
// if file doesn't exist, fetch it
if _, err := os.Stat(topHostsFilename); os.IsNotExist(err) {
// file does not exist, fetch it
fetchTopHostsFromNet()
}
func NewForTest() *Dnsfilter {
d := New(nil, nil)
purgeCaches()
return d
}
func TestLotsOfRulesLotsOfHostsMemoryUsage(t *testing.T) {
start := getRSS()
log.Tracef("RSS before loading rules - %d kB\n", start/1024)
dumpMemProfile("tests/" + _Func() + "1.pprof")
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
log.Tracef("Have %d rules", d.Count())
afterLoad := getRSS()
log.Tracef("RSS after loading rules - %d kB (%d kB diff)\n", afterLoad/1024, (afterLoad-start)/1024)
dumpMemProfile("tests/" + _Func() + "2.pprof")
getTopHosts()
hostnames, err := os.Open(topHostsFilename)
if err != nil {
t.Fatal(err)
}
defer hostnames.Close()
afterHosts := getRSS()
log.Tracef("RSS after loading hosts - %d kB (%d kB diff)\n", afterHosts/1024, (afterHosts-afterLoad)/1024)
dumpMemProfile("tests/" + _Func() + "2.pprof")
{
scanner := bufio.NewScanner(hostnames)
for scanner.Scan() {
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
t.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
}
afterMatch := getRSS()
log.Tracef("RSS after matching - %d kB (%d kB diff)\n", afterMatch/1024, (afterMatch-afterLoad)/1024)
dumpMemProfile("tests/" + _Func() + "3.pprof")
}
func TestRuleToRegexp(t *testing.T) {
tests := []struct {
rule string
result string
err error
}{
{"/doubleclick/", "doubleclick", nil},
{"/", "", ErrInvalidSyntax},
{`|double*?.+[]|(){}#$\|`, `^double.*\?\.\+\[\]\|\(\)\{\}\#\$\\$`, nil},
{`||doubleclick.net^`, `(?:^|\.)doubleclick\.net$`, nil},
}
for _, testcase := range tests {
converted, err := ruleToRegexp(testcase.rule)
if err != testcase.err {
t.Error("Errors do not match, got ", err, " expected ", testcase.err)
}
if converted != testcase.result {
t.Error("Results do not match, got ", converted, " expected ", testcase.result)
}
}
}
func TestSuffixRule(t *testing.T) {
for _, testcase := range []struct {
rule string
isSuffix bool
suffix string
}{
{`||doubleclick.net^`, true, `doubleclick.net`}, // entire string or subdomain match
{`||doubleclick.net|`, true, `doubleclick.net`}, // entire string or subdomain match
{`|doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`|*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`||*doubleclick.net^`, false, ``}, // TODO: ends with doubleclick.net
{`||*doubleclick.net|`, false, ``}, // TODO: ends with doubleclick.net
{`||*doublec*lick.net^`, false, ``}, // has a wildcard inside, has to be regexp
{`||*doublec|lick.net^`, false, ``}, // has a special symbol inside, has to be regexp
{`/abracadabra/`, false, ``}, // regexp, not anchored
{`/abracadabra$/`, false, ``}, // TODO: simplify simple suffix regexes
} {
isSuffix, suffix := getSuffix(testcase.rule)
if testcase.isSuffix != isSuffix {
t.Errorf("Results do not match for \"%s\": got %v expected %v", testcase.rule, isSuffix, testcase.isSuffix)
continue
}
if testcase.isSuffix && testcase.suffix != suffix {
t.Errorf("Result suffix does not match for \"%s\": got \"%s\" expected \"%s\"", testcase.rule, suffix, testcase.suffix)
continue
}
// log.Tracef("\"%s\": %v: %s", testcase.rule, isSuffix, suffix)
}
}
//
// helper functions
//
func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) {
t.Helper()
err := d.AddRule(rule, 0)
if err == nil {
// nothing to report
return
}
if err == ErrInvalidSyntax {
t.Errorf("This rule has invalid syntax: %s", rule)
}
if err != nil {
t.Errorf("Error while adding rule %s: %s", rule, err)
}
}
func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) {
t.Helper()
err := d.AddRule(rule, 0)
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
return
}
if err != nil {
t.Errorf("Error while adding rule %s: %s", rule, err)
}
t.Errorf("Adding this rule should have failed: %s", rule)
func NewForTestFilters(filters map[int]string) *Dnsfilter {
d := New(nil, filters)
purgeCaches()
return d
}
func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
@ -311,232 +84,21 @@ func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
}
}
func loadTestRules(d *Dnsfilter) error {
filterFileName := "tests/dns.txt"
file, err := os.Open(filterFileName)
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
rule := scanner.Text()
err = d.AddRule(rule, 0)
if err == ErrInvalidSyntax || err == ErrAlreadyExists {
continue
}
if err != nil {
return err
}
}
err = scanner.Err()
return err
}
func mustLoadTestRules(d *Dnsfilter) {
err := loadTestRules(d)
if err != nil {
panic(err)
}
}
func NewForTest() *Dnsfilter {
d := New(nil)
purgeCaches()
return d
}
//
// tests
//
func TestSanityCheck(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
}
func TestEtcHostsMatching(t *testing.T) {
d := NewForTest()
defer d.Destroy()
addr := "216.239.38.120"
text := fmt.Sprintf(" %s google.com www.google.com # enforce google's safesearch ", addr)
filters := make(map[int]string)
filters[0] = text
d := NewForTestFilters(filters)
defer d.Destroy()
d.checkAddRule(t, text)
d.checkMatchIP(t, "google.com", addr)
d.checkMatchIP(t, "www.google.com", addr)
d.checkMatchEmpty(t, "subdomain.google.com")
d.checkMatchEmpty(t, "example.org")
}
func TestSuffixMatching1(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching2(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "|doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatchEmpty(t, "www.doubleclick.net")
d.checkMatchEmpty(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching3(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching4(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching5(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "|*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestSuffixMatching6(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||*doubleclick.net^")
d.checkMatch(t, "doubleclick.net")
d.checkMatch(t, "www.doubleclick.net")
d.checkMatch(t, "nodoubleclick.net")
d.checkMatchEmpty(t, "doubleclick.net.ru")
}
func TestCount(t *testing.T) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
t.Fatal(err)
}
count := d.Count()
expected := 12747
if count != expected {
t.Fatalf("Number of rules parsed should be %d, but it is %d\n", expected, count)
}
}
func TestDnsFilterBlocking(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||example.org^")
d.checkMatch(t, "example.org")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test.test.example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "onemoreexample.org")
}
func TestDnsFilterWhitelist(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "||example.org^")
d.checkAddRule(t, "@@||test.example.org")
d.checkMatch(t, "example.org")
d.checkMatchEmpty(t, "test.example.org")
d.checkMatchEmpty(t, "test.test.example.org")
d.checkAddRule(t, "||googleadapis.l.google.com^|")
d.checkMatch(t, "googleadapis.l.google.com")
d.checkMatch(t, "test.googleadapis.l.google.com")
d.checkAddRule(t, "@@||googleadapis.l.google.com|")
d.checkMatchEmpty(t, "googleadapis.l.google.com")
d.checkMatchEmpty(t, "test.googleadapis.l.google.com")
}
func TestDnsFilterImportant(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "@@||example.org^")
d.checkAddRule(t, "||test.example.org^$important")
d.checkMatchEmpty(t, "example.org")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test.test.example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "onemoreexample.org")
}
func TestDnsFilterRegexrule(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "/example\\.org/")
d.checkAddRule(t, "@@||test.example.org^")
d.checkMatch(t, "example.org")
d.checkMatchEmpty(t, "test.example.org")
d.checkMatchEmpty(t, "test.test.example.org")
d.checkMatch(t, "testexample.org")
d.checkMatch(t, "onemoreexample.org")
}
func TestDomainMask(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRule(t, "test*.example.org^")
d.checkAddRule(t, "exam*.com")
d.checkMatch(t, "test.example.org")
d.checkMatch(t, "test2.example.org")
d.checkMatch(t, "example.com")
d.checkMatch(t, "exampleeee.com")
d.checkMatchEmpty(t, "example.org")
d.checkMatchEmpty(t, "testexample.org")
d.checkMatchEmpty(t, "example.co.uk")
}
func TestAddRuleFail(t *testing.T) {
d := NewForTest()
defer d.Destroy()
d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
}
// SAFE BROWSING
func TestSafeBrowsing(t *testing.T) {
testCases := []string{
@ -608,6 +170,25 @@ func TestSafeBrowsingCustomServerFail(t *testing.T) {
d.checkMatchEmpty(t, "wmconvirus.narod.ru")
}
// SAFE SEARCH
func TestSafeSearch(t *testing.T) {
d := NewForTest()
defer d.Destroy()
_, ok := d.SafeSearchDomain("www.google.com")
if ok {
t.Errorf("Expected safesearch to error when disabled")
}
d.SafeSearchEnabled = true
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
t.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
func TestCheckHostSafeSearchYandex(t *testing.T) {
d := NewForTest()
defer d.Destroy()
@ -757,6 +338,8 @@ func TestSafeSearchCacheGoogle(t *testing.T) {
}
}
// PARENTAL
func TestParentalControl(t *testing.T) {
d := NewForTest()
defer d.Destroy()
@ -785,63 +368,50 @@ func TestParentalControl(t *testing.T) {
d.checkMatchEmpty(t, "api.jquery.com")
}
func TestSafeSearch(t *testing.T) {
d := NewForTest()
defer d.Destroy()
_, ok := d.SafeSearchDomain("www.google.com")
if ok {
t.Errorf("Expected safesearch to error when disabled")
}
d.SafeSearchEnabled = true
val, ok := d.SafeSearchDomain("www.google.com")
if !ok {
t.Errorf("Expected safesearch to find result for www.google.com")
}
if val != "forcesafesearch.google.com" {
t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
}
}
// FILTERING
//
// parametrized testing
//
var blockingRules = []string{"||example.org^"}
var whitelistRules = []string{"||example.org^", "@@||test.example.org"}
var importantRules = []string{"@@||example.org^", "||test.example.org^$important"}
var regexRules = []string{"/example\\.org/", "@@||test.example.org^"}
var maskRules = []string{"test*.example.org^", "exam*.com"}
var blockingRules = "||example.org^\n"
var whitelistRules = "||example.org^\n@@||test.example.org\n"
var importantRules = "@@||example.org^\n||test.example.org^$important\n"
var regexRules = "/example\\.org/\n@@||test.example.org^\n"
var maskRules = "test*.example.org^\nexam*.com\n"
var tests = []struct {
testname string
rules []string
rules string
hostname string
isFiltered bool
reason Reason
}{
{"sanity", []string{"||doubleclick.net^"}, "www.doubleclick.net", true, FilteredBlackList},
{"sanity", []string{"||doubleclick.net^"}, "nodoubleclick.net", false, NotFilteredNotFound},
{"sanity", []string{"||doubleclick.net^"}, "doubleclick.net.ru", false, NotFilteredNotFound},
{"sanity", []string{"||doubleclick.net^"}, "wmconvirus.narod.ru", false, NotFilteredNotFound},
{"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList},
{"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound},
{"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound},
{"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound},
{"blocking", blockingRules, "example.org", true, FilteredBlackList},
{"blocking", blockingRules, "test.example.org", true, FilteredBlackList},
{"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList},
{"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound},
{"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound},
{"whitelist", whitelistRules, "example.org", true, FilteredBlackList},
{"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList},
{"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList},
{"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound},
{"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound},
{"important", importantRules, "example.org", false, NotFilteredWhiteList},
{"important", importantRules, "test.example.org", true, FilteredBlackList},
{"important", importantRules, "test.test.example.org", true, FilteredBlackList},
{"important", importantRules, "testexample.org", false, NotFilteredNotFound},
{"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound},
{"regex", regexRules, "example.org", true, FilteredBlackList},
{"regex", regexRules, "test.example.org", false, NotFilteredWhiteList},
{"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList},
{"regex", regexRules, "testexample.org", true, FilteredBlackList},
{"regex", regexRules, "onemoreexample.org", true, FilteredBlackList},
{"mask", maskRules, "test.example.org", true, FilteredBlackList},
{"mask", maskRules, "test2.example.org", true, FilteredBlackList},
{"mask", maskRules, "example.com", true, FilteredBlackList},
@ -855,14 +425,11 @@ var tests = []struct {
func TestMatching(t *testing.T) {
for _, test := range tests {
t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) {
d := NewForTest()
filters := make(map[int]string)
filters[0] = test.rules
d := NewForTestFilters(filters)
defer d.Destroy()
for _, rule := range test.rules {
err := d.AddRule(rule, 0)
if err != nil {
t.Fatal(err)
}
}
ret, err := d.CheckHost(test.hostname)
if err != nil {
t.Errorf("Error while matching host %s: %s", test.hostname, err)
@ -877,204 +444,7 @@ func TestMatching(t *testing.T) {
}
}
//
// benchmarks
//
func BenchmarkAddRule(b *testing.B) {
d := NewForTest()
defer d.Destroy()
for n := 0; n < b.N; n++ {
rule := "||doubleclick.net^"
err := d.AddRule(rule, 0)
switch err {
case nil:
case ErrAlreadyExists: // ignore rules which were already added
case ErrInvalidSyntax: // ignore invalid syntax
default:
b.Fatalf("Error while adding rule %s: %s", rule, err)
}
}
}
func BenchmarkAddRuleParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
rule := "||doubleclick.net^"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
var err error
for pb.Next() {
err = d.AddRule(rule, 0)
}
switch err {
case nil:
case ErrAlreadyExists: // ignore rules which were already added
case ErrInvalidSyntax: // ignore invalid syntax
default:
b.Fatalf("Error while adding rule %s: %s", rule, err)
}
})
}
func BenchmarkLotsOfRulesNoMatch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
b.Errorf("Expected hostname %s to not match", hostname)
}
}
}
func BenchmarkLotsOfRulesNoMatchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if ret.IsFiltered {
b.Errorf("Expected hostname %s to not match", hostname)
}
}
})
}
func BenchmarkLotsOfRulesMatch(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
}
func BenchmarkLotsOfRulesMatchParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
err := loadTestRules(d)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ret, err := d.CheckHost(hostname)
if err != nil {
b.Errorf("Error while matching host %s: %s", hostname, err)
}
if !ret.IsFiltered {
b.Errorf("Expected hostname %s to match", hostname)
}
}
})
}
func BenchmarkLotsOfRulesLotsOfHosts(b *testing.B) {
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
getTopHosts()
hostnames, err := os.Open(topHostsFilename)
if err != nil {
b.Fatal(err)
}
defer hostnames.Close()
scanner := bufio.NewScanner(hostnames)
b.ResetTimer()
for n := 0; n < b.N; n++ {
havedata := scanner.Scan()
if !havedata {
_, _ = hostnames.Seek(0, 0)
scanner = bufio.NewScanner(hostnames)
havedata = scanner.Scan()
}
if !havedata {
b.Fatal(scanner.Err())
}
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
b.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
}
func BenchmarkLotsOfRulesLotsOfHostsParallel(b *testing.B) {
d := NewForTest()
defer d.Destroy()
mustLoadTestRules(d)
getTopHosts()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
hostnames, err := os.Open(topHostsFilename)
if err != nil {
b.Fatal(err)
}
defer hostnames.Close()
scanner := bufio.NewScanner(hostnames)
for pb.Next() {
havedata := scanner.Scan()
if !havedata {
_, _ = hostnames.Seek(0, 0)
scanner = bufio.NewScanner(hostnames)
havedata = scanner.Scan()
}
if !havedata {
b.Fatal(scanner.Err())
}
line := scanner.Text()
records := strings.Split(line, ",")
ret, err := d.CheckHost(records[1] + "." + records[1])
if err != nil {
b.Error(err)
}
if ret.Reason.Matched() {
// log.Printf("host \"%s\" mathed. Rule \"%s\", reason: %v", host, ret.Rule, ret.Reason)
}
}
})
}
// BENCHMARKS
func BenchmarkSafeBrowsing(b *testing.B) {
d := NewForTest()
@ -1141,26 +511,3 @@ func BenchmarkSafeSearchParallel(b *testing.B) {
}
})
}
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
//
// helper functions for debugging and testing
//
func purgeCaches() {
if safebrowsingCache != nil {
safebrowsingCache.Purge()
}
if parentalCache != nil {
parentalCache.Purge()
}
}
func _Func() string {
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])
return path.Base(f.Name())
}

40
dnsfilter/helpers.go
View file

@ -1,49 +1,9 @@
package dnsfilter
import (
"strings"
"sync/atomic"
)
func isValidRule(rule string) bool {
if len(rule) < 4 {
return false
}
if rule[0] == '!' {
return false
}
if rule[0] == '#' {
return false
}
if strings.HasPrefix(rule, "[Adblock") {
return false
}
// Filter out all sorts of cosmetic rules:
// https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#cosmetic-rules
masks := []string{
"##",
"#@#",
"#?#",
"#@?#",
"#$#",
"#@$#",
"#?$#",
"#@?$#",
"$$",
"$@$",
"#%#",
"#@%#",
}
for _, mask := range masks {
if strings.Contains(rule, mask) {
return false
}
}
return true
}
func updateMax(valuePtr *int64, maxPtr *int64) {
for {
current := atomic.LoadInt64(valuePtr)

91
dnsfilter/rule_to_regexp.go
View file

@ -1,91 +0,0 @@
package dnsfilter
import (
"strings"
)
func ruleToRegexp(rule string) (string, error) {
const hostStart = `(?:^|\.)`
const hostEnd = `$`
// empty or short rule -- do nothing
if !isValidRule(rule) {
return "", ErrInvalidSyntax
}
// if starts with / and ends with /, it's already a regexp, just strip the slashes
if rule[0] == '/' && rule[len(rule)-1] == '/' {
return rule[1 : len(rule)-1], nil
}
var sb strings.Builder
if rule[0] == '|' && rule[1] == '|' {
sb.WriteString(hostStart)
rule = rule[2:]
}
for i, r := range rule {
switch {
case r == '?' || r == '.' || r == '+' || r == '[' || r == ']' || r == '(' || r == ')' || r == '{' || r == '}' || r == '#' || r == '\\' || r == '$':
sb.WriteRune('\\')
sb.WriteRune(r)
case r == '|' && i == 0:
// | at start and it's not || at start
sb.WriteRune('^')
case r == '|' && i == len(rule)-1:
// | at end
sb.WriteRune('$')
case r == '|' && i != 0 && i != len(rule)-1:
sb.WriteString(`\|`)
case r == '*':
sb.WriteString(`.*`)
case r == '^':
sb.WriteString(hostEnd)
default:
sb.WriteRune(r)
}
}
return sb.String(), nil
}
// handle suffix rule ||example.com^ -- either entire string is example.com or *.example.com
func getSuffix(rule string) (bool, string) {
// if starts with / and ends with /, it's already a regexp
// TODO: if a regexp is simple `/abracadabra$/`, then simplify it maybe?
if rule[0] == '/' && rule[len(rule)-1] == '/' {
return false, ""
}
// must start with ||
if rule[0] != '|' || rule[1] != '|' {
return false, ""
}
rule = rule[2:]
// suffix rule must end with ^ or |
lastChar := rule[len(rule)-1]
if lastChar != '^' && lastChar != '|' {
return false, ""
}
// last char was checked, eat it
rule = rule[:len(rule)-1]
// it might also end with ^|
if rule[len(rule)-1] == '^' {
rule = rule[:len(rule)-1]
}
// check that it doesn't have any special characters inside
for _, r := range rule {
switch r {
case '|':
return false, ""
case '*':
return false, ""
}
}
return true, rule
}

56
dnsforward/dnsforward.go
View file

@ -44,7 +44,7 @@ type Server struct {
once sync.Once
sync.RWMutex
ServerConfig
conf ServerConfig
}
// NewServer creates a new instance of the dnsforward.Server
@ -123,7 +123,7 @@ func (s *Server) Start(config *ServerConfig) error {
// startInternal starts without locking
func (s *Server) startInternal(config *ServerConfig) error {
if config != nil {
s.ServerConfig = *config
s.conf = *config
}
if s.dnsFilter != nil || s.dnsProxy != nil {
@ -158,21 +158,21 @@ func (s *Server) startInternal(config *ServerConfig) error {
})
proxyConfig := proxy.Config{
UDPListenAddr: s.UDPListenAddr,
TCPListenAddr: s.TCPListenAddr,
Ratelimit: s.Ratelimit,
RatelimitWhitelist: s.RatelimitWhitelist,
RefuseAny: s.RefuseAny,
UDPListenAddr: s.conf.UDPListenAddr,
TCPListenAddr: s.conf.TCPListenAddr,
Ratelimit: s.conf.Ratelimit,
RatelimitWhitelist: s.conf.RatelimitWhitelist,
RefuseAny: s.conf.RefuseAny,
CacheEnabled: true,
Upstreams: s.Upstreams,
DomainsReservedUpstreams: s.DomainsReservedUpstreams,
Upstreams: s.conf.Upstreams,
DomainsReservedUpstreams: s.conf.DomainsReservedUpstreams,
Handler: s.handleDNSRequest,
AllServers: s.AllServers,
AllServers: s.conf.AllServers,
}
if s.TLSListenAddr != nil && s.CertificateChain != "" && s.PrivateKey != "" {
proxyConfig.TLSListenAddr = s.TLSListenAddr
keypair, err := tls.X509KeyPair([]byte(s.CertificateChain), []byte(s.PrivateKey))
if s.conf.TLSListenAddr != nil && s.conf.CertificateChain != "" && s.conf.PrivateKey != "" {
proxyConfig.TLSListenAddr = s.conf.TLSListenAddr
keypair, err := tls.X509KeyPair([]byte(s.conf.CertificateChain), []byte(s.conf.PrivateKey))
if err != nil {
return errorx.Decorate(err, "Failed to parse TLS keypair")
}
@ -202,14 +202,20 @@ func (s *Server) startInternal(config *ServerConfig) error {
// Initializes the DNS filter
func (s *Server) initDNSFilter() error {
log.Tracef("Creating dnsfilter")
s.dnsFilter = dnsfilter.New(&s.Config)
// add rules only if they are enabled
if s.FilteringEnabled {
err := s.dnsFilter.AddRules(s.Filters)
if err != nil {
return errorx.Decorate(err, "could not initialize dnsfilter")
var filters map[int]string
filters = nil
if s.conf.FilteringEnabled {
filters = make(map[int]string)
for _, f := range s.conf.Filters {
filters[int(f.ID)] = string(f.Data)
}
}
s.dnsFilter = dnsfilter.New(&s.conf.Config, filters)
if s.dnsFilter == nil {
return fmt.Errorf("could not initialize dnsfilter")
}
return nil
}
@ -336,11 +342,11 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
msg := d.Req
// don't log ANY request if refuseAny is enabled
if len(msg.Question) >= 1 && msg.Question[0].Qtype == dns.TypeANY && s.RefuseAny {
if len(msg.Question) >= 1 && msg.Question[0].Qtype == dns.TypeANY && s.conf.RefuseAny {
shouldLog = false
}
if s.QueryLogEnabled && shouldLog {
if s.conf.QueryLogEnabled && shouldLog {
elapsed := time.Since(start)
upstreamAddr := ""
if d.Upstream != nil {
@ -361,7 +367,7 @@ func (s *Server) filterDNSRequest(d *proxy.DNSContext) (*dnsfilter.Result, error
host := strings.TrimSuffix(msg.Question[0].Name, ".")
s.RLock()
protectionEnabled := s.ProtectionEnabled
protectionEnabled := s.conf.ProtectionEnabled
dnsFilter := s.dnsFilter
s.RUnlock()
@ -402,7 +408,7 @@ func (s *Server) genDNSFilterMessage(d *proxy.DNSContext, result *dnsfilter.Resu
return s.genARecord(m, result.IP)
}
if s.BlockingMode == "null_ip" {
if s.conf.BlockingMode == "null_ip" {
return s.genARecord(m, net.IPv4zero)
}
@ -420,7 +426,7 @@ func (s *Server) genServerFailure(request *dns.Msg) *dns.Msg {
func (s *Server) genARecord(request *dns.Msg, ip net.IP) *dns.Msg {
resp := dns.Msg{}
resp.SetReply(request)
answer, err := dns.NewRR(fmt.Sprintf("%s %d A %s", request.Question[0].Name, s.BlockedResponseTTL, ip.String()))
answer, err := dns.NewRR(fmt.Sprintf("%s %d A %s", request.Question[0].Name, s.conf.BlockedResponseTTL, ip.String()))
if err != nil {
log.Printf("Couldn't generate A record for replacement host '%s': %s", ip.String(), err)
return s.genServerFailure(request)
@ -489,7 +495,7 @@ func (s *Server) genSOA(request *dns.Msg) []dns.RR {
Hdr: dns.RR_Header{
Name: zone,
Rrtype: dns.TypeSOA,
Ttl: s.BlockedResponseTTL,
Ttl: s.conf.BlockedResponseTTL,
Class: dns.ClassINET,
},
Mbox: "hostmaster.", // zone will be appended later if it's not empty or "."

22
dnsforward/dnsforward_test.go
View file

@ -86,7 +86,7 @@ func TestDotServer(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
s.TLSConfig = TLSConfig{
s.conf.TLSConfig = TLSConfig{
TLSListenAddr: &net.TCPAddr{Port: 0},
CertificateChain: string(certPem),
PrivateKey: string(keyPem),
@ -149,7 +149,7 @@ func TestServerRace(t *testing.T) {
func TestSafeSearch(t *testing.T) {
s := createTestServer(t)
s.SafeSearchEnabled = true
s.conf.SafeSearchEnabled = true
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
@ -295,7 +295,7 @@ func TestBlockedRequest(t *testing.T) {
func TestNullBlockedRequest(t *testing.T) {
s := createTestServer(t)
s.FilteringConfig.BlockingMode = "null_ip"
s.conf.FilteringConfig.BlockingMode = "null_ip"
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
@ -451,14 +451,14 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
func createTestServer(t *testing.T) *Server {
s := NewServer(createDataDir(t))
s.UDPListenAddr = &net.UDPAddr{Port: 0}
s.TCPListenAddr = &net.TCPAddr{Port: 0}
s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
s.QueryLogEnabled = true
s.FilteringConfig.FilteringEnabled = true
s.FilteringConfig.ProtectionEnabled = true
s.FilteringConfig.SafeBrowsingEnabled = true
s.Filters = make([]dnsfilter.Filter, 0)
s.conf.QueryLogEnabled = true
s.conf.FilteringConfig.FilteringEnabled = true
s.conf.FilteringConfig.ProtectionEnabled = true
s.conf.FilteringConfig.SafeBrowsingEnabled = true
s.conf.Filters = make([]dnsfilter.Filter, 0)
rules := []string{
"||nxdomain.example.org^",
@ -466,7 +466,7 @@ func createTestServer(t *testing.T) *Server {
"127.0.0.1 host.example.org",
}
filter := dnsfilter.Filter{ID: 1, Rules: rules}
s.Filters = append(s.Filters, filter)
s.conf.Filters = append(s.conf.Filters, filter)
return s
}

26
filter.go
View file

@ -35,13 +35,12 @@ type filter struct {
// Creates a helper object for working with the user rules
func userFilter() filter {
return filter{
f := filter{
// User filter always has constant ID=0
Enabled: true,
Filter: dnsfilter.Filter{
Rules: config.UserRules,
},
}
f.Filter.Data = []byte(strings.Join(config.UserRules, "\n"))
return f
}
// Enable or disable a filter
@ -242,7 +241,7 @@ func refreshFiltersIfNecessary(force bool) int {
log.Info("Updated filter #%d. Rules: %d -> %d",
f.ID, f.RulesCount, uf.RulesCount)
f.Name = uf.Name
f.Rules = uf.Rules
f.Data = uf.Data
f.RulesCount = uf.RulesCount
f.checksum = uf.checksum
updateCount++
@ -261,7 +260,7 @@ func refreshFiltersIfNecessary(force bool) int {
}
// A helper function that parses filter contents and returns a number of rules and a filter name (if there's any)
func parseFilterContents(contents []byte) (int, string, []string) {
func parseFilterContents(contents []byte) (int, string) {
lines := strings.Split(string(contents), "\n")
rulesCount := 0
name := ""
@ -286,7 +285,7 @@ func parseFilterContents(contents []byte) (int, string, []string) {
}
}
return rulesCount, name, lines
return rulesCount, name
}
// Perform upgrade on a filter
@ -327,13 +326,13 @@ func (filter *filter) update() (bool, error) {
}
// Extract filter name and count number of rules
rulesCount, filterName, rules := parseFilterContents(body)
rulesCount, filterName := parseFilterContents(body)
log.Printf("Filter %d has been updated: %d bytes, %d rules", filter.ID, len(body), rulesCount)
if filterName != "" {
filter.Name = filterName
}
filter.RulesCount = rulesCount
filter.Rules = rules
filter.Data = body
filter.checksum = checksum
return true, nil
@ -343,9 +342,8 @@ func (filter *filter) update() (bool, error) {
func (filter *filter) save() error {
filterFilePath := filter.Path()
log.Printf("Saving filter %d contents to: %s", filter.ID, filterFilePath)
body := []byte(strings.Join(filter.Rules, "\n"))
err := file.SafeWrite(filterFilePath, body)
err := file.SafeWrite(filterFilePath, filter.Data)
// update LastUpdated field after saving the file
filter.LastUpdated = filter.LastTimeUpdated()
@ -368,10 +366,10 @@ func (filter *filter) load() error {
}
log.Tracef("File %s, id %d, length %d", filterFilePath, filter.ID, len(filterFileContents))
rulesCount, _, rules := parseFilterContents(filterFileContents)
rulesCount, _ := parseFilterContents(filterFileContents)
filter.RulesCount = rulesCount
filter.Rules = rules
filter.Data = filterFileContents
filter.checksum = crc32.ChecksumIEEE(filterFileContents)
filter.LastUpdated = filter.LastTimeUpdated()
@ -380,7 +378,7 @@ func (filter *filter) load() error {
// Clear filter rules
func (filter *filter) unload() {
filter.Rules = []string{}
filter.Data = nil
filter.RulesCount = 0
}

10
go.mod
View file

@ -5,10 +5,9 @@ go 1.12
require (
github.com/AdguardTeam/dnsproxy v0.12.0
github.com/AdguardTeam/golibs v0.1.3
github.com/AdguardTeam/urlfilter v0.3.0
github.com/NYTimes/gziphandler v1.1.1
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-test/deep v1.0.1
github.com/gobuffalo/packr v1.19.0
github.com/joomcode/errorx v0.1.0
@ -16,13 +15,10 @@ require (
github.com/kardianos/service v0.0.0-20181115005516-4c239ee84e7b
github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
github.com/miekg/dns v1.1.1
github.com/shirou/gopsutil v2.18.10+incompatible
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0
github.com/stretchr/testify v1.3.0
go.uber.org/goleak v0.10.0
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e
golang.org/x/sys v0.0.0-20190122071731-054c452bb702
golang.org/x/net v0.0.0-20190313220215-9f648a60d977
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477
gopkg.in/yaml.v2 v2.2.1
)

31
go.sum
View file

@ -3,10 +3,12 @@ github.com/AdguardTeam/dnsproxy v0.12.0/go.mod h1:lcZM2QPwcWGEL3pz8RYy06nQdbjj4p
github.com/AdguardTeam/golibs v0.1.2/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg=
github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
github.com/AdguardTeam/urlfilter v0.3.0 h1:WNd3uZEYWwxylUuA8QS6V5DqHNsVFw3ZD/E2rd5HGpo=
github.com/AdguardTeam/urlfilter v0.3.0/go.mod h1:9xfZ6R2vB8LlT8G9LxtbNhDsbr/xybUOSwmJvpXhl/c=
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY=
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s=
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 h1:52m0LGchQBBVqJRyYYufQuIbVqRawmubW3OFGqK1ekw=
@ -15,6 +17,9 @@ github.com/ameshkov/dnscrypt v1.0.6 h1:55wfnNF8c4E3JXDNlwPl2Pbs7UPPIh+kI6KK3THqY
github.com/ameshkov/dnscrypt v1.0.6/go.mod h1:ZvT9LaNaJfDNXKIbkYFf24HUgHuQR6MNT6nwVvN4jMQ=
github.com/ameshkov/dnsstamps v1.0.1 h1:LhGvgWDzhNJh+kBQd/AfUlq1vfVe109huiXw4JhnPug=
github.com/ameshkov/dnsstamps v1.0.1/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A=
github.com/ameshkov/goproxy v0.0.0-20190328085534-e9f6fabc24d4/go.mod h1:tKA6C/1BQYejT7L6ZX0klDrqloYenYETv3BCk8xCbrQ=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 h1:KXlsf+qt/X5ttPGEjR0tPH1xaWWoKBEg9Q1THAj2h3I=
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0 h1:vUdUwmQLnT/yuk8PsDhhMVkrfr4aMdcv/0GWzIqOjEY=
@ -22,8 +27,8 @@ github.com/bluele/gcache v0.0.0-20190203144525-2016d595ccb0/go.mod h1:8c4/i2Vlov
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg=
github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4=
@ -32,6 +37,9 @@ github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264 h1:roWyi0eEdiFreSq
github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI=
github.com/gobuffalo/packr v1.19.0 h1:3UDmBDxesCOPF8iZdMDBBWKfkBoYujIMIZePnobqIUI=
github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU=
github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@ -55,8 +63,9 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs=
github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0 h1:mu7brOsdaH5Dqf93vdch+mr/0To8Sgc+yInt/jE/RJM=
@ -70,15 +79,18 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4=
go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=
golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190122013713-64072686203f h1:u1CmMhe3a44hy8VIgpInORnI01UVaUYheqR7x9BxT3c=
golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e h1:MDa3fSUp6MdYHouVmCCNz/zaH2a6CRcxY3VhT/K3C5Q=
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
@ -87,6 +99,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190122071731-054c452bb702 h1:Lk4tbZFnlyPgV+sLgTw5yGfzrlOn9kx4vSombi2FFlY=
golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+lg4zao9W4HIDzlFbMYgSgtvNVHh00MEHvbGpQ=
@ -95,3 +109,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=