From 1011b8f93d2746ed1365de87eaa9dfb65bdc3c5c Mon Sep 17 00:00:00 2001 From: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Tue, 7 Mar 2023 17:52:03 +0300 Subject: [PATCH] Pull request 1163 safesearch package vol.1 Merge in DNS/adguard-home from 1163-safesearch-1-1 to master Squashed commit of the following: commit ccc8393304441b0edbcd15598d29764fb5d5fc34 Merge: 7d3901b5 6b265c64 Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Tue Mar 7 21:36:24 2023 +0700 Merge remote-tracking branch 'origin/master' into 1163-safesearch-1-1 commit 7d3901b5753f6456192f3533c257de7ae3bef6ac Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Sat Mar 4 10:15:18 2023 +0700 all: safesearch imp code commit 94a50dc61d955f659cda123825389225d0ebfc3e Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Fri Mar 3 13:22:41 2023 +0700 safesearch: imp code commit aa6e30fed8f67ffbc336f619f299fbd789c86ada Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 21:43:28 2023 +0700 safesearch: imp code commit 30e75b6fa85edfdf66b7ace68cad4cbc61b28ea6 Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 21:38:14 2023 +0700 safesearch: imp code commit 7d95ec73392b20519eab0e869ccba0c30404ae6f Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 21:24:49 2023 +0700 safesearch: imp code commit d6c0a0fbf9ad836606fbbfe31702e84773a2709d Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 21:04:53 2023 +0700 safesearch: imp code commit fa179b767bf75ed931cd62f282220d1f7025c641 Merge: 7a43ca00 012e5beb Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 20:52:01 2023 +0700 Merge remote-tracking branch 'origin/master' into 1163-safesearch-1-1 commit 7a43ca001ccab4d53b059c0e7843f2fc5b7dcefc Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 11:59:35 2023 +0700 safesearch: embed rules commit 2b1a83d6ffacc89a3e81a8327653018ed803f15b Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 11:50:29 2023 +0700 safesearch: imp code commit 400d463d32490cde7b4f55f3bb5a68e022e1c762 Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 11:30:57 2023 +0700 safesearch: imp code commit 60cce36bdef0cf0def28c760a3d767314a03f176 Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Wed Mar 1 11:16:56 2023 +0700 all: imp code commit 4eee3237b736fd16870835458757da550bcef295 Author: Dimitry Kolyshev <dkolyshev@adguard.com> Date: Tue Feb 28 23:26:17 2023 +0700 all: safesearch package --- internal/filtering/safesearch.go | 29 ++ internal/filtering/safesearch/rules.go | 34 +++ internal/filtering/safesearch/rules/bing.txt | 1 + .../filtering/safesearch/rules/duckduckgo.txt | 3 + .../filtering/safesearch/rules/google.txt | 191 +++++++++++++ .../filtering/safesearch/rules/pixabay.txt | 1 + .../filtering/safesearch/rules/yandex.txt | 52 ++++ .../filtering/safesearch/rules/youtube.txt | 5 + internal/filtering/safesearch/safesearch.go | 269 ++++++++++++++++++ .../filtering/safesearch/safesearch_test.go | 202 +++++++++++++ 10 files changed, 787 insertions(+) create mode 100644 internal/filtering/safesearch/rules.go create mode 100644 internal/filtering/safesearch/rules/bing.txt create mode 100644 internal/filtering/safesearch/rules/duckduckgo.txt create mode 100644 internal/filtering/safesearch/rules/google.txt create mode 100644 internal/filtering/safesearch/rules/pixabay.txt create mode 100644 internal/filtering/safesearch/rules/yandex.txt create mode 100644 internal/filtering/safesearch/rules/youtube.txt create mode 100644 internal/filtering/safesearch/safesearch.go create mode 100644 internal/filtering/safesearch/safesearch_test.go diff --git a/internal/filtering/safesearch.go b/internal/filtering/safesearch.go index f7661dd6..b3eea28e 100644 --- a/internal/filtering/safesearch.go +++ b/internal/filtering/safesearch.go @@ -13,8 +13,37 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/golibs/cache" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" ) +// SafeSearch interface describes a service for search engines hosts rewrites. +type SafeSearch interface { + // SearchHost returns a replacement address for the search engine host. + SearchHost(host string, qtype uint16) (res *rules.DNSRewrite) + + // CheckHost checks host with safe search engine. + CheckHost(host string, qtype uint16) (res Result, err error) +} + +// SafeSearchConfig is a struct with safe search related settings. +type SafeSearchConfig struct { + // CustomResolver is the resolver used by safe search. + CustomResolver Resolver `yaml:"-"` + + // Enabled indicates if safe search is enabled entirely. + Enabled bool `yaml:"enabled" json:"enabled"` + + // Services flags. Each flag indicates if the corresponding service is + // enabled or disabled. + + Bing bool `yaml:"bing" json:"bing"` + DuckDuckGo bool `yaml:"duckduckgo" json:"duckduckgo"` + Google bool `yaml:"google" json:"google"` + Pixabay bool `yaml:"pixabay" json:"pixabay"` + Yandex bool `yaml:"yandex" json:"yandex"` + YouTube bool `yaml:"youtube" json:"youtube"` +} + /* expire byte[4] res Result diff --git a/internal/filtering/safesearch/rules.go b/internal/filtering/safesearch/rules.go new file mode 100644 index 00000000..75e512e8 --- /dev/null +++ b/internal/filtering/safesearch/rules.go @@ -0,0 +1,34 @@ +package safesearch + +import _ "embed" + +//go:embed rules/bing.txt +var bing string + +//go:embed rules/google.txt +var google string + +//go:embed rules/pixabay.txt +var pixabay string + +//go:embed rules/duckduckgo.txt +var duckduckgo string + +//go:embed rules/yandex.txt +var yandex string + +//go:embed rules/youtube.txt +var youtube string + +// safeSearchRules is a map with rules texts grouped by search providers. +// Source rules downloaded from: +// https://adguardteam.github.io/HostlistsRegistry/assets/engines_safe_search.txt, +// https://adguardteam.github.io/HostlistsRegistry/assets/youtube_safe_search.txt. +var safeSearchRules = map[Service]string{ + Bing: bing, + DuckDuckGo: duckduckgo, + Google: google, + Pixabay: pixabay, + Yandex: yandex, + YouTube: youtube, +} diff --git a/internal/filtering/safesearch/rules/bing.txt b/internal/filtering/safesearch/rules/bing.txt new file mode 100644 index 00000000..8c61b63c --- /dev/null +++ b/internal/filtering/safesearch/rules/bing.txt @@ -0,0 +1 @@ +|www.bing.com^$dnsrewrite=NOERROR;CNAME;strict.bing.com \ No newline at end of file diff --git a/internal/filtering/safesearch/rules/duckduckgo.txt b/internal/filtering/safesearch/rules/duckduckgo.txt new file mode 100644 index 00000000..084f1be0 --- /dev/null +++ b/internal/filtering/safesearch/rules/duckduckgo.txt @@ -0,0 +1,3 @@ +|duckduckgo.com^$dnsrewrite=NOERROR;CNAME;safe.duckduckgo.com +|start.duckduckgo.com^$dnsrewrite=NOERROR;CNAME;safe.duckduckgo.com +|www.duckduckgo.com^$dnsrewrite=NOERROR;CNAME;safe.duckduckgo.com \ No newline at end of file diff --git a/internal/filtering/safesearch/rules/google.txt b/internal/filtering/safesearch/rules/google.txt new file mode 100644 index 00000000..62f13067 --- /dev/null +++ b/internal/filtering/safesearch/rules/google.txt @@ -0,0 +1,191 @@ +|www.google.ad^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ae^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.al^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.am^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.as^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.at^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.az^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ba^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.be^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.bf^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.bg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.bi^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.bj^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.bs^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.bt^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.by^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ca^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cat^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cd^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cf^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ch^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ci^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cl^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ao^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.bw^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ck^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.cr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.id^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.il^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.in^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.jp^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ke^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.kr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ls^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ma^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.mz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.nz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.th^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.tz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ug^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.uk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.uz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.ve^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.co.vi^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.af^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ag^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ai^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ar^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.au^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.bd^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.bh^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.bn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.bo^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.br^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.bz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.co^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.cu^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.cy^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.do^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ec^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.eg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.et^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.fj^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.gh^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.gi^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.gt^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.hk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.jm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.kh^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.kw^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.lb^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ly^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.mm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.mt^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.mx^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.my^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.na^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.nf^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ng^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ni^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.np^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.om^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.pa^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.pe^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.pg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ph^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.pk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.pr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.py^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.qa^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.sa^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.sb^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.sg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.sl^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.sv^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.tj^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.tr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.tw^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.ua^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.uy^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.vc^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com.vn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.com^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cv^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.cz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.de^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.dj^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.dk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.dm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.dz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ee^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.es^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.fi^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.fm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.fr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ga^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ge^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.gg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.gl^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.gm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.gp^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.gr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.gy^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.hn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.hr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ht^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.hu^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ie^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.im^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.iq^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.is^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.it^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.je^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.jo^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.kg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ki^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.kz^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.la^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.li^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.lk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.lt^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.lu^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.lv^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.md^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.me^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.mg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.mk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ml^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.mn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ms^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.mu^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.mv^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.mw^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ne^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.nl^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.no^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.nr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.nu^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.pl^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.pn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ps^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.pt^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ro^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.rs^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ru^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.rw^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.sc^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.se^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.sh^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.si^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.sk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.sm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.sn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.so^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.sr^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.st^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.td^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.tg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.tk^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.tl^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.tm^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.tn^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.to^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.tt^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.vg^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.vu^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com +|www.google.ws^$dnsrewrite=NOERROR;CNAME;forcesafesearch.google.com \ No newline at end of file diff --git a/internal/filtering/safesearch/rules/pixabay.txt b/internal/filtering/safesearch/rules/pixabay.txt new file mode 100644 index 00000000..0ab07746 --- /dev/null +++ b/internal/filtering/safesearch/rules/pixabay.txt @@ -0,0 +1 @@ +|pixabay.com^$dnsrewrite=NOERROR;CNAME;safesearch.pixabay.com \ No newline at end of file diff --git a/internal/filtering/safesearch/rules/yandex.txt b/internal/filtering/safesearch/rules/yandex.txt new file mode 100644 index 00000000..b6f4afb7 --- /dev/null +++ b/internal/filtering/safesearch/rules/yandex.txt @@ -0,0 +1,52 @@ +|www.xn--d1acpjx3f.xn--p1ai^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.ya.ru^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.az^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.by^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.co.il^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.com.am^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.com.ge^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.com.ru^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.com.tr^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.com^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.de^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.ee^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.eu^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.fi^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.fr^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.kz^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.lt^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.lv^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.md^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.net^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.org^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.pl^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.ru^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.tj^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.tm^$dnsrewrite=NOERROR;A;213.180.193.56 +|www.yandex.uz^$dnsrewrite=NOERROR;A;213.180.193.56 +|xn--d1acpjx3f.xn--p1ai^$dnsrewrite=NOERROR;A;213.180.193.56 +|ya.ru^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.az^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.by^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.co.il^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.com.am^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.com.ge^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.com.ru^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.com.tr^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.com^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.de^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.ee^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.eu^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.fi^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.fr^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.kz^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.lt^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.lv^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.md^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.net^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.org^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.pl^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.ru^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.tj^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.tm^$dnsrewrite=NOERROR;A;213.180.193.56 +|yandex.uz^$dnsrewrite=NOERROR;A;213.180.193.56 \ No newline at end of file diff --git a/internal/filtering/safesearch/rules/youtube.txt b/internal/filtering/safesearch/rules/youtube.txt new file mode 100644 index 00000000..70e3ae46 --- /dev/null +++ b/internal/filtering/safesearch/rules/youtube.txt @@ -0,0 +1,5 @@ +|www.youtube.com^$dnsrewrite=NOERROR;CNAME;restrictmoderate.youtube.com +|m.youtube.com^$dnsrewrite=NOERROR;CNAME;restrictmoderate.youtube.com +|youtubei.googleapis.com^$dnsrewrite=NOERROR;CNAME;restrictmoderate.youtube.com +|youtube.googleapis.com^$dnsrewrite=NOERROR;CNAME;restrictmoderate.youtube.com +|www.youtube-nocookie.com^$dnsrewrite=NOERROR;CNAME;restrictmoderate.youtube.com \ No newline at end of file diff --git a/internal/filtering/safesearch/safesearch.go b/internal/filtering/safesearch/safesearch.go new file mode 100644 index 00000000..e944e217 --- /dev/null +++ b/internal/filtering/safesearch/safesearch.go @@ -0,0 +1,269 @@ +// Package safesearch implements safesearch host matching. +package safesearch + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/gob" + "fmt" + "net" + "strings" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/filtering" + "github.com/AdguardTeam/golibs/cache" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter" + "github.com/AdguardTeam/urlfilter/filterlist" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" +) + +// Service is a enum with service names used as search providers. +type Service string + +// Service enum members. +const ( + Bing Service = "bing" + DuckDuckGo Service = "duckduckgo" + Google Service = "google" + Pixabay Service = "pixabay" + Yandex Service = "yandex" + YouTube Service = "youtube" +) + +// isServiceProtected returns true if the service safe search is active. +func isServiceProtected(s filtering.SafeSearchConfig, service Service) (ok bool) { + switch service { + case Bing: + return s.Bing + case DuckDuckGo: + return s.DuckDuckGo + case Google: + return s.Google + case Pixabay: + return s.Pixabay + case Yandex: + return s.Yandex + case YouTube: + return s.YouTube + default: + panic(fmt.Errorf("safesearch: invalid sources: not found service %q", service)) + } +} + +// DefaultSafeSearch is the default safesearch struct. +type DefaultSafeSearch struct { + engine *urlfilter.DNSEngine + safeSearchCache cache.Cache + resolver filtering.Resolver + cacheTime time.Duration +} + +// NewDefaultSafeSearch returns new safesearch struct. CacheTime is an element +// TTL (in minutes). +func NewDefaultSafeSearch( + conf filtering.SafeSearchConfig, + cacheSize uint, + cacheTime time.Duration, +) (ss *DefaultSafeSearch, err error) { + engine, err := newEngine(filtering.SafeSearchListID, conf) + if err != nil { + return nil, err + } + + var resolver filtering.Resolver = net.DefaultResolver + if conf.CustomResolver != nil { + resolver = conf.CustomResolver + } + + return &DefaultSafeSearch{ + engine: engine, + safeSearchCache: cache.New(cache.Config{ + EnableLRU: true, + MaxSize: cacheSize, + }), + cacheTime: cacheTime, + resolver: resolver, + }, nil +} + +// newEngine creates new engine for provided safe search configuration. +func newEngine(listID int, conf filtering.SafeSearchConfig) (engine *urlfilter.DNSEngine, err error) { + var sb strings.Builder + for service, serviceRules := range safeSearchRules { + if isServiceProtected(conf, service) { + sb.WriteString(serviceRules) + } + } + + strList := &filterlist.StringRuleList{ + ID: listID, + RulesText: sb.String(), + IgnoreCosmetic: true, + } + + rs, err := filterlist.NewRuleStorage([]filterlist.RuleList{strList}) + if err != nil { + return nil, fmt.Errorf("creating rule storage: %w", err) + } + + engine = urlfilter.NewDNSEngine(rs) + log.Info("safesearch: filter %d: reset %d rules", listID, engine.RulesCount) + + return engine, nil +} + +// type check +var _ filtering.SafeSearch = (*DefaultSafeSearch)(nil) + +// SearchHost implements the [filtering.SafeSearch] interface for *DefaultSafeSearch. +func (ss *DefaultSafeSearch) SearchHost(host string, qtype uint16) (res *rules.DNSRewrite) { + r, _ := ss.engine.MatchRequest(&urlfilter.DNSRequest{ + Hostname: strings.ToLower(host), + DNSType: qtype, + }) + + rewritesRules := r.DNSRewrites() + if len(rewritesRules) > 0 { + return rewritesRules[0].DNSRewrite + } + + return nil +} + +// CheckHost implements the [filtering.SafeSearch] interface for +// *DefaultSafeSearch. +func (ss *DefaultSafeSearch) CheckHost( + host string, + qtype uint16, +) (res filtering.Result, err error) { + if log.GetLevel() >= log.DEBUG { + timer := log.StartTimer() + defer timer.LogElapsed("safesearch: lookup for %s", host) + } + + // Check cache. Return cached result if it was found + cachedValue, isFound := ss.getCachedResult(host) + if isFound { + log.Debug("safesearch: found in cache: %s", host) + + return cachedValue, nil + } + + rewrite := ss.SearchHost(host, qtype) + if rewrite == nil { + return filtering.Result{}, nil + } + + dRes, err := ss.newResult(rewrite, qtype) + if err != nil { + log.Debug("safesearch: failed to lookup addresses for %s: %s", host, err) + + return filtering.Result{}, err + } + + if dRes != nil { + res = *dRes + ss.setCacheResult(host, res) + + return res, nil + } + + return filtering.Result{}, fmt.Errorf("no ipv4 addresses in safe search response for %s", host) +} + +// newResult creates Result object from rewrite rule. +func (ss *DefaultSafeSearch) newResult( + rewrite *rules.DNSRewrite, + qtype uint16, +) (res *filtering.Result, err error) { + res = &filtering.Result{ + Rules: []*filtering.ResultRule{{ + FilterListID: filtering.SafeSearchListID, + }}, + Reason: filtering.FilteredSafeSearch, + IsFiltered: true, + } + + if rewrite.RRType == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) { + ip, ok := rewrite.Value.(net.IP) + if !ok || ip == nil { + return nil, nil + } + + res.Rules[0].IP = ip + + return res, nil + } + + if rewrite.NewCNAME == "" { + return nil, nil + } + + ips, err := ss.resolver.LookupIP(context.Background(), "ip", rewrite.NewCNAME) + if err != nil { + return nil, err + } + + for _, ip := range ips { + if ip = ip.To4(); ip == nil { + continue + } + + res.Rules[0].IP = ip + + return res, nil + } + + return nil, nil +} + +// setCacheResult stores data in cache for host. +func (ss *DefaultSafeSearch) setCacheResult(host string, res filtering.Result) { + expire := uint32(time.Now().Add(ss.cacheTime).Unix()) + exp := make([]byte, 4) + binary.BigEndian.PutUint32(exp, expire) + buf := bytes.NewBuffer(exp) + + err := gob.NewEncoder(buf).Encode(res) + if err != nil { + log.Error("safesearch: cache encoding: %s", err) + + return + } + + val := buf.Bytes() + _ = ss.safeSearchCache.Set([]byte(host), val) + + log.Debug("safesearch: stored in cache: %s (%d bytes)", host, len(val)) +} + +// getCachedResult returns stored data from cache for host. +func (ss *DefaultSafeSearch) getCachedResult(host string) (res filtering.Result, ok bool) { + res = filtering.Result{} + + data := ss.safeSearchCache.Get([]byte(host)) + if data == nil { + return res, false + } + + exp := binary.BigEndian.Uint32(data[:4]) + if exp <= uint32(time.Now().Unix()) { + ss.safeSearchCache.Del([]byte(host)) + + return res, false + } + + buf := bytes.NewBuffer(data[4:]) + + err := gob.NewDecoder(buf).Decode(&res) + if err != nil { + log.Debug("safesearch: cache decoding: %s", err) + + return filtering.Result{}, false + } + + return res, true +} diff --git a/internal/filtering/safesearch/safesearch_test.go b/internal/filtering/safesearch/safesearch_test.go new file mode 100644 index 00000000..97d18f95 --- /dev/null +++ b/internal/filtering/safesearch/safesearch_test.go @@ -0,0 +1,202 @@ +package safesearch + +import ( + "context" + "net" + "testing" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghtest" + "github.com/AdguardTeam/AdGuardHome/internal/filtering" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + safeSearchCacheSize = 5000 + cacheTime = 30 * time.Minute +) + +var defaultSafeSearchConf = filtering.SafeSearchConfig{ + Enabled: true, + Bing: true, + DuckDuckGo: true, + Google: true, + Pixabay: true, + Yandex: true, + YouTube: true, +} + +var yandexIP = net.IPv4(213, 180, 193, 56) + +func newForTest(t testing.TB, ssConf filtering.SafeSearchConfig) (ss *DefaultSafeSearch) { + ss, err := NewDefaultSafeSearch(ssConf, safeSearchCacheSize, cacheTime) + require.NoError(t, err) + + return ss +} + +func TestSafeSearch(t *testing.T) { + ss := newForTest(t, defaultSafeSearchConf) + val := ss.SearchHost("www.google.com", dns.TypeA) + + assert.Equal(t, &rules.DNSRewrite{NewCNAME: "forcesafesearch.google.com"}, val) +} + +func TestCheckHostSafeSearchYandex(t *testing.T) { + ss := newForTest(t, defaultSafeSearchConf) + + // Check host for each domain. + for _, host := range []string{ + "yandex.ru", + "yAndeX.ru", + "YANdex.COM", + "yandex.by", + "yandex.kz", + "www.yandex.com", + } { + res, err := ss.CheckHost(host, dns.TypeA) + require.NoError(t, err) + + assert.True(t, res.IsFiltered) + + require.Len(t, res.Rules, 1) + + assert.Equal(t, yandexIP, res.Rules[0].IP) + assert.EqualValues(t, filtering.SafeSearchListID, res.Rules[0].FilterListID) + } +} + +func TestCheckHostSafeSearchGoogle(t *testing.T) { + resolver := &aghtest.TestResolver{} + ip, _ := resolver.HostToIPs("forcesafesearch.google.com") + + ss := newForTest(t, defaultSafeSearchConf) + ss.resolver = resolver + + // Check host for each domain. + for _, host := range []string{ + "www.google.com", + "www.google.im", + "www.google.co.in", + "www.google.iq", + "www.google.is", + "www.google.it", + "www.google.je", + } { + t.Run(host, func(t *testing.T) { + res, err := ss.CheckHost(host, dns.TypeA) + require.NoError(t, err) + + assert.True(t, res.IsFiltered) + + require.Len(t, res.Rules, 1) + + assert.Equal(t, ip, res.Rules[0].IP) + assert.EqualValues(t, filtering.SafeSearchListID, res.Rules[0].FilterListID) + }) + } +} + +func TestSafeSearchCacheYandex(t *testing.T) { + const domain = "yandex.ru" + + ss := newForTest(t, filtering.SafeSearchConfig{Enabled: false}) + + // Check host with disabled safesearch. + res, err := ss.CheckHost(domain, dns.TypeA) + require.NoError(t, err) + + assert.False(t, res.IsFiltered) + assert.Empty(t, res.Rules) + + ss = newForTest(t, defaultSafeSearchConf) + res, err = ss.CheckHost(domain, dns.TypeA) + require.NoError(t, err) + + // For yandex we already know valid IP. + require.Len(t, res.Rules, 1) + + assert.Equal(t, res.Rules[0].IP, yandexIP) + + // Check cache. + cachedValue, isFound := ss.getCachedResult(domain) + require.True(t, isFound) + require.Len(t, cachedValue.Rules, 1) + + assert.Equal(t, cachedValue.Rules[0].IP, yandexIP) +} + +func TestSafeSearchCacheGoogle(t *testing.T) { + const domain = "www.google.ru" + + ss := newForTest(t, filtering.SafeSearchConfig{Enabled: false}) + + res, err := ss.CheckHost(domain, dns.TypeA) + require.NoError(t, err) + + assert.False(t, res.IsFiltered) + assert.Empty(t, res.Rules) + + resolver := &aghtest.TestResolver{} + ss = newForTest(t, defaultSafeSearchConf) + ss.resolver = resolver + + // Lookup for safesearch domain. + rewrite := ss.SearchHost(domain, dns.TypeA) + + ips, err := resolver.LookupIP(context.Background(), "ip", rewrite.NewCNAME) + require.NoError(t, err) + + var foundIP net.IP + for _, ip := range ips { + if ip.To4() != nil { + foundIP = ip + + break + } + } + + res, err = ss.CheckHost(domain, dns.TypeA) + require.NoError(t, err) + require.Len(t, res.Rules, 1) + + assert.True(t, res.Rules[0].IP.Equal(foundIP)) + + // Check cache. + cachedValue, isFound := ss.getCachedResult(domain) + require.True(t, isFound) + require.Len(t, cachedValue.Rules, 1) + + assert.True(t, cachedValue.Rules[0].IP.Equal(foundIP)) +} + +const googleHost = "www.google.com" + +var dnsRewriteSink *rules.DNSRewrite + +func BenchmarkSafeSearch(b *testing.B) { + ss := newForTest(b, defaultSafeSearchConf) + + for n := 0; n < b.N; n++ { + dnsRewriteSink = ss.SearchHost(googleHost, dns.TypeA) + } + + assert.Equal(b, "forcesafesearch.google.com", dnsRewriteSink.NewCNAME) +} + +var dnsRewriteParallelSink *rules.DNSRewrite + +func BenchmarkSafeSearch_parallel(b *testing.B) { + ss := newForTest(b, defaultSafeSearchConf) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + dnsRewriteParallelSink = ss.SearchHost(googleHost, dns.TypeA) + } + }) + + assert.Equal(b, "forcesafesearch.google.com", dnsRewriteParallelSink.NewCNAME) +}