mirrors/AdGuardHome
1
0
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-12-18 13:01:51 +03:00
Code Issues Projects Releases Packages Wiki Activity

Pull request 1751: imp-querylog

Browse source 
Merge in DNS/adguard-home from imp-querylog to master

Squashed commit of the following:

commit 40b88f9dac46576399cd4d1fad52ecffd8f5945d
Merge: fcfe40b3 bb80a7c2
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 27 17:14:34 2023 +0300

    Merge branch 'master' into imp-querylog

commit fcfe40b33143f82fe4ef03fd883c3159dfb06026
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 27 17:13:45 2023 +0300

    querylog: imp docs, names

commit 21722c6d853465c97e6f18693095a0bc86308ea3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Feb 22 20:28:49 2023 +0300

    querylog: fix race; refactor
This commit is contained in:
Ainar Garipov 2023-02-27 17:18:56 +03:00
parent bb80a7c215
commit a772212d05
7 changed files with 107 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
internal/querylog/decode.go
View file

@ -247,55 +247,20 @@ var resultHandlers = map[string]logEntryHandler{
} }
func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) { func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
var vToken json.Token
switch key { switch key {
case "FilterListID": case "FilterListID":
vToken, err := dec.Token() ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return
}
if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}
if n, ok := vToken.(json.Number); ok { if n, ok := vToken.(json.Number); ok {
ent.Result.Rules[i].FilterListID, _ = n.Int64() ent.Result.Rules[i].FilterListID, _ = n.Int64()
} }
case "IP": case "IP":
vToken, err := dec.Token() ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return
}
if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}
if ipStr, ok := vToken.(string); ok { if ipStr, ok := vToken.(string); ok {
ent.Result.Rules[i].IP = net.ParseIP(ipStr) ent.Result.Rules[i].IP = net.ParseIP(ipStr)
} }
case "Text": case "Text":
vToken, err := dec.Token() ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return
}
if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}
if s, ok := vToken.(string); ok { if s, ok := vToken.(string); ok {
ent.Result.Rules[i].Text = s ent.Result.Rules[i].Text = s
} }
@ -304,6 +269,30 @@ func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
} }
} }
func decodeVTokenAndAddRule(
key string,
i int,
dec *json.Decoder,
rules []*filtering.ResultRule,
) (newRules []*filtering.ResultRule, vToken json.Token) {
newRules = rules
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return newRules, nil
}
if len(rules) < i+1 {
newRules = append(newRules, &filtering.ResultRule{})
}
return newRules, vToken
}
func decodeResultRules(dec *json.Decoder, ent *logEntry) { func decodeResultRules(dec *json.Decoder, ent *logEntry) {
for { for {
delimToken, err := dec.Token() delimToken, err := dec.Token()

3
internal/querylog/http.go
View file

@ -54,10 +54,7 @@ func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
return return
} }
// search for the log entries
entries, oldest := l.search(params) entries, oldest := l.search(params)
// convert log entries to JSON
data := l.entriesToJSON(entries, oldest) data := l.entriesToJSON(entries, oldest)
_ = aghhttp.WriteJSONResponse(w, r, data) _ = aghhttp.WriteJSONResponse(w, r, data)

58
internal/querylog/json.go
View file

@ -1,7 +1,6 @@
package querylog package querylog
import ( import (
"fmt"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@ -117,7 +116,7 @@ func (l *queryLog) setMsgData(entry *logEntry, jsonEntry jobject) {
// it from there as well. // it from there as well.
jsonEntry["answer_dnssec"] = entry.AuthenticatedData || msg.AuthenticatedData jsonEntry["answer_dnssec"] = entry.AuthenticatedData || msg.AuthenticatedData
if a := answerToMap(msg); a != nil { if a := answerToJSON(msg); a != nil {
jsonEntry["answer"] = a jsonEntry["answer"] = a
} }
} }
@ -136,7 +135,7 @@ func (l *queryLog) setOrigAns(entry *logEntry, jsonEntry jobject) {
return return
} }
if a := answerToMap(orig); a != nil { if a := answerToJSON(orig); a != nil {
jsonEntry["original_answer"] = a jsonEntry["original_answer"] = a
} }
} }
@ -159,55 +158,24 @@ type dnsAnswer struct {
TTL uint32 `json:"ttl"` TTL uint32 `json:"ttl"`
} }
func answerToMap(a *dns.Msg) (answers []*dnsAnswer) { // answerToJSON converts the answer records of msg, if any, to their JSON form.
if a == nil || len(a.Answer) == 0 { func answerToJSON(msg *dns.Msg) (answers []*dnsAnswer) {
if msg == nil || len(msg.Answer) == 0 {
return nil return nil
} }
answers = make([]*dnsAnswer, 0, len(a.Answer)) answers = make([]*dnsAnswer, 0, len(msg.Answer))
for _, k := range a.Answer { for _, rr := range msg.Answer {
header := k.Header() header := rr.Header()
answer := &dnsAnswer{ a := &dnsAnswer{
Type: dns.TypeToString[header.Rrtype], Type: dns.TypeToString[header.Rrtype],
// Remove the header string from the answer value since it's mostly
// unnecessary in the log.
Value: strings.TrimPrefix(rr.String(), header.String()),
TTL: header.Ttl, TTL: header.Ttl,
} }
// Some special treatment for some well-known types. answers = append(answers, a)
//
// TODO(a.garipov): Consider just calling String() for everyone
// instead.
switch v := k.(type) {
case nil:
// Probably unlikely, but go on.
case *dns.A:
answer.Value = v.A.String()
case *dns.AAAA:
answer.Value = v.AAAA.String()
case *dns.MX:
answer.Value = fmt.Sprintf("%v %v", v.Preference, v.Mx)
case *dns.CNAME:
answer.Value = v.Target
case *dns.NS:
answer.Value = v.Ns
case *dns.SPF:
answer.Value = strings.Join(v.Txt, "\n")
case *dns.TXT:
answer.Value = strings.Join(v.Txt, "\n")
case *dns.PTR:
answer.Value = v.Ptr
case *dns.SOA:
answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v", v.Ns, v.Mbox, v.Serial, v.Refresh, v.Retry, v.Expire, v.Minttl)
case *dns.CAA:
answer.Value = fmt.Sprintf("%v %v \"%v\"", v.Flag, v.Tag, v.Value)
case *dns.HINFO:
answer.Value = fmt.Sprintf("\"%v\" \"%v\"", v.Cpu, v.Os)
case *dns.RRSIG:
answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v %v %v", dns.TypeToString[v.TypeCovered], v.Algorithm, v.Labels, v.OrigTtl, v.Expiration, v.Inception, v.KeyTag, v.SignerName, v.Signature)
default:
answer.Value = v.String()
}
answers = append(answers, answer)
} }
return answers return answers

10
internal/querylog/qlog.go
View file

@ -31,7 +31,8 @@ type queryLog struct {
// bufferLock protects buffer. // bufferLock protects buffer.
bufferLock sync.RWMutex bufferLock sync.RWMutex
// buffer contains recent log entries. // buffer contains recent log entries. The entries in this buffer must not
// be modified.
buffer []*logEntry buffer []*logEntry
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
@ -100,6 +101,13 @@ type logEntry struct {
AuthenticatedData bool `json:"AD,omitempty"` AuthenticatedData bool `json:"AD,omitempty"`
} }
// shallowClone returns a shallow clone of e.
func (e *logEntry) shallowClone() (clone *logEntry) {
cloneVal := *e
return &cloneVal
}
func (l *queryLog) Start() { func (l *queryLog) Start() {
if l.conf.HTTPRegister != nil { if l.conf.HTTPRegister != nil {
l.initWeb() l.initWeb()

6
internal/querylog/search.go
View file

@ -52,7 +52,9 @@ func (l *queryLog) searchMemory(params *searchParams, cache clientCache) (entrie
// Go through the buffer in the reverse order, from newer to older. // Go through the buffer in the reverse order, from newer to older.
var err error var err error
for i := len(l.buffer) - 1; i >= 0; i-- { for i := len(l.buffer) - 1; i >= 0; i-- {
e := l.buffer[i] // A shallow clone is enough, since the only thing that this loop
// modifies is the client field.
e := l.buffer[i].shallowClone()
e.client, err = l.client(e.ClientID, e.IP.String(), cache) e.client, err = l.client(e.ClientID, e.IP.String(), cache)
if err != nil { if err != nil {
@ -130,7 +132,7 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
// searchFiles looks up log records from all log files. It optionally uses the // searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than // client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all // maxFileScanEntries so callers may need to call it several times to get all
// results. oldset and total are the time of the oldest processed entry and the // results. oldest and total are the time of the oldest processed entry and the
// total number of processed entries, including discarded ones, correspondingly. // total number of processed entries, including discarded ones, correspondingly.
func (l *queryLog) searchFiles( func (l *queryLog) searchFiles(
params *searchParams, params *searchParams,

85
internal/querylog/searchcriterion.go
View file

@ -1,6 +1,7 @@
package querylog package querylog
import ( import (
"fmt"
"strings" "strings"
"github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/filtering"
@ -118,7 +119,7 @@ func (c *searchCriterion) match(entry *logEntry) bool {
case ctTerm: case ctTerm:
return c.ctDomainOrClientCase(entry) return c.ctDomainOrClientCase(entry)
case ctFilteringStatus: case ctFilteringStatus:
return c.ctFilteringStatusCase(entry.Result) return c.ctFilteringStatusCase(entry.Result.Reason, entry.Result.IsFiltered)
} }
return false return false
@ -141,54 +142,70 @@ func (c *searchCriterion) ctDomainOrClientCase(e *logEntry) bool {
return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip) return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip)
} }
func (c *searchCriterion) ctFilteringStatusCase(res filtering.Result) bool { // ctFilteringStatusCase returns true if the result matches the value.
func (c *searchCriterion) ctFilteringStatusCase(
reason filtering.Reason,
isFiltered bool,
) (matched bool) {
switch c.value { switch c.value {
case filteringStatusAll: case filteringStatusAll:
return true return true
case
case filteringStatusFiltered: filteringStatusBlocked,
return res.IsFiltered || filteringStatusBlockedParental,
res.Reason.In( filteringStatusBlockedSafebrowsing,
filtering.NotFilteredAllowList, filteringStatusBlockedService,
filtering.Rewritten, filteringStatusFiltered,
filtering.RewrittenAutoHosts, filteringStatusSafeSearch:
filtering.RewrittenRule, return isFiltered && c.isFilteredWithReason(reason)
)
case filteringStatusBlocked:
return res.IsFiltered &&
res.Reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
case filteringStatusBlockedService:
return res.IsFiltered && res.Reason == filtering.FilteredBlockedService
case filteringStatusBlockedParental:
return res.IsFiltered && res.Reason == filtering.FilteredParental
case filteringStatusBlockedSafebrowsing:
return res.IsFiltered && res.Reason == filtering.FilteredSafeBrowsing
case filteringStatusWhitelisted: case filteringStatusWhitelisted:
return res.Reason == filtering.NotFilteredAllowList return reason == filtering.NotFilteredAllowList
case filteringStatusRewritten: case filteringStatusRewritten:
return res.Reason.In( return reason.In(
filtering.Rewritten, filtering.Rewritten,
filtering.RewrittenAutoHosts, filtering.RewrittenAutoHosts,
filtering.RewrittenRule, filtering.RewrittenRule,
) )
case filteringStatusSafeSearch:
return res.IsFiltered && res.Reason == filtering.FilteredSafeSearch
case filteringStatusProcessed: case filteringStatusProcessed:
return !res.Reason.In( return !reason.In(
filtering.FilteredBlockList, filtering.FilteredBlockList,
filtering.FilteredBlockedService, filtering.FilteredBlockedService,
filtering.NotFilteredAllowList, filtering.NotFilteredAllowList,
) )
default: default:
return false return false
} }
} }
// isFilteredWithReason returns true if reason matches the criterion value.
// c.value must be one of:
//
// - filteringStatusBlocked
// - filteringStatusBlockedParental
// - filteringStatusBlockedSafebrowsing
// - filteringStatusBlockedService
// - filteringStatusFiltered
// - filteringStatusSafeSearch
func (c *searchCriterion) isFilteredWithReason(reason filtering.Reason) (matched bool) {
switch c.value {
case filteringStatusBlocked:
return reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
case filteringStatusBlockedParental:
return reason == filtering.FilteredParental
case filteringStatusBlockedSafebrowsing:
return reason == filtering.FilteredSafeBrowsing
case filteringStatusBlockedService:
return reason == filtering.FilteredBlockedService
case filteringStatusFiltered:
return reason.In(
filtering.NotFilteredAllowList,
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)
case filteringStatusSafeSearch:
return reason == filtering.FilteredSafeSearch
default:
panic(fmt.Errorf("unexpected value %q", c.value))
}
}

2
scripts/make/go-lint.sh
View file

@ -161,7 +161,7 @@ run_linter "$GO" vet ./...
run_linter govulncheck ./... run_linter govulncheck ./...
# Apply more lax standards to the code we haven't properly refactored yet. # Apply more lax standards to the code we haven't properly refactored yet.
run_linter gocyclo --over 17 ./internal/querylog/ run_linter gocyclo --over 14 ./internal/querylog/
run_linter gocyclo --over 13\ run_linter gocyclo --over 13\
./internal/dhcpd\ ./internal/dhcpd\
./internal/filtering/\ ./internal/filtering/\