From 2e845e4f4d5f35bfd3d68b4946fb1f576e7362c3 Mon Sep 17 00:00:00 2001
From: Simon Zolin <s.zolin@adguard.com>
Date: Tue, 3 Mar 2020 20:21:53 +0300
Subject: [PATCH 1/2] + qlog: hide_client_ip setting
---
AGHTechDoc.md | 11 +++++++++++
home/config.go | 8 +++++---
home/dns.go | 22 ++++++++++++----------
openapi/openapi.yaml | 3 +++
querylog/qlog.go | 33 +++++++++++++++++++++++++++++----
querylog/qlog_http.go | 9 +++++++--
querylog/querylog.go | 16 +++++++++-------
stats/stats.go | 7 ++++---
stats/stats_unit.go | 22 +++++++++++++++++++++-
9 files changed, 101 insertions(+), 30 deletions(-)
diff --git a/AGHTechDoc.md b/AGHTechDoc.md
index b1303621..8adcc068 100644
--- a/AGHTechDoc.md
+++ b/AGHTechDoc.md
@@ -1287,12 +1287,22 @@ Request:
{
"enabled": true | false
"interval": 1 | 7 | 30 | 90
+ "anonymize_client_ip": true | false // anonymize clients' IP addresses
}
Response:
200 OK
+`anonymize_client_ip`:
+1. New log entries written to a log file will contain modified client IP addresses. Note that there's no way to obtain the full IP address later for these entries.
+2. `GET /control/querylog` response data will contain modified client IP addresses (masked /24 or /112).
+3. Searching by client IP won't work for the previously stored entries.
+
+How `anonymize_client_ip` affects Stats:
+1. After AGH restart, new stats entries will contain modified client IP addresses.
+2. Existing entries are not affected.
+
### API: Get querylog parameters
@@ -1307,6 +1317,7 @@ Response:
{
"enabled": true | false
"interval": 1 | 7 | 30 | 90
+ "anonymize_client_ip": true | false
}
diff --git a/home/config.go b/home/config.go
index 78c5dfda..2d78b42c 100644
--- a/home/config.go
+++ b/home/config.go
@@ -77,9 +77,10 @@ type dnsConfig struct {
// time interval for statistics (in days)
StatsInterval uint32 `yaml:"statistics_interval"`
- QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
- QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
- QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
+ QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
+ QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
+ QueryLogMemSize uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
+ AnonymizeClientIP bool `yaml:"anonymize_client_ip"` // anonymize clients' IP addresses in logs and stats
dnsforward.FilteringConfig `yaml:",inline"`
@@ -242,6 +243,7 @@ func (c *configuration) write() error {
config.DNS.QueryLogEnabled = dc.Enabled
config.DNS.QueryLogInterval = dc.Interval
config.DNS.QueryLogMemSize = dc.MemSize
+ config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
}
if Context.dnsFilter != nil {
diff --git a/home/dns.go b/home/dns.go
index 29cec636..c9cfb513 100644
--- a/home/dns.go
+++ b/home/dns.go
@@ -29,22 +29,24 @@ func initDNSServer() error {
baseDir := Context.getDataDir()
statsConf := stats.Config{
- Filename: filepath.Join(baseDir, "stats.db"),
- LimitDays: config.DNS.StatsInterval,
- ConfigModified: onConfigModified,
- HTTPRegister: httpRegister,
+ Filename: filepath.Join(baseDir, "stats.db"),
+ LimitDays: config.DNS.StatsInterval,
+ AnonymizeClientIP: config.DNS.AnonymizeClientIP,
+ ConfigModified: onConfigModified,
+ HTTPRegister: httpRegister,
}
Context.stats, err = stats.New(statsConf)
if err != nil {
return fmt.Errorf("Couldn't initialize statistics module")
}
conf := querylog.Config{
- Enabled: config.DNS.QueryLogEnabled,
- BaseDir: baseDir,
- Interval: config.DNS.QueryLogInterval,
- MemSize: config.DNS.QueryLogMemSize,
- ConfigModified: onConfigModified,
- HTTPRegister: httpRegister,
+ Enabled: config.DNS.QueryLogEnabled,
+ BaseDir: baseDir,
+ Interval: config.DNS.QueryLogInterval,
+ MemSize: config.DNS.QueryLogMemSize,
+ AnonymizeClientIP: config.DNS.AnonymizeClientIP,
+ ConfigModified: onConfigModified,
+ HTTPRegister: httpRegister,
}
Context.queryLog = querylog.New(conf)
diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml
index e597786f..64fb5e5d 100644
--- a/openapi/openapi.yaml
+++ b/openapi/openapi.yaml
@@ -1585,6 +1585,9 @@ definitions:
interval:
type: "integer"
description: "Time period to keep data (1 | 7 | 30 | 90)"
+ anonymize_client_ip:
+ type: "boolean"
+ description: "Anonymize clients' IP addresses"
TlsConfig:
type: "object"
diff --git a/querylog/qlog.go b/querylog/qlog.go
index 2cf1c013..9dab0acb 100644
--- a/querylog/qlog.go
+++ b/querylog/qlog.go
@@ -2,6 +2,7 @@ package querylog
import (
"fmt"
+ "net"
"os"
"path/filepath"
"strconv"
@@ -66,6 +67,7 @@ func (l *queryLog) WriteDiskConfig(dc *DiskConfig) {
dc.Enabled = l.conf.Enabled
dc.Interval = l.conf.Interval
dc.MemSize = l.conf.MemSize
+ dc.AnonymizeClientIP = l.conf.AnonymizeClientIP
}
// Clear memory buffer and remove log files
@@ -123,7 +125,7 @@ func (l *queryLog) Add(params AddParams) {
now := time.Now()
entry := logEntry{
- IP: params.ClientIP.String(),
+ IP: l.getClientIP(params.ClientIP.String()),
Time: now,
Result: *params.Result,
@@ -196,6 +198,10 @@ const (
func (l *queryLog) getData(params getDataParams) map[string]interface{} {
now := time.Now()
+ if len(params.Client) != 0 && l.conf.AnonymizeClientIP {
+ params.Client = l.getClientIP(params.Client)
+ }
+
// add from file
fileEntries, oldest, total := l.searchFiles(params)
@@ -246,7 +252,7 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} {
// the elements order is already reversed (from newer to older)
for i := 0; i < len(entries); i++ {
entry := entries[i]
- jsonEntry := logEntryToJSONEntry(entry)
+ jsonEntry := l.logEntryToJSONEntry(entry)
data = append(data, jsonEntry)
}
@@ -262,7 +268,26 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} {
return result
}
-func logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
+// Get Client IP address
+func (l *queryLog) getClientIP(clientIP string) string {
+ if l.conf.AnonymizeClientIP {
+ ip := net.ParseIP(clientIP)
+ if ip != nil {
+ ip4 := ip.To4()
+ const AnonymizeClientIP4Mask = 24
+ const AnonymizeClientIP6Mask = 112
+ if ip4 != nil {
+ clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String()
+ } else {
+ clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String()
+ }
+ }
+ }
+
+ return clientIP
+}
+
+func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
var msg *dns.Msg
if len(entry.Answer) > 0 {
@@ -277,7 +302,7 @@ func logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
- "client": entry.IP,
+ "client": l.getClientIP(entry.IP),
}
jsonEntry["question"] = map[string]interface{}{
"host": entry.QHost,
diff --git a/querylog/qlog_http.go b/querylog/qlog_http.go
index 07a3aadd..fae8dba6 100644
--- a/querylog/qlog_http.go
+++ b/querylog/qlog_http.go
@@ -106,8 +106,9 @@ func (l *queryLog) handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
}
type qlogConfig struct {
- Enabled bool `json:"enabled"`
- Interval uint32 `json:"interval"`
+ Enabled bool `json:"enabled"`
+ Interval uint32 `json:"interval"`
+ AnonymizeClientIP bool `json:"anonymize_client_ip"`
}
// Get configuration
@@ -115,6 +116,7 @@ func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = l.conf.Enabled
resp.Interval = l.conf.Interval
+ resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
jsonVal, err := json.Marshal(resp)
if err != nil {
@@ -151,6 +153,9 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
if req.Exists("interval") {
conf.Interval = d.Interval
}
+ if req.Exists("anonymize_client_ip") {
+ conf.AnonymizeClientIP = d.AnonymizeClientIP
+ }
l.conf = &conf
l.lock.Unlock()
diff --git a/querylog/querylog.go b/querylog/querylog.go
index dcca14dd..0e079ec3 100644
--- a/querylog/querylog.go
+++ b/querylog/querylog.go
@@ -11,9 +11,10 @@ import (
// DiskConfig - configuration settings that are stored on disk
type DiskConfig struct {
- Enabled bool
- Interval uint32
- MemSize uint32
+ Enabled bool
+ Interval uint32
+ MemSize uint32
+ AnonymizeClientIP bool
}
// QueryLog - main interface
@@ -32,10 +33,11 @@ type QueryLog interface {
// Config - configuration object
type Config struct {
- Enabled bool
- BaseDir string // directory where log file is stored
- Interval uint32 // interval to rotate logs (in days)
- MemSize uint32 // number of entries kept in memory before they are flushed to disk
+ Enabled bool
+ BaseDir string // directory where log file is stored
+ Interval uint32 // interval to rotate logs (in days)
+ MemSize uint32 // number of entries kept in memory before they are flushed to disk
+ AnonymizeClientIP bool // anonymize clients' IP addresses
// Called when the configuration is changed by HTTP request
ConfigModified func()
diff --git a/stats/stats.go b/stats/stats.go
index 91b6b25f..8f77425f 100644
--- a/stats/stats.go
+++ b/stats/stats.go
@@ -16,9 +16,10 @@ type DiskConfig struct {
// Config - module configuration
type Config struct {
- Filename string // database file name
- LimitDays uint32 // time limit (in days)
- UnitID unitIDCallback // user function to get the current unit ID. If nil, the current time hour is used.
+ Filename string // database file name
+ LimitDays uint32 // time limit (in days)
+ UnitID unitIDCallback // user function to get the current unit ID. If nil, the current time hour is used.
+ AnonymizeClientIP bool // anonymize clients' IP addresses
// Called when the configuration is changed by HTTP request
ConfigModified func()
diff --git a/stats/stats_unit.go b/stats/stats_unit.go
index 44aa66d5..5126e69c 100644
--- a/stats/stats_unit.go
+++ b/stats/stats_unit.go
@@ -5,6 +5,7 @@ import (
"encoding/binary"
"encoding/gob"
"fmt"
+ "net"
"os"
"sort"
"sync"
@@ -442,6 +443,25 @@ func (s *statsCtx) clear() {
log.Debug("Stats: cleared")
}
+// Get Client IP address
+func (s *statsCtx) getClientIP(clientIP string) string {
+ if s.conf.AnonymizeClientIP {
+ ip := net.ParseIP(clientIP)
+ if ip != nil {
+ ip4 := ip.To4()
+ const AnonymizeClientIP4Mask = 24
+ const AnonymizeClientIP6Mask = 112
+ if ip4 != nil {
+ clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String()
+ } else {
+ clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String()
+ }
+ }
+ }
+
+ return clientIP
+}
+
func (s *statsCtx) Update(e Entry) {
if e.Result == 0 ||
e.Result >= rLast ||
@@ -449,7 +469,7 @@ func (s *statsCtx) Update(e Entry) {
!(len(e.Client) == 4 || len(e.Client) == 16) {
return
}
- client := e.Client.String()
+ client := s.getClientIP(e.Client.String())
s.unitLock.Lock()
u := s.unit
From a0be7f5566a7d5986eb04ed7ffb79c5230171e2a Mon Sep 17 00:00:00 2001
From: Ildar Kamalov <i.kamalov@adguard.com>
Date: Wed, 11 Mar 2020 16:54:05 +0300
Subject: [PATCH 2/2] + client: handle hide_client_ip
---
client/src/__locales/en.json | 2 ++
client/src/components/Settings/LogsConfig/Form.js | 10 ++++++++++
client/src/components/Settings/LogsConfig/index.js | 4 +++-
client/src/components/Settings/index.js | 1 +
client/src/reducers/queryLogs.js | 1 +
5 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json
index a4f2293e..27d3dd7f 100644
--- a/client/src/__locales/en.json
+++ b/client/src/__locales/en.json
@@ -199,6 +199,8 @@
"query_log_disabled": "The query log is disabled and can be configured in the <0>settings</0>",
"query_log_strict_search": "Use double quotes for strict search",
"query_log_retention_confirm": "Are you sure you want to change query log retention? If you decrease the interval value, some data will be lost",
+ "anonymize_client_ip": "Anonymize client IP",
+ "anonymize_client_ip_desc": "Don't save the full IP address of the client in logs and statistics",
"dns_config": "DNS server configuration",
"blocking_mode": "Blocking mode",
"default": "Default",
diff --git a/client/src/components/Settings/LogsConfig/Form.js b/client/src/components/Settings/LogsConfig/Form.js
index 3daf2b8d..a05c4f10 100644
--- a/client/src/components/Settings/LogsConfig/Form.js
+++ b/client/src/components/Settings/LogsConfig/Form.js
@@ -42,6 +42,16 @@ const Form = (props) => {
disabled={processing}
/>
</div>
+ <div className="form__group form__group--settings">
+ <Field
+ name="anonymize_client_ip"
+ type="checkbox"
+ component={renderSelectField}
+ placeholder={t('anonymize_client_ip')}
+ subtitle={t('anonymize_client_ip_desc')}
+ disabled={processing}
+ />
+ </div>
<label className="form__label">
<Trans>query_log_retention</Trans>
</label>
diff --git a/client/src/components/Settings/LogsConfig/index.js b/client/src/components/Settings/LogsConfig/index.js
index e45b785c..a4af6d36 100644
--- a/client/src/components/Settings/LogsConfig/index.js
+++ b/client/src/components/Settings/LogsConfig/index.js
@@ -30,7 +30,7 @@ class LogsConfig extends Component {
render() {
const {
- t, enabled, interval, processing, processingClear,
+ t, enabled, interval, processing, processingClear, anonymize_client_ip,
} = this.props;
return (
@@ -44,6 +44,7 @@ class LogsConfig extends Component {
initialValues={{
enabled,
interval,
+ anonymize_client_ip,
}}
onSubmit={this.handleFormSubmit}
processing={processing}
@@ -59,6 +60,7 @@ class LogsConfig extends Component {
LogsConfig.propTypes = {
interval: PropTypes.number.isRequired,
enabled: PropTypes.bool.isRequired,
+ anonymize_client_ip: PropTypes.bool.isRequired,
processing: PropTypes.bool.isRequired,
processingClear: PropTypes.bool.isRequired,
setLogsConfig: PropTypes.func.isRequired,
diff --git a/client/src/components/Settings/index.js b/client/src/components/Settings/index.js
index 0603f2cd..33901605 100644
--- a/client/src/components/Settings/index.js
+++ b/client/src/components/Settings/index.js
@@ -106,6 +106,7 @@ class Settings extends Component {
<LogsConfig
enabled={queryLogs.enabled}
interval={queryLogs.interval}
+ anonymize_client_ip={queryLogs.anonymize_client_ip}
processing={queryLogs.processingSetConfig}
processingClear={queryLogs.processingClear}
setLogsConfig={setLogsConfig}
diff --git a/client/src/reducers/queryLogs.js b/client/src/reducers/queryLogs.js
index 52f98fd0..de384461 100644
--- a/client/src/reducers/queryLogs.js
+++ b/client/src/reducers/queryLogs.js
@@ -134,6 +134,7 @@ const queryLogs = handleActions(
oldest: '',
filter: DEFAULT_LOGS_FILTER,
isFiltered: false,
+ anonymize_client_ip: false,
},
);