AdGuardHome/internal/querylog/qlogfile_test.go

349 lines
7.6 KiB
Go
Raw Normal View History

package querylog
import (
2020-02-20 14:11:33 +03:00
"encoding/binary"
"fmt"
"io"
"math"
2020-02-20 14:11:33 +03:00
"net"
"os"
"strings"
"testing"
2020-02-20 14:11:33 +03:00
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// prepareTestFile prepares one test query log file with the specified lines
// count.
func prepareTestFile(t *testing.T, dir string, linesNum int) (name string) {
t.Helper()
f, err := os.CreateTemp(dir, "*.txt")
require.NoError(t, err)
// Use defer and not t.Cleanup to make sure that the file is closed
// after this function is done.
defer func() {
derr := f.Close()
require.NoError(t, derr)
}()
const ans = `"AAAAAAABAAEAAAAAB2V4YW1wbGUDb3JnAAABAAEHZXhhbXBsZQNvcmcAAAEAAQAAAAAABAECAwQ="`
const format = `{"IP":%q,"T":%q,"QH":"example.org","QT":"A","QC":"IN",` +
`"Answer":` + ans + `,"Result":{},"Elapsed":0,"Upstream":"upstream"}` + "\n"
var lineIP uint32
lineTime := time.Date(2020, 2, 18, 19, 36, 35, 920973000, time.UTC)
2024-05-15 13:34:12 +03:00
for range linesNum {
lineIP++
lineTime = lineTime.Add(time.Second)
ip := make(net.IP, 4)
binary.BigEndian.PutUint32(ip, lineIP)
line := fmt.Sprintf(format, ip, lineTime.Format(time.RFC3339Nano))
2020-02-20 14:11:33 +03:00
_, err = f.WriteString(line)
require.NoError(t, err)
}
return f.Name()
}
// prepareTestFiles prepares several test query log files, each with the
// specified lines count.
func prepareTestFiles(t *testing.T, filesNum, linesNum int) []string {
t.Helper()
if filesNum == 0 {
return []string{}
}
dir := t.TempDir()
files := make([]string, filesNum)
for i := range files {
files[filesNum-i-1] = prepareTestFile(t, dir, linesNum)
}
return files
}
2023-06-07 20:04:01 +03:00
// newTestQLogFile creates new *qLogFile for tests and registers the required
// cleanup functions.
2023-06-07 20:04:01 +03:00
func newTestQLogFile(t *testing.T, linesNum int) (file *qLogFile) {
t.Helper()
testFile := prepareTestFiles(t, 1, linesNum)[0]
2023-06-07 20:04:01 +03:00
// Create the new qLogFile instance.
file, err := newQLogFile(testFile)
require.NoError(t, err)
assert.NotNil(t, file)
testutil.CleanupAndRequireSuccess(t, file.Close)
return file
}
func TestQLogFile_ReadNext(t *testing.T) {
testCases := []struct {
name string
linesNum int
}{{
name: "empty",
linesNum: 0,
}, {
name: "large",
linesNum: 50000,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
q := newTestQLogFile(t, tc.linesNum)
// Calculate the expected position.
fileInfo, err := q.file.Stat()
require.NoError(t, err)
var expPos int64
if expPos = fileInfo.Size(); expPos > 0 {
expPos--
}
// Seek to the start.
pos, err := q.SeekStart()
require.NoError(t, err)
require.EqualValues(t, expPos, pos)
var read int
var line string
for err == nil {
line, err = q.ReadNext()
if err == nil {
assert.NotEmpty(t, line)
read++
}
}
require.Equal(t, io.EOF, err)
assert.Equal(t, tc.linesNum, read)
})
}
}
func TestQLogFile_SeekTS_good(t *testing.T) {
linesCases := []struct {
name string
num int
}{{
name: "large",
num: 10000,
}, {
name: "small",
num: 10,
}}
for _, l := range linesCases {
testCases := []struct {
name string
linesNum int
line int
}{{
name: "not_too_old",
line: 2,
}, {
name: "old",
line: l.num - 2,
}, {
name: "first",
line: 0,
}, {
name: "last",
line: l.num,
}}
q := newTestQLogFile(t, l.num)
for _, tc := range testCases {
t.Run(l.name+"_"+tc.name, func(t *testing.T) {
line, err := getQLogFileLine(q, tc.line)
require.NoError(t, err)
ts := readQLogTimestamp(line)
assert.NotEqualValues(t, 0, ts)
// Try seeking to that line now.
pos, _, err := q.seekTS(ts)
require.NoError(t, err)
assert.NotEqualValues(t, 0, pos)
testLine, err := q.ReadNext()
require.NoError(t, err)
assert.Equal(t, line, testLine)
})
}
}
}
func TestQLogFile_SeekTS_bad(t *testing.T) {
linesCases := []struct {
name string
num int
}{{
name: "large",
num: 10000,
}, {
name: "small",
num: 10,
}}
for _, l := range linesCases {
testCases := []struct {
name string
ts int64
leq bool
}{{
name: "non-existent_long_ago",
}, {
name: "non-existent_far_ahead",
}, {
name: "almost",
leq: true,
}}
q := newTestQLogFile(t, l.num)
testCases[0].ts = 123
lateTS, _ := time.Parse(time.RFC3339, "2100-01-02T15:04:05Z07:00")
testCases[1].ts = lateTS.UnixNano()
line, err := getQLogFileLine(q, l.num/2)
require.NoError(t, err)
testCases[2].ts = readQLogTimestamp(line) - 1
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.NotEqualValues(t, 0, tc.ts)
var depth int
_, depth, err = q.seekTS(tc.ts)
assert.NotEmpty(t, l.num)
require.Error(t, err)
if tc.leq {
assert.LessOrEqual(t, depth, int(math.Log2(float64(l.num))+3))
}
})
}
}
}
2023-06-07 20:04:01 +03:00
func getQLogFileLine(q *qLogFile, lineNumber int) (line string, err error) {
if _, err = q.SeekStart(); err != nil {
return line, err
}
for i := 1; i < lineNumber; i++ {
if _, err = q.ReadNext(); err != nil {
return line, err
}
}
return q.ReadNext()
2020-02-20 14:11:33 +03:00
}
// Check adding and loading (with filtering) entries from disk and memory.
2020-02-20 14:11:33 +03:00
func TestQLogFile(t *testing.T) {
2023-06-07 20:04:01 +03:00
// Create the new qLogFile instance.
q := newTestQLogFile(t, 2)
// Seek to the start.
pos, err := q.SeekStart()
require.NoError(t, err)
Pull request: 2508 ip conversion vol.1 Merge in DNS/adguard-home from 2508-ip-conversion to master Updates #2508. Squashed commit of the following: commit 3f64709fbc73ef74c11b910997be1e9bc337193c Merge: 5ac7faaaa 0d67aa251 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 16:21:34 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit 5ac7faaaa9dda570fdb872acad5d13d078f46b64 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 12:00:11 2021 +0300 all: replace conditions with appropriate functions in tests commit 9e3fa9a115ed23024c57dd5192d5173477ddbf71 Merge: db992a42a bba74859e Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 10:47:10 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit db992a42a2c6f315421e78a6a0492e2bfb3ce89d Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:55:53 2021 +0300 sysutil: fix linux tests commit f629b15d62349323ce2da05e68dc9cc0b5f6e194 Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:41:20 2021 +0300 all: improve code quality commit 3bf03a75524040738562298bd1de6db536af130f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 17:33:26 2021 +0300 sysutil: fix linux net.IP conversion commit 5d5b6994916923636e635588631b63b7e7b74e5f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 14:57:26 2021 +0300 dnsforward: remove redundant net.IP <-> string conversion commit 0b955d99b7fad40942f21d1dd8734adb99126195 Author: Eugene Burkov <e.burkov@adguard.com> Date: Mon Jan 11 18:04:25 2021 +0300 dhcpd: remove net.IP <-> string conversion
2021-01-13 16:56:05 +03:00
assert.Greater(t, pos, int64(0))
// Read first line.
line, err := q.ReadNext()
require.NoError(t, err)
Pull request: 2508 ip conversion vol.1 Merge in DNS/adguard-home from 2508-ip-conversion to master Updates #2508. Squashed commit of the following: commit 3f64709fbc73ef74c11b910997be1e9bc337193c Merge: 5ac7faaaa 0d67aa251 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 16:21:34 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit 5ac7faaaa9dda570fdb872acad5d13d078f46b64 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 12:00:11 2021 +0300 all: replace conditions with appropriate functions in tests commit 9e3fa9a115ed23024c57dd5192d5173477ddbf71 Merge: db992a42a bba74859e Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 10:47:10 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit db992a42a2c6f315421e78a6a0492e2bfb3ce89d Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:55:53 2021 +0300 sysutil: fix linux tests commit f629b15d62349323ce2da05e68dc9cc0b5f6e194 Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:41:20 2021 +0300 all: improve code quality commit 3bf03a75524040738562298bd1de6db536af130f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 17:33:26 2021 +0300 sysutil: fix linux net.IP conversion commit 5d5b6994916923636e635588631b63b7e7b74e5f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 14:57:26 2021 +0300 dnsforward: remove redundant net.IP <-> string conversion commit 0b955d99b7fad40942f21d1dd8734adb99126195 Author: Eugene Burkov <e.burkov@adguard.com> Date: Mon Jan 11 18:04:25 2021 +0300 dhcpd: remove net.IP <-> string conversion
2021-01-13 16:56:05 +03:00
assert.Contains(t, line, "0.0.0.2")
assert.True(t, strings.HasPrefix(line, "{"), line)
assert.True(t, strings.HasSuffix(line, "}"), line)
// Read second line.
line, err = q.ReadNext()
require.NoError(t, err)
Pull request: 2508 ip conversion vol.1 Merge in DNS/adguard-home from 2508-ip-conversion to master Updates #2508. Squashed commit of the following: commit 3f64709fbc73ef74c11b910997be1e9bc337193c Merge: 5ac7faaaa 0d67aa251 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 16:21:34 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit 5ac7faaaa9dda570fdb872acad5d13d078f46b64 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 12:00:11 2021 +0300 all: replace conditions with appropriate functions in tests commit 9e3fa9a115ed23024c57dd5192d5173477ddbf71 Merge: db992a42a bba74859e Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 10:47:10 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit db992a42a2c6f315421e78a6a0492e2bfb3ce89d Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:55:53 2021 +0300 sysutil: fix linux tests commit f629b15d62349323ce2da05e68dc9cc0b5f6e194 Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:41:20 2021 +0300 all: improve code quality commit 3bf03a75524040738562298bd1de6db536af130f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 17:33:26 2021 +0300 sysutil: fix linux net.IP conversion commit 5d5b6994916923636e635588631b63b7e7b74e5f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 14:57:26 2021 +0300 dnsforward: remove redundant net.IP <-> string conversion commit 0b955d99b7fad40942f21d1dd8734adb99126195 Author: Eugene Burkov <e.burkov@adguard.com> Date: Mon Jan 11 18:04:25 2021 +0300 dhcpd: remove net.IP <-> string conversion
2021-01-13 16:56:05 +03:00
assert.EqualValues(t, 0, q.position)
assert.Contains(t, line, "0.0.0.1")
assert.True(t, strings.HasPrefix(line, "{"), line)
assert.True(t, strings.HasSuffix(line, "}"), line)
// Try reading again (there's nothing to read anymore).
line, err = q.ReadNext()
require.Equal(t, io.EOF, err)
Pull request: 2508 ip conversion vol.1 Merge in DNS/adguard-home from 2508-ip-conversion to master Updates #2508. Squashed commit of the following: commit 3f64709fbc73ef74c11b910997be1e9bc337193c Merge: 5ac7faaaa 0d67aa251 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 16:21:34 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit 5ac7faaaa9dda570fdb872acad5d13d078f46b64 Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 12:00:11 2021 +0300 all: replace conditions with appropriate functions in tests commit 9e3fa9a115ed23024c57dd5192d5173477ddbf71 Merge: db992a42a bba74859e Author: Eugene Burkov <e.burkov@adguard.com> Date: Wed Jan 13 10:47:10 2021 +0300 Merge branch 'master' into 2508-ip-conversion commit db992a42a2c6f315421e78a6a0492e2bfb3ce89d Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:55:53 2021 +0300 sysutil: fix linux tests commit f629b15d62349323ce2da05e68dc9cc0b5f6e194 Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 18:41:20 2021 +0300 all: improve code quality commit 3bf03a75524040738562298bd1de6db536af130f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 17:33:26 2021 +0300 sysutil: fix linux net.IP conversion commit 5d5b6994916923636e635588631b63b7e7b74e5f Author: Eugene Burkov <e.burkov@adguard.com> Date: Tue Jan 12 14:57:26 2021 +0300 dnsforward: remove redundant net.IP <-> string conversion commit 0b955d99b7fad40942f21d1dd8734adb99126195 Author: Eugene Burkov <e.burkov@adguard.com> Date: Mon Jan 11 18:04:25 2021 +0300 dhcpd: remove net.IP <-> string conversion
2021-01-13 16:56:05 +03:00
assert.Empty(t, line)
}
2020-02-20 14:11:33 +03:00
2023-06-07 20:04:01 +03:00
func newTestQLogFileData(t *testing.T, data string) (file *qLogFile) {
f, err := os.CreateTemp(t.TempDir(), "*.txt")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, f.Close)
_, err = f.WriteString(data)
require.NoError(t, err)
2023-06-07 20:04:01 +03:00
file, err = newQLogFile(f.Name())
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, file.Close)
return file
}
func TestQLog_Seek(t *testing.T) {
const nl = "\n"
const strV = "%s"
const recs = `{"T":"` + strV + `","QH":"wfqvjymurpwegyv","QT":"A","QC":"IN","CP":"","Answer":"","Result":{},"Elapsed":66286385,"Upstream":"tls://unfiltered.adguard-dns.com:853"}` + nl +
`{"T":"` + strV + `"}` + nl +
`{"T":"` + strV + `"}` + nl
timestamp, _ := time.Parse(time.RFC3339Nano, "2020-08-31T18:44:25.376690873+03:00")
testCases := []struct {
2023-06-07 20:04:01 +03:00
wantErr error
name string
delta int
wantDepth int
}{{
name: "ok",
delta: 0,
wantErr: nil,
wantDepth: 2,
}, {
name: "too_late",
delta: 2,
2023-06-07 20:04:01 +03:00
wantErr: errTSTooLate,
wantDepth: 2,
}, {
name: "too_early",
delta: -2,
2023-06-07 20:04:01 +03:00
wantErr: errTSTooEarly,
wantDepth: 1,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
data := fmt.Sprintf(recs,
timestamp.Add(-time.Second).Format(time.RFC3339Nano),
timestamp.Format(time.RFC3339Nano),
timestamp.Add(time.Second).Format(time.RFC3339Nano),
)
2023-06-07 20:04:01 +03:00
q := newTestQLogFileData(t, data)
_, depth, err := q.seekTS(timestamp.Add(time.Second * time.Duration(tc.delta)).UnixNano())
require.Truef(t, errors.Is(err, tc.wantErr), "%v", err)
assert.Equal(t, tc.wantDepth, depth)
})
}
}