AdGuardHome/scripts/translations/main.go
Ainar Garipov ac5a96fada Pull request 2302: upd-all
Squashed commit of the following:

commit f920006277f39b74c803139af2a9039aa45effae
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Nov 8 16:14:41 2024 +0300

    all: fix pre-commit; upd dnsproxy

commit 391f79b244
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Nov 7 18:53:28 2024 +0300

    scripts: imp install

commit 35324db80b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Nov 7 18:20:23 2024 +0300

    all: imp docs, scripts

commit d2724cfaef
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Nov 7 17:26:23 2024 +0300

    all: upd go, deps, tools, scripts
2024-11-08 17:18:16 +03:00

505 lines
11 KiB
Go

// translations downloads translations, uploads translations, prints summary
// for translations, prints unused strings.
package main
import (
"bufio"
"bytes"
"cmp"
"context"
"encoding/json"
"fmt"
"log/slog"
"maps"
"net/url"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/logutil/slogutil"
)
const (
twoskyConfFile = "./.twosky.json"
localesDir = "./client/src/__locales"
defaultBaseFile = "en.json"
defaultProjectID = "home"
srcDir = "./client/src"
twoskyURI = "https://twosky.int.agrd.dev/api/v1"
readLimit = 1 * 1024 * 1024
uploadTimeout = 20 * time.Second
)
// blockerLangCodes is the codes of languages which need to be fully translated.
var blockerLangCodes = []langCode{
"de",
"en",
"es",
"fr",
"it",
"ja",
"ko",
"pt-br",
"pt-pt",
"ru",
"zh-cn",
"zh-tw",
}
// langCode is a language code.
type langCode string
// languages is a map, where key is language code and value is display name.
type languages map[langCode]string
// textlabel is a text label of localization.
type textLabel string
// locales is a map, where key is text label and value is translation.
type locales map[textLabel]string
func main() {
ctx := context.Background()
l := slogutil.New(nil)
if len(os.Args) == 1 {
usage("need a command")
}
if os.Args[1] == "help" {
usage("")
}
conf := errors.Must(readTwoskyConfig())
var cli *twoskyClient
switch os.Args[1] {
case "summary":
errors.Check(summary(conf.Languages))
case "download":
cli = errors.Must(conf.toClient())
errors.Check(cli.download(ctx, l))
case "unused":
err := unused(ctx, l, conf.LocalizableFiles[0])
errors.Check(err)
case "upload":
cli = errors.Must(conf.toClient())
errors.Check(cli.upload())
case "auto-add":
err := autoAdd(conf.LocalizableFiles[0])
errors.Check(err)
default:
usage("unknown command")
}
}
// usage prints usage. If addStr is not empty print addStr and exit with code
// 1, otherwise exit with code 0.
func usage(addStr string) {
const usageStr = `Usage: go run main.go <command> [<args>]
Commands:
help
Print usage.
summary
Print summary.
download [-n <count>]
Download translations. count is a number of concurrent downloads.
unused
Print unused strings.
upload
Upload translations.
auto-add
Add locales with additions to the git and restore locales with
deletions.`
if addStr != "" {
fmt.Printf("%s\n%s\n", addStr, usageStr)
os.Exit(1)
}
fmt.Println(usageStr)
os.Exit(0)
}
// twoskyConfig is the configuration structure for localization.
type twoskyConfig struct {
Languages languages `json:"languages"`
ProjectID string `json:"project_id"`
BaseLangcode langCode `json:"base_locale"`
LocalizableFiles []string `json:"localizable_files"`
}
// readTwoskyConfig returns twosky configuration.
func readTwoskyConfig() (t *twoskyConfig, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
b, err := os.ReadFile(twoskyConfFile)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
var tsc []twoskyConfig
err = json.Unmarshal(b, &tsc)
if err != nil {
return nil, fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
}
if len(tsc) == 0 {
return nil, fmt.Errorf("%q is empty", twoskyConfFile)
}
conf := tsc[0]
for _, lang := range conf.Languages {
if lang == "" {
return nil, errors.Error("language is empty")
}
}
if len(conf.LocalizableFiles) == 0 {
return nil, errors.Error("no localizable files specified")
}
return &conf, nil
}
// twoskyClient is the twosky client with methods for download and upload
// translations.
type twoskyClient struct {
// uri is the base URL.
uri *url.URL
// projectID is the name of the project.
projectID string
// baseLang is the base language code.
baseLang langCode
// langs is the list of codes of languages to download.
langs []langCode
}
// toClient reads values from environment variables or defaults, validates
// them, and returns the twosky client.
func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
defer func() { err = errors.Annotate(err, "filling config: %w") }()
uriStr := cmp.Or(os.Getenv("TWOSKY_URI"), twoskyURI)
uri, err := url.Parse(uriStr)
if err != nil {
return nil, err
}
projectID := cmp.Or(os.Getenv("TWOSKY_PROJECT_ID"), defaultProjectID)
baseLang := t.BaseLangcode
uLangStr := os.Getenv("UPLOAD_LANGUAGE")
if uLangStr != "" {
baseLang = langCode(uLangStr)
}
langs := slices.Sorted(maps.Keys(t.Languages))
dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
if dlLangStr == "blocker" {
langs = blockerLangCodes
} else if dlLangStr != "" {
var dlLangs []langCode
dlLangs, err = validateLanguageStr(dlLangStr, t.Languages)
if err != nil {
return nil, err
}
langs = dlLangs
}
return &twoskyClient{
uri: uri,
projectID: projectID,
baseLang: baseLang,
langs: langs,
}, nil
}
// validateLanguageStr validates languages codes that contain in the str and
// returns them or error.
func validateLanguageStr(str string, all languages) (langs []langCode, err error) {
codes := strings.Fields(str)
langs = make([]langCode, 0, len(codes))
for _, k := range codes {
lc := langCode(k)
_, ok := all[lc]
if !ok {
return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
}
langs = append(langs, lc)
}
return langs, nil
}
// readLocales reads file with name fn and returns a map, where key is text
// label and value is localization.
func readLocales(fn string) (loc locales, err error) {
b, err := os.ReadFile(fn)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil, err
}
loc = make(locales)
err = json.Unmarshal(b, &loc)
if err != nil {
err = fmt.Errorf("unmarshalling %q: %w", fn, err)
return nil, err
}
return loc, nil
}
// summary prints summary for translations.
func summary(langs languages) (err error) {
basePath := filepath.Join(localesDir, defaultBaseFile)
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("summary: %w", err)
}
size := float64(len(baseLoc))
keys := slices.Sorted(maps.Keys(langs))
for _, lang := range keys {
name := filepath.Join(localesDir, string(lang)+".json")
if name == basePath {
continue
}
var loc locales
loc, err = readLocales(name)
if err != nil {
return fmt.Errorf("summary: reading locales: %w", err)
}
f := float64(len(loc)) * 100 / size
blocker := ""
// N is small enough to not raise performance questions.
ok := slices.Contains(blockerLangCodes, lang)
if ok {
blocker = " (blocker)"
}
fmt.Printf("%s\t %6.2f %%%s\n", lang, f, blocker)
}
return nil
}
// unused prints unused text labels.
func unused(ctx context.Context, l *slog.Logger, basePath string) (err error) {
defer func() { err = errors.Annotate(err, "unused: %w") }()
baseLoc, err := readLocales(basePath)
if err != nil {
return err
}
locDir := filepath.Clean(localesDir)
js, err := findJS(ctx, l, locDir)
if err != nil {
return err
}
return findUnused(js, baseLoc)
}
// findJS returns list of JavaScript and JSON files or error.
func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []string, err error) {
walkFn := func(name string, _ os.FileInfo, err error) error {
if err != nil {
l.WarnContext(ctx, "accessing a path", slogutil.KeyError, err)
return nil
}
if strings.HasPrefix(name, locDir) {
return nil
}
ext := filepath.Ext(name)
if ext == ".js" || ext == ".json" {
fileNames = append(fileNames, name)
}
return nil
}
err = filepath.Walk(srcDir, walkFn)
if err != nil {
return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
return fileNames, nil
}
// findUnused prints unused text labels from fileNames.
func findUnused(fileNames []string, loc locales) (err error) {
knownUsed := []textLabel{
"blocking_mode_refused",
"blocking_mode_nxdomain",
"blocking_mode_custom_ip",
}
for _, v := range knownUsed {
delete(loc, v)
}
for _, fn := range fileNames {
var buf []byte
buf, err = os.ReadFile(fn)
if err != nil {
return fmt.Errorf("finding unused: %w", err)
}
for k := range loc {
if bytes.Contains(buf, []byte(k)) {
delete(loc, k)
}
}
}
for _, v := range slices.Sorted(maps.Keys(loc)) {
fmt.Println(v)
}
return nil
}
// autoAdd adds locales with additions to the git and restores locales with
// deletions.
func autoAdd(basePath string) (err error) {
defer func() { err = errors.Annotate(err, "auto add: %w") }()
adds, dels, err := changedLocales()
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if slices.Contains(dels, basePath) {
return errors.Error("base locale contains deletions")
}
err = handleAdds(adds)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil
}
err = handleDels(dels)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil
}
return nil
}
// handleAdds adds locales with additions to the git.
func handleAdds(locales []string) (err error) {
if len(locales) == 0 {
return nil
}
args := append([]string{"add"}, locales...)
code, out, err := aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
return nil
}
// handleDels restores locales with deletions.
func handleDels(locales []string) (err error) {
if len(locales) == 0 {
return nil
}
args := append([]string{"restore"}, locales...)
code, out, err := aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
return nil
}
// changedLocales returns cleaned paths of locales with changes or error. adds
// is the list of locales with only additions. dels is the list of locales
// with only deletions.
func changedLocales() (adds, dels []string, err error) {
defer func() { err = errors.Annotate(err, "getting changes: %w") }()
cmd := exec.Command("git", "diff", "--numstat", localesDir)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, fmt.Errorf("piping: %w", err)
}
err = cmd.Start()
if err != nil {
return nil, nil, fmt.Errorf("starting: %w", err)
}
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 3 {
return nil, nil, fmt.Errorf("invalid input: %q", line)
}
path := fields[2]
if fields[0] == "0" {
dels = append(dels, path)
} else if fields[1] == "0" {
adds = append(adds, path)
}
}
err = scanner.Err()
if err != nil {
return nil, nil, fmt.Errorf("scanning: %w", err)
}
err = cmd.Wait()
if err != nil {
return nil, nil, fmt.Errorf("waiting: %w", err)
}
return adds, dels, nil
}