mirrors/AdGuardHome
1
0
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-11-21 12:35:33 +03:00
Code Issues Projects Releases Packages Wiki Activity

Pull request 1908: AG-23497-scripts-download-languages

Browse source 
Squashed commit of the following:

commit 874e847fc9bbfaeb8af1c02eb0ba1dbb98bd008f
Merge: 4becdd809 a79deda66
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 12 16:01:45 2023 +0300

    Merge branch 'master' into AG-23497-scripts-download-languages

commit 4becdd8092558b15d783674f5b9d1e9c151e3a8c
Merge: 1e5385c33 40884624c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 12 13:34:34 2023 +0300

    Merge branch 'master' into AG-23497-scripts-download-languages

commit 1e5385c33a298b0b8563fee6704f6bb3ded12d60
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jul 11 19:56:29 2023 +0300

    all: upd golibs, imp code

commit 0498960b00be21b1294f8b71108b234554e5847f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jul 7 19:05:58 2023 +0300

    scripts: imp naming

commit 6e36ed83c6bec2fe6159442a9e6805c0720e27f5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jul 6 16:37:13 2023 +0300

    scripts: separate files

commit 55027cfa1c04b0a36e5267b024b53a45f26dd974
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 5 13:51:40 2023 +0300

    scripts: add download languages
This commit is contained in:
Stanislav Chzhen 2023-07-12 16:06:17 +03:00
parent a79deda665
commit 55335c4061
7 changed files with 476 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -5,7 +5,7 @@ go 1.19
require (
// TODO(a.garipov): Update to a tagged version when it's released.
github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768
github.com/AdguardTeam/golibs v0.13.3
github.com/AdguardTeam/golibs v0.13.4
github.com/AdguardTeam/urlfilter v0.16.1
github.com/NYTimes/gziphandler v1.1.1
github.com/ameshkov/dnscrypt/v2 v2.2.7

4
go.sum
View file

@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+
github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
github.com/AdguardTeam/golibs v0.13.4 h1:ACTwIR1pEENBijHcEWtiMbSh4wWQOlIHRxmUB8oBHf8=
github.com/AdguardTeam/golibs v0.13.4/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=

20
scripts/README.md
View file

@ -269,25 +269,29 @@ Optional environment:
### Usage
* `go run main.go help`: print usage.
* `go run ./scripts/translations help`: print usage.
* `go run main.go download [-n <count>]`: download and save all translations.
`n` is optional flag where count is a number of concurrent downloads.
* `go run ./scripts/translations download [-n <count>]`: download and save
all translations. `n` is optional flag where count is a number of
concurrent downloads.
* `go run main.go upload`: upload the base `en` locale.
* `go run ./scripts/translations upload`: upload the base `en` locale.
* `go run main.go summary`: show the current locales summary.
* `go run ./scripts/translations summary`: show the current locales summary.
* `go run main.go unused`: show the list of unused strings.
* `go run ./scripts/translations unused`: show the list of unused strings.
* `go run main.go auto-add`: add locales with additions to the git and
restore locales with deletions.
* `go run ./scripts/translations auto-add`: add locales with additions to the
git and restore locales with deletions.
After the download you'll find the output locales in the `client/src/__locales/`
directory.
Optional environment:
* `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`. For
example `ar be bg`.
* `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
* `TWOSKY_URI`: set an alternative URL for `download` or `upload`.

1
scripts/make/go-lint.sh
View file

@ -183,6 +183,7 @@ run_linter gocognit --over 10\
./internal/tools/\
./internal/version/\
./internal/whois/\
./scripts/\
;
run_linter ineffassign ./...

177
scripts/translations/download.go Normal file
View file

@ -0,0 +1,177 @@
package main
import (
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/slices"
)
// download and save all translations.
func (c *twoskyClient) download() (err error) {
var numWorker int
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
flagSet.Usage = func() {
usage("download command error")
}
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
err = flagSet.Parse(os.Args[2:])
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if numWorker < 1 {
usage("count must be positive")
}
downloadURI := c.uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
}
wg := &sync.WaitGroup{}
failed := &sync.Map{}
uriCh := make(chan *url.URL, len(c.langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, failed, client, uriCh)
}
for lang := range c.langs {
uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
uriCh <- uri
}
close(uriCh)
wg.Wait()
printFailedLocales(failed)
return nil
}
// printFailedLocales prints sorted list of failed downloads, if any.
func printFailedLocales(failed *sync.Map) {
keys := []string{}
failed.Range(func(k, _ any) bool {
s, ok := k.(string)
if !ok {
panic("unexpected type")
}
keys = append(keys, s)
return true
})
if len(keys) == 0 {
return
}
slices.Sort(keys)
log.Info("failed locales: %s", strings.Join(keys, " "))
}
// downloadWorker downloads translations by received urls and saves them.
// Where failed is a map for storing failed downloads.
func downloadWorker(
wg *sync.WaitGroup,
failed *sync.Map,
client *http.Client,
uriCh <-chan *url.URL,
) {
defer wg.Done()
for uri := range uriCh {
q := uri.Query()
code := q.Get("language")
err := saveToFile(client, uri, code)
if err != nil {
log.Error("download: worker: %s", err)
failed.Store(code, struct{}{})
}
}
}
// saveToFile downloads translation by url and saves it to a file, or returns
// error.
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
data, err := getTranslation(client, uri.String())
if err != nil {
log.Info("%s", data)
return fmt.Errorf("getting translation: %s", err)
}
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
if err != nil {
return fmt.Errorf("writing file: %s", err)
}
fmt.Println(name)
return nil
}
// getTranslation returns received translation data and error. If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
// Go on and download the body for inspection.
}
limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
if lrErr != nil {
// Generally shouldn't happen, since the only error returned by
// [aghio.LimitReader] is an argument error.
panic(fmt.Errorf("limit reading: %w", lrErr))
}
data, readErr := io.ReadAll(limitReader)
return data, errors.WithDeferred(err, readErr)
}
// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
uri = &url.URL{}
*uri = *oldURL
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)
q.Set("project", projectID)
q.Set("language", string(lang))
uri.RawQuery = q.Encode()
return uri
}

480
scripts/translations/main.go
View file

@ -6,25 +6,16 @@ import (
"bufio"
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/httphdr"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
@ -38,7 +29,8 @@ const (
srcDir = "./client/src"
twoskyURI = "https://twosky.int.agrd.dev/api/v1"
readLimit = 1 * 1024 * 1024
readLimit = 1 * 1024 * 1024
uploadTimeout = 10 * time.Second
)
// langCode is a language code.
@ -62,31 +54,26 @@ func main() {
usage("")
}
uriStr := os.Getenv("TWOSKY_URI")
if uriStr == "" {
uriStr = twoskyURI
}
uri, err := url.Parse(uriStr)
conf, err := readTwoskyConfig()
check(err)
projectID := os.Getenv("TWOSKY_PROJECT_ID")
if projectID == "" {
projectID = defaultProjectID
}
conf, err := readTwoskyConf()
check(err)
var cli *twoskyClient
switch os.Args[1] {
case "summary":
err = summary(conf.Languages)
case "download":
err = download(uri, projectID, conf.Languages)
cli, err = conf.toClient()
check(err)
err = cli.download()
case "unused":
err = unused(conf.LocalizableFiles[0])
case "upload":
err = upload(uri, projectID, conf.BaseLangcode)
cli, err = conf.toClient()
check(err)
err = cli.upload()
case "auto-add":
err = autoAdd(conf.LocalizableFiles[0])
default:
@ -133,51 +120,131 @@ Commands:
os.Exit(0)
}
// twoskyConf is the configuration structure for localization.
type twoskyConf struct {
// twoskyConfig is the configuration structure for localization.
type twoskyConfig struct {
Languages languages `json:"languages"`
ProjectID string `json:"project_id"`
BaseLangcode langCode `json:"base_locale"`
LocalizableFiles []string `json:"localizable_files"`
}
// readTwoskyConf returns configuration.
func readTwoskyConf() (t twoskyConf, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
// readTwoskyConfig returns twosky configuration.
func readTwoskyConfig() (t *twoskyConfig, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
b, err := os.ReadFile(twoskyConfFile)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return twoskyConf{}, err
return nil, err
}
var tsc []twoskyConf
var tsc []twoskyConfig
err = json.Unmarshal(b, &tsc)
if err != nil {
err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
return twoskyConf{}, err
return nil, err
}
if len(tsc) == 0 {
err = fmt.Errorf("%q is empty", twoskyConfFile)
return twoskyConf{}, err
return nil, err
}
conf := tsc[0]
for _, lang := range conf.Languages {
if lang == "" {
return twoskyConf{}, errors.Error("language is empty")
return nil, errors.Error("language is empty")
}
}
if len(conf.LocalizableFiles) == 0 {
return twoskyConf{}, errors.Error("no localizable files specified")
return nil, errors.Error("no localizable files specified")
}
return conf, nil
return &conf, nil
}
// twoskyClient is the twosky client with methods for download and upload
// translations.
type twoskyClient struct {
// uri is the base URL.
uri *url.URL
// langs is the map of languages to download.
langs languages
// projectID is the name of the project.
projectID string
// baseLang is the base language code.
baseLang langCode
}
// toClient reads values from environment variables or defaults, validates
// them, and returns the twosky client.
func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
defer func() { err = errors.Annotate(err, "filling config: %w") }()
uriStr := os.Getenv("TWOSKY_URI")
if uriStr == "" {
uriStr = twoskyURI
}
uri, err := url.Parse(uriStr)
if err != nil {
return nil, err
}
projectID := os.Getenv("TWOSKY_PROJECT_ID")
if projectID == "" {
projectID = defaultProjectID
}
baseLang := t.BaseLangcode
uLangStr := os.Getenv("UPLOAD_LANGUAGE")
if uLangStr != "" {
baseLang = langCode(uLangStr)
}
langs := t.Languages
dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
if dlLangStr != "" {
var dlLangs languages
dlLangs, err = validateLanguageStr(dlLangStr, langs)
if err != nil {
return nil, err
}
langs = dlLangs
}
return &twoskyClient{
uri: uri,
projectID: projectID,
baseLang: baseLang,
langs: langs,
}, nil
}
// validateLanguageStr validates languages codes that contain in the str and
// returns language map, where key is language code and value is display name.
func validateLanguageStr(str string, all languages) (langs languages, err error) {
langs = make(languages)
codes := strings.Fields(str)
for _, k := range codes {
lc := langCode(k)
name, ok := all[lc]
if !ok {
return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
}
langs[lc] = name
}
return langs, nil
}
// readLocales reads file with name fn and returns a map, where key is text
@ -233,163 +300,33 @@ func summary(langs languages) (err error) {
return nil
}
// download and save all translations. uri is the base URL. projectID is the
// name of the project.
func download(uri *url.URL, projectID string, langs languages) (err error) {
var numWorker int
// unused prints unused text labels.
func unused(basePath string) (err error) {
defer func() { err = errors.Annotate(err, "unused: %w") }()
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
flagSet.Usage = func() {
usage("download command error")
}
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
err = flagSet.Parse(os.Args[2:])
baseLoc, err := readLocales(basePath)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if numWorker < 1 {
usage("count must be positive")
}
downloadURI := uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
}
wg := &sync.WaitGroup{}
uriCh := make(chan *url.URL, len(langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, client, uriCh)
}
for lang := range langs {
uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
uriCh <- uri
}
close(uriCh)
wg.Wait()
return nil
}
// downloadWorker downloads translations by received urls and saves them.
func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
defer wg.Done()
for uri := range uriCh {
data, err := getTranslation(client, uri.String())
if err != nil {
log.Error("download worker: getting translation: %s", err)
log.Info("download worker: error response:\n%s", data)
continue
}
q := uri.Query()
code := q.Get("language")
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
code = strings.ToLower(code)
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
if err != nil {
log.Error("download worker: writing file: %s", err)
continue
}
fmt.Println(name)
}
}
// getTranslation returns received translation data and error. If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
// Go on and download the body for inspection.
}
limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
if lrErr != nil {
// Generally shouldn't happen, since the only error returned by
// [aghio.LimitReader] is an argument error.
panic(fmt.Errorf("limit reading: %w", lrErr))
}
data, readErr := io.ReadAll(limitReader)
return data, errors.WithDeferred(err, readErr)
}
// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
uri = &url.URL{}
*uri = *oldURL
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
switch lang {
case "si-lk":
lang = "si-LK"
case "zh-hk":
lang = "zh-HK"
default:
// Go on.
}
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)
q.Set("project", projectID)
q.Set("language", string(lang))
uri.RawQuery = q.Encode()
return uri
}
// unused prints unused text labels.
func unused(basePath string) (err error) {
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("unused: %w", err)
}
locDir := filepath.Clean(localesDir)
js, err := findJS(locDir)
if err != nil {
return err
}
fileNames := []string{}
err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
return findUnused(js, baseLoc)
}
// findJS returns list of JavaScript and JSON files or error.
func findJS(locDir string) (fileNames []string, err error) {
walkFn := func(name string, _ os.FileInfo, err error) error {
if err != nil {
log.Info("warning: accessing a path %q: %s", name, err)
return nil
}
if info.IsDir() {
return nil
}
if strings.HasPrefix(name, locDir) {
return nil
}
@ -400,13 +337,14 @@ func unused(basePath string) (err error) {
}
return nil
})
if err != nil {
return fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
return findUnused(fileNames, baseLoc)
err = filepath.Walk(srcDir, walkFn)
if err != nil {
return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
return fileNames, nil
}
// findUnused prints unused text labels from fileNames.
@ -445,118 +383,6 @@ func findUnused(fileNames []string, loc locales) (err error) {
return nil
}
// upload base translation. uri is the base URL. projectID is the name of the
// project. baseLang is the base language code.
func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
defer func() { err = errors.Annotate(err, "upload: %w") }()
uploadURI := uri.JoinPath("upload")
lang := baseLang
langStr := os.Getenv("UPLOAD_LANGUAGE")
if langStr != "" {
lang = langCode(langStr)
}
basePath := filepath.Join(localesDir, defaultBaseFile)
formData := map[string]string{
"format": "json",
"language": string(lang),
"filename": defaultBaseFile,
"project": projectID,
}
buf, cType, err := prepareMultipartMsg(formData, basePath)
if err != nil {
return fmt.Errorf("preparing multipart msg: %w", err)
}
err = send(uploadURI.String(), cType, buf)
if err != nil {
return fmt.Errorf("sending multipart msg: %w", err)
}
return nil
}
// prepareMultipartMsg prepares translation data for upload.
func prepareMultipartMsg(
formData map[string]string,
basePath string,
) (buf *bytes.Buffer, cType string, err error) {
buf = &bytes.Buffer{}
w := multipart.NewWriter(buf)
var fw io.Writer
for k, v := range formData {
err = w.WriteField(k, v)
if err != nil {
return nil, "", fmt.Errorf("writing field: %w", err)
}
}
file, err := os.Open(basePath)
if err != nil {
return nil, "", fmt.Errorf("opening file: %w", err)
}
defer func() {
err = errors.WithDeferred(err, file.Close())
}()
h := make(textproto.MIMEHeader)
h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
h.Set(httphdr.ContentDisposition, d)
fw, err = w.CreatePart(h)
if err != nil {
return nil, "", fmt.Errorf("creating part: %w", err)
}
_, err = io.Copy(fw, file)
if err != nil {
return nil, "", fmt.Errorf("copying: %w", err)
}
err = w.Close()
if err != nil {
return nil, "", fmt.Errorf("closing writer: %w", err)
}
return buf, w.FormDataContentType(), nil
}
// send POST request to uriStr.
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
var client http.Client
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
if err != nil {
return fmt.Errorf("bad request: %w", err)
}
req.Header.Set(httphdr.ContentType, cType)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("client post form: %w", err)
}
defer func() {
err = errors.WithDeferred(err, resp.Body.Close())
}()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
}
return nil
}
// autoAdd adds locales with additions to the git and restores locales with
// deletions.
func autoAdd(basePath string) (err error) {
@ -572,28 +398,48 @@ func autoAdd(basePath string) (err error) {
return errors.Error("base locale contains deletions")
}
var (
args []string
code int
out []byte
)
if len(adds) > 0 {
args = append([]string{"add"}, adds...)
code, out, err = aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
err = handleAdds(adds)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil
}
if len(dels) > 0 {
args = append([]string{"restore"}, dels...)
code, out, err = aghos.RunCommand("git", args...)
err = handleDels(dels)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil
}
if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
return nil
}
// handleAdds adds locales with additions to the git.
func handleAdds(locales []string) (err error) {
if len(locales) == 0 {
return nil
}
args := append([]string{"add"}, locales...)
code, out, err := aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
return nil
}
// handleDels restores locales with deletions.
func handleDels(locales []string) (err error) {
if len(locales) == 0 {
return nil
}
args := append([]string{"restore"}, locales...)
code, out, err := aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
return nil

120
scripts/translations/upload.go Normal file
View file

@ -0,0 +1,120 @@
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"os"
"path/filepath"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/httphdr"
"github.com/AdguardTeam/golibs/mapsutil"
)
// upload base translation.
func (c *twoskyClient) upload() (err error) {
defer func() { err = errors.Annotate(err, "upload: %w") }()
uploadURI := c.uri.JoinPath("upload")
basePath := filepath.Join(localesDir, defaultBaseFile)
formData := map[string]string{
"format": "json",
"language": string(c.baseLang),
"filename": defaultBaseFile,
"project": c.projectID,
}
buf, cType, err := prepareMultipartMsg(formData, basePath)
if err != nil {
return fmt.Errorf("preparing multipart msg: %w", err)
}
err = send(uploadURI.String(), cType, buf)
if err != nil {
return fmt.Errorf("sending multipart msg: %w", err)
}
return nil
}
// prepareMultipartMsg prepares translation data for upload.
func prepareMultipartMsg(
formData map[string]string,
basePath string,
) (buf *bytes.Buffer, cType string, err error) {
buf = &bytes.Buffer{}
w := multipart.NewWriter(buf)
var fw io.Writer
err = mapsutil.OrderedRangeError(formData, w.WriteField)
if err != nil {
return nil, "", fmt.Errorf("writing field: %w", err)
}
file, err := os.Open(basePath)
if err != nil {
return nil, "", fmt.Errorf("opening file: %w", err)
}
defer func() {
err = errors.WithDeferred(err, file.Close())
}()
h := make(textproto.MIMEHeader)
h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
h.Set(httphdr.ContentDisposition, d)
fw, err = w.CreatePart(h)
if err != nil {
return nil, "", fmt.Errorf("creating part: %w", err)
}
_, err = io.Copy(fw, file)
if err != nil {
return nil, "", fmt.Errorf("copying: %w", err)
}
err = w.Close()
if err != nil {
return nil, "", fmt.Errorf("closing writer: %w", err)
}
return buf, w.FormDataContentType(), nil
}
// send POST request to uriStr.
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
client := http.Client{
Timeout: uploadTimeout,
}
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
if err != nil {
return fmt.Errorf("bad request: %w", err)
}
req.Header.Set(httphdr.ContentType, cType)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("client post form: %w", err)
}
defer func() {
err = errors.WithDeferred(err, resp.Body.Close())
}()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
}
return nil
}