AdGuardHome/scripts/translations/download.go

package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/AdguardTeam/golibs/errors"
	"github.com/AdguardTeam/golibs/ioutil"
	"github.com/AdguardTeam/golibs/log"
	"golang.org/x/exp/slices"
)

// download and save all translations.
func (c *twoskyClient) download() (err error) {
	var numWorker int

	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
	flagSet.Usage = func() {
		usage("download command error")
	}
	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")

	err = flagSet.Parse(os.Args[2:])
	if err != nil {
		// Don't wrap the error since it's informative enough as is.
		return err
	}

	if numWorker < 1 {
		usage("count must be positive")
	}

	downloadURI := c.uri.JoinPath("download")

	client := &http.Client{
		Timeout: 10 * time.Second,
	}

	wg := &sync.WaitGroup{}
	failed := &sync.Map{}
	uriCh := make(chan *url.URL, len(c.langs))

	for i := 0; i < numWorker; i++ {
		wg.Add(1)
		go downloadWorker(wg, failed, client, uriCh)
	}

	for _, lang := range c.langs {
		uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)

		uriCh <- uri
	}

	close(uriCh)
	wg.Wait()

	printFailedLocales(failed)

	return nil
}

// printFailedLocales prints sorted list of failed downloads, if any.
func printFailedLocales(failed *sync.Map) {
	keys := []string{}
	failed.Range(func(k, _ any) bool {
		s, ok := k.(string)
		if !ok {
			panic("unexpected type")
		}

		keys = append(keys, s)

		return true
	})

	if len(keys) == 0 {
		return
	}

	slices.Sort(keys)
	log.Info("failed locales: %s", strings.Join(keys, " "))
}

// downloadWorker downloads translations by received urls and saves them.
// Where failed is a map for storing failed downloads.
func downloadWorker(
	wg *sync.WaitGroup,
	failed *sync.Map,
	client *http.Client,
	uriCh <-chan *url.URL,
) {
	defer wg.Done()

	for uri := range uriCh {
		q := uri.Query()
		code := q.Get("language")

		err := saveToFile(client, uri, code)
		if err != nil {
			log.Error("download: worker: %s", err)
			failed.Store(code, struct{}{})
		}
	}
}

// saveToFile downloads translation by url and saves it to a file, or returns
// error.
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
	data, err := getTranslation(client, uri.String())
	if err != nil {
		log.Info("%s", data)

		return fmt.Errorf("getting translation: %s", err)
	}

	name := filepath.Join(localesDir, code+".json")
	err = os.WriteFile(name, data, 0o664)
	if err != nil {
		return fmt.Errorf("writing file: %s", err)
	}

	fmt.Println(name)

	return nil
}

// getTranslation returns received translation data and error.  If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
	resp, err := client.Get(url)
	if err != nil {
		return nil, fmt.Errorf("requesting: %w", err)
	}

	defer log.OnCloserError(resp.Body, log.ERROR)

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))

		// Go on and download the body for inspection.
	}

	limitReader := ioutil.LimitReader(resp.Body, readLimit)

	data, readErr := io.ReadAll(limitReader)

	return data, errors.WithDeferred(err, readErr)
}

// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
	uri = &url.URL{}
	*uri = *oldURL

	q := uri.Query()
	q.Set("format", "json")
	q.Set("filename", baseFile)
	q.Set("project", projectID)
	q.Set("language", string(lang))

	uri.RawQuery = q.Encode()

	return uri
}