diff --git a/build/generate-emoji.go b/build/generate-emoji.go index 5a88e456ee..98c2f15d75 100644 --- a/build/generate-emoji.go +++ b/build/generate-emoji.go @@ -53,8 +53,6 @@ func (e Emoji) MarshalJSON() ([]byte, error) { } func main() { - var err error - flag.Parse() // generate data @@ -83,8 +81,6 @@ var replacer = strings.NewReplacer( var emojiRE = regexp.MustCompile(`\{Emoji:"([^"]*)"`) func generate() ([]byte, error) { - var err error - // load gemoji data res, err := http.Get(gemojiURL) if err != nil { diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index 800a15e215..56745e9a38 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -91,7 +91,7 @@ func TestMigrateActionsArtifacts(t *testing.T) { srcStorage, _ := createLocalStorage(t) defer test.MockVariableValue(&storage.ActionsArtifacts, srcStorage)() - id := int64(0) + id := int64(42) addArtifact := func(storagePath string, status actions.ArtifactStatus) { id++ diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 3f0e9c447d..f0fd40da5f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -328,6 +328,10 @@ RUN_USER = ; git ;; Maximum number of locks returned per page ;LFS_LOCKS_PAGING_NUM = 50 ;; +;; When clients make lfs batch requests, reject them if there are more pointers than this number +;; zero means 'unlimited' +;LFS_MAX_BATCH_SIZE = 0 +;; ;; Allow graceful restarts using SIGHUP to fork ;ALLOW_GRACEFUL_RESTARTS = true ;; @@ -2672,6 +2676,10 @@ LEVEL = Info ;; override the minio base path if storage type is minio ;MINIO_BASE_PATH = lfs/ +;[lfs_client] +;; When mirroring an upstream lfs endpoint, limit the number of pointers in each batch request to this number +;BATCH_SIZE = 20 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; settings for packages, will override storage setting diff --git a/models/fixtures/action_artifact.yml b/models/fixtures/action_artifact.yml new file mode 100644 index 0000000000..2c51c11ebd --- /dev/null +++ b/models/fixtures/action_artifact.yml @@ -0,0 +1,71 @@ +- + id: 1 + run_id: 791 + runner_id: 1 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + storage_path: "26/1/1712166500347189545.chunk" + file_size: 1024 + file_compressed_size: 1024 + content_encoding: "" + artifact_path: "abc.txt" + artifact_name: "artifact-download" + status: 1 + created_unix: 1712338649 + updated_unix: 1712338649 + expired_unix: 1720114649 + +- + id: 19 + run_id: 791 + runner_id: 1 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + storage_path: "26/19/1712348022422036662.chunk" + file_size: 1024 + file_compressed_size: 1024 + content_encoding: "" + artifact_path: "abc.txt" + artifact_name: "multi-file-download" + status: 2 + created_unix: 1712348022 + updated_unix: 1712348022 + expired_unix: 1720124022 + +- + id: 20 + run_id: 791 + runner_id: 1 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + storage_path: "26/20/1712348022423431524.chunk" + file_size: 1024 + file_compressed_size: 1024 + content_encoding: "" + artifact_path: "xyz/def.txt" + artifact_name: "multi-file-download" + status: 2 + created_unix: 1712348022 + updated_unix: 1712348022 + expired_unix: 1720124022 + +- + id: 22 + run_id: 792 + runner_id: 1 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + storage_path: "27/5/1730330775594233150.chunk" + file_size: 1024 + file_compressed_size: 1024 + content_encoding: "application/zip" + artifact_path: "artifact-v4-download.zip" + artifact_name: "artifact-v4-download" + status: 2 + created_unix: 1730330775 + updated_unix: 1730330775 + expired_unix: 1738106775 diff --git a/models/git/lfs.go b/models/git/lfs.go index 44b741c4c8..635d70d9bc 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -136,8 +136,6 @@ var ErrLFSObjectNotExist = db.ErrNotExist{Resource: "LFS Meta object"} // NewLFSMetaObject stores a given populated LFSMetaObject structure in the database // if it is not already present. func NewLFSMetaObject(ctx context.Context, repoID int64, p lfs.Pointer) (*LFSMetaObject, error) { - var err error - ctx, committer, err := db.TxContext(ctx) if err != nil { return nil, err diff --git a/models/git/protected_branch.go b/models/git/protected_branch.go index a8b8c81bbe..4fc08020e0 100644 --- a/models/git/protected_branch.go +++ b/models/git/protected_branch.go @@ -79,14 +79,20 @@ func IsRuleNameSpecial(ruleName string) bool { } func (protectBranch *ProtectedBranch) loadGlob() { - if protectBranch.globRule == nil { - var err error - protectBranch.globRule, err = glob.Compile(protectBranch.RuleName, '/') - if err != nil { - log.Warn("Invalid glob rule for ProtectedBranch[%d]: %s %v", protectBranch.ID, protectBranch.RuleName, err) - protectBranch.globRule = glob.MustCompile(glob.QuoteMeta(protectBranch.RuleName), '/') - } - protectBranch.isPlainName = !IsRuleNameSpecial(protectBranch.RuleName) + if protectBranch.isPlainName || protectBranch.globRule != nil { + return + } + // detect if it is not glob + if !IsRuleNameSpecial(protectBranch.RuleName) { + protectBranch.isPlainName = true + return + } + // now we load the glob + var err error + protectBranch.globRule, err = glob.Compile(protectBranch.RuleName, '/') + if err != nil { + log.Warn("Invalid glob rule for ProtectedBranch[%d]: %s %v", protectBranch.ID, protectBranch.RuleName, err) + protectBranch.globRule = glob.MustCompile(glob.QuoteMeta(protectBranch.RuleName), '/') } } diff --git a/models/git/protected_banch_list_test.go b/models/git/protected_branch_list_test.go similarity index 79% rename from models/git/protected_banch_list_test.go rename to models/git/protected_branch_list_test.go index 09319d21a8..db7e54f685 100644 --- a/models/git/protected_banch_list_test.go +++ b/models/git/protected_branch_list_test.go @@ -75,3 +75,32 @@ func TestBranchRuleMatchPriority(t *testing.T) { } } } + +func TestBranchRuleSort(t *testing.T) { + in := []*ProtectedBranch{{ + RuleName: "b", + CreatedUnix: 1, + }, { + RuleName: "b/*", + CreatedUnix: 3, + }, { + RuleName: "a/*", + CreatedUnix: 2, + }, { + RuleName: "c", + CreatedUnix: 0, + }, { + RuleName: "a", + CreatedUnix: 4, + }} + expect := []string{"c", "b", "a", "a/*", "b/*"} + + pbr := ProtectedBranchRules(in) + pbr.sort() + + var got []string + for i := range pbr { + got = append(got, pbr[i].RuleName) + } + assert.Equal(t, expect, got) +} diff --git a/models/issues/label_test.go b/models/issues/label_test.go index b03fc1cd20..3e3d097458 100644 --- a/models/issues/label_test.go +++ b/models/issues/label_test.go @@ -231,8 +231,7 @@ func TestGetLabelsByOrgID(t *testing.T) { testSuccess(3, "reversealphabetically", []int64{4, 3}) testSuccess(3, "default", []int64{3, 4}) - var err error - _, err = issues_model.GetLabelsByOrgID(db.DefaultContext, 0, "leastissues", db.ListOptions{}) + _, err := issues_model.GetLabelsByOrgID(db.DefaultContext, 0, "leastissues", db.ListOptions{}) assert.True(t, issues_model.IsErrOrgLabelNotExist(err)) _, err = issues_model.GetLabelsByOrgID(db.DefaultContext, -1, "leastissues", db.ListOptions{}) diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 42c8415376..f7ea2beccf 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -41,14 +41,12 @@ func TestMaybeRemoveBOM(t *testing.T) { func TestToUTF8(t *testing.T) { resetDefaultCharsetsOrder() - var res string - var err error // Note: golang compiler seems so behave differently depending on the current // locale, so some conversions might behave differently. For that reason, we don't // depend on particular conversions but in expected behaviors. - res, err = ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) + res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) require.NoError(t, err) assert.Equal(t, "ABC", res) diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index 8390570098..f45b6e6191 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -50,25 +50,35 @@ func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) er } // ReadTreeToTemporaryIndex reads a treeish to a temporary index file -func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename, tmpDir string, cancel context.CancelFunc, err error) { - tmpDir, err = os.MkdirTemp("", "index") - if err != nil { - return filename, tmpDir, cancel, err - } +func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilename, tmpDir string, cancel context.CancelFunc, err error) { + defer func() { + // if error happens and there is a cancel function, do clean up + if err != nil && cancel != nil { + cancel() + cancel = nil + } + }() - filename = filepath.Join(tmpDir, ".tmp-index") - cancel = func() { - err := util.RemoveAll(tmpDir) - if err != nil { - log.Error("failed to remove tmp index file: %v", err) + removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs + return func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("failed to remove tmp index dir: %v", err) + } } } - err = repo.ReadTreeToIndex(treeish, filename) + + tmpDir, err = os.MkdirTemp("", "index") if err != nil { - defer cancel() - return "", "", func() {}, err + return "", "", nil, err } - return filename, tmpDir, cancel, err + + tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index") + cancel = removeDirFn(tmpDir) + err = repo.ReadTreeToIndex(treeish, tmpIndexFilename) + if err != nil { + return "", "", cancel, err + } + return tmpIndexFilename, tmpDir, cancel, err } // EmptyIndex empties the index diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 4859fe61e1..aa9e744d72 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -16,10 +16,9 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" ) -const httpBatchSize = 20 - // HTTPClient is used to communicate with the LFS server // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md type HTTPClient struct { @@ -30,7 +29,7 @@ type HTTPClient struct { // BatchSize returns the preferred size of batchs to process func (c *HTTPClient) BatchSize() int { - return httpBatchSize + return setting.LFSClient.BatchSize } func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient { diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 0290e1312d..1d3e04224f 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -203,8 +203,7 @@ func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node return ast.WalkContinue, nil } - var err error - _, err = w.WriteString(fmt.Sprintf(``, name)) + _, err := w.WriteString(fmt.Sprintf(``, name)) if err != nil { return ast.WalkStop, err } diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index da93e6cf6b..6438fb174f 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -37,8 +37,8 @@ func (s *ContentStore) ShouldServeDirect() bool { return setting.Packages.Storage.MinioConfig.ServeDirect } -func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string) (*url.URL, error) { - return s.store.URL(KeyToRelativePath(key), filename) +func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string, reqParams url.Values) (*url.URL, error) { + return s.store.URL(KeyToRelativePath(key), filename, reqParams) } // FIXME: Workaround to be removed in v1.20 diff --git a/modules/repository/repo.go b/modules/repository/repo.go index e15526d78d..9ce330b218 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -341,9 +341,10 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git for _, tag := range updates { if _, err := db.GetEngine(ctx).Where("repo_id = ? AND lower_tag_name = ?", repo.ID, strings.ToLower(tag.Name)). - Cols("sha1"). + Cols("sha1", "created_unix"). Update(&repo_model.Release{ - Sha1: tag.Object.String(), + Sha1: tag.Object.String(), + CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()), }); err != nil { return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) } diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go index 750101747f..ebb234a3ef 100644 --- a/modules/setting/lfs.go +++ b/modules/setting/lfs.go @@ -10,22 +10,31 @@ import ( "code.gitea.io/gitea/modules/generate" ) -// LFS represents the configuration for Git LFS +// LFS represents the server-side configuration for Git LFS. +// Ideally these options should be in a section like "[lfs_server]", +// but they are in "[server]" section due to historical reasons. +// Could be refactored in the future while keeping backwards compatibility. var LFS = struct { StartServer bool `ini:"LFS_START_SERVER"` JWTSecretBytes []byte `ini:"-"` HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"` MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"` LocksPagingNum int `ini:"LFS_LOCKS_PAGING_NUM"` + MaxBatchSize int `ini:"LFS_MAX_BATCH_SIZE"` Storage *Storage }{} +// LFSClient represents configuration for Gitea's LFS clients, for example: mirroring upstream Git LFS +var LFSClient = struct { + BatchSize int `ini:"BATCH_SIZE"` +}{} + func loadLFSFrom(rootCfg ConfigProvider) error { + mustMapSetting(rootCfg, "lfs_client", &LFSClient) + + mustMapSetting(rootCfg, "server", &LFS) sec := rootCfg.Section("server") - if err := sec.MapTo(&LFS); err != nil { - return fmt.Errorf("failed to map LFS settings: %v", err) - } lfsSec, _ := rootCfg.GetSection("lfs") @@ -52,6 +61,10 @@ func loadLFSFrom(rootCfg ConfigProvider) error { LFS.LocksPagingNum = 50 } + if LFSClient.BatchSize < 1 { + LFSClient.BatchSize = 20 + } + LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour) if !LFS.StartServer || !InstallLock { diff --git a/modules/setting/lfs_test.go b/modules/setting/lfs_test.go index c7f16379b2..324965781d 100644 --- a/modules/setting/lfs_test.go +++ b/modules/setting/lfs_test.go @@ -100,3 +100,19 @@ STORAGE_TYPE = minio assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket) assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath) } + +func Test_LFSClientServerConfigs(t *testing.T) { + iniStr := ` +[server] +LFS_MAX_BATCH_SIZE = 100 +[lfs_client] +# will default to 20 +BATCH_SIZE = 0 +` + cfg, err := NewConfigProviderFromData(iniStr) + assert.NoError(t, err) + + assert.NoError(t, loadLFSFrom(cfg)) + assert.EqualValues(t, 100, LFS.MaxBatchSize) + assert.EqualValues(t, 20, LFSClient.BatchSize) +} diff --git a/modules/storage/helper.go b/modules/storage/helper.go index 95f1c7b9a8..8bec3a0042 100644 --- a/modules/storage/helper.go +++ b/modules/storage/helper.go @@ -30,7 +30,7 @@ func (s DiscardStorage) Delete(_ string) error { return fmt.Errorf("%s", s) } -func (s DiscardStorage) URL(_, _ string) (*url.URL, error) { +func (s DiscardStorage) URL(_, _ string, _ url.Values) (*url.URL, error) { return nil, fmt.Errorf("%s", s) } diff --git a/modules/storage/helper_test.go b/modules/storage/helper_test.go index 60a7c61289..dd30c9b8ac 100644 --- a/modules/storage/helper_test.go +++ b/modules/storage/helper_test.go @@ -38,7 +38,7 @@ func Test_discardStorage(t *testing.T) { require.Error(t, err, string(tt)) } { - got, err := tt.URL("path", "name") + got, err := tt.URL("path", "name", nil) assert.Nil(t, got) require.Errorf(t, err, string(tt)) } diff --git a/modules/storage/local.go b/modules/storage/local.go index 9bb532f1df..00c7f668aa 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -114,7 +114,7 @@ func (l *LocalStorage) Delete(path string) error { } // URL gets the redirect URL to a file -func (l *LocalStorage) URL(path, name string) (*url.URL, error) { +func (l *LocalStorage) URL(path, name string, reqParams url.Values) (*url.URL, error) { return nil, ErrURLNotSupported } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index d0c2dec65b..b02eec7aa0 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -276,8 +276,12 @@ func (m *MinioStorage) Delete(path string) error { } // URL gets the redirect URL to a file. The presigned link is valid for 5 minutes. -func (m *MinioStorage) URL(path, name string) (*url.URL, error) { - reqParams := make(url.Values) +func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) { + // copy serveDirectReqParams + reqParams, err := url.ParseQuery(serveDirectReqParams.Encode()) + if err != nil { + return nil, err + } // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we? reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"") u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams) diff --git a/modules/storage/storage.go b/modules/storage/storage.go index b83b1c7929..d944b8618f 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -63,7 +63,7 @@ type ObjectStorage interface { Save(path string, r io.Reader, size int64) (int64, error) Stat(path string) (os.FileInfo, error) Delete(path string) error - URL(path, name string) (*url.URL, error) + URL(path, name string, reqParams url.Values) (*url.URL, error) IterateObjects(path string, iterator func(path string, obj Object) error) error } @@ -131,7 +131,7 @@ var ( ActionsArtifacts ObjectStorage = UninitializedStorage ) -// Init init the stoarge +// Init init the storage func Init() error { for _, f := range []func() error{ initAttachments, diff --git a/release-notes/5789.md b/release-notes/5789.md new file mode 100644 index 0000000000..0c0763a46e --- /dev/null +++ b/release-notes/5789.md @@ -0,0 +1,6 @@ +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/362ad0ba39bdbc87202e349678e21fc2a75ff7cb) Update force-pushed tags too when syncing mirrors +chore: [commit](https://codeberg.org/forgejo/forgejo/commit/b308bcca7c950b7f0d127ee4282019c2a9923299) Improved diff view performance +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/4c5bdddf7751a35985c08ba6506f1f30103749d6) Fix `missing signature key` error when pulling Docker images with `SERVE_DIRECT` enabled +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/2c5fdb108ff9e23e8f907fb6afe59177c6bb202e) Fix the missing menu in organization project view page +feat: [commit](https://codeberg.org/forgejo/forgejo/commit/1e595979625e54d375a0eaa440b84ef5e17af160) Add new [lfs_client].BATCH_SIZE and [server].LFS_MAX_BATCH_SIZE config settings. +fix: [commit](https://codeberg.org/forgejo/forgejo/commit/2358c0d899faec8311e46dcb0550041496bcd532) Properly clean temporary index files diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index bc29e4481d..405686a058 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -437,7 +437,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { for _, artifact := range artifacts { var downloadURL string if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { - u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName) + u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName, nil) if err != nil && !errors.Is(err, storage.ErrURLNotSupported) { log.Error("Error getting serve direct url: %v", err) } diff --git a/routers/api/actions/artifactsv4.go b/routers/api/actions/artifactsv4.go index 677e89da2f..0417f98242 100644 --- a/routers/api/actions/artifactsv4.go +++ b/routers/api/actions/artifactsv4.go @@ -530,7 +530,7 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) { respData := GetSignedArtifactURLResponse{} if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { - u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath) + u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath, nil) if u != nil && err == nil { respData.SignedUrl = u.String() } diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index f376e7bc59..9c9da38424 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -689,7 +689,9 @@ func DeleteManifest(ctx *context.Context) { } func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor) { - s, u, _, err := packages_service.GetPackageBlobStream(ctx, pfd.File, pfd.Blob) + serveDirectReqParams := make(url.Values) + serveDirectReqParams.Set("response-content-type", pfd.Properties.GetByName(container_module.PropertyMediaType)) + s, u, _, err := packages_service.GetPackageBlobStream(ctx, pfd.File, pfd.Blob, serveDirectReqParams) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index 4181577454..521ef2209a 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -217,7 +217,7 @@ func servePackageFile(ctx *context.Context, params parameters, serveContent bool return } - s, u, _, err := packages_service.GetPackageBlobStream(ctx, pf, pb) + s, u, _, err := packages_service.GetPackageBlobStream(ctx, pf, pb, nil) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 1fa44d50c4..50d2786ec8 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -214,7 +214,7 @@ func GetRawFileOrLFS(ctx *context.APIContext) { if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) + u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name(), nil) if u != nil && err == nil { ctx.Redirect(u.String()) return @@ -341,7 +341,7 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. rPath := archiver.RelativePath() if setting.RepoArchive.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.RepoArchives.URL(rPath, downloadName) + u, err := storage.RepoArchives.URL(rPath, downloadName, nil) if u != nil && err == nil { ctx.Redirect(u.String()) return diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index e2e784a671..2c4a65934d 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -202,7 +202,6 @@ func Search(ctx *context.APIContext) { } } - var err error repos, count, err := repo_model.SearchRepository(ctx, opts) if err != nil { ctx.JSON(http.StatusInternalServerError, api.SearchError{ diff --git a/routers/api/v1/user/repo.go b/routers/api/v1/user/repo.go index 86716ff44f..3b304c30b0 100644 --- a/routers/api/v1/user/repo.go +++ b/routers/api/v1/user/repo.go @@ -130,7 +130,6 @@ func ListMyRepos(ctx *context.APIContext) { return } - var err error repos, count, err := repo_model.SearchRepository(ctx, opts) if err != nil { ctx.Error(http.StatusInternalServerError, "SearchRepository", err) diff --git a/routers/web/base.go b/routers/web/base.go index 78dde57fa6..285d1ecddc 100644 --- a/routers/web/base.go +++ b/routers/web/base.go @@ -39,7 +39,7 @@ func storageHandler(storageSetting *setting.Storage, prefix string, objStore sto rPath := strings.TrimPrefix(req.URL.Path, "/"+prefix+"/") rPath = util.PathJoinRelX(rPath) - u, err := objStore.URL(rPath, path.Base(rPath)) + u, err := objStore.URL(rPath, path.Base(rPath), nil) if err != nil { if os.IsNotExist(err) || errors.Is(err, os.ErrNotExist) { log.Warn("Unable to find %s %s", prefix, rPath) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index bc1ecbfc1e..e7dbb6d975 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -307,7 +307,6 @@ func ViewPost(ctx *context_module.Context) { if validCursor { length := step.LogLength - cursor.Cursor offset := task.LogIndexes[index] - var err error logRows, err := actions.ReadLogs(ctx, task.LogInStorage, task.LogFilename, offset, length) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) @@ -689,7 +688,8 @@ func ArtifactsDownloadView(ctx *context_module.Context) { if len(artifacts) == 1 && artifacts[0].ArtifactName+".zip" == artifacts[0].ArtifactPath && artifacts[0].ContentEncoding == "application/zip" { art := artifacts[0] if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { - u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath) + u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath, nil) + if u != nil && err == nil { ctx.Redirect(u.String()) return diff --git a/routers/web/repo/activity.go b/routers/web/repo/activity.go index 954446be36..af9cea0f33 100644 --- a/routers/web/repo/activity.go +++ b/routers/web/repo/activity.go @@ -94,7 +94,6 @@ func ActivityAuthors(ctx *context.Context) { timeFrom = timeUntil.Add(-time.Hour * 168) } - var err error authors, err := activities_model.GetActivityStatsTopAuthors(ctx, ctx.Repo.Repository, timeFrom, 10) if err != nil { ctx.ServerError("GetActivityStatsTopAuthors", err) diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index b42effd8c3..b5078e1f63 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -134,7 +134,7 @@ func ServeAttachment(ctx *context.Context, uuid string) { if setting.Attachment.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name) + u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name, nil) if u != nil && err == nil { ctx.Redirect(u.String()) diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go index 0e5d1f0a1f..1428238074 100644 --- a/routers/web/repo/commit.go +++ b/routers/web/repo/commit.go @@ -338,6 +338,7 @@ func Diff(ctx *context.Context) { MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters, MaxFiles: maxFiles, WhitespaceBehavior: gitdiff.GetWhitespaceFlag(ctx.Data["WhitespaceBehavior"].(string)), + FileOnly: fileOnly, }, files...) if err != nil { ctx.NotFound("GetDiff", err) diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index 38d6004ec6..e5eab2bffa 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -611,6 +611,8 @@ func PrepareCompareDiff( maxLines, maxFiles = -1, -1 } + fileOnly := ctx.FormBool("file-only") + diff, err := gitdiff.GetDiff(ctx, ci.HeadGitRepo, &gitdiff.DiffOptions{ BeforeCommitID: beforeCommitID, @@ -621,6 +623,7 @@ func PrepareCompareDiff( MaxFiles: maxFiles, WhitespaceBehavior: whitespaceBehavior, DirectComparison: ci.DirectComparison, + FileOnly: fileOnly, }, ctx.FormStrings("files")...) if err != nil { ctx.ServerError("GetDiffRangeWithWhitespaceBehavior", err) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index c4a8baecca..1e87bbf015 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -54,8 +54,8 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim } if setting.LFS.Storage.MinioConfig.ServeDirect { - // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) + // If we have a signed url (S3, object storage, blob storage), redirect to this directly. + u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name(), nil) if u != nil && err == nil { ctx.Redirect(u.String()) return nil diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 9d0dcad61e..98dacc1a0d 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -614,12 +614,12 @@ func PrepareViewPullInfo(ctx *context.Context, issue *issues_model.Issue) *git.C var headBranchSha string // HeadRepo may be missing if pull.HeadRepo != nil { - headGitRepo, err := gitrepo.OpenRepository(ctx, pull.HeadRepo) + headGitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, pull.HeadRepo) if err != nil { - ctx.ServerError("OpenRepository", err) + ctx.ServerError("RepositoryFromContextOrOpen", err) return nil } - defer headGitRepo.Close() + defer closer.Close() if pull.Flow == issues_model.PullRequestFlowGithub { headBranchExist = headGitRepo.IsBranchExist(pull.HeadBranch) @@ -966,6 +966,7 @@ func viewPullFiles(ctx *context.Context, specifiedStartCommit, specifiedEndCommi MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters, MaxFiles: maxFiles, WhitespaceBehavior: gitdiff.GetWhitespaceFlag(ctx.Data["WhitespaceBehavior"].(string)), + FileOnly: fileOnly, } if !willShowSpecifiedCommit { diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 9562491440..8036bcae67 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -505,7 +505,7 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep rPath := archiver.RelativePath() if setting.RepoArchive.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.RepoArchives.URL(rPath, downloadName) + u, err := storage.RepoArchives.URL(rPath, downloadName, nil) if u != nil && err == nil { if archiver.ReleaseID != 0 { err = repo_model.CountArchiveDownload(ctx, ctx.Repo.Repository.ID, archiver.ReleaseID, archiver.Type) diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index f1445c580a..41ff5f97f7 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -147,7 +147,6 @@ func FindReadmeFileInEntries(ctx *context.Context, entries []*git.TreeEntry, try // this should be impossible; if subTreeEntry exists so should this. continue } - var err error childEntries, err := subTree.ListEntries() if err != nil { return "", nil, err diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 91b1f135c4..7d137fb214 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -379,18 +379,11 @@ func (diffFile *DiffFile) GetType() int { } // GetTailSection creates a fake DiffLineSection if the last section is not the end of the file -func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommitID, rightCommitID string) *DiffSection { +func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommit, rightCommit *git.Commit) *DiffSection { if len(diffFile.Sections) == 0 || diffFile.Type != DiffFileChange || diffFile.IsBin || diffFile.IsLFSFile { return nil } - leftCommit, err := gitRepo.GetCommit(leftCommitID) - if err != nil { - return nil - } - rightCommit, err := gitRepo.GetCommit(rightCommitID) - if err != nil { - return nil - } + lastSection := diffFile.Sections[len(diffFile.Sections)-1] lastLine := lastSection.Lines[len(lastSection.Lines)-1] leftLineCount := getCommitFileLineCount(leftCommit, diffFile.Name) @@ -532,11 +525,6 @@ parsingLoop: lastFile := createDiffFile(diff, line) diff.End = lastFile.Name diff.IsIncomplete = true - _, err := io.Copy(io.Discard, reader) - if err != nil { - // By the definition of io.Copy this never returns io.EOF - return diff, fmt.Errorf("error during io.Copy: %w", err) - } break parsingLoop } @@ -1097,6 +1085,7 @@ type DiffOptions struct { MaxFiles int WhitespaceBehavior git.TrustedCmdArgs DirectComparison bool + FileOnly bool } // GetDiff builds a Diff between two commits of a repository. @@ -1105,12 +1094,16 @@ type DiffOptions struct { func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) { repoPath := gitRepo.Path + var beforeCommit *git.Commit commit, err := gitRepo.GetCommit(opts.AfterCommitID) if err != nil { return nil, err } - cmdDiff := git.NewCommand(gitRepo.Ctx) + cmdCtx, cmdCancel := context.WithCancel(ctx) + defer cmdCancel() + + cmdDiff := git.NewCommand(cmdCtx) objectFormat, err := gitRepo.GetObjectFormat() if err != nil { return nil, err @@ -1132,6 +1125,12 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi AddArguments(opts.WhitespaceBehavior...). AddDynamicArguments(actualBeforeCommitID, opts.AfterCommitID) opts.BeforeCommitID = actualBeforeCommitID + + var err error + beforeCommit, err = gitRepo.GetCommit(opts.BeforeCommitID) + if err != nil { + return nil, err + } } // In git 2.31, git diff learned --skip-to which we can use to shortcut skip to file @@ -1166,7 +1165,9 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi _ = writer.Close() }() - diff, err := ParsePatch(ctx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile) + diff, err := ParsePatch(cmdCtx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile) + // Ensure the git process is killed if it didn't exit already + cmdCancel() if err != nil { return nil, fmt.Errorf("unable to ParsePatch: %w", err) } @@ -1207,37 +1208,28 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi diffFile.IsGenerated = analyze.IsGenerated(diffFile.Name) } - tailSection := diffFile.GetTailSection(gitRepo, opts.BeforeCommitID, opts.AfterCommitID) + tailSection := diffFile.GetTailSection(gitRepo, beforeCommit, commit) if tailSection != nil { diffFile.Sections = append(diffFile.Sections, tailSection) } } - separator := "..." - if opts.DirectComparison { - separator = ".." + if opts.FileOnly { + return diff, nil } - diffPaths := []string{opts.BeforeCommitID + separator + opts.AfterCommitID} - if len(opts.BeforeCommitID) == 0 || opts.BeforeCommitID == objectFormat.EmptyObjectID().String() { - diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID} - } - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // git >= 2.28 now returns an error if base and head have become unrelated. - // previously it would return the results of git diff --shortstat base head so let's try that... - diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID} - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) - } + stats, err := GetPullDiffStats(gitRepo, opts) if err != nil { return nil, err } + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion = stats.NumFiles, stats.TotalAddition, stats.TotalDeletion + return diff, nil } type PullDiffStats struct { - TotalAddition, TotalDeletion int + NumFiles, TotalAddition, TotalDeletion int } // GetPullDiffStats @@ -1261,12 +1253,12 @@ func GetPullDiffStats(gitRepo *git.Repository, opts *DiffOptions) (*PullDiffStat diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID} } - _, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) if err != nil && strings.Contains(err.Error(), "no merge base") { // git >= 2.28 now returns an error if base and head have become unrelated. // previously it would return the results of git diff --shortstat base head so let's try that... diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID} - _, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) } if err != nil { return nil, err diff --git a/services/issue/template.go b/services/issue/template.go index 47633e5d85..9a2b048401 100644 --- a/services/issue/template.go +++ b/services/issue/template.go @@ -56,8 +56,6 @@ func GetTemplateConfig(gitRepo *git.Repository, path string, commit *git.Commit) return GetDefaultTemplateConfig(), nil } - var err error - treeEntry, err := commit.GetTreeEntryByPath(path) if err != nil { return GetDefaultTemplateConfig(), err diff --git a/services/lfs/server.go b/services/lfs/server.go index a300de19c4..51d6f42776 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -192,6 +192,11 @@ func BatchHandler(ctx *context.Context) { } } + if setting.LFS.MaxBatchSize != 0 && len(br.Objects) > setting.LFS.MaxBatchSize { + writeStatus(ctx, http.StatusRequestEntityTooLarge) + return + } + contentStore := lfs_module.NewContentStore() var responseObjects []*lfs_module.ObjectResponse @@ -480,7 +485,7 @@ func buildObjectResponse(rc *requestContext, pointer lfs_module.Pointer, downloa var link *lfs_module.Link if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid) + u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid, nil) if u != nil && err == nil { // Presigned url does not need the Authorization header // https://github.com/go-gitea/gitea/issues/21525 diff --git a/services/packages/packages.go b/services/packages/packages.go index a5b84506de..72ab19ee27 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -602,12 +602,12 @@ func GetPackageFileStream(ctx context.Context, pf *packages_model.PackageFile) ( return nil, nil, nil, err } - return GetPackageBlobStream(ctx, pf, pb) + return GetPackageBlobStream(ctx, pf, pb, nil) } // GetPackageBlobStream returns the content of the specific package blob // If the storage supports direct serving and it's enabled, only the direct serving url is returned. -func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { +func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { key := packages_module.BlobHash256Key(pb.HashSHA256) cs := packages_module.NewContentStore() @@ -617,7 +617,7 @@ func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, p var err error if cs.ShouldServeDirect() { - u, err = cs.GetServeDirectURL(key, pf.Name) + u, err = cs.GetServeDirectURL(key, pf.Name, serveDirectReqParams) if err != nil && !errors.Is(err, storage.ErrURLNotSupported) { log.Error("Error getting serve direct url: %v", err) } diff --git a/templates/org/projects/view.tmpl b/templates/org/projects/view.tmpl index e1ab81c4cd..bd74114fe2 100644 --- a/templates/org/projects/view.tmpl +++ b/templates/org/projects/view.tmpl @@ -1,9 +1,13 @@ {{template "base/head" .}} -