From 4a5af4edca3883bcb45056de11b9fd59f9759b9b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 9 Apr 2025 09:34:38 -0700 Subject: [PATCH 1/2] Cache GPG keys, emails and users when list commits (#34086) When list commits, some of the commits authors are the same at many situations. But current logic will always fetch the same GPG keys from database. This PR will cache the GPG keys, emails and users for the context so that reducing the database queries. --------- Co-authored-by: wxiaoguang --- models/asymkey/gpg_key.go | 7 +++++++ models/user/user.go | 27 +++++++++++++-------------- modules/cache/context.go | 8 ++++---- modules/cache/context_test.go | 3 ++- modules/cachegroup/cachegroup.go | 12 ++++++++++++ modules/repository/commits.go | 3 ++- routers/private/hook_post_receive.go | 5 ++--- services/asymkey/commit.go | 25 ++++++++++--------------- services/convert/pull.go | 13 +++++++------ services/git/commit.go | 10 ++++++---- 10 files changed, 65 insertions(+), 48 deletions(-) create mode 100644 modules/cachegroup/cachegroup.go diff --git a/models/asymkey/gpg_key.go b/models/asymkey/gpg_key.go index 24c76f7b5c..220f46ad1d 100644 --- a/models/asymkey/gpg_key.go +++ b/models/asymkey/gpg_key.go @@ -240,3 +240,10 @@ func DeleteGPGKey(ctx context.Context, doer *user_model.User, id int64) (err err return committer.Commit() } + +func FindGPGKeyWithSubKeys(ctx context.Context, keyID string) ([]*GPGKey, error) { + return db.Find[GPGKey](ctx, FindGPGKeyOptions{ + KeyID: keyID, + IncludeSubKeys: true, + }) +} diff --git a/models/user/user.go b/models/user/user.go index 5989be74f0..100f924cc6 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -1187,29 +1187,28 @@ func GetUsersByEmails(ctx context.Context, emails []string) (map[string]*User, e for _, email := range emailAddresses { userIDs.Add(email.UID) } - users, err := GetUsersMapByIDs(ctx, userIDs.Values()) - if err != nil { - return nil, err - } - results := make(map[string]*User, len(emails)) - for _, email := range emailAddresses { - user := users[email.UID] - if user != nil { - if user.KeepEmailPrivate { - results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user - } else { - results[email.Email] = user + + if len(userIDs) > 0 { + users, err := GetUsersMapByIDs(ctx, userIDs.Values()) + if err != nil { + return nil, err + } + + for _, email := range emailAddresses { + user := users[email.UID] + if user != nil { + results[user.GetEmail()] = user } } } - users = make(map[int64]*User, len(needCheckUserNames)) + users := make(map[int64]*User, len(needCheckUserNames)) if err := db.GetEngine(ctx).In("lower_name", needCheckUserNames.Values()).Find(&users); err != nil { return nil, err } for _, user := range users { - results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user + results[user.GetPlaceholderEmail()] = user } return results, nil } diff --git a/modules/cache/context.go b/modules/cache/context.go index 484cee659a..85eb9e6790 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -166,15 +166,15 @@ func RemoveContextData(ctx context.Context, tp, key any) { } // GetWithContextCache returns the cache value of the given key in the given context. -func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) { - v := GetContextData(ctx, cacheGroupKey, cacheTargetID) +func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + v := GetContextData(ctx, groupKey, targetKey) if vv, ok := v.(T); ok { return vv, nil } - t, err := f() + t, err := f(ctx, targetKey) if err != nil { return t, err } - SetContextData(ctx, cacheGroupKey, cacheTargetID, t) + SetContextData(ctx, groupKey, targetKey, t) return t, nil } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index decb532937..23dd789dbc 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -4,6 +4,7 @@ package cache import ( + "context" "testing" "time" @@ -30,7 +31,7 @@ func TestWithCacheContext(t *testing.T) { v = GetContextData(ctx, field, "my_config1") assert.Nil(t, v) - vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) { + vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) { return 1, nil }) assert.NoError(t, err) diff --git a/modules/cachegroup/cachegroup.go b/modules/cachegroup/cachegroup.go new file mode 100644 index 0000000000..06085f860f --- /dev/null +++ b/modules/cachegroup/cachegroup.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cachegroup + +const ( + User = "user" + EmailAvatarLink = "email_avatar_link" + UserEmailAddresses = "user_email_addresses" + GPGKeyWithSubKeys = "gpg_key_with_subkeys" + RepoUserPermission = "repo_user_permission" +) diff --git a/modules/repository/commits.go b/modules/repository/commits.go index 16520fb28a..878fdc1603 100644 --- a/modules/repository/commits.go +++ b/modules/repository/commits.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -131,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string { size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor - v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) { + v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) { u, err := user_model.GetUserByEmail(ctx, email) if err != nil { if !user_model.IsErrUserNotExist(err) { diff --git a/routers/private/hook_post_receive.go b/routers/private/hook_post_receive.go index 442d0a76c9..8b1e849e7a 100644 --- a/routers/private/hook_post_receive.go +++ b/routers/private/hook_post_receive.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" @@ -326,9 +327,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { } func loadContextCacheUser(ctx context.Context, id int64) (*user_model.User, error) { - return cache.GetWithContextCache(ctx, "hook_post_receive_user", id, func() (*user_model.User, error) { - return user_model.GetUserByID(ctx, id) - }) + return cache.GetWithContextCache(ctx, cachegroup.User, id, user_model.GetUserByID) } // handlePullRequestMerging handle pull request merging, a pull request action should push at least 1 commit diff --git a/services/asymkey/commit.go b/services/asymkey/commit.go index 5d85be56f1..105782a93a 100644 --- a/services/asymkey/commit.go +++ b/services/asymkey/commit.go @@ -11,6 +11,8 @@ import ( asymkey_model "code.gitea.io/gitea/models/asymkey" "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -115,7 +117,7 @@ func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, commi } } - committerEmailAddresses, _ := user_model.GetEmailAddresses(ctx, committer.ID) + committerEmailAddresses, _ := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses) activated := false for _, e := range committerEmailAddresses { if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) { @@ -209,10 +211,9 @@ func checkKeyEmails(ctx context.Context, email string, keys ...*asymkey_model.GP } if key.Verified && key.OwnerID != 0 { if uid != key.OwnerID { - userEmails, _ = user_model.GetEmailAddresses(ctx, key.OwnerID) + userEmails, _ = cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, key.OwnerID, user_model.GetEmailAddresses) uid = key.OwnerID - user = &user_model.User{ID: uid} - _, _ = user_model.GetUser(ctx, user) + user, _ = cache.GetWithContextCache(ctx, cachegroup.User, uid, user_model.GetUserByID) } for _, e := range userEmails { if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) { @@ -231,10 +232,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s if keyID == "" { return nil } - keys, err := db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{ - KeyID: keyID, - IncludeSubKeys: true, - }) + keys, err := cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, keyID, asymkey_model.FindGPGKeyWithSubKeys) if err != nil { log.Error("GetGPGKeysByKeyID: %v", err) return &asymkey_model.CommitVerification{ @@ -249,10 +247,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s for _, key := range keys { var primaryKeys []*asymkey_model.GPGKey if key.PrimaryKeyID != "" { - primaryKeys, err = db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{ - KeyID: key.PrimaryKeyID, - IncludeSubKeys: true, - }) + primaryKeys, err = cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, key.PrimaryKeyID, asymkey_model.FindGPGKeyWithSubKeys) if err != nil { log.Error("GetGPGKeysByKeyID: %v", err) return &asymkey_model.CommitVerification{ @@ -272,8 +267,8 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s Name: name, Email: email, } - if key.OwnerID != 0 { - owner, err := user_model.GetUserByID(ctx, key.OwnerID) + if key.OwnerID > 0 { + owner, err := cache.GetWithContextCache(ctx, cachegroup.User, key.OwnerID, user_model.GetUserByID) if err == nil { signer = owner } else if !user_model.IsErrUserNotExist(err) { @@ -381,7 +376,7 @@ func ParseCommitWithSSHSignature(ctx context.Context, c *git.Commit, committer * } } - committerEmailAddresses, err := user_model.GetEmailAddresses(ctx, committer.ID) + committerEmailAddresses, err := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses) if err != nil { log.Error("GetEmailAddresses: %v", err) } diff --git a/services/convert/pull.go b/services/convert/pull.go index 34c3b1bf9a..7798bebb08 100644 --- a/services/convert/pull.go +++ b/services/convert/pull.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" @@ -60,14 +61,14 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u doerID = doer.ID } - const repoDoerPermCacheKey = "repo_doer_perm_cache" - p, err := cache.GetWithContextCache(ctx, repoDoerPermCacheKey, fmt.Sprintf("%d_%d", pr.BaseRepoID, doerID), - func() (access_model.Permission, error) { + repoUserPerm, err := cache.GetWithContextCache(ctx, cachegroup.RepoUserPermission, fmt.Sprintf("%d-%d", pr.BaseRepoID, doerID), + func(ctx context.Context, _ string) (access_model.Permission, error) { return access_model.GetUserRepoPermission(ctx, pr.BaseRepo, doer) - }) + }, + ) if err != nil { log.Error("GetUserRepoPermission[%d]: %v", pr.BaseRepoID, err) - p.AccessMode = perm.AccessModeNone + repoUserPerm.AccessMode = perm.AccessModeNone } apiPullRequest := &api.PullRequest{ @@ -107,7 +108,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u Name: pr.BaseBranch, Ref: pr.BaseBranch, RepoID: pr.BaseRepoID, - Repository: ToRepo(ctx, pr.BaseRepo, p), + Repository: ToRepo(ctx, pr.BaseRepo, repoUserPerm), }, Head: &api.PRBranchInfo{ Name: pr.HeadBranch, diff --git a/services/git/commit.go b/services/git/commit.go index 8ab8f3d369..3faef76782 100644 --- a/services/git/commit.go +++ b/services/git/commit.go @@ -17,7 +17,7 @@ import ( ) // ParseCommitsWithSignature checks if signaute of commits are corresponding to users gpg keys. -func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType, isOwnerMemberCollaborator func(*user_model.User) (bool, error)) ([]*asymkey_model.SignCommit, error) { +func ParseCommitsWithSignature(ctx context.Context, repo *repo_model.Repository, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType) ([]*asymkey_model.SignCommit, error) { newCommits := make([]*asymkey_model.SignCommit, 0, len(oldCommits)) keyMap := map[string]bool{} @@ -47,6 +47,10 @@ func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.Use Verification: asymkey_service.ParseCommitWithSignatureCommitter(ctx, c.Commit, committer), } + isOwnerMemberCollaborator := func(user *user_model.User) (bool, error) { + return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID) + } + _ = asymkey_model.CalculateTrustStatus(signCommit.Verification, repoTrustModel, isOwnerMemberCollaborator, &keyMap) newCommits = append(newCommits, signCommit) @@ -62,11 +66,9 @@ func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo } signedCommits, err := ParseCommitsWithSignature( ctx, + repo, validatedCommits, repo.GetTrustModel(), - func(user *user_model.User) (bool, error) { - return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID) - }, ) if err != nil { return nil, err From fac6b87dd24be5021d9c656edc2072397cfd6bed Mon Sep 17 00:00:00 2001 From: ManInDark <61268856+ManInDark@users.noreply.github.com> Date: Wed, 9 Apr 2025 21:21:54 +0200 Subject: [PATCH 2/2] bugfix check for alternate ssh host certificate location (#34146) fixes #34145 Edited all locations to actually be correct. --------- Co-authored-by: wxiaoguang Co-authored-by: Giteabot --- docker/root/etc/s6/openssh/setup | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/root/etc/s6/openssh/setup b/docker/root/etc/s6/openssh/setup index 6fbc599cc5..48e7d4b211 100755 --- a/docker/root/etc/s6/openssh/setup +++ b/docker/root/etc/s6/openssh/setup @@ -31,16 +31,19 @@ if [ -e /data/ssh/ssh_host_ecdsa_cert ]; then SSH_ECDSA_CERT=${SSH_ECDSA_CERT:-"/data/ssh/ssh_host_ecdsa_cert"} fi -if [ -e /data/ssh/ssh_host_ed25519-cert.pub ]; then - SSH_ED25519_CERT=${SSH_ED25519_CERT:-"/data/ssh/ssh_host_ed25519-cert.pub"} +# In case someone wants to sign the `{keyname}.pub` key by `ssh-keygen -s ca -I identity ...` to +# make use of the ssh-key certificate authority feature (see ssh-keygen CERTIFICATES section), +# the generated key file name is `{keyname}-cert.pub` +if [ -e /data/ssh/ssh_host_ed25519_key-cert.pub ]; then + SSH_ED25519_CERT=${SSH_ED25519_CERT:-"/data/ssh/ssh_host_ed25519_key-cert.pub"} fi -if [ -e /data/ssh/ssh_host_rsa-cert.pub ]; then - SSH_RSA_CERT=${SSH_RSA_CERT:-"/data/ssh/ssh_host_rsa-cert.pub"} +if [ -e /data/ssh/ssh_host_rsa_key-cert.pub ]; then + SSH_RSA_CERT=${SSH_RSA_CERT:-"/data/ssh/ssh_host_rsa_key-cert.pub"} fi -if [ -e /data/ssh/ssh_host_ecdsa-cert.pub ]; then - SSH_ECDSA_CERT=${SSH_ECDSA_CERT:-"/data/ssh/ssh_host_ecdsa-cert.pub"} +if [ -e /data/ssh/ssh_host_ecdsa_key-cert.pub ]; then + SSH_ECDSA_CERT=${SSH_ECDSA_CERT:-"/data/ssh/ssh_host_ecdsa_key-cert.pub"} fi if [ -d /etc/ssh ]; then