diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index 2264c9e822..f459e3910d 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -59,6 +59,8 @@ jobs: aws s3 sync dist/release s3://${{ secrets.AWS_S3_BUCKET }}/gitea/${{ steps.clean_name.outputs.branch }} --no-progress nightly-docker-rootful: runs-on: namespace-profile-gitea-release-docker + permissions: + packages: write # to publish to ghcr.io steps: - uses: actions/checkout@v4 # fetch all commits instead of only the last as some branches are long lived and could have many between versions @@ -85,6 +87,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GHCR using PAT + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: fetch go modules run: make vendor - name: build rootful docker image @@ -93,9 +101,13 @@ jobs: context: . platforms: linux/amd64,linux/arm64 push: true - tags: gitea/gitea:${{ steps.clean_name.outputs.branch }} + tags: |- + gitea/gitea:${{ steps.clean_name.outputs.branch }} + ghcr.io/go-gitea/gitea:${{ steps.clean_name.outputs.branch }} nightly-docker-rootless: runs-on: namespace-profile-gitea-release-docker + permissions: + packages: write # to publish to ghcr.io steps: - uses: actions/checkout@v4 # fetch all commits instead of only the last as some branches are long lived and could have many between versions @@ -122,6 +134,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GHCR using PAT + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: fetch go modules run: make vendor - name: build rootless docker image @@ -131,4 +149,6 @@ jobs: platforms: linux/amd64,linux/arm64 push: true file: Dockerfile.rootless - tags: gitea/gitea:${{ steps.clean_name.outputs.branch }}-rootless + tags: |- + gitea/gitea:${{ steps.clean_name.outputs.branch }}-rootless + ghcr.io/go-gitea/gitea:${{ steps.clean_name.outputs.branch }}-rootless diff --git a/.github/workflows/release-tag-rc.yml b/.github/workflows/release-tag-rc.yml index a406602dc0..02da6d1eab 100644 --- a/.github/workflows/release-tag-rc.yml +++ b/.github/workflows/release-tag-rc.yml @@ -69,6 +69,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} docker-rootful: runs-on: namespace-profile-gitea-release-docker + permissions: + packages: write # to publish to ghcr.io steps: - uses: actions/checkout@v4 # fetch all commits instead of only the last as some branches are long lived and could have many between versions @@ -79,7 +81,9 @@ jobs: - uses: docker/metadata-action@v5 id: meta with: - images: gitea/gitea + images: |- + gitea/gitea + ghcr.io/go-gitea/gitea flavor: | latest=false # 1.2.3-rc0 @@ -90,6 +94,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GHCR using PAT + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: build rootful docker image uses: docker/build-push-action@v5 with: @@ -100,6 +110,8 @@ jobs: labels: ${{ steps.meta.outputs.labels }} docker-rootless: runs-on: namespace-profile-gitea-release-docker + permissions: + packages: write # to publish to ghcr.io steps: - uses: actions/checkout@v4 # fetch all commits instead of only the last as some branches are long lived and could have many between versions @@ -110,7 +122,9 @@ jobs: - uses: docker/metadata-action@v5 id: meta with: - images: gitea/gitea + images: |- + gitea/gitea + ghcr.io/go-gitea/gitea # each tag below will have the suffix of -rootless flavor: | latest=false @@ -123,6 +137,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GHCR using PAT + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: build rootless docker image uses: docker/build-push-action@v5 with: diff --git a/.github/workflows/release-tag-version.yml b/.github/workflows/release-tag-version.yml index 08bb9baecf..158945b615 100644 --- a/.github/workflows/release-tag-version.yml +++ b/.github/workflows/release-tag-version.yml @@ -14,6 +14,8 @@ concurrency: jobs: binary: runs-on: namespace-profile-gitea-release-binary + permissions: + packages: write # to publish to ghcr.io steps: - uses: actions/checkout@v4 # fetch all commits instead of only the last as some branches are long lived and could have many between versions @@ -71,6 +73,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} docker-rootful: runs-on: namespace-profile-gitea-release-docker + permissions: + packages: write # to publish to ghcr.io steps: - uses: actions/checkout@v4 # fetch all commits instead of only the last as some branches are long lived and could have many between versions @@ -81,7 +85,9 @@ jobs: - uses: docker/metadata-action@v5 id: meta with: - images: gitea/gitea + images: |- + gitea/gitea + ghcr.io/go-gitea/gitea # this will generate tags in the following format: # latest # 1 @@ -96,6 +102,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GHCR using PAT + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: build rootful docker image uses: docker/build-push-action@v5 with: @@ -116,7 +128,9 @@ jobs: - uses: docker/metadata-action@v5 id: meta with: - images: gitea/gitea + images: |- + gitea/gitea + ghcr.io/go-gitea/gitea # each tag below will have the suffix of -rootless flavor: | suffix=-rootless,onlatest=true @@ -134,6 +148,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GHCR using PAT + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: build rootless docker image uses: docker/build-push-action@v5 with: diff --git a/cmd/web.go b/cmd/web.go index dc5c6de48a..e47b171455 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -213,6 +213,10 @@ func serveInstalled(ctx *cli.Context) error { log.Fatal("Can not find APP_DATA_PATH %q", setting.AppDataPath) } + // the AppDataTempDir is fully managed by us with a safe sub-path + // so it's safe to automatically remove the outdated files + setting.AppDataTempDir("").RemoveOutdated(3 * 24 * time.Hour) + // Override the provided port number within the configuration if ctx.IsSet("port") { if err := setPort(ctx.String("port")); err != nil { diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 07a6ebdcf2..8d39551168 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -197,13 +197,6 @@ RUN_USER = ; git ;; relative paths are made absolute relative to the APP_DATA_PATH ;SSH_SERVER_HOST_KEYS=ssh/gitea.rsa, ssh/gogs.rsa ;; -;; Directory to create temporary files in when testing public keys using ssh-keygen, -;; default is the system temporary directory. -;SSH_KEY_TEST_PATH = -;; -;; Use `ssh-keygen` to parse public SSH keys. The value is passed to the shell. By default, Gitea does the parsing itself. -;SSH_KEYGEN_PATH = -;; ;; Enable SSH Authorized Key Backup when rewriting all keys, default is false ;SSH_AUTHORIZED_KEYS_BACKUP = false ;; @@ -294,6 +287,9 @@ RUN_USER = ; git ;; Default path for App data ;APP_DATA_PATH = data ; relative paths will be made absolute with _`AppWorkPath`_ ;; +;; Base path for App's temp files, leave empty to use the managed tmp directory in APP_DATA_PATH +;APP_TEMP_PATH = +;; ;; Enable gzip compression for runtime-generated content, static resources excluded ;ENABLE_GZIP = false ;; @@ -1069,15 +1065,6 @@ LEVEL = Info ;; Separate extensions with a comma. To line wrap files without an extension, just put a comma ;LINE_WRAP_EXTENSIONS = .txt,.md,.markdown,.mdown,.mkd,.livemd, -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;[repository.local] -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; -;; Path for local repository copy. Defaults to `tmp/local-repo` (content gets deleted on gitea restart) -;LOCAL_COPY_PATH = tmp/local-repo - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;[repository.upload] @@ -1087,9 +1074,6 @@ LEVEL = Info ;; Whether repository file uploads are enabled. Defaults to `true` ;ENABLED = true ;; -;; Path for uploads. Defaults to `data/tmp/uploads` (content gets deleted on gitea restart) -;TEMP_PATH = data/tmp/uploads -;; ;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. ;ALLOWED_TYPES = ;; @@ -2473,7 +2457,7 @@ LEVEL = Info ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Set the maximum number of characters in a mermaid source. (Set to -1 to disable limits) -;MERMAID_MAX_SOURCE_CHARACTERS = 5000 +;MERMAID_MAX_SOURCE_CHARACTERS = 50000 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2594,9 +2578,6 @@ LEVEL = Info ;; Currently, only `minio` and `azureblob` is supported. ;SERVE_DIRECT = false ;; -;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` -;CHUNKED_UPLOAD_PATH = tmp/package-upload -;; ;; Maximum count of package versions a single owner can have (`-1` means no limits) ;LIMIT_TOTAL_OWNER_COUNT = -1 ;; Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docker/root/etc/s6/openssh/setup b/docker/root/etc/s6/openssh/setup index 6fbc599cc5..48e7d4b211 100755 --- a/docker/root/etc/s6/openssh/setup +++ b/docker/root/etc/s6/openssh/setup @@ -31,16 +31,19 @@ if [ -e /data/ssh/ssh_host_ecdsa_cert ]; then SSH_ECDSA_CERT=${SSH_ECDSA_CERT:-"/data/ssh/ssh_host_ecdsa_cert"} fi -if [ -e /data/ssh/ssh_host_ed25519-cert.pub ]; then - SSH_ED25519_CERT=${SSH_ED25519_CERT:-"/data/ssh/ssh_host_ed25519-cert.pub"} +# In case someone wants to sign the `{keyname}.pub` key by `ssh-keygen -s ca -I identity ...` to +# make use of the ssh-key certificate authority feature (see ssh-keygen CERTIFICATES section), +# the generated key file name is `{keyname}-cert.pub` +if [ -e /data/ssh/ssh_host_ed25519_key-cert.pub ]; then + SSH_ED25519_CERT=${SSH_ED25519_CERT:-"/data/ssh/ssh_host_ed25519_key-cert.pub"} fi -if [ -e /data/ssh/ssh_host_rsa-cert.pub ]; then - SSH_RSA_CERT=${SSH_RSA_CERT:-"/data/ssh/ssh_host_rsa-cert.pub"} +if [ -e /data/ssh/ssh_host_rsa_key-cert.pub ]; then + SSH_RSA_CERT=${SSH_RSA_CERT:-"/data/ssh/ssh_host_rsa_key-cert.pub"} fi -if [ -e /data/ssh/ssh_host_ecdsa-cert.pub ]; then - SSH_ECDSA_CERT=${SSH_ECDSA_CERT:-"/data/ssh/ssh_host_ecdsa-cert.pub"} +if [ -e /data/ssh/ssh_host_ecdsa_key-cert.pub ]; then + SSH_ECDSA_CERT=${SSH_ECDSA_CERT:-"/data/ssh/ssh_host_ecdsa_key-cert.pub"} fi if [ -d /etc/ssh ]; then diff --git a/models/asymkey/gpg_key.go b/models/asymkey/gpg_key.go index 24c76f7b5c..220f46ad1d 100644 --- a/models/asymkey/gpg_key.go +++ b/models/asymkey/gpg_key.go @@ -240,3 +240,10 @@ func DeleteGPGKey(ctx context.Context, doer *user_model.User, id int64) (err err return committer.Commit() } + +func FindGPGKeyWithSubKeys(ctx context.Context, keyID string) ([]*GPGKey, error) { + return db.Find[GPGKey](ctx, FindGPGKeyOptions{ + KeyID: keyID, + IncludeSubKeys: true, + }) +} diff --git a/models/asymkey/ssh_key_fingerprint.go b/models/asymkey/ssh_key_fingerprint.go index 1ed3b5df2a..4dcfe1f279 100644 --- a/models/asymkey/ssh_key_fingerprint.go +++ b/models/asymkey/ssh_key_fingerprint.go @@ -6,27 +6,13 @@ package asymkey import ( "context" "fmt" - "strings" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "golang.org/x/crypto/ssh" "xorm.io/builder" ) -// ___________.__ .__ __ -// \_ _____/|__| ____ ____ ________________________|__| _____/ |_ -// | __) | |/ \ / ___\_/ __ \_ __ \____ \_ __ \ |/ \ __\ -// | \ | | | \/ /_/ > ___/| | \/ |_> > | \/ | | \ | -// \___ / |__|___| /\___ / \___ >__| | __/|__| |__|___| /__| -// \/ \//_____/ \/ |__| \/ -// -// This file contains functions for fingerprinting SSH keys -// // The database is used in checkKeyFingerprint however most of these functions probably belong in a module // checkKeyFingerprint only checks if key fingerprint has been used as public key, @@ -41,29 +27,6 @@ func checkKeyFingerprint(ctx context.Context, fingerprint string) error { return nil } -func calcFingerprintSSHKeygen(publicKeyContent string) (string, error) { - // Calculate fingerprint. - tmpPath, err := writeTmpKeyFile(publicKeyContent) - if err != nil { - return "", err - } - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary key file: %s: Error: %v", tmpPath, err) - } - }() - stdout, stderr, err := process.GetManager().Exec("AddPublicKey", "ssh-keygen", "-lf", tmpPath) - if err != nil { - if strings.Contains(stderr, "is not a public key file") { - return "", ErrKeyUnableVerify{stderr} - } - return "", util.NewInvalidArgumentErrorf("'ssh-keygen -lf %s' failed with error '%s': %s", tmpPath, err, stderr) - } else if len(stdout) < 2 { - return "", util.NewInvalidArgumentErrorf("not enough output for calculating fingerprint: %s", stdout) - } - return strings.Split(stdout, " ")[1], nil -} - func calcFingerprintNative(publicKeyContent string) (string, error) { // Calculate fingerprint. pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKeyContent)) @@ -75,15 +38,12 @@ func calcFingerprintNative(publicKeyContent string) (string, error) { // CalcFingerprint calculate public key's fingerprint func CalcFingerprint(publicKeyContent string) (string, error) { - // Call the method based on configuration - useNative := setting.SSH.KeygenPath == "" - calcFn := util.Iif(useNative, calcFingerprintNative, calcFingerprintSSHKeygen) - fp, err := calcFn(publicKeyContent) + fp, err := calcFingerprintNative(publicKeyContent) if err != nil { if IsErrKeyUnableVerify(err) { return "", err } - return "", fmt.Errorf("CalcFingerprint(%s): %w", util.Iif(useNative, "native", "ssh-keygen"), err) + return "", fmt.Errorf("CalcFingerprint: %w", err) } return fp, nil } diff --git a/models/asymkey/ssh_key_parse.go b/models/asymkey/ssh_key_parse.go index c843525718..46dcf4d894 100644 --- a/models/asymkey/ssh_key_parse.go +++ b/models/asymkey/ssh_key_parse.go @@ -13,12 +13,9 @@ import ( "errors" "fmt" "math/big" - "os" - "strconv" "strings" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -175,20 +172,9 @@ func CheckPublicKeyString(content string) (_ string, err error) { return content, nil } - var ( - fnName string - keyType string - length int - ) - if len(setting.SSH.KeygenPath) == 0 { - fnName = "SSHNativeParsePublicKey" - keyType, length, err = SSHNativeParsePublicKey(content) - } else { - fnName = "SSHKeyGenParsePublicKey" - keyType, length, err = SSHKeyGenParsePublicKey(content) - } + keyType, length, err := SSHNativeParsePublicKey(content) if err != nil { - return "", fmt.Errorf("%s: %w", fnName, err) + return "", fmt.Errorf("SSHNativeParsePublicKey: %w", err) } log.Trace("Key info [native: %v]: %s-%d", setting.SSH.StartBuiltinServer, keyType, length) @@ -258,56 +244,3 @@ func SSHNativeParsePublicKey(keyLine string) (string, int, error) { } return "", 0, fmt.Errorf("unsupported key length detection for type: %s", pkey.Type()) } - -// writeTmpKeyFile writes key content to a temporary file -// and returns the name of that file, along with any possible errors. -func writeTmpKeyFile(content string) (string, error) { - tmpFile, err := os.CreateTemp(setting.SSH.KeyTestPath, "gitea_keytest") - if err != nil { - return "", fmt.Errorf("TempFile: %w", err) - } - defer tmpFile.Close() - - if _, err = tmpFile.WriteString(content); err != nil { - return "", fmt.Errorf("WriteString: %w", err) - } - return tmpFile.Name(), nil -} - -// SSHKeyGenParsePublicKey extracts key type and length using ssh-keygen. -func SSHKeyGenParsePublicKey(key string) (string, int, error) { - tmpName, err := writeTmpKeyFile(key) - if err != nil { - return "", 0, fmt.Errorf("writeTmpKeyFile: %w", err) - } - defer func() { - if err := util.Remove(tmpName); err != nil { - log.Warn("Unable to remove temporary key file: %s: Error: %v", tmpName, err) - } - }() - - keygenPath := setting.SSH.KeygenPath - if len(keygenPath) == 0 { - keygenPath = "ssh-keygen" - } - - stdout, stderr, err := process.GetManager().Exec("SSHKeyGenParsePublicKey", keygenPath, "-lf", tmpName) - if err != nil { - return "", 0, fmt.Errorf("fail to parse public key: %s - %s", err, stderr) - } - if strings.Contains(stdout, "is not a public key file") { - return "", 0, ErrKeyUnableVerify{stdout} - } - - fields := strings.Split(stdout, " ") - if len(fields) < 4 { - return "", 0, fmt.Errorf("invalid public key line: %s", stdout) - } - - keyType := strings.Trim(fields[len(fields)-1], "()\r\n") - length, err := strconv.ParseInt(fields[0], 10, 32) - if err != nil { - return "", 0, err - } - return strings.ToLower(keyType), int(length), nil -} diff --git a/models/asymkey/ssh_key_test.go b/models/asymkey/ssh_key_test.go index b33d16030d..21e4ddf62e 100644 --- a/models/asymkey/ssh_key_test.go +++ b/models/asymkey/ssh_key_test.go @@ -18,7 +18,6 @@ import ( "github.com/42wim/sshsig" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func Test_SSHParsePublicKey(t *testing.T) { @@ -45,27 +44,6 @@ func Test_SSHParsePublicKey(t *testing.T) { assert.Equal(t, tc.keyType, keyTypeN) assert.Equal(t, tc.length, lengthN) }) - if tc.skipSSHKeygen { - return - } - t.Run("SSHKeygen", func(t *testing.T) { - keyTypeK, lengthK, err := SSHKeyGenParsePublicKey(tc.content) - if err != nil { - // Some servers do not support ecdsa format. - if !strings.Contains(err.Error(), "line 1 too long:") { - require.NoError(t, err) - } - } - assert.Equal(t, tc.keyType, keyTypeK) - assert.Equal(t, tc.length, lengthK) - }) - t.Run("SSHParseKeyNative", func(t *testing.T) { - keyTypeK, lengthK, err := SSHNativeParsePublicKey(tc.content) - require.NoError(t, err) - - assert.Equal(t, tc.keyType, keyTypeK) - assert.Equal(t, tc.length, lengthK) - }) }) } } @@ -186,14 +164,6 @@ func Test_calcFingerprint(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tc.fp, fpN) }) - if tc.skipSSHKeygen { - return - } - t.Run("SSHKeygen", func(t *testing.T) { - fpK, err := calcFingerprintSSHKeygen(tc.content) - assert.NoError(t, err) - assert.Equal(t, tc.fp, fpK) - }) }) } } diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 737b69f154..f9e1fbeb14 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -21,6 +21,8 @@ import ( "xorm.io/xorm" ) +const ScopeSortPrefix = "scope-" + // IssuesOptions represents options of an issue. type IssuesOptions struct { //nolint Paginator *db.ListOptions @@ -70,6 +72,17 @@ func (o *IssuesOptions) Copy(edit ...func(options *IssuesOptions)) *IssuesOption // applySorts sort an issues-related session based on the provided // sortType string func applySorts(sess *xorm.Session, sortType string, priorityRepoID int64) { + // Since this sortType is dynamically created, it has to be treated specially. + if strings.HasPrefix(sortType, ScopeSortPrefix) { + scope := strings.TrimPrefix(sortType, ScopeSortPrefix) + sess.Join("LEFT", "issue_label", "issue.id = issue_label.issue_id") + // "exclusive_order=0" means "no order is set", so exclude it from the JOIN criteria and then "LEFT JOIN" result is also null + sess.Join("LEFT", "label", "label.id = issue_label.label_id AND label.exclusive_order <> 0 AND label.name LIKE ?", scope+"/%") + // Use COALESCE to make sure we sort NULL last regardless of backend DB (2147483647 == max int) + sess.OrderBy("COALESCE(label.exclusive_order, 2147483647) ASC").Desc("issue.id") + return + } + switch sortType { case "oldest": sess.Asc("issue.created_unix").Asc("issue.id") diff --git a/models/issues/issue_update.go b/models/issues/issue_update.go index 5e9c012c05..7ddf7ee901 100644 --- a/models/issues/issue_update.go +++ b/models/issues/issue_update.go @@ -845,6 +845,7 @@ func DeleteOrphanedIssues(ctx context.Context) error { // Remove issue attachment files. for i := range attachmentPaths { + // FIXME: it's not right, because the attachment might not be on local filesystem system_model.RemoveAllWithNotice(ctx, "Delete issue attachment", attachmentPaths[i]) } return nil diff --git a/models/issues/label.go b/models/issues/label.go index 8a5d9321cc..cfbe100926 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -87,6 +87,7 @@ type Label struct { OrgID int64 `xorm:"INDEX"` Name string Exclusive bool + ExclusiveOrder int `xorm:"DEFAULT 0"` // 0 means no exclusive order Description string Color string `xorm:"VARCHAR(7)"` NumIssues int @@ -236,7 +237,7 @@ func UpdateLabel(ctx context.Context, l *Label) error { } l.Color = color - return updateLabelCols(ctx, l, "name", "description", "color", "exclusive", "archived_unix") + return updateLabelCols(ctx, l, "name", "description", "color", "exclusive", "exclusive_order", "archived_unix") } // DeleteLabel delete a label diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index fe6de9c517..7da426fef0 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/tempdir" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/testlogger" @@ -114,15 +115,16 @@ func MainTest(m *testing.M) { setting.CustomConf = giteaConf } - tmpDataPath, err := os.MkdirTemp("", "data") + tmpDataPath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("data") if err != nil { testlogger.Fatalf("Unable to create temporary data path %v\n", err) } + defer cleanup() setting.CustomPath = filepath.Join(setting.AppWorkPath, "custom") setting.AppDataPath = tmpDataPath - unittest.InitSettings() + unittest.InitSettingsForTesting() if err = git.InitFull(context.Background()); err != nil { testlogger.Fatalf("Unable to InitFull: %v\n", err) } @@ -134,8 +136,5 @@ func MainTest(m *testing.M) { if err := removeAllWithRetry(setting.RepoRootPath); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } - if err := removeAllWithRetry(tmpDataPath); err != nil { - fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) - } os.Exit(exitStatus) } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 6a60067782..31b035eb31 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -380,6 +380,7 @@ func prepareMigrationTasks() []*migration { newMigration(316, "Add description for secrets and variables", v1_24.AddDescriptionForSecretsAndVariables), newMigration(317, "Add new index for action for heatmap", v1_24.AddNewIndexForUserDashboard), newMigration(318, "Add anonymous_access_mode for repo_unit", v1_24.AddRepoUnitAnonymousAccessMode), + newMigration(319, "Add ExclusiveOrder to Label table", v1_24.AddExclusiveOrderColumnToLabelTable), } return preparedMigrations } diff --git a/models/migrations/v1_24/v319.go b/models/migrations/v1_24/v319.go new file mode 100644 index 0000000000..6983c38605 --- /dev/null +++ b/models/migrations/v1_24/v319.go @@ -0,0 +1,16 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_24 //nolint + +import ( + "xorm.io/xorm" +) + +func AddExclusiveOrderColumnToLabelTable(x *xorm.Engine) error { + type Label struct { + ExclusiveOrder int `xorm:"DEFAULT 0"` + } + + return x.Sync(new(Label)) +} diff --git a/models/repo/upload.go b/models/repo/upload.go index cae11df96a..fb57fb6c51 100644 --- a/models/repo/upload.go +++ b/models/repo/upload.go @@ -51,14 +51,10 @@ func init() { db.RegisterModel(new(Upload)) } -// UploadLocalPath returns where uploads is stored in local file system based on given UUID. -func UploadLocalPath(uuid string) string { - return filepath.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid) -} - -// LocalPath returns where uploads are temporarily stored in local file system. +// LocalPath returns where uploads are temporarily stored in local file system based on given UUID. func (upload *Upload) LocalPath() string { - return UploadLocalPath(upload.UUID) + uuid := upload.UUID + return setting.AppDataTempDir("repo-uploads").JoinPath(uuid[0:1], uuid[1:2], uuid) } // NewUpload creates a new upload object. diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 7a9ca9698d..cb60cf5f85 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting/config" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/tempdir" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" @@ -35,8 +36,8 @@ func fatalTestError(fmtStr string, args ...any) { os.Exit(1) } -// InitSettings initializes config provider and load common settings for tests -func InitSettings() { +// InitSettingsForTesting initializes config provider and load common settings for tests +func InitSettingsForTesting() { setting.IsInTesting = true log.OsExiter = func(code int) { if code != 0 { @@ -75,7 +76,7 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { testOpts := util.OptionalArg(testOptsArg, &TestOptions{}) giteaRoot = test.SetupGiteaRoot() setting.CustomPath = filepath.Join(giteaRoot, "custom") - InitSettings() + InitSettingsForTesting() fixturesOpts := FixturesOptions{Dir: filepath.Join(giteaRoot, "models", "fixtures"), Files: testOpts.FixtureFiles} if err := CreateTestEngine(fixturesOpts); err != nil { @@ -92,15 +93,19 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { setting.SSH.Domain = "try.gitea.io" setting.Database.Type = "sqlite3" setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - repoRootPath, err := os.MkdirTemp(os.TempDir(), "repos") + repoRootPath, cleanup1, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("repos") if err != nil { fatalTestError("TempDir: %v\n", err) } + defer cleanup1() + setting.RepoRootPath = repoRootPath - appDataPath, err := os.MkdirTemp(os.TempDir(), "appdata") + appDataPath, cleanup2, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("appdata") if err != nil { fatalTestError("TempDir: %v\n", err) } + defer cleanup2() + setting.AppDataPath = appDataPath setting.AppWorkPath = giteaRoot setting.StaticRootPath = giteaRoot @@ -153,13 +158,6 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { fatalTestError("tear down failed: %v\n", err) } } - - if err = util.RemoveAll(repoRootPath); err != nil { - fatalTestError("util.RemoveAll: %v\n", err) - } - if err = util.RemoveAll(appDataPath); err != nil { - fatalTestError("util.RemoveAll: %v\n", err) - } os.Exit(exitStatus) } diff --git a/models/user/user.go b/models/user/user.go index 5989be74f0..100f924cc6 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -1187,29 +1187,28 @@ func GetUsersByEmails(ctx context.Context, emails []string) (map[string]*User, e for _, email := range emailAddresses { userIDs.Add(email.UID) } - users, err := GetUsersMapByIDs(ctx, userIDs.Values()) - if err != nil { - return nil, err - } - results := make(map[string]*User, len(emails)) - for _, email := range emailAddresses { - user := users[email.UID] - if user != nil { - if user.KeepEmailPrivate { - results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user - } else { - results[email.Email] = user + + if len(userIDs) > 0 { + users, err := GetUsersMapByIDs(ctx, userIDs.Values()) + if err != nil { + return nil, err + } + + for _, email := range emailAddresses { + user := users[email.UID] + if user != nil { + results[user.GetEmail()] = user } } } - users = make(map[int64]*User, len(needCheckUserNames)) + users := make(map[int64]*User, len(needCheckUserNames)) if err := db.GetEngine(ctx).In("lower_name", needCheckUserNames.Values()).Find(&users); err != nil { return nil, err } for _, user := range users { - results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user + results[user.GetPlaceholderEmail()] = user } return results, nil } diff --git a/modules/cache/context.go b/modules/cache/context.go index 484cee659a..85eb9e6790 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -166,15 +166,15 @@ func RemoveContextData(ctx context.Context, tp, key any) { } // GetWithContextCache returns the cache value of the given key in the given context. -func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) { - v := GetContextData(ctx, cacheGroupKey, cacheTargetID) +func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) { + v := GetContextData(ctx, groupKey, targetKey) if vv, ok := v.(T); ok { return vv, nil } - t, err := f() + t, err := f(ctx, targetKey) if err != nil { return t, err } - SetContextData(ctx, cacheGroupKey, cacheTargetID, t) + SetContextData(ctx, groupKey, targetKey, t) return t, nil } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index decb532937..23dd789dbc 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -4,6 +4,7 @@ package cache import ( + "context" "testing" "time" @@ -30,7 +31,7 @@ func TestWithCacheContext(t *testing.T) { v = GetContextData(ctx, field, "my_config1") assert.Nil(t, v) - vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) { + vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) { return 1, nil }) assert.NoError(t, err) diff --git a/modules/cachegroup/cachegroup.go b/modules/cachegroup/cachegroup.go new file mode 100644 index 0000000000..06085f860f --- /dev/null +++ b/modules/cachegroup/cachegroup.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cachegroup + +const ( + User = "user" + EmailAvatarLink = "email_avatar_link" + UserEmailAddresses = "user_email_addresses" + GPGKeyWithSubKeys = "gpg_key_with_subkeys" + RepoUserPermission = "repo_user_permission" +) diff --git a/modules/git/attribute.go b/modules/git/attribute.go deleted file mode 100644 index 4dfa510369..0000000000 --- a/modules/git/attribute.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "code.gitea.io/gitea/modules/optional" -) - -const ( - AttributeLinguistVendored = "linguist-vendored" - AttributeLinguistGenerated = "linguist-generated" - AttributeLinguistDocumentation = "linguist-documentation" - AttributeLinguistDetectable = "linguist-detectable" - AttributeLinguistLanguage = "linguist-language" - AttributeGitlabLanguage = "gitlab-language" -) - -// true if "set"/"true", false if "unset"/"false", none otherwise -func AttributeToBool(attr map[string]string, name string) optional.Option[bool] { - switch attr[name] { - case "set", "true": - return optional.Some(true) - case "unset", "false": - return optional.Some(false) - } - return optional.None[bool]() -} - -func AttributeToString(attr map[string]string, name string) optional.Option[string] { - if value, has := attr[name]; has && value != "unspecified" { - return optional.Some(value) - } - return optional.None[string]() -} diff --git a/modules/git/attribute/attribute.go b/modules/git/attribute/attribute.go new file mode 100644 index 0000000000..adf323ef41 --- /dev/null +++ b/modules/git/attribute/attribute.go @@ -0,0 +1,114 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "strings" + + "code.gitea.io/gitea/modules/optional" +) + +type Attribute string + +const ( + LinguistVendored = "linguist-vendored" + LinguistGenerated = "linguist-generated" + LinguistDocumentation = "linguist-documentation" + LinguistDetectable = "linguist-detectable" + LinguistLanguage = "linguist-language" + GitlabLanguage = "gitlab-language" + Lockable = "lockable" + Filter = "filter" +) + +var LinguistAttributes = []string{ + LinguistVendored, + LinguistGenerated, + LinguistDocumentation, + LinguistDetectable, + LinguistLanguage, + GitlabLanguage, +} + +func (a Attribute) IsUnspecified() bool { + return a == "" || a == "unspecified" +} + +func (a Attribute) ToString() optional.Option[string] { + if !a.IsUnspecified() { + return optional.Some(string(a)) + } + return optional.None[string]() +} + +// ToBool converts the attribute value to optional boolean: true if "set"/"true", false if "unset"/"false", none otherwise +func (a Attribute) ToBool() optional.Option[bool] { + switch a { + case "set", "true": + return optional.Some(true) + case "unset", "false": + return optional.Some(false) + } + return optional.None[bool]() +} + +type Attributes struct { + m map[string]Attribute +} + +func NewAttributes() *Attributes { + return &Attributes{m: make(map[string]Attribute)} +} + +func (attrs *Attributes) Get(name string) Attribute { + if value, has := attrs.m[name]; has { + return value + } + return "" +} + +func (attrs *Attributes) GetVendored() optional.Option[bool] { + return attrs.Get(LinguistVendored).ToBool() +} + +func (attrs *Attributes) GetGenerated() optional.Option[bool] { + return attrs.Get(LinguistGenerated).ToBool() +} + +func (attrs *Attributes) GetDocumentation() optional.Option[bool] { + return attrs.Get(LinguistDocumentation).ToBool() +} + +func (attrs *Attributes) GetDetectable() optional.Option[bool] { + return attrs.Get(LinguistDetectable).ToBool() +} + +func (attrs *Attributes) GetLinguistLanguage() optional.Option[string] { + return attrs.Get(LinguistLanguage).ToString() +} + +func (attrs *Attributes) GetGitlabLanguage() optional.Option[string] { + attrStr := attrs.Get(GitlabLanguage).ToString() + if attrStr.Has() { + raw := attrStr.Value() + // gitlab-language may have additional parameters after the language + // ignore them and just use the main language + // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type + if idx := strings.IndexByte(raw, '?'); idx >= 0 { + return optional.Some(raw[:idx]) + } + } + return attrStr +} + +func (attrs *Attributes) GetLanguage() optional.Option[string] { + // prefer linguist-language over gitlab-language + // if linguist-language is not set, use gitlab-language + // if both are not set, return none + language := attrs.GetLinguistLanguage() + if language.Value() == "" { + language = attrs.GetGitlabLanguage() + } + return language +} diff --git a/modules/git/attribute/attribute_test.go b/modules/git/attribute/attribute_test.go new file mode 100644 index 0000000000..dadb5582a3 --- /dev/null +++ b/modules/git/attribute/attribute_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Attribute(t *testing.T) { + assert.Empty(t, Attribute("").ToString().Value()) + assert.Empty(t, Attribute("unspecified").ToString().Value()) + assert.Equal(t, "python", Attribute("python").ToString().Value()) + assert.Equal(t, "Java", Attribute("Java").ToString().Value()) + + attributes := Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "true", + LinguistDocumentation: "false", + LinguistDetectable: "set", + LinguistLanguage: "Python", + GitlabLanguage: "Java", + "filter": "unspecified", + "test": "", + }, + } + + assert.Empty(t, attributes.Get("test").ToString().Value()) + assert.Empty(t, attributes.Get("filter").ToString().Value()) + assert.Equal(t, "Python", attributes.Get(LinguistLanguage).ToString().Value()) + assert.Equal(t, "Java", attributes.Get(GitlabLanguage).ToString().Value()) + assert.True(t, attributes.Get(LinguistGenerated).ToBool().Value()) + assert.False(t, attributes.Get(LinguistDocumentation).ToBool().Value()) + assert.True(t, attributes.Get(LinguistDetectable).ToBool().Value()) +} diff --git a/modules/git/attribute/batch.go b/modules/git/attribute/batch.go new file mode 100644 index 0000000000..4e31fda575 --- /dev/null +++ b/modules/git/attribute/batch.go @@ -0,0 +1,216 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" +) + +// BatchChecker provides a reader for check-attribute content that can be long running +type BatchChecker struct { + attributesNum int + repo *git.Repository + stdinWriter *os.File + stdOut *nulSeparatedAttributeWriter + ctx context.Context + cancel context.CancelFunc + cmd *git.Command +} + +// NewBatchChecker creates a check attribute reader for the current repository and provided commit ID +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func NewBatchChecker(repo *git.Repository, treeish string, attributes []string) (checker *BatchChecker, returnedErr error) { + ctx, cancel := context.WithCancel(repo.Ctx) + defer func() { + if returnedErr != nil { + cancel() + } + }() + + cmd, envs, cleanup, err := checkAttrCommand(repo, treeish, nil, attributes) + if err != nil { + return nil, err + } + defer func() { + if returnedErr != nil { + cleanup() + } + }() + + cmd.AddArguments("--stdin") + + checker = &BatchChecker{ + attributesNum: len(attributes), + repo: repo, + ctx: ctx, + cmd: cmd, + cancel: func() { + cancel() + cleanup() + }, + } + + stdinReader, stdinWriter, err := os.Pipe() + if err != nil { + return nil, err + } + checker.stdinWriter = stdinWriter + + lw := new(nulSeparatedAttributeWriter) + lw.attributes = make(chan attributeTriple, len(attributes)) + lw.closed = make(chan struct{}) + checker.stdOut = lw + + go func() { + defer func() { + _ = stdinReader.Close() + _ = lw.Close() + }() + stdErr := new(bytes.Buffer) + err := cmd.Run(ctx, &git.RunOpts{ + Env: envs, + Dir: repo.Path, + Stdin: stdinReader, + Stdout: lw, + Stderr: stdErr, + }) + + if err != nil && !git.IsErrCanceledOrKilled(err) { + log.Error("Attribute checker for commit %s exits with error: %v", treeish, err) + } + checker.cancel() + }() + + return checker, nil +} + +// CheckPath check attr for given path +func (c *BatchChecker) CheckPath(path string) (rs *Attributes, err error) { + defer func() { + if err != nil && err != c.ctx.Err() { + log.Error("Unexpected error when checking path %s in %s, error: %v", path, filepath.Base(c.repo.Path), err) + } + }() + + select { + case <-c.ctx.Done(): + return nil, c.ctx.Err() + default: + } + + if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { + defer c.Close() + return nil, err + } + + reportTimeout := func() error { + stdOutClosed := false + select { + case <-c.stdOut.closed: + stdOutClosed = true + default: + } + debugMsg := fmt.Sprintf("check path %q in repo %q", path, filepath.Base(c.repo.Path)) + debugMsg += fmt.Sprintf(", stdOut: tmp=%q, pos=%d, closed=%v", string(c.stdOut.tmp), c.stdOut.pos, stdOutClosed) + if c.cmd != nil { + debugMsg += fmt.Sprintf(", process state: %q", c.cmd.ProcessState()) + } + _ = c.Close() + return fmt.Errorf("CheckPath timeout: %s", debugMsg) + } + + rs = NewAttributes() + for i := 0; i < c.attributesNum; i++ { + select { + case <-time.After(5 * time.Second): + // there is no "hang" problem now. This code is just used to catch other potential problems. + return nil, reportTimeout() + case attr, ok := <-c.stdOut.ReadAttribute(): + if !ok { + return nil, c.ctx.Err() + } + rs.m[attr.Attribute] = Attribute(attr.Value) + case <-c.ctx.Done(): + return nil, c.ctx.Err() + } + } + return rs, nil +} + +func (c *BatchChecker) Close() error { + c.cancel() + err := c.stdinWriter.Close() + return err +} + +type attributeTriple struct { + Filename string + Attribute string + Value string +} + +type nulSeparatedAttributeWriter struct { + tmp []byte + attributes chan attributeTriple + closed chan struct{} + working attributeTriple + pos int +} + +func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { + l, read := len(p), 0 + + nulIdx := bytes.IndexByte(p, '\x00') + for nulIdx >= 0 { + wr.tmp = append(wr.tmp, p[:nulIdx]...) + switch wr.pos { + case 0: + wr.working = attributeTriple{ + Filename: string(wr.tmp), + } + case 1: + wr.working.Attribute = string(wr.tmp) + case 2: + wr.working.Value = string(wr.tmp) + } + wr.tmp = wr.tmp[:0] + wr.pos++ + if wr.pos > 2 { + wr.attributes <- wr.working + wr.pos = 0 + } + read += nulIdx + 1 + if l > read { + p = p[nulIdx+1:] + nulIdx = bytes.IndexByte(p, '\x00') + } else { + return l, nil + } + } + wr.tmp = append(wr.tmp, p...) + return l, nil +} + +func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { + return wr.attributes +} + +func (wr *nulSeparatedAttributeWriter) Close() error { + select { + case <-wr.closed: + return nil + default: + } + close(wr.attributes) + close(wr.closed) + return nil +} diff --git a/modules/git/repo_attribute_test.go b/modules/git/attribute/batch_test.go similarity index 50% rename from modules/git/repo_attribute_test.go rename to modules/git/attribute/batch_test.go index d8fd9f0e8d..30a3d805fe 100644 --- a/modules/git/repo_attribute_test.go +++ b/modules/git/attribute/batch_test.go @@ -1,13 +1,19 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package attribute import ( + "path/filepath" "testing" "time" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { @@ -24,7 +30,7 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { select { case attr := <-wr.ReadAttribute(): assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) + assert.Equal(t, LinguistVendored, attr.Attribute) assert.Equal(t, "unspecified", attr.Value) case <-time.After(100 * time.Millisecond): assert.FailNow(t, "took too long to read an attribute from the list") @@ -38,7 +44,7 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { select { case attr := <-wr.ReadAttribute(): assert.Equal(t, ".gitignore\"\n", attr.Filename) - assert.Equal(t, AttributeLinguistVendored, attr.Attribute) + assert.Equal(t, LinguistVendored, attr.Attribute) assert.Equal(t, "unspecified", attr.Value) case <-time.After(100 * time.Millisecond): assert.FailNow(t, "took too long to read an attribute from the list") @@ -77,21 +83,90 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) { assert.NoError(t, err) assert.Equal(t, attributeTriple{ Filename: "shouldbe.vendor", - Attribute: AttributeLinguistVendored, + Attribute: LinguistVendored, Value: "set", }, attr) attr = <-wr.ReadAttribute() assert.NoError(t, err) assert.Equal(t, attributeTriple{ Filename: "shouldbe.vendor", - Attribute: AttributeLinguistGenerated, + Attribute: LinguistGenerated, Value: "unspecified", }, attr) attr = <-wr.ReadAttribute() assert.NoError(t, err) assert.Equal(t, attributeTriple{ Filename: "shouldbe.vendor", - Attribute: AttributeLinguistLanguage, + Attribute: LinguistLanguage, Value: "unspecified", }, attr) } + +func expectedAttrs() *Attributes { + return &Attributes{ + m: map[string]Attribute{ + LinguistGenerated: "unspecified", + LinguistDetectable: "unspecified", + LinguistDocumentation: "unspecified", + LinguistVendored: "unspecified", + LinguistLanguage: "Python", + GitlabLanguage: "unspecified", + }, + } +} + +func Test_BatchChecker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + checker, err := NewBatchChecker(tempRepo, "", LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + checker, err := NewBatchChecker(gitRepo, commitID, LinguistAttributes) + assert.NoError(t, err) + defer checker.Close() + + attributes, err := checker.CheckPath("i-am-a-python.p") + assert.NoError(t, err) + assert.Equal(t, expectedAttrs(), attributes) + }) +} diff --git a/modules/git/attribute/checker.go b/modules/git/attribute/checker.go new file mode 100644 index 0000000000..c17006a154 --- /dev/null +++ b/modules/git/attribute/checker.go @@ -0,0 +1,96 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + + "code.gitea.io/gitea/modules/git" +) + +func checkAttrCommand(gitRepo *git.Repository, treeish string, filenames, attributes []string) (*git.Command, []string, func(), error) { + cancel := func() {} + envs := []string{"GIT_FLUSH=1"} + cmd := git.NewCommand("check-attr", "-z") + if len(attributes) == 0 { + cmd.AddArguments("--all") + } + + // there is treeish, read from bare repo or temp index created by "read-tree" + if treeish != "" { + if git.DefaultFeatures().SupportCheckAttrOnBare { + cmd.AddArguments("--source") + cmd.AddDynamicArguments(treeish) + } else { + indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(treeish) + if err != nil { + return nil, nil, nil, err + } + + cmd.AddArguments("--cached") + envs = append(envs, + "GIT_INDEX_FILE="+indexFilename, + "GIT_WORK_TREE="+worktree, + ) + cancel = deleteTemporaryFile + } + } // else: no treeish, assume it is a not a bare repo, read from working directory + + cmd.AddDynamicArguments(attributes...) + if len(filenames) > 0 { + cmd.AddDashesAndList(filenames...) + } + return cmd, envs, cancel, nil +} + +type CheckAttributeOpts struct { + Filenames []string + Attributes []string +} + +// CheckAttributes return the attributes of the given filenames and attributes in the given treeish. +// If treeish is empty, then it will use current working directory, otherwise it will use the provided treeish on the bare repo +func CheckAttributes(ctx context.Context, gitRepo *git.Repository, treeish string, opts CheckAttributeOpts) (map[string]*Attributes, error) { + cmd, envs, cancel, err := checkAttrCommand(gitRepo, treeish, opts.Filenames, opts.Attributes) + if err != nil { + return nil, err + } + defer cancel() + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := cmd.Run(ctx, &git.RunOpts{ + Env: append(os.Environ(), envs...), + Dir: gitRepo.Path, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + if len(fields)%3 != 1 { + return nil, errors.New("wrong number of fields in return from check-attr") + } + + attributesMap := make(map[string]*Attributes) + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info, ok := attributesMap[filename] + if !ok { + attribute2info = NewAttributes() + attributesMap[filename] = attribute2info + } + attribute2info.m[attribute] = Attribute(info) + } + + return attributesMap, nil +} diff --git a/modules/git/attribute/checker_test.go b/modules/git/attribute/checker_test.go new file mode 100644 index 0000000000..97db43460b --- /dev/null +++ b/modules/git/attribute/checker_test.go @@ -0,0 +1,74 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_Checker(t *testing.T) { + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + require.NoError(t, err) + defer gitRepo.Close() + + commitID := "8fee858da5796dfb37704761701bb8e800ad9ef3" + + t.Run("Create index file to run git check-attr", func(t *testing.T) { + defer test.MockVariableValue(&git.DefaultFeatures().SupportCheckAttrOnBare, false)() + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + // run git check-attr on work tree + t.Run("Run git check-attr on git work tree", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "test-repo") + err := git.Clone(t.Context(), repoPath, dir, git.CloneRepoOptions{ + Shared: true, + Branch: "master", + }) + assert.NoError(t, err) + + tempRepo, err := git.OpenRepository(t.Context(), dir) + assert.NoError(t, err) + defer tempRepo.Close() + + attrs, err := CheckAttributes(t.Context(), tempRepo, "", CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) + + if !git.DefaultFeatures().SupportCheckAttrOnBare { + t.Skip("git version 2.40 is required to support run check-attr on bare repo") + return + } + + t.Run("Run git check-attr in bare repository", func(t *testing.T) { + attrs, err := CheckAttributes(t.Context(), gitRepo, commitID, CheckAttributeOpts{ + Filenames: []string{"i-am-a-python.p"}, + Attributes: LinguistAttributes, + }) + assert.NoError(t, err) + assert.Len(t, attrs, 1) + assert.Equal(t, expectedAttrs(), attrs["i-am-a-python.p"]) + }) +} diff --git a/modules/git/attribute/main_test.go b/modules/git/attribute/main_test.go new file mode 100644 index 0000000000..df8241bfb0 --- /dev/null +++ b/modules/git/attribute/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package attribute + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/blame.go b/modules/git/blame.go index d1d732c716..6eb583a6b9 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -11,7 +11,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // BlamePart represents block of blame - continuous lines with one sha @@ -29,12 +29,13 @@ type BlameReader struct { bufferedReader *bufio.Reader done chan error lastSha *string - ignoreRevsFile *string + ignoreRevsFile string objectFormat ObjectFormat + cleanupFuncs []func() } func (r *BlameReader) UsesIgnoreRevs() bool { - return r.ignoreRevsFile != nil + return r.ignoreRevsFile != "" } // NextPart returns next part of blame (sequential code lines with the same commit) @@ -122,36 +123,37 @@ func (r *BlameReader) Close() error { r.bufferedReader = nil _ = r.reader.Close() _ = r.output.Close() - if r.ignoreRevsFile != nil { - _ = util.Remove(*r.ignoreRevsFile) + for _, cleanup := range r.cleanupFuncs { + if cleanup != nil { + cleanup() + } } return err } // CreateBlameReader creates reader for given repository, commit and file func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (*BlameReader, error) { - var ignoreRevsFile *string - if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { - ignoreRevsFile = tryCreateBlameIgnoreRevsFile(commit) - } - - cmd := NewCommandNoGlobals("blame", "--porcelain") - if ignoreRevsFile != nil { - // Possible improvement: use --ignore-revs-file /dev/stdin on unix - // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. - cmd.AddOptionValues("--ignore-revs-file", *ignoreRevsFile) - } - cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) reader, stdout, err := os.Pipe() if err != nil { - if ignoreRevsFile != nil { - _ = util.Remove(*ignoreRevsFile) - } return nil, err } - done := make(chan error, 1) + cmd := NewCommandNoGlobals("blame", "--porcelain") + var ignoreRevsFileName string + var ignoreRevsFileCleanup func() // TODO: maybe it should check the returned err in a defer func to make sure the cleanup could always be executed correctly + if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { + ignoreRevsFileName, ignoreRevsFileCleanup = tryCreateBlameIgnoreRevsFile(commit) + if ignoreRevsFileName != "" { + // Possible improvement: use --ignore-revs-file /dev/stdin on unix + // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. + cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) + } + } + + cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) + + done := make(chan error, 1) go func() { stderr := bytes.Buffer{} // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" @@ -169,40 +171,44 @@ func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath }() bufferedReader := bufio.NewReader(reader) - return &BlameReader{ output: stdout, reader: reader, bufferedReader: bufferedReader, done: done, - ignoreRevsFile: ignoreRevsFile, + ignoreRevsFile: ignoreRevsFileName, objectFormat: objectFormat, + cleanupFuncs: []func(){ignoreRevsFileCleanup}, }, nil } -func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { +func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func()) { entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") if err != nil { - return nil + log.Error("Unable to get .git-blame-ignore-revs file: GetTreeEntryByPath: %v", err) + return "", nil } r, err := entry.Blob().DataAsync() if err != nil { - return nil + log.Error("Unable to get .git-blame-ignore-revs file data: DataAsync: %v", err) + return "", nil } defer r.Close() - f, err := os.CreateTemp("", "gitea_git-blame-ignore-revs") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") if err != nil { - return nil + log.Error("Unable to get .git-blame-ignore-revs file data: CreateTempFileRandom: %v", err) + return "", nil } - + filename := f.Name() _, err = io.Copy(f, r) _ = f.Close() if err != nil { - _ = util.Remove(f.Name()) - return nil + cleanup() + log.Error("Unable to get .git-blame-ignore-revs file data: Copy: %v", err) + return "", nil } - return util.ToPointer(f.Name()) + return filename, cleanup } diff --git a/modules/git/blame_sha256_test.go b/modules/git/blame_sha256_test.go index 99c23429e2..c0a97bed3b 100644 --- a/modules/git/blame_sha256_test.go +++ b/modules/git/blame_sha256_test.go @@ -7,10 +7,13 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutputSha256(t *testing.T) { + setting.AppDataPath = t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() diff --git a/modules/git/blame_test.go b/modules/git/blame_test.go index 36b5fb9349..809d6fbcf7 100644 --- a/modules/git/blame_test.go +++ b/modules/git/blame_test.go @@ -7,10 +7,13 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutput(t *testing.T) { + setting.AppDataPath = t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() diff --git a/modules/git/command.go b/modules/git/command.go index f449f1ff0e..eaaa4969d0 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -80,6 +80,13 @@ func (c *Command) LogString() string { return strings.Join(a, " ") } +func (c *Command) ProcessState() string { + if c.cmd == nil { + return "" + } + return c.cmd.ProcessState.String() +} + // NewCommand creates and returns a new Git Command based on given command and arguments. // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. func NewCommand(args ...internal.CmdArg) *Command { diff --git a/modules/git/git.go b/modules/git/git.go index 2b593910a2..a2ffd6d289 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -30,6 +30,7 @@ type Features struct { SupportProcReceive bool // >= 2.29 SupportHashSha256 bool // >= 2.42, SHA-256 repositories no longer an ‘experimental curiosity’ SupportedObjectFormats []ObjectFormat // sha1, sha256 + SupportCheckAttrOnBare bool // >= 2.40 } var ( @@ -77,6 +78,7 @@ func loadGitVersionFeatures() (*Features, error) { if features.SupportHashSha256 { features.SupportedObjectFormats = append(features.SupportedObjectFormats, Sha256ObjectFormat) } + features.SupportCheckAttrOnBare = features.CheckVersionAtLeast("2.40") return features, nil } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index 5472842b76..58ba01cabc 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,18 +10,19 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/tempdir" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" ) func testRun(m *testing.M) error { - gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } - defer util.RemoveAll(gitHomePath) + defer cleanup() + setting.Git.HomePath = gitHomePath if err = InitFull(context.Background()); err != nil { diff --git a/modules/git/repo_language_stats.go b/modules/git/languagestats/language_stats.go similarity index 59% rename from modules/git/repo_language_stats.go rename to modules/git/languagestats/language_stats.go index 8551ea9d24..a71284c3e4 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/languagestats/language_stats.go @@ -1,13 +1,15 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package languagestats import ( + "context" "strings" "unicode" - "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" ) const ( @@ -49,19 +51,15 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 { return res } -func TryReadLanguageAttribute(attrs map[string]string) optional.Option[string] { - language := AttributeToString(attrs, AttributeLinguistLanguage) - if language.Value() == "" { - language = AttributeToString(attrs, AttributeGitlabLanguage) - if language.Has() { - raw := language.Value() - // gitlab-language may have additional parameters after the language - // ignore them and just use the main language - // https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type - if idx := strings.IndexByte(raw, '?'); idx >= 0 { - language = optional.Some(raw[:idx]) - } - } +// GetFileLanguage tries to get the (linguist) language of the file content +func GetFileLanguage(ctx context.Context, gitRepo *git.Repository, treeish, treePath string) (string, error) { + attributesMap, err := attribute.CheckAttributes(ctx, gitRepo, treeish, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.LinguistLanguage, attribute.GitlabLanguage}, + Filenames: []string{treePath}, + }) + if err != nil { + return "", err } - return language + + return attributesMap[treePath].GetLanguage().Value(), nil } diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/languagestats/language_stats_gogit.go similarity index 73% rename from modules/git/repo_language_stats_gogit.go rename to modules/git/languagestats/language_stats_gogit.go index a34c03c781..418c05b157 100644 --- a/modules/git/repo_language_stats_gogit.go +++ b/modules/git/languagestats/language_stats_gogit.go @@ -3,13 +3,15 @@ //go:build gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + git_module "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/optional" "github.com/go-enry/go-enry/v2" @@ -19,7 +21,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git_module.Repository, commitID string) (map[string]int64, error) { r, err := git.PlainOpen(repo.Path) if err != nil { return nil, err @@ -40,8 +42,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() // sizes contains the current calculated size of all files by language sizes := make(map[string]int64) @@ -62,43 +67,41 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - return nil + attrs, err := checker.CheckPath(f.Name) + if err == nil { + isVendored = attrs.GetVendored() + if isVendored.ValueOrDefault(false) { + return nil + } + + isGenerated = attrs.GetGenerated() + if isGenerated.ValueOrDefault(false) { + return nil + } + + isDocumentation = attrs.GetDocumentation() + if isDocumentation.ValueOrDefault(false) { + return nil + } + + isDetectable = attrs.GetDetectable() + if !isDetectable.ValueOrDefault(true) { + return nil + } + + hasLanguage := attrs.GetLanguage() + if hasLanguage.Value() != "" { + language := hasLanguage.Value() + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - return nil - } - - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - return nil - } - - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - return nil - } - - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() - - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } - - // this language will always be added to the size - sizes[language] += f.Size - return nil - } + // this language will always be added to the size + sizes[language] += f.Size + return nil } } diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/languagestats/language_stats_nogogit.go similarity index 73% rename from modules/git/repo_language_stats_nogogit.go rename to modules/git/languagestats/language_stats_nogogit.go index de7707bd6c..34797263a6 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/languagestats/language_stats_nogogit.go @@ -3,13 +3,15 @@ //go:build !gogit -package git +package languagestats import ( "bytes" "io" "code.gitea.io/gitea/modules/analyze" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -17,7 +19,7 @@ import ( ) // GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { +func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // so let's create a batch stdin and stdout batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) @@ -34,19 +36,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(commitID); err != nil { return nil, err } - shaBytes, typ, size, err := ReadBatchLine(batchReader) + shaBytes, typ, size, err := git.ReadBatchLine(batchReader) if typ != "commit" { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - sha, err := NewIDFromString(string(shaBytes)) + sha, err := git.NewIDFromString(string(shaBytes)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} + return nil, git.ErrNotExist{ID: commitID} } - commit, err := CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) + commit, err := git.CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) if err != nil { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) return nil, err @@ -62,8 +64,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() + checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) + if err != nil { + return nil, err + } + defer checker.Close() contentBuf := bytes.Buffer{} var content []byte @@ -96,43 +101,36 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err isDocumentation := optional.None[bool]() isDetectable := optional.None[bool]() - if checker != nil { - attrs, err := checker.CheckPath(f.Name()) - if err == nil { - isVendored = AttributeToBool(attrs, AttributeLinguistVendored) - if isVendored.ValueOrDefault(false) { - continue + attrs, err := checker.CheckPath(f.Name()) + if err == nil { + if isVendored = attrs.GetVendored(); isVendored.ValueOrDefault(false) { + continue + } + + if isGenerated = attrs.GetGenerated(); isGenerated.ValueOrDefault(false) { + continue + } + + if isDocumentation = attrs.GetDocumentation(); isDocumentation.ValueOrDefault(false) { + continue + } + + if isDetectable = attrs.GetDetectable(); !isDetectable.ValueOrDefault(true) { + continue + } + + if hasLanguage := attrs.GetLanguage(); hasLanguage.Value() != "" { + language := hasLanguage.Value() + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group } - isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated) - if isGenerated.ValueOrDefault(false) { - continue - } - - isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation) - if isDocumentation.ValueOrDefault(false) { - continue - } - - isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable) - if !isDetectable.ValueOrDefault(true) { - continue - } - - hasLanguage := TryReadLanguageAttribute(attrs) - if hasLanguage.Value() != "" { - language := hasLanguage.Value() - - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } - - // this language will always be added to the size - sizes[language] += f.Size() - continue - } + // this language will always be added to the size + sizes[language] += f.Size() + continue } } @@ -149,7 +147,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err if err := writeID(f.ID.String()); err != nil { return nil, err } - _, _, size, err := ReadBatchLine(batchReader) + _, _, size, err := git.ReadBatchLine(batchReader) if err != nil { log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) return nil, err @@ -167,7 +165,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err return nil, err } content = contentBuf.Bytes() - if err := DiscardFull(batchReader, discard); err != nil { + if err := git.DiscardFull(batchReader, discard); err != nil { return nil, err } } diff --git a/modules/git/repo_language_stats_test.go b/modules/git/languagestats/language_stats_test.go similarity index 67% rename from modules/git/repo_language_stats_test.go rename to modules/git/languagestats/language_stats_test.go index 81f130bacb..b908ae6413 100644 --- a/modules/git/repo_language_stats_test.go +++ b/modules/git/languagestats/language_stats_test.go @@ -3,24 +3,26 @@ //go:build !gogit -package git +package languagestats import ( - "path/filepath" "testing" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRepository_GetLanguageStats(t *testing.T) { - repoPath := filepath.Join(testReposDir, "language_stats_repo") - gitRepo, err := openRepositoryWithDefaultContext(repoPath) + setting.AppDataPath = t.TempDir() + repoPath := "../tests/repos/language_stats_repo" + gitRepo, err := git.OpenRepository(t.Context(), repoPath) require.NoError(t, err) - defer gitRepo.Close() - stats, err := gitRepo.GetLanguageStats("8fee858da5796dfb37704761701bb8e800ad9ef3") + stats, err := GetLanguageStats(gitRepo, "8fee858da5796dfb37704761701bb8e800ad9ef3") require.NoError(t, err) assert.Equal(t, map[string]int64{ diff --git a/modules/git/languagestats/main_test.go b/modules/git/languagestats/main_test.go new file mode 100644 index 0000000000..707d268c81 --- /dev/null +++ b/modules/git/languagestats/main_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package languagestats + +import ( + "context" + "fmt" + "os" + "testing" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func testRun(m *testing.M) error { + gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitHomePath) + setting.Git.HomePath = gitHomePath + + if err = git.InitFull(context.Background()); err != nil { + return fmt.Errorf("failed to call Init: %w", err) + } + + exitCode := m.Run() + if exitCode != 0 { + return fmt.Errorf("run test failed, ExitCode=%d", exitCode) + } + return nil +} + +func TestMain(m *testing.M) { + if err := testRun(m); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Test failed: %v", err) + os.Exit(1) + } +} diff --git a/modules/git/repo.go b/modules/git/repo.go index 6459adf851..45937a8d5f 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" ) // GPGSettings represents the default GPG settings for this repository @@ -266,11 +267,11 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, err := os.MkdirTemp(os.TempDir(), "gitea-bundle") + tmp, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-bundle") if err != nil { return err } - defer os.RemoveAll(tmp) + defer cleanup() env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) _, _, err = NewCommand("init", "--bare").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go deleted file mode 100644 index fde42d4730..0000000000 --- a/modules/git/repo_attribute.go +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// CheckAttributeOpts represents the possible options to CheckAttribute -type CheckAttributeOpts struct { - CachedOnly bool - AllAttributes bool - Attributes []string - Filenames []string - IndexFile string - WorkTree string -} - -// CheckAttribute return the Blame object of file -func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[string]string, error) { - env := []string{} - - if len(opts.IndexFile) > 0 { - env = append(env, "GIT_INDEX_FILE="+opts.IndexFile) - } - if len(opts.WorkTree) > 0 { - env = append(env, "GIT_WORK_TREE="+opts.WorkTree) - } - - if len(env) > 0 { - env = append(os.Environ(), env...) - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - cmd := NewCommand("check-attr", "-z") - - if opts.AllAttributes { - cmd.AddArguments("-a") - } else { - for _, attribute := range opts.Attributes { - if attribute != "" { - cmd.AddDynamicArguments(attribute) - } - } - } - - if opts.CachedOnly { - cmd.AddArguments("--cached") - } - - cmd.AddDashesAndList(opts.Filenames...) - - if err := cmd.Run(repo.Ctx, &RunOpts{ - Env: env, - Dir: repo.Path, - Stdout: stdOut, - Stderr: stdErr, - }); err != nil { - return nil, fmt.Errorf("failed to run check-attr: %w\n%s\n%s", err, stdOut.String(), stdErr.String()) - } - - // FIXME: This is incorrect on versions < 1.8.5 - fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) - - if len(fields)%3 != 1 { - return nil, errors.New("wrong number of fields in return from check-attr") - } - - name2attribute2info := make(map[string]map[string]string) - - for i := 0; i < (len(fields) / 3); i++ { - filename := string(fields[3*i]) - attribute := string(fields[3*i+1]) - info := string(fields[3*i+2]) - attribute2info := name2attribute2info[filename] - if attribute2info == nil { - attribute2info = make(map[string]string) - } - attribute2info[attribute] = info - name2attribute2info[filename] = attribute2info - } - - return name2attribute2info, nil -} - -// CheckAttributeReader provides a reader for check-attribute content that can be long running -type CheckAttributeReader struct { - // params - Attributes []string - Repo *Repository - IndexFile string - WorkTree string - - stdinReader io.ReadCloser - stdinWriter *os.File - stdOut *nulSeparatedAttributeWriter - cmd *Command - env []string - ctx context.Context - cancel context.CancelFunc -} - -// Init initializes the CheckAttributeReader -func (c *CheckAttributeReader) Init(ctx context.Context) error { - if len(c.Attributes) == 0 { - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple) - lw.closed = make(chan struct{}) - - c.stdOut = lw - c.stdOut.Close() - return errors.New("no provided Attributes to check") - } - - c.ctx, c.cancel = context.WithCancel(ctx) - c.cmd = NewCommand("check-attr", "--stdin", "-z") - - if len(c.IndexFile) > 0 { - c.cmd.AddArguments("--cached") - c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile) - } - - if len(c.WorkTree) > 0 { - c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree) - } - - c.env = append(c.env, "GIT_FLUSH=1") - - c.cmd.AddDynamicArguments(c.Attributes...) - - var err error - - c.stdinReader, c.stdinWriter, err = os.Pipe() - if err != nil { - c.cancel() - return err - } - - lw := new(nulSeparatedAttributeWriter) - lw.attributes = make(chan attributeTriple, 5) - lw.closed = make(chan struct{}) - c.stdOut = lw - return nil -} - -func (c *CheckAttributeReader) Run() error { - defer func() { - _ = c.stdinReader.Close() - _ = c.stdOut.Close() - }() - stdErr := new(bytes.Buffer) - err := c.cmd.Run(c.ctx, &RunOpts{ - Env: c.env, - Dir: c.Repo.Path, - Stdin: c.stdinReader, - Stdout: c.stdOut, - Stderr: stdErr, - }) - if err != nil && !IsErrCanceledOrKilled(err) { - return fmt.Errorf("failed to run attr-check. Error: %w\nStderr: %s", err, stdErr.String()) - } - return nil -} - -// CheckPath check attr for given path -func (c *CheckAttributeReader) CheckPath(path string) (rs map[string]string, err error) { - defer func() { - if err != nil && err != c.ctx.Err() { - log.Error("Unexpected error when checking path %s in %s, error: %v", path, filepath.Base(c.Repo.Path), err) - } - }() - - select { - case <-c.ctx.Done(): - return nil, c.ctx.Err() - default: - } - - if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil { - defer c.Close() - return nil, err - } - - reportTimeout := func() error { - stdOutClosed := false - select { - case <-c.stdOut.closed: - stdOutClosed = true - default: - } - debugMsg := fmt.Sprintf("check path %q in repo %q", path, filepath.Base(c.Repo.Path)) - debugMsg += fmt.Sprintf(", stdOut: tmp=%q, pos=%d, closed=%v", string(c.stdOut.tmp), c.stdOut.pos, stdOutClosed) - if c.cmd.cmd != nil { - debugMsg += fmt.Sprintf(", process state: %q", c.cmd.cmd.ProcessState.String()) - } - _ = c.Close() - return fmt.Errorf("CheckPath timeout: %s", debugMsg) - } - - rs = make(map[string]string) - for range c.Attributes { - select { - case <-time.After(5 * time.Second): - // There is a strange "hang" problem in gitdiff.GetDiff -> CheckPath - // So add a timeout here to mitigate the problem, and output more logs for debug purpose - // In real world, if CheckPath runs long than seconds, it blocks the end user's operation, - // and at the moment the CheckPath result is not so important, so we can just ignore it. - return nil, reportTimeout() - case attr, ok := <-c.stdOut.ReadAttribute(): - if !ok { - return nil, c.ctx.Err() - } - rs[attr.Attribute] = attr.Value - case <-c.ctx.Done(): - return nil, c.ctx.Err() - } - } - return rs, nil -} - -func (c *CheckAttributeReader) Close() error { - c.cancel() - err := c.stdinWriter.Close() - return err -} - -type attributeTriple struct { - Filename string - Attribute string - Value string -} - -type nulSeparatedAttributeWriter struct { - tmp []byte - attributes chan attributeTriple - closed chan struct{} - working attributeTriple - pos int -} - -func (wr *nulSeparatedAttributeWriter) Write(p []byte) (n int, err error) { - l, read := len(p), 0 - - nulIdx := bytes.IndexByte(p, '\x00') - for nulIdx >= 0 { - wr.tmp = append(wr.tmp, p[:nulIdx]...) - switch wr.pos { - case 0: - wr.working = attributeTriple{ - Filename: string(wr.tmp), - } - case 1: - wr.working.Attribute = string(wr.tmp) - case 2: - wr.working.Value = string(wr.tmp) - } - wr.tmp = wr.tmp[:0] - wr.pos++ - if wr.pos > 2 { - wr.attributes <- wr.working - wr.pos = 0 - } - read += nulIdx + 1 - if l > read { - p = p[nulIdx+1:] - nulIdx = bytes.IndexByte(p, '\x00') - } else { - return l, nil - } - } - wr.tmp = append(wr.tmp, p...) - return l, nil -} - -func (wr *nulSeparatedAttributeWriter) ReadAttribute() <-chan attributeTriple { - return wr.attributes -} - -func (wr *nulSeparatedAttributeWriter) Close() error { - select { - case <-wr.closed: - return nil - default: - } - close(wr.attributes) - close(wr.closed) - return nil -} - -// CheckAttributeReader creates a check attribute reader for the current repository and provided commit ID -func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeReader, context.CancelFunc) { - indexFilename, worktree, deleteTemporaryFile, err := repo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return nil, func() {} - } - - checker := &CheckAttributeReader{ - Attributes: []string{ - AttributeLinguistVendored, - AttributeLinguistGenerated, - AttributeLinguistDocumentation, - AttributeLinguistDetectable, - AttributeLinguistLanguage, - AttributeGitlabLanguage, - }, - Repo: repo, - IndexFile: indexFilename, - WorkTree: worktree, - } - ctx, cancel := context.WithCancel(repo.Ctx) - if err := checker.Init(ctx); err != nil { - log.Error("Unable to open attribute checker for commit %s, error: %v", commitID, err) - } else { - go func() { - err := checker.Run() - if err != nil && !IsErrCanceledOrKilled(err) { - log.Error("Attribute checker for commit %s exits with error: %v", commitID, err) - } - cancel() - }() - } - deferrable := func() { - _ = checker.Close() - cancel() - deleteTemporaryFile() - } - - return checker, deferrable -} diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index 1c7fcc063e..443a3a20d1 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -10,8 +10,7 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // ReadTreeToIndex reads a treeish to the index @@ -59,26 +58,18 @@ func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilena } }() - removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs - return func() { - if err := util.RemoveAll(dir); err != nil { - log.Error("failed to remove tmp index dir: %v", err) - } - } - } - - tmpDir, err = os.MkdirTemp("", "index") + tmpDir, cancel, err = setting.AppDataTempDir("git-repo-content").MkdirTempRandom("index") if err != nil { return "", "", nil, err } tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index") - cancel = removeDirFn(tmpDir) + err = repo.ReadTreeToIndex(treeish, tmpIndexFilename) if err != nil { return "", "", cancel, err } - return tmpIndexFilename, tmpDir, cancel, err + return tmpIndexFilename, tmpDir, cancel, nil } // EmptyIndex empties the index diff --git a/modules/git/tests/repos/language_stats_repo/config b/modules/git/tests/repos/language_stats_repo/config index 515f483629..a4ef456cbc 100644 --- a/modules/git/tests/repos/language_stats_repo/config +++ b/modules/git/tests/repos/language_stats_repo/config @@ -1,5 +1,5 @@ [core] repositoryformatversion = 0 filemode = true - bare = false + bare = true logallrefupdates = true diff --git a/modules/git/tests/repos/repo3_notes/config b/modules/git/tests/repos/repo3_notes/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo3_notes/config +++ b/modules/git/tests/repos/repo3_notes/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/git/tests/repos/repo4_commitsbetween/config b/modules/git/tests/repos/repo4_commitsbetween/config index d545cdabdb..5ed22e23d1 100644 --- a/modules/git/tests/repos/repo4_commitsbetween/config +++ b/modules/git/tests/repos/repo4_commitsbetween/config @@ -1,7 +1,7 @@ [core] repositoryformatversion = 0 filemode = false - bare = false + bare = true logallrefupdates = true symlinks = false ignorecase = true diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 6124ad2515..50951f9c88 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -6,6 +6,7 @@ package db import ( "context" "strings" + "sync" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -18,7 +19,7 @@ import ( "xorm.io/builder" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = (*Indexer)(nil) // Indexer implements Indexer interface to use database's like search type Indexer struct { @@ -29,11 +30,9 @@ func (i *Indexer) SupportedSearchModes() []indexer.SearchMode { return indexer.SearchModesExactWords() } -func NewIndexer() *Indexer { - return &Indexer{ - Indexer: &inner_db.Indexer{}, - } -} +var GetIndexer = sync.OnceValue(func() *Indexer { + return &Indexer{Indexer: &inner_db.Indexer{}} +}) // Index dummy function func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error { @@ -122,7 +121,11 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }, nil } - ids, total, err := issue_model.IssueIDs(ctx, opt, cond) + return i.FindWithIssueOptions(ctx, opt, cond) +} + +func (i *Indexer) FindWithIssueOptions(ctx context.Context, opt *issue_model.IssuesOptions, otherConds ...builder.Cond) (*internal.SearchResult, error) { + ids, total, err := issue_model.IssueIDs(ctx, opt, otherConds...) if err != nil { return nil, err } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 3b19d742ba..380a25dc23 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -6,6 +6,7 @@ package db import ( "context" "fmt" + "strings" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -34,7 +35,11 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m case internal.SortByDeadlineAsc: sortType = "nearduedate" default: - sortType = "newest" + if strings.HasPrefix(string(options.SortBy), issue_model.ScopeSortPrefix) { + sortType = string(options.SortBy) + } else { + sortType = "newest" + } } // See the comment of issues_model.SearchOptions for the reason why we need to convert @@ -68,7 +73,6 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m ExcludedLabelNames: nil, IncludeMilestones: nil, SortType: sortType, - IssueIDs: nil, UpdatedAfterUnix: options.UpdatedAfterUnix.Value(), UpdatedBeforeUnix: options.UpdatedBeforeUnix.Value(), PriorityRepoID: 0, diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index 4e2dff572a..f17724664d 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -4,12 +4,19 @@ package issues import ( + "strings" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" ) func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions { + if opts.IssueIDs != nil { + setting.PanicInDevOrTesting("Indexer SearchOptions doesn't support IssueIDs") + } searchOpt := &SearchOptions{ Keyword: keyword, RepoIDs: opts.RepoIDs, @@ -95,7 +102,11 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp // Unsupported sort type for search fallthrough default: - searchOpt.SortBy = SortByUpdatedDesc + if strings.HasPrefix(opts.SortType, issues_model.ScopeSortPrefix) { + searchOpt.SortBy = internal.SortBy(opts.SortType) + } else { + searchOpt.SortBy = SortByUpdatedDesc + } } return searchOpt diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 4741235d47..9e63ad1ad8 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -103,7 +103,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) } case "db": - issueIndexer = db.NewIndexer() + issueIndexer = db.GetIndexer() case "meilisearch": issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName) existed, err = issueIndexer.Init(ctx) @@ -291,19 +291,22 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err // So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue. // Even worse, the external indexer like elastic search may not be available for a while, // and the user may not be able to list issues completely until it is available again. - ix = db.NewIndexer() + ix = db.GetIndexer() } + result, err := ix.Search(ctx, opts) if err != nil { return nil, 0, err } + return SearchResultToIDSlice(result), result.Total, nil +} +func SearchResultToIDSlice(result *internal.SearchResult) []int64 { ret := make([]int64, 0, len(result.Hits)) for _, hit := range result.Hits { ret = append(ret, hit.ID) } - - return ret, result.Total, nil + return ret } // CountIssues counts issues by options. It is a shortcut of SearchIssues(ctx, opts) but only returns the total count. diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go index 067a6f609b..199d493e97 100644 --- a/modules/indexer/stats/db.go +++ b/modules/indexer/stats/db.go @@ -8,6 +8,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/languagestats" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" @@ -62,7 +63,7 @@ func (db *DBIndexer) Index(id int64) error { } // Calculate and save language statistics to database - stats, err := gitRepo.GetLanguageStats(commitID) + stats, err := languagestats.GetLanguageStats(gitRepo, commitID) if err != nil { if !setting.IsInTesting { log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) diff --git a/modules/label/label.go b/modules/label/label.go index d3ef0e1dc9..ce028aa9f3 100644 --- a/modules/label/label.go +++ b/modules/label/label.go @@ -14,10 +14,11 @@ var colorPattern = regexp.MustCompile("^#?(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{3})$") // Label represents label information loaded from template type Label struct { - Name string `yaml:"name"` - Color string `yaml:"color"` - Description string `yaml:"description,omitempty"` - Exclusive bool `yaml:"exclusive,omitempty"` + Name string `yaml:"name"` + Color string `yaml:"color"` + Description string `yaml:"description,omitempty"` + Exclusive bool `yaml:"exclusive,omitempty"` + ExclusiveOrder int `yaml:"exclusive_order,omitempty"` } // NormalizeColor normalizes a color string to a 6-character hex code diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index f708457853..39861ade12 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,11 +12,9 @@ import ( "runtime" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) // RegisterRenderers registers all supported third part renderers according settings @@ -88,16 +86,11 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. if p.IsInputFile { // write to temp file - f, err := os.CreateTemp("", "gitea_input") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } - tmpPath := f.Name() - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) - } - }() + defer cleanup() _, err = io.Copy(f, input) if err != nil { diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 4ab45edcec..0cd657cd44 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -6,6 +6,7 @@ package packages import ( "io" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util/filebuffer" ) @@ -34,11 +35,11 @@ func NewHashedBuffer() (*HashedBuffer, error) { // NewHashedBufferWithSize creates a hashed buffer with a specific memory size func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { - b, err := filebuffer.New(maxMemorySize) + tempDir, err := setting.AppDataTempDir("package-hashed-buffer").MkdirAllSub("") if err != nil { return nil, err } - + b := filebuffer.New(maxMemorySize, tempDir) hash := NewMultiHasher() combinedWriter := io.MultiWriter(b, hash) diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index 564e782f18..5104c1fb25 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -9,10 +9,13 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestHashedBuffer(t *testing.T) { + setting.AppDataPath = t.TempDir() cases := []struct { MaxMemorySize int Data string diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go index fa1b80ee82..711ad6d096 100644 --- a/modules/packages/nuget/symbol_extractor_test.go +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -9,6 +9,8 @@ import ( "encoding/base64" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) @@ -17,6 +19,7 @@ fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` func TestExtractPortablePdb(t *testing.T) { + setting.AppDataPath = t.TempDir() createArchive := func(name string, content []byte) []byte { var buf bytes.Buffer archive := zip.NewWriter(&buf) diff --git a/modules/repository/commits.go b/modules/repository/commits.go index 16520fb28a..878fdc1603 100644 --- a/modules/repository/commits.go +++ b/modules/repository/commits.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/cachegroup" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -131,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string { size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor - v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) { + v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) { u, err := user_model.GetUserByEmail(ctx, email) if err != nil { if !user_model.IsErrUserNotExist(err) { diff --git a/modules/repository/init.go b/modules/repository/init.go index e6331966ba..91d4889782 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -127,10 +127,11 @@ func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg labels := make([]*issues_model.Label, len(list)) for i := 0; i < len(list); i++ { labels[i] = &issues_model.Label{ - Name: list[i].Name, - Exclusive: list[i].Exclusive, - Description: list[i].Description, - Color: list[i].Color, + Name: list[i].Name, + Exclusive: list[i].Exclusive, + ExclusiveOrder: list[i].ExclusiveOrder, + Description: list[i].Description, + Color: list[i].Color, } if isOrg { labels[i].OrgID = id diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 76b9bda4ad..d7253d9e02 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -4,41 +4,19 @@ package repository import ( + "context" "fmt" - "os" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -// LocalCopyPath returns the local repository temporary copy path. -func LocalCopyPath() string { - if filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return setting.Repository.Local.LocalCopyPath - } - return filepath.Join(setting.AppDataPath, setting.Repository.Local.LocalCopyPath) -} - // CreateTemporaryPath creates a temporary path -func CreateTemporaryPath(prefix string) (string, error) { - if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) - return "", fmt.Errorf("Failed to create localcopypath directory %s: %w", LocalCopyPath(), err) - } - basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") +func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { + basePath, cleanup, err := setting.AppDataTempDir("local-repo").MkdirTempRandom(prefix + ".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) - return "", fmt.Errorf("Failed to create dir %s-*.git: %w", prefix, err) + return "", nil, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } - return basePath, nil -} - -// RemoveTemporaryPath removes the temporary path -func RemoveTemporaryPath(basePath string) error { - if _, err := os.Stat(basePath); !os.IsNotExist(err) { - return util.RemoveAll(basePath) - } - return nil + return basePath, cleanup, nil } diff --git a/modules/setting/markup.go b/modules/setting/markup.go index 3bd368f831..365af05fcf 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -127,7 +127,7 @@ func loadMarkupFrom(rootCfg ConfigProvider) { } } - MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(5000) + MermaidMaxSourceCharacters = rootCfg.Section("markup").Key("MERMAID_MAX_SOURCE_CHARACTERS").MustInt(50000) ExternalMarkupRenderers = make([]*MarkupRenderer, 0, 10) ExternalSanitizerRules = make([]MarkupSanitizerRule, 0, 10) diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3f618cfd64..845f6d4e12 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -6,8 +6,6 @@ package setting import ( "fmt" "math" - "os" - "path/filepath" "github.com/dustin/go-humanize" ) @@ -67,14 +65,10 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { return err } - Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) - if !filepath.IsAbs(Packages.ChunkedUploadPath) { - Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) - } - if HasInstallLock(rootCfg) { - if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) + Packages.ChunkedUploadPath, err = AppDataTempDir("package-upload").MkdirAllSub("") + if err != nil { + return fmt.Errorf("unable to create chunked upload directory: %w", err) } } diff --git a/modules/setting/path.go b/modules/setting/path.go index 0fdc305aa1..f51457a620 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/tempdir" ) var ( @@ -196,3 +197,18 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP CustomPath = tmpCustomPath.Value CustomConf = tmpCustomConf.Value } + +// AppDataTempDir returns a managed temporary directory for the application data. +// Using empty sub will get the managed base temp directory, and it's safe to delete it. +// Gitea only creates subdirectories under it, but not the APP_TEMP_PATH directory itself. +// * When APP_TEMP_PATH="/tmp": the managed temp directory is "/tmp/gitea-tmp" +// * When APP_TEMP_PATH is not set: the managed temp directory is "/{APP_DATA_PATH}/tmp" +func AppDataTempDir(sub string) *tempdir.TempDir { + if appTempPathInternal != "" { + return tempdir.New(appTempPathInternal, "gitea-tmp/"+sub) + } + if AppDataPath == "" { + panic("setting.AppDataPath is not set") + } + return tempdir.New(AppDataPath, "tmp/"+sub) +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 43bfb3256d..f99c854e4f 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -62,17 +62,11 @@ var ( // Repository upload settings Upload struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int } `ini:"-"` - // Repository local settings - Local struct { - LocalCopyPath string - } `ini:"-"` - // Pull request settings PullRequest struct { WorkInProgressPrefixes []string @@ -181,25 +175,16 @@ var ( // Repository upload settings Upload: struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int }{ Enabled: true, - TempPath: "data/tmp/uploads", AllowedTypes: "", FileMaxSize: 50, MaxFiles: 5, }, - // Repository local settings - Local: struct { - LocalCopyPath string - }{ - LocalCopyPath: "tmp/local-repo", - }, - // Pull request settings PullRequest: struct { WorkInProgressPrefixes []string @@ -308,8 +293,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { log.Fatal("Failed to map Repository.Editor settings: %v", err) } else if err = rootCfg.Section("repository.upload").MapTo(&Repository.Upload); err != nil { log.Fatal("Failed to map Repository.Upload settings: %v", err) - } else if err = rootCfg.Section("repository.local").MapTo(&Repository.Local); err != nil { - log.Fatal("Failed to map Repository.Local settings: %v", err) } else if err = rootCfg.Section("repository.pull-request").MapTo(&Repository.PullRequest); err != nil { log.Fatal("Failed to map Repository.PullRequest settings: %v", err) } @@ -361,10 +344,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } } - if !filepath.IsAbs(Repository.Upload.TempPath) { - Repository.Upload.TempPath = filepath.Join(AppWorkPath, Repository.Upload.TempPath) - } - if err := loadRepoArchiveFrom(rootCfg); err != nil { log.Fatal("loadRepoArchiveFrom: %v", err) } diff --git a/modules/setting/server.go b/modules/setting/server.go index e15b790906..ca635c8abe 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "net" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -59,6 +60,8 @@ var ( // AssetVersion holds a opaque value that is used for cache-busting assets AssetVersion string + appTempPathInternal string // the temporary path for the app, it is only an internal variable, do not use it, always use AppDataTempDir + Protocol Scheme UseProxyProtocol bool // `ini:"USE_PROXY_PROTOCOL"` ProxyProtocolTLSBridging bool //`ini:"PROXY_PROTOCOL_TLS_BRIDGING"` @@ -330,6 +333,19 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(AppDataPath) { AppDataPath = filepath.ToSlash(filepath.Join(AppWorkPath, AppDataPath)) } + if IsInTesting && HasInstallLock(rootCfg) { + // FIXME: in testing, the "app data" directory is not correctly initialized before loading settings + if _, err := os.Stat(AppDataPath); err != nil { + _ = os.MkdirAll(AppDataPath, os.ModePerm) + } + } + + appTempPathInternal = sec.Key("APP_TEMP_PATH").String() + if appTempPathInternal != "" { + if _, err := os.Stat(appTempPathInternal); err != nil { + log.Fatal("APP_TEMP_PATH %q is not accessible: %v", appTempPathInternal, err) + } + } EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index 46eb49bfd4..da8cdf58d2 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,7 +4,6 @@ package setting import ( - "os" "path/filepath" "strings" "text/template" @@ -31,8 +30,6 @@ var SSH = struct { ServerKeyExchanges []string `ini:"SSH_SERVER_KEY_EXCHANGES"` ServerMACs []string `ini:"SSH_SERVER_MACS"` ServerHostKeys []string `ini:"SSH_SERVER_HOST_KEYS"` - KeyTestPath string `ini:"SSH_KEY_TEST_PATH"` - KeygenPath string `ini:"SSH_KEYGEN_PATH"` AuthorizedKeysBackup bool `ini:"SSH_AUTHORIZED_KEYS_BACKUP"` AuthorizedPrincipalsBackup bool `ini:"SSH_AUTHORIZED_PRINCIPALS_BACKUP"` AuthorizedKeysCommandTemplate string `ini:"SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE"` @@ -57,7 +54,6 @@ var SSH = struct { ServerCiphers: []string{"chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"}, ServerKeyExchanges: []string{"curve25519-sha256", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1"}, ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1"}, - KeygenPath: "", MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, @@ -123,7 +119,6 @@ func loadSSHFrom(rootCfg ConfigProvider) { if len(serverMACs) > 0 { SSH.ServerMACs = serverMACs } - SSH.KeyTestPath = os.TempDir() if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } @@ -133,7 +128,6 @@ func loadSSHFrom(rootCfg ConfigProvider) { } } - SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) SSH.UseProxyProtocol = sec.Key("SSH_SERVER_USE_PROXY_PROTOCOL").MustBool(false) diff --git a/modules/ssh/init.go b/modules/ssh/init.go index 21d4f89936..fdc11632e2 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -32,11 +32,6 @@ func Init() error { builtinUnused() - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(setting.SSH.KeyTestPath, 0o644); err != nil { - return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.SSH.KeyTestPath, err) - } - if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go new file mode 100644 index 0000000000..22c2e4ea16 --- /dev/null +++ b/modules/tempdir/tempdir.go @@ -0,0 +1,112 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +type TempDir struct { + // base is the base directory for temporary files, it must exist before accessing and won't be created automatically. + // for example: base="/system-tmpdir", sub="gitea-tmp" + base, sub string +} + +func (td *TempDir) JoinPath(elems ...string) string { + return filepath.Join(append([]string{td.base, td.sub}, elems...)...) +} + +// MkdirAllSub works like os.MkdirAll, but the base directory must exist +func (td *TempDir) MkdirAllSub(dir string) (string, error) { + if _, err := os.Stat(td.base); err != nil { + return "", err + } + full := filepath.Join(td.base, td.sub, dir) + if err := os.MkdirAll(full, os.ModePerm); err != nil { + return "", err + } + return full, nil +} + +func (td *TempDir) prepareDirWithPattern(elems ...string) (dir, pattern string, err error) { + if _, err = os.Stat(td.base); err != nil { + return "", "", err + } + dir, pattern = filepath.Split(filepath.Join(append([]string{td.base, td.sub}, elems...)...)) + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return "", "", err + } + return dir, pattern, nil +} + +// MkdirTempRandom works like os.MkdirTemp, the last path field is the "pattern" +func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return "", nil, err + } + dir, err = os.MkdirTemp(dir, pattern) + if err != nil { + return "", nil, err + } + return dir, func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("Failed to remove temp directory %s: %v", dir, err) + } + }, nil +} + +// CreateTempFileRandom works like os.CreateTemp, the last path field is the "pattern" +func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return nil, nil, err + } + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, nil, err + } + filename := f.Name() + return f, func() { + _ = f.Close() + if err := util.Remove(filename); err != nil { + log.Error("Unable to remove temporary file: %s: Error: %v", filename, err) + } + }, err +} + +func (td *TempDir) RemoveOutdated(d time.Duration) { + var remove func(path string) + remove = func(path string) { + entries, _ := os.ReadDir(path) + for _, entry := range entries { + full := filepath.Join(path, entry.Name()) + if entry.IsDir() { + remove(full) + _ = os.Remove(full) + continue + } + info, err := entry.Info() + if err == nil && time.Since(info.ModTime()) > d { + _ = os.Remove(full) + } + } + } + remove(td.JoinPath("")) +} + +// New create a new TempDir instance, "base" must be an existing directory, +// "sub" could be a multi-level directory and will be created if not exist +func New(base, sub string) *TempDir { + return &TempDir{base: base, sub: sub} +} + +func OsTempDir(sub string) *TempDir { + return New(os.TempDir(), sub) +} diff --git a/modules/tempdir/tempdir_test.go b/modules/tempdir/tempdir_test.go new file mode 100644 index 0000000000..d6afcb7bed --- /dev/null +++ b/modules/tempdir/tempdir_test.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTempDir(t *testing.T) { + base := t.TempDir() + + t.Run("Create", func(t *testing.T) { + td := New(base, "sub1/sub2") // make sure the sub dir supports "/" in the path + assert.Equal(t, filepath.Join(base, "sub1", "sub2"), td.JoinPath()) + assert.Equal(t, filepath.Join(base, "sub1", "sub2/test"), td.JoinPath("test")) + + t.Run("MkdirTempRandom", func(t *testing.T) { + s, cleanup, err := td.MkdirTempRandom("foo") + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(s, filepath.Join(base, "sub1/sub2", "foo"))) + + _, err = os.Stat(s) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(s) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("CreateTempFileRandom", func(t *testing.T) { + f, cleanup, err := td.CreateTempFileRandom("foo", "bar") + filename := f.Name() + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filename, filepath.Join(base, "sub1/sub2", "foo", "bar"))) + _, err = os.Stat(filename) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(filename) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("RemoveOutDated", func(t *testing.T) { + fa1, _, err := td.CreateTempFileRandom("dir-a", "f1") + assert.NoError(t, err) + fa2, _, err := td.CreateTempFileRandom("dir-a", "f2") + assert.NoError(t, err) + _ = os.Chtimes(fa2.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + fb1, _, err := td.CreateTempFileRandom("dir-b", "f1") + assert.NoError(t, err) + _ = os.Chtimes(fb1.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + _, _, _ = fa1.Close(), fa2.Close(), fb1.Close() + + td.RemoveOutdated(time.Minute) + + _, err = os.Stat(fa1.Name()) + assert.NoError(t, err) + _, err = os.Stat(fa2.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(fb1.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + }) + + t.Run("BaseNotExist", func(t *testing.T) { + td := New(filepath.Join(base, "not-exist"), "sub") + _, _, err := td.MkdirTempRandom("foo") + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} diff --git a/modules/templates/util_render.go b/modules/templates/util_render.go index ae397d87c9..521233db40 100644 --- a/modules/templates/util_render.go +++ b/modules/templates/util_render.go @@ -170,13 +170,28 @@ func (ut *RenderUtils) RenderLabel(label *issues_model.Label) template.HTML { itemColor := "#" + hex.EncodeToString(itemBytes) scopeColor := "#" + hex.EncodeToString(scopeBytes) + if label.ExclusiveOrder > 0 { + // |