nordic-dev.net/gitea
nordic-dev.net/gitea
2
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2025-04-16 14:19:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a config option to block "expensive" pages for anonymous users (#34024) (#34071)

Browse Source 
Backport #34024 since there are too many AI crawlers. The new code is
covered by tests and it does nothing if users don't set it.
This commit is contained in:
wxiaoguang 2025-03-30 14:16:32 +08:00 committed by GitHub
parent 5a9b3bfa50
commit 15e93a751c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 228 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
custom/conf/app.example.ini
View File

@ -774,6 +774,9 @@ LEVEL = Info
;ALLOW_ONLY_EXTERNAL_REGISTRATION = false
;;
;; User must sign in to view anything.
;; After 1.23.7, it could be set to "expensive" to block anonymous users accessing some pages which consume a lot of resources,
;; for example: block anonymous AI crawlers from accessing repo code pages.
;; The "expensive" mode is experimental and subject to change.
;REQUIRE_SIGNIN_VIEW = false
;;
;; Mail notification

1
modules/setting/config_provider.go
View File

@ -26,6 +26,7 @@ type ConfigKey interface {
In(defaultVal string, candidates []string) string
String() string
Strings(delim string) []string
Bool() (bool, error)
MustString(defaultVal string) string
MustBool(defaultVal ...bool) bool

16
modules/setting/service.go
View File

@ -43,7 +43,8 @@ var Service = struct {
ShowRegistrationButton bool
EnablePasswordSignInForm bool
ShowMilestonesDashboardPage bool
RequireSignInView bool
RequireSignInViewStrict bool
BlockAnonymousAccessExpensive bool
EnableNotifyMail bool
EnableBasicAuth bool
EnablePasskeyAuth bool
@ -159,7 +160,18 @@ func loadServiceFrom(rootCfg ConfigProvider) {
Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST")
Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration))
Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true)
Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool()
// boolean values are considered as "strict"
var err error
Service.RequireSignInViewStrict, err = sec.Key("REQUIRE_SIGNIN_VIEW").Bool()
if s := sec.Key("REQUIRE_SIGNIN_VIEW").String(); err != nil && s != "" {
// non-boolean value only supports "expensive" at the moment
Service.BlockAnonymousAccessExpensive = s == "expensive"
if !Service.BlockAnonymousAccessExpensive {
log.Error("Invalid config option: REQUIRE_SIGNIN_VIEW = %s", s)
}
}
Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true)
Service.EnablePasswordSignInForm = sec.Key("ENABLE_PASSWORD_SIGNIN_FORM").MustBool(true)
Service.EnablePasskeyAuth = sec.Key("ENABLE_PASSKEY_AUTHENTICATION").MustBool(true)

41
modules/setting/service_test.go
View File

@ -7,16 +7,14 @@ import (
"testing"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test"
"github.com/gobwas/glob"
"github.com/stretchr/testify/assert"
)
func TestLoadServices(t *testing.T) {
oldService := Service
defer func() {
Service = oldService
}()
defer test.MockVariableValue(&Service)()
cfg, err := NewConfigProviderFromData(`
[service]
@ -48,10 +46,7 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b
}
func TestLoadServiceVisibilityModes(t *testing.T) {
oldService := Service
defer func() {
Service = oldService
}()
defer test.MockVariableValue(&Service)()
kases := map[string]func(){
`
@ -130,3 +125,33 @@ ALLOWED_USER_VISIBILITY_MODES = public, limit, privated
})
}
}
func TestLoadServiceRequireSignInView(t *testing.T) {
defer test.MockVariableValue(&Service)()
cfg, err := NewConfigProviderFromData(`
[service]
`)
assert.NoError(t, err)
loadServiceFrom(cfg)
assert.False(t, Service.RequireSignInViewStrict)
assert.False(t, Service.BlockAnonymousAccessExpensive)
cfg, err = NewConfigProviderFromData(`
[service]
REQUIRE_SIGNIN_VIEW = true
`)
assert.NoError(t, err)
loadServiceFrom(cfg)
assert.True(t, Service.RequireSignInViewStrict)
assert.False(t, Service.BlockAnonymousAccessExpensive)
cfg, err = NewConfigProviderFromData(`
[service]
REQUIRE_SIGNIN_VIEW = expensive
`)
assert.NoError(t, err)
loadServiceFrom(cfg)
assert.False(t, Service.RequireSignInViewStrict)
assert.True(t, Service.BlockAnonymousAccessExpensive)
}

2
routers/api/packages/cargo/cargo.go
View File

@ -51,7 +51,7 @@ func apiError(ctx *context.Context, status int, obj any) {
// https://rust-lang.github.io/rfcs/2789-sparse-index.html
func RepositoryConfig(ctx *context.Context) {
ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner, setting.Service.RequireSignInView || ctx.Package.Owner.Visibility != structs.VisibleTypePublic))
ctx.JSON(http.StatusOK, cargo_service.BuildConfig(ctx.Package.Owner, setting.Service.RequireSignInViewStrict || ctx.Package.Owner.Visibility != structs.VisibleTypePublic))
}
func EnumeratePackageVersions(ctx *context.Context) {

4
routers/api/packages/container/container.go
View File

@ -126,7 +126,7 @@ func apiUnauthorizedError(ctx *context.Context) {
// ReqContainerAccess is a middleware which checks the current user valid (real user or ghost if anonymous access is enabled)
func ReqContainerAccess(ctx *context.Context) {
if ctx.Doer == nil || (setting.Service.RequireSignInView && ctx.Doer.IsGhost()) {
if ctx.Doer == nil || (setting.Service.RequireSignInViewStrict && ctx.Doer.IsGhost()) {
apiUnauthorizedError(ctx)
}
}
@ -152,7 +152,7 @@ func Authenticate(ctx *context.Context) {
u := ctx.Doer
packageScope := auth_service.GetAccessScope(ctx.Data)
if u == nil {
if setting.Service.RequireSignInView {
if setting.Service.RequireSignInViewStrict {
apiUnauthorizedError(ctx)
return
}

4
routers/api/v1/api.go
View File

@ -356,7 +356,7 @@ func reqToken() func(ctx *context.APIContext) {
func reqExploreSignIn() func(ctx *context.APIContext) {
return func(ctx *context.APIContext) {
if (setting.Service.RequireSignInView || setting.Service.Explore.RequireSigninView) && !ctx.IsSigned {
if (setting.Service.RequireSignInViewStrict || setting.Service.Explore.RequireSigninView) && !ctx.IsSigned {
ctx.Error(http.StatusUnauthorized, "reqExploreSignIn", "you must be signed in to search for users")
}
}
@ -874,7 +874,7 @@ func Routes() *web.Router {
m.Use(apiAuth(buildAuthGroup()))
m.Use(verifyAuthWithOptions(&common.VerifyOptions{
SignInRequired: setting.Service.RequireSignInView,
SignInRequired: setting.Service.RequireSignInViewStrict,
}))
addActionsRoutes := func(

91
routers/common/blockexpensive.go Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"context"
"net/http"
"strings"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/web/middleware"
"github.com/go-chi/chi/v5"
)
func BlockExpensive() func(next http.Handler) http.Handler {
if !setting.Service.BlockAnonymousAccessExpensive {
return nil
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ret := determineRequestPriority(req.Context())
if !ret.SignedIn {
if ret.Expensive || ret.LongPolling {
http.Redirect(w, req, setting.AppSubURL+"/user/login", http.StatusSeeOther)
return
}
}
next.ServeHTTP(w, req)
})
}
}
func isRoutePathExpensive(routePattern string) bool {
if strings.HasPrefix(routePattern, "/user/") || strings.HasPrefix(routePattern, "/login/") {
return false
}
expensivePaths := []string{
// code related
"/{username}/{reponame}/archive/",
"/{username}/{reponame}/blame/",
"/{username}/{reponame}/commit/",
"/{username}/{reponame}/commits/",
"/{username}/{reponame}/graph",
"/{username}/{reponame}/media/",
"/{username}/{reponame}/raw/",
"/{username}/{reponame}/src/",
// issue & PR related (no trailing slash)
"/{username}/{reponame}/issues",
"/{username}/{reponame}/{type:issues}",
"/{username}/{reponame}/pulls",
"/{username}/{reponame}/{type:pulls}",
// wiki
"/{username}/{reponame}/wiki/",
// activity
"/{username}/{reponame}/activity/",
}
for _, path := range expensivePaths {
if strings.HasPrefix(routePattern, path) {
return true
}
}
return false
}
func isRoutePathForLongPolling(routePattern string) bool {
return routePattern == "/user/events"
}
func determineRequestPriority(ctx context.Context) (ret struct {
SignedIn bool
Expensive bool
LongPolling bool
},
) {
dataStore := middleware.GetContextData(ctx)
chiRoutePath := chi.RouteContext(ctx).RoutePattern()
if _, ok := dataStore[middleware.ContextDataKeySignedUser].(*user_model.User); ok {
ret.SignedIn = true
} else {
ret.Expensive = isRoutePathExpensive(chiRoutePath)
ret.LongPolling = isRoutePathForLongPolling(chiRoutePath)
}
return ret
}

30
routers/common/blockexpensive_test.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestBlockExpensive(t *testing.T) {
cases := []struct {
expensive bool
routePath string
}{
{false, "/user/xxx"},
{false, "/login/xxx"},
{true, "/{username}/{reponame}/archive/xxx"},
{true, "/{username}/{reponame}/graph"},
{true, "/{username}/{reponame}/src/xxx"},
{true, "/{username}/{reponame}/wiki/xxx"},
{true, "/{username}/{reponame}/activity/xxx"},
}
for _, c := range cases {
assert.Equal(t, c.expensive, isRoutePathExpensive(c.routePath), "routePath: %s", c.routePath)
}
assert.True(t, isRoutePathForLongPolling("/user/events"))
}

2
routers/install/install.go
View File

@ -156,7 +156,7 @@ func Install(ctx *context.Context) {
form.DisableRegistration = setting.Service.DisableRegistration
form.AllowOnlyExternalRegistration = setting.Service.AllowOnlyExternalRegistration
form.EnableCaptcha = setting.Service.EnableCaptcha
form.RequireSignInView = setting.Service.RequireSignInView
form.RequireSignInView = setting.Service.RequireSignInViewStrict
form.DefaultKeepEmailPrivate = setting.Service.DefaultKeepEmailPrivate
form.DefaultAllowCreateOrganization = setting.Service.DefaultAllowCreateOrganization
form.DefaultEnableTimetracking = setting.Service.DefaultEnableTimetracking

2
routers/private/serv.go
View File

@ -286,7 +286,7 @@ func ServCommand(ctx *context.PrivateContext) {
repo.IsPrivate ||
owner.Visibility.IsPrivate() ||
(user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey
setting.Service.RequireSignInView) {
setting.Service.RequireSignInViewStrict) {
if key.Type == asymkey_model.KeyTypeDeploy {
if deployKey.Mode < mode {
ctx.JSON(http.StatusUnauthorized, private.Response{

2
routers/web/repo/githttp.go
View File

@ -127,7 +127,7 @@ func httpBase(ctx *context.Context) *serviceHandler {
// Only public pull don't need auth.
isPublicPull := repoExist && !repo.IsPrivate && isPull
var (
askAuth = !isPublicPull || setting.Service.RequireSignInView
askAuth = !isPublicPull || setting.Service.RequireSignInViewStrict
environ []string
)

16
routers/web/web.go
View File

@ -285,23 +285,23 @@ func Routes() *web.Router {
mid = append(mid, repo.GetActiveStopwatch)
mid = append(mid, goGet)
others := web.NewRouter()
others.Use(mid...)
registerRoutes(others)
routes.Mount("", others)
webRoutes := web.NewRouter()
webRoutes.Use(mid...)
webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive())
routes.Mount("", webRoutes)
return routes
}
var optSignInIgnoreCsrf = verifyAuthWithOptions(&common.VerifyOptions{DisableCSRF: true})
// registerRoutes register routes
func registerRoutes(m *web.Router) {
// registerWebRoutes register routes
func registerWebRoutes(m *web.Router) {
// required to be signed in or signed out
reqSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: true})
reqSignOut := verifyAuthWithOptions(&common.VerifyOptions{SignOutRequired: true})
// optional sign in (if signed in, use the user as doer, if not, no doer)
optSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInView})
optExploreSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInView || setting.Service.Explore.RequireSigninView})
optSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInViewStrict})
optExploreSignIn := verifyAuthWithOptions(&common.VerifyOptions{SignInRequired: setting.Service.RequireSignInViewStrict || setting.Service.Explore.RequireSigninView})
validation.AddBindingRules()

2
services/context/package.go
View File

@ -93,7 +93,7 @@ func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any))
}
func determineAccessMode(ctx *Base, pkg *Package, doer *user_model.User) (perm.AccessMode, error) {
if setting.Service.RequireSignInView && (doer == nil || doer.IsGhost()) {
if setting.Service.RequireSignInViewStrict && (doer == nil || doer.IsGhost()) {
return perm.AccessModeNone, nil
}

2
services/packages/cargo/index.go
View File

@ -248,7 +248,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository,
"Initialize Cargo Config",
func(t *files_service.TemporaryUploadRepository) error {
var b bytes.Buffer
err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInView || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate))
err := json.NewEncoder(&b).Encode(BuildConfig(owner, setting.Service.RequireSignInViewStrict || owner.Visibility != structs.VisibleTypePublic || repo.IsPrivate))
if err != nil {
return err
}

2
templates/admin/config.tmpl
View File

@ -148,7 +148,7 @@
<dt>{{ctx.Locale.Tr "admin.config.enable_openid_signin"}}</dt>
<dd>{{svg (Iif .Service.EnableOpenIDSignIn "octicon-check" "octicon-x")}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.require_sign_in_view"}}</dt>
<dd>{{svg (Iif .Service.RequireSignInView "octicon-check" "octicon-x")}}</dd>
<dd>{{svg (Iif .Service.RequireSignInViewStrict "octicon-check" "octicon-x")}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.mail_notify"}}</dt>
<dd>{{svg (Iif .Service.EnableNotifyMail "octicon-check" "octicon-x")}}</dd>
<dt>{{ctx.Locale.Tr "admin.config.enable_captcha"}}</dt>

4
tests/integration/api_org_test.go
View File

@ -148,9 +148,9 @@ func TestAPIOrgEditBadVisibility(t *testing.T) {
func TestAPIOrgDeny(t *testing.T) {
onGiteaRun(t, func(*testing.T, *url.URL) {
setting.Service.RequireSignInView = true
setting.Service.RequireSignInViewStrict = true
defer func() {
setting.Service.RequireSignInView = false
setting.Service.RequireSignInViewStrict = false
}()
orgName := "user1_org"

2
tests/integration/api_packages_container_test.go
View File

@ -111,7 +111,7 @@ func TestPackageContainer(t *testing.T) {
AddTokenAuth(anonymousToken)
MakeRequest(t, req, http.StatusOK)
defer test.MockVariableValue(&setting.Service.RequireSignInView, true)()
defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
req = NewRequest(t, "GET", fmt.Sprintf("%sv2", setting.AppURL))
MakeRequest(t, req, http.StatusUnauthorized)

7
tests/integration/api_packages_generic_test.go
View File

@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
@ -131,11 +132,7 @@ func TestPackageGeneric(t *testing.T) {
t.Run("RequireSignInView", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
setting.Service.RequireSignInView = true
defer func() {
setting.Service.RequireSignInView = false
}()
defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
req = NewRequest(t, "GET", url+"/dummy.bin")
MakeRequest(t, req, http.StatusUnauthorized)

2
tests/integration/git_smart_http_test.go
View File

@ -74,7 +74,7 @@ func testGitSmartHTTP(t *testing.T, u *url.URL) {
}
func testRenamedRepoRedirect(t *testing.T) {
defer test.MockVariableValue(&setting.Service.RequireSignInView, true)()
defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
// git client requires to get a 301 redirect response before 401 unauthorized response
req := NewRequest(t, "GET", "/user2/oldrepo1/info/refs")

31
tests/integration/signin_test.go
View File

@ -15,11 +15,13 @@ import (
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/modules/translation"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers"
"code.gitea.io/gitea/services/context"
"code.gitea.io/gitea/tests"
"github.com/markbates/goth"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testLoginFailed(t *testing.T, username, password, message string) {
@ -158,3 +160,32 @@ func TestEnablePasswordSignInFormAndEnablePasskeyAuth(t *testing.T) {
NewHTMLParser(t, resp.Body).AssertElement(t, ".signin-passkey", true)
})
}
func TestRequireSignInView(t *testing.T) {
defer tests.PrepareTestEnv(t)()
t.Run("NoRequireSignInView", func(t *testing.T) {
require.False(t, setting.Service.RequireSignInViewStrict)
require.False(t, setting.Service.BlockAnonymousAccessExpensive)
req := NewRequest(t, "GET", "/user2/repo1/src/branch/master")
MakeRequest(t, req, http.StatusOK)
})
t.Run("RequireSignInView", func(t *testing.T) {
defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)()
defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
req := NewRequest(t, "GET", "/user2/repo1/src/branch/master")
resp := MakeRequest(t, req, http.StatusSeeOther)
assert.Equal(t, "/user/login", resp.Header().Get("Location"))
})
t.Run("BlockAnonymousAccessExpensive", func(t *testing.T) {
defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, false)()
defer test.MockVariableValue(&setting.Service.BlockAnonymousAccessExpensive, true)()
defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
req := NewRequest(t, "GET", "/user2/repo1")
MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", "/user2/repo1/src/branch/master")
resp := MakeRequest(t, req, http.StatusSeeOther)
assert.Equal(t, "/user/login", resp.Header().Get("Location"))
})
}