Merge remote-tracking branch 'upstream/release/v1.16' into codeberg-1.16

Author: ashimokawa

CHANGELOG.md

@@ -4,6 +4,28 @@ This changelog goes through all the changes that have been made in each release
 without substantial changes to our git log; to see the highlights of what has
 been added to each release, please refer to the [blog](https://blog.gitea.io).
 
+## [1.16.4](https://github.com/go-gitea/gitea/releases/tag/v1.16.4) - 2022-03-14
+
+* SECURITY
+  * Restrict email address validation (#17688) (#19085)
+  * Fix lfs bug (#19072) (#19080)
+* ENHANCEMENTS
+  * Improve SyncMirrors logging (#19045) (#19050)
+* BUGFIXES
+  * Refactor mirror code & fix `StartToMirror` (#18904) (#19075)
+  * Update the webauthn_credential_id_sequence in Postgres (#19048) (#19060)
+  * Prevent 500 when there is an error during new auth source post (#19041) (#19059)
+  * If rendering has failed due to a net.OpError stop rendering (attempt 2) (#19049) (#19056)
+  * Fix flag validation (#19046) (#19051)
+  * Add pam account authorization check (#19040) (#19047)
+  * Ignore missing comment for user notifications (#18954) (#19043)
+  * Set `rel="nofollow noindex"` on new issue links (#19023) (#19042)
+  * Upgrading binding package (#19034) (#19035)
+  * Don't show context cancelled errors in attribute reader (#19006) (#19027)
+  * Fix update hint bug (#18996) (#19002)
+* MISC
+  *  Fix potential assignee query for repo (#18994) (#18999)
+
 ## [1.16.3](https://github.com/go-gitea/gitea/releases/tag/v1.16.3) - 2022-03-02
 
 * SECURITY

cmd/cmd.go

@@ -31,7 +31,7 @@ func argsSet(c *cli.Context, args ...string) error {
 			return errors.New(a + " is not set")
 		}
 
-		if util.IsEmptyString(a) {
+		if util.IsEmptyString(c.String(a)) {
 			return errors.New(a + " is required")
 		}
 	}

docs/content/doc/developers/guidelines-backend.md

@@ -42,7 +42,7 @@ To maintain understandable code and avoid circular dependencies it is important
   - `modules/setting`: Store all system configurations read from ini files and has been referenced by everywhere. But they should be used as function parameters when possible.
   - `modules/git`: Package to interactive with `Git` command line or Gogit package.
 - `public`: Compiled frontend files (javascript, images, css, etc.)
-- `routers`: Handling of server requests. As it uses other Gitea packages to serve the request, other packages (models, modules or services) shall not depend on routers.
+- `routers`: Handling of server requests. As it uses other Gitea packages to serve the request, other packages (models, modules or services) must not depend on routers.
   - `routers/api` Contains routers for `/api/v1` aims to handle RESTful API requests. 
   - `routers/install` Could only respond when system is in INSTALL mode (INSTALL_LOCK=false). 
   - `routers/private` will only be invoked by internal sub commands, especially `serv` and `hooks`. 
@@ -106,10 +106,20 @@ i.e. `servcies/user`, `models/repository`.
 Since there are some packages which use the same package name, it is possible that you find packages like `modules/user`, `models/user`, and `services/user`. When these packages are imported in one Go file, it's difficult to know which package we are using and if it's a variable name or an import name. So, we always recommend to use import aliases. To differ from package variables which are commonly in camelCase, just use **snake_case** for import aliases.
 i.e. `import user_service "code.gitea.io/gitea/services/user"`
 
+### Important Gotchas
+
+- Never write `x.Update(exemplar)` without an explicit `WHERE` clause:
+  - This will cause all rows in the table to be updated with the non-zero values of the exemplar - including IDs.
+  - You should usually write `x.ID(id).Update(exemplar)`.
+- If during a migration you are inserting into a table using `x.Insert(exemplar)` where the ID is preset:
+  - You will need to ``SET IDENTITY_INSERT `table` ON`` for the MSSQL variant (the migration will fail otherwise)
+  - However, you will also need to update the id sequence for postgres - the migration will silently pass here but later insertions will fail:
+    ``SELECT setval('table_name_id_seq', COALESCE((SELECT MAX(id)+1 FROM `table_name`), 1), false)``
+
 ### Future Tasks
 
 Currently, we are creating some refactors to do the following things:
 
 - Correct that codes which doesn't follow the rules.
 - There are too many files in `models`, so we are moving some of them into a sub package `models/xxx`.
-- Some `modules` sub packages should be moved to `services` because they depends on `models`.
+- Some `modules` sub packages should be moved to `services` because they depend on `models`.

go.mod

@@ -6,7 +6,7 @@ require (
 	cloud.google.com/go v0.78.0 // indirect
 	code.gitea.io/gitea-vet v0.2.1
 	code.gitea.io/sdk/gitea v0.15.1
-	gitea.com/go-chi/binding v0.0.0-20211013065440-d16dc407c2be
+	gitea.com/go-chi/binding v0.0.0-20220309004920-114340dabecb
 	gitea.com/go-chi/cache v0.0.0-20211013020926-78790b11abf1
 	gitea.com/go-chi/captcha v0.0.0-20211013065431-70641c1a35d5
 	gitea.com/go-chi/session v0.0.0-20211218221615-e3605d8b28b8

go.sum

@@ -41,8 +41,8 @@ code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFj
 code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M=
 code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-gitea.com/go-chi/binding v0.0.0-20211013065440-d16dc407c2be h1:IzSwPVzd2hE6e67ujY8ReBCrQ5IFNd0uiBmC7Ux5IaY=
-gitea.com/go-chi/binding v0.0.0-20211013065440-d16dc407c2be/go.mod h1:/vR0YjlusOYvosKYW7QKcSnrY0nPLe4RQ/DGi3+i/Do=
+gitea.com/go-chi/binding v0.0.0-20220309004920-114340dabecb h1:Yy0Bxzc8R2wxiwXoG/rECGplJUSpXqCsog9PuJFgiHs=
+gitea.com/go-chi/binding v0.0.0-20220309004920-114340dabecb/go.mod h1:77TZu701zMXWJFvB8gvTbQ92zQ3DQq/H7l5wAEjQRKc=
 gitea.com/go-chi/cache v0.0.0-20210110083709-82c4c9ce2d5e/go.mod h1:k2V/gPDEtXGjjMGuBJiapffAXTv76H4snSmlJRLUhH0=
 gitea.com/go-chi/cache v0.0.0-20211013020926-78790b11abf1 h1:Z7DcvTkxt8ovcENgPsQ7xzrGNSQmmIjGS9fJEb1l8jk=
 gitea.com/go-chi/cache v0.0.0-20211013020926-78790b11abf1/go.mod h1:k2V/gPDEtXGjjMGuBJiapffAXTv76H4snSmlJRLUhH0=
@@ -489,8 +489,9 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/goccy/go-json v0.7.4 h1:B44qRUFwz/vxPKPISQ1KhvzRi9kZ28RAf6YtjriBZ5k=
 github.com/goccy/go-json v0.7.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.9.5 h1:ooSMW526ZjK+EaL5elrSyN2EzIfi/3V0m4+HJEDYLik=
+github.com/goccy/go-json v0.9.5/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

models/lfs.go

@@ -193,12 +193,13 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
 		// admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key),
 		// even if error occurs, it won't hurt users and won't make things worse
 		for i := range metas {
+			p := lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size}
 			_, err = sess.Insert(&LFSMetaObject{
-				Pointer:      lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size},
+				Pointer:      p,
 				RepositoryID: repoID,
 			})
 			if err != nil {
-				log.Warn("failed to insert LFS meta object into database, err=%v", err)
+				log.Warn("failed to insert LFS meta object %-v for repo_id: %d into database, err=%v", p, repoID, err)
 			}
 		}
 	}

models/migrations/v210.go

@@ -174,5 +174,11 @@ func remigrateU2FCredentials(x *xorm.Engine) error {
 		regs = regs[:0]
 	}
 
+	if x.Dialect().URI().DBType == schemas.POSTGRES {
+		if _, err := x.Exec("SELECT setval('webauthn_credential_id_seq', COALESCE((SELECT MAX(id)+1 FROM `webauthn_credential`), 1), false)"); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

models/notification.go

@@ -498,14 +498,15 @@ func (n *Notification) APIURL() string {
 type NotificationList []*Notification
 
 // LoadAttributes load Repo Issue User and Comment if not loaded
-func (nl NotificationList) LoadAttributes() (err error) {
+func (nl NotificationList) LoadAttributes() error {
+	var err error
 	for i := 0; i < len(nl); i++ {
 		err = nl[i].LoadAttributes()
 		if err != nil && !IsErrCommentNotExist(err) {
-			return
+			return err
 		}
 	}
-	return
+	return nil
 }
 
 func (nl NotificationList) getPendingRepoIDs() []int64 {

models/user/email_address.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"net/mail"
+	"regexp"
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
@@ -21,10 +22,23 @@ import (
 	"xorm.io/builder"
 )
 
-var (
-	// ErrEmailNotActivated e-mail address has not been activated error
-	ErrEmailNotActivated = errors.New("E-mail address has not been activated")
-)
+// ErrEmailNotActivated e-mail address has not been activated error
+var ErrEmailNotActivated = errors.New("e-mail address has not been activated")
+
+// ErrEmailCharIsNotSupported e-mail address contains unsupported character
+type ErrEmailCharIsNotSupported struct {
+	Email string
+}
+
+// IsErrEmailCharIsNotSupported checks if an error is an ErrEmailCharIsNotSupported
+func IsErrEmailCharIsNotSupported(err error) bool {
+	_, ok := err.(ErrEmailCharIsNotSupported)
+	return ok
+}
+
+func (err ErrEmailCharIsNotSupported) Error() string {
+	return fmt.Sprintf("e-mail address contains unsupported character [email: %s]", err.Email)
+}
 
 // ErrEmailInvalid represents an error where the email address does not comply with RFC 5322
 type ErrEmailInvalid struct {
@@ -108,12 +122,24 @@ func (email *EmailAddress) BeforeInsert() {
 	}
 }
 
+var emailRegexp = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+-/=?^_`{|}~]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
+
 // ValidateEmail check if email is a allowed address
 func ValidateEmail(email string) error {
 	if len(email) == 0 {
 		return nil
 	}
 
+	if !emailRegexp.MatchString(email) {
+		return ErrEmailCharIsNotSupported{email}
+	}
+
+	if !(email[0] >= 'a' && email[0] <= 'z') &&
+		!(email[0] >= 'A' && email[0] <= 'Z') &&
+		!(email[0] >= '0' && email[0] <= '9') {
+		return ErrEmailInvalid{email}
+	}
+
 	if _, err := mail.ParseAddress(email); err != nil {
 		return ErrEmailInvalid{email}
 	}

models/user/email_address_test.go

@@ -252,3 +252,58 @@ func TestListEmails(t *testing.T) {
 	assert.Len(t, emails, 5)
 	assert.Greater(t, count, int64(len(emails)))
 }
+
+func TestEmailAddressValidate(t *testing.T) {
+	kases := map[string]error{
+		"abc@gmail.com":                  nil,
+		"132@hotmail.com":                nil,
+		"1-3-2@test.org":                 nil,
+		"1.3.2@test.org":                 nil,
+		"a_123@test.org.cn":              nil,
+		`first.last@iana.org`:            nil,
+		`first!last@iana.org`:            nil,
+		`first#last@iana.org`:            nil,
+		`first$last@iana.org`:            nil,
+		`first%last@iana.org`:            nil,
+		`first&last@iana.org`:            nil,
+		`first'last@iana.org`:            nil,
+		`first*last@iana.org`:            nil,
+		`first+last@iana.org`:            nil,
+		`first/last@iana.org`:            nil,
+		`first=last@iana.org`:            nil,
+		`first?last@iana.org`:            nil,
+		`first^last@iana.org`:            nil,
+		"first`last@iana.org":            nil,
+		`first{last@iana.org`:            nil,
+		`first|last@iana.org`:            nil,
+		`first}last@iana.org`:            nil,
+		`first~last@iana.org`:            nil,
+		`first;last@iana.org`:            ErrEmailCharIsNotSupported{`first;last@iana.org`},
+		".233@qq.com":                    ErrEmailInvalid{".233@qq.com"},
+		"!233@qq.com":                    ErrEmailInvalid{"!233@qq.com"},
+		"#233@qq.com":                    ErrEmailInvalid{"#233@qq.com"},
+		"$233@qq.com":                    ErrEmailInvalid{"$233@qq.com"},
+		"%233@qq.com":                    ErrEmailInvalid{"%233@qq.com"},
+		"&233@qq.com":                    ErrEmailInvalid{"&233@qq.com"},
+		"'233@qq.com":                    ErrEmailInvalid{"'233@qq.com"},
+		"*233@qq.com":                    ErrEmailInvalid{"*233@qq.com"},
+		"+233@qq.com":                    ErrEmailInvalid{"+233@qq.com"},
+		"/233@qq.com":                    ErrEmailInvalid{"/233@qq.com"},
+		"=233@qq.com":                    ErrEmailInvalid{"=233@qq.com"},
+		"?233@qq.com":                    ErrEmailInvalid{"?233@qq.com"},
+		"^233@qq.com":                    ErrEmailInvalid{"^233@qq.com"},
+		"`233@qq.com":                    ErrEmailInvalid{"`233@qq.com"},
+		"{233@qq.com":                    ErrEmailInvalid{"{233@qq.com"},
+		"|233@qq.com":                    ErrEmailInvalid{"|233@qq.com"},
+		"}233@qq.com":                    ErrEmailInvalid{"}233@qq.com"},
+		"~233@qq.com":                    ErrEmailInvalid{"~233@qq.com"},
+		";233@qq.com":                    ErrEmailCharIsNotSupported{";233@qq.com"},
+		"Foo <foo@bar.com>":              ErrEmailCharIsNotSupported{"Foo <foo@bar.com>"},
+		string([]byte{0xE2, 0x84, 0xAA}): ErrEmailCharIsNotSupported{string([]byte{0xE2, 0x84, 0xAA})},
+	}
+	for kase, err := range kases {
+		t.Run(kase, func(t *testing.T) {
+			assert.EqualValues(t, err, ValidateEmail(kase))
+		})
+	}
+}