Cleanup test generation

Author: kdumontnu

docs/content/doc/developers/guidelines-backend.md

@@ -33,8 +33,9 @@ To maintain understandable code and avoid circular dependencies it is important
 
 - `build`: Scripts to help build Gitea.
 - `cmd`: All Gitea actual sub commands includes web, doctor, serv, hooks, admin and etc. `web` will start the web service. `serv` and `hooks` will be invoked by Git or OpenSSH. Other sub commands could help to maintain Gitea.
-- `tests`: Tests
-  - `tests/integration`: Integration tests
+- `tests`: Common test utility functions
+  - `tests/integration`: Integration tests, to test back-end regressions
+  - `tests/e2e`: E2e tests, to test test front-end <> back-end compatibility and visual regressions.
 - `models`: Contains the data structures used by xorm to construct database tables. It also contains functions to query and update the database. Dependencies to other Gitea code should be avoided. You can make exceptions in cases such as logging.
   - `models/db`: Basic database operations. All other `models/xxx` packages should depend on this package. The `GetEngine` function should only be invoked from `models/`.
   - `models/fixtures`: Sample data used in unit tests and integration tests. One `yml` file means one table which will be loaded into database when beginning the tests.

playwright.config.js

@@ -33,7 +33,7 @@ const config = {
   // workers: process.env.CI ? 1 : undefined,
 
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: process.env.CI ? 'dot' : [['html', {outputFolder: 'tools/e2e/reports/', open: 'never'}]],
+  reporter: process.env.CI ? 'list' : [['list'], ['html', {outputFolder: 'tools/e2e/reports/', open: 'never'}]],
 
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {

tests/e2e/e2e_test.go

@@ -67,20 +67,37 @@ func TestMain(m *testing.M) {
 }
 
 func TestE2e(t *testing.T) {
-	// Default 5 minute timeout
-	onGiteaRun(t, func(*testing.T, *url.URL) {
-		cmd := exec.Command("npx", "playwright", "test")
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, fmt.Sprintf("GITEA_URL=%s", setting.AppURL))
-		var out bytes.Buffer
-		cmd.Stdout = &out
-		err := cmd.Run()
-		if err != nil {
-			// Currently colored output is conflicting. Using Printf until that is resolved.
-			fmt.Printf("%v", out.String())
-			log.Fatal("Playwright Failed: %s", err)
-		} else {
-			fmt.Printf("%v", out.String())
-		}
-	})
+	// Find the paths of all e2e test files in test test directory.
+	search_glob := filepath.Join(filepath.Dir(setting.AppPath), "tests", "e2e", "*.test.e2e.js")
+	paths, err := filepath.Glob(search_glob)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(paths) == 0 {
+		t.Fatal(fmt.Errorf("No e2e tests found in %s", search_glob))
+	}
+
+	// Create new test for each input file
+	for _, path := range paths {
+		_, filename := filepath.Split(path)
+		testname := filename[:len(filename)-len(filepath.Ext(path))]
+
+		t.Run(testname, func(t *testing.T) {
+			// Default 2 minute timeout
+			onGiteaRun(t, func(*testing.T, *url.URL) {
+				cmd := exec.Command("npx", "playwright", "test")
+				cmd.Env = os.Environ()
+				cmd.Env = append(cmd.Env, fmt.Sprintf("GITEA_URL=%s", setting.AppURL))
+				var out bytes.Buffer
+				cmd.Stdout = &out
+				err := cmd.Run()
+				if err != nil {
+					// Currently colored output is conflicting. Using Printf until that is resolved.
+					fmt.Printf("%v", out.String())
+					log.Fatal("Playwright Failed: %s", err)
+				} else {
+					fmt.Printf("%v", out.String())
+				}
+			})
+		})
+	}
 }