Substitute variables in path names of template repos too (#25294)

### Summary

Extend the template variable substitution to replace file paths. This
can be helpful for setting up log files & directories that should match
the repository name.

### PR Changes

 - Move files matching glob pattern when setting up repos from template
- For security, added ~escaping~ sanitization for cross-platform support
and to prevent directory traversal (thanks @silverwind for the
reference)
 - Added unit testing for escaping function 
- Fixed the integration tests for repo template generation by passing
the repo_template_id
- Updated the integration testfiles to add some variable substitution &
assert the outputs

I had to fix the existing repo template integration test and extend it
to add a check for variable substitutions.

Example:

![image](https://github.com/go-gitea/gitea/assets/12700993/621feb09-0ef3-460e-afa8-da74cd84fa4e)

docs/content/doc/usage/template-repositories.en-us.md

@@ -51,6 +51,8 @@ a/b/c/d.json
 
 In any file matched by the above globs, certain variables will be expanded.
 
+Matching filenames and paths can also be expanded, and are conservatively sanitized to support cross-platform filesystems.
+
 All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}`
 
 | Variable             | Expands To                                          | Transformable |

modules/repository/generate.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -48,7 +49,7 @@ var defaultTransformers = []transformer{
 	{Name: "TITLE", Transform: util.ToTitleCase},
 }
 
-func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository) string {
+func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository, sanitizeFileName bool) string {
 	expansions := []expansion{
 		{Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers},
 		{Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers},
@@ -74,6 +75,9 @@ func generateExpansion(src string, templateRepo, generateRepo *repo_model.Reposi
 
 	return os.Expand(src, func(key string) string {
 		if expansion, ok := expansionMap[key]; ok {
+			if sanitizeFileName {
+				return fileNameSanitize(expansion)
+			}
 			return expansion
 		}
 		return key
@@ -191,10 +195,24 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r
 						}
 
 						if err := os.WriteFile(path,
-							[]byte(generateExpansion(string(content), templateRepo, generateRepo)),
+							[]byte(generateExpansion(string(content), templateRepo, generateRepo, false)),
 							0o644); err != nil {
 							return err
 						}
+
+						substPath := filepath.FromSlash(filepath.Join(tmpDirSlash,
+							generateExpansion(base, templateRepo, generateRepo, true)))
+
+						// Create parent subdirectories if needed or continue silently if it exists
+						if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil {
+							return err
+						}
+
+						// Substitute filename variables
+						if err := os.Rename(path, substPath); err != nil {
+							return err
+						}
+
 						break
 					}
 				}
@@ -353,3 +371,13 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ
 
 	return generateRepo, nil
 }
+
+// Sanitize user input to valid OS filenames
+//
+//		Based on https://github.com/sindresorhus/filename-reserved-regex
+//	 Adds ".." to prevent directory traversal
+func fileNameSanitize(s string) string {
+	re := regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`)
+
+	return strings.TrimSpace(re.ReplaceAllString(s, "_"))
+}

modules/repository/generate_test.go

@@ -54,3 +54,14 @@ func TestGiteaTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestFileNameSanitize(t *testing.T) {
+	assert.Equal(t, "test_CON", fileNameSanitize("test_CON"))
+	assert.Equal(t, "test CON", fileNameSanitize("test CON "))
+	assert.Equal(t, "__traverse__", fileNameSanitize("../traverse/.."))
+	assert.Equal(t, "http___localhost_3003_user_test.git", fileNameSanitize("http://localhost:3003/user/test.git"))
+	assert.Equal(t, "_", fileNameSanitize("CON"))
+	assert.Equal(t, "_", fileNameSanitize("con"))
+	assert.Equal(t, "_", fileNameSanitize("\u0000"))
+	assert.Equal(t, "目标", fileNameSanitize("目标"))
+}

tests/gitea-repositories-meta/user27/template1.git/objects/2a/83b349fa234131fc5db6f2a0498d3f4d3d6038

@@ -0,0 +1,2 @@
+x
<EFBFBD>ÎAJÅ0€a×9Å\@Ij2™ÂCÞwž"™Ìh±i¥<EFBFBD>ÞÞ·qïö‡~Þ{_¦	ì<EFBFBD>çæ+cÔ)M<EFBFBD>•³*rȉSD&’ŠM³û*‡l¥pm*³Ž5fE_ªP 8û˜´D™QCËÉ•aû
o?«À+\>ÛèûfÛ¸¾÷²¬O¼÷HH9G"xôÑ{w¯÷;“ÿ8
+iþsîÖœ£ž¶Ø0<EFBFBD>ï²9Ý/å
IH

tests/gitea-repositories-meta/user27/template1.git/refs/heads/master

@@ -1 +1 @@
-aacbdfe9e1c4b47f60abe81849045fa4e96f1d75
+2a83b349fa234131fc5db6f2a0498d3f4d3d6038

tests/integration/repo_generate_test.go

@@ -7,16 +7,18 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/tests"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, templateRepoName, generateOwnerName, generateRepoName string) *httptest.ResponseRecorder {
+func testRepoGenerate(t *testing.T, session *TestSession, templateID, templateOwnerName, templateRepoName, generateOwnerName, generateRepoName string) *httptest.ResponseRecorder {
 	generateOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: generateOwnerName})
 
 	// Step0: check the existence of the generated repo
@@ -41,16 +43,38 @@ func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, tem
 	_, exists = htmlDoc.doc.Find(fmt.Sprintf(".owner.dropdown .item[data-value=\"%d\"]", generateOwner.ID)).Attr("data-value")
 	assert.True(t, exists, fmt.Sprintf("Generate owner '%s' is not present in select box", generateOwnerName))
 	req = NewRequestWithValues(t, "POST", link, map[string]string{
-		"_csrf":       htmlDoc.GetCSRF(),
+		"_csrf":         htmlDoc.GetCSRF(),
-		"uid":         fmt.Sprintf("%d", generateOwner.ID),
+		"uid":           fmt.Sprintf("%d", generateOwner.ID),
-		"repo_name":   generateRepoName,
+		"repo_name":     generateRepoName,
-		"git_content": "true",
+		"repo_template": templateID,
+		"git_content":   "true",
 	})
 	session.MakeRequest(t, req, http.StatusSeeOther)
 
 	// Step4: check the existence of the generated repo
 	req = NewRequestf(t, "GET", "/%s/%s", generateOwnerName, generateRepoName)
+	session.MakeRequest(t, req, http.StatusOK)
+
+	// Step5: check substituted values in Readme
+	req = NewRequestf(t, "GET", "/%s/%s/raw/branch/master/README.md", generateOwnerName, generateRepoName)
+	resp = session.MakeRequest(t, req, http.StatusOK)
+	body := fmt.Sprintf(`# %s Readme
+Owner: %s
+Link: /%s/%s
+Clone URL: %s%s/%s.git`,
+		generateRepoName,
+		strings.ToUpper(generateOwnerName),
+		generateOwnerName,
+		generateRepoName,
+		setting.AppURL,
+		generateOwnerName,
+		generateRepoName)
+	assert.Equal(t, body, resp.Body.String())
+
+	// Step6: check substituted values in substituted file path ${REPO_NAME}
+	req = NewRequestf(t, "GET", "/%s/%s/raw/branch/master/%s.log", generateOwnerName, generateRepoName, generateRepoName)
 	resp = session.MakeRequest(t, req, http.StatusOK)
+	assert.Equal(t, generateRepoName, resp.Body.String())
 
 	return resp
 }
@@ -58,11 +82,11 @@ func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, tem
 func TestRepoGenerate(t *testing.T) {
 	defer tests.PrepareTestEnv(t)()
 	session := loginUser(t, "user1")
-	testRepoGenerate(t, session, "user27", "template1", "user1", "generated1")
+	testRepoGenerate(t, session, "44", "user27", "template1", "user1", "generated1")
 }
 
 func TestRepoGenerateToOrg(t *testing.T) {
 	defer tests.PrepareTestEnv(t)()
 	session := loginUser(t, "user2")
-	testRepoGenerate(t, session, "user27", "template1", "user2", "generated2")
+	testRepoGenerate(t, session, "44", "user27", "template1", "user2", "generated2")
 }