Adjustments to remove dangling repository locks

Signed-off-by: Bruno Sofiato <bruno.sofiato@gmail.com>

custom/conf/app.example.ini

@@ -1039,6 +1039,9 @@ LEVEL = Info
 ;; Allow fork repositories without maximum number limit
 ;ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT = true
 
+;; The elapsed time for dangling repository lock to be removed
+;DANGLING_LOCK_THRESHOLD = 1h
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;[repository.editor]

modules/git/command.go

@@ -269,8 +269,43 @@ func CommonCmdServEnvs() []string {
 
 var ErrBrokenCommand = errors.New("git command is broken")
 
-// Run runs the command with the RunOpts
 func (c *Command) Run(opts *RunOpts) error {
+	// Check if there is some dangling locks older than the given threshold
+	if err := ForciblyUnlockRepositoryIfNeeded(c.parentContext, opts.Dir); err != nil {
+		log.Error("Error while trying to unlock repository: %v", err)
+		return err
+	}
+	// Execute the git command
+	if err := c.doRun(opts); err != nil {
+		unlockUponCrashing(c.parentContext, err, opts.Dir)
+		return err
+	}
+	return nil
+}
+
+func unlockUponCrashing(ctx context.Context, originalError error, repoDir string) {
+	if hasGitProcessCrashed(originalError) {
+		log.Warn("The git process has crashed. Attempting to forcbily unlock the underlying repo at %s", repoDir)
+		if err := ForciblyUnlockRepository(ctx, repoDir); err != nil {
+			log.Error("Error while trying to unlock repository at %v", err)
+		}
+	}
+}
+
+func hasGitProcessCrashed(err error) bool {
+	if exitError, ok := err.(*exec.ExitError); ok {
+		if runtime.GOOS == "windows" {
+			log.Warn("Cannot realiably detected if the git process has crashed in windows. Assuming it hasn't [exitCode: %s, pid: %s]", exitError.ExitCode(), exitError.Pid())
+			return false
+		}
+		return exitError.ExitCode() > 128
+	}
+	log.Debug("The given error is not an ExitError [err: %v]. Assuming it the git process hasn't crashed", err)
+	return false
+}
+
+// Run runs the command with the RunOpts
+func (c *Command) doRun(opts *RunOpts) error {
 	if len(c.brokenArgs) != 0 {
 		log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " "))
 		return ErrBrokenCommand

modules/git/repo_cleanup.go

@@ -0,0 +1,60 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+func ForciblyUnlockRepository(ctx context.Context, repoPath string) error {
+	return cleanLocksIfNeeded(repoPath, time.Now())
+}
+
+func ForciblyUnlockRepositoryIfNeeded(ctx context.Context, repoPath string) error {
+	lockThreshold := time.Now().Add(-1 * setting.Repository.DanglingLockThreshold)
+	return cleanLocksIfNeeded(repoPath, lockThreshold)
+}
+
+func cleanLocksIfNeeded(repoPath string, threshold time.Time) error {
+	if repoPath == "" {
+		return nil
+	}
+	log.Trace("Checking if repository %s is locked [lock threshold is %s]", repoPath, threshold)
+	return filepath.Walk(repoPath, func(filePath string, fileInfo os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if err := cleanLockIfNeeded(filePath, fileInfo, threshold); err != nil {
+			log.Error("Failed to remove lock file %s: %v", filePath, err)
+			return err
+		}
+		return nil
+	})
+}
+
+func cleanLockIfNeeded(filePath string, fileInfo os.FileInfo, threshold time.Time) error {
+	if isLock(fileInfo) {
+		if fileInfo.ModTime().Before(threshold) {
+			if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			log.Info("Lock file %s has been removed since its older than %s [timestamp: %s]", filePath, threshold, fileInfo.ModTime())
+			return nil
+		}
+		log.Warn("Cannot exclude lock file %s because it is younger than the threshold %s [timestamp: %s]", filePath, threshold, fileInfo.ModTime())
+		return nil
+	}
+	return nil
+}
+
+func isLock(lockFile os.FileInfo) bool {
+	return !lockFile.IsDir() && strings.HasSuffix(lockFile.Name(), ".lock")
+}

modules/git/repo_cleanup_test.go

@@ -0,0 +1,101 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"runtime"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+	"github.com/stretchr/testify/assert"
+)
+
+// This test mimics a repository having dangling locks. If the locks are older than the threshold, they should be
+// removed. Otherwise, they'll remain and the command will fail.
+
+func TestMaintainExistentLock(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		// Need to use touch to change the last access time of the lock files
+		t.Skip("Skipping test on non-linux OS")
+	}
+
+	shouldRemainLocked := func(lockFiles []string, err error) {
+		assert.Error(t, err)
+		for _, lockFile := range lockFiles {
+			assert.FileExists(t, lockFile)
+		}
+	}
+
+	shouldBeUnlocked := func(lockFiles []string, err error) {
+		assert.NoError(t, err)
+		for _, lockFile := range lockFiles {
+			assert.NoFileExists(t, lockFile)
+		}
+	}
+
+	t.Run("2 days lock file (1 hour threshold)", func(t *testing.T) {
+		doTestLockCleanup(t, "2 days", time.Hour, shouldBeUnlocked)
+	})
+
+	t.Run("1 hour lock file (1 hour threshold)", func(t *testing.T) {
+		doTestLockCleanup(t, "1 hour", time.Hour, shouldBeUnlocked)
+	})
+
+	t.Run("1 minutes lock file (1 hour threshold)", func(t *testing.T) {
+		doTestLockCleanup(t, "1 minutes", time.Hour, shouldRemainLocked)
+	})
+
+	t.Run("1 hour lock file (2 hour threshold)", func(t *testing.T) {
+		doTestLockCleanup(t, "1 hour", 2*time.Hour, shouldRemainLocked)
+	})
+}
+
+func doTestLockCleanup(t *testing.T, lockAge string, threshold time.Duration, expectedResult func(lockFiles []string, err error)) {
+	defer test.MockVariableValue(&setting.Repository, setting.Repository)()
+
+	setting.Repository.DanglingLockThreshold = threshold
+
+	if tmpDir, err := os.MkdirTemp("", "cleanup-after-crash"); err != nil {
+		t.Fatal(err)
+	} else {
+		defer os.RemoveAll(tmpDir)
+
+		if err := os.CopyFS(tmpDir, os.DirFS("../../tests/gitea-repositories-meta/org3/repo3.git")); err != nil {
+			t.Fatal(err)
+		}
+
+		lockFiles := lockFilesFor(tmpDir)
+
+		os.MkdirAll(tmpDir+"/objects/info/commit-graphs", os.ModeSticky|os.ModePerm)
+
+		for _, lockFile := range lockFiles {
+			createLockFiles(t, lockFile, lockAge)
+		}
+
+		cmd := NewCommand(context.Background(), "fetch")
+		_, _, cmdErr := cmd.RunStdString(&RunOpts{Dir: tmpDir})
+
+		expectedResult(lockFiles, cmdErr)
+	}
+}
+
+func lockFilesFor(path string) []string {
+	return []string{
+		path + "/config.lock",
+		path + "/HEAD.lock",
+		path + "/objects/info/commit-graphs/commit-graph-chain.lock",
+	}
+}
+
+func createLockFiles(t *testing.T, file, lockAge string) {
+	cmd := exec.Command("touch", "-m", "-a", "-d", "-"+lockAge, file)
+	if err := cmd.Run(); err != nil {
+		t.Error(err)
+	}
+}

modules/setting/repository.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/log"
 )
@@ -53,7 +54,7 @@ var (
 		AllowDeleteOfUnadoptedRepositories      bool
 		DisableDownloadSourceArchives           bool
 		AllowForkWithoutMaximumLimit            bool
-
+		DanglingLockThreshold                   time.Duration
 		// Repository editor settings
 		Editor struct {
 			LineWrapExtensions []string
@@ -283,6 +284,8 @@ func loadRepositoryFrom(rootCfg ConfigProvider) {
 	Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https")
 	Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1)
 	Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch)
+	Repository.DanglingLockThreshold = sec.Key("DANGLING_LOCK_THRESHOLD").MustDuration(time.Hour)
+
 	RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories"))
 	if !filepath.IsAbs(RepoRootPath) {
 		RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath)