mirror of https://github.com/go-gitea/gitea
Refactor LFS SSH and internal routers (#32473)
Gitea instance keeps reporting a lot of errors like "LFS SSH transfer connection denied, pure SSH protocol is disabled". When starting debugging the problem, there are more problems found. Try to address most of them: * avoid unnecessary server side error logs (change `fail()` to not log them) * figure out the broken tests/user2/lfs.git (added comments) * avoid `migratePushMirrors` failure when a repository doesn't exist (ignore them) * avoid "Authorization" (internal&lfs) header conflicts, remove the tricky "swapAuth" and use "X-Gitea-Internal-Auth" * make internal token comparing constant time (it wasn't a serous problem because in a real world it's nearly impossible to timing-attack the token, but good to fix and backport) * avoid duplicate routers (introduce AddOwnerRepoGitLFSRoutes) * avoid "internal (private)" routes using session/web context (they should use private context) * fix incorrect "path" usages (use "filepath") * fix incorrect mocked route point handling (need to check func nil correctly) * split some tests from "git general tests" to "git misc tests" (to keep "git_general_test.go" simple) Still no correct result for Git LFS SSH tests. So the code is kept there (`tests/integration/git_lfs_ssh_test.go`) and a FIXME explains the details.pull/32430/head^2
wxiaoguang
1 week ago
committed by
GitHub
parent
f35e2b0cd1
commit
580e21dd2e
@ -0,0 +1,29 @@ |
|||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package common |
||||||
|
|
||||||
|
import ( |
||||||
|
"net/http" |
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/web" |
||||||
|
"code.gitea.io/gitea/services/lfs" |
||||||
|
) |
||||||
|
|
||||||
|
func AddOwnerRepoGitLFSRoutes(m *web.Router, middlewares ...any) { |
||||||
|
// shared by web and internal routers
|
||||||
|
m.Group("/{username}/{reponame}/info/lfs", func() { |
||||||
|
m.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler) |
||||||
|
m.Put("/objects/{oid}/{size}", lfs.UploadHandler) |
||||||
|
m.Get("/objects/{oid}/{filename}", lfs.DownloadHandler) |
||||||
|
m.Get("/objects/{oid}", lfs.DownloadHandler) |
||||||
|
m.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler) |
||||||
|
m.Group("/locks", func() { |
||||||
|
m.Get("/", lfs.GetListLockHandler) |
||||||
|
m.Post("/", lfs.PostLockHandler) |
||||||
|
m.Post("/verify", lfs.VerifyLockHandler) |
||||||
|
m.Post("/{lid}/unlock", lfs.UnLockHandler) |
||||||
|
}, lfs.CheckAcceptMediaType) |
||||||
|
m.Any("/*", http.NotFound) |
||||||
|
}, middlewares...) |
||||||
|
} |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package integration |
||||||
|
|
||||||
|
import ( |
||||||
|
"net/url" |
||||||
|
"sync" |
||||||
|
"testing" |
||||||
|
|
||||||
|
auth_model "code.gitea.io/gitea/models/auth" |
||||||
|
"code.gitea.io/gitea/modules/setting" |
||||||
|
"code.gitea.io/gitea/modules/web" |
||||||
|
"code.gitea.io/gitea/routers/private" |
||||||
|
"code.gitea.io/gitea/services/context" |
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert" |
||||||
|
"github.com/stretchr/testify/require" |
||||||
|
) |
||||||
|
|
||||||
|
func TestGitLFSSSH(t *testing.T) { |
||||||
|
onGiteaRun(t, func(t *testing.T, u *url.URL) { |
||||||
|
dstPath := t.TempDir() |
||||||
|
apiTestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) |
||||||
|
|
||||||
|
var mu sync.Mutex |
||||||
|
var routerCalls []string |
||||||
|
web.RouteMock(private.RouterMockPointInternalLFS, func(ctx *context.PrivateContext) { |
||||||
|
mu.Lock() |
||||||
|
routerCalls = append(routerCalls, ctx.Req.Method+" "+ctx.Req.URL.Path) |
||||||
|
mu.Unlock() |
||||||
|
}) |
||||||
|
|
||||||
|
withKeyFile(t, "my-testing-key", func(keyFile string) { |
||||||
|
t.Run("CreateUserKey", doAPICreateUserKey(apiTestContext, "test-key", keyFile)) |
||||||
|
cloneURL := createSSHUrl(apiTestContext.GitPath(), u) |
||||||
|
t.Run("Clone", doGitClone(dstPath, cloneURL)) |
||||||
|
|
||||||
|
cfg, err := setting.CfgProvider.PrepareSaving() |
||||||
|
require.NoError(t, err) |
||||||
|
cfg.Section("server").Key("LFS_ALLOW_PURE_SSH").SetValue("true") |
||||||
|
setting.LFS.AllowPureSSH = true |
||||||
|
require.NoError(t, cfg.Save()) |
||||||
|
|
||||||
|
// do LFS SSH transfer?
|
||||||
|
lfsCommitAndPushTest(t, dstPath, 10) |
||||||
|
}) |
||||||
|
|
||||||
|
// FIXME: Here we only see the following calls, but actually there should be calls to "PUT"?
|
||||||
|
// 0 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||||
|
// 1 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/objects/batch"
|
||||||
|
// 2 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||||
|
// 3 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||||
|
// 4 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||||
|
// 5 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||||
|
// 6 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||||
|
// 7 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/locks/24/unlock"
|
||||||
|
assert.NotEmpty(t, routerCalls) |
||||||
|
// assert.Contains(t, routerCalls, "PUT /api/internal/repo/user2/repo1.git/info/lfs/objects/....")
|
||||||
|
}) |
||||||
|
} |
||||||
@ -0,0 +1,138 @@ |
|||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package integration |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"context" |
||||||
|
"io" |
||||||
|
"net/url" |
||||||
|
"sync" |
||||||
|
"testing" |
||||||
|
|
||||||
|
auth_model "code.gitea.io/gitea/models/auth" |
||||||
|
"code.gitea.io/gitea/models/db" |
||||||
|
issues_model "code.gitea.io/gitea/models/issues" |
||||||
|
repo_model "code.gitea.io/gitea/models/repo" |
||||||
|
"code.gitea.io/gitea/models/unittest" |
||||||
|
user_model "code.gitea.io/gitea/models/user" |
||||||
|
"code.gitea.io/gitea/modules/git" |
||||||
|
"code.gitea.io/gitea/modules/gitrepo" |
||||||
|
files_service "code.gitea.io/gitea/services/repository/files" |
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert" |
||||||
|
) |
||||||
|
|
||||||
|
func TestDataAsyncDoubleRead_Issue29101(t *testing.T) { |
||||||
|
onGiteaRun(t, func(t *testing.T, u *url.URL) { |
||||||
|
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) |
||||||
|
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) |
||||||
|
|
||||||
|
testContent := bytes.Repeat([]byte{'a'}, 10000) |
||||||
|
resp, err := files_service.ChangeRepoFiles(db.DefaultContext, repo, user, &files_service.ChangeRepoFilesOptions{ |
||||||
|
Files: []*files_service.ChangeRepoFile{ |
||||||
|
{ |
||||||
|
Operation: "create", |
||||||
|
TreePath: "test.txt", |
||||||
|
ContentReader: bytes.NewReader(testContent), |
||||||
|
}, |
||||||
|
}, |
||||||
|
OldBranch: repo.DefaultBranch, |
||||||
|
NewBranch: repo.DefaultBranch, |
||||||
|
}) |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
sha := resp.Commit.SHA |
||||||
|
|
||||||
|
gitRepo, err := gitrepo.OpenRepository(db.DefaultContext, repo) |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
commit, err := gitRepo.GetCommit(sha) |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
entry, err := commit.GetTreeEntryByPath("test.txt") |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
b := entry.Blob() |
||||||
|
r1, err := b.DataAsync() |
||||||
|
assert.NoError(t, err) |
||||||
|
defer r1.Close() |
||||||
|
r2, err := b.DataAsync() |
||||||
|
assert.NoError(t, err) |
||||||
|
defer r2.Close() |
||||||
|
|
||||||
|
var data1, data2 []byte |
||||||
|
wg := sync.WaitGroup{} |
||||||
|
wg.Add(2) |
||||||
|
go func() { |
||||||
|
data1, _ = io.ReadAll(r1) |
||||||
|
assert.NoError(t, err) |
||||||
|
wg.Done() |
||||||
|
}() |
||||||
|
go func() { |
||||||
|
data2, _ = io.ReadAll(r2) |
||||||
|
assert.NoError(t, err) |
||||||
|
wg.Done() |
||||||
|
}() |
||||||
|
wg.Wait() |
||||||
|
assert.Equal(t, testContent, data1) |
||||||
|
assert.Equal(t, testContent, data2) |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func TestAgitPullPush(t *testing.T) { |
||||||
|
onGiteaRun(t, func(t *testing.T, u *url.URL) { |
||||||
|
baseAPITestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) |
||||||
|
|
||||||
|
u.Path = baseAPITestContext.GitPath() |
||||||
|
u.User = url.UserPassword("user2", userPassword) |
||||||
|
|
||||||
|
dstPath := t.TempDir() |
||||||
|
doGitClone(dstPath, u)(t) |
||||||
|
|
||||||
|
gitRepo, err := git.OpenRepository(context.Background(), dstPath) |
||||||
|
assert.NoError(t, err) |
||||||
|
defer gitRepo.Close() |
||||||
|
|
||||||
|
doGitCreateBranch(dstPath, "test-agit-push") |
||||||
|
|
||||||
|
// commit 1
|
||||||
|
_, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
// push to create an agit pull request
|
||||||
|
err = git.NewCommand(git.DefaultContext, "push", "origin", |
||||||
|
"-o", "title=test-title", "-o", "description=test-description", |
||||||
|
"HEAD:refs/for/master/test-agit-push", |
||||||
|
).Run(&git.RunOpts{Dir: dstPath}) |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
// check pull request exist
|
||||||
|
pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{BaseRepoID: 1, Flow: issues_model.PullRequestFlowAGit, HeadBranch: "user2/test-agit-push"}) |
||||||
|
assert.NoError(t, pr.LoadIssue(db.DefaultContext)) |
||||||
|
assert.Equal(t, "test-title", pr.Issue.Title) |
||||||
|
assert.Equal(t, "test-description", pr.Issue.Content) |
||||||
|
|
||||||
|
// commit 2
|
||||||
|
_, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-2-") |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
// push 2
|
||||||
|
err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").Run(&git.RunOpts{Dir: dstPath}) |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
// reset to first commit
|
||||||
|
err = git.NewCommand(git.DefaultContext, "reset", "--hard", "HEAD~1").Run(&git.RunOpts{Dir: dstPath}) |
||||||
|
assert.NoError(t, err) |
||||||
|
|
||||||
|
// test force push without confirm
|
||||||
|
_, stderr, err := git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").RunStdString(&git.RunOpts{Dir: dstPath}) |
||||||
|
assert.Error(t, err) |
||||||
|
assert.Contains(t, stderr, "[remote rejected] HEAD -> refs/for/master/test-agit-push (request `force-push` push option)") |
||||||
|
|
||||||
|
// test force push with confirm
|
||||||
|
err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push", "-o", "force-push").Run(&git.RunOpts{Dir: dstPath}) |
||||||
|
assert.NoError(t, err) |
||||||
|
}) |
||||||
|
} |
||||||
Loading…
Reference in new issue