mirror of https://github.com/go-gitea/gitea
Refactor LFS SSH and internal routers (#32473)
Gitea instance keeps reporting a lot of errors like "LFS SSH transfer connection denied, pure SSH protocol is disabled". When starting debugging the problem, there are more problems found. Try to address most of them: * avoid unnecessary server side error logs (change `fail()` to not log them) * figure out the broken tests/user2/lfs.git (added comments) * avoid `migratePushMirrors` failure when a repository doesn't exist (ignore them) * avoid "Authorization" (internal&lfs) header conflicts, remove the tricky "swapAuth" and use "X-Gitea-Internal-Auth" * make internal token comparing constant time (it wasn't a serous problem because in a real world it's nearly impossible to timing-attack the token, but good to fix and backport) * avoid duplicate routers (introduce AddOwnerRepoGitLFSRoutes) * avoid "internal (private)" routes using session/web context (they should use private context) * fix incorrect "path" usages (use "filepath") * fix incorrect mocked route point handling (need to check func nil correctly) * split some tests from "git general tests" to "git misc tests" (to keep "git_general_test.go" simple) Still no correct result for Git LFS SSH tests. So the code is kept there (`tests/integration/git_lfs_ssh_test.go`) and a FIXME explains the details.pull/32430/head^2
wxiaoguang
1 week ago
committed by
GitHub
parent
f35e2b0cd1
commit
580e21dd2e
@ -0,0 +1,29 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common |
||||
|
||||
import ( |
||||
"net/http" |
||||
|
||||
"code.gitea.io/gitea/modules/web" |
||||
"code.gitea.io/gitea/services/lfs" |
||||
) |
||||
|
||||
func AddOwnerRepoGitLFSRoutes(m *web.Router, middlewares ...any) { |
||||
// shared by web and internal routers
|
||||
m.Group("/{username}/{reponame}/info/lfs", func() { |
||||
m.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler) |
||||
m.Put("/objects/{oid}/{size}", lfs.UploadHandler) |
||||
m.Get("/objects/{oid}/{filename}", lfs.DownloadHandler) |
||||
m.Get("/objects/{oid}", lfs.DownloadHandler) |
||||
m.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler) |
||||
m.Group("/locks", func() { |
||||
m.Get("/", lfs.GetListLockHandler) |
||||
m.Post("/", lfs.PostLockHandler) |
||||
m.Post("/verify", lfs.VerifyLockHandler) |
||||
m.Post("/{lid}/unlock", lfs.UnLockHandler) |
||||
}, lfs.CheckAcceptMediaType) |
||||
m.Any("/*", http.NotFound) |
||||
}, middlewares...) |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package integration |
||||
|
||||
import ( |
||||
"net/url" |
||||
"sync" |
||||
"testing" |
||||
|
||||
auth_model "code.gitea.io/gitea/models/auth" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/web" |
||||
"code.gitea.io/gitea/routers/private" |
||||
"code.gitea.io/gitea/services/context" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestGitLFSSSH(t *testing.T) { |
||||
onGiteaRun(t, func(t *testing.T, u *url.URL) { |
||||
dstPath := t.TempDir() |
||||
apiTestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) |
||||
|
||||
var mu sync.Mutex |
||||
var routerCalls []string |
||||
web.RouteMock(private.RouterMockPointInternalLFS, func(ctx *context.PrivateContext) { |
||||
mu.Lock() |
||||
routerCalls = append(routerCalls, ctx.Req.Method+" "+ctx.Req.URL.Path) |
||||
mu.Unlock() |
||||
}) |
||||
|
||||
withKeyFile(t, "my-testing-key", func(keyFile string) { |
||||
t.Run("CreateUserKey", doAPICreateUserKey(apiTestContext, "test-key", keyFile)) |
||||
cloneURL := createSSHUrl(apiTestContext.GitPath(), u) |
||||
t.Run("Clone", doGitClone(dstPath, cloneURL)) |
||||
|
||||
cfg, err := setting.CfgProvider.PrepareSaving() |
||||
require.NoError(t, err) |
||||
cfg.Section("server").Key("LFS_ALLOW_PURE_SSH").SetValue("true") |
||||
setting.LFS.AllowPureSSH = true |
||||
require.NoError(t, cfg.Save()) |
||||
|
||||
// do LFS SSH transfer?
|
||||
lfsCommitAndPushTest(t, dstPath, 10) |
||||
}) |
||||
|
||||
// FIXME: Here we only see the following calls, but actually there should be calls to "PUT"?
|
||||
// 0 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||
// 1 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/objects/batch"
|
||||
// 2 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||
// 3 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||
// 4 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||
// 5 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||
// 6 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks"
|
||||
// 7 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/locks/24/unlock"
|
||||
assert.NotEmpty(t, routerCalls) |
||||
// assert.Contains(t, routerCalls, "PUT /api/internal/repo/user2/repo1.git/info/lfs/objects/....")
|
||||
}) |
||||
} |
||||
@ -0,0 +1,138 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package integration |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"io" |
||||
"net/url" |
||||
"sync" |
||||
"testing" |
||||
|
||||
auth_model "code.gitea.io/gitea/models/auth" |
||||
"code.gitea.io/gitea/models/db" |
||||
issues_model "code.gitea.io/gitea/models/issues" |
||||
repo_model "code.gitea.io/gitea/models/repo" |
||||
"code.gitea.io/gitea/models/unittest" |
||||
user_model "code.gitea.io/gitea/models/user" |
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/gitrepo" |
||||
files_service "code.gitea.io/gitea/services/repository/files" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestDataAsyncDoubleRead_Issue29101(t *testing.T) { |
||||
onGiteaRun(t, func(t *testing.T, u *url.URL) { |
||||
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) |
||||
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) |
||||
|
||||
testContent := bytes.Repeat([]byte{'a'}, 10000) |
||||
resp, err := files_service.ChangeRepoFiles(db.DefaultContext, repo, user, &files_service.ChangeRepoFilesOptions{ |
||||
Files: []*files_service.ChangeRepoFile{ |
||||
{ |
||||
Operation: "create", |
||||
TreePath: "test.txt", |
||||
ContentReader: bytes.NewReader(testContent), |
||||
}, |
||||
}, |
||||
OldBranch: repo.DefaultBranch, |
||||
NewBranch: repo.DefaultBranch, |
||||
}) |
||||
assert.NoError(t, err) |
||||
|
||||
sha := resp.Commit.SHA |
||||
|
||||
gitRepo, err := gitrepo.OpenRepository(db.DefaultContext, repo) |
||||
assert.NoError(t, err) |
||||
|
||||
commit, err := gitRepo.GetCommit(sha) |
||||
assert.NoError(t, err) |
||||
|
||||
entry, err := commit.GetTreeEntryByPath("test.txt") |
||||
assert.NoError(t, err) |
||||
|
||||
b := entry.Blob() |
||||
r1, err := b.DataAsync() |
||||
assert.NoError(t, err) |
||||
defer r1.Close() |
||||
r2, err := b.DataAsync() |
||||
assert.NoError(t, err) |
||||
defer r2.Close() |
||||
|
||||
var data1, data2 []byte |
||||
wg := sync.WaitGroup{} |
||||
wg.Add(2) |
||||
go func() { |
||||
data1, _ = io.ReadAll(r1) |
||||
assert.NoError(t, err) |
||||
wg.Done() |
||||
}() |
||||
go func() { |
||||
data2, _ = io.ReadAll(r2) |
||||
assert.NoError(t, err) |
||||
wg.Done() |
||||
}() |
||||
wg.Wait() |
||||
assert.Equal(t, testContent, data1) |
||||
assert.Equal(t, testContent, data2) |
||||
}) |
||||
} |
||||
|
||||
func TestAgitPullPush(t *testing.T) { |
||||
onGiteaRun(t, func(t *testing.T, u *url.URL) { |
||||
baseAPITestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) |
||||
|
||||
u.Path = baseAPITestContext.GitPath() |
||||
u.User = url.UserPassword("user2", userPassword) |
||||
|
||||
dstPath := t.TempDir() |
||||
doGitClone(dstPath, u)(t) |
||||
|
||||
gitRepo, err := git.OpenRepository(context.Background(), dstPath) |
||||
assert.NoError(t, err) |
||||
defer gitRepo.Close() |
||||
|
||||
doGitCreateBranch(dstPath, "test-agit-push") |
||||
|
||||
// commit 1
|
||||
_, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") |
||||
assert.NoError(t, err) |
||||
|
||||
// push to create an agit pull request
|
||||
err = git.NewCommand(git.DefaultContext, "push", "origin", |
||||
"-o", "title=test-title", "-o", "description=test-description", |
||||
"HEAD:refs/for/master/test-agit-push", |
||||
).Run(&git.RunOpts{Dir: dstPath}) |
||||
assert.NoError(t, err) |
||||
|
||||
// check pull request exist
|
||||
pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{BaseRepoID: 1, Flow: issues_model.PullRequestFlowAGit, HeadBranch: "user2/test-agit-push"}) |
||||
assert.NoError(t, pr.LoadIssue(db.DefaultContext)) |
||||
assert.Equal(t, "test-title", pr.Issue.Title) |
||||
assert.Equal(t, "test-description", pr.Issue.Content) |
||||
|
||||
// commit 2
|
||||
_, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-2-") |
||||
assert.NoError(t, err) |
||||
|
||||
// push 2
|
||||
err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").Run(&git.RunOpts{Dir: dstPath}) |
||||
assert.NoError(t, err) |
||||
|
||||
// reset to first commit
|
||||
err = git.NewCommand(git.DefaultContext, "reset", "--hard", "HEAD~1").Run(&git.RunOpts{Dir: dstPath}) |
||||
assert.NoError(t, err) |
||||
|
||||
// test force push without confirm
|
||||
_, stderr, err := git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").RunStdString(&git.RunOpts{Dir: dstPath}) |
||||
assert.Error(t, err) |
||||
assert.Contains(t, stderr, "[remote rejected] HEAD -> refs/for/master/test-agit-push (request `force-push` push option)") |
||||
|
||||
// test force push with confirm
|
||||
err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push", "-o", "force-push").Run(&git.RunOpts{Dir: dstPath}) |
||||
assert.NoError(t, err) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue