From 511f6138d4b5b7a464a8fa3d7f8fc52bec3789a4 Mon Sep 17 00:00:00 2001 From: zeripath Date: Thu, 17 Dec 2020 14:00:47 +0000 Subject: [PATCH] Use native git variants by default with go-git variants as build tag (#13673) * Move last commit cache back into modules/git Signed-off-by: Andrew Thornton * Remove go-git from the interface for last commit cache Signed-off-by: Andrew Thornton * move cacheref to last_commit_cache Signed-off-by: Andrew Thornton * Remove go-git from routers/private/hook Signed-off-by: Andrew Thornton * Move FindLFSFiles to pipeline Signed-off-by: Andrew Thornton * Make no-go-git variants Signed-off-by: Andrew Thornton * Submodule RefID Signed-off-by: Andrew Thornton * fix issue with GetCommitsInfo Signed-off-by: Andrew Thornton * fix GetLastCommitForPaths Signed-off-by: Andrew Thornton * Improve efficiency Signed-off-by: Andrew Thornton * More efficiency Signed-off-by: Andrew Thornton * even faster Signed-off-by: Andrew Thornton * Reduce duplication * As per @lunny Signed-off-by: Andrew Thornton * attempt to fix drone Signed-off-by: Andrew Thornton * fix test-tags Signed-off-by: Andrew Thornton * default to use no-go-git variants and add gogit build tag Signed-off-by: Andrew Thornton * placate lint Signed-off-by: Andrew Thornton * as per @6543 Signed-off-by: Andrew Thornton Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick --- .drone.yml | 29 +- Makefile | 19 +- .../doc/installation/from-source.en-us.md | 1 + modules/cache/cache.go | 23 ++ modules/cache/last_commit.go | 70 ---- modules/convert/git_commit_test.go | 3 +- modules/git/batch_reader_nogogit.go | 243 ++++++++++++ modules/git/blob.go | 21 +- modules/git/blob_gogit.go | 33 ++ modules/git/blob_nogogit.go | 77 ++++ modules/git/cache.go | 13 - modules/git/command.go | 2 +- modules/git/commit.go | 59 +-- modules/git/commit_convert_gogit.go | 70 ++++ modules/git/commit_info.go | 287 +------------- modules/git/commit_info_gogit.go | 291 ++++++++++++++ modules/git/commit_info_nogogit.go | 370 ++++++++++++++++++ modules/git/commit_info_test.go | 16 +- modules/git/commit_reader.go | 46 +-- modules/git/last_commit_cache.go | 29 ++ modules/git/last_commit_cache_gogit.go | 113 ++++++ modules/git/last_commit_cache_nogogit.go | 103 +++++ modules/git/notes.go | 65 --- modules/git/notes_gogit.go | 72 ++++ modules/git/notes_nogogit.go | 59 +++ modules/git/{parse.go => parse_gogit.go} | 2 + .../{parse_test.go => parse_gogit_test.go} | 2 + modules/git/parse_nogogit.go | 78 ++++ modules/git/pipeline/lfs.go | 159 ++++++++ modules/git/pipeline/lfs_nogogit.go | 266 +++++++++++++ modules/git/repo.go | 64 --- modules/git/repo_base_gogit.go | 76 ++++ modules/git/repo_base_nogogit.go | 40 ++ modules/git/repo_blob.go | 18 +- modules/git/repo_blob_gogit.go | 23 ++ modules/git/repo_blob_nogogit.go | 17 + modules/git/repo_branch.go | 33 -- modules/git/repo_branch_gogit.go | 45 +++ modules/git/repo_branch_nogogit.go | 82 ++++ modules/git/repo_commit.go | 98 ----- modules/git/repo_commit_gogit.go | 110 ++++++ modules/git/repo_commit_nogogit.go | 109 ++++++ ...mmitgraph.go => repo_commitgraph_gogit.go} | 2 + modules/git/repo_language_stats.go | 106 ----- modules/git/repo_language_stats_gogit.go | 113 ++++++ modules/git/repo_language_stats_nogogit.go | 109 ++++++ modules/git/repo_object.go | 5 + modules/git/repo_ref.go | 45 --- modules/git/repo_ref_gogit.go | 52 +++ modules/git/repo_ref_nogogit.go | 84 ++++ modules/git/repo_tag.go | 31 -- modules/git/repo_tag_gogit.go | 43 ++ modules/git/repo_tag_nogogit.go | 18 + modules/git/repo_tree.go | 41 +- modules/git/repo_tree_gogit.go | 47 +++ modules/git/repo_tree_nogogit.go | 98 +++++ modules/git/sha1.go | 5 - modules/git/sha1_gogit.go | 20 + modules/git/sha1_nogogit.go | 62 +++ modules/git/signature.go | 46 --- modules/git/signature_gogit.go | 54 +++ modules/git/signature_nogogit.go | 95 +++++ modules/git/tag.go | 31 +- modules/git/tree.go | 83 ---- modules/git/tree_blob.go | 58 --- modules/git/tree_blob_gogit.go | 66 ++++ modules/git/tree_blob_nogogit.go | 49 +++ modules/git/tree_entry.go | 104 ----- modules/git/tree_entry_gogit.go | 96 +++++ modules/git/tree_entry_mode.go | 36 ++ modules/git/tree_entry_nogogit.go | 91 +++++ modules/git/tree_entry_test.go | 2 + modules/git/tree_gogit.go | 94 +++++ modules/git/tree_nogogit.go | 69 ++++ modules/git/utils.go | 32 +- modules/indexer/stats/db.go | 3 + modules/repository/cache.go | 54 +-- routers/private/hook.go | 3 +- routers/repo/lfs.go | 146 +------ routers/repo/view.go | 4 +- templates/repo/view_list.tmpl | 11 +- 81 files changed, 3952 insertions(+), 1492 deletions(-) delete mode 100644 modules/cache/last_commit.go create mode 100644 modules/git/batch_reader_nogogit.go create mode 100644 modules/git/blob_gogit.go create mode 100644 modules/git/blob_nogogit.go delete mode 100644 modules/git/cache.go create mode 100644 modules/git/commit_convert_gogit.go create mode 100644 modules/git/commit_info_gogit.go create mode 100644 modules/git/commit_info_nogogit.go create mode 100644 modules/git/last_commit_cache.go create mode 100644 modules/git/last_commit_cache_gogit.go create mode 100644 modules/git/last_commit_cache_nogogit.go create mode 100644 modules/git/notes_gogit.go create mode 100644 modules/git/notes_nogogit.go rename modules/git/{parse.go => parse_gogit.go} (99%) rename modules/git/{parse_test.go => parse_gogit_test.go} (99%) create mode 100644 modules/git/parse_nogogit.go create mode 100644 modules/git/pipeline/lfs.go create mode 100644 modules/git/pipeline/lfs_nogogit.go create mode 100644 modules/git/repo_base_gogit.go create mode 100644 modules/git/repo_base_nogogit.go create mode 100644 modules/git/repo_blob_gogit.go create mode 100644 modules/git/repo_blob_nogogit.go create mode 100644 modules/git/repo_branch_gogit.go create mode 100644 modules/git/repo_branch_nogogit.go create mode 100644 modules/git/repo_commit_gogit.go create mode 100644 modules/git/repo_commit_nogogit.go rename modules/git/{repo_commitgraph.go => repo_commitgraph_gogit.go} (98%) create mode 100644 modules/git/repo_language_stats_gogit.go create mode 100644 modules/git/repo_language_stats_nogogit.go create mode 100644 modules/git/repo_ref_gogit.go create mode 100644 modules/git/repo_ref_nogogit.go create mode 100644 modules/git/repo_tag_gogit.go create mode 100644 modules/git/repo_tag_nogogit.go create mode 100644 modules/git/repo_tree_gogit.go create mode 100644 modules/git/repo_tree_nogogit.go create mode 100644 modules/git/sha1_gogit.go create mode 100644 modules/git/sha1_nogogit.go create mode 100644 modules/git/signature_gogit.go create mode 100644 modules/git/signature_nogogit.go create mode 100644 modules/git/tree_blob_gogit.go create mode 100644 modules/git/tree_blob_nogogit.go create mode 100644 modules/git/tree_entry_gogit.go create mode 100644 modules/git/tree_entry_mode.go create mode 100644 modules/git/tree_entry_nogogit.go create mode 100644 modules/git/tree_gogit.go create mode 100644 modules/git/tree_nogogit.go diff --git a/.drone.yml b/.drone.yml index 0f7f72b8433..e97d65e5db9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,6 +33,16 @@ steps: GOSUMDB: sum.golang.org TAGS: bindata sqlite sqlite_unlock_notify + - name: lint-backend-gogit + pull: always + image: golang:1.15 + commands: + - make lint-backend + environment: + GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata gogit sqlite sqlite_unlock_notify + - name: checks-frontend image: node:14 commands: @@ -69,7 +79,7 @@ steps: GOPROXY: off GOOS: linux GOARCH: arm64 - TAGS: bindata + TAGS: bindata gogit commands: - make backend # test cross compile - rm ./gitea # clean @@ -173,6 +183,17 @@ steps: GITHUB_READ_TOKEN: from_secret: github_read_token + - name: unit-test-gogit + pull: always + image: golang:1.15 + commands: + - make unit-test-coverage test-check + environment: + GOPROXY: off + TAGS: bindata gogit sqlite sqlite_unlock_notify + GITHUB_READ_TOKEN: + from_secret: github_read_token + - name: test-mysql image: golang:1.15 commands: @@ -305,7 +326,8 @@ steps: - timeout -s ABRT 40m make test-sqlite-migration test-sqlite environment: GOPROXY: off - TAGS: bindata + TAGS: bindata gogit sqlite sqlite_unlock_notify + TEST_TAGS: gogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 depends_on: - build @@ -318,7 +340,8 @@ steps: - timeout -s ABRT 40m make test-pgsql-migration test-pgsql environment: GOPROXY: off - TAGS: bindata + TAGS: bindata gogit + TEST_TAGS: gogit TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 depends_on: diff --git a/Makefile b/Makefile index e21cf20f84f..fe26a413bd0 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,10 @@ TAGS ?= TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS)) TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags +TEST_TAGS ?= sqlite sqlite_unlock_notify + GO_DIRS := cmd integrations models modules routers build services vendor tools + GO_SOURCES := $(wildcard *.go) GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go) @@ -339,8 +342,8 @@ watch-backend: go-check .PHONY: test test: - @echo "Running go test..." - @$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' $(GO_PACKAGES) + @echo "Running go test with -tags '$(TEST_TAGS)'..." + @$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES) .PHONY: test-check test-check: @@ -356,8 +359,8 @@ test-check: .PHONY: test\#% test\#%: - @echo "Running go test..." - @$(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -run $(subst .,/,$*) $(GO_PACKAGES) + @echo "Running go test with -tags '$(TEST_TAGS)'..." + @$(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES) .PHONY: coverage coverage: @@ -365,8 +368,8 @@ coverage: .PHONY: unit-test-coverage unit-test-coverage: - @echo "Running unit-test-coverage..." - @$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + @echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..." + @$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 .PHONY: vendor vendor: @@ -511,7 +514,7 @@ integrations.mssql.test: git-check $(GO_SOURCES) $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test integrations.sqlite.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify' + $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags '$(TEST_TAGS)' integrations.cover.test: git-check $(GO_SOURCES) $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(GO_PACKAGES) | tr ' ' ',') -o integrations.cover.test @@ -534,7 +537,7 @@ migrations.mssql.test: $(GO_SOURCES) .PHONY: migrations.sqlite.test migrations.sqlite.test: $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify' + $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)' .PHONY: check check: test diff --git a/docs/content/doc/installation/from-source.en-us.md b/docs/content/doc/installation/from-source.en-us.md index e83495166f2..bff206a8621 100644 --- a/docs/content/doc/installation/from-source.en-us.md +++ b/docs/content/doc/installation/from-source.en-us.md @@ -101,6 +101,7 @@ Depending on requirements, the following build tags can be included. - `pam`: Enable support for PAM (Linux Pluggable Authentication Modules). Can be used to authenticate local users or extend authentication to methods available to PAM. +* `gogit`: (EXPERIMENTAL) Use go-git variants of git commands. Bundling assets into the binary using the `bindata` build tag is recommended for production deployments. It is possible to serve the static assets directly via a reverse proxy, diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 60865d8335d..42227f92895 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -27,6 +27,24 @@ func newCache(cacheConfig setting.Cache) (mc.Cache, error) { }) } +// Cache is the interface that operates the cache data. +type Cache interface { + // Put puts value into cache with key and expire time. + Put(key string, val interface{}, timeout int64) error + // Get gets cached value by given key. + Get(key string) interface{} + // Delete deletes cached value by given key. + Delete(key string) error + // Incr increases cached int-type value by given key as a counter. + Incr(key string) error + // Decr decreases cached int-type value by given key as a counter. + Decr(key string) error + // IsExist returns true if cached value exists. + IsExist(key string) bool + // Flush deletes all cached data. + Flush() error +} + // NewContext start cache service func NewContext() error { var err error @@ -40,6 +58,11 @@ func NewContext() error { return err } +// GetCache returns the currently configured cache +func GetCache() Cache { + return conn +} + // GetString returns the key value from cache with callback when no key exists in cache func GetString(key string, getFunc func() (string, error)) (string, error) { if conn == nil || setting.CacheService.TTL == 0 { diff --git a/modules/cache/last_commit.go b/modules/cache/last_commit.go deleted file mode 100644 index 660a9250d63..00000000000 --- a/modules/cache/last_commit.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package cache - -import ( - "crypto/sha256" - "fmt" - - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - - mc "gitea.com/macaron/cache" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// LastCommitCache represents a cache to store last commit -type LastCommitCache struct { - repoPath string - ttl int64 - repo *git.Repository - commitCache map[string]*object.Commit - mc.Cache -} - -// NewLastCommitCache creates a new last commit cache for repo -func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache { - return &LastCommitCache{ - repoPath: repoPath, - repo: gitRepo, - commitCache: make(map[string]*object.Commit), - ttl: ttl, - Cache: conn, - } -} - -func (c LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) - return fmt.Sprintf("last_commit:%x", hashBytes) -} - -// Get get the last commit information by commit id and entry path -func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) { - v := c.Cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) - if vs, ok := v.(string); ok { - log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) - if commit, ok := c.commitCache[vs]; ok { - log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) - return commit, nil - } - id, err := c.repo.ConvertToSHA1(vs) - if err != nil { - return nil, err - } - commit, err := c.repo.GoGitRepo().CommitObject(id) - if err != nil { - return nil, err - } - c.commitCache[vs] = commit - return commit, nil - } - return nil, nil -} - -// Put put the last commit id with commit and entry path -func (c LastCommitCache) Put(ref, entryPath, commitID string) error { - log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) - return c.Cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) -} diff --git a/modules/convert/git_commit_test.go b/modules/convert/git_commit_test.go index 2158d0d7779..aa35571706e 100644 --- a/modules/convert/git_commit_test.go +++ b/modules/convert/git_commit_test.go @@ -13,7 +13,6 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" - "github.com/go-git/go-git/v5/plumbing/object" "github.com/stretchr/testify/assert" ) @@ -21,7 +20,7 @@ func TestToCommitMeta(t *testing.T) { assert.NoError(t, models.PrepareTestDatabase()) headRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) sha1, _ := git.NewIDFromString("0000000000000000000000000000000000000000") - signature := &object.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)} + signature := &git.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)} tag := &git.Tag{ Name: "Test Tag", ID: sha1, diff --git a/modules/git/batch_reader_nogogit.go b/modules/git/batch_reader_nogogit.go new file mode 100644 index 00000000000..6a236e50027 --- /dev/null +++ b/modules/git/batch_reader_nogogit.go @@ -0,0 +1,243 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "bytes" + "math" + "strconv" +) + +// ReadBatchLine reads the header line from cat-file --batch +// We expect: +// SP SP LF +func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { + sha, err = rd.ReadBytes(' ') + if err != nil { + return + } + sha = sha[:len(sha)-1] + + typ, err = rd.ReadString(' ') + if err != nil { + return + } + typ = typ[:len(typ)-1] + + var sizeStr string + sizeStr, err = rd.ReadString('\n') + if err != nil { + return + } + + size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64) + return +} + +// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream. +func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) { + id := "" + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "object" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + // Discard the rest of the tag + discard := size - n + for discard > math.MaxInt32 { + _, err := rd.Discard(math.MaxInt32) + if err != nil { + return id, err + } + discard -= math.MaxInt32 + } + _, err := rd.Discard(int(discard)) + return id, err +} + +// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream. +func ReadTreeID(rd *bufio.Reader, size int64) (string, error) { + id := "" + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "tree" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + // Discard the rest of the commit + discard := size - n + for discard > math.MaxInt32 { + _, err := rd.Discard(math.MaxInt32) + if err != nil { + return id, err + } + discard -= math.MaxInt32 + } + _, err := rd.Discard(int(discard)) + return id, err +} + +// git tree files are a list: +// SP NUL <20-byte SHA> +// +// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools +// Therefore we need some method to convert these 20-byte SHAs to a 40-byte SHA + +// constant hextable to help quickly convert between 20byte and 40byte hashes +const hextable = "0123456789abcdef" + +// to40ByteSHA converts a 20-byte SHA in a 40-byte slice into a 40-byte sha in place +// without allocations. This is at least 100x quicker that hex.EncodeToString +// NB This requires that sha is a 40-byte slice +func to40ByteSHA(sha []byte) []byte { + for i := 19; i >= 0; i-- { + v := sha[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + sha[i*2], sha[i*2+1] = shi, slo + } + return sha +} + +// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream +// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small. +// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations +// +// Each line is composed of: +// SP NUL <20-byte SHA> +// +// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time +func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) { + var readBytes []byte + // Skip the Mode + readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER + if err != nil { + return + } + n += len(readBytes) + + // Deal with the fname + readBytes, err = rd.ReadSlice('\x00') + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits + } + for err == bufio.ErrBufferFull { // Then we need to read more + readBytes, err = rd.ReadSlice('\x00') + fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend + } + n += len(fnameBuf) + if err != nil { + return + } + fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL + fname = fnameBuf // set the returnable fname to the slice + + // Now deal with the 20-byte SHA + idx := 0 + for idx < 20 { + read := 0 + read, err = rd.Read(shaBuf[idx:20]) + n += read + if err != nil { + return + } + idx += read + } + sha = shaBuf + return +} + +// ParseTreeLine reads an entry from a tree in a cat-file --batch stream +// This carefully avoids allocations - except where fnameBuf is too small. +// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations +// +// Each line is composed of: +// SP NUL <20-byte SHA> +// +// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time +func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { + var readBytes []byte + + // Read the Mode + readBytes, err = rd.ReadSlice(' ') + if err != nil { + return + } + n += len(readBytes) + copy(modeBuf, readBytes) + if len(modeBuf) > len(readBytes) { + modeBuf = modeBuf[:len(readBytes)] + } else { + modeBuf = append(modeBuf, readBytes[len(modeBuf):]...) + + } + mode = modeBuf[:len(modeBuf)-1] // Drop the SP + + // Deal with the fname + readBytes, err = rd.ReadSlice('\x00') + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) + } + for err == bufio.ErrBufferFull { + readBytes, err = rd.ReadSlice('\x00') + fnameBuf = append(fnameBuf, readBytes...) + } + n += len(fnameBuf) + if err != nil { + return + } + fnameBuf = fnameBuf[:len(fnameBuf)-1] + fname = fnameBuf + + // Deal with the 20-byte SHA + idx := 0 + for idx < 20 { + read := 0 + read, err = rd.Read(shaBuf[idx:20]) + n += read + if err != nil { + return + } + idx += read + } + sha = shaBuf + return +} diff --git a/modules/git/blob.go b/modules/git/blob.go index 98545f2f909..674a6a95927 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -10,28 +10,9 @@ import ( "encoding/base64" "io" "io/ioutil" - - "github.com/go-git/go-git/v5/plumbing" ) -// Blob represents a Git object. -type Blob struct { - ID SHA1 - - gogitEncodedObj plumbing.EncodedObject - name string -} - -// DataAsync gets a ReadCloser for the contents of a blob without reading it all. -// Calling the Close function on the result will discard all unread output. -func (b *Blob) DataAsync() (io.ReadCloser, error) { - return b.gogitEncodedObj.Reader() -} - -// Size returns the uncompressed size of the blob -func (b *Blob) Size() int64 { - return b.gogitEncodedObj.Size() -} +// This file contains common functions between the gogit and !gogit variants for git Blobs // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { diff --git a/modules/git/blob_gogit.go b/modules/git/blob_gogit.go new file mode 100644 index 00000000000..7a82eb5c370 --- /dev/null +++ b/modules/git/blob_gogit.go @@ -0,0 +1,33 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "io" + + "github.com/go-git/go-git/v5/plumbing" +) + +// Blob represents a Git object. +type Blob struct { + ID SHA1 + + gogitEncodedObj plumbing.EncodedObject + name string +} + +// DataAsync gets a ReadCloser for the contents of a blob without reading it all. +// Calling the Close function on the result will discard all unread output. +func (b *Blob) DataAsync() (io.ReadCloser, error) { + return b.gogitEncodedObj.Reader() +} + +// Size returns the uncompressed size of the blob +func (b *Blob) Size() int64 { + return b.gogitEncodedObj.Size() +} diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go new file mode 100644 index 00000000000..401b172860e --- /dev/null +++ b/modules/git/blob_nogogit.go @@ -0,0 +1,77 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "io" + "strconv" + "strings" +) + +// Blob represents a Git object. +type Blob struct { + ID SHA1 + + gotSize bool + size int64 + repoPath string + name string +} + +// DataAsync gets a ReadCloser for the contents of a blob without reading it all. +// Calling the Close function on the result will discard all unread output. +func (b *Blob) DataAsync() (io.ReadCloser, error) { + stdoutReader, stdoutWriter := io.Pipe() + var err error + + go func() { + stderr := &strings.Builder{} + err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n")) + if err != nil { + err = ConcatenateError(err, stderr.String()) + _ = stdoutWriter.CloseWithError(err) + } else { + _ = stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + _, _, size, err := ReadBatchLine(bufReader) + if err != nil { + stdoutReader.Close() + return nil, err + } + + return &LimitedReaderCloser{ + R: bufReader, + C: stdoutReader, + N: int64(size), + }, err +} + +// Size returns the uncompressed size of the blob +func (b *Blob) Size() int64 { + if b.gotSize { + return b.size + } + + size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath) + if err != nil { + log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err) + return 0 + } + + b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64) + if err != nil { + log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err) + return 0 + } + b.gotSize = true + + return b.size +} diff --git a/modules/git/cache.go b/modules/git/cache.go deleted file mode 100644 index a1f0f8a57b1..00000000000 --- a/modules/git/cache.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package git - -import "github.com/go-git/go-git/v5/plumbing/object" - -// LastCommitCache cache -type LastCommitCache interface { - Get(ref, entryPath string) (*object.Commit, error) - Put(ref, entryPath, commitID string) error -} diff --git a/modules/git/command.go b/modules/git/command.go index c9d17324168..fe258954628 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -189,7 +189,7 @@ func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir st stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil { - return nil, concatenateError(err, stderr.String()) + return nil, ConcatenateError(err, stderr.String()) } if stdout.Len() > 0 { diff --git a/modules/git/commit.go b/modules/git/commit.go index 6425345ea8e..ce82c2f5820 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -19,8 +19,6 @@ import ( "net/http" "strconv" "strings" - - "github.com/go-git/go-git/v5/plumbing/object" ) // Commit represents a git commit. @@ -43,61 +41,6 @@ type CommitGPGSignature struct { Payload string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data } -func convertPGPSignature(c *object.Commit) *CommitGPGSignature { - if c.PGPSignature == "" { - return nil - } - - var w strings.Builder - var err error - - if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil { - return nil - } - - for _, parent := range c.ParentHashes { - if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil { - return nil - } - } - - if _, err = fmt.Fprint(&w, "author "); err != nil { - return nil - } - - if err = c.Author.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil { - return nil - } - - if err = c.Committer.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil { - return nil - } - - return &CommitGPGSignature{ - Signature: c.PGPSignature, - Payload: w.String(), - } -} - -func convertCommit(c *object.Commit) *Commit { - return &Commit{ - ID: c.Hash, - CommitMessage: c.Message, - Committer: &c.Committer, - Author: &c.Author, - Signature: convertPGPSignature(c), - Parents: c.ParentHashes, - } -} - // Message returns the commit message. Same as retrieving CommitMessage directly. func (c *Commit) Message() string { return c.CommitMessage @@ -576,7 +519,7 @@ func GetCommitFileStatus(repoPath, commitID string) (*CommitFileStatus, error) { err := NewCommand("show", "--name-status", "--pretty=format:''", commitID).RunInDirPipeline(repoPath, w, stderr) w.Close() // Close writer to exit parsing goroutine if err != nil { - return nil, concatenateError(err, stderr.String()) + return nil, ConcatenateError(err, stderr.String()) } <-done diff --git a/modules/git/commit_convert_gogit.go b/modules/git/commit_convert_gogit.go new file mode 100644 index 00000000000..be2b948b363 --- /dev/null +++ b/modules/git/commit_convert_gogit.go @@ -0,0 +1,70 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +func convertPGPSignature(c *object.Commit) *CommitGPGSignature { + if c.PGPSignature == "" { + return nil + } + + var w strings.Builder + var err error + + if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil { + return nil + } + + for _, parent := range c.ParentHashes { + if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil { + return nil + } + } + + if _, err = fmt.Fprint(&w, "author "); err != nil { + return nil + } + + if err = c.Author.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil { + return nil + } + + if err = c.Committer.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil { + return nil + } + + return &CommitGPGSignature{ + Signature: c.PGPSignature, + Payload: w.String(), + } +} + +func convertCommit(c *object.Commit) *Commit { + return &Commit{ + ID: c.Hash, + CommitMessage: c.Message, + Committer: &c.Committer, + Author: &c.Author, + Signature: convertPGPSignature(c), + Parents: c.ParentHashes, + } +} diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index e03ea00fc69..83e23545de5 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -4,286 +4,9 @@ package git -import ( - "path" - - "github.com/emirpasic/gods/trees/binaryheap" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" -) - -// GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) { - entryPaths := make([]string, len(tes)+1) - // Get the commit for the treePath itself - entryPaths[0] = "" - for i, entry := range tes { - entryPaths[i+1] = entry.Name() - } - - commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex() - if commitGraphFile != nil { - defer commitGraphFile.Close() - } - - c, err := commitNodeIndex.Get(commit.ID) - if err != nil { - return nil, nil, err - } - - var revs map[string]*object.Commit - if cache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) - if err != nil { - return nil, nil, err - } - if len(unHitPaths) > 0 { - revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for k, v := range revs2 { - if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil { - return nil, nil, err - } - revs[k] = v - } - } - } else { - revs, err = GetLastCommitForPaths(c, treePath, entryPaths) - } - if err != nil { - return nil, nil, err - } - - commit.repo.gogitStorage.Close() - - commitsInfo := make([][]interface{}, len(tes)) - for i, entry := range tes { - if rev, ok := revs[entry.Name()]; ok { - entryCommit := convertCommit(rev) - if entry.IsSubModule() { - subModuleURL := "" - var fullPath string - if len(treePath) > 0 { - fullPath = treePath + "/" + entry.Name() - } else { - fullPath = entry.Name() - } - if subModule, err := commit.GetSubModule(fullPath); err != nil { - return nil, nil, err - } else if subModule != nil { - subModuleURL = subModule.URL - } - subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) - commitsInfo[i] = []interface{}{entry, subModuleFile} - } else { - commitsInfo[i] = []interface{}{entry, entryCommit} - } - } else { - commitsInfo[i] = []interface{}{entry, nil} - } - } - - // Retrieve the commit for the treePath itself (see above). We basically - // get it for free during the tree traversal and it's used for listing - // pages to display information about newest commit for a given path. - var treeCommit *Commit - if treePath == "" { - treeCommit = commit - } else if rev, ok := revs[""]; ok { - treeCommit = convertCommit(rev) - treeCommit.repo = commit.repo - } - return commitsInfo, treeCommit, nil -} - -type commitAndPaths struct { - commit cgobject.CommitNode - // Paths that are still on the branch represented by commit - paths []string - // Set of hashes for the paths - hashes map[string]plumbing.Hash -} - -func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) { - tree, err := c.Tree() - if err != nil { - return nil, err - } - - // Optimize deep traversals by focusing only on the specific tree - if treePath != "" { - tree, err = tree.Tree(treePath) - if err != nil { - return nil, err - } - } - - return tree, nil -} - -func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) { - tree, err := getCommitTree(c, treePath) - if err == object.ErrDirectoryNotFound { - // The whole tree didn't exist, so return empty map - return make(map[string]plumbing.Hash), nil - } - if err != nil { - return nil, err - } - - hashes := make(map[string]plumbing.Hash) - for _, path := range paths { - if path != "" { - entry, err := tree.FindEntry(path) - if err == nil { - hashes[path] = entry.Hash - } - } else { - hashes[path] = tree.Hash - } - } - - return hashes, nil -} - -func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) { - var unHitEntryPaths []string - var results = make(map[string]*object.Commit) - for _, p := range paths { - lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) - if err != nil { - return nil, nil, err - } - if lastCommit != nil { - results[p] = lastCommit - continue - } - - unHitEntryPaths = append(unHitEntryPaths, p) - } - - return results, unHitEntryPaths, nil -} - -// GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) { - // We do a tree traversal with nodes sorted by commit time - heap := binaryheap.NewWith(func(a, b interface{}) int { - if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) { - return 1 - } - return -1 - }) - - resultNodes := make(map[string]cgobject.CommitNode) - initialHashes, err := getFileHashes(c, treePath, paths) - if err != nil { - return nil, err - } - - // Start search from the root commit and with full set of paths - heap.Push(&commitAndPaths{c, paths, initialHashes}) - - for { - cIn, ok := heap.Pop() - if !ok { - break - } - current := cIn.(*commitAndPaths) - - // Load the parent commits for the one we are currently examining - numParents := current.commit.NumParents() - var parents []cgobject.CommitNode - for i := 0; i < numParents; i++ { - parent, err := current.commit.ParentNode(i) - if err != nil { - break - } - parents = append(parents, parent) - } - - // Examine the current commit and set of interesting paths - pathUnchanged := make([]bool, len(current.paths)) - parentHashes := make([]map[string]plumbing.Hash, len(parents)) - for j, parent := range parents { - parentHashes[j], err = getFileHashes(parent, treePath, current.paths) - if err != nil { - break - } - - for i, path := range current.paths { - if parentHashes[j][path] == current.hashes[path] { - pathUnchanged[i] = true - } - } - } - - var remainingPaths []string - for i, path := range current.paths { - // The results could already contain some newer change for the same path, - // so don't override that and bail out on the file early. - if resultNodes[path] == nil { - if pathUnchanged[i] { - // The path existed with the same hash in at least one parent so it could - // not have been changed in this commit directly. - remainingPaths = append(remainingPaths, path) - } else { - // There are few possible cases how can we get here: - // - The path didn't exist in any parent, so it must have been created by - // this commit. - // - The path did exist in the parent commit, but the hash of the file has - // changed. - // - We are looking at a merge commit and the hash of the file doesn't - // match any of the hashes being merged. This is more common for directories, - // but it can also happen if a file is changed through conflict resolution. - resultNodes[path] = current.commit - } - } - } - - if len(remainingPaths) > 0 { - // Add the parent nodes along with remaining paths to the heap for further - // processing. - for j, parent := range parents { - // Combine remainingPath with paths available on the parent branch - // and make union of them - remainingPathsForParent := make([]string, 0, len(remainingPaths)) - newRemainingPaths := make([]string, 0, len(remainingPaths)) - for _, path := range remainingPaths { - if parentHashes[j][path] == current.hashes[path] { - remainingPathsForParent = append(remainingPathsForParent, path) - } else { - newRemainingPaths = append(newRemainingPaths, path) - } - } - - if remainingPathsForParent != nil { - heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]}) - } - - if len(newRemainingPaths) == 0 { - break - } else { - remainingPaths = newRemainingPaths - } - } - } - } - - // Post-processing - result := make(map[string]*object.Commit) - for path, commitNode := range resultNodes { - var err error - result[path], err = commitNode.Commit() - if err != nil { - return nil, err - } - } - - return result, nil +// CommitInfo describes the first commit with the provided entry +type CommitInfo struct { + Entry *TreeEntry + Commit *Commit + SubModuleFile *SubModuleFile } diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go new file mode 100644 index 00000000000..6d95e22d0c7 --- /dev/null +++ b/modules/git/commit_info_gogit.go @@ -0,0 +1,291 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "path" + + "github.com/emirpasic/gods/trees/binaryheap" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" +) + +// GetCommitsInfo gets information of all commits that are corresponding to these entries +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { + entryPaths := make([]string, len(tes)+1) + // Get the commit for the treePath itself + entryPaths[0] = "" + for i, entry := range tes { + entryPaths[i+1] = entry.Name() + } + + commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex() + if commitGraphFile != nil { + defer commitGraphFile.Close() + } + + c, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return nil, nil, err + } + + var revs map[string]*object.Commit + if cache != nil { + var unHitPaths []string + revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + if err != nil { + return nil, nil, err + } + if len(unHitPaths) > 0 { + revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths) + if err != nil { + return nil, nil, err + } + + for k, v := range revs2 { + if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil { + return nil, nil, err + } + revs[k] = v + } + } + } else { + revs, err = GetLastCommitForPaths(c, treePath, entryPaths) + } + if err != nil { + return nil, nil, err + } + + commit.repo.gogitStorage.Close() + + commitsInfo := make([]CommitInfo, len(tes)) + for i, entry := range tes { + commitsInfo[i] = CommitInfo{ + Entry: entry, + } + if rev, ok := revs[entry.Name()]; ok { + entryCommit := convertCommit(rev) + commitsInfo[i].Commit = entryCommit + if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile + } + } + } + + // Retrieve the commit for the treePath itself (see above). We basically + // get it for free during the tree traversal and it's used for listing + // pages to display information about newest commit for a given path. + var treeCommit *Commit + if treePath == "" { + treeCommit = commit + } else if rev, ok := revs[""]; ok { + treeCommit = convertCommit(rev) + treeCommit.repo = commit.repo + } + return commitsInfo, treeCommit, nil +} + +type commitAndPaths struct { + commit cgobject.CommitNode + // Paths that are still on the branch represented by commit + paths []string + // Set of hashes for the paths + hashes map[string]plumbing.Hash +} + +func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) { + tree, err := c.Tree() + if err != nil { + return nil, err + } + + // Optimize deep traversals by focusing only on the specific tree + if treePath != "" { + tree, err = tree.Tree(treePath) + if err != nil { + return nil, err + } + } + + return tree, nil +} + +func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) { + tree, err := getCommitTree(c, treePath) + if err == object.ErrDirectoryNotFound { + // The whole tree didn't exist, so return empty map + return make(map[string]plumbing.Hash), nil + } + if err != nil { + return nil, err + } + + hashes := make(map[string]plumbing.Hash) + for _, path := range paths { + if path != "" { + entry, err := tree.FindEntry(path) + if err == nil { + hashes[path] = entry.Hash + } + } else { + hashes[path] = tree.Hash + } + } + + return hashes, nil +} + +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) { + var unHitEntryPaths []string + var results = make(map[string]*object.Commit) + for _, p := range paths { + lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) + if err != nil { + return nil, nil, err + } + if lastCommit != nil { + results[p] = lastCommit.(*object.Commit) + continue + } + + unHitEntryPaths = append(unHitEntryPaths, p) + } + + return results, unHitEntryPaths, nil +} + +// GetLastCommitForPaths returns last commit information +func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) { + // We do a tree traversal with nodes sorted by commit time + heap := binaryheap.NewWith(func(a, b interface{}) int { + if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) { + return 1 + } + return -1 + }) + + resultNodes := make(map[string]cgobject.CommitNode) + initialHashes, err := getFileHashes(c, treePath, paths) + if err != nil { + return nil, err + } + + // Start search from the root commit and with full set of paths + heap.Push(&commitAndPaths{c, paths, initialHashes}) + + for { + cIn, ok := heap.Pop() + if !ok { + break + } + current := cIn.(*commitAndPaths) + + // Load the parent commits for the one we are currently examining + numParents := current.commit.NumParents() + var parents []cgobject.CommitNode + for i := 0; i < numParents; i++ { + parent, err := current.commit.ParentNode(i) + if err != nil { + break + } + parents = append(parents, parent) + } + + // Examine the current commit and set of interesting paths + pathUnchanged := make([]bool, len(current.paths)) + parentHashes := make([]map[string]plumbing.Hash, len(parents)) + for j, parent := range parents { + parentHashes[j], err = getFileHashes(parent, treePath, current.paths) + if err != nil { + break + } + + for i, path := range current.paths { + if parentHashes[j][path] == current.hashes[path] { + pathUnchanged[i] = true + } + } + } + + var remainingPaths []string + for i, path := range current.paths { + // The results could already contain some newer change for the same path, + // so don't override that and bail out on the file early. + if resultNodes[path] == nil { + if pathUnchanged[i] { + // The path existed with the same hash in at least one parent so it could + // not have been changed in this commit directly. + remainingPaths = append(remainingPaths, path) + } else { + // There are few possible cases how can we get here: + // - The path didn't exist in any parent, so it must have been created by + // this commit. + // - The path did exist in the parent commit, but the hash of the file has + // changed. + // - We are looking at a merge commit and the hash of the file doesn't + // match any of the hashes being merged. This is more common for directories, + // but it can also happen if a file is changed through conflict resolution. + resultNodes[path] = current.commit + } + } + } + + if len(remainingPaths) > 0 { + // Add the parent nodes along with remaining paths to the heap for further + // processing. + for j, parent := range parents { + // Combine remainingPath with paths available on the parent branch + // and make union of them + remainingPathsForParent := make([]string, 0, len(remainingPaths)) + newRemainingPaths := make([]string, 0, len(remainingPaths)) + for _, path := range remainingPaths { + if parentHashes[j][path] == current.hashes[path] { + remainingPathsForParent = append(remainingPathsForParent, path) + } else { + newRemainingPaths = append(newRemainingPaths, path) + } + } + + if remainingPathsForParent != nil { + heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]}) + } + + if len(newRemainingPaths) == 0 { + break + } else { + remainingPaths = newRemainingPaths + } + } + } + } + + // Post-processing + result := make(map[string]*object.Commit) + for path, commitNode := range resultNodes { + var err error + result[path], err = commitNode.Commit() + if err != nil { + return nil, err + } + } + + return result, nil +} diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go new file mode 100644 index 00000000000..ac0c7cff5dc --- /dev/null +++ b/modules/git/commit_info_nogogit.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "bytes" + "fmt" + "io" + "math" + "path" + "sort" + "strings" +) + +// GetCommitsInfo gets information of all commits that are corresponding to these entries +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { + entryPaths := make([]string, len(tes)+1) + // Get the commit for the treePath itself + entryPaths[0] = "" + for i, entry := range tes { + entryPaths[i+1] = entry.Name() + } + + var err error + + var revs map[string]*Commit + if cache != nil { + var unHitPaths []string + revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + if err != nil { + return nil, nil, err + } + if len(unHitPaths) > 0 { + sort.Strings(unHitPaths) + commits, err := GetLastCommitForPaths(commit, treePath, unHitPaths) + if err != nil { + return nil, nil, err + } + + for i, found := range commits { + if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil { + return nil, nil, err + } + revs[unHitPaths[i]] = found + } + } + } else { + sort.Strings(entryPaths) + revs = map[string]*Commit{} + var foundCommits []*Commit + foundCommits, err = GetLastCommitForPaths(commit, treePath, entryPaths) + for i, found := range foundCommits { + revs[entryPaths[i]] = found + } + } + if err != nil { + return nil, nil, err + } + + commitsInfo := make([]CommitInfo, len(tes)) + for i, entry := range tes { + commitsInfo[i] = CommitInfo{ + Entry: entry, + } + if entryCommit, ok := revs[entry.Name()]; ok { + commitsInfo[i].Commit = entryCommit + if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile + } + } + } + + // Retrieve the commit for the treePath itself (see above). We basically + // get it for free during the tree traversal and it's used for listing + // pages to display information about newest commit for a given path. + var treeCommit *Commit + var ok bool + if treePath == "" { + treeCommit = commit + } else if treeCommit, ok = revs[""]; ok { + treeCommit.repo = commit.repo + } + return commitsInfo, treeCommit, nil +} + +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { + var unHitEntryPaths []string + var results = make(map[string]*Commit) + for _, p := range paths { + lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) + if err != nil { + return nil, nil, err + } + if lastCommit != nil { + results[p] = lastCommit.(*Commit) + continue + } + + unHitEntryPaths = append(unHitEntryPaths, p) + } + + return results, unHitEntryPaths, nil +} + +// GetLastCommitForPaths returns last commit information +func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) { + // We read backwards from the commit to obtain all of the commits + + // We'll do this by using rev-list to provide us with parent commits in order + revListReader, revListWriter := io.Pipe() + defer func() { + _ = revListWriter.Close() + _ = revListReader.Close() + }() + + go func() { + stderr := strings.Builder{} + err := NewCommand("rev-list", "--format=%T", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr) + if err != nil { + _ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // We feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. + // so let's create a batch stdin and stdout + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := io.Pipe() + defer func() { + _ = batchStdinReader.Close() + _ = batchStdinWriter.Close() + _ = batchStdoutReader.Close() + _ = batchStdoutWriter.Close() + }() + + go func() { + stderr := strings.Builder{} + err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(commit.repo.Path, batchStdoutWriter, &stderr, batchStdinReader) + if err != nil { + _ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // For simplicities sake we'll us a buffered reader + batchReader := bufio.NewReader(batchStdoutReader) + + mapsize := 4096 + if len(paths) > mapsize { + mapsize = len(paths) + } + + path2idx := make(map[string]int, mapsize) + for i, path := range paths { + path2idx[path] = i + } + + fnameBuf := make([]byte, 4096) + modeBuf := make([]byte, 40) + + allShaBuf := make([]byte, (len(paths)+1)*20) + shaBuf := make([]byte, 20) + tmpTreeID := make([]byte, 40) + + // commits is the returnable commits matching the paths provided + commits := make([]string, len(paths)) + // ids are the blob/tree ids for the paths + ids := make([][]byte, len(paths)) + + // We'll use a scanner for the revList because it's simpler than a bufio.Reader + scan := bufio.NewScanner(revListReader) +revListLoop: + for scan.Scan() { + // Get the next parent commit ID + commitID := scan.Text() + if !scan.Scan() { + break revListLoop + } + commitID = commitID[7:] + rootTreeID := scan.Text() + + // push the tree to the cat-file --batch process + _, err := batchStdinWriter.Write([]byte(rootTreeID + "\n")) + if err != nil { + return nil, err + } + + currentPath := "" + + // OK if the target tree path is "" and the "" is in the paths just set this now + if treePath == "" && paths[0] == "" { + // If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit + if len(ids[0]) == 0 { + ids[0] = []byte(rootTreeID) + commits[0] = string(commitID) + } else if bytes.Equal(ids[0], []byte(rootTreeID)) { + commits[0] = string(commitID) + } + } + + treeReadingLoop: + for { + _, _, size, err := ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + + // Handle trees + + // n is counter for file position in the tree file + var n int64 + + // Two options: currentPath is the targetTreepath + if treePath == currentPath { + // We are in the right directory + // Parse each tree line in turn. (don't care about mode here.) + for n < size { + fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf) + shaBuf = sha + if err != nil { + return nil, err + } + n += int64(count) + idx, ok := path2idx[string(fname)] + if ok { + // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit + if len(ids[idx]) == 0 { + copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf) + ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)] + commits[idx] = string(commitID) + } else if bytes.Equal(ids[idx], shaBuf) { + commits[idx] = string(commitID) + } + } + // FIXME: is there any order to the way strings are emitted from cat-file? + // if there is - then we could skip once we've passed all of our data + } + break treeReadingLoop + } + + var treeID []byte + + // We're in the wrong directory + // Find target directory in this directory + idx := len(currentPath) + if idx > 0 { + idx++ + } + target := strings.SplitN(treePath[idx:], "/", 2)[0] + + for n < size { + // Read each tree entry in turn + mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf) + if err != nil { + return nil, err + } + n += int64(count) + + // if we have found the target directory + if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) { + copy(tmpTreeID, sha) + treeID = tmpTreeID + break + } + } + + if n < size { + // Discard any remaining entries in the current tree + discard := size - n + for discard > math.MaxInt32 { + _, err := batchReader.Discard(math.MaxInt32) + if err != nil { + return nil, err + } + discard -= math.MaxInt32 + } + _, err := batchReader.Discard(int(discard)) + if err != nil { + return nil, err + } + } + + // if we haven't found a treeID for the target directory our search is over + if len(treeID) == 0 { + break treeReadingLoop + } + + // add the target to the current path + if idx > 0 { + currentPath += "/" + } + currentPath += target + + // if we've now found the current path check its sha id and commit status + if treePath == currentPath && paths[0] == "" { + if len(ids[0]) == 0 { + copy(allShaBuf[0:20], treeID) + ids[0] = allShaBuf[0:20] + commits[0] = string(commitID) + } else if bytes.Equal(ids[0], treeID) { + commits[0] = string(commitID) + } + } + treeID = to40ByteSHA(treeID) + _, err = batchStdinWriter.Write(treeID) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte("\n")) + if err != nil { + return nil, err + } + } + } + + commitsMap := make(map[string]*Commit, len(commits)) + commitsMap[commit.ID.String()] = commit + + commitCommits := make([]*Commit, len(commits)) + for i, commitID := range commits { + c, ok := commitsMap[commitID] + if ok { + commitCommits[i] = c + continue + } + + if len(commitID) == 0 { + continue + } + + _, err := batchStdinWriter.Write([]byte(commitID + "\n")) + if err != nil { + return nil, err + } + _, typ, size, err := ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + if typ != "commit" { + return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) + } + c, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size))) + if err != nil { + return nil, err + } + commitCommits[i] = c + } + + return commitCommits, scan.Err() +} diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 8bdf1a769be..3966419bc14 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -58,17 +58,27 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { for _, testCase := range testCases { commit, err := repo1.GetCommit(testCase.CommitID) assert.NoError(t, err) + assert.NotNil(t, commit) + assert.NotNil(t, commit.Tree) + assert.NotNil(t, commit.Tree.repo) + tree, err := commit.Tree.SubTree(testCase.Path) + assert.NotNil(t, tree, "tree is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path) + assert.NotNil(t, tree.repo, "repo is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path) + assert.NoError(t, err) entries, err := tree.ListEntries() assert.NoError(t, err) commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil) - assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String()) assert.NoError(t, err) + if err != nil { + t.FailNow() + } + assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String()) assert.Len(t, commitsInfo, len(testCase.ExpectedIDs)) for _, commitInfo := range commitsInfo { - entry := commitInfo[0].(*TreeEntry) - commit := commitInfo[1].(*Commit) + entry := commitInfo.Entry + commit := commitInfo.Commit expectedID, ok := testCase.ExpectedIDs[entry.Name()] if !assert.True(t, ok) { continue diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index fdcb6dca84a..4eb861040e6 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -9,13 +9,13 @@ import ( "bytes" "io" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // CommitFromReader will generate a Commit from a provided reader -// We will need this to interpret commits from cat-file -func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) (*Commit, error) { +// We need this to interpret commits from cat-file or cat-file --batch +// +// If used as part of a cat-file --batch stream you need to limit the reader to the correct size +func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) { commit := &Commit{ ID: sha, } @@ -26,26 +26,20 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) message := false pgpsig := false - scanner := bufio.NewScanner(reader) - // Split by '\n' but include the '\n' - scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - if i := bytes.IndexByte(data, '\n'); i >= 0 { - // We have a full newline-terminated line. - return i + 1, data[0 : i+1], nil - } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil - } - // Request more data. - return 0, nil, nil - }) + bufReader, ok := reader.(*bufio.Reader) + if !ok { + bufReader = bufio.NewReader(reader) + } - for scanner.Scan() { - line := scanner.Bytes() +readLoop: + for { + line, err := bufReader.ReadBytes('\n') + if err != nil { + if err == io.EOF { + break readLoop + } + return nil, err + } if pgpsig { if len(line) > 0 && line[0] == ' ' { _, _ = signatureSB.Write(line[1:]) @@ -72,10 +66,10 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) switch string(split[0]) { case "tree": - commit.Tree = *NewTree(gitRepo, plumbing.NewHash(string(data))) + commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data))) _, _ = payloadSB.Write(line) case "parent": - commit.Parents = append(commit.Parents, plumbing.NewHash(string(data))) + commit.Parents = append(commit.Parents, MustIDFromString(string(data))) _, _ = payloadSB.Write(line) case "author": commit.Author = &Signature{} @@ -104,5 +98,5 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) commit.Signature = nil } - return commit, scanner.Err() + return commit, nil } diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go new file mode 100644 index 00000000000..7cca601226a --- /dev/null +++ b/modules/git/last_commit_cache.go @@ -0,0 +1,29 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import ( + "crypto/sha256" + "fmt" +) + +// Cache represents a caching interface +type Cache interface { + // Put puts value into cache with key and expire time. + Put(key string, val interface{}, timeout int64) error + // Get gets cached value by given key. + Get(key string) interface{} +} + +func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { + hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) + return fmt.Sprintf("last_commit:%x", hashBytes) +} + +// Put put the last commit id with commit and entry path +func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { + log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) + return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) +} diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go new file mode 100644 index 00000000000..76c97a4cc04 --- /dev/null +++ b/modules/git/last_commit_cache_gogit.go @@ -0,0 +1,113 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "path" + + "github.com/go-git/go-git/v5/plumbing/object" + cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" +) + +// LastCommitCache represents a cache to store last commit +type LastCommitCache struct { + repoPath string + ttl int64 + repo *Repository + commitCache map[string]*object.Commit + cache Cache +} + +// NewLastCommitCache creates a new last commit cache for repo +func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache { + if cache == nil { + return nil + } + return &LastCommitCache{ + repoPath: repoPath, + repo: gitRepo, + commitCache: make(map[string]*object.Commit), + ttl: ttl, + cache: cache, + } +} + +// Get get the last commit information by commit id and entry path +func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + if vs, ok := v.(string); ok { + log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) + if commit, ok := c.commitCache[vs]; ok { + log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) + return commit, nil + } + id, err := c.repo.ConvertToSHA1(vs) + if err != nil { + return nil, err + } + commit, err := c.repo.GoGitRepo().CommitObject(id) + if err != nil { + return nil, err + } + c.commitCache[vs] = commit + return commit, nil + } + return nil, nil +} + +// CacheCommit will cache the commit from the gitRepository +func (c *LastCommitCache) CacheCommit(commit *Commit) error { + + commitNodeIndex, _ := commit.repo.CommitNodeIndex() + + index, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return err + } + + return c.recursiveCache(index, &commit.Tree, "", 1) +} + +func (c *LastCommitCache) recursiveCache(index cgobject.CommitNode, tree *Tree, treePath string, level int) error { + if level == 0 { + return nil + } + + entries, err := tree.ListEntries() + if err != nil { + return err + } + + entryPaths := make([]string, len(entries)) + entryMap := make(map[string]*TreeEntry) + for i, entry := range entries { + entryPaths[i] = entry.Name() + entryMap[entry.Name()] = entry + } + + commits, err := GetLastCommitForPaths(index, treePath, entryPaths) + if err != nil { + return err + } + + for entry, cm := range commits { + if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil { + return err + } + if entryMap[entry].IsDir() { + subTree, err := tree.SubTree(entry) + if err != nil { + return err + } + if err := c.recursiveCache(index, subTree, entry, level-1); err != nil { + return err + } + } + } + + return nil +} diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go new file mode 100644 index 00000000000..b9c50b5cfb4 --- /dev/null +++ b/modules/git/last_commit_cache_nogogit.go @@ -0,0 +1,103 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "path" +) + +// LastCommitCache represents a cache to store last commit +type LastCommitCache struct { + repoPath string + ttl int64 + repo *Repository + commitCache map[string]*Commit + cache Cache +} + +// NewLastCommitCache creates a new last commit cache for repo +func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache { + if cache == nil { + return nil + } + return &LastCommitCache{ + repoPath: repoPath, + repo: gitRepo, + commitCache: make(map[string]*Commit), + ttl: ttl, + cache: cache, + } +} + +// Get get the last commit information by commit id and entry path +func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + if vs, ok := v.(string); ok { + log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) + if commit, ok := c.commitCache[vs]; ok { + log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) + return commit, nil + } + id, err := c.repo.ConvertToSHA1(vs) + if err != nil { + return nil, err + } + commit, err := c.repo.getCommit(id) + if err != nil { + return nil, err + } + c.commitCache[vs] = commit + return commit, nil + } + return nil, nil +} + +// CacheCommit will cache the commit from the gitRepository +func (c *LastCommitCache) CacheCommit(commit *Commit) error { + return c.recursiveCache(commit, &commit.Tree, "", 1) +} + +func (c *LastCommitCache) recursiveCache(commit *Commit, tree *Tree, treePath string, level int) error { + if level == 0 { + return nil + } + + entries, err := tree.ListEntries() + if err != nil { + return err + } + + entryPaths := make([]string, len(entries)) + entryMap := make(map[string]*TreeEntry) + for i, entry := range entries { + entryPaths[i] = entry.Name() + entryMap[entry.Name()] = entry + } + + commits, err := GetLastCommitForPaths(commit, treePath, entryPaths) + if err != nil { + return err + } + + for i, entryCommit := range commits { + entry := entryPaths[i] + if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil { + return err + } + if entryMap[entry].IsDir() { + subTree, err := tree.SubTree(entry) + if err != nil { + return err + } + if err := c.recursiveCache(commit, subTree, entry, level-1); err != nil { + return err + } + } + } + + return nil +} diff --git a/modules/git/notes.go b/modules/git/notes.go index ba19fa48934..a8dd66df0bc 100644 --- a/modules/git/notes.go +++ b/modules/git/notes.go @@ -4,12 +4,6 @@ package git -import ( - "io/ioutil" - - "github.com/go-git/go-git/v5/plumbing/object" -) - // NotesRef is the git ref where Gitea will look for git-notes data. // The value ("refs/notes/commits") is the default ref used by git-notes. const NotesRef = "refs/notes/commits" @@ -19,62 +13,3 @@ type Note struct { Message []byte Commit *Commit } - -// GetNote retrieves the git-notes data for a given commit. -func GetNote(repo *Repository, commitID string, note *Note) error { - notes, err := repo.GetCommit(NotesRef) - if err != nil { - return err - } - - remainingCommitID := commitID - path := "" - currentTree := notes.Tree.gogitTree - var file *object.File - for len(remainingCommitID) > 2 { - file, err = currentTree.File(remainingCommitID) - if err == nil { - path += remainingCommitID - break - } - if err == object.ErrFileNotFound { - currentTree, err = currentTree.Tree(remainingCommitID[0:2]) - path += remainingCommitID[0:2] + "/" - remainingCommitID = remainingCommitID[2:] - } - if err != nil { - return err - } - } - - blob := file.Blob - dataRc, err := blob.Reader() - if err != nil { - return err - } - - defer dataRc.Close() - d, err := ioutil.ReadAll(dataRc) - if err != nil { - return err - } - note.Message = d - - commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() - if commitGraphFile != nil { - defer commitGraphFile.Close() - } - - commitNode, err := commitNodeIndex.Get(notes.ID) - if err != nil { - return err - } - - lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path}) - if err != nil { - return err - } - note.Commit = convertCommit(lastCommits[path]) - - return nil -} diff --git a/modules/git/notes_gogit.go b/modules/git/notes_gogit.go new file mode 100644 index 00000000000..173d29cee6b --- /dev/null +++ b/modules/git/notes_gogit.go @@ -0,0 +1,72 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "io/ioutil" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetNote retrieves the git-notes data for a given commit. +func GetNote(repo *Repository, commitID string, note *Note) error { + notes, err := repo.GetCommit(NotesRef) + if err != nil { + return err + } + + remainingCommitID := commitID + path := "" + currentTree := notes.Tree.gogitTree + var file *object.File + for len(remainingCommitID) > 2 { + file, err = currentTree.File(remainingCommitID) + if err == nil { + path += remainingCommitID + break + } + if err == object.ErrFileNotFound { + currentTree, err = currentTree.Tree(remainingCommitID[0:2]) + path += remainingCommitID[0:2] + "/" + remainingCommitID = remainingCommitID[2:] + } + if err != nil { + return err + } + } + + blob := file.Blob + dataRc, err := blob.Reader() + if err != nil { + return err + } + + defer dataRc.Close() + d, err := ioutil.ReadAll(dataRc) + if err != nil { + return err + } + note.Message = d + + commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() + if commitGraphFile != nil { + defer commitGraphFile.Close() + } + + commitNode, err := commitNodeIndex.Get(notes.ID) + if err != nil { + return err + } + + lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path}) + if err != nil { + return err + } + note.Commit = convertCommit(lastCommits[path]) + + return nil +} diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go new file mode 100644 index 00000000000..613efd2e0e0 --- /dev/null +++ b/modules/git/notes_nogogit.go @@ -0,0 +1,59 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "io/ioutil" +) + +// GetNote retrieves the git-notes data for a given commit. +func GetNote(repo *Repository, commitID string, note *Note) error { + notes, err := repo.GetCommit(NotesRef) + if err != nil { + return err + } + + path := "" + + tree := ¬es.Tree + + var entry *TreeEntry + for len(commitID) > 2 { + entry, err = tree.GetTreeEntryByPath(commitID) + if err == nil { + path += commitID + break + } + if IsErrNotExist(err) { + tree, err = tree.SubTree(commitID[0:2]) + path += commitID[0:2] + "/" + commitID = commitID[2:] + } + if err != nil { + return err + } + } + + dataRc, err := entry.Blob().DataAsync() + if err != nil { + return err + } + defer dataRc.Close() + d, err := ioutil.ReadAll(dataRc) + if err != nil { + return err + } + note.Message = d + + lastCommits, err := GetLastCommitForPaths(notes, "", []string{path}) + if err != nil { + return err + } + note.Commit = lastCommits[0] + + return nil +} diff --git a/modules/git/parse.go b/modules/git/parse_gogit.go similarity index 99% rename from modules/git/parse.go rename to modules/git/parse_gogit.go index 89b4488600e..434fb4160f7 100644 --- a/modules/git/parse.go +++ b/modules/git/parse_gogit.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build gogit + package git import ( diff --git a/modules/git/parse_test.go b/modules/git/parse_gogit_test.go similarity index 99% rename from modules/git/parse_test.go rename to modules/git/parse_gogit_test.go index 8e0be828b3c..cf38c29932f 100644 --- a/modules/git/parse_test.go +++ b/modules/git/parse_gogit_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build gogit + package git import ( diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go new file mode 100644 index 00000000000..26dd700af73 --- /dev/null +++ b/modules/git/parse_nogogit.go @@ -0,0 +1,78 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bytes" + "fmt" + "strconv" +) + +// ParseTreeEntries parses the output of a `git ls-tree` command. +func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { + return parseTreeEntries(data, nil) +} + +func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { + entries := make([]*TreeEntry, 0, 10) + for pos := 0; pos < len(data); { + // expect line to be of the form " \t" + entry := new(TreeEntry) + entry.ptree = ptree + if pos+6 > len(data) { + return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) + } + switch string(data[pos : pos+6]) { + case "100644": + entry.entryMode = EntryModeBlob + pos += 12 // skip over "100644 blob " + case "100755": + entry.entryMode = EntryModeExec + pos += 12 // skip over "100755 blob " + case "120000": + entry.entryMode = EntryModeSymlink + pos += 12 // skip over "120000 blob " + case "160000": + entry.entryMode = EntryModeCommit + pos += 14 // skip over "160000 object " + case "040000": + entry.entryMode = EntryModeTree + pos += 12 // skip over "040000 tree " + default: + return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6])) + } + + if pos+40 > len(data) { + return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) + } + id, err := NewIDFromString(string(data[pos : pos+40])) + if err != nil { + return nil, fmt.Errorf("Invalid ls-tree output: %v", err) + } + entry.ID = id + pos += 41 // skip over sha and trailing space + + end := pos + bytes.IndexByte(data[pos:], '\n') + if end < pos { + return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) + } + + // In case entry name is surrounded by double quotes(it happens only in git-shell). + if data[pos] == '"' { + entry.name, err = strconv.Unquote(string(data[pos:end])) + if err != nil { + return nil, fmt.Errorf("Invalid ls-tree output: %v", err) + } + } else { + entry.name = string(data[pos:end]) + } + + pos = end + 1 + entries = append(entries, entry) + } + return entries, nil +} diff --git a/modules/git/pipeline/lfs.go b/modules/git/pipeline/lfs.go new file mode 100644 index 00000000000..d47b7d91ea8 --- /dev/null +++ b/modules/git/pipeline/lfs.go @@ -0,0 +1,159 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package pipeline + +import ( + "bufio" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/git" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// LFSResult represents commits found using a provided pointer file hash +type LFSResult struct { + Name string + SHA string + Summary string + When time.Time + ParentHashes []git.SHA1 + BranchName string + FullCommitName string +} + +type lfsResultSlice []*LFSResult + +func (a lfsResultSlice) Len() int { return len(a) } +func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } + +// FindLFSFile finds commits that contain a provided pointer file hash +func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { + resultsMap := map[string]*LFSResult{} + results := make([]*LFSResult, 0) + + basePath := repo.Path + gogitRepo := repo.GoGitRepo() + + commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ + Order: gogit.LogOrderCommitterTime, + All: true, + }) + if err != nil { + return nil, fmt.Errorf("Failed to get GoGit CommitsIter. Error: %w", err) + } + + err = commitsIter.ForEach(func(gitCommit *object.Commit) error { + tree, err := gitCommit.Tree() + if err != nil { + return err + } + treeWalker := object.NewTreeWalker(tree, true, nil) + defer treeWalker.Close() + for { + name, entry, err := treeWalker.Next() + if err == io.EOF { + break + } + if entry.Hash == hash { + result := LFSResult{ + Name: name, + SHA: gitCommit.Hash.String(), + Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], + When: gitCommit.Author.When, + ParentHashes: gitCommit.ParentHashes, + } + resultsMap[gitCommit.Hash.String()+":"+name] = &result + } + } + return nil + }) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("Failure in CommitIter.ForEach: %w", err) + } + + for _, result := range resultsMap { + hasParent := false + for _, parentHash := range result.ParentHashes { + if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { + break + } + } + if !hasParent { + results = append(results, result) + } + } + + sort.Sort(lfsResultSlice(results)) + + // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple + shasToNameReader, shasToNameWriter := io.Pipe() + nameRevStdinReader, nameRevStdinWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(3) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(nameRevStdinReader) + i := 0 + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + result := results[i] + result.FullCommitName = line + result.BranchName = strings.Split(line, "~")[0] + i++ + } + }() + go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) + go func() { + defer wg.Done() + defer shasToNameWriter.Close() + for _, result := range results { + i := 0 + if i < len(result.SHA) { + n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) + if err != nil { + errChan <- err + break + } + i += n + } + n := 0 + for n < 1 { + n, err = shasToNameWriter.Write([]byte{'\n'}) + if err != nil { + errChan <- err + break + } + + } + + } + }() + + wg.Wait() + + select { + case err, has := <-errChan: + if has { + return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err) + } + default: + } + + return results, nil +} diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go new file mode 100644 index 00000000000..30d33e27e03 --- /dev/null +++ b/modules/git/pipeline/lfs_nogogit.go @@ -0,0 +1,266 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package pipeline + +import ( + "bufio" + "bytes" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/git" +) + +// LFSResult represents commits found using a provided pointer file hash +type LFSResult struct { + Name string + SHA string + Summary string + When time.Time + ParentHashes []git.SHA1 + BranchName string + FullCommitName string +} + +type lfsResultSlice []*LFSResult + +func (a lfsResultSlice) Len() int { return len(a) } +func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } + +// FindLFSFile finds commits that contain a provided pointer file hash +func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { + resultsMap := map[string]*LFSResult{} + results := make([]*LFSResult, 0) + + basePath := repo.Path + + hashStr := hash.String() + + // Use rev-list to provide us with all commits in order + revListReader, revListWriter := io.Pipe() + defer func() { + _ = revListWriter.Close() + _ = revListReader.Close() + }() + + go func() { + stderr := strings.Builder{} + err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr) + if err != nil { + _ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. + // so let's create a batch stdin and stdout + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := io.Pipe() + defer func() { + _ = batchStdinReader.Close() + _ = batchStdinWriter.Close() + _ = batchStdoutReader.Close() + _ = batchStdoutWriter.Close() + }() + + go func() { + stderr := strings.Builder{} + err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, batchStdoutWriter, &stderr, batchStdinReader) + if err != nil { + _ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // For simplicities sake we'll us a buffered reader to read from the cat-file --batch + batchReader := bufio.NewReader(batchStdoutReader) + + // We'll use a scanner for the revList because it's simpler than a bufio.Reader + scan := bufio.NewScanner(revListReader) + trees := [][]byte{} + paths := []string{} + + fnameBuf := make([]byte, 4096) + modeBuf := make([]byte, 40) + workingShaBuf := make([]byte, 40) + + for scan.Scan() { + // Get the next commit ID + commitID := scan.Bytes() + + // push the commit to the cat-file --batch process + _, err := batchStdinWriter.Write(commitID) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte{'\n'}) + if err != nil { + return nil, err + } + + var curCommit *git.Commit + curPath := "" + + commitReadingLoop: + for { + _, typ, size, err := git.ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + // This shouldn't happen but if it does well just get the commit and try again + id, err := git.ReadTagObjectID(batchReader, size) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte(id + "\n")) + if err != nil { + return nil, err + } + continue + case "commit": + // Read in the commit to get its tree and in case this is one of the last used commits + curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size))) + if err != nil { + return nil, err + } + + _, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n")) + if err != nil { + return nil, err + } + curPath = "" + case "tree": + var n int64 + for n < size { + mode, fname, sha, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf) + if err != nil { + return nil, err + } + n += int64(count) + if bytes.Equal(sha, []byte(hashStr)) { + result := LFSResult{ + Name: curPath + string(fname), + SHA: curCommit.ID.String(), + Summary: strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0], + When: curCommit.Author.When, + ParentHashes: curCommit.Parents, + } + resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result + } else if string(mode) == git.EntryModeTree.String() { + trees = append(trees, sha) + paths = append(paths, curPath+string(fname)+"/") + } + } + if len(trees) > 0 { + _, err := batchStdinWriter.Write(trees[len(trees)-1]) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte("\n")) + if err != nil { + return nil, err + } + curPath = paths[len(paths)-1] + trees = trees[:len(trees)-1] + paths = paths[:len(paths)-1] + } else { + break commitReadingLoop + } + } + } + } + + if err := scan.Err(); err != nil { + return nil, err + } + + for _, result := range resultsMap { + hasParent := false + for _, parentHash := range result.ParentHashes { + if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { + break + } + } + if !hasParent { + results = append(results, result) + } + } + + sort.Sort(lfsResultSlice(results)) + + // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple + shasToNameReader, shasToNameWriter := io.Pipe() + nameRevStdinReader, nameRevStdinWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(3) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(nameRevStdinReader) + i := 0 + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + result := results[i] + result.FullCommitName = line + result.BranchName = strings.Split(line, "~")[0] + i++ + } + }() + go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) + go func() { + defer wg.Done() + defer shasToNameWriter.Close() + for _, result := range results { + i := 0 + if i < len(result.SHA) { + n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) + if err != nil { + errChan <- err + break + } + i += n + } + var err error + n := 0 + for n < 1 { + n, err = shasToNameWriter.Write([]byte{'\n'}) + if err != nil { + errChan <- err + break + } + + } + + } + }() + + wg.Wait() + + select { + case err, has := <-errChan: + if has { + return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err) + } + default: + } + + return results, nil +} diff --git a/modules/git/repo.go b/modules/git/repo.go index 9b1da87a32e..e824dcc3f29 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -9,34 +9,16 @@ import ( "bytes" "container/list" "context" - "errors" "fmt" "os" "path" - "path/filepath" "strconv" "strings" "time" - gitealog "code.gitea.io/gitea/modules/log" - "github.com/go-git/go-billy/v5/osfs" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/storage/filesystem" "github.com/unknwon/com" ) -// Repository represents a Git repository. -type Repository struct { - Path string - - tagCache *ObjectCache - - gogitRepo *gogit.Repository - gogitStorage *filesystem.Storage - gpgSettings *GPGSettings -} - // GPGSettings represents the default GPG settings for this repository type GPGSettings struct { Sign bool @@ -93,52 +75,6 @@ func InitRepository(repoPath string, bare bool) error { return err } -// OpenRepository opens the repository at the given path. -func OpenRepository(repoPath string) (*Repository, error) { - repoPath, err := filepath.Abs(repoPath) - if err != nil { - return nil, err - } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") - } - - fs := osfs.New(repoPath) - _, err = fs.Stat(".git") - if err == nil { - fs, err = fs.Chroot(".git") - if err != nil { - return nil, err - } - } - storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true}) - gogitRepo, err := gogit.Open(storage, fs) - if err != nil { - return nil, err - } - - return &Repository{ - Path: repoPath, - gogitRepo: gogitRepo, - gogitStorage: storage, - tagCache: newObjectCache(), - }, nil -} - -// Close this repository, in particular close the underlying gogitStorage if this is not nil -func (repo *Repository) Close() { - if repo == nil || repo.gogitStorage == nil { - return - } - if err := repo.gogitStorage.Close(); err != nil { - gitealog.Error("Error closing storage: %v", err) - } -} - -// GoGitRepo gets the go-git repo representation -func (repo *Repository) GoGitRepo() *gogit.Repository { - return repo.gogitRepo -} - // IsEmpty Check if repository is empty. func (repo *Repository) IsEmpty() (bool, error) { var errbuf strings.Builder diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go new file mode 100644 index 00000000000..19a3f84571f --- /dev/null +++ b/modules/git/repo_base_gogit.go @@ -0,0 +1,76 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "errors" + "path/filepath" + + gitealog "code.gitea.io/gitea/modules/log" + "github.com/go-git/go-billy/v5/osfs" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/storage/filesystem" +) + +// Repository represents a Git repository. +type Repository struct { + Path string + + tagCache *ObjectCache + + gogitRepo *gogit.Repository + gogitStorage *filesystem.Storage + gpgSettings *GPGSettings +} + +// OpenRepository opens the repository at the given path. +func OpenRepository(repoPath string) (*Repository, error) { + repoPath, err := filepath.Abs(repoPath) + if err != nil { + return nil, err + } else if !isDir(repoPath) { + return nil, errors.New("no such file or directory") + } + + fs := osfs.New(repoPath) + _, err = fs.Stat(".git") + if err == nil { + fs, err = fs.Chroot(".git") + if err != nil { + return nil, err + } + } + storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true}) + gogitRepo, err := gogit.Open(storage, fs) + if err != nil { + return nil, err + } + + return &Repository{ + Path: repoPath, + gogitRepo: gogitRepo, + gogitStorage: storage, + tagCache: newObjectCache(), + }, nil +} + +// Close this repository, in particular close the underlying gogitStorage if this is not nil +func (repo *Repository) Close() { + if repo == nil || repo.gogitStorage == nil { + return + } + if err := repo.gogitStorage.Close(); err != nil { + gitealog.Error("Error closing storage: %v", err) + } +} + +// GoGitRepo gets the go-git repo representation +func (repo *Repository) GoGitRepo() *gogit.Repository { + return repo.gogitRepo +} diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go new file mode 100644 index 00000000000..e05219a4e70 --- /dev/null +++ b/modules/git/repo_base_nogogit.go @@ -0,0 +1,40 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "errors" + "path/filepath" +) + +// Repository represents a Git repository. +type Repository struct { + Path string + + tagCache *ObjectCache + + gpgSettings *GPGSettings +} + +// OpenRepository opens the repository at the given path. +func OpenRepository(repoPath string) (*Repository, error) { + repoPath, err := filepath.Abs(repoPath) + if err != nil { + return nil, err + } else if !isDir(repoPath) { + return nil, errors.New("no such file or directory") + } + return &Repository{ + Path: repoPath, + tagCache: newObjectCache(), + }, nil +} + +// Close this repository, in particular close the underlying gogitStorage if this is not nil +func (repo *Repository) Close() { +} diff --git a/modules/git/repo_blob.go b/modules/git/repo_blob.go index ce0ad6b50fb..5397f24cb62 100644 --- a/modules/git/repo_blob.go +++ b/modules/git/repo_blob.go @@ -1,25 +1,9 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package git -import ( - "github.com/go-git/go-git/v5/plumbing" -) - -func (repo *Repository) getBlob(id SHA1) (*Blob, error) { - encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id) - if err != nil { - return nil, ErrNotExist{id.String(), ""} - } - - return &Blob{ - ID: id, - gogitEncodedObj: encodedObj, - }, nil -} - // GetBlob finds the blob object in the repository. func (repo *Repository) GetBlob(idStr string) (*Blob, error) { id, err := NewIDFromString(idStr) diff --git a/modules/git/repo_blob_gogit.go b/modules/git/repo_blob_gogit.go new file mode 100644 index 00000000000..485c233ff89 --- /dev/null +++ b/modules/git/repo_blob_gogit.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" +) + +func (repo *Repository) getBlob(id SHA1) (*Blob, error) { + encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id) + if err != nil { + return nil, ErrNotExist{id.String(), ""} + } + + return &Blob{ + ID: id, + gogitEncodedObj: encodedObj, + }, nil +} diff --git a/modules/git/repo_blob_nogogit.go b/modules/git/repo_blob_nogogit.go new file mode 100644 index 00000000000..9959420df4e --- /dev/null +++ b/modules/git/repo_blob_nogogit.go @@ -0,0 +1,17 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +func (repo *Repository) getBlob(id SHA1) (*Blob, error) { + if id.IsZero() { + return nil, ErrNotExist{id.String(), ""} + } + return &Blob{ + ID: id, + repoPath: repo.Path, + }, nil +} diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index cd30c191ea3..25438530f55 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -8,8 +8,6 @@ package git import ( "fmt" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // BranchPrefix base dir of the branch information file store on git @@ -26,18 +24,6 @@ func IsBranchExist(repoPath, name string) bool { return IsReferenceExist(repoPath, BranchPrefix+name) } -// IsBranchExist returns true if given branch exists in current repository. -func (repo *Repository) IsBranchExist(name string) bool { - if name == "" { - return false - } - reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true) - if err != nil { - return false - } - return reference.Type() != plumbing.InvalidReference -} - // Branch represents a Git branch. type Branch struct { Name string @@ -79,25 +65,6 @@ func (repo *Repository) GetDefaultBranch() (string, error) { return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path) } -// GetBranches returns all branches of the repository. -func (repo *Repository) GetBranches() ([]string, error) { - var branchNames []string - - branches, err := repo.gogitRepo.Branches() - if err != nil { - return nil, err - } - - _ = branches.ForEach(func(branch *plumbing.Reference) error { - branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) - return nil - }) - - // TODO: Sort? - - return branchNames, nil -} - // GetBranch returns a branch by it's name func (repo *Repository) GetBranch(branch string) (*Branch, error) { if !repo.IsBranchExist(branch) { diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go new file mode 100644 index 00000000000..65cb77a8b55 --- /dev/null +++ b/modules/git/repo_branch_gogit.go @@ -0,0 +1,45 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +// IsBranchExist returns true if given branch exists in current repository. +func (repo *Repository) IsBranchExist(name string) bool { + if name == "" { + return false + } + reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true) + if err != nil { + return false + } + return reference.Type() != plumbing.InvalidReference +} + +// GetBranches returns all branches of the repository. +func (repo *Repository) GetBranches() ([]string, error) { + var branchNames []string + + branches, err := repo.gogitRepo.Branches() + if err != nil { + return nil, err + } + + _ = branches.ForEach(func(branch *plumbing.Reference) error { + branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) + return nil + }) + + // TODO: Sort? + + return branchNames, nil +} diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go new file mode 100644 index 00000000000..5ec46d725e4 --- /dev/null +++ b/modules/git/repo_branch_nogogit.go @@ -0,0 +1,82 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "io" + "strings" +) + +// IsBranchExist returns true if given branch exists in current repository. +func (repo *Repository) IsBranchExist(name string) bool { + if name == "" { + return false + } + return IsReferenceExist(repo.Path, BranchPrefix+name) +} + +// GetBranches returns all branches of the repository. +func (repo *Repository) GetBranches() ([]string, error) { + return callShowRef(repo.Path, BranchPrefix, "--heads") +} + +func callShowRef(repoPath, prefix, arg string) ([]string, error) { + var branchNames []string + + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + if err != nil { + if stderrBuilder.Len() == 0 { + _ = stdoutWriter.Close() + return + } + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + for { + // The output of show-ref is simply a list: + // SP LF + _, err := bufReader.ReadSlice(' ') + for err == bufio.ErrBufferFull { + // This shouldn't happen but we'll tolerate it for the sake of peace + _, err = bufReader.ReadSlice(' ') + } + if err == io.EOF { + return branchNames, nil + } + if err != nil { + return nil, err + } + + branchName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This shouldn't happen... but we'll tolerate it for the sake of peace + return branchNames, nil + } + if err != nil { + return nil, err + } + branchName = strings.TrimPrefix(branchName, prefix) + if len(branchName) > 0 { + branchName = branchName[:len(branchName)-1] + } + branchNames = append(branchNames, branchName) + } +} diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index ee3b05447b8..c31f4166283 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -8,36 +8,10 @@ package git import ( "bytes" "container/list" - "fmt" "strconv" "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" ) -// GetRefCommitID returns the last commit ID string of given reference (branch or tag). -func (repo *Repository) GetRefCommitID(name string) (string, error) { - ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) - if err != nil { - if err == plumbing.ErrReferenceNotFound { - return "", ErrNotExist{ - ID: name, - } - } - return "", err - } - - return ref.Hash().String(), nil -} - -// IsCommitExist returns true if given commit exists in current repository. -func (repo *Repository) IsCommitExist(name string) bool { - hash := plumbing.NewHash(name) - _, err := repo.gogitRepo.CommitObject(hash) - return err == nil -} - // GetBranchCommitID returns last commit ID string of given branch. func (repo *Repository) GetBranchCommitID(name string) (string, error) { return repo.GetRefCommitID(BranchPrefix + name) @@ -55,78 +29,6 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) { return strings.TrimSpace(stdout), nil } -func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature { - if t.PGPSignature == "" { - return nil - } - - var w strings.Builder - var err error - - if _, err = fmt.Fprintf(&w, - "object %s\ntype %s\ntag %s\ntagger ", - t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil { - return nil - } - - if err = t.Tagger.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, "\n\n"); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, t.Message); err != nil { - return nil - } - - return &CommitGPGSignature{ - Signature: t.PGPSignature, - Payload: strings.TrimSpace(w.String()) + "\n", - } -} - -func (repo *Repository) getCommit(id SHA1) (*Commit, error) { - var tagObject *object.Tag - - gogitCommit, err := repo.gogitRepo.CommitObject(id) - if err == plumbing.ErrObjectNotFound { - tagObject, err = repo.gogitRepo.TagObject(id) - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - ID: id.String(), - } - } - if err == nil { - gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target) - } - // if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500 - } - if err != nil { - return nil, err - } - - commit := convertCommit(gogitCommit) - commit.repo = repo - - if tagObject != nil { - commit.CommitMessage = strings.TrimSpace(tagObject.Message) - commit.Author = &tagObject.Tagger - commit.Signature = convertPGPSignatureForTag(tagObject) - } - - tree, err := gogitCommit.Tree() - if err != nil { - return nil, err - } - - commit.Tree.ID = tree.Hash - commit.Tree.gogitTree = tree - - return commit, nil -} - // ConvertToSHA1 returns a Hash object from a potential ID string func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { if len(commitID) != 40 { diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go new file mode 100644 index 00000000000..48b0cfe19d0 --- /dev/null +++ b/modules/git/repo_commit_gogit.go @@ -0,0 +1,110 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +func (repo *Repository) GetRefCommitID(name string) (string, error) { + ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) + if err != nil { + if err == plumbing.ErrReferenceNotFound { + return "", ErrNotExist{ + ID: name, + } + } + return "", err + } + + return ref.Hash().String(), nil +} + +// IsCommitExist returns true if given commit exists in current repository. +func (repo *Repository) IsCommitExist(name string) bool { + hash := plumbing.NewHash(name) + _, err := repo.gogitRepo.CommitObject(hash) + return err == nil +} + +func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature { + if t.PGPSignature == "" { + return nil + } + + var w strings.Builder + var err error + + if _, err = fmt.Fprintf(&w, + "object %s\ntype %s\ntag %s\ntagger ", + t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil { + return nil + } + + if err = t.Tagger.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, "\n\n"); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, t.Message); err != nil { + return nil + } + + return &CommitGPGSignature{ + Signature: t.PGPSignature, + Payload: strings.TrimSpace(w.String()) + "\n", + } +} + +func (repo *Repository) getCommit(id SHA1) (*Commit, error) { + var tagObject *object.Tag + + gogitCommit, err := repo.gogitRepo.CommitObject(id) + if err == plumbing.ErrObjectNotFound { + tagObject, err = repo.gogitRepo.TagObject(id) + if err == plumbing.ErrObjectNotFound { + return nil, ErrNotExist{ + ID: id.String(), + } + } + if err == nil { + gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target) + } + // if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500 + } + if err != nil { + return nil, err + } + + commit := convertCommit(gogitCommit) + commit.repo = repo + + if tagObject != nil { + commit.CommitMessage = strings.TrimSpace(tagObject.Message) + commit.Author = &tagObject.Tagger + commit.Signature = convertPGPSignatureForTag(tagObject) + } + + tree, err := gogitCommit.Tree() + if err != nil { + return nil, err + } + + commit.Tree.ID = tree.Hash + commit.Tree.gogitTree = tree + + return commit, nil +} diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go new file mode 100644 index 00000000000..a43fe4b3349 --- /dev/null +++ b/modules/git/repo_commit_nogogit.go @@ -0,0 +1,109 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "strings" +) + +// ResolveReference resolves a name to a reference +func (repo *Repository) ResolveReference(name string) (string, error) { + stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "not a valid ref") { + return "", ErrNotExist{name, ""} + } + return "", err + } + stdout = strings.TrimSpace(stdout) + if stdout == "" { + return "", ErrNotExist{name, ""} + } + + return stdout, nil +} + +// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +func (repo *Repository) GetRefCommitID(name string) (string, error) { + stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "not a valid ref") { + return "", ErrNotExist{name, ""} + } + return "", err + } + + return strings.TrimSpace(stdout), nil +} + +// IsCommitExist returns true if given commit exists in current repository. +func (repo *Repository) IsCommitExist(name string) bool { + _, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path) + return err == nil +} + +func (repo *Repository) getCommit(id SHA1) (*Commit, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderr := strings.Builder{} + err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n")) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + _, typ, size, err := ReadBatchLine(bufReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + // then we need to parse the tag + // and load the commit + data, err := ioutil.ReadAll(io.LimitReader(bufReader, size)) + if err != nil { + return nil, err + } + tag, err := parseTagData(data) + if err != nil { + return nil, err + } + tag.repo = repo + + commit, err := tag.Commit() + if err != nil { + return nil, err + } + + commit.CommitMessage = strings.TrimSpace(tag.Message) + commit.Author = tag.Tagger + commit.Signature = tag.Signature + + return commit, nil + case "commit": + return CommitFromReader(repo, id, io.LimitReader(bufReader, size)) + default: + _ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) + log("Unknown typ: %s", typ) + return nil, ErrNotExist{ + ID: id.String(), + } + } +} diff --git a/modules/git/repo_commitgraph.go b/modules/git/repo_commitgraph_gogit.go similarity index 98% rename from modules/git/repo_commitgraph.go rename to modules/git/repo_commitgraph_gogit.go index 00111f55037..67731094513 100644 --- a/modules/git/repo_commitgraph.go +++ b/modules/git/repo_commitgraph_gogit.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build gogit + package git import ( diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go index b721b996e49..ac23caa0fc0 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/repo_language_stats.go @@ -4,111 +4,5 @@ package git -import ( - "bytes" - "io" - "io/ioutil" - - "code.gitea.io/gitea/modules/analyze" - - "github.com/go-enry/go-enry/v2" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - const fileSizeLimit int64 = 16 * 1024 // 16 KiB const bigFileSize int64 = 1024 * 1024 // 1 MiB - -// GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { - r, err := git.PlainOpen(repo.Path) - if err != nil { - return nil, err - } - - rev, err := r.ResolveRevision(plumbing.Revision(commitID)) - if err != nil { - return nil, err - } - - commit, err := r.CommitObject(*rev) - if err != nil { - return nil, err - } - - tree, err := commit.Tree() - if err != nil { - return nil, err - } - - sizes := make(map[string]int64) - err = tree.Files().ForEach(func(f *object.File) error { - if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) || - enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) { - return nil - } - - // If content can not be read or file is too big just do detection by filename - var content []byte - if f.Size <= bigFileSize { - content, _ = readFile(f, fileSizeLimit) - } - if enry.IsGenerated(f.Name, content) { - return nil - } - - // TODO: Use .gitattributes file for linguist overrides - - language := analyze.GetCodeLanguage(f.Name, content) - if language == enry.OtherLanguage || language == "" { - return nil - } - - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if group != "" { - language = group - } - - sizes[language] += f.Size - - return nil - }) - if err != nil { - return nil, err - } - - // filter special languages unless they are the only language - if len(sizes) > 1 { - for language := range sizes { - langtype := enry.GetLanguageType(language) - if langtype != enry.Programming && langtype != enry.Markup { - delete(sizes, language) - } - } - } - - return sizes, nil -} - -func readFile(f *object.File, limit int64) ([]byte, error) { - r, err := f.Reader() - if err != nil { - return nil, err - } - defer r.Close() - - if limit <= 0 { - return ioutil.ReadAll(r) - } - - size := f.Size - if limit > 0 && size > limit { - size = limit - } - buf := bytes.NewBuffer(nil) - buf.Grow(int(size)) - _, err = io.Copy(buf, io.LimitReader(r, limit)) - return buf.Bytes(), err -} diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/repo_language_stats_gogit.go new file mode 100644 index 00000000000..b5a235921c8 --- /dev/null +++ b/modules/git/repo_language_stats_gogit.go @@ -0,0 +1,113 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "bytes" + "io" + "io/ioutil" + + "code.gitea.io/gitea/modules/analyze" + + "github.com/go-enry/go-enry/v2" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetLanguageStats calculates language stats for git repository at specified commit +func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { + r, err := git.PlainOpen(repo.Path) + if err != nil { + return nil, err + } + + rev, err := r.ResolveRevision(plumbing.Revision(commitID)) + if err != nil { + return nil, err + } + + commit, err := r.CommitObject(*rev) + if err != nil { + return nil, err + } + + tree, err := commit.Tree() + if err != nil { + return nil, err + } + + sizes := make(map[string]int64) + err = tree.Files().ForEach(func(f *object.File) error { + if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) || + enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) { + return nil + } + + // If content can not be read or file is too big just do detection by filename + var content []byte + if f.Size <= bigFileSize { + content, _ = readFile(f, fileSizeLimit) + } + if enry.IsGenerated(f.Name, content) { + return nil + } + + // TODO: Use .gitattributes file for linguist overrides + + language := analyze.GetCodeLanguage(f.Name, content) + if language == enry.OtherLanguage || language == "" { + return nil + } + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if group != "" { + language = group + } + + sizes[language] += f.Size + + return nil + }) + if err != nil { + return nil, err + } + + // filter special languages unless they are the only language + if len(sizes) > 1 { + for language := range sizes { + langtype := enry.GetLanguageType(language) + if langtype != enry.Programming && langtype != enry.Markup { + delete(sizes, language) + } + } + } + + return sizes, nil +} + +func readFile(f *object.File, limit int64) ([]byte, error) { + r, err := f.Reader() + if err != nil { + return nil, err + } + defer r.Close() + + if limit <= 0 { + return ioutil.ReadAll(r) + } + + size := f.Size + if limit > 0 && size > limit { + size = limit + } + buf := bytes.NewBuffer(nil) + buf.Grow(int(size)) + _, err = io.Copy(buf, io.LimitReader(r, limit)) + return buf.Bytes(), err +} diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go new file mode 100644 index 00000000000..5607e4591ad --- /dev/null +++ b/modules/git/repo_language_stats_nogogit.go @@ -0,0 +1,109 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bytes" + "io" + "io/ioutil" + + "code.gitea.io/gitea/modules/analyze" + + "github.com/go-enry/go-enry/v2" +) + +// GetLanguageStats calculates language stats for git repository at specified commit +func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { + // FIXME: We can be more efficient here... + // + // We're expecting that we will be reading a lot of blobs and the trees + // Thus we should use a shared `cat-file --batch` to get all of this data + // And keep the buffers around with resets as necessary. + // + // It's more complicated so... + commit, err := repo.GetCommit(commitID) + if err != nil { + log("Unable to get commit for: %s", commitID) + return nil, err + } + + tree := commit.Tree + + entries, err := tree.ListEntriesRecursive() + if err != nil { + return nil, err + } + + sizes := make(map[string]int64) + for _, f := range entries { + if f.Size() == 0 || enry.IsVendor(f.Name()) || enry.IsDotFile(f.Name()) || + enry.IsDocumentation(f.Name()) || enry.IsConfiguration(f.Name()) { + continue + } + + // If content can not be read or file is too big just do detection by filename + var content []byte + if f.Size() <= bigFileSize { + content, _ = readFile(f, fileSizeLimit) + } + if enry.IsGenerated(f.Name(), content) { + continue + } + + // TODO: Use .gitattributes file for linguist overrides + // FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary? + // - eg. do the all the detection tests using filename first before reading content. + language := analyze.GetCodeLanguage(f.Name(), content) + if language == enry.OtherLanguage || language == "" { + continue + } + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if group != "" { + language = group + } + + sizes[language] += f.Size() + + continue + } + + // filter special languages unless they are the only language + if len(sizes) > 1 { + for language := range sizes { + langtype := enry.GetLanguageType(language) + if langtype != enry.Programming && langtype != enry.Markup { + delete(sizes, language) + } + } + } + + return sizes, nil +} + +func readFile(entry *TreeEntry, limit int64) ([]byte, error) { + // FIXME: We can probably be a little more efficient here... see above + r, err := entry.Blob().DataAsync() + if err != nil { + return nil, err + } + defer r.Close() + + if limit <= 0 { + return ioutil.ReadAll(r) + } + + size := entry.Size() + if limit > 0 && size > limit { + size = limit + } + buf := bytes.NewBuffer(nil) + buf.Grow(int(size)) + _, err = io.Copy(buf, io.LimitReader(r, limit)) + return buf.Bytes(), err +} diff --git a/modules/git/repo_object.go b/modules/git/repo_object.go index d4d638a7434..f054c349029 100644 --- a/modules/git/repo_object.go +++ b/modules/git/repo_object.go @@ -27,6 +27,11 @@ const ( ObjectBranch ObjectType = "branch" ) +// Bytes returns the byte array for the Object Type +func (o ObjectType) Bytes() []byte { + return []byte(o) +} + // HashObject takes a reader and returns SHA1 hash for that reader func (repo *Repository) HashObject(reader io.Reader) (SHA1, error) { idStr, err := repo.hashObject(reader) diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index be2a38c5f0a..397434e12f2 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -4,52 +4,7 @@ package git -import ( - "strings" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" -) - // GetRefs returns all references of the repository. func (repo *Repository) GetRefs() ([]*Reference, error) { return repo.GetRefsFiltered("") } - -// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. -func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { - r, err := git.PlainOpen(repo.Path) - if err != nil { - return nil, err - } - - refsIter, err := r.References() - if err != nil { - return nil, err - } - refs := make([]*Reference, 0) - if err = refsIter.ForEach(func(ref *plumbing.Reference) error { - if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() && - (pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) { - refType := string(ObjectCommit) - if ref.Name().IsTag() { - // tags can be of type `commit` (lightweight) or `tag` (annotated) - if tagType, _ := repo.GetTagType(ref.Hash()); err == nil { - refType = tagType - } - } - r := &Reference{ - Name: ref.Name().String(), - Object: ref.Hash(), - Type: refType, - repo: repo, - } - refs = append(refs, r) - } - return nil - }); err != nil { - return nil, err - } - - return refs, nil -} diff --git a/modules/git/repo_ref_gogit.go b/modules/git/repo_ref_gogit.go new file mode 100644 index 00000000000..2e83e6c4627 --- /dev/null +++ b/modules/git/repo_ref_gogit.go @@ -0,0 +1,52 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. +func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { + r, err := git.PlainOpen(repo.Path) + if err != nil { + return nil, err + } + + refsIter, err := r.References() + if err != nil { + return nil, err + } + refs := make([]*Reference, 0) + if err = refsIter.ForEach(func(ref *plumbing.Reference) error { + if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() && + (pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) { + refType := string(ObjectCommit) + if ref.Name().IsTag() { + // tags can be of type `commit` (lightweight) or `tag` (annotated) + if tagType, _ := repo.GetTagType(ref.Hash()); err == nil { + refType = tagType + } + } + r := &Reference{ + Name: ref.Name().String(), + Object: ref.Hash(), + Type: refType, + repo: repo, + } + refs = append(refs, r) + } + return nil + }); err != nil { + return nil, err + } + + return refs, nil +} diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go new file mode 100644 index 00000000000..540961592b4 --- /dev/null +++ b/modules/git/repo_ref_nogogit.go @@ -0,0 +1,84 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "io" + "strings" +) + +// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. +func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("for-each-ref").RunInDirPipeline(repo.Path, stdoutWriter, stderrBuilder) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + refs := make([]*Reference, 0) + bufReader := bufio.NewReader(stdoutReader) + for { + // The output of for-each-ref is simply a list: + // SP TAB LF + sha, err := bufReader.ReadString(' ') + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + sha = sha[:len(sha)-1] + + typ, err := bufReader.ReadString('\t') + if err == io.EOF { + // This should not happen, but we'll tolerate it + break + } + if err != nil { + return nil, err + } + typ = typ[:len(typ)-1] + + refName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This should not happen, but we'll tolerate it + break + } + if err != nil { + return nil, err + } + refName = refName[:len(refName)-1] + + // refName cannot be HEAD but can be remotes or stash + if strings.HasPrefix(refName, "/refs/remotes/") || refName == "/refs/stash" { + continue + } + + if pattern == "" || strings.HasPrefix(refName, pattern) { + r := &Reference{ + Name: refName, + Object: MustIDFromString(sha), + Type: typ, + repo: repo, + } + refs = append(refs, r) + } + } + + return refs, nil +} diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 376a699502b..3e8f80fe82d 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -8,8 +8,6 @@ package git import ( "fmt" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // TagPrefix tags prefix path on the repository @@ -20,12 +18,6 @@ func IsTagExist(repoPath, name string) bool { return IsReferenceExist(repoPath, TagPrefix+name) } -// IsTagExist returns true if given tag exists in the repository. -func (repo *Repository) IsTagExist(name string) bool { - _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true) - return err == nil -} - // CreateTag create one tag in the repository func (repo *Repository) CreateTag(name, revision string) error { _, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path) @@ -224,29 +216,6 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) { return tags, nil } -// GetTags returns all tags of the repository. -func (repo *Repository) GetTags() ([]string, error) { - var tagNames []string - - tags, err := repo.gogitRepo.Tags() - if err != nil { - return nil, err - } - - _ = tags.ForEach(func(tag *plumbing.Reference) error { - tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) - return nil - }) - - // Reverse order - for i := 0; i < len(tagNames)/2; i++ { - j := len(tagNames) - i - 1 - tagNames[i], tagNames[j] = tagNames[j], tagNames[i] - } - - return tagNames, nil -} - // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) func (repo *Repository) GetTagType(id SHA1) (string, error) { // Get tag type diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go new file mode 100644 index 00000000000..3ac097c9a8c --- /dev/null +++ b/modules/git/repo_tag_gogit.go @@ -0,0 +1,43 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +// IsTagExist returns true if given tag exists in the repository. +func (repo *Repository) IsTagExist(name string) bool { + _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true) + return err == nil +} + +// GetTags returns all tags of the repository. +func (repo *Repository) GetTags() ([]string, error) { + var tagNames []string + + tags, err := repo.gogitRepo.Tags() + if err != nil { + return nil, err + } + + _ = tags.ForEach(func(tag *plumbing.Reference) error { + tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) + return nil + }) + + // Reverse order + for i := 0; i < len(tagNames)/2; i++ { + j := len(tagNames) - i - 1 + tagNames[i], tagNames[j] = tagNames[j], tagNames[i] + } + + return tagNames, nil +} diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go new file mode 100644 index 00000000000..83cbc58e342 --- /dev/null +++ b/modules/git/repo_tag_nogogit.go @@ -0,0 +1,18 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +// IsTagExist returns true if given tag exists in the repository. +func (repo *Repository) IsTagExist(name string) bool { + return IsReferenceExist(repo.Path, TagPrefix+name) +} + +// GetTags returns all tags of the repository. +func (repo *Repository) GetTags() ([]string, error) { + return callShowRef(repo.Path, TagPrefix, "--tags") +} diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index 0b08a10d554..2053b6a1de1 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -13,45 +13,6 @@ import ( "time" ) -func (repo *Repository) getTree(id SHA1) (*Tree, error) { - gogitTree, err := repo.gogitRepo.TreeObject(id) - if err != nil { - return nil, err - } - - tree := NewTree(repo, id) - tree.gogitTree = gogitTree - return tree, nil -} - -// GetTree find the tree object in the repository. -func (repo *Repository) GetTree(idStr string) (*Tree, error) { - if len(idStr) != 40 { - res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) - if err != nil { - return nil, err - } - if len(res) > 0 { - idStr = res[:len(res)-1] - } - } - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - resolvedID := id - commitObject, err := repo.gogitRepo.CommitObject(id) - if err == nil { - id = SHA1(commitObject.TreeHash) - } - treeObject, err := repo.getTree(id) - if err != nil { - return nil, err - } - treeObject.ResolvedID = resolvedID - return treeObject, nil -} - // CommitTreeOpts represents the possible options to CommitTree type CommitTreeOpts struct { Parents []string @@ -102,7 +63,7 @@ func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes) if err != nil { - return SHA1{}, concatenateError(err, stderr.String()) + return SHA1{}, ConcatenateError(err, stderr.String()) } return NewIDFromString(strings.TrimSpace(stdout.String())) } diff --git a/modules/git/repo_tree_gogit.go b/modules/git/repo_tree_gogit.go new file mode 100644 index 00000000000..d878f5e7a72 --- /dev/null +++ b/modules/git/repo_tree_gogit.go @@ -0,0 +1,47 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +func (repo *Repository) getTree(id SHA1) (*Tree, error) { + gogitTree, err := repo.gogitRepo.TreeObject(id) + if err != nil { + return nil, err + } + + tree := NewTree(repo, id) + tree.gogitTree = gogitTree + return tree, nil +} + +// GetTree find the tree object in the repository. +func (repo *Repository) GetTree(idStr string) (*Tree, error) { + if len(idStr) != 40 { + res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) + if err != nil { + return nil, err + } + if len(res) > 0 { + idStr = res[:len(res)-1] + } + } + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + resolvedID := id + commitObject, err := repo.gogitRepo.CommitObject(id) + if err == nil { + id = SHA1(commitObject.TreeHash) + } + treeObject, err := repo.getTree(id) + if err != nil { + return nil, err + } + treeObject.ResolvedID = resolvedID + return treeObject, nil +} diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go new file mode 100644 index 00000000000..416205d8a09 --- /dev/null +++ b/modules/git/repo_tree_nogogit.go @@ -0,0 +1,98 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "strings" +) + +func (repo *Repository) getTree(id SHA1) (*Tree, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderr := &strings.Builder{} + err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n")) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + // ignore the SHA + _, typ, _, err := ReadBatchLine(bufReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + resolvedID := id + data, err := ioutil.ReadAll(bufReader) + if err != nil { + return nil, err + } + tag, err := parseTagData(data) + if err != nil { + return nil, err + } + commit, err := tag.Commit() + if err != nil { + return nil, err + } + commit.Tree.ResolvedID = resolvedID + log("tag.commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo) + return &commit.Tree, nil + case "commit": + commit, err := CommitFromReader(repo, id, bufReader) + if err != nil { + _ = stdoutReader.CloseWithError(err) + return nil, err + } + commit.Tree.ResolvedID = commit.ID + log("commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo) + return &commit.Tree, nil + case "tree": + stdoutReader.Close() + tree := NewTree(repo, id) + tree.ResolvedID = id + return tree, nil + default: + _ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) + return nil, ErrNotExist{ + ID: id.String(), + } + } +} + +// GetTree find the tree object in the repository. +func (repo *Repository) GetTree(idStr string) (*Tree, error) { + if len(idStr) != 40 { + res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) + if err != nil { + return nil, err + } + if len(res) > 0 { + idStr = res[:len(res)-1] + } + } + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + + return repo.getTree(id) +} diff --git a/modules/git/sha1.go b/modules/git/sha1.go index 06c8ad14b55..2da74733df7 100644 --- a/modules/git/sha1.go +++ b/modules/git/sha1.go @@ -10,8 +10,6 @@ import ( "fmt" "regexp" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // EmptySHA defines empty git SHA @@ -23,9 +21,6 @@ const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" // SHAPattern can be used to determine if a string is an valid sha var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`) -// SHA1 a git commit name -type SHA1 = plumbing.Hash - // MustID always creates a new SHA1 from a [20]byte array with no validation of input. func MustID(b []byte) SHA1 { var id SHA1 diff --git a/modules/git/sha1_gogit.go b/modules/git/sha1_gogit.go new file mode 100644 index 00000000000..5953af58bfc --- /dev/null +++ b/modules/git/sha1_gogit.go @@ -0,0 +1,20 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" +) + +// SHA1 a git commit name +type SHA1 = plumbing.Hash + +// ComputeBlobHash compute the hash for a given blob content +func ComputeBlobHash(content []byte) SHA1 { + return plumbing.ComputeHash(plumbing.BlobObject, content) +} diff --git a/modules/git/sha1_nogogit.go b/modules/git/sha1_nogogit.go new file mode 100644 index 00000000000..09b5baacd5c --- /dev/null +++ b/modules/git/sha1_nogogit.go @@ -0,0 +1,62 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "crypto/sha1" + "encoding/hex" + "hash" + "strconv" +) + +// SHA1 a git commit name +type SHA1 [20]byte + +// String returns a string representation of the SHA +func (s SHA1) String() string { + return hex.EncodeToString(s[:]) +} + +// IsZero returns whether this SHA1 is all zeroes +func (s SHA1) IsZero() bool { + var empty SHA1 + return s == empty +} + +// ComputeBlobHash compute the hash for a given blob content +func ComputeBlobHash(content []byte) SHA1 { + return ComputeHash(ObjectBlob, content) +} + +// ComputeHash compute the hash for a given ObjectType and content +func ComputeHash(t ObjectType, content []byte) SHA1 { + h := NewHasher(t, int64(len(content))) + _, _ = h.Write(content) + return h.Sum() +} + +// Hasher is a struct that will generate a SHA1 +type Hasher struct { + hash.Hash +} + +// NewHasher takes an object type and size and creates a hasher to generate a SHA +func NewHasher(t ObjectType, size int64) Hasher { + h := Hasher{sha1.New()} + _, _ = h.Write(t.Bytes()) + _, _ = h.Write([]byte(" ")) + _, _ = h.Write([]byte(strconv.FormatInt(size, 10))) + _, _ = h.Write([]byte{0}) + return h +} + +// Sum generates a SHA1 for the provided hash +func (h Hasher) Sum() (sha1 SHA1) { + copy(sha1[:], h.Hash.Sum(nil)) + return +} diff --git a/modules/git/signature.go b/modules/git/signature.go index 4cb56b29f44..b59db8f490e 100644 --- a/modules/git/signature.go +++ b/modules/git/signature.go @@ -5,53 +5,7 @@ package git -import ( - "bytes" - "strconv" - "time" - - "github.com/go-git/go-git/v5/plumbing/object" -) - -// Signature represents the Author or Committer information. -type Signature = object.Signature - const ( // GitTimeLayout is the (default) time layout used by git. GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700" ) - -// Helper to get a signature from the commit line, which looks like these: -// author Patrick Gundlach 1378823654 +0200 -// author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 -// but without the "author " at the beginning (this method should) -// be used for author and committer. -// -// FIXME: include timezone for timestamp! -func newSignatureFromCommitline(line []byte) (_ *Signature, err error) { - sig := new(Signature) - emailStart := bytes.IndexByte(line, '<') - sig.Name = string(line[:emailStart-1]) - emailEnd := bytes.IndexByte(line, '>') - sig.Email = string(line[emailStart+1 : emailEnd]) - - // Check date format. - if len(line) > emailEnd+2 { - firstChar := line[emailEnd+2] - if firstChar >= 48 && firstChar <= 57 { - timestop := bytes.IndexByte(line[emailEnd+2:], ' ') - timestring := string(line[emailEnd+2 : emailEnd+2+timestop]) - seconds, _ := strconv.ParseInt(timestring, 10, 64) - sig.When = time.Unix(seconds, 0) - } else { - sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:])) - if err != nil { - return nil, err - } - } - } else { - // Fall back to unix 0 time - sig.When = time.Unix(0, 0) - } - return sig, nil -} diff --git a/modules/git/signature_gogit.go b/modules/git/signature_gogit.go new file mode 100644 index 00000000000..804c0074d35 --- /dev/null +++ b/modules/git/signature_gogit.go @@ -0,0 +1,54 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "bytes" + "strconv" + "time" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +// Signature represents the Author or Committer information. +type Signature = object.Signature + +// Helper to get a signature from the commit line, which looks like these: +// author Patrick Gundlach 1378823654 +0200 +// author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 +// but without the "author " at the beginning (this method should) +// be used for author and committer. +// +// FIXME: include timezone for timestamp! +func newSignatureFromCommitline(line []byte) (_ *Signature, err error) { + sig := new(Signature) + emailStart := bytes.IndexByte(line, '<') + sig.Name = string(line[:emailStart-1]) + emailEnd := bytes.IndexByte(line, '>') + sig.Email = string(line[emailStart+1 : emailEnd]) + + // Check date format. + if len(line) > emailEnd+2 { + firstChar := line[emailEnd+2] + if firstChar >= 48 && firstChar <= 57 { + timestop := bytes.IndexByte(line[emailEnd+2:], ' ') + timestring := string(line[emailEnd+2 : emailEnd+2+timestop]) + seconds, _ := strconv.ParseInt(timestring, 10, 64) + sig.When = time.Unix(seconds, 0) + } else { + sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:])) + if err != nil { + return nil, err + } + } + } else { + // Fall back to unix 0 time + sig.When = time.Unix(0, 0) + } + return sig, nil +} diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go new file mode 100644 index 00000000000..753d87b6059 --- /dev/null +++ b/modules/git/signature_nogogit.go @@ -0,0 +1,95 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "bytes" + "fmt" + "strconv" + "time" +) + +// Signature represents the Author or Committer information. +type Signature struct { + // Name represents a person name. It is an arbitrary string. + Name string + // Email is an email, but it cannot be assumed to be well-formed. + Email string + // When is the timestamp of the signature. + When time.Time +} + +func (s *Signature) String() string { + return fmt.Sprintf("%s <%s>", s.Name, s.Email) +} + +// Decode decodes a byte array representing a signature to signature +func (s *Signature) Decode(b []byte) { + sig, _ := newSignatureFromCommitline(b) + s.Email = sig.Email + s.Name = sig.Name + s.When = sig.When +} + +// Helper to get a signature from the commit line, which looks like these: +// author Patrick Gundlach 1378823654 +0200 +// author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 +// but without the "author " at the beginning (this method should) +// be used for author and committer. +func newSignatureFromCommitline(line []byte) (sig *Signature, err error) { + sig = new(Signature) + emailStart := bytes.LastIndexByte(line, '<') + emailEnd := bytes.LastIndexByte(line, '>') + if emailStart == -1 || emailEnd == -1 || emailEnd < emailStart { + return + } + + sig.Name = string(line[:emailStart-1]) + sig.Email = string(line[emailStart+1 : emailEnd]) + + hasTime := emailEnd+2 < len(line) + if !hasTime { + return + } + + // Check date format. + firstChar := line[emailEnd+2] + if firstChar >= 48 && firstChar <= 57 { + idx := bytes.IndexByte(line[emailEnd+2:], ' ') + if idx < 0 { + return + } + + timestring := string(line[emailEnd+2 : emailEnd+2+idx]) + seconds, _ := strconv.ParseInt(timestring, 10, 64) + sig.When = time.Unix(seconds, 0) + + idx += emailEnd + 3 + if idx >= len(line) || idx+5 > len(line) { + return + } + + timezone := string(line[idx : idx+5]) + tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64) + tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64) + if err1 != nil || err2 != nil { + return + } + if tzhours < 0 { + tzmins *= -1 + } + tz := time.FixedZone("", int(tzhours*60*60+tzmins*60)) + sig.When = sig.When.In(tz) + } else { + sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:])) + if err != nil { + return + } + } + return +} diff --git a/modules/git/tag.go b/modules/git/tag.go index c97f574fa62..d58a9a202dc 100644 --- a/modules/git/tag.go +++ b/modules/git/tag.go @@ -10,15 +10,19 @@ import ( "strings" ) +const beginpgp = "\n-----BEGIN PGP SIGNATURE-----\n" +const endpgp = "\n-----END PGP SIGNATURE-----" + // Tag represents a Git tag. type Tag struct { - Name string - ID SHA1 - repo *Repository - Object SHA1 // The id of this commit object - Type string - Tagger *Signature - Message string + Name string + ID SHA1 + repo *Repository + Object SHA1 // The id of this commit object + Type string + Tagger *Signature + Message string + Signature *CommitGPGSignature } // Commit return the commit of the tag reference @@ -60,12 +64,23 @@ l: } nextline += eol + 1 case eol == 0: - tag.Message = strings.TrimRight(string(data[nextline+1:]), "\n") + tag.Message = string(data[nextline+1 : len(data)-1]) break l default: break l } } + idx := strings.LastIndex(tag.Message, beginpgp) + if idx > 0 { + endSigIdx := strings.Index(tag.Message[idx:], endpgp) + if endSigIdx > 0 { + tag.Signature = &CommitGPGSignature{ + Signature: tag.Message[idx+1 : idx+endSigIdx+len(endpgp)], + Payload: string(data[:bytes.LastIndex(data, []byte(beginpgp))+1]), + } + tag.Message = tag.Message[:idx+1] + } + } return tag, nil } diff --git a/modules/git/tree.go b/modules/git/tree.go index 258b11aaac5..059f0a8287d 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -6,25 +6,9 @@ package git import ( - "io" "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" ) -// Tree represents a flat directory listing. -type Tree struct { - ID SHA1 - ResolvedID SHA1 - repo *Repository - - gogitTree *object.Tree - - // parent tree - ptree *Tree -} - // NewTree create a new tree according the repository and tree id func NewTree(repo *Repository, id SHA1) *Tree { return &Tree{ @@ -61,70 +45,3 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) { } return g, nil } - -func (t *Tree) loadTreeObject() error { - gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID) - if err != nil { - return err - } - - t.gogitTree = gogitTree - return nil -} - -// ListEntries returns all entries of current tree. -func (t *Tree) ListEntries() (Entries, error) { - if t.gogitTree == nil { - err := t.loadTreeObject() - if err != nil { - return nil, err - } - } - - entries := make([]*TreeEntry, len(t.gogitTree.Entries)) - for i, entry := range t.gogitTree.Entries { - entries[i] = &TreeEntry{ - ID: entry.Hash, - gogitTreeEntry: &t.gogitTree.Entries[i], - ptree: t, - } - } - - return entries, nil -} - -// ListEntriesRecursive returns all entries of current tree recursively including all subtrees -func (t *Tree) ListEntriesRecursive() (Entries, error) { - if t.gogitTree == nil { - err := t.loadTreeObject() - if err != nil { - return nil, err - } - } - - var entries []*TreeEntry - seen := map[plumbing.Hash]bool{} - walker := object.NewTreeWalker(t.gogitTree, true, seen) - for { - fullName, entry, err := walker.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if seen[entry.Hash] { - continue - } - - convertedEntry := &TreeEntry{ - ID: entry.Hash, - gogitTreeEntry: &entry, - ptree: t, - fullName: fullName, - } - entries = append(entries, convertedEntry) - } - - return entries, nil -} diff --git a/modules/git/tree_blob.go b/modules/git/tree_blob.go index f9fc6db497c..19edcf4c6cb 100644 --- a/modules/git/tree_blob.go +++ b/modules/git/tree_blob.go @@ -5,64 +5,6 @@ package git -import ( - "path" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { - if len(relpath) == 0 { - return &TreeEntry{ - ID: t.ID, - //Type: ObjectTree, - gogitTreeEntry: &object.TreeEntry{ - Name: "", - Mode: filemode.Dir, - Hash: t.ID, - }, - }, nil - } - - relpath = path.Clean(relpath) - parts := strings.Split(relpath, "/") - var err error - tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - RelPath: relpath, - } - } - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - RelPath: relpath, - } - } - return nil, err - } - } - } - return nil, ErrNotExist{"", relpath} -} - // GetBlobByPath get the blob object according the path func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) { entry, err := t.GetTreeEntryByPath(relpath) diff --git a/modules/git/tree_blob_gogit.go b/modules/git/tree_blob_gogit.go new file mode 100644 index 00000000000..93ebc8a367d --- /dev/null +++ b/modules/git/tree_blob_gogit.go @@ -0,0 +1,66 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "path" + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetTreeEntryByPath get the tree entries according the sub dir +func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { + if len(relpath) == 0 { + return &TreeEntry{ + ID: t.ID, + //Type: ObjectTree, + gogitTreeEntry: &object.TreeEntry{ + Name: "", + Mode: filemode.Dir, + Hash: t.ID, + }, + }, nil + } + + relpath = path.Clean(relpath) + parts := strings.Split(relpath, "/") + var err error + tree := t + for i, name := range parts { + if i == len(parts)-1 { + entries, err := tree.ListEntries() + if err != nil { + if err == plumbing.ErrObjectNotFound { + return nil, ErrNotExist{ + RelPath: relpath, + } + } + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil + } + } + } else { + tree, err = tree.SubTree(name) + if err != nil { + if err == plumbing.ErrObjectNotFound { + return nil, ErrNotExist{ + RelPath: relpath, + } + } + return nil, err + } + } + } + return nil, ErrNotExist{"", relpath} +} diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go new file mode 100644 index 00000000000..6da0ccfe8e5 --- /dev/null +++ b/modules/git/tree_blob_nogogit.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "path" + "strings" +) + +// GetTreeEntryByPath get the tree entries according the sub dir +func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { + if len(relpath) == 0 { + return &TreeEntry{ + ID: t.ID, + name: "", + fullName: "", + entryMode: EntryModeTree, + }, nil + } + + // FIXME: This should probably use git cat-file --batch to be a bit more efficient + relpath = path.Clean(relpath) + parts := strings.Split(relpath, "/") + var err error + tree := t + for i, name := range parts { + if i == len(parts)-1 { + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil + } + } + } else { + tree, err = tree.SubTree(name) + if err != nil { + return nil, err + } + } + } + return nil, ErrNotExist{"", relpath} +} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index d9814120953..498767a63eb 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -9,55 +9,8 @@ import ( "io" "sort" "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" ) -// EntryMode the type of the object in the git tree -type EntryMode int - -// There are only a few file modes in Git. They look like unix file modes, but they can only be -// one of these. -const ( - // EntryModeBlob - EntryModeBlob EntryMode = 0100644 - // EntryModeExec - EntryModeExec EntryMode = 0100755 - // EntryModeSymlink - EntryModeSymlink EntryMode = 0120000 - // EntryModeCommit - EntryModeCommit EntryMode = 0160000 - // EntryModeTree - EntryModeTree EntryMode = 0040000 -) - -// TreeEntry the leaf in the git tree -type TreeEntry struct { - ID SHA1 - - gogitTreeEntry *object.TreeEntry - ptree *Tree - - size int64 - sized bool - fullName string -} - -// Name returns the name of the entry -func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } - return te.gogitTreeEntry.Name -} - -// Mode returns the mode of the entry -func (te *TreeEntry) Mode() EntryMode { - return EntryMode(te.gogitTreeEntry.Mode) -} - // Type returns the type of the entry (commit, tree, blob) func (te *TreeEntry) Type() string { switch te.Mode() { @@ -70,63 +23,6 @@ func (te *TreeEntry) Type() string { } } -// Size returns the size of the entry -func (te *TreeEntry) Size() int64 { - if te.IsDir() { - return 0 - } else if te.sized { - return te.size - } - - file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry) - if err != nil { - return 0 - } - - te.sized = true - te.size = file.Size - return te.size -} - -// IsSubModule if the entry is a sub module -func (te *TreeEntry) IsSubModule() bool { - return te.gogitTreeEntry.Mode == filemode.Submodule -} - -// IsDir if the entry is a sub dir -func (te *TreeEntry) IsDir() bool { - return te.gogitTreeEntry.Mode == filemode.Dir -} - -// IsLink if the entry is a symlink -func (te *TreeEntry) IsLink() bool { - return te.gogitTreeEntry.Mode == filemode.Symlink -} - -// IsRegular if the entry is a regular file -func (te *TreeEntry) IsRegular() bool { - return te.gogitTreeEntry.Mode == filemode.Regular -} - -// IsExecutable if the entry is an executable file (not necessarily binary) -func (te *TreeEntry) IsExecutable() bool { - return te.gogitTreeEntry.Mode == filemode.Executable -} - -// Blob returns the blob object the entry -func (te *TreeEntry) Blob() *Blob { - encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash) - if err != nil { - return nil - } - - return &Blob{ - ID: te.gogitTreeEntry.Hash, - gogitEncodedObj: encodedObj, - name: te.Name(), - } -} - // FollowLink returns the entry pointed to by a symlink func (te *TreeEntry) FollowLink() (*TreeEntry, error) { if !te.IsLink() { diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go new file mode 100644 index 00000000000..219251a77e4 --- /dev/null +++ b/modules/git/tree_entry_gogit.go @@ -0,0 +1,96 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// TreeEntry the leaf in the git tree +type TreeEntry struct { + ID SHA1 + + gogitTreeEntry *object.TreeEntry + ptree *Tree + + size int64 + sized bool + fullName string +} + +// Name returns the name of the entry +func (te *TreeEntry) Name() string { + if te.fullName != "" { + return te.fullName + } + return te.gogitTreeEntry.Name +} + +// Mode returns the mode of the entry +func (te *TreeEntry) Mode() EntryMode { + return EntryMode(te.gogitTreeEntry.Mode) +} + +// Size returns the size of the entry +func (te *TreeEntry) Size() int64 { + if te.IsDir() { + return 0 + } else if te.sized { + return te.size + } + + file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry) + if err != nil { + return 0 + } + + te.sized = true + te.size = file.Size + return te.size +} + +// IsSubModule if the entry is a sub module +func (te *TreeEntry) IsSubModule() bool { + return te.gogitTreeEntry.Mode == filemode.Submodule +} + +// IsDir if the entry is a sub dir +func (te *TreeEntry) IsDir() bool { + return te.gogitTreeEntry.Mode == filemode.Dir +} + +// IsLink if the entry is a symlink +func (te *TreeEntry) IsLink() bool { + return te.gogitTreeEntry.Mode == filemode.Symlink +} + +// IsRegular if the entry is a regular file +func (te *TreeEntry) IsRegular() bool { + return te.gogitTreeEntry.Mode == filemode.Regular +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (te *TreeEntry) IsExecutable() bool { + return te.gogitTreeEntry.Mode == filemode.Executable +} + +// Blob returns the blob object the entry +func (te *TreeEntry) Blob() *Blob { + encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash) + if err != nil { + return nil + } + + return &Blob{ + ID: te.gogitTreeEntry.Hash, + gogitEncodedObj: encodedObj, + name: te.Name(), + } +} diff --git a/modules/git/tree_entry_mode.go b/modules/git/tree_entry_mode.go new file mode 100644 index 00000000000..b029c6fc476 --- /dev/null +++ b/modules/git/tree_entry_mode.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import "strconv" + +// EntryMode the type of the object in the git tree +type EntryMode int + +// There are only a few file modes in Git. They look like unix file modes, but they can only be +// one of these. +const ( + // EntryModeBlob + EntryModeBlob EntryMode = 0100644 + // EntryModeExec + EntryModeExec EntryMode = 0100755 + // EntryModeSymlink + EntryModeSymlink EntryMode = 0120000 + // EntryModeCommit + EntryModeCommit EntryMode = 0160000 + // EntryModeTree + EntryModeTree EntryMode = 0040000 +) + +// String converts an EntryMode to a string +func (e EntryMode) String() string { + return strconv.FormatInt(int64(e), 8) +} + +// ToEntryMode converts a string to an EntryMode +func ToEntryMode(value string) EntryMode { + v, _ := strconv.ParseInt(value, 8, 32) + return EntryMode(v) +} diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go new file mode 100644 index 00000000000..f18daee7788 --- /dev/null +++ b/modules/git/tree_entry_nogogit.go @@ -0,0 +1,91 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "strconv" + "strings" +) + +// TreeEntry the leaf in the git tree +type TreeEntry struct { + ID SHA1 + + ptree *Tree + + entryMode EntryMode + name string + + size int64 + sized bool + fullName string +} + +// Name returns the name of the entry +func (te *TreeEntry) Name() string { + if te.fullName != "" { + return te.fullName + } + return te.name +} + +// Mode returns the mode of the entry +func (te *TreeEntry) Mode() EntryMode { + return te.entryMode +} + +// Size returns the size of the entry +func (te *TreeEntry) Size() int64 { + if te.IsDir() { + return 0 + } else if te.sized { + return te.size + } + + stdout, err := NewCommand("cat-file", "-s", te.ID.String()).RunInDir(te.ptree.repo.Path) + if err != nil { + return 0 + } + + te.sized = true + te.size, _ = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) + return te.size +} + +// IsSubModule if the entry is a sub module +func (te *TreeEntry) IsSubModule() bool { + return te.entryMode == EntryModeCommit +} + +// IsDir if the entry is a sub dir +func (te *TreeEntry) IsDir() bool { + return te.entryMode == EntryModeTree +} + +// IsLink if the entry is a symlink +func (te *TreeEntry) IsLink() bool { + return te.entryMode == EntryModeSymlink +} + +// IsRegular if the entry is a regular file +func (te *TreeEntry) IsRegular() bool { + return te.entryMode == EntryModeBlob +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (te *TreeEntry) IsExecutable() bool { + return te.entryMode == EntryModeExec +} + +// Blob returns the blob object the entry +func (te *TreeEntry) Blob() *Blob { + return &Blob{ + ID: te.ID, + repoPath: te.ptree.repo.Path, + name: te.Name(), + } +} diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 4878fce0b84..16cfbc4fc3a 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build gogit + package git import ( diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go new file mode 100644 index 00000000000..79132c5548a --- /dev/null +++ b/modules/git/tree_gogit.go @@ -0,0 +1,94 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package git + +import ( + "io" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// Tree represents a flat directory listing. +type Tree struct { + ID SHA1 + ResolvedID SHA1 + repo *Repository + + gogitTree *object.Tree + + // parent tree + ptree *Tree +} + +func (t *Tree) loadTreeObject() error { + gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID) + if err != nil { + return err + } + + t.gogitTree = gogitTree + return nil +} + +// ListEntries returns all entries of current tree. +func (t *Tree) ListEntries() (Entries, error) { + if t.gogitTree == nil { + err := t.loadTreeObject() + if err != nil { + return nil, err + } + } + + entries := make([]*TreeEntry, len(t.gogitTree.Entries)) + for i, entry := range t.gogitTree.Entries { + entries[i] = &TreeEntry{ + ID: entry.Hash, + gogitTreeEntry: &t.gogitTree.Entries[i], + ptree: t, + } + } + + return entries, nil +} + +// ListEntriesRecursive returns all entries of current tree recursively including all subtrees +func (t *Tree) ListEntriesRecursive() (Entries, error) { + if t.gogitTree == nil { + err := t.loadTreeObject() + if err != nil { + return nil, err + } + } + + var entries []*TreeEntry + seen := map[plumbing.Hash]bool{} + walker := object.NewTreeWalker(t.gogitTree, true, seen) + for { + fullName, entry, err := walker.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if seen[entry.Hash] { + continue + } + + convertedEntry := &TreeEntry{ + ID: entry.Hash, + gogitTreeEntry: &entry, + ptree: t, + fullName: fullName, + } + entries = append(entries, convertedEntry) + } + + return entries, nil +} diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go new file mode 100644 index 00000000000..e78115b7776 --- /dev/null +++ b/modules/git/tree_nogogit.go @@ -0,0 +1,69 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package git + +import ( + "strings" +) + +// Tree represents a flat directory listing. +type Tree struct { + ID SHA1 + ResolvedID SHA1 + repo *Repository + + // parent tree + ptree *Tree + + entries Entries + entriesParsed bool + + entriesRecursive Entries + entriesRecursiveParsed bool +} + +// ListEntries returns all entries of current tree. +func (t *Tree) ListEntries() (Entries, error) { + if t.entriesParsed { + return t.entries, nil + } + + stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path) + if err != nil { + if strings.Contains(err.Error(), "fatal: Not a valid object name") || strings.Contains(err.Error(), "fatal: not a tree object") { + return nil, ErrNotExist{ + ID: t.ID.String(), + } + } + return nil, err + } + + t.entries, err = parseTreeEntries(stdout, t) + if err == nil { + t.entriesParsed = true + } + + return t.entries, err +} + +// ListEntriesRecursive returns all entries of current tree recursively including all subtrees +func (t *Tree) ListEntriesRecursive() (Entries, error) { + if t.entriesRecursiveParsed { + return t.entriesRecursive, nil + } + stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path) + if err != nil { + return nil, err + } + + t.entriesRecursive, err = parseTreeEntries(stdout, t) + if err == nil { + t.entriesRecursiveParsed = true + } + + return t.entriesRecursive, err +} diff --git a/modules/git/utils.go b/modules/git/utils.go index 83209924c8d..d9521894160 100644 --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -6,6 +6,7 @@ package git import ( "fmt" + "io" "os" "strconv" "strings" @@ -68,11 +69,12 @@ func isExist(path string) bool { return err == nil || os.IsExist(err) } -func concatenateError(err error, stderr string) error { +// ConcatenateError concatenats an error with stderr string +func ConcatenateError(err error, stderr string) error { if len(stderr) == 0 { return err } - return fmt.Errorf("%v - %s", err, stderr) + return fmt.Errorf("%w - %s", err, stderr) } // RefEndName return the end name of a ref name @@ -140,3 +142,29 @@ func ParseBool(value string) (result bool, valid bool) { } return intValue != 0, true } + +// LimitedReaderCloser is a limited reader closer +type LimitedReaderCloser struct { + R io.Reader + C io.Closer + N int64 +} + +// Read implements io.Reader +func (l *LimitedReaderCloser) Read(p []byte) (n int, err error) { + if l.N <= 0 { + _ = l.C.Close() + return 0, io.EOF + } + if int64(len(p)) > l.N { + p = p[0:l.N] + } + n, err = l.R.Read(p) + l.N -= int64(n) + return +} + +// Close implements io.Closer +func (l *LimitedReaderCloser) Close() error { + return l.C.Close() +} diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go index 6e10ee20527..bc3fbc13d89 100644 --- a/modules/indexer/stats/db.go +++ b/modules/indexer/stats/db.go @@ -7,6 +7,7 @@ package stats import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" ) // DBIndexer implements Indexer interface to use database's like search @@ -37,6 +38,7 @@ func (db *DBIndexer) Index(id int64) error { // Get latest commit for default branch commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch) if err != nil { + log.Error("Unable to get commit ID for defaultbranch %s in %s", repo.DefaultBranch, repo.RepoPath()) return err } @@ -48,6 +50,7 @@ func (db *DBIndexer) Index(id int64) error { // Calculate and save language statistics to database stats, err := gitRepo.GetLanguageStats(commitID) if err != nil { + log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err) return err } return repo.UpdateLanguageStats(commitID, stats) diff --git a/modules/repository/cache.go b/modules/repository/cache.go index 508e5bec0b7..0852771a559 100644 --- a/modules/repository/cache.go +++ b/modules/repository/cache.go @@ -5,57 +5,14 @@ package repository import ( - "path" "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" - - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) -func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *cache.LastCommitCache, level int) error { - if level == 0 { - return nil - } - - entries, err := tree.ListEntries() - if err != nil { - return err - } - - entryPaths := make([]string, len(entries)) - entryMap := make(map[string]*git.TreeEntry) - for i, entry := range entries { - entryPaths[i] = entry.Name() - entryMap[entry.Name()] = entry - } - - commits, err := git.GetLastCommitForPaths(c, treePath, entryPaths) - if err != nil { - return err - } - - for entry, cm := range commits { - if err := ca.Put(c.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil { - return err - } - if entryMap[entry].IsDir() { - subTree, err := tree.SubTree(entry) - if err != nil { - return err - } - if err := recusiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil { - return err - } - } - } - - return nil -} - func getRefName(fullRefName string) string { if strings.HasPrefix(fullRefName, git.TagPrefix) { return fullRefName[len(git.TagPrefix):] @@ -84,14 +41,7 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri return nil } - commitNodeIndex, _ := gitRepo.CommitNodeIndex() - - c, err := commitNodeIndex.Get(commit.ID) - if err != nil { - return err - } - - ca := cache.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds())) + commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) - return recusiveCache(gitRepo, c, &commit.Tree, "", ca, 1) + return commitCache.CacheCommit(commit) } diff --git a/routers/private/hook.go b/routers/private/hook.go index dac39407562..1b9ab000e4b 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -25,7 +25,6 @@ import ( repo_service "code.gitea.io/gitea/services/repository" "gitea.com/macaron/macaron" - "github.com/go-git/go-git/v5/plumbing" ) func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error { @@ -82,7 +81,7 @@ func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error { _ = stdoutReader.Close() _ = stdoutWriter.Close() }() - hash := plumbing.NewHash(sha) + hash := git.MustIDFromString(sha) return git.NewCommand("cat-file", "commit", sha). RunInDirTimeoutEnvFullPipelineFunc(env, -1, repo.Path, diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index be95e56d3b1..c74b088e2e4 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -12,11 +12,9 @@ import ( "io" "io/ioutil" "path" - "sort" "strconv" "strings" "sync" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" @@ -29,9 +27,6 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" "github.com/unknwon/com" ) @@ -363,22 +358,6 @@ func LFSDelete(ctx *context.Context) { ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") } -type lfsResult struct { - Name string - SHA string - Summary string - When time.Time - ParentHashes []plumbing.Hash - BranchName string - FullCommitName string -} - -type lfsResultSlice []*lfsResult - -func (a lfsResultSlice) Len() int { return len(a) } -func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } - // LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha func LFSFileFind(ctx *context.Context) { if !setting.LFS.StartServer { @@ -394,140 +373,27 @@ func LFSFileFind(ctx *context.Context) { sha := ctx.Query("sha") ctx.Data["Title"] = oid ctx.Data["PageIsSettingsLFS"] = true - var hash plumbing.Hash + var hash git.SHA1 if len(sha) == 0 { meta := models.LFSMetaObject{Oid: oid, Size: size} pointer := meta.Pointer() - hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer)) + hash = git.ComputeBlobHash([]byte(pointer)) sha = hash.String() } else { - hash = plumbing.NewHash(sha) + hash = git.MustIDFromString(sha) } ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" ctx.Data["Oid"] = oid ctx.Data["Size"] = size ctx.Data["SHA"] = sha - resultsMap := map[string]*lfsResult{} - results := make([]*lfsResult, 0) - - basePath := ctx.Repo.Repository.RepoPath() - gogitRepo := ctx.Repo.GitRepo.GoGitRepo() - - commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ - Order: gogit.LogOrderCommitterTime, - All: true, - }) - if err != nil { - log.Error("Failed to get GoGit CommitsIter: %v", err) - ctx.ServerError("LFSFind: Iterate Commits", err) - return - } - - err = commitsIter.ForEach(func(gitCommit *object.Commit) error { - tree, err := gitCommit.Tree() - if err != nil { - return err - } - treeWalker := object.NewTreeWalker(tree, true, nil) - defer treeWalker.Close() - for { - name, entry, err := treeWalker.Next() - if err == io.EOF { - break - } - if entry.Hash == hash { - result := lfsResult{ - Name: name, - SHA: gitCommit.Hash.String(), - Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], - When: gitCommit.Author.When, - ParentHashes: gitCommit.ParentHashes, - } - resultsMap[gitCommit.Hash.String()+":"+name] = &result - } - } - return nil - }) + results, err := pipeline.FindLFSFile(ctx.Repo.GitRepo, hash) if err != nil && err != io.EOF { - log.Error("Failure in CommitIter.ForEach: %v", err) - ctx.ServerError("LFSFind: IterateCommits ForEach", err) + log.Error("Failure in FindLFSFile: %v", err) + ctx.ServerError("LFSFind: FindLFSFile.", err) return } - for _, result := range resultsMap { - hasParent := false - for _, parentHash := range result.ParentHashes { - if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { - break - } - } - if !hasParent { - results = append(results, result) - } - } - - sort.Sort(lfsResultSlice(results)) - - // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple - shasToNameReader, shasToNameWriter := io.Pipe() - nameRevStdinReader, nameRevStdinWriter := io.Pipe() - errChan := make(chan error, 1) - wg := sync.WaitGroup{} - wg.Add(3) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(nameRevStdinReader) - i := 0 - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - result := results[i] - result.FullCommitName = line - result.BranchName = strings.Split(line, "~")[0] - i++ - } - }() - go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) - go func() { - defer wg.Done() - defer shasToNameWriter.Close() - for _, result := range results { - i := 0 - if i < len(result.SHA) { - n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) - if err != nil { - errChan <- err - break - } - i += n - } - n := 0 - for n < 1 { - n, err = shasToNameWriter.Write([]byte{'\n'}) - if err != nil { - errChan <- err - break - } - - } - - } - }() - - wg.Wait() - - select { - case err, has := <-errChan: - if has { - ctx.ServerError("LFSPointerFiles", err) - } - default: - } - ctx.Data["Results"] = results ctx.HTML(200, tplSettingsLFSFileFind) } diff --git a/routers/repo/view.go b/routers/repo/view.go index 2df5b30ce8f..7d69ee4cf88 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -137,9 +137,9 @@ func renderDirectory(ctx *context.Context, treeLink string) { } entries.CustomSort(base.NaturalSortLess) - var c git.LastCommitCache + var c *git.LastCommitCache if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { - c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds())) + c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) } var latestCommit *git.Commit diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index 3266a813e38..a99efab0206 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -40,18 +40,19 @@ {{end}} {{range $item := .Files}} - {{$entry := index $item 0}} - {{$commit := index $item 1}} + {{$entry := $item.Entry}} + {{$commit := $item.Commit}} + {{$subModuleFile := $item.SubModuleFile}} {{if $entry.IsSubModule}} {{svg "octicon-file-submodule"}} - {{$refURL := $commit.RefURL AppUrl $.Repository.FullName $.SSHDomain}} + {{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}} {{if $refURL}} - {{$entry.Name}}@{{ShortSha $commit.RefID}} + {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} {{else}} - {{$entry.Name}}@{{ShortSha $commit.RefID}} + {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} {{end}} {{else}} {{if $entry.IsDir}}