diff --git a/go.mod b/go.mod
index de67f582dac..f28b199f0dc 100644
--- a/go.mod
+++ b/go.mod
@@ -85,6 +85,7 @@ require (
github.com/sergi/go-diff v1.0.0
github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
+ github.com/src-d/enry/v2 v2.1.0
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect
github.com/stretchr/testify v1.4.0
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect
diff --git a/go.sum b/go.sum
index ff5c52bc234..30109a24e40 100644
--- a/go.sum
+++ b/go.sum
@@ -508,8 +508,13 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/src-d/enry v1.7.3 h1:jG2fmEaQaURh0qqU/sn82BRzVa6d4EVHJIw6gc98bak=
+github.com/src-d/enry/v2 v2.1.0 h1:z1L8t+B8bh3mmjPkJrgOTnVRpFGmTPJsplHX9wAn6BI=
+github.com/src-d/enry/v2 v2.1.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww=
+github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s=
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -530,6 +535,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/toqueteos/trie v1.0.0 h1:8i6pXxNUXNRAqP246iibb7w/pSFquNTQ+uNfriG7vlk=
+github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM=
github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM=
github.com/tstranex/u2f v1.0.0 h1:HhJkSzDDlVSVIVt7pDJwCHQj67k7A5EeBgPmeD+pVsQ=
@@ -747,6 +754,8 @@ gopkg.in/testfixtures.v2 v2.5.0 h1:N08B7l2GzFQenyYbzqthDnKAA+cmb17iAZhhFxr7JHw=
gopkg.in/testfixtures.v2 v2.5.0/go.mod h1:vyAq+MYCgNpR29qitQdLZhdbLFf4mR/2MFJRFoQZZ2M=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE=
+gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
diff --git a/integrations/testlogger.go b/integrations/testlogger.go
index b2ad257a9b8..eed0bf788d4 100644
--- a/integrations/testlogger.go
+++ b/integrations/testlogger.go
@@ -13,7 +13,6 @@ import (
"strings"
"sync"
"testing"
- "time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
@@ -101,7 +100,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
}
writerCloser.setT(&t)
return func() {
- if err := queue.GetManager().FlushAll(context.Background(), 20*time.Second); err != nil {
+ if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
t.Errorf("Flushing queues failed with error %v", err)
}
_ = writerCloser.Close()
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index 2de2a455686..ce3f77ba4e2 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -186,6 +186,8 @@ var migrations = []Migration{
NewMigration("Add some columns on review for migration", addReviewMigrateInfo),
// v126 -> v127
NewMigration("Fix topic repository count", fixTopicRepositoryCount),
+ // v127 -> v128
+ NewMigration("add repository code language statistics", addLanguageStats),
}
// Migrate database to current version
diff --git a/models/migrations/v127.go b/models/migrations/v127.go
new file mode 100644
index 00000000000..d8f0de4a6e1
--- /dev/null
+++ b/models/migrations/v127.go
@@ -0,0 +1,45 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "fmt"
+
+ "code.gitea.io/gitea/modules/timeutil"
+
+ "xorm.io/xorm"
+)
+
+func addLanguageStats(x *xorm.Engine) error {
+ // LanguageStat see models/repo_language_stats.go
+ type LanguageStat struct {
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
+ CommitID string
+ IsPrimary bool
+ Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
+ Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
+ Color string `xorm:"-"`
+ CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+ }
+
+ type RepoIndexerType int
+
+ // RepoIndexerStatus see models/repo_stats_indexer.go
+ type RepoIndexerStatus struct {
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"INDEX(s)"`
+ CommitSha string `xorm:"VARCHAR(40)"`
+ IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
+ }
+
+ if err := x.Sync2(new(LanguageStat)); err != nil {
+ return fmt.Errorf("Sync2: %v", err)
+ }
+ if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
+ return fmt.Errorf("Sync2: %v", err)
+ }
+ return nil
+}
diff --git a/models/models.go b/models/models.go
index 239a9cf2809..b84a179e377 100644
--- a/models/models.go
+++ b/models/models.go
@@ -116,6 +116,7 @@ func init() {
new(OAuth2AuthorizationCode),
new(OAuth2Grant),
new(Task),
+ new(LanguageStat),
)
gonicNames := []string{"SSL", "UID"}
diff --git a/models/repo.go b/models/repo.go
index 2d0f6fccbdf..4bb5e3eb01e 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -175,8 +175,9 @@ type Repository struct {
*Mirror `xorm:"-"`
Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
- RenderingMetas map[string]string `xorm:"-"`
- Units []*RepoUnit `xorm:"-"`
+ RenderingMetas map[string]string `xorm:"-"`
+ Units []*RepoUnit `xorm:"-"`
+ PrimaryLanguage *LanguageStat `xorm:"-"`
IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"`
ForkID int64 `xorm:"INDEX"`
@@ -185,7 +186,8 @@ type Repository struct {
TemplateID int64 `xorm:"INDEX"`
TemplateRepo *Repository `xorm:"-"`
Size int64 `xorm:"NOT NULL DEFAULT 0"`
- IndexerStatus *RepoIndexerStatus `xorm:"-"`
+ CodeIndexerStatus *RepoIndexerStatus `xorm:"-"`
+ StatsIndexerStatus *RepoIndexerStatus `xorm:"-"`
IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"`
CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"`
Topics []string `xorm:"TEXT JSON"`
@@ -1504,6 +1506,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
&Notification{RepoID: repoID},
&CommitStatus{RepoID: repoID},
&RepoIndexerStatus{RepoID: repoID},
+ &LanguageStat{RepoID: repoID},
&Comment{RefRepoID: repoID},
&Task{RepoID: repoID},
); err != nil {
diff --git a/models/repo_indexer.go b/models/repo_indexer.go
index a9a516175d9..1f5ab928ab7 100644
--- a/models/repo_indexer.go
+++ b/models/repo_indexer.go
@@ -10,21 +10,32 @@ import (
"xorm.io/builder"
)
+// RepoIndexerType specifies the repository indexer type
+type RepoIndexerType int
+
+const (
+ // RepoIndexerTypeCode code indexer
+ RepoIndexerTypeCode RepoIndexerType = iota // 0
+ // RepoIndexerTypeStats repository stats indexer
+ RepoIndexerTypeStats // 1
+)
+
// RepoIndexerStatus status of a repo's entry in the repo indexer
// For now, implicitly refers to default branch
type RepoIndexerStatus struct {
- ID int64 `xorm:"pk autoincr"`
- RepoID int64 `xorm:"INDEX"`
- CommitSha string `xorm:"VARCHAR(40)"`
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"INDEX(s)"`
+ CommitSha string `xorm:"VARCHAR(40)"`
+ IndexerType RepoIndexerType `xorm:"INDEX(s) NOT NULL DEFAULT 0"`
}
// GetUnindexedRepos returns repos which do not have an indexer status
-func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
+func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageSize int) ([]int64, error) {
ids := make([]int64, 0, 50)
cond := builder.Cond(builder.IsNull{
"repo_indexer_status.id",
})
- sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id")
+ sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType)
if maxRepoID > 0 {
cond = builder.And(cond, builder.Lte{
"repository.id": maxRepoID,
@@ -43,40 +54,64 @@ func GetUnindexedRepos(maxRepoID int64, page, pageSize int) ([]int64, error) {
return ids, err
}
-// GetIndexerStatus loads repo codes indxer status
-func (repo *Repository) GetIndexerStatus() error {
- if repo.IndexerStatus != nil {
- return nil
+// getIndexerStatus loads repo codes indxer status
+func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
+ switch indexerType {
+ case RepoIndexerTypeCode:
+ if repo.CodeIndexerStatus != nil {
+ return repo.CodeIndexerStatus, nil
+ }
+ case RepoIndexerTypeStats:
+ if repo.StatsIndexerStatus != nil {
+ return repo.StatsIndexerStatus, nil
+ }
}
- status := &RepoIndexerStatus{RepoID: repo.ID}
- has, err := x.Get(status)
+ status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType}
+ has, err := e.Get(status)
if err != nil {
- return err
+ return nil, err
} else if !has {
status.CommitSha = ""
}
- repo.IndexerStatus = status
- return nil
+ switch indexerType {
+ case RepoIndexerTypeCode:
+ repo.CodeIndexerStatus = status
+ case RepoIndexerTypeStats:
+ repo.StatsIndexerStatus = status
+ }
+ return status, nil
}
-// UpdateIndexerStatus updates indexer status
-func (repo *Repository) UpdateIndexerStatus(sha string) error {
- if err := repo.GetIndexerStatus(); err != nil {
+// GetIndexerStatus loads repo codes indxer status
+func (repo *Repository) GetIndexerStatus(indexerType RepoIndexerType) (*RepoIndexerStatus, error) {
+ return repo.getIndexerStatus(x, indexerType)
+}
+
+// updateIndexerStatus updates indexer status
+func (repo *Repository) updateIndexerStatus(e Engine, indexerType RepoIndexerType, sha string) error {
+ status, err := repo.getIndexerStatus(e, indexerType)
+ if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s Error: %v", repo.FullName(), err)
}
- if len(repo.IndexerStatus.CommitSha) == 0 {
- repo.IndexerStatus.CommitSha = sha
- _, err := x.Insert(repo.IndexerStatus)
+
+ if len(status.CommitSha) == 0 {
+ status.CommitSha = sha
+ _, err := e.Insert(status)
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
}
return nil
}
- repo.IndexerStatus.CommitSha = sha
- _, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
- Update(repo.IndexerStatus)
+ status.CommitSha = sha
+ _, err = e.ID(status.ID).Cols("commit_sha").
+ Update(status)
if err != nil {
return fmt.Errorf("UpdateIndexerStatus: Unable to update repoIndexerStatus for repo: %s Sha: %s Error: %v", repo.FullName(), sha, err)
}
return nil
}
+
+// UpdateIndexerStatus updates indexer status
+func (repo *Repository) UpdateIndexerStatus(indexerType RepoIndexerType, sha string) error {
+ return repo.updateIndexerStatus(x, indexerType, sha)
+}
diff --git a/models/repo_language_stats.go b/models/repo_language_stats.go
new file mode 100644
index 00000000000..4c3171e2926
--- /dev/null
+++ b/models/repo_language_stats.go
@@ -0,0 +1,137 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+ "math"
+ "strings"
+
+ "code.gitea.io/gitea/modules/timeutil"
+
+ "github.com/src-d/enry/v2"
+)
+
+// LanguageStat describes language statistics of a repository
+type LanguageStat struct {
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
+ CommitID string
+ IsPrimary bool
+ Language string `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
+ Percentage float32 `xorm:"NUMERIC(5,2) NOT NULL DEFAULT 0"`
+ Color string `xorm:"-"`
+ CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+}
+
+// LanguageStatList defines a list of language statistics
+type LanguageStatList []*LanguageStat
+
+func (stats LanguageStatList) loadAttributes() {
+ for i := range stats {
+ stats[i].Color = enry.GetColor(stats[i].Language)
+ }
+}
+
+func (repo *Repository) getLanguageStats(e Engine) (LanguageStatList, error) {
+ stats := make(LanguageStatList, 0, 6)
+ if err := e.Where("`repo_id` = ?", repo.ID).Desc("`percentage`").Find(&stats); err != nil {
+ return nil, err
+ }
+ stats.loadAttributes()
+ return stats, nil
+}
+
+// GetLanguageStats returns the language statistics for a repository
+func (repo *Repository) GetLanguageStats() (LanguageStatList, error) {
+ return repo.getLanguageStats(x)
+}
+
+// GetTopLanguageStats returns the top language statistics for a repository
+func (repo *Repository) GetTopLanguageStats(limit int) (LanguageStatList, error) {
+ stats, err := repo.getLanguageStats(x)
+ if err != nil {
+ return nil, err
+ }
+ topstats := make(LanguageStatList, 0, limit)
+ var other float32
+ for i := range stats {
+ if stats[i].Language == "other" || len(topstats) >= limit {
+ other += stats[i].Percentage
+ continue
+ }
+ topstats = append(topstats, stats[i])
+ }
+ if other > 0 {
+ topstats = append(topstats, &LanguageStat{
+ RepoID: repo.ID,
+ Language: "other",
+ Color: "#cccccc",
+ Percentage: float32(math.Round(float64(other)*10) / 10),
+ })
+ }
+ return topstats, nil
+}
+
+// UpdateLanguageStats updates the language statistics for repository
+func (repo *Repository) UpdateLanguageStats(commitID string, stats map[string]float32) error {
+ sess := x.NewSession()
+ if err := sess.Begin(); err != nil {
+ return err
+ }
+ defer sess.Close()
+
+ oldstats, err := repo.getLanguageStats(sess)
+ if err != nil {
+ return err
+ }
+ var topLang string
+ var p float32
+ for lang, perc := range stats {
+ if perc > p {
+ p = perc
+ topLang = strings.ToLower(lang)
+ }
+ }
+
+ for lang, perc := range stats {
+ upd := false
+ llang := strings.ToLower(lang)
+ for _, s := range oldstats {
+ // Update already existing language
+ if strings.ToLower(s.Language) == llang {
+ s.CommitID = commitID
+ s.IsPrimary = llang == topLang
+ s.Percentage = perc
+ if _, err := sess.ID(s.ID).Cols("`commit_id`", "`percentage`", "`is_primary`").Update(s); err != nil {
+ return err
+ }
+ upd = true
+ break
+ }
+ }
+ // Insert new language
+ if !upd {
+ if _, err := sess.Insert(&LanguageStat{
+ RepoID: repo.ID,
+ CommitID: commitID,
+ IsPrimary: llang == topLang,
+ Language: lang,
+ Percentage: perc,
+ }); err != nil {
+ return err
+ }
+ }
+ }
+ // Delete old languages
+ if _, err := sess.Where("`id` IN (SELECT `id` FROM `language_stat` WHERE `repo_id` = ? AND `commit_id` != ?)", repo.ID, commitID).Delete(&LanguageStat{}); err != nil {
+ return err
+ }
+
+ if err = repo.updateIndexerStatus(sess, RepoIndexerTypeStats, commitID); err != nil {
+ return err
+ }
+
+ return sess.Commit()
+}
diff --git a/models/repo_list.go b/models/repo_list.go
index d3a113d26cd..6385de4b326 100644
--- a/models/repo_list.go
+++ b/models/repo_list.go
@@ -46,11 +46,14 @@ func (repos RepositoryList) loadAttributes(e Engine) error {
return nil
}
- // Load owners.
set := make(map[int64]struct{})
+ repoIDs := make([]int64, len(repos))
for i := range repos {
set[repos[i].OwnerID] = struct{}{}
+ repoIDs[i] = repos[i].ID
}
+
+ // Load owners.
users := make(map[int64]*User, len(set))
if err := e.
Where("id > 0").
@@ -61,6 +64,25 @@ func (repos RepositoryList) loadAttributes(e Engine) error {
for i := range repos {
repos[i].Owner = users[repos[i].OwnerID]
}
+
+ // Load primary language.
+ stats := make(LanguageStatList, 0, len(repos))
+ if err := e.
+ Where("`is_primary` = ? AND `language` != ?", true, "other").
+ In("`repo_id`", repoIDs).
+ Find(&stats); err != nil {
+ return fmt.Errorf("find primary languages: %v", err)
+ }
+ stats.loadAttributes()
+ for i := range repos {
+ for _, st := range stats {
+ if st.RepoID == repos[i].ID {
+ repos[i].PrimaryLanguage = st
+ break
+ }
+ }
+ }
+
return nil
}
@@ -119,7 +141,6 @@ type SearchRepoOptions struct {
OrderBy SearchOrderBy
Private bool // Include private repositories in results
StarredByID int64
- IsProfile bool
AllPublic bool // Include also all public repositories of users and public organisations
AllLimited bool // Include also all public repositories of limited organisations
// None -> include collaborative AND non-collaborative
@@ -306,10 +327,8 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
return nil, 0, fmt.Errorf("Repo: %v", err)
}
- if !opts.IsProfile {
- if err = repos.loadAttributes(sess); err != nil {
- return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
- }
+ if err = repos.loadAttributes(sess); err != nil {
+ return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
}
return repos, count, nil
diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go
new file mode 100644
index 00000000000..ffe6dd0848a
--- /dev/null
+++ b/modules/git/repo_language_stats.go
@@ -0,0 +1,116 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "math"
+ "path/filepath"
+
+ "github.com/src-d/enry/v2"
+ "gopkg.in/src-d/go-git.v4"
+ "gopkg.in/src-d/go-git.v4/plumbing"
+ "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+const fileSizeLimit int64 = 16 * 1024 * 1024
+
+// GetLanguageStats calculates language stats for git repository at specified commit
+func (repo *Repository) GetLanguageStats(commitID string) (map[string]float32, error) {
+ r, err := git.PlainOpen(repo.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ rev, err := r.ResolveRevision(plumbing.Revision(commitID))
+ if err != nil {
+ return nil, err
+ }
+
+ commit, err := r.CommitObject(*rev)
+ if err != nil {
+ return nil, err
+ }
+
+ tree, err := commit.Tree()
+ if err != nil {
+ return nil, err
+ }
+
+ sizes := make(map[string]int64)
+ var total int64
+ err = tree.Files().ForEach(func(f *object.File) error {
+ if enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
+ enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
+ return nil
+ }
+
+ // TODO: Use .gitattributes file for linguist overrides
+
+ language, ok := enry.GetLanguageByExtension(f.Name)
+ if !ok {
+ if language, ok = enry.GetLanguageByFilename(f.Name); !ok {
+ content, err := readFile(f, fileSizeLimit)
+ if err != nil {
+ return nil
+ }
+
+ language = enry.GetLanguage(filepath.Base(f.Name), content)
+ if language == enry.OtherLanguage {
+ return nil
+ }
+ }
+ }
+
+ if language != "" {
+ sizes[language] += f.Size
+ total += f.Size
+ }
+
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ stats := make(map[string]float32)
+ var otherPerc float32 = 100
+ for language, size := range sizes {
+ perc := float32(math.Round(float64(size)/float64(total)*1000) / 10)
+ if perc <= 0.1 {
+ continue
+ }
+ otherPerc -= perc
+ stats[language] = perc
+ }
+ otherPerc = float32(math.Round(float64(otherPerc)*10) / 10)
+ if otherPerc > 0 {
+ stats["other"] = otherPerc
+ }
+ return stats, nil
+}
+
+func readFile(f *object.File, limit int64) ([]byte, error) {
+ r, err := f.Reader()
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+
+ if limit <= 0 {
+ return ioutil.ReadAll(r)
+ }
+
+ size := f.Size
+ if limit > 0 && size > limit {
+ size = limit
+ }
+ buf := bytes.NewBuffer(nil)
+ buf.Grow(int(size))
+ _, err = io.Copy(buf, io.LimitReader(r, limit))
+ return buf.Bytes(), err
+}
diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go
index 339dca74a12..6052304f83c 100644
--- a/modules/indexer/code/bleve.go
+++ b/modules/indexer/code/bleve.go
@@ -267,7 +267,7 @@ func (b *BleveIndexer) Index(repoID int64) error {
if err = batch.Flush(); err != nil {
return err
}
- return repo.UpdateIndexerStatus(sha)
+ return repo.UpdateIndexerStatus(models.RepoIndexerTypeCode, sha)
}
// Delete deletes indexes by ids
diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go
index 114d5a9e6d4..37ab5ac3d32 100644
--- a/modules/indexer/code/git.go
+++ b/modules/indexer/code/git.go
@@ -35,11 +35,12 @@ func getDefaultBranchSha(repo *models.Repository) (string, error) {
// getRepoChanges returns changes to repo since last indexer update
func getRepoChanges(repo *models.Repository, revision string) (*repoChanges, error) {
- if err := repo.GetIndexerStatus(); err != nil {
+ status, err := repo.GetIndexerStatus(models.RepoIndexerTypeCode)
+ if err != nil {
return nil, err
}
- if len(repo.IndexerStatus.CommitSha) == 0 {
+ if len(status.CommitSha) == 0 {
return genesisChanges(repo, revision)
}
return nonGenesisChanges(repo, revision)
@@ -98,7 +99,7 @@ func genesisChanges(repo *models.Repository, revision string) (*repoChanges, err
// nonGenesisChanges get changes since the previous indexer update
func nonGenesisChanges(repo *models.Repository, revision string) (*repoChanges, error) {
diffCmd := git.NewCommand("diff", "--name-status",
- repo.IndexerStatus.CommitSha, revision)
+ repo.CodeIndexerStatus.CommitSha, revision)
stdout, err := diffCmd.RunInDir(repo.RepoPath())
if err != nil {
// previous commit sha may have been removed by a force push, so
diff --git a/modules/indexer/code/queue.go b/modules/indexer/code/queue.go
index 4eeb6ac7d40..94675559eaf 100644
--- a/modules/indexer/code/queue.go
+++ b/modules/indexer/code/queue.go
@@ -109,7 +109,7 @@ func populateRepoIndexer() {
return
default:
}
- ids, err := models.GetUnindexedRepos(maxRepoID, 0, 50)
+ ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeCode, maxRepoID, 0, 50)
if err != nil {
log.Error("populateRepoIndexer: %v", err)
return
diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go
new file mode 100644
index 00000000000..fe219b443f0
--- /dev/null
+++ b/modules/indexer/stats/db.go
@@ -0,0 +1,54 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package stats
+
+import (
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/git"
+)
+
+// DBIndexer implements Indexer interface to use database's like search
+type DBIndexer struct {
+}
+
+// Index repository status function
+func (db *DBIndexer) Index(id int64) error {
+ repo, err := models.GetRepositoryByID(id)
+ if err != nil {
+ return err
+ }
+ status, err := repo.GetIndexerStatus(models.RepoIndexerTypeStats)
+ if err != nil {
+ return err
+ }
+
+ gitRepo, err := git.OpenRepository(repo.RepoPath())
+ if err != nil {
+ return err
+ }
+ defer gitRepo.Close()
+
+ // Get latest commit for default branch
+ commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
+ if err != nil {
+ return err
+ }
+
+ // Do not recalculate stats if already calculated for this commit
+ if status.CommitSha == commitID {
+ return nil
+ }
+
+ // Calculate and save language statistics to database
+ stats, err := gitRepo.GetLanguageStats(commitID)
+ if err != nil {
+ return err
+ }
+ return repo.UpdateLanguageStats(commitID, stats)
+}
+
+// Close dummy function
+func (db *DBIndexer) Close() {
+}
diff --git a/modules/indexer/stats/indexer.go b/modules/indexer/stats/indexer.go
new file mode 100644
index 00000000000..4d8a174ff9a
--- /dev/null
+++ b/modules/indexer/stats/indexer.go
@@ -0,0 +1,85 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package stats
+
+import (
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/graceful"
+ "code.gitea.io/gitea/modules/log"
+)
+
+// Indexer defines an interface to index repository stats
+type Indexer interface {
+ Index(id int64) error
+ Close()
+}
+
+// indexer represents a indexer instance
+var indexer Indexer
+
+// Init initialize the repo indexer
+func Init() error {
+ indexer = &DBIndexer{}
+
+ if err := initStatsQueue(); err != nil {
+ return err
+ }
+
+ go populateRepoIndexer()
+
+ return nil
+}
+
+// populateRepoIndexer populate the repo indexer with pre-existing data. This
+// should only be run when the indexer is created for the first time.
+func populateRepoIndexer() {
+ log.Info("Populating the repo stats indexer with existing repositories")
+
+ isShutdown := graceful.GetManager().IsShutdown()
+
+ exist, err := models.IsTableNotEmpty("repository")
+ if err != nil {
+ log.Fatal("System error: %v", err)
+ } else if !exist {
+ return
+ }
+
+ var maxRepoID int64
+ if maxRepoID, err = models.GetMaxID("repository"); err != nil {
+ log.Fatal("System error: %v", err)
+ }
+
+ // start with the maximum existing repo ID and work backwards, so that we
+ // don't include repos that are created after gitea starts; such repos will
+ // already be added to the indexer, and we don't need to add them again.
+ for maxRepoID > 0 {
+ select {
+ case <-isShutdown:
+ log.Info("Repository Stats Indexer population shutdown before completion")
+ return
+ default:
+ }
+ ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeStats, maxRepoID, 0, 50)
+ if err != nil {
+ log.Error("populateRepoIndexer: %v", err)
+ return
+ } else if len(ids) == 0 {
+ break
+ }
+ for _, id := range ids {
+ select {
+ case <-isShutdown:
+ log.Info("Repository Stats Indexer population shutdown before completion")
+ return
+ default:
+ }
+ if err := statsQueue.Push(id); err != nil {
+ log.Error("statsQueue.Push: %v", err)
+ }
+ maxRepoID = id - 1
+ }
+ }
+ log.Info("Done (re)populating the repo stats indexer with existing repositories")
+}
diff --git a/modules/indexer/stats/indexer_test.go b/modules/indexer/stats/indexer_test.go
new file mode 100644
index 00000000000..29d0f6dbe4a
--- /dev/null
+++ b/modules/indexer/stats/indexer_test.go
@@ -0,0 +1,42 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package stats
+
+import (
+ "path/filepath"
+ "testing"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/setting"
+
+ "gopkg.in/ini.v1"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMain(m *testing.M) {
+ models.MainTest(m, filepath.Join("..", "..", ".."))
+}
+
+func TestRepoStatsIndex(t *testing.T) {
+ assert.NoError(t, models.PrepareTestDatabase())
+ setting.Cfg = ini.Empty()
+
+ setting.NewQueueService()
+
+ err := Init()
+ assert.NoError(t, err)
+
+ time.Sleep(5 * time.Second)
+
+ repo, err := models.GetRepositoryByID(1)
+ assert.NoError(t, err)
+ langs, err := repo.GetTopLanguageStats(5)
+ assert.NoError(t, err)
+ assert.Len(t, langs, 1)
+ assert.Equal(t, "other", langs[0].Language)
+ assert.Equal(t, float32(100), langs[0].Percentage)
+}
diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go
new file mode 100644
index 00000000000..43a4de5ac90
--- /dev/null
+++ b/modules/indexer/stats/queue.go
@@ -0,0 +1,43 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package stats
+
+import (
+ "fmt"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/graceful"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/queue"
+)
+
+// statsQueue represents a queue to handle repository stats updates
+var statsQueue queue.Queue
+
+// handle passed PR IDs and test the PRs
+func handle(data ...queue.Data) {
+ for _, datum := range data {
+ opts := datum.(int64)
+ if err := indexer.Index(opts); err != nil {
+ log.Error("stats queue idexer.Index(%d) failed: %v", opts, err)
+ }
+ }
+}
+
+func initStatsQueue() error {
+ statsQueue = queue.CreateQueue("repo_stats_update", handle, int64(0)).(queue.Queue)
+ if statsQueue == nil {
+ return fmt.Errorf("Unable to create repo_stats_update Queue")
+ }
+
+ go graceful.GetManager().RunWithShutdownFns(statsQueue.Run)
+
+ return nil
+}
+
+// UpdateRepoIndexer update a repository's entries in the indexer
+func UpdateRepoIndexer(repo *models.Repository) error {
+ return statsQueue.Push(repo.ID)
+}
diff --git a/modules/notification/indexer/indexer.go b/modules/notification/indexer/indexer.go
index 4bce9907397..6caae6fa65a 100644
--- a/modules/notification/indexer/indexer.go
+++ b/modules/notification/indexer/indexer.go
@@ -9,6 +9,7 @@ import (
"code.gitea.io/gitea/modules/git"
code_indexer "code.gitea.io/gitea/modules/indexer/code"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
+ stats_indexer "code.gitea.io/gitea/modules/indexer/stats"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification/base"
"code.gitea.io/gitea/modules/repository"
@@ -117,12 +118,18 @@ func (r *indexerNotifier) NotifyMigrateRepository(doer *models.User, u *models.U
if setting.Indexer.RepoIndexerEnabled && !repo.IsEmpty {
code_indexer.UpdateRepoIndexer(repo)
}
+ if err := stats_indexer.UpdateRepoIndexer(repo); err != nil {
+ log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err)
+ }
}
func (r *indexerNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) {
if setting.Indexer.RepoIndexerEnabled && refName == git.BranchPrefix+repo.DefaultBranch {
code_indexer.UpdateRepoIndexer(repo)
}
+ if err := stats_indexer.UpdateRepoIndexer(repo); err != nil {
+ log.Error("stats_indexer.UpdateRepoIndexer(%d) failed: %v", repo.ID, err)
+ }
}
func (r *indexerNotifier) NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent string) {
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 8cd76ad2be0..5b4d491bd78 100644
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -641,6 +641,7 @@ forks = Forks
pick_reaction = Pick your reaction
reactions_more = and %d more
unit_disabled = The site administrator has disabled this repository section.
+language_other = Other
template.items = Template Items
template.git_content = Git Content (Default Branch)
diff --git a/routers/init.go b/routers/init.go
index f86a7ad4b2d..724bf84c10f 100644
--- a/routers/init.go
+++ b/routers/init.go
@@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/highlight"
code_indexer "code.gitea.io/gitea/modules/indexer/code"
issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
+ stats_indexer "code.gitea.io/gitea/modules/indexer/stats"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/external"
@@ -111,6 +112,9 @@ func GlobalInit(ctx context.Context) {
cron.NewContext()
issue_indexer.InitIssueIndexer(false)
code_indexer.Init()
+ if err := stats_indexer.Init(); err != nil {
+ log.Fatal("Failed to initialize repository stats indexer queue: %v", err)
+ }
mirror_service.InitSyncMirrors()
webhook.InitDeliverHooks()
if err := pull_service.Init(); err != nil {
diff --git a/routers/org/home.go b/routers/org/home.go
index e1bea5b7a6a..fa61218d3fd 100644
--- a/routers/org/home.go
+++ b/routers/org/home.go
@@ -85,7 +85,6 @@ func Home(ctx *context.Context) {
OrderBy: orderBy,
Private: ctx.IsSigned,
Actor: ctx.User,
- IsProfile: true,
IncludeDescription: setting.UI.SearchRepoDescription,
})
if err != nil {
diff --git a/routers/repo/view.go b/routers/repo/view.go
index f56c5243598..9183aea030b 100644
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -457,6 +457,16 @@ func Home(ctx *context.Context) {
ctx.NotFound("Home", fmt.Errorf(ctx.Tr("units.error.no_unit_allowed_repo")))
}
+func renderLanguageStats(ctx *context.Context) {
+ langs, err := ctx.Repo.Repository.GetTopLanguageStats(5)
+ if err != nil {
+ ctx.ServerError("Repo.GetTopLanguageStats", err)
+ return
+ }
+
+ ctx.Data["LanguageStats"] = langs
+}
+
func renderCode(ctx *context.Context) {
ctx.Data["PageIsViewCode"] = true
@@ -497,6 +507,11 @@ func renderCode(ctx *context.Context) {
return
}
+ renderLanguageStats(ctx)
+ if ctx.Written() {
+ return
+ }
+
if entry.IsDir() {
renderDirectory(ctx, treeLink)
} else {
diff --git a/routers/user/profile.go b/routers/user/profile.go
index a151884d768..215dff0084b 100644
--- a/routers/user/profile.go
+++ b/routers/user/profile.go
@@ -220,7 +220,6 @@ func Profile(ctx *context.Context) {
OwnerID: ctxUser.ID,
OrderBy: orderBy,
Private: ctx.IsSigned,
- IsProfile: true,
Collaborate: util.OptionalBoolFalse,
TopicOnly: topicOnly,
IncludeDescription: setting.UI.SearchRepoDescription,
diff --git a/templates/explore/repo_list.tmpl b/templates/explore/repo_list.tmpl
index 8c7ba51a540..fec304cc917 100644
--- a/templates/explore/repo_list.tmpl
+++ b/templates/explore/repo_list.tmpl
@@ -21,6 +21,9 @@
{{end}}
{{end}}
+ {{if .PrimaryLanguage }}
+ {{ .PrimaryLanguage.Language }}
+ {{end}}
{{.NumStars}}
{{.NumForks}}
diff --git a/templates/repo/sub_menu.tmpl b/templates/repo/sub_menu.tmpl
index b97fe902eb6..96128fb2af0 100644
--- a/templates/repo/sub_menu.tmpl
+++ b/templates/repo/sub_menu.tmpl
@@ -1,17 +1,42 @@
-