diff --git a/conf/app.ini b/conf/app.ini index 47fd4b1182a..4f7dc9946b8 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -442,6 +442,16 @@ SCHEDULE = @every 24h ; Archives created more than OLDER_THAN ago are subject to deletion OLDER_THAN = 24h +; Synchronize external user data (only LDAP user synchronization is supported) +[cron.sync_external_users] +; Syncronize external user data when starting server (default false) +RUN_AT_START = false +; Interval as a duration between each synchronization (default every 24h) +SCHEDULE = @every 24h +; Create new users, update existing user data and disable users that are not in external source anymore (default) +; or only create new users if UPDATE_EXISTING is set to false +UPDATE_EXISTING = true + [git] ; Disables highlight of added and removed changes DISABLE_DIFF_HIGHLIGHT = false diff --git a/models/login_source.go b/models/login_source.go index 3c7bff8cb87..60110708cb6 100644 --- a/models/login_source.go +++ b/models/login_source.go @@ -140,11 +140,12 @@ func (cfg *OAuth2Config) ToDB() ([]byte, error) { // LoginSource represents an external way for authorizing users. type LoginSource struct { - ID int64 `xorm:"pk autoincr"` - Type LoginType - Name string `xorm:"UNIQUE"` - IsActived bool `xorm:"INDEX NOT NULL DEFAULT false"` - Cfg core.Conversion `xorm:"TEXT"` + ID int64 `xorm:"pk autoincr"` + Type LoginType + Name string `xorm:"UNIQUE"` + IsActived bool `xorm:"INDEX NOT NULL DEFAULT false"` + IsSyncEnabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + Cfg core.Conversion `xorm:"TEXT"` Created time.Time `xorm:"-"` CreatedUnix int64 `xorm:"INDEX"` @@ -294,6 +295,10 @@ func CreateLoginSource(source *LoginSource) error { } else if has { return ErrLoginSourceAlreadyExist{source.Name} } + // Synchronization is only aviable with LDAP for now + if !source.IsLDAP() { + source.IsSyncEnabled = false + } _, err = x.Insert(source) if err == nil && source.IsOAuth2() && source.IsActived { @@ -405,8 +410,8 @@ func composeFullName(firstname, surname, username string) string { // LoginViaLDAP queries if login/password is valid against the LDAP directory pool, // and create a local user if success when enabled. func LoginViaLDAP(user *User, login, password string, source *LoginSource, autoRegister bool) (*User, error) { - username, fn, sn, mail, isAdmin, succeed := source.Cfg.(*LDAPConfig).SearchEntry(login, password, source.Type == LoginDLDAP) - if !succeed { + sr := source.Cfg.(*LDAPConfig).SearchEntry(login, password, source.Type == LoginDLDAP) + if sr == nil { // User not in LDAP, do nothing return nil, ErrUserNotExist{0, login, 0} } @@ -416,28 +421,28 @@ func LoginViaLDAP(user *User, login, password string, source *LoginSource, autoR } // Fallback. - if len(username) == 0 { - username = login + if len(sr.Username) == 0 { + sr.Username = login } // Validate username make sure it satisfies requirement. - if binding.AlphaDashDotPattern.MatchString(username) { - return nil, fmt.Errorf("Invalid pattern for attribute 'username' [%s]: must be valid alpha or numeric or dash(-_) or dot characters", username) + if binding.AlphaDashDotPattern.MatchString(sr.Username) { + return nil, fmt.Errorf("Invalid pattern for attribute 'username' [%s]: must be valid alpha or numeric or dash(-_) or dot characters", sr.Username) } - if len(mail) == 0 { - mail = fmt.Sprintf("%s@localhost", username) + if len(sr.Mail) == 0 { + sr.Mail = fmt.Sprintf("%s@localhost", sr.Username) } user = &User{ - LowerName: strings.ToLower(username), - Name: username, - FullName: composeFullName(fn, sn, username), - Email: mail, + LowerName: strings.ToLower(sr.Username), + Name: sr.Username, + FullName: composeFullName(sr.Name, sr.Surname, sr.Username), + Email: sr.Mail, LoginType: source.Type, LoginSource: source.ID, LoginName: login, IsActive: true, - IsAdmin: isAdmin, + IsAdmin: sr.IsAdmin, } return user, CreateUser(user) } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 4877a9fb02d..000412ae37f 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -110,6 +110,8 @@ var migrations = []Migration{ NewMigration("add commit status table", addCommitStatus), // v30 -> 31 NewMigration("add primary key to external login user", addExternalLoginUserPK), + // 31 -> 32 + NewMigration("add field for login source synchronization", addLoginSourceSyncEnabledColumn), } // Migrate database to current version diff --git a/models/migrations/v31.go b/models/migrations/v31.go new file mode 100644 index 00000000000..1166a5f6c4f --- /dev/null +++ b/models/migrations/v31.go @@ -0,0 +1,35 @@ +// Copyright 2017 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "time" + + "github.com/go-xorm/core" + "github.com/go-xorm/xorm" +) + +func addLoginSourceSyncEnabledColumn(x *xorm.Engine) error { + // LoginSource see models/login_source.go + type LoginSource struct { + ID int64 `xorm:"pk autoincr"` + Type int + Name string `xorm:"UNIQUE"` + IsActived bool `xorm:"INDEX NOT NULL DEFAULT false"` + IsSyncEnabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + Cfg core.Conversion `xorm:"TEXT"` + + Created time.Time `xorm:"-"` + CreatedUnix int64 `xorm:"INDEX"` + Updated time.Time `xorm:"-"` + UpdatedUnix int64 `xorm:"INDEX"` + } + + if err := x.Sync2(new(LoginSource)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + return nil +} diff --git a/models/user.go b/models/user.go index e95bf5cd44f..7e6bbd5dc30 100644 --- a/models/user.go +++ b/models/user.go @@ -50,6 +50,8 @@ const ( UserTypeOrganization ) +const syncExternalUsers = "sync_external_users" + var ( // ErrUserNotKeyOwner user does not own this key error ErrUserNotKeyOwner = errors.New("User does not own this public key") @@ -1322,3 +1324,128 @@ func GetWatchedRepos(userID int64, private bool) ([]*Repository, error) { } return repos, nil } + +// SyncExternalUsers is used to synchronize users with external authorization source +func SyncExternalUsers() { + if taskStatusTable.IsRunning(syncExternalUsers) { + return + } + taskStatusTable.Start(syncExternalUsers) + defer taskStatusTable.Stop(syncExternalUsers) + + log.Trace("Doing: SyncExternalUsers") + + ls, err := LoginSources() + if err != nil { + log.Error(4, "SyncExternalUsers: %v", err) + return + } + + updateExisting := setting.Cron.SyncExternalUsers.UpdateExisting + + for _, s := range ls { + if !s.IsActived || !s.IsSyncEnabled { + continue + } + if s.IsLDAP() { + log.Trace("Doing: SyncExternalUsers[%s]", s.Name) + + var existingUsers []int64 + + // Find all users with this login type + var users []User + x.Where("login_type = ?", LoginLDAP). + And("login_source = ?", s.ID). + Find(&users) + + sr := s.LDAP().SearchEntries() + + for _, su := range sr { + if len(su.Username) == 0 { + continue + } + + if len(su.Mail) == 0 { + su.Mail = fmt.Sprintf("%s@localhost", su.Username) + } + + var usr *User + // Search for existing user + for _, du := range users { + if du.LowerName == strings.ToLower(su.Username) { + usr = &du + break + } + } + + fullName := composeFullName(su.Name, su.Surname, su.Username) + // If no existing user found, create one + if usr == nil { + log.Trace("SyncExternalUsers[%s]: Creating user %s", s.Name, su.Username) + + usr = &User{ + LowerName: strings.ToLower(su.Username), + Name: su.Username, + FullName: fullName, + LoginType: s.Type, + LoginSource: s.ID, + LoginName: su.Username, + Email: su.Mail, + IsAdmin: su.IsAdmin, + IsActive: true, + } + + err = CreateUser(usr) + if err != nil { + log.Error(4, "SyncExternalUsers[%s]: Error creating user %s: %v", s.Name, su.Username, err) + } + } else if updateExisting { + existingUsers = append(existingUsers, usr.ID) + // Check if user data has changed + if (len(s.LDAP().AdminFilter) > 0 && usr.IsAdmin != su.IsAdmin) || + strings.ToLower(usr.Email) != strings.ToLower(su.Mail) || + usr.FullName != fullName || + !usr.IsActive { + + log.Trace("SyncExternalUsers[%s]: Updating user %s", s.Name, usr.Name) + + usr.FullName = fullName + usr.Email = su.Mail + // Change existing admin flag only if AdminFilter option is set + if len(s.LDAP().AdminFilter) > 0 { + usr.IsAdmin = su.IsAdmin + } + usr.IsActive = true + + err = UpdateUser(usr) + if err != nil { + log.Error(4, "SyncExternalUsers[%s]: Error updating user %s: %v", s.Name, usr.Name, err) + } + } + } + } + + // Deactivate users not present in LDAP + if updateExisting { + for _, usr := range users { + found := false + for _, uid := range existingUsers { + if usr.ID == uid { + found = true + break + } + } + if !found { + log.Trace("SyncExternalUsers[%s]: Deactivating user %s", s.Name, usr.Name) + + usr.IsActive = false + err = UpdateUser(&usr) + if err != nil { + log.Error(4, "SyncExternalUsers[%s]: Error deactivating user %s: %v", s.Name, usr.Name, err) + } + } + } + } + } + } +} diff --git a/modules/auth/auth_form.go b/modules/auth/auth_form.go index 8dc039835fd..7c452bbc353 100644 --- a/modules/auth/auth_form.go +++ b/modules/auth/auth_form.go @@ -28,6 +28,7 @@ type AuthenticationForm struct { Filter string AdminFilter string IsActive bool + IsSyncEnabled bool SMTPAuth string SMTPHost string SMTPPort int diff --git a/modules/auth/ldap/ldap.go b/modules/auth/ldap/ldap.go index 3064b319588..7754cc81826 100644 --- a/modules/auth/ldap/ldap.go +++ b/modules/auth/ldap/ldap.go @@ -47,6 +47,15 @@ type Source struct { Enabled bool // if this source is disabled } +// SearchResult : user data +type SearchResult struct { + Username string // Username + Name string // Name + Surname string // Surname + Mail string // E-mail address + IsAdmin bool // if user is administrator +} + func (ls *Source) sanitizedUserQuery(username string) (string, bool) { // See http://tools.ietf.org/search/rfc4515 badCharacters := "\x00()*\\" @@ -149,18 +158,39 @@ func bindUser(l *ldap.Conn, userDN, passwd string) error { return err } +func checkAdmin(l *ldap.Conn, ls *Source, userDN string) bool { + if len(ls.AdminFilter) > 0 { + log.Trace("Checking admin with filter %s and base %s", ls.AdminFilter, userDN) + search := ldap.NewSearchRequest( + userDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, ls.AdminFilter, + []string{ls.AttributeName}, + nil) + + sr, err := l.Search(search) + + if err != nil { + log.Error(4, "LDAP Admin Search failed unexpectedly! (%v)", err) + } else if len(sr.Entries) < 1 { + log.Error(4, "LDAP Admin Search failed") + } else { + return true + } + } + return false +} + // SearchEntry : search an LDAP source if an entry (name, passwd) is valid and in the specific filter -func (ls *Source) SearchEntry(name, passwd string, directBind bool) (string, string, string, string, bool, bool) { +func (ls *Source) SearchEntry(name, passwd string, directBind bool) *SearchResult { // See https://tools.ietf.org/search/rfc4513#section-5.1.2 if len(passwd) == 0 { log.Debug("Auth. failed for %s, password cannot be empty") - return "", "", "", "", false, false + return nil } l, err := dial(ls) if err != nil { log.Error(4, "LDAP Connect error, %s:%v", ls.Host, err) ls.Enabled = false - return "", "", "", "", false, false + return nil } defer l.Close() @@ -171,7 +201,7 @@ func (ls *Source) SearchEntry(name, passwd string, directBind bool) (string, str var ok bool userDN, ok = ls.sanitizedUserDN(name) if !ok { - return "", "", "", "", false, false + return nil } } else { log.Trace("LDAP will use BindDN.") @@ -179,7 +209,7 @@ func (ls *Source) SearchEntry(name, passwd string, directBind bool) (string, str var found bool userDN, found = ls.findUserDN(l, name) if !found { - return "", "", "", "", false, false + return nil } } @@ -187,13 +217,13 @@ func (ls *Source) SearchEntry(name, passwd string, directBind bool) (string, str // binds user (checking password) before looking-up attributes in user context err = bindUser(l, userDN, passwd) if err != nil { - return "", "", "", "", false, false + return nil } } userFilter, ok := ls.sanitizedUserQuery(name) if !ok { - return "", "", "", "", false, false + return nil } log.Trace("Fetching attributes '%v', '%v', '%v', '%v' with filter %s and base %s", ls.AttributeUsername, ls.AttributeName, ls.AttributeSurname, ls.AttributeMail, userFilter, userDN) @@ -205,7 +235,7 @@ func (ls *Source) SearchEntry(name, passwd string, directBind bool) (string, str sr, err := l.Search(search) if err != nil { log.Error(4, "LDAP Search failed unexpectedly! (%v)", err) - return "", "", "", "", false, false + return nil } else if len(sr.Entries) < 1 { if directBind { log.Error(4, "User filter inhibited user login.") @@ -213,39 +243,78 @@ func (ls *Source) SearchEntry(name, passwd string, directBind bool) (string, str log.Error(4, "LDAP Search failed unexpectedly! (0 entries)") } - return "", "", "", "", false, false + return nil } username := sr.Entries[0].GetAttributeValue(ls.AttributeUsername) firstname := sr.Entries[0].GetAttributeValue(ls.AttributeName) surname := sr.Entries[0].GetAttributeValue(ls.AttributeSurname) mail := sr.Entries[0].GetAttributeValue(ls.AttributeMail) + isAdmin := checkAdmin(l, ls, userDN) - isAdmin := false - if len(ls.AdminFilter) > 0 { - log.Trace("Checking admin with filter %s and base %s", ls.AdminFilter, userDN) - search = ldap.NewSearchRequest( - userDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, ls.AdminFilter, - []string{ls.AttributeName}, - nil) - - sr, err = l.Search(search) + if !directBind && ls.AttributesInBind { + // binds user (checking password) after looking-up attributes in BindDN context + err = bindUser(l, userDN, passwd) if err != nil { - log.Error(4, "LDAP Admin Search failed unexpectedly! (%v)", err) - } else if len(sr.Entries) < 1 { - log.Error(4, "LDAP Admin Search failed") - } else { - isAdmin = true + return nil } } - if !directBind && ls.AttributesInBind { - // binds user (checking password) after looking-up attributes in BindDN context - err = bindUser(l, userDN, passwd) + return &SearchResult{ + Username: username, + Name: firstname, + Surname: surname, + Mail: mail, + IsAdmin: isAdmin, + } +} + +// SearchEntries : search an LDAP source for all users matching userFilter +func (ls *Source) SearchEntries() []*SearchResult { + l, err := dial(ls) + if err != nil { + log.Error(4, "LDAP Connect error, %s:%v", ls.Host, err) + ls.Enabled = false + return nil + } + defer l.Close() + + if ls.BindDN != "" && ls.BindPassword != "" { + err := l.Bind(ls.BindDN, ls.BindPassword) if err != nil { - return "", "", "", "", false, false + log.Debug("Failed to bind as BindDN[%s]: %v", ls.BindDN, err) + return nil + } + log.Trace("Bound as BindDN %s", ls.BindDN) + } else { + log.Trace("Proceeding with anonymous LDAP search.") + } + + userFilter := fmt.Sprintf(ls.Filter, "*") + + log.Trace("Fetching attributes '%v', '%v', '%v', '%v' with filter %s and base %s", ls.AttributeUsername, ls.AttributeName, ls.AttributeSurname, ls.AttributeMail, userFilter, ls.UserBase) + search := ldap.NewSearchRequest( + ls.UserBase, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, userFilter, + []string{ls.AttributeUsername, ls.AttributeName, ls.AttributeSurname, ls.AttributeMail}, + nil) + + sr, err := l.Search(search) + if err != nil { + log.Error(4, "LDAP Search failed unexpectedly! (%v)", err) + return nil + } + + result := make([]*SearchResult, len(sr.Entries)) + + for i, v := range sr.Entries { + result[i] = &SearchResult{ + Username: v.GetAttributeValue(ls.AttributeUsername), + Name: v.GetAttributeValue(ls.AttributeName), + Surname: v.GetAttributeValue(ls.AttributeSurname), + Mail: v.GetAttributeValue(ls.AttributeMail), + IsAdmin: checkAdmin(l, ls, v.DN), } } - return username, firstname, surname, mail, isAdmin, true + return result } diff --git a/modules/cron/cron.go b/modules/cron/cron.go index 785bf44ada0..a64b51253cc 100644 --- a/modules/cron/cron.go +++ b/modules/cron/cron.go @@ -66,6 +66,17 @@ func NewContext() { go models.DeleteOldRepositoryArchives() } } + if setting.Cron.SyncExternalUsers.Enabled { + entry, err = c.AddFunc("Synchronize external users", setting.Cron.SyncExternalUsers.Schedule, models.SyncExternalUsers) + if err != nil { + log.Fatal(4, "Cron[Synchronize external users]: %v", err) + } + if setting.Cron.SyncExternalUsers.RunAtStart { + entry.Prev = time.Now() + entry.ExecTimes++ + go models.SyncExternalUsers() + } + } c.Start() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c3ed4ef9717..4acad423933 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -336,6 +336,12 @@ var ( Schedule string OlderThan time.Duration } `ini:"cron.archive_cleanup"` + SyncExternalUsers struct { + Enabled bool + RunAtStart bool + Schedule string + UpdateExisting bool + } `ini:"cron.sync_external_users"` }{ UpdateMirror: struct { Enabled bool @@ -379,6 +385,17 @@ var ( Schedule: "@every 24h", OlderThan: 24 * time.Hour, }, + SyncExternalUsers: struct { + Enabled bool + RunAtStart bool + Schedule string + UpdateExisting bool + }{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 24h", + UpdateExisting: true, + }, } // Git settings diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index fadb90a9e36..cd24ec93490 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1065,7 +1065,8 @@ dashboard.resync_all_hooks = Resync pre-receive, update and post-receive hooks o dashboard.resync_all_hooks_success = All repositories' pre-receive, update and post-receive hooks have been resynced successfully. dashboard.reinit_missing_repos = Reinitialize all lost Git repositories for which records exist dashboard.reinit_missing_repos_success = All lost Git repositories for which records existed have been reinitialized successfully. - +dashboard.sync_external_users = Synchronize external user data +dashboard.sync_external_users_started = External user synchronization started dashboard.server_uptime = Server Uptime dashboard.current_goroutine = Current Goroutines dashboard.current_memory_usage = Current Memory Usage @@ -1147,6 +1148,7 @@ auths.new = Add New Source auths.name = Name auths.type = Type auths.enabled = Enabled +auths.syncenabled = Enable user synchronization auths.updated = Updated auths.auth_type = Authentication Type auths.auth_name = Authentication Name diff --git a/routers/admin/admin.go b/routers/admin/admin.go index 6b5b33f734d..8ae45048476 100644 --- a/routers/admin/admin.go +++ b/routers/admin/admin.go @@ -121,6 +121,7 @@ const ( syncSSHAuthorizedKey syncRepositoryUpdateHook reinitMissingRepository + syncExternalUsers ) // Dashboard show admin panel dashboard @@ -157,6 +158,9 @@ func Dashboard(ctx *context.Context) { case reinitMissingRepository: success = ctx.Tr("admin.dashboard.reinit_missing_repos_success") err = models.ReinitMissingRepositories() + case syncExternalUsers: + success = ctx.Tr("admin.dashboard.sync_external_users_started") + go models.SyncExternalUsers() } if err != nil { diff --git a/routers/admin/auths.go b/routers/admin/auths.go index eb7c7e8e939..590e45a4f41 100644 --- a/routers/admin/auths.go +++ b/routers/admin/auths.go @@ -74,6 +74,7 @@ func NewAuthSource(ctx *context.Context) { ctx.Data["CurrentSecurityProtocol"] = models.SecurityProtocolNames[ldap.SecurityProtocolUnencrypted] ctx.Data["smtp_auth"] = "PLAIN" ctx.Data["is_active"] = true + ctx.Data["is_sync_enabled"] = true ctx.Data["AuthSources"] = authSources ctx.Data["SecurityProtocols"] = securityProtocols ctx.Data["SMTPAuths"] = models.SMTPAuths @@ -186,10 +187,11 @@ func NewAuthSourcePost(ctx *context.Context, form auth.AuthenticationForm) { } if err := models.CreateLoginSource(&models.LoginSource{ - Type: models.LoginType(form.Type), - Name: form.Name, - IsActived: form.IsActive, - Cfg: config, + Type: models.LoginType(form.Type), + Name: form.Name, + IsActived: form.IsActive, + IsSyncEnabled: form.IsSyncEnabled, + Cfg: config, }); err != nil { if models.IsErrLoginSourceAlreadyExist(err) { ctx.Data["Err_Name"] = true @@ -273,6 +275,7 @@ func EditAuthSourcePost(ctx *context.Context, form auth.AuthenticationForm) { source.Name = form.Name source.IsActived = form.IsActive + source.IsSyncEnabled = form.IsSyncEnabled source.Cfg = config if err := models.UpdateSource(source); err != nil { if models.IsErrOpenIDConnectInitialize(err) { diff --git a/templates/admin/auth/edit.tmpl b/templates/admin/auth/edit.tmpl index 3c74b2ad175..e3048b21836 100644 --- a/templates/admin/auth/edit.tmpl +++ b/templates/admin/auth/edit.tmpl @@ -211,6 +211,14 @@ + {{if .Source.IsLDAP}} +
+
+ + +
+
+ {{end}}
diff --git a/templates/admin/auth/new.tmpl b/templates/admin/auth/new.tmpl index 00239b0462c..46db82c3a7d 100644 --- a/templates/admin/auth/new.tmpl +++ b/templates/admin/auth/new.tmpl @@ -61,6 +61,12 @@
+
+
+ + +
+
diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index 229cd305b93..23fc4a422d8 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -45,6 +45,10 @@ {{.i18n.Tr "admin.dashboard.reinit_missing_repos"}} {{.i18n.Tr "admin.dashboard.operation_run"}} + + {{.i18n.Tr "admin.dashboard.sync_external_users"}} + {{.i18n.Tr "admin.dashboard.operation_run"}} +