From f83db078f0603c775cd1b1bb016f996b65a04835 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 24 Aug 2019 17:24:45 +0800 Subject: [PATCH] Move database settings from models to setting (#7806) * move database settings from models to setting * update docs * fix checkout pr * fix tests * fix lint * remove unsupported tidb options * correct wrong variable name * remove tidb totally --- cmd/cmd.go | 2 +- cmd/convert.go | 4 +- cmd/dump.go | 5 +- cmd/migrate.go | 2 +- contrib/pr/checkout.go | 10 +- custom/conf/app.ini.sample | 4 + .../doc/advanced/config-cheat-sheet.en-us.md | 2 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 2 + integrations/integration_test.go | 26 +-- integrations/lfs_getobject_test.go | 2 +- integrations/migration-test/migration_test.go | 48 ++--- models/convert.go | 8 +- models/migrations/migrations.go | 8 +- models/migrations/v19.go | 2 +- models/migrations/v22.go | 2 +- models/migrations/v26.go | 2 +- models/migrations/v27.go | 2 - models/migrations/v33.go | 4 +- models/migrations/v45.go | 4 +- models/migrations/v50.go | 4 +- models/migrations/v67.go | 4 +- models/migrations/v81.go | 2 - models/models.go | 148 ++------------- models/models_test.go | 91 +--------- models/repo.go | 6 +- models/unit_tests.go | 2 +- models/user.go | 5 +- models/user_heatmap.go | 8 +- modules/setting/database.go | 171 ++++++++++++++++++ .../setting/database_sqlite.go | 4 +- modules/setting/database_test.go | 94 ++++++++++ modules/setting/setting.go | 38 ++-- routers/admin/admin.go | 4 +- routers/init.go | 19 +- routers/install.go | 62 +++---- 35 files changed, 424 insertions(+), 377 deletions(-) create mode 100644 modules/setting/database.go rename models/models_sqlite.go => modules/setting/database_sqlite.go (77%) create mode 100644 modules/setting/database_test.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 5a55ac318ca..d05eb8b1a2c 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -38,7 +38,7 @@ func initDB() error { func initDBDisableConsole(disableConsole bool) error { setting.NewContext() - models.LoadConfigs() + setting.InitDBConfig() setting.NewXORMLogService(disableConsole) if err := models.SetEngine(); err != nil { diff --git a/cmd/convert.go b/cmd/convert.go index cb0510c722f..23a3d8dbe9d 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -31,9 +31,9 @@ func runConvert(ctx *cli.Context) error { log.Trace("AppWorkPath: %s", setting.AppWorkPath) log.Trace("Custom path: %s", setting.CustomPath) log.Trace("Log path: %s", setting.LogRootPath) - models.LoadConfigs() + setting.InitDBConfig() - if models.DbCfg.Type != "mysql" { + if !setting.Database.UseMySQL { fmt.Println("This command can only be used with a MySQL database") return nil } diff --git a/cmd/dump.go b/cmd/dump.go index de289270d46..1bf6901769f 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -58,7 +58,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`, func runDump(ctx *cli.Context) error { setting.NewContext() setting.NewServices() // cannot access session settings otherwise - models.LoadConfigs() err := models.SetEngine() if err != nil { @@ -104,8 +103,8 @@ func runDump(ctx *cli.Context) error { } targetDBType := ctx.String("database") - if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type { - log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) + if len(targetDBType) > 0 && targetDBType != setting.Database.Type { + log.Printf("Dumping database %s => %s...", setting.Database.Type, targetDBType) } else { log.Printf("Dumping database...") } diff --git a/cmd/migrate.go b/cmd/migrate.go index dde50a455f0..1fa1d09e253 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -30,7 +30,7 @@ func runMigrate(ctx *cli.Context) error { log.Trace("AppWorkPath: %s", setting.AppWorkPath) log.Trace("Custom path: %s", setting.CustomPath) log.Trace("Log path: %s", setting.LogRootPath) - models.LoadConfigs() + setting.InitDBConfig() if err := models.NewEngine(migrations.Migrate); err != nil { log.Fatal("Failed to initialize ORM engine: %v", err) diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index 7e6351cc632..490d6760c83 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -79,16 +79,16 @@ func runPR() { setting.CheckLFSVersion() //models.LoadConfigs() /* - models.DbCfg.Type = "sqlite3" - models.DbCfg.Path = ":memory:" - models.DbCfg.Timeout = 500 + setting.Database.Type = "sqlite3" + setting.Database.Path = ":memory:" + setting.Database.Timeout = 500 */ db := setting.Cfg.Section("database") db.NewKey("DB_TYPE", "sqlite3") db.NewKey("PATH", ":memory:") - setting.LogSQL = true - models.LoadConfigs() + routers.NewServices() + setting.Database.LogSQL = true //x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared") var helper testfixtures.Helper = &testfixtures.SQLite{} diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 73ca5e0030f..f2763e44df7 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -275,6 +275,10 @@ LOG_SQL = true DB_RETRIES = 10 ; Backoff time per DB retry (time.Duration) DB_RETRY_BACKOFF = 3s +; Max idle database connections on connnection pool, default is 0 +MAX_IDLE_CONNS = 0 +; Database connection max life time, default is 3s +CONN_MAX_LIFETIME = 3s [indexer] ; Issue indexer type, currently support: bleve or db, default is bleve diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index ac309cecfa5..8b5235f7164 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -166,6 +166,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `LOG_SQL`: **true**: Log the executed SQL. - `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed. - `DB_RETRY_BACKOFF`: **3s**: time.Duration to wait before trying another ORM init / DB connect attempt, if failure occured. +- `MAX_IDLE_CONNS` **0**: Max idle database connections on connnection pool, default is 0 +- `CONN_MAX_LIFETIME` **3s**: Database connection max lifetime ## Indexer (`indexer`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 7a00216b9f4..b4b9e4e3a92 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -82,6 +82,8 @@ menu: - `CHARSET`: **utf8**: 仅当数据库为 MySQL 时有效, 可以为 "utf8" 或 "utf8mb4"。注意：如果使用 "utf8mb4"，你的 MySQL InnoDB 版本必须在 5.6 以上。 - `PATH`: Tidb 或者 SQLite3 数据文件存放路径。 - `LOG_SQL`: **true**: 显示生成的SQL，默认为真。 +- `MAX_IDLE_CONNS` **0**: 最大空闲数据库连接 +- `CONN_MAX_LIFETIME` **3s**: 数据库连接最大存活时间 ## Indexer (`indexer`) diff --git a/integrations/integration_test.go b/integrations/integration_test.go index eaec2d509ee..e4d506ee7d3 100644 --- a/integrations/integration_test.go +++ b/integrations/integration_test.go @@ -59,13 +59,13 @@ func TestMain(m *testing.M) { routes.RegisterRoutes(mac) var helper testfixtures.Helper - if setting.UseMySQL { + if setting.Database.UseMySQL { helper = &testfixtures.MySQL{} - } else if setting.UsePostgreSQL { + } else if setting.Database.UsePostgreSQL { helper = &testfixtures.PostgreSQL{} - } else if setting.UseSQLite3 { + } else if setting.Database.UseSQLite3 { helper = &testfixtures.SQLite{} - } else if setting.UseMSSQL { + } else if setting.Database.UseMSSQL { helper = &testfixtures.SQLServer{} } else { fmt.Println("Unsupported RDBMS for integration tests") @@ -121,12 +121,12 @@ func initIntegrationTest() { setting.SetCustomPathAndConf("", "", "") setting.NewContext() setting.CheckLFSVersion() - models.LoadConfigs() + setting.InitDBConfig() switch { - case setting.UseMySQL: + case setting.Database.UseMySQL: db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", - models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host)) + setting.Database.User, setting.Database.Passwd, setting.Database.Host)) defer db.Close() if err != nil { log.Fatalf("sql.Open: %v", err) @@ -134,14 +134,14 @@ func initIntegrationTest() { if _, err = db.Exec("CREATE DATABASE IF NOT EXISTS testgitea"); err != nil { log.Fatalf("db.Exec: %v", err) } - case setting.UsePostgreSQL: + case setting.Database.UsePostgreSQL: db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", - models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode)) + setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) defer db.Close() if err != nil { log.Fatalf("sql.Open: %v", err) } - rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", models.DbCfg.Name)) + rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name)) if err != nil { log.Fatalf("db.Query: %v", err) } @@ -153,10 +153,10 @@ func initIntegrationTest() { if _, err = db.Exec("CREATE DATABASE testgitea"); err != nil { log.Fatalf("db.Exec: %v", err) } - case setting.UseMSSQL: - host, port := models.ParseMSSQLHostPort(models.DbCfg.Host) + case setting.Database.UseMSSQL: + host, port := setting.ParseMSSQLHostPort(setting.Database.Host) db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, "master", models.DbCfg.User, models.DbCfg.Passwd)) + host, port, "master", setting.Database.User, setting.Database.Passwd)) if err != nil { log.Fatalf("sql.Open: %v", err) } diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index 42b64612fd9..bb269d5eeba 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -39,7 +39,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string assert.NoError(t, err) var lfsMetaObject *models.LFSMetaObject - if setting.UsePostgreSQL { + if setting.Database.UsePostgreSQL { lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} } else { lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 858f339b89f..3b47f0d7fc0 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -54,7 +54,7 @@ func initMigrationTest(t *testing.T) { setting.NewContext() setting.CheckLFSVersion() - models.LoadConfigs() + setting.InitDBConfig() setting.NewLogServices(true) } @@ -64,7 +64,7 @@ func availableVersions() ([]string, error) { return nil, err } defer migrationsDir.Close() - versionRE, err := regexp.Compile("gitea-v(?P.+)\\." + regexp.QuoteMeta(models.DbCfg.Type) + "\\.sql.gz") + versionRE, err := regexp.Compile("gitea-v(?P.+)\\." + regexp.QuoteMeta(setting.Database.Type) + "\\.sql.gz") if err != nil { return nil, err } @@ -85,7 +85,7 @@ func availableVersions() ([]string, error) { } func readSQLFromFile(version string) (string, error) { - filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, models.DbCfg.Type) + filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type) if _, err := os.Stat(filename); os.IsNotExist(err) { return "", nil @@ -114,17 +114,17 @@ func restoreOldDB(t *testing.T, version string) bool { data, err := readSQLFromFile(version) assert.NoError(t, err) if len(data) == 0 { - integrations.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version) + integrations.Printf("No db found to restore for %s version: %s\n", setting.Database.Type, version) return false } switch { - case setting.UseSQLite3: - os.Remove(models.DbCfg.Path) - err := os.MkdirAll(path.Dir(models.DbCfg.Path), os.ModePerm) + case setting.Database.UseSQLite3: + os.Remove(setting.Database.Path) + err := os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm) assert.NoError(t, err) - db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", models.DbCfg.Path, models.DbCfg.Timeout)) + db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", setting.Database.Path, setting.Database.Timeout)) assert.NoError(t, err) defer db.Close() @@ -132,20 +132,20 @@ func restoreOldDB(t *testing.T, version string) bool { assert.NoError(t, err) db.Close() - case setting.UseMySQL: + case setting.Database.UseMySQL: db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", - models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host)) + setting.Database.User, setting.Database.Passwd, setting.Database.Host)) assert.NoError(t, err) defer db.Close() - _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name)) + _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name)) assert.NoError(t, err) - _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", models.DbCfg.Name)) + _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)) assert.NoError(t, err) db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true", - models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name)) + setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name)) assert.NoError(t, err) defer db.Close() @@ -153,21 +153,21 @@ func restoreOldDB(t *testing.T, version string) bool { assert.NoError(t, err) db.Close() - case setting.UsePostgreSQL: + case setting.Database.UsePostgreSQL: db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", - models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode)) + setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) assert.NoError(t, err) defer db.Close() - _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name)) + _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name)) assert.NoError(t, err) - _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", models.DbCfg.Name)) + _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)) assert.NoError(t, err) db.Close() db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", - models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name, models.DbCfg.SSLMode)) + setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode)) assert.NoError(t, err) defer db.Close() @@ -175,10 +175,10 @@ func restoreOldDB(t *testing.T, version string) bool { assert.NoError(t, err) db.Close() - case setting.UseMSSQL: - host, port := models.ParseMSSQLHostPort(models.DbCfg.Host) + case setting.Database.UseMSSQL: + host, port := setting.ParseMSSQLHostPort(setting.Database.Host) db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, "master", models.DbCfg.User, models.DbCfg.Passwd)) + host, port, "master", setting.Database.User, setting.Database.Passwd)) assert.NoError(t, err) defer db.Close() @@ -191,7 +191,7 @@ func restoreOldDB(t *testing.T, version string) bool { dbname := statement[5 : len(statement)-1] db.Close() db, err = sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, dbname, models.DbCfg.User, models.DbCfg.Passwd)) + host, port, dbname, setting.Database.User, setting.Database.Passwd)) assert.NoError(t, err) defer db.Close() } @@ -210,7 +210,7 @@ func wrappedMigrate(x *xorm.Engine) error { func doMigrationTest(t *testing.T, version string) { integrations.PrintCurrentTest(t) - integrations.Printf("Performing migration test for %s version: %s\n", models.DbCfg.Type, version) + integrations.Printf("Performing migration test for %s version: %s\n", setting.Database.Type, version) if !restoreOldDB(t, version) { return } @@ -227,7 +227,7 @@ func doMigrationTest(t *testing.T, version string) { func TestMigrations(t *testing.T) { initMigrationTest(t) - dialect := models.DbCfg.Type + dialect := setting.Database.Type versions, err := availableVersions() assert.NoError(t, err) diff --git a/models/convert.go b/models/convert.go index c34be973c6b..025f88b5038 100644 --- a/models/convert.go +++ b/models/convert.go @@ -4,11 +4,15 @@ package models -import "fmt" +import ( + "fmt" + + "code.gitea.io/gitea/modules/setting" +) // ConvertUtf8ToUtf8mb4 converts database and tables from utf8 to utf8mb4 if it's mysql func ConvertUtf8ToUtf8mb4() error { - _, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", DbCfg.Name)) + _, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", setting.Database.Name)) if err != nil { return err } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index fc086bfd464..7d97741a20d 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -296,7 +296,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin // TODO: This will not work if there are foreign keys switch { - case setting.UseSQLite3: + case setting.Database.UseSQLite3: // First drop the indexes on the columns res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName)) if errIndex != nil { @@ -372,7 +372,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin return err } - case setting.UsePostgreSQL: + case setting.Database.UsePostgreSQL: cols := "" for _, col := range columnNames { if cols != "" { @@ -383,7 +383,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil { return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) } - case setting.UseMySQL, setting.UseTiDB: + case setting.Database.UseMySQL: // Drop indexes on columns first sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','")) res, err := sess.Query(sql) @@ -409,7 +409,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil { return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) } - case setting.UseMSSQL: + case setting.Database.UseMSSQL: cols := "" for _, col := range columnNames { if cols != "" { diff --git a/models/migrations/v19.go b/models/migrations/v19.go index 9793f405ade..7728f5add6e 100644 --- a/models/migrations/v19.go +++ b/models/migrations/v19.go @@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) { } ) - return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository), + return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository), func(idx int, bean interface{}) error { repo := bean.(*Repository) user := new(User) diff --git a/models/migrations/v22.go b/models/migrations/v22.go index f991ae25fa6..faac74343b5 100644 --- a/models/migrations/v22.go +++ b/models/migrations/v22.go @@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) { } ) - return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository), + return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository), func(idx int, bean interface{}) error { repo := bean.(*Repository) user := new(User) diff --git a/models/migrations/v26.go b/models/migrations/v26.go index 3d88c62923e..04277191f5e 100644 --- a/models/migrations/v26.go +++ b/models/migrations/v26.go @@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) { hookTpl = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType) ) - return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository), + return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository), func(idx int, bean interface{}) error { repo := bean.(*Repository) user := new(User) diff --git a/models/migrations/v27.go b/models/migrations/v27.go index e87c7ab68f5..12e5fbcdbfe 100644 --- a/models/migrations/v27.go +++ b/models/migrations/v27.go @@ -41,8 +41,6 @@ func convertIntervalToDuration(x *xorm.Engine) (err error) { _, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT") case "postgres": _, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" SET DATA TYPE bigint") - case "tidb": - _, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT") case "mssql": _, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" BIGINT") case "sqlite3": diff --git a/models/migrations/v33.go b/models/migrations/v33.go index ae7612e68cd..566951db962 100644 --- a/models/migrations/v33.go +++ b/models/migrations/v33.go @@ -15,9 +15,9 @@ import ( func removeActionColumns(x *xorm.Engine) error { switch { - case setting.UseSQLite3: + case setting.Database.UseSQLite3: log.Warn("Unable to drop columns in SQLite") - case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB: + case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL: if _, err := x.Exec("ALTER TABLE action DROP COLUMN act_user_name"); err != nil { return fmt.Errorf("DROP COLUMN act_user_name: %v", err) } else if _, err = x.Exec("ALTER TABLE action DROP COLUMN repo_user_name"); err != nil { diff --git a/models/migrations/v45.go b/models/migrations/v45.go index 92cb8178197..99baff2c8b3 100644 --- a/models/migrations/v45.go +++ b/models/migrations/v45.go @@ -13,9 +13,9 @@ import ( func removeIndexColumnFromRepoUnitTable(x *xorm.Engine) (err error) { switch { - case setting.UseSQLite3: + case setting.Database.UseSQLite3: log.Warn("Unable to drop columns in SQLite") - case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB: + case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL: if _, err := x.Exec("ALTER TABLE repo_unit DROP COLUMN `index`"); err != nil { // Ignoring this error in case we run this migration second time (after migration reordering) log.Warn("DROP COLUMN index: %v", err) diff --git a/models/migrations/v50.go b/models/migrations/v50.go index a15914f0ee6..23b1bb526eb 100644 --- a/models/migrations/v50.go +++ b/models/migrations/v50.go @@ -40,9 +40,9 @@ func migrateProtectedBranchStruct(x *xorm.Engine) error { } switch { - case setting.UseSQLite3: + case setting.Database.UseSQLite3: log.Warn("Unable to drop columns in SQLite") - case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB: + case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL: if _, err := x.Exec("ALTER TABLE protected_branch DROP COLUMN can_push"); err != nil { // Ignoring this error in case we run this migration second time (after migration reordering) log.Warn("DROP COLUMN can_push (skipping): %v", err) diff --git a/models/migrations/v67.go b/models/migrations/v67.go index 74d3f379cd2..6cf3dd4d193 100644 --- a/models/migrations/v67.go +++ b/models/migrations/v67.go @@ -80,7 +80,7 @@ func removeStaleWatches(x *xorm.Engine) error { } repoCache := make(map[int64]*Repository) - err := sess.BufferSize(setting.IterateBufferSize).Iterate(new(Watch), + err := sess.BufferSize(setting.Database.IterateBufferSize).Iterate(new(Watch), func(idx int, bean interface{}) error { watch := bean.(*Watch) @@ -117,7 +117,7 @@ func removeStaleWatches(x *xorm.Engine) error { } repoCache = make(map[int64]*Repository) - err = sess.BufferSize(setting.IterateBufferSize). + err = sess.BufferSize(setting.Database.IterateBufferSize). Distinct("issue_watch.user_id", "issue.repo_id"). Join("INNER", "issue", "issue_watch.issue_id = issue.id"). Where("issue_watch.is_watching = ?", true). diff --git a/models/migrations/v81.go b/models/migrations/v81.go index 56bb8477e6b..48e96508d9c 100644 --- a/models/migrations/v81.go +++ b/models/migrations/v81.go @@ -14,8 +14,6 @@ func changeU2FCounterType(x *xorm.Engine) error { var err error switch x.Dialect().DriverName() { - case "tidb": - fallthrough case "mysql": _, err = x.Exec("ALTER TABLE `u2f_registration` MODIFY `counter` BIGINT") case "postgres": diff --git a/models/models.go b/models/models.go index 4c925fa5706..04acc77aa94 100644 --- a/models/models.go +++ b/models/models.go @@ -9,12 +9,6 @@ import ( "database/sql" "errors" "fmt" - "net/url" - "os" - "path" - "path/filepath" - "strings" - "time" "code.gitea.io/gitea/modules/setting" @@ -52,24 +46,11 @@ type Engine interface { } var ( - x *xorm.Engine - supportedDatabases = []string{"mysql", "postgres", "mssql"} - tables []interface{} + x *xorm.Engine + tables []interface{} // HasEngine specifies if we have a xorm.Engine HasEngine bool - - // DbCfg holds the database settings - DbCfg struct { - Type, Host, Name, User, Passwd, Path, SSLMode, Charset string - Timeout int - } - - // EnableSQLite3 use SQLite3 - EnableSQLite3 bool - - // EnableTiDB enable TiDB - EnableTiDB bool ) func init() { @@ -139,120 +120,13 @@ func init() { } } -// LoadConfigs loads the database settings -func LoadConfigs() { - sec := setting.Cfg.Section("database") - DbCfg.Type = sec.Key("DB_TYPE").String() - switch DbCfg.Type { - case "sqlite3": - setting.UseSQLite3 = true - case "mysql": - setting.UseMySQL = true - case "postgres": - setting.UsePostgreSQL = true - case "tidb": - setting.UseTiDB = true - case "mssql": - setting.UseMSSQL = true - } - DbCfg.Host = sec.Key("HOST").String() - DbCfg.Name = sec.Key("NAME").String() - DbCfg.User = sec.Key("USER").String() - if len(DbCfg.Passwd) == 0 { - DbCfg.Passwd = sec.Key("PASSWD").String() - } - DbCfg.SSLMode = sec.Key("SSL_MODE").MustString("disable") - DbCfg.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"}) - DbCfg.Path = sec.Key("PATH").MustString(filepath.Join(setting.AppDataPath, "gitea.db")) - DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) -} - -// parsePostgreSQLHostPort parses given input in various forms defined in -// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING -// and returns proper host and port number. -func parsePostgreSQLHostPort(info string) (string, string) { - host, port := "127.0.0.1", "5432" - if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") { - idx := strings.LastIndex(info, ":") - host = info[:idx] - port = info[idx+1:] - } else if len(info) > 0 { - host = info - } - return host, port -} - -func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) { - host, port := parsePostgreSQLHostPort(dbHost) - if host[0] == '/' { // looks like a unix socket - connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s", - url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host) - } else { - connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s", - url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode) - } - return -} - -// ParseMSSQLHostPort splits the host into host and port -func ParseMSSQLHostPort(info string) (string, string) { - host, port := "127.0.0.1", "1433" - if strings.Contains(info, ":") { - host = strings.Split(info, ":")[0] - port = strings.Split(info, ":")[1] - } else if strings.Contains(info, ",") { - host = strings.Split(info, ",")[0] - port = strings.TrimSpace(strings.Split(info, ",")[1]) - } else if len(info) > 0 { - host = info - } - return host, port -} - func getEngine() (*xorm.Engine, error) { - connStr := "" - var Param = "?" - if strings.Contains(DbCfg.Name, Param) { - Param = "&" - } - switch DbCfg.Type { - case "mysql": - connType := "tcp" - if DbCfg.Host[0] == '/' { // looks like a unix socket - connType = "unix" - } - tls := DbCfg.SSLMode - if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL - tls = "false" - } - connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s", - DbCfg.User, DbCfg.Passwd, connType, DbCfg.Host, DbCfg.Name, Param, DbCfg.Charset, tls) - case "postgres": - connStr = getPostgreSQLConnectionString(DbCfg.Host, DbCfg.User, DbCfg.Passwd, DbCfg.Name, Param, DbCfg.SSLMode) - case "mssql": - host, port := ParseMSSQLHostPort(DbCfg.Host) - connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, DbCfg.Name, DbCfg.User, DbCfg.Passwd) - case "sqlite3": - if !EnableSQLite3 { - return nil, errors.New("this binary version does not build support for SQLite3") - } - if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil { - return nil, fmt.Errorf("Failed to create directories: %v", err) - } - connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", DbCfg.Path, DbCfg.Timeout) - case "tidb": - if !EnableTiDB { - return nil, errors.New("this binary version does not build support for TiDB") - } - if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil { - return nil, fmt.Errorf("Failed to create directories: %v", err) - } - connStr = "goleveldb://" + DbCfg.Path - default: - return nil, fmt.Errorf("Unknown database type: %s", DbCfg.Type) + connStr, err := setting.DBConnStr() + if err != nil { + return nil, err } - return xorm.NewEngine(DbCfg.Type, connStr) + return xorm.NewEngine(setting.Database.Type, connStr) } // NewTestEngine sets a new test xorm.Engine @@ -280,11 +154,11 @@ func SetEngine() (err error) { x.SetMapper(core.GonicMapper{}) // WARNING: for serv command, MUST remove the output to os.stdout, // so use log file to instead print to stdout. - x.SetLogger(NewXORMLogger(setting.LogSQL)) - x.ShowSQL(setting.LogSQL) - if DbCfg.Type == "mysql" { - x.SetMaxIdleConns(0) - x.SetConnMaxLifetime(3 * time.Second) + x.SetLogger(NewXORMLogger(setting.Database.LogSQL)) + x.ShowSQL(setting.Database.LogSQL) + if setting.Database.UseMySQL { + x.SetMaxIdleConns(setting.Database.MaxIdleConns) + x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) } return nil diff --git a/models/models_test.go b/models/models_test.go index 6df3b4e0486..37e9a352f8a 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -1,5 +1,4 @@ -// Copyright 2016 The Gogs Authors. All rights reserved. -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -11,99 +10,19 @@ import ( "path/filepath" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) -func Test_parsePostgreSQLHostPort(t *testing.T) { - tests := []struct { - HostPort string - Host string - Port string - }{ - { - HostPort: "127.0.0.1:1234", - Host: "127.0.0.1", - Port: "1234", - }, - { - HostPort: "127.0.0.1", - Host: "127.0.0.1", - Port: "5432", - }, - { - HostPort: "[::1]:1234", - Host: "[::1]", - Port: "1234", - }, - { - HostPort: "[::1]", - Host: "[::1]", - Port: "5432", - }, - { - HostPort: "/tmp/pg.sock:1234", - Host: "/tmp/pg.sock", - Port: "1234", - }, - { - HostPort: "/tmp/pg.sock", - Host: "/tmp/pg.sock", - Port: "5432", - }, - } - for _, test := range tests { - host, port := parsePostgreSQLHostPort(test.HostPort) - assert.Equal(t, test.Host, host) - assert.Equal(t, test.Port, port) - } -} - -func Test_getPostgreSQLConnectionString(t *testing.T) { - tests := []struct { - Host string - Port string - User string - Passwd string - Name string - Param string - SSLMode string - Output string - }{ - { - Host: "/tmp/pg.sock", - Port: "4321", - User: "testuser", - Passwd: "space space !#$%^^%^```-=?=", - Name: "gitea", - Param: "", - SSLMode: "false", - Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/giteasslmode=false&host=/tmp/pg.sock", - }, - { - Host: "localhost", - Port: "1234", - User: "pgsqlusername", - Passwd: "I love Gitea!", - Name: "gitea", - Param: "", - SSLMode: "true", - Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/giteasslmode=true", - }, - } - - for _, test := range tests { - connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.Param, test.SSLMode) - assert.Equal(t, test.Output, connStr) - } -} - func TestDumpDatabase(t *testing.T) { assert.NoError(t, PrepareTestDatabase()) dir, err := ioutil.TempDir(os.TempDir(), "dump") assert.NoError(t, err) - for _, dbType := range supportedDatabases { + for _, dbName := range setting.SupportedDatabases { + dbType := setting.GetDBTypeByName(dbName) assert.NoError(t, DumpDatabase(filepath.Join(dir, dbType+".sql"), dbType)) } } diff --git a/models/repo.go b/models/repo.go index a70350576e8..507521357e3 100644 --- a/models/repo.go +++ b/models/repo.go @@ -2201,7 +2201,7 @@ func GitFsck() { log.Trace("Doing: GitFsck") if err := x. - Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.IterateBufferSize). + Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.Database.IterateBufferSize). Iterate(new(Repository), func(idx int, bean interface{}) error { repo := bean.(*Repository) @@ -2225,7 +2225,7 @@ func GitFsck() { func GitGcRepos() error { args := append([]string{"gc"}, setting.Git.GCArgs...) return x. - Where("id > 0").BufferSize(setting.IterateBufferSize). + Where("id > 0").BufferSize(setting.Database.IterateBufferSize). Iterate(new(Repository), func(idx int, bean interface{}) error { repo := bean.(*Repository) @@ -2568,7 +2568,7 @@ func (repo *Repository) generateRandomAvatar(e Engine) error { // RemoveRandomAvatars removes the randomly generated avatars that were created for repositories func RemoveRandomAvatars() error { return x. - Where("id > 0").BufferSize(setting.IterateBufferSize). + Where("id > 0").BufferSize(setting.Database.IterateBufferSize). Iterate(new(Repository), func(idx int, bean interface{}) error { repository := bean.(*Repository) diff --git a/models/unit_tests.go b/models/unit_tests.go index 80dc4feb96d..b53302dad4b 100644 --- a/models/unit_tests.go +++ b/models/unit_tests.go @@ -49,7 +49,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) { setting.RunUser = "runuser" setting.SSH.Port = 3000 setting.SSH.Domain = "try.gitea.io" - setting.UseSQLite3 = true + setting.Database.UseSQLite3 = true setting.RepoRootPath, err = ioutil.TempDir(os.TempDir(), "repos") if err != nil { fatalTestError("TempDir: %v\n", err) diff --git a/models/user.go b/models/user.go index 037ae0397ba..c8664cc4c80 100644 --- a/models/user.go +++ b/models/user.go @@ -40,7 +40,6 @@ import ( "golang.org/x/crypto/scrypt" "golang.org/x/crypto/ssh" "xorm.io/builder" - "xorm.io/core" ) // UserType defines the user type @@ -1432,9 +1431,9 @@ func (opts *SearchUserOptions) toConds() builder.Cond { if opts.OwnerID > 0 { var exprCond builder.Cond - if DbCfg.Type == core.MYSQL { + if setting.Database.UseMySQL { exprCond = builder.Expr("org_user.org_id = user.id") - } else if DbCfg.Type == core.MSSQL { + } else if setting.Database.UseMSSQL { exprCond = builder.Expr("org_user.org_id = [user].id") } else { exprCond = builder.Expr("org_user.org_id = \"user\".id") diff --git a/models/user_heatmap.go b/models/user_heatmap.go index b511b5fa62d..3d9e0683fc7 100644 --- a/models/user_heatmap.go +++ b/models/user_heatmap.go @@ -21,13 +21,13 @@ func GetUserHeatmapDataByUser(user *User) ([]*UserHeatmapData, error) { var groupBy string var groupByName = "timestamp" // We need this extra case because mssql doesn't allow grouping by alias switch { - case setting.UseSQLite3: + case setting.Database.UseSQLite3: groupBy = "strftime('%s', strftime('%Y-%m-%d', created_unix, 'unixepoch'))" - case setting.UseMySQL: + case setting.Database.UseMySQL: groupBy = "UNIX_TIMESTAMP(DATE(FROM_UNIXTIME(created_unix)))" - case setting.UsePostgreSQL: + case setting.Database.UsePostgreSQL: groupBy = "extract(epoch from date_trunc('day', to_timestamp(created_unix)))" - case setting.UseMSSQL: + case setting.Database.UseMSSQL: groupBy = "datediff(SECOND, '19700101', dateadd(DAY, 0, datediff(day, 0, dateadd(s, created_unix, '19700101'))))" groupByName = groupBy } diff --git a/modules/setting/database.go b/modules/setting/database.go new file mode 100644 index 00000000000..2cac4824dfb --- /dev/null +++ b/modules/setting/database.go @@ -0,0 +1,171 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "errors" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +var ( + // SupportedDatabases includes all supported databases type + SupportedDatabases = []string{"MySQL", "PostgreSQL", "MSSQL"} + dbTypes = map[string]string{"MySQL": "mysql", "PostgreSQL": "postgres", "MSSQL": "mssql", "SQLite3": "sqlite3"} + + // EnableSQLite3 use SQLite3, set by build flag + EnableSQLite3 bool + + // Database holds the database settings + Database = struct { + Type string + Host string + Name string + User string + Passwd string + SSLMode string + Path string + LogSQL bool + Charset string + Timeout int // seconds + UseSQLite3 bool + UseMySQL bool + UseMSSQL bool + UsePostgreSQL bool + DBConnectRetries int + DBConnectBackoff time.Duration + MaxIdleConns int + ConnMaxLifetime time.Duration + IterateBufferSize int + }{ + Timeout: 500, + MaxIdleConns: 0, + ConnMaxLifetime: 3 * time.Second, + } +) + +// GetDBTypeByName returns the dataase type as it defined on XORM according the given name +func GetDBTypeByName(name string) string { + return dbTypes[name] +} + +// InitDBConfig loads the database settings +func InitDBConfig() { + sec := Cfg.Section("database") + Database.Type = sec.Key("DB_TYPE").String() + switch Database.Type { + case "sqlite3": + Database.UseSQLite3 = true + case "mysql": + Database.UseMySQL = true + case "postgres": + Database.UsePostgreSQL = true + case "mssql": + Database.UseMSSQL = true + } + Database.Host = sec.Key("HOST").String() + Database.Name = sec.Key("NAME").String() + Database.User = sec.Key("USER").String() + if len(Database.Passwd) == 0 { + Database.Passwd = sec.Key("PASSWD").String() + } + Database.SSLMode = sec.Key("SSL_MODE").MustString("disable") + Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"}) + Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db")) + Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) + Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(0) + Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second) + + Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50) + Database.LogSQL = sec.Key("LOG_SQL").MustBool(true) + Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10) + Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second) +} + +// DBConnStr returns database connection string +func DBConnStr() (string, error) { + connStr := "" + var Param = "?" + if strings.Contains(Database.Name, Param) { + Param = "&" + } + switch Database.Type { + case "mysql": + connType := "tcp" + if Database.Host[0] == '/' { // looks like a unix socket + connType = "unix" + } + tls := Database.SSLMode + if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL + tls = "false" + } + connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s", + Database.User, Database.Passwd, connType, Database.Host, Database.Name, Param, Database.Charset, tls) + case "postgres": + connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Param, Database.SSLMode) + case "mssql": + host, port := ParseMSSQLHostPort(Database.Host) + connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd) + case "sqlite3": + if !EnableSQLite3 { + return "", errors.New("this binary version does not build support for SQLite3") + } + if err := os.MkdirAll(path.Dir(Database.Path), os.ModePerm); err != nil { + return "", fmt.Errorf("Failed to create directories: %v", err) + } + connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", Database.Path, Database.Timeout) + default: + return "", fmt.Errorf("Unknown database type: %s", Database.Type) + } + + return connStr, nil +} + +// parsePostgreSQLHostPort parses given input in various forms defined in +// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING +// and returns proper host and port number. +func parsePostgreSQLHostPort(info string) (string, string) { + host, port := "127.0.0.1", "5432" + if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") { + idx := strings.LastIndex(info, ":") + host = info[:idx] + port = info[idx+1:] + } else if len(info) > 0 { + host = info + } + return host, port +} + +func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) { + host, port := parsePostgreSQLHostPort(dbHost) + if host[0] == '/' { // looks like a unix socket + connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s", + url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host) + } else { + connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s", + url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode) + } + return +} + +// ParseMSSQLHostPort splits the host into host and port +func ParseMSSQLHostPort(info string) (string, string) { + host, port := "127.0.0.1", "1433" + if strings.Contains(info, ":") { + host = strings.Split(info, ":")[0] + port = strings.Split(info, ":")[1] + } else if strings.Contains(info, ",") { + host = strings.Split(info, ",")[0] + port = strings.TrimSpace(strings.Split(info, ",")[1]) + } else if len(info) > 0 { + host = info + } + return host, port +} diff --git a/models/models_sqlite.go b/modules/setting/database_sqlite.go similarity index 77% rename from models/models_sqlite.go rename to modules/setting/database_sqlite.go index 3889aceb847..623326ddc2f 100644 --- a/models/models_sqlite.go +++ b/modules/setting/database_sqlite.go @@ -4,7 +4,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package models +package setting import ( _ "github.com/mattn/go-sqlite3" @@ -12,5 +12,5 @@ import ( func init() { EnableSQLite3 = true - supportedDatabases = append(supportedDatabases, "sqlite3") + SupportedDatabases = append(SupportedDatabases, "SQLite3") } diff --git a/modules/setting/database_test.go b/modules/setting/database_test.go new file mode 100644 index 00000000000..a90be2a6875 --- /dev/null +++ b/modules/setting/database_test.go @@ -0,0 +1,94 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_parsePostgreSQLHostPort(t *testing.T) { + tests := []struct { + HostPort string + Host string + Port string + }{ + { + HostPort: "127.0.0.1:1234", + Host: "127.0.0.1", + Port: "1234", + }, + { + HostPort: "127.0.0.1", + Host: "127.0.0.1", + Port: "5432", + }, + { + HostPort: "[::1]:1234", + Host: "[::1]", + Port: "1234", + }, + { + HostPort: "[::1]", + Host: "[::1]", + Port: "5432", + }, + { + HostPort: "/tmp/pg.sock:1234", + Host: "/tmp/pg.sock", + Port: "1234", + }, + { + HostPort: "/tmp/pg.sock", + Host: "/tmp/pg.sock", + Port: "5432", + }, + } + for _, test := range tests { + host, port := parsePostgreSQLHostPort(test.HostPort) + assert.Equal(t, test.Host, host) + assert.Equal(t, test.Port, port) + } +} + +func Test_getPostgreSQLConnectionString(t *testing.T) { + tests := []struct { + Host string + Port string + User string + Passwd string + Name string + Param string + SSLMode string + Output string + }{ + { + Host: "/tmp/pg.sock", + Port: "4321", + User: "testuser", + Passwd: "space space !#$%^^%^```-=?=", + Name: "gitea", + Param: "", + SSLMode: "false", + Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/giteasslmode=false&host=/tmp/pg.sock", + }, + { + Host: "localhost", + Port: "1234", + User: "pgsqlusername", + Passwd: "I love Gitea!", + Name: "gitea", + Param: "", + SSLMode: "true", + Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/giteasslmode=true", + }, + } + + for _, test := range tests { + connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.Param, test.SSLMode) + assert.Equal(t, test.Output, connStr) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 123aa8f1039..77b8f20640c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -148,16 +148,6 @@ var ( DisableGitHooks bool PasswordHashAlgo string - // Database settings - UseSQLite3 bool - UseMySQL bool - UseMSSQL bool - UsePostgreSQL bool - UseTiDB bool - LogSQL bool - DBConnectRetries int - DBConnectBackoff time.Duration - // UI settings UI = struct { ExplorePagingNum int @@ -348,16 +338,19 @@ var ( ShowFooterTemplateLoadTime bool // Global setting objects - Cfg *ini.File - CustomPath string // Custom directory path - CustomConf string - CustomPID string - ProdMode bool - RunUser string - IsWindows bool - HasRobotsTxt bool - InternalToken string // internal access token - IterateBufferSize int + Cfg *ini.File + CustomPath string // Custom directory path + CustomConf string + CustomPID string + ProdMode bool + RunUser string + IsWindows bool + HasRobotsTxt bool + InternalToken string // internal access token + + // UILocation is the location on the UI, so that we can display the time on UI. + // Currently only show the default time.Local, it could be added to app.ini after UI is ready + UILocation = time.Local ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -775,10 +768,6 @@ func NewContext() { CSRFCookieHTTPOnly = sec.Key("CSRF_COOKIE_HTTP_ONLY").MustBool(true) InternalToken = loadInternalToken(sec) - IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50) - LogSQL = Cfg.Section("database").Key("LOG_SQL").MustBool(true) - DBConnectRetries = Cfg.Section("database").Key("DB_RETRIES").MustInt(10) - DBConnectBackoff = Cfg.Section("database").Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second) sec = Cfg.Section("attachment") AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments")) @@ -1037,6 +1026,7 @@ func loadOrGenerateInternalToken(sec *ini.Section) string { // NewServices initializes the services func NewServices() { + InitDBConfig() newService() NewLogServices(false) newCacheService() diff --git a/routers/admin/admin.go b/routers/admin/admin.go index 54fbddc58e1..496aca375f5 100644 --- a/routers/admin/admin.go +++ b/routers/admin/admin.go @@ -289,7 +289,7 @@ func Config(ctx *context.Context) { ctx.Data["LFS"] = setting.LFS ctx.Data["Service"] = setting.Service - ctx.Data["DbCfg"] = models.DbCfg + ctx.Data["DbCfg"] = setting.Database ctx.Data["Webhook"] = setting.Webhook ctx.Data["MailerEnabled"] = false @@ -333,7 +333,7 @@ func Config(ctx *context.Context) { ctx.Data["AccessLogTemplate"] = setting.AccessLogTemplate ctx.Data["DisableRouterLog"] = setting.DisableRouterLog ctx.Data["EnableXORMLog"] = setting.EnableXORMLog - ctx.Data["LogSQL"] = setting.LogSQL + ctx.Data["LogSQL"] = setting.Database.LogSQL ctx.HTML(200, tplConfig) } diff --git a/routers/init.go b/routers/init.go index 4724da8627a..fdf90904ce7 100644 --- a/routers/init.go +++ b/routers/init.go @@ -47,16 +47,16 @@ func NewServices() { // In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology func initDBEngine() (err error) { log.Info("Beginning ORM engine initialization.") - for i := 0; i < setting.DBConnectRetries; i++ { - log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.DBConnectRetries) + for i := 0; i < setting.Database.DBConnectRetries; i++ { + log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.Database.DBConnectRetries) if err = models.NewEngine(migrations.Migrate); err == nil { break - } else if i == setting.DBConnectRetries-1 { + } else if i == setting.Database.DBConnectRetries-1 { return err } - log.Error("ORM engine initialization attempt #%d/%d failed. Error: %v", i+1, setting.DBConnectRetries, err) - log.Info("Backing off for %d seconds", int64(setting.DBConnectBackoff/time.Second)) - time.Sleep(setting.DBConnectBackoff) + log.Error("ORM engine initialization attempt #%d/%d failed. Error: %v", i+1, setting.Database.DBConnectRetries, err) + log.Info("Backing off for %d seconds", int64(setting.Database.DBConnectBackoff/time.Second)) + time.Sleep(setting.Database.DBConnectBackoff) } models.HasEngine = true return nil @@ -73,7 +73,7 @@ func GlobalInit() { log.Trace("AppWorkPath: %s", setting.AppWorkPath) log.Trace("Custom path: %s", setting.CustomPath) log.Trace("Log path: %s", setting.LogRootPath) - models.LoadConfigs() + NewServices() if setting.InstallLock { @@ -102,12 +102,9 @@ func GlobalInit() { models.InitDeliverHooks() models.InitTestPullRequests() } - if models.EnableSQLite3 { + if setting.EnableSQLite3 { log.Info("SQLite3 Supported") } - if models.EnableTiDB { - log.Info("TiDB Supported") - } checkRunMode() if setting.InstallLock && setting.SSH.StartBuiltinServer { diff --git a/routers/install.go b/routers/install.go index 6d29bcf887a..16888adc824 100644 --- a/routers/install.go +++ b/routers/install.go @@ -40,11 +40,7 @@ func InstallInit(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("install.install") ctx.Data["PageIsInstall"] = true - dbOpts := []string{"MySQL", "PostgreSQL", "MSSQL"} - if models.EnableSQLite3 { - dbOpts = append(dbOpts, "SQLite3") - } - ctx.Data["DbOptions"] = dbOpts + ctx.Data["DbOptions"] = setting.SupportedDatabases } // Install render installation page @@ -52,21 +48,21 @@ func Install(ctx *context.Context) { form := auth.InstallForm{} // Database settings - form.DbHost = models.DbCfg.Host - form.DbUser = models.DbCfg.User - form.DbPasswd = models.DbCfg.Passwd - form.DbName = models.DbCfg.Name - form.DbPath = models.DbCfg.Path - form.Charset = models.DbCfg.Charset + form.DbHost = setting.Database.Host + form.DbUser = setting.Database.User + form.DbPasswd = setting.Database.Passwd + form.DbName = setting.Database.Name + form.DbPath = setting.Database.Path + form.Charset = setting.Database.Charset ctx.Data["CurDbOption"] = "MySQL" - switch models.DbCfg.Type { + switch setting.Database.Type { case "postgres": ctx.Data["CurDbOption"] = "PostgreSQL" case "mssql": ctx.Data["CurDbOption"] = "MSSQL" case "sqlite3": - if models.EnableSQLite3 { + if setting.EnableSQLite3 { ctx.Data["CurDbOption"] = "SQLite3" } } @@ -144,18 +140,18 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) { // Pass basic check, now test configuration. // Test database setting. - dbTypes := map[string]string{"MySQL": "mysql", "PostgreSQL": "postgres", "MSSQL": "mssql", "SQLite3": "sqlite3"} - models.DbCfg.Type = dbTypes[form.DbType] - models.DbCfg.Host = form.DbHost - models.DbCfg.User = form.DbUser - models.DbCfg.Passwd = form.DbPasswd - models.DbCfg.Name = form.DbName - models.DbCfg.SSLMode = form.SSLMode - models.DbCfg.Charset = form.Charset - models.DbCfg.Path = form.DbPath - - if (models.DbCfg.Type == "sqlite3") && - len(models.DbCfg.Path) == 0 { + + setting.Database.Type = setting.GetDBTypeByName(form.DbType) + setting.Database.Host = form.DbHost + setting.Database.User = form.DbUser + setting.Database.Passwd = form.DbPasswd + setting.Database.Name = form.DbName + setting.Database.SSLMode = form.SSLMode + setting.Database.Charset = form.Charset + setting.Database.Path = form.DbPath + + if (setting.Database.Type == "sqlite3") && + len(setting.Database.Path) == 0 { ctx.Data["Err_DbPath"] = true ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, &form) return @@ -265,14 +261,14 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) { log.Error("Failed to load custom conf '%s': %v", setting.CustomConf, err) } } - cfg.Section("database").Key("DB_TYPE").SetValue(models.DbCfg.Type) - cfg.Section("database").Key("HOST").SetValue(models.DbCfg.Host) - cfg.Section("database").Key("NAME").SetValue(models.DbCfg.Name) - cfg.Section("database").Key("USER").SetValue(models.DbCfg.User) - cfg.Section("database").Key("PASSWD").SetValue(models.DbCfg.Passwd) - cfg.Section("database").Key("SSL_MODE").SetValue(models.DbCfg.SSLMode) - cfg.Section("database").Key("CHARSET").SetValue(models.DbCfg.Charset) - cfg.Section("database").Key("PATH").SetValue(models.DbCfg.Path) + cfg.Section("database").Key("DB_TYPE").SetValue(setting.Database.Type) + cfg.Section("database").Key("HOST").SetValue(setting.Database.Host) + cfg.Section("database").Key("NAME").SetValue(setting.Database.Name) + cfg.Section("database").Key("USER").SetValue(setting.Database.User) + cfg.Section("database").Key("PASSWD").SetValue(setting.Database.Passwd) + cfg.Section("database").Key("SSL_MODE").SetValue(setting.Database.SSLMode) + cfg.Section("database").Key("CHARSET").SetValue(setting.Database.Charset) + cfg.Section("database").Key("PATH").SetValue(setting.Database.Path) cfg.Section("").Key("APP_NAME").SetValue(form.AppName) cfg.Section("repository").Key("ROOT").SetValue(form.RepoRootPath)