mirror of https://github.com/go-gitea/gitea
Refactor fixture loading for testing (#33024)
To help binary size and testing performancepull/26402/head^2
wxiaoguang
1 month ago
committed by
GitHub
parent
f4ccbd38dc
commit
dafadf0028
File diff suppressed because one or more lines are too long
@ -0,0 +1,201 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package unittest |
||||
|
||||
import ( |
||||
"database/sql" |
||||
"encoding/hex" |
||||
"fmt" |
||||
"os" |
||||
"path/filepath" |
||||
"slices" |
||||
"strings" |
||||
|
||||
"gopkg.in/yaml.v3" |
||||
"xorm.io/xorm" |
||||
"xorm.io/xorm/schemas" |
||||
) |
||||
|
||||
type fixtureItem struct { |
||||
tableName string |
||||
tableNameQuoted string |
||||
sqlInserts []string |
||||
sqlInsertArgs [][]any |
||||
|
||||
mssqlHasIdentityColumn bool |
||||
} |
||||
|
||||
type fixturesLoaderInternal struct { |
||||
db *sql.DB |
||||
dbType schemas.DBType |
||||
files []string |
||||
fixtures map[string]*fixtureItem |
||||
quoteObject func(string) string |
||||
paramPlaceholder func(idx int) string |
||||
} |
||||
|
||||
func (f *fixturesLoaderInternal) mssqlTableHasIdentityColumn(db *sql.DB, tableName string) (bool, error) { |
||||
row := db.QueryRow(`SELECT COUNT(*) FROM sys.identity_columns WHERE OBJECT_ID = OBJECT_ID(?)`, tableName) |
||||
var count int |
||||
if err := row.Scan(&count); err != nil { |
||||
return false, err |
||||
} |
||||
return count > 0, nil |
||||
} |
||||
|
||||
func (f *fixturesLoaderInternal) preprocessFixtureRow(row []map[string]any) (err error) { |
||||
for _, m := range row { |
||||
for k, v := range m { |
||||
if s, ok := v.(string); ok { |
||||
if strings.HasPrefix(s, "0x") { |
||||
if m[k], err = hex.DecodeString(s[2:]); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (f *fixturesLoaderInternal) prepareFixtureItem(file string) (_ *fixtureItem, err error) { |
||||
fixture := &fixtureItem{} |
||||
fixture.tableName, _, _ = strings.Cut(filepath.Base(file), ".") |
||||
fixture.tableNameQuoted = f.quoteObject(fixture.tableName) |
||||
|
||||
if f.dbType == schemas.MSSQL { |
||||
fixture.mssqlHasIdentityColumn, err = f.mssqlTableHasIdentityColumn(f.db, fixture.tableName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
data, err := os.ReadFile(file) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("failed to read file %q: %w", file, err) |
||||
} |
||||
|
||||
var rows []map[string]any |
||||
if err = yaml.Unmarshal(data, &rows); err != nil { |
||||
return nil, fmt.Errorf("failed to unmarshal yaml data from %q: %w", file, err) |
||||
} |
||||
if err = f.preprocessFixtureRow(rows); err != nil { |
||||
return nil, fmt.Errorf("failed to preprocess fixture rows from %q: %w", file, err) |
||||
} |
||||
|
||||
var sqlBuf []byte |
||||
var sqlArguments []any |
||||
for _, row := range rows { |
||||
sqlBuf = append(sqlBuf, fmt.Sprintf("INSERT INTO %s (", fixture.tableNameQuoted)...) |
||||
for k, v := range row { |
||||
sqlBuf = append(sqlBuf, f.quoteObject(k)...) |
||||
sqlBuf = append(sqlBuf, ","...) |
||||
sqlArguments = append(sqlArguments, v) |
||||
} |
||||
sqlBuf = sqlBuf[:len(sqlBuf)-1] |
||||
sqlBuf = append(sqlBuf, ") VALUES ("...) |
||||
paramIdx := 1 |
||||
for range row { |
||||
sqlBuf = append(sqlBuf, f.paramPlaceholder(paramIdx)...) |
||||
sqlBuf = append(sqlBuf, ',') |
||||
paramIdx++ |
||||
} |
||||
sqlBuf[len(sqlBuf)-1] = ')' |
||||
fixture.sqlInserts = append(fixture.sqlInserts, string(sqlBuf)) |
||||
fixture.sqlInsertArgs = append(fixture.sqlInsertArgs, slices.Clone(sqlArguments)) |
||||
sqlBuf = sqlBuf[:0] |
||||
sqlArguments = sqlArguments[:0] |
||||
} |
||||
return fixture, nil |
||||
} |
||||
|
||||
func (f *fixturesLoaderInternal) loadFixtures(tx *sql.Tx, file string) (err error) { |
||||
fixture := f.fixtures[file] |
||||
if fixture == nil { |
||||
if fixture, err = f.prepareFixtureItem(file); err != nil { |
||||
return err |
||||
} |
||||
f.fixtures[file] = fixture |
||||
} |
||||
|
||||
_, err = tx.Exec(fmt.Sprintf("DELETE FROM %s", fixture.tableNameQuoted)) // sqlite3 doesn't support truncate
|
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if fixture.mssqlHasIdentityColumn { |
||||
_, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s ON", fixture.tableNameQuoted)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer func() { _, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s OFF", fixture.tableNameQuoted)) }() |
||||
} |
||||
for i := range fixture.sqlInserts { |
||||
_, err = tx.Exec(fixture.sqlInserts[i], fixture.sqlInsertArgs[i]...) |
||||
} |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (f *fixturesLoaderInternal) Load() error { |
||||
tx, err := f.db.Begin() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer func() { _ = tx.Rollback() }() |
||||
|
||||
for _, file := range f.files { |
||||
if err := f.loadFixtures(tx, file); err != nil { |
||||
return fmt.Errorf("failed to load fixtures from %s: %w", file, err) |
||||
} |
||||
} |
||||
return tx.Commit() |
||||
} |
||||
|
||||
func FixturesFileFullPaths(dir string, files []string) ([]string, error) { |
||||
if files != nil && len(files) == 0 { |
||||
return nil, nil // load nothing
|
||||
} |
||||
files = slices.Clone(files) |
||||
if len(files) == 0 { |
||||
entries, err := os.ReadDir(dir) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
for _, e := range entries { |
||||
files = append(files, e.Name()) |
||||
} |
||||
} |
||||
for i, file := range files { |
||||
if !filepath.IsAbs(file) { |
||||
files[i] = filepath.Join(dir, file) |
||||
} |
||||
} |
||||
return files, nil |
||||
} |
||||
|
||||
func NewFixturesLoader(x *xorm.Engine, opts FixturesOptions) (FixturesLoader, error) { |
||||
files, err := FixturesFileFullPaths(opts.Dir, opts.Files) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("failed to get fixtures files: %w", err) |
||||
} |
||||
f := &fixturesLoaderInternal{db: x.DB().DB, dbType: x.Dialect().URI().DBType, files: files, fixtures: map[string]*fixtureItem{}} |
||||
switch f.dbType { |
||||
case schemas.SQLITE: |
||||
f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) } |
||||
f.paramPlaceholder = func(idx int) string { return "?" } |
||||
case schemas.POSTGRES: |
||||
f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) } |
||||
f.paramPlaceholder = func(idx int) string { return fmt.Sprintf(`$%d`, idx) } |
||||
case schemas.MYSQL: |
||||
f.quoteObject = func(s string) string { return fmt.Sprintf("`%s`", s) } |
||||
f.paramPlaceholder = func(idx int) string { return "?" } |
||||
case schemas.MSSQL: |
||||
f.quoteObject = func(s string) string { return fmt.Sprintf("[%s]", s) } |
||||
f.paramPlaceholder = func(idx int) string { return "?" } |
||||
} |
||||
return f, nil |
||||
} |
||||
@ -0,0 +1,114 @@ |
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package unittest_test |
||||
|
||||
import ( |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/models/unittest" |
||||
user_model "code.gitea.io/gitea/models/user" |
||||
"code.gitea.io/gitea/modules/test" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
"xorm.io/xorm" |
||||
) |
||||
|
||||
var NewFixturesLoaderVendor = func(e *xorm.Engine, opts unittest.FixturesOptions) (unittest.FixturesLoader, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
/* |
||||
// the old code is kept here in case we are still interested in benchmarking the two implementations
|
||||
func init() { |
||||
NewFixturesLoaderVendor = func(e *xorm.Engine, opts unittest.FixturesOptions) (unittest.FixturesLoader, error) { |
||||
return NewFixturesLoaderVendorGoTestfixtures(e, opts) |
||||
} |
||||
} |
||||
|
||||
func NewFixturesLoaderVendorGoTestfixtures(e *xorm.Engine, opts unittest.FixturesOptions) (*testfixtures.Loader, error) { |
||||
files, err := unittest.FixturesFileFullPaths(opts.Dir, opts.Files) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("failed to get fixtures files: %w", err) |
||||
} |
||||
var dialect string |
||||
switch e.Dialect().URI().DBType { |
||||
case schemas.POSTGRES: |
||||
dialect = "postgres" |
||||
case schemas.MYSQL: |
||||
dialect = "mysql" |
||||
case schemas.MSSQL: |
||||
dialect = "mssql" |
||||
case schemas.SQLITE: |
||||
dialect = "sqlite3" |
||||
default: |
||||
return nil, fmt.Errorf("unsupported RDBMS for integration tests: %q", e.Dialect().URI().DBType) |
||||
} |
||||
loaderOptions := []func(loader *testfixtures.Loader) error{ |
||||
testfixtures.Database(e.DB().DB), |
||||
testfixtures.Dialect(dialect), |
||||
testfixtures.DangerousSkipTestDatabaseCheck(), |
||||
testfixtures.Files(files...), |
||||
} |
||||
if e.Dialect().URI().DBType == schemas.POSTGRES { |
||||
loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences()) |
||||
} |
||||
return testfixtures.New(loaderOptions...) |
||||
} |
||||
*/ |
||||
|
||||
func prepareTestFixturesLoaders(t testing.TB) unittest.FixturesOptions { |
||||
_ = user_model.User{} |
||||
opts := unittest.FixturesOptions{Dir: filepath.Join(test.SetupGiteaRoot(), "models", "fixtures"), Files: []string{ |
||||
"user.yml", |
||||
}} |
||||
require.NoError(t, unittest.CreateTestEngine(opts)) |
||||
return opts |
||||
} |
||||
|
||||
func TestFixturesLoader(t *testing.T) { |
||||
opts := prepareTestFixturesLoaders(t) |
||||
loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts) |
||||
require.NoError(t, err) |
||||
loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts) |
||||
require.NoError(t, err) |
||||
t.Run("Internal", func(t *testing.T) { |
||||
require.NoError(t, loaderInternal.Load()) |
||||
require.NoError(t, loaderInternal.Load()) |
||||
}) |
||||
t.Run("Vendor", func(t *testing.T) { |
||||
if loaderVendor == nil { |
||||
t.Skip() |
||||
} |
||||
require.NoError(t, loaderVendor.Load()) |
||||
require.NoError(t, loaderVendor.Load()) |
||||
}) |
||||
} |
||||
|
||||
func BenchmarkFixturesLoader(b *testing.B) { |
||||
opts := prepareTestFixturesLoaders(b) |
||||
require.NoError(b, unittest.CreateTestEngine(opts)) |
||||
loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts) |
||||
require.NoError(b, err) |
||||
loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts) |
||||
require.NoError(b, err) |
||||
|
||||
// BenchmarkFixturesLoader/Vendor
|
||||
// BenchmarkFixturesLoader/Vendor-12 1696 719416 ns/op
|
||||
// BenchmarkFixturesLoader/Internal
|
||||
// BenchmarkFixturesLoader/Internal-12 1746 670457 ns/op
|
||||
b.Run("Internal", func(b *testing.B) { |
||||
for i := 0; i < b.N; i++ { |
||||
require.NoError(b, loaderInternal.Load()) |
||||
} |
||||
}) |
||||
b.Run("Vendor", func(b *testing.B) { |
||||
if loaderVendor == nil { |
||||
b.Skip() |
||||
} |
||||
for i := 0; i < b.N; i++ { |
||||
require.NoError(b, loaderVendor.Load()) |
||||
} |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue