mirror of https://github.com/go-gitea/gitea.git
Refactor fixture loading for testing (#33024)
To help binary size and testing performancepull/26402/head^2
wxiaoguang
1 month ago
committed by
GitHub
parent
f4ccbd38dc
commit
dafadf0028
File diff suppressed because one or more lines are too long
@ -0,0 +1,201 @@
|
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package unittest
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"xorm.io/xorm"
|
||||
"xorm.io/xorm/schemas"
|
||||
)
|
||||
|
||||
type fixtureItem struct {
|
||||
tableName string
|
||||
tableNameQuoted string
|
||||
sqlInserts []string
|
||||
sqlInsertArgs [][]any
|
||||
|
||||
mssqlHasIdentityColumn bool
|
||||
}
|
||||
|
||||
type fixturesLoaderInternal struct {
|
||||
db *sql.DB
|
||||
dbType schemas.DBType
|
||||
files []string
|
||||
fixtures map[string]*fixtureItem
|
||||
quoteObject func(string) string
|
||||
paramPlaceholder func(idx int) string
|
||||
}
|
||||
|
||||
func (f *fixturesLoaderInternal) mssqlTableHasIdentityColumn(db *sql.DB, tableName string) (bool, error) {
|
||||
row := db.QueryRow(`SELECT COUNT(*) FROM sys.identity_columns WHERE OBJECT_ID = OBJECT_ID(?)`, tableName)
|
||||
var count int
|
||||
if err := row.Scan(&count); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func (f *fixturesLoaderInternal) preprocessFixtureRow(row []map[string]any) (err error) {
|
||||
for _, m := range row {
|
||||
for k, v := range m {
|
||||
if s, ok := v.(string); ok {
|
||||
if strings.HasPrefix(s, "0x") {
|
||||
if m[k], err = hex.DecodeString(s[2:]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fixturesLoaderInternal) prepareFixtureItem(file string) (_ *fixtureItem, err error) {
|
||||
fixture := &fixtureItem{}
|
||||
fixture.tableName, _, _ = strings.Cut(filepath.Base(file), ".")
|
||||
fixture.tableNameQuoted = f.quoteObject(fixture.tableName)
|
||||
|
||||
if f.dbType == schemas.MSSQL {
|
||||
fixture.mssqlHasIdentityColumn, err = f.mssqlTableHasIdentityColumn(f.db, fixture.tableName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file %q: %w", file, err)
|
||||
}
|
||||
|
||||
var rows []map[string]any
|
||||
if err = yaml.Unmarshal(data, &rows); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal yaml data from %q: %w", file, err)
|
||||
}
|
||||
if err = f.preprocessFixtureRow(rows); err != nil {
|
||||
return nil, fmt.Errorf("failed to preprocess fixture rows from %q: %w", file, err)
|
||||
}
|
||||
|
||||
var sqlBuf []byte
|
||||
var sqlArguments []any
|
||||
for _, row := range rows {
|
||||
sqlBuf = append(sqlBuf, fmt.Sprintf("INSERT INTO %s (", fixture.tableNameQuoted)...)
|
||||
for k, v := range row {
|
||||
sqlBuf = append(sqlBuf, f.quoteObject(k)...)
|
||||
sqlBuf = append(sqlBuf, ","...)
|
||||
sqlArguments = append(sqlArguments, v)
|
||||
}
|
||||
sqlBuf = sqlBuf[:len(sqlBuf)-1]
|
||||
sqlBuf = append(sqlBuf, ") VALUES ("...)
|
||||
paramIdx := 1
|
||||
for range row {
|
||||
sqlBuf = append(sqlBuf, f.paramPlaceholder(paramIdx)...)
|
||||
sqlBuf = append(sqlBuf, ',')
|
||||
paramIdx++
|
||||
}
|
||||
sqlBuf[len(sqlBuf)-1] = ')'
|
||||
fixture.sqlInserts = append(fixture.sqlInserts, string(sqlBuf))
|
||||
fixture.sqlInsertArgs = append(fixture.sqlInsertArgs, slices.Clone(sqlArguments))
|
||||
sqlBuf = sqlBuf[:0]
|
||||
sqlArguments = sqlArguments[:0]
|
||||
}
|
||||
return fixture, nil
|
||||
}
|
||||
|
||||
func (f *fixturesLoaderInternal) loadFixtures(tx *sql.Tx, file string) (err error) {
|
||||
fixture := f.fixtures[file]
|
||||
if fixture == nil {
|
||||
if fixture, err = f.prepareFixtureItem(file); err != nil {
|
||||
return err
|
||||
}
|
||||
f.fixtures[file] = fixture
|
||||
}
|
||||
|
||||
_, err = tx.Exec(fmt.Sprintf("DELETE FROM %s", fixture.tableNameQuoted)) // sqlite3 doesn't support truncate
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if fixture.mssqlHasIdentityColumn {
|
||||
_, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s ON", fixture.tableNameQuoted))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s OFF", fixture.tableNameQuoted)) }()
|
||||
}
|
||||
for i := range fixture.sqlInserts {
|
||||
_, err = tx.Exec(fixture.sqlInserts[i], fixture.sqlInsertArgs[i]...)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fixturesLoaderInternal) Load() error {
|
||||
tx, err := f.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
for _, file := range f.files {
|
||||
if err := f.loadFixtures(tx, file); err != nil {
|
||||
return fmt.Errorf("failed to load fixtures from %s: %w", file, err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func FixturesFileFullPaths(dir string, files []string) ([]string, error) {
|
||||
if files != nil && len(files) == 0 {
|
||||
return nil, nil // load nothing
|
||||
}
|
||||
files = slices.Clone(files)
|
||||
if len(files) == 0 {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
files = append(files, e.Name())
|
||||
}
|
||||
}
|
||||
for i, file := range files {
|
||||
if !filepath.IsAbs(file) {
|
||||
files[i] = filepath.Join(dir, file)
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func NewFixturesLoader(x *xorm.Engine, opts FixturesOptions) (FixturesLoader, error) {
|
||||
files, err := FixturesFileFullPaths(opts.Dir, opts.Files)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get fixtures files: %w", err)
|
||||
}
|
||||
f := &fixturesLoaderInternal{db: x.DB().DB, dbType: x.Dialect().URI().DBType, files: files, fixtures: map[string]*fixtureItem{}}
|
||||
switch f.dbType {
|
||||
case schemas.SQLITE:
|
||||
f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) }
|
||||
f.paramPlaceholder = func(idx int) string { return "?" }
|
||||
case schemas.POSTGRES:
|
||||
f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) }
|
||||
f.paramPlaceholder = func(idx int) string { return fmt.Sprintf(`$%d`, idx) }
|
||||
case schemas.MYSQL:
|
||||
f.quoteObject = func(s string) string { return fmt.Sprintf("`%s`", s) }
|
||||
f.paramPlaceholder = func(idx int) string { return "?" }
|
||||
case schemas.MSSQL:
|
||||
f.quoteObject = func(s string) string { return fmt.Sprintf("[%s]", s) }
|
||||
f.paramPlaceholder = func(idx int) string { return "?" }
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
@ -0,0 +1,114 @@
|
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package unittest_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/test"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
var NewFixturesLoaderVendor = func(e *xorm.Engine, opts unittest.FixturesOptions) (unittest.FixturesLoader, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
/*
|
||||
// the old code is kept here in case we are still interested in benchmarking the two implementations
|
||||
func init() {
|
||||
NewFixturesLoaderVendor = func(e *xorm.Engine, opts unittest.FixturesOptions) (unittest.FixturesLoader, error) {
|
||||
return NewFixturesLoaderVendorGoTestfixtures(e, opts)
|
||||
}
|
||||
}
|
||||
|
||||
func NewFixturesLoaderVendorGoTestfixtures(e *xorm.Engine, opts unittest.FixturesOptions) (*testfixtures.Loader, error) {
|
||||
files, err := unittest.FixturesFileFullPaths(opts.Dir, opts.Files)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get fixtures files: %w", err)
|
||||
}
|
||||
var dialect string
|
||||
switch e.Dialect().URI().DBType {
|
||||
case schemas.POSTGRES:
|
||||
dialect = "postgres"
|
||||
case schemas.MYSQL:
|
||||
dialect = "mysql"
|
||||
case schemas.MSSQL:
|
||||
dialect = "mssql"
|
||||
case schemas.SQLITE:
|
||||
dialect = "sqlite3"
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported RDBMS for integration tests: %q", e.Dialect().URI().DBType)
|
||||
}
|
||||
loaderOptions := []func(loader *testfixtures.Loader) error{
|
||||
testfixtures.Database(e.DB().DB),
|
||||
testfixtures.Dialect(dialect),
|
||||
testfixtures.DangerousSkipTestDatabaseCheck(),
|
||||
testfixtures.Files(files...),
|
||||
}
|
||||
if e.Dialect().URI().DBType == schemas.POSTGRES {
|
||||
loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
|
||||
}
|
||||
return testfixtures.New(loaderOptions...)
|
||||
}
|
||||
*/
|
||||
|
||||
func prepareTestFixturesLoaders(t testing.TB) unittest.FixturesOptions {
|
||||
_ = user_model.User{}
|
||||
opts := unittest.FixturesOptions{Dir: filepath.Join(test.SetupGiteaRoot(), "models", "fixtures"), Files: []string{
|
||||
"user.yml",
|
||||
}}
|
||||
require.NoError(t, unittest.CreateTestEngine(opts))
|
||||
return opts
|
||||
}
|
||||
|
||||
func TestFixturesLoader(t *testing.T) {
|
||||
opts := prepareTestFixturesLoaders(t)
|
||||
loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts)
|
||||
require.NoError(t, err)
|
||||
loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts)
|
||||
require.NoError(t, err)
|
||||
t.Run("Internal", func(t *testing.T) {
|
||||
require.NoError(t, loaderInternal.Load())
|
||||
require.NoError(t, loaderInternal.Load())
|
||||
})
|
||||
t.Run("Vendor", func(t *testing.T) {
|
||||
if loaderVendor == nil {
|
||||
t.Skip()
|
||||
}
|
||||
require.NoError(t, loaderVendor.Load())
|
||||
require.NoError(t, loaderVendor.Load())
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkFixturesLoader(b *testing.B) {
|
||||
opts := prepareTestFixturesLoaders(b)
|
||||
require.NoError(b, unittest.CreateTestEngine(opts))
|
||||
loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts)
|
||||
require.NoError(b, err)
|
||||
loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts)
|
||||
require.NoError(b, err)
|
||||
|
||||
// BenchmarkFixturesLoader/Vendor
|
||||
// BenchmarkFixturesLoader/Vendor-12 1696 719416 ns/op
|
||||
// BenchmarkFixturesLoader/Internal
|
||||
// BenchmarkFixturesLoader/Internal-12 1746 670457 ns/op
|
||||
b.Run("Internal", func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
require.NoError(b, loaderInternal.Load())
|
||||
}
|
||||
})
|
||||
b.Run("Vendor", func(b *testing.B) {
|
||||
if loaderVendor == nil {
|
||||
b.Skip()
|
||||
}
|
||||
for i := 0; i < b.N; i++ {
|
||||
require.NoError(b, loaderVendor.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue