mirror of https://github.com/go-gitea/gitea
Rewrite queue (#24505)
# ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: pull/24588/head^2
wxiaoguang
2 years ago
committed by
GitHub
parent
cb700aedd1
commit
6f9c278559
@ -1,180 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package base |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"os" |
||||
"runtime" |
||||
"strings" |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/json" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/queue" |
||||
) |
||||
|
||||
var ( |
||||
prefix string |
||||
slowTest = 10 * time.Second |
||||
slowFlush = 5 * time.Second |
||||
) |
||||
|
||||
// TestLogger is a logger which will write to the testing log
|
||||
type TestLogger struct { |
||||
log.WriterLogger |
||||
} |
||||
|
||||
var writerCloser = &testLoggerWriterCloser{} |
||||
|
||||
type testLoggerWriterCloser struct { |
||||
sync.RWMutex |
||||
t []*testing.TB |
||||
} |
||||
|
||||
func (w *testLoggerWriterCloser) setT(t *testing.TB) { |
||||
w.Lock() |
||||
w.t = append(w.t, t) |
||||
w.Unlock() |
||||
} |
||||
|
||||
func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { |
||||
w.RLock() |
||||
var t *testing.TB |
||||
if len(w.t) > 0 { |
||||
t = w.t[len(w.t)-1] |
||||
} |
||||
w.RUnlock() |
||||
if t != nil && *t != nil { |
||||
if len(p) > 0 && p[len(p)-1] == '\n' { |
||||
p = p[:len(p)-1] |
||||
} |
||||
|
||||
defer func() { |
||||
err := recover() |
||||
if err == nil { |
||||
return |
||||
} |
||||
var errString string |
||||
errErr, ok := err.(error) |
||||
if ok { |
||||
errString = errErr.Error() |
||||
} else { |
||||
errString, ok = err.(string) |
||||
} |
||||
if !ok { |
||||
panic(err) |
||||
} |
||||
if !strings.HasPrefix(errString, "Log in goroutine after ") { |
||||
panic(err) |
||||
} |
||||
}() |
||||
|
||||
(*t).Log(string(p)) |
||||
return len(p), nil |
||||
} |
||||
return len(p), nil |
||||
} |
||||
|
||||
func (w *testLoggerWriterCloser) Close() error { |
||||
w.Lock() |
||||
if len(w.t) > 0 { |
||||
w.t = w.t[:len(w.t)-1] |
||||
} |
||||
w.Unlock() |
||||
return nil |
||||
} |
||||
|
||||
// PrintCurrentTest prints the current test to os.Stdout
|
||||
func PrintCurrentTest(t testing.TB, skip ...int) func() { |
||||
start := time.Now() |
||||
actualSkip := 1 |
||||
if len(skip) > 0 { |
||||
actualSkip = skip[0] |
||||
} |
||||
_, filename, line, _ := runtime.Caller(actualSkip) |
||||
|
||||
if log.CanColorStdout { |
||||
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line) |
||||
} else { |
||||
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) |
||||
} |
||||
writerCloser.setT(&t) |
||||
return func() { |
||||
took := time.Since(start) |
||||
if took > slowTest { |
||||
if log.CanColorStdout { |
||||
fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow))) |
||||
} else { |
||||
fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took) |
||||
} |
||||
} |
||||
timer := time.AfterFunc(slowFlush, func() { |
||||
if log.CanColorStdout { |
||||
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), slowFlush) |
||||
} else { |
||||
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush) |
||||
} |
||||
}) |
||||
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { |
||||
t.Errorf("Flushing queues failed with error %v", err) |
||||
} |
||||
timer.Stop() |
||||
flushTook := time.Since(start) - took |
||||
if flushTook > slowFlush { |
||||
if log.CanColorStdout { |
||||
fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed))) |
||||
} else { |
||||
fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) |
||||
} |
||||
} |
||||
_ = writerCloser.Close() |
||||
} |
||||
} |
||||
|
||||
// Printf takes a format and args and prints the string to os.Stdout
|
||||
func Printf(format string, args ...interface{}) { |
||||
if log.CanColorStdout { |
||||
for i := 0; i < len(args); i++ { |
||||
args[i] = log.NewColoredValue(args[i]) |
||||
} |
||||
} |
||||
fmt.Fprintf(os.Stdout, "\t"+format, args...) |
||||
} |
||||
|
||||
// NewTestLogger creates a TestLogger as a log.LoggerProvider
|
||||
func NewTestLogger() log.LoggerProvider { |
||||
logger := &TestLogger{} |
||||
logger.Colorize = log.CanColorStdout |
||||
logger.Level = log.TRACE |
||||
return logger |
||||
} |
||||
|
||||
// Init inits connection writer with json config.
|
||||
// json config only need key "level".
|
||||
func (log *TestLogger) Init(config string) error { |
||||
err := json.Unmarshal([]byte(config), log) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
log.NewWriterLogger(writerCloser) |
||||
return nil |
||||
} |
||||
|
||||
// Flush when log should be flushed
|
||||
func (log *TestLogger) Flush() { |
||||
} |
||||
|
||||
// ReleaseReopen does nothing
|
||||
func (log *TestLogger) ReleaseReopen() error { |
||||
return nil |
||||
} |
||||
|
||||
// GetName returns the default name for this implementation
|
||||
func (log *TestLogger) GetName() string { |
||||
return "test" |
||||
} |
||||
@ -0,0 +1,63 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
) |
||||
|
||||
const ( |
||||
backoffBegin = 50 * time.Millisecond |
||||
backoffUpper = 2 * time.Second |
||||
) |
||||
|
||||
type ( |
||||
backoffFuncRetErr[T any] func() (retry bool, ret T, err error) |
||||
backoffFuncErr func() (retry bool, err error) |
||||
) |
||||
|
||||
func backoffRetErr[T any](ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncRetErr[T]) (ret T, err error) { |
||||
d := begin |
||||
for { |
||||
// check whether the context has been cancelled or has reached the deadline, return early
|
||||
select { |
||||
case <-ctx.Done(): |
||||
return ret, ctx.Err() |
||||
case <-end: |
||||
return ret, context.DeadlineExceeded |
||||
default: |
||||
} |
||||
|
||||
// call the target function
|
||||
retry, ret, err := fn() |
||||
if err != nil { |
||||
return ret, err |
||||
} |
||||
if !retry { |
||||
return ret, nil |
||||
} |
||||
|
||||
// wait for a while before retrying, and also respect the context & deadline
|
||||
select { |
||||
case <-ctx.Done(): |
||||
return ret, ctx.Err() |
||||
case <-time.After(d): |
||||
d *= 2 |
||||
if d > upper { |
||||
d = upper |
||||
} |
||||
case <-end: |
||||
return ret, context.DeadlineExceeded |
||||
} |
||||
} |
||||
} |
||||
|
||||
func backoffErr(ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncErr) error { |
||||
_, err := backoffRetErr(ctx, begin, upper, end, func() (retry bool, ret any, err error) { |
||||
retry, err = fn() |
||||
return retry, nil, err |
||||
}) |
||||
return err |
||||
} |
||||
@ -0,0 +1,42 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
) |
||||
|
||||
var pushBlockTime = 5 * time.Second |
||||
|
||||
type baseQueue interface { |
||||
PushItem(ctx context.Context, data []byte) error |
||||
PopItem(ctx context.Context) ([]byte, error) |
||||
HasItem(ctx context.Context, data []byte) (bool, error) |
||||
Len(ctx context.Context) (int, error) |
||||
Close() error |
||||
RemoveAll(ctx context.Context) error |
||||
} |
||||
|
||||
func popItemByChan(ctx context.Context, popItemFn func(ctx context.Context) ([]byte, error)) (chanItem chan []byte, chanErr chan error) { |
||||
chanItem = make(chan []byte) |
||||
chanErr = make(chan error) |
||||
go func() { |
||||
for { |
||||
it, err := popItemFn(ctx) |
||||
if err != nil { |
||||
close(chanItem) |
||||
chanErr <- err |
||||
return |
||||
} |
||||
if it == nil { |
||||
close(chanItem) |
||||
close(chanErr) |
||||
return |
||||
} |
||||
chanItem <- it |
||||
} |
||||
}() |
||||
return chanItem, chanErr |
||||
} |
||||
@ -0,0 +1,123 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/container" |
||||
) |
||||
|
||||
var errChannelClosed = errors.New("channel is closed") |
||||
|
||||
type baseChannel struct { |
||||
c chan []byte |
||||
set container.Set[string] |
||||
mu sync.Mutex |
||||
|
||||
isUnique bool |
||||
} |
||||
|
||||
var _ baseQueue = (*baseChannel)(nil) |
||||
|
||||
func newBaseChannelGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { |
||||
q := &baseChannel{c: make(chan []byte, cfg.Length), isUnique: unique} |
||||
if unique { |
||||
q.set = container.Set[string]{} |
||||
} |
||||
return q, nil |
||||
} |
||||
|
||||
func newBaseChannelSimple(cfg *BaseConfig) (baseQueue, error) { |
||||
return newBaseChannelGeneric(cfg, false) |
||||
} |
||||
|
||||
func newBaseChannelUnique(cfg *BaseConfig) (baseQueue, error) { |
||||
return newBaseChannelGeneric(cfg, true) |
||||
} |
||||
|
||||
func (q *baseChannel) PushItem(ctx context.Context, data []byte) error { |
||||
if q.c == nil { |
||||
return errChannelClosed |
||||
} |
||||
|
||||
if q.isUnique { |
||||
q.mu.Lock() |
||||
has := q.set.Contains(string(data)) |
||||
q.mu.Unlock() |
||||
if has { |
||||
return ErrAlreadyInQueue |
||||
} |
||||
} |
||||
|
||||
select { |
||||
case q.c <- data: |
||||
if q.isUnique { |
||||
q.mu.Lock() |
||||
q.set.Add(string(data)) |
||||
q.mu.Unlock() |
||||
} |
||||
return nil |
||||
case <-time.After(pushBlockTime): |
||||
return context.DeadlineExceeded |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
} |
||||
} |
||||
|
||||
func (q *baseChannel) PopItem(ctx context.Context) ([]byte, error) { |
||||
select { |
||||
case data, ok := <-q.c: |
||||
if !ok { |
||||
return nil, errChannelClosed |
||||
} |
||||
q.mu.Lock() |
||||
q.set.Remove(string(data)) |
||||
q.mu.Unlock() |
||||
return data, nil |
||||
case <-ctx.Done(): |
||||
return nil, ctx.Err() |
||||
} |
||||
} |
||||
|
||||
func (q *baseChannel) HasItem(ctx context.Context, data []byte) (bool, error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
return q.set.Contains(string(data)), nil |
||||
} |
||||
|
||||
func (q *baseChannel) Len(ctx context.Context) (int, error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
if q.c == nil { |
||||
return 0, errChannelClosed |
||||
} |
||||
|
||||
return len(q.c), nil |
||||
} |
||||
|
||||
func (q *baseChannel) Close() error { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
close(q.c) |
||||
q.set = container.Set[string]{} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (q *baseChannel) RemoveAll(ctx context.Context) error { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
for q.c != nil && len(q.c) > 0 { |
||||
<-q.c |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,11 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import "testing" |
||||
|
||||
func TestBaseChannel(t *testing.T) { |
||||
testQueueBasic(t, newBaseChannelSimple, &BaseConfig{ManagedName: "baseChannel", Length: 10}, false) |
||||
testQueueBasic(t, newBaseChannelUnique, &BaseConfig{ManagedName: "baseChannel", Length: 10}, true) |
||||
} |
||||
@ -0,0 +1,38 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import "context" |
||||
|
||||
type baseDummy struct{} |
||||
|
||||
var _ baseQueue = (*baseDummy)(nil) |
||||
|
||||
func newBaseDummy(cfg *BaseConfig, unique bool) (baseQueue, error) { |
||||
return &baseDummy{}, nil |
||||
} |
||||
|
||||
func (q *baseDummy) PushItem(ctx context.Context, data []byte) error { |
||||
return nil |
||||
} |
||||
|
||||
func (q *baseDummy) PopItem(ctx context.Context) ([]byte, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (q *baseDummy) Len(ctx context.Context) (int, error) { |
||||
return 0, nil |
||||
} |
||||
|
||||
func (q *baseDummy) HasItem(ctx context.Context, data []byte) (bool, error) { |
||||
return false, nil |
||||
} |
||||
|
||||
func (q *baseDummy) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
func (q *baseDummy) RemoveAll(ctx context.Context) error { |
||||
return nil |
||||
} |
||||
@ -0,0 +1,72 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"gitea.com/lunny/levelqueue" |
||||
) |
||||
|
||||
type baseLevelQueue struct { |
||||
internal *levelqueue.Queue |
||||
conn string |
||||
cfg *BaseConfig |
||||
} |
||||
|
||||
var _ baseQueue = (*baseLevelQueue)(nil) |
||||
|
||||
func newBaseLevelQueueGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { |
||||
if unique { |
||||
return newBaseLevelQueueUnique(cfg) |
||||
} |
||||
return newBaseLevelQueueSimple(cfg) |
||||
} |
||||
|
||||
func newBaseLevelQueueSimple(cfg *BaseConfig) (baseQueue, error) { |
||||
conn, db, err := prepareLevelDB(cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
q := &baseLevelQueue{conn: conn, cfg: cfg} |
||||
q.internal, err = levelqueue.NewQueue(db, []byte(cfg.QueueFullName), false) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return q, nil |
||||
} |
||||
|
||||
func (q *baseLevelQueue) PushItem(ctx context.Context, data []byte) error { |
||||
return baseLevelQueueCommon(q.cfg, q.internal, nil).PushItem(ctx, data) |
||||
} |
||||
|
||||
func (q *baseLevelQueue) PopItem(ctx context.Context) ([]byte, error) { |
||||
return baseLevelQueueCommon(q.cfg, q.internal, nil).PopItem(ctx) |
||||
} |
||||
|
||||
func (q *baseLevelQueue) HasItem(ctx context.Context, data []byte) (bool, error) { |
||||
return false, nil |
||||
} |
||||
|
||||
func (q *baseLevelQueue) Len(ctx context.Context) (int, error) { |
||||
return int(q.internal.Len()), nil |
||||
} |
||||
|
||||
func (q *baseLevelQueue) Close() error { |
||||
err := q.internal.Close() |
||||
_ = nosql.GetManager().CloseLevelDB(q.conn) |
||||
return err |
||||
} |
||||
|
||||
func (q *baseLevelQueue) RemoveAll(ctx context.Context) error { |
||||
for q.internal.Len() > 0 { |
||||
if _, err := q.internal.LPop(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,92 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"path/filepath" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"gitea.com/lunny/levelqueue" |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
) |
||||
|
||||
type baseLevelQueuePushPoper interface { |
||||
RPush(data []byte) error |
||||
LPop() ([]byte, error) |
||||
Len() int64 |
||||
} |
||||
|
||||
type baseLevelQueueCommonImpl struct { |
||||
length int |
||||
internal baseLevelQueuePushPoper |
||||
mu *sync.Mutex |
||||
} |
||||
|
||||
func (q *baseLevelQueueCommonImpl) PushItem(ctx context.Context, data []byte) error { |
||||
return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) { |
||||
if q.mu != nil { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
} |
||||
|
||||
cnt := int(q.internal.Len()) |
||||
if cnt >= q.length { |
||||
return true, nil |
||||
} |
||||
retry, err = false, q.internal.RPush(data) |
||||
if err == levelqueue.ErrAlreadyInQueue { |
||||
err = ErrAlreadyInQueue |
||||
} |
||||
return retry, err |
||||
}) |
||||
} |
||||
|
||||
func (q *baseLevelQueueCommonImpl) PopItem(ctx context.Context) ([]byte, error) { |
||||
return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) { |
||||
if q.mu != nil { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
} |
||||
|
||||
data, err = q.internal.LPop() |
||||
if err == levelqueue.ErrNotFound { |
||||
return true, nil, nil |
||||
} |
||||
if err != nil { |
||||
return false, nil, err |
||||
} |
||||
return false, data, nil |
||||
}) |
||||
} |
||||
|
||||
func baseLevelQueueCommon(cfg *BaseConfig, internal baseLevelQueuePushPoper, mu *sync.Mutex) *baseLevelQueueCommonImpl { |
||||
return &baseLevelQueueCommonImpl{length: cfg.Length, internal: internal} |
||||
} |
||||
|
||||
func prepareLevelDB(cfg *BaseConfig) (conn string, db *leveldb.DB, err error) { |
||||
if cfg.ConnStr == "" { // use data dir as conn str
|
||||
if !filepath.IsAbs(cfg.DataFullDir) { |
||||
return "", nil, fmt.Errorf("invalid leveldb data dir (not absolute): %q", cfg.DataFullDir) |
||||
} |
||||
conn = cfg.DataFullDir |
||||
} else { |
||||
if !strings.HasPrefix(cfg.ConnStr, "leveldb://") { |
||||
return "", nil, fmt.Errorf("invalid leveldb connection string: %q", cfg.ConnStr) |
||||
} |
||||
conn = cfg.ConnStr |
||||
} |
||||
for i := 0; i < 10; i++ { |
||||
if db, err = nosql.GetManager().GetLevelDB(conn); err == nil { |
||||
break |
||||
} |
||||
time.Sleep(1 * time.Second) |
||||
} |
||||
return conn, db, err |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/modules/setting" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestBaseLevelDB(t *testing.T) { |
||||
_, err := newBaseLevelQueueGeneric(&BaseConfig{ConnStr: "redis://"}, false) |
||||
assert.ErrorContains(t, err, "invalid leveldb connection string") |
||||
|
||||
_, err = newBaseLevelQueueGeneric(&BaseConfig{DataFullDir: "relative"}, false) |
||||
assert.ErrorContains(t, err, "invalid leveldb data dir") |
||||
|
||||
testQueueBasic(t, newBaseLevelQueueSimple, toBaseConfig("baseLevelQueue", setting.QueueSettings{Datadir: t.TempDir() + "/queue-test", Length: 10}), false) |
||||
testQueueBasic(t, newBaseLevelQueueUnique, toBaseConfig("baseLevelQueueUnique", setting.QueueSettings{ConnStr: "leveldb://" + t.TempDir() + "/queue-test", Length: 10}), true) |
||||
} |
||||
@ -0,0 +1,93 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
"unsafe" |
||||
|
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"gitea.com/lunny/levelqueue" |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
) |
||||
|
||||
type baseLevelQueueUnique struct { |
||||
internal *levelqueue.UniqueQueue |
||||
conn string |
||||
cfg *BaseConfig |
||||
|
||||
mu sync.Mutex // the levelqueue.UniqueQueue is not thread-safe, there is no mutex protecting the underlying queue&set together
|
||||
} |
||||
|
||||
var _ baseQueue = (*baseLevelQueueUnique)(nil) |
||||
|
||||
func newBaseLevelQueueUnique(cfg *BaseConfig) (baseQueue, error) { |
||||
conn, db, err := prepareLevelDB(cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
q := &baseLevelQueueUnique{conn: conn, cfg: cfg} |
||||
q.internal, err = levelqueue.NewUniqueQueue(db, []byte(cfg.QueueFullName), []byte(cfg.SetFullName), false) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return q, nil |
||||
} |
||||
|
||||
func (q *baseLevelQueueUnique) PushItem(ctx context.Context, data []byte) error { |
||||
return baseLevelQueueCommon(q.cfg, q.internal, &q.mu).PushItem(ctx, data) |
||||
} |
||||
|
||||
func (q *baseLevelQueueUnique) PopItem(ctx context.Context) ([]byte, error) { |
||||
return baseLevelQueueCommon(q.cfg, q.internal, &q.mu).PopItem(ctx) |
||||
} |
||||
|
||||
func (q *baseLevelQueueUnique) HasItem(ctx context.Context, data []byte) (bool, error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
return q.internal.Has(data) |
||||
} |
||||
|
||||
func (q *baseLevelQueueUnique) Len(ctx context.Context) (int, error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
return int(q.internal.Len()), nil |
||||
} |
||||
|
||||
func (q *baseLevelQueueUnique) Close() error { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
err := q.internal.Close() |
||||
_ = nosql.GetManager().CloseLevelDB(q.conn) |
||||
return err |
||||
} |
||||
|
||||
func (q *baseLevelQueueUnique) RemoveAll(ctx context.Context) error { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
type levelUniqueQueue struct { |
||||
q *levelqueue.Queue |
||||
set *levelqueue.Set |
||||
db *leveldb.DB |
||||
} |
||||
lq := (*levelUniqueQueue)(unsafe.Pointer(q.internal)) |
||||
|
||||
members, err := lq.set.Members() |
||||
if err != nil { |
||||
return err // seriously corrupted
|
||||
} |
||||
for _, v := range members { |
||||
_, _ = lq.set.Remove(v) |
||||
} |
||||
for lq.q.Len() > 0 { |
||||
if _, err = lq.q.LPop(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,135 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/graceful" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"github.com/redis/go-redis/v9" |
||||
) |
||||
|
||||
type baseRedis struct { |
||||
client redis.UniversalClient |
||||
isUnique bool |
||||
cfg *BaseConfig |
||||
|
||||
mu sync.Mutex // the old implementation is not thread-safe, the queue operation and set operation should be protected together
|
||||
} |
||||
|
||||
var _ baseQueue = (*baseRedis)(nil) |
||||
|
||||
func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { |
||||
client := nosql.GetManager().GetRedisClient(cfg.ConnStr) |
||||
|
||||
var err error |
||||
for i := 0; i < 10; i++ { |
||||
err = client.Ping(graceful.GetManager().ShutdownContext()).Err() |
||||
if err == nil { |
||||
break |
||||
} |
||||
log.Warn("Redis is not ready, waiting for 1 second to retry: %v", err) |
||||
time.Sleep(time.Second) |
||||
} |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &baseRedis{cfg: cfg, client: client, isUnique: unique}, nil |
||||
} |
||||
|
||||
func newBaseRedisSimple(cfg *BaseConfig) (baseQueue, error) { |
||||
return newBaseRedisGeneric(cfg, false) |
||||
} |
||||
|
||||
func newBaseRedisUnique(cfg *BaseConfig) (baseQueue, error) { |
||||
return newBaseRedisGeneric(cfg, true) |
||||
} |
||||
|
||||
func (q *baseRedis) PushItem(ctx context.Context, data []byte) error { |
||||
return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result() |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
if int(cnt) >= q.cfg.Length { |
||||
return true, nil |
||||
} |
||||
|
||||
if q.isUnique { |
||||
added, err := q.client.SAdd(ctx, q.cfg.SetFullName, data).Result() |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
if added == 0 { |
||||
return false, ErrAlreadyInQueue |
||||
} |
||||
} |
||||
return false, q.client.RPush(ctx, q.cfg.QueueFullName, data).Err() |
||||
}) |
||||
} |
||||
|
||||
func (q *baseRedis) PopItem(ctx context.Context) ([]byte, error) { |
||||
return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
|
||||
data, err = q.client.LPop(ctx, q.cfg.QueueFullName).Bytes() |
||||
if err == redis.Nil { |
||||
return true, nil, nil |
||||
} |
||||
if err != nil { |
||||
return true, nil, nil |
||||
} |
||||
if q.isUnique { |
||||
// the data has been popped, even if there is any error we can't do anything
|
||||
_ = q.client.SRem(ctx, q.cfg.SetFullName, data).Err() |
||||
} |
||||
return false, data, err |
||||
}) |
||||
} |
||||
|
||||
func (q *baseRedis) HasItem(ctx context.Context, data []byte) (bool, error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
if !q.isUnique { |
||||
return false, nil |
||||
} |
||||
return q.client.SIsMember(ctx, q.cfg.SetFullName, data).Result() |
||||
} |
||||
|
||||
func (q *baseRedis) Len(ctx context.Context) (int, error) { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result() |
||||
return int(cnt), err |
||||
} |
||||
|
||||
func (q *baseRedis) Close() error { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
return q.client.Close() |
||||
} |
||||
|
||||
func (q *baseRedis) RemoveAll(ctx context.Context) error { |
||||
q.mu.Lock() |
||||
defer q.mu.Unlock() |
||||
c1 := q.client.Del(ctx, q.cfg.QueueFullName) |
||||
c2 := q.client.Del(ctx, q.cfg.SetFullName) |
||||
if c1.Err() != nil { |
||||
return c1.Err() |
||||
} |
||||
if c2.Err() != nil { |
||||
return c2.Err() |
||||
} |
||||
return nil // actually, checking errors doesn't make sense here because the state could be out-of-sync
|
||||
} |
||||
@ -0,0 +1,71 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"os" |
||||
"os/exec" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/nosql" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func waitRedisReady(conn string, dur time.Duration) (ready bool) { |
||||
ctxTimed, cancel := context.WithTimeout(context.Background(), time.Second*5) |
||||
defer cancel() |
||||
for t := time.Now(); ; time.Sleep(50 * time.Millisecond) { |
||||
ret := nosql.GetManager().GetRedisClient(conn).Ping(ctxTimed) |
||||
if ret.Err() == nil { |
||||
return true |
||||
} |
||||
if time.Since(t) > dur { |
||||
return false |
||||
} |
||||
} |
||||
} |
||||
|
||||
func redisServerCmd(t *testing.T) *exec.Cmd { |
||||
redisServerProg, err := exec.LookPath("redis-server") |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
c := &exec.Cmd{ |
||||
Path: redisServerProg, |
||||
Args: []string{redisServerProg, "--bind", "127.0.0.1", "--port", "6379"}, |
||||
Dir: t.TempDir(), |
||||
Stdin: os.Stdin, |
||||
Stdout: os.Stdout, |
||||
Stderr: os.Stderr, |
||||
} |
||||
return c |
||||
} |
||||
|
||||
func TestBaseRedis(t *testing.T) { |
||||
var redisServer *exec.Cmd |
||||
defer func() { |
||||
if redisServer != nil { |
||||
_ = redisServer.Process.Signal(os.Interrupt) |
||||
_ = redisServer.Wait() |
||||
} |
||||
}() |
||||
if !waitRedisReady("redis://127.0.0.1:6379/0", 0) { |
||||
redisServer = redisServerCmd(t) |
||||
if redisServer == nil && os.Getenv("CI") != "" { |
||||
t.Skip("redis-server not found") |
||||
return |
||||
} |
||||
assert.NoError(t, redisServer.Start()) |
||||
if !assert.True(t, waitRedisReady("redis://127.0.0.1:6379/0", 5*time.Second), "start redis-server") { |
||||
return |
||||
} |
||||
} |
||||
|
||||
testQueueBasic(t, newBaseRedisSimple, toBaseConfig("baseRedis", setting.QueueSettings{Length: 10}), false) |
||||
testQueueBasic(t, newBaseRedisUnique, toBaseConfig("baseRedisUnique", setting.QueueSettings{Length: 10}), true) |
||||
} |
||||
@ -0,0 +1,140 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error), cfg *BaseConfig, isUnique bool) { |
||||
t.Run(fmt.Sprintf("testQueueBasic-%s-unique:%v", cfg.ManagedName, isUnique), func(t *testing.T) { |
||||
q, err := newFn(cfg) |
||||
assert.NoError(t, err) |
||||
|
||||
ctx := context.Background() |
||||
_ = q.RemoveAll(ctx) |
||||
cnt, err := q.Len(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 0, cnt) |
||||
|
||||
// push the first item
|
||||
err = q.PushItem(ctx, []byte("foo")) |
||||
assert.NoError(t, err) |
||||
|
||||
cnt, err = q.Len(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 1, cnt) |
||||
|
||||
// push a duplicate item
|
||||
err = q.PushItem(ctx, []byte("foo")) |
||||
if !isUnique { |
||||
assert.NoError(t, err) |
||||
} else { |
||||
assert.ErrorIs(t, err, ErrAlreadyInQueue) |
||||
} |
||||
|
||||
// check the duplicate item
|
||||
cnt, err = q.Len(ctx) |
||||
assert.NoError(t, err) |
||||
has, err := q.HasItem(ctx, []byte("foo")) |
||||
assert.NoError(t, err) |
||||
if !isUnique { |
||||
assert.EqualValues(t, 2, cnt) |
||||
assert.EqualValues(t, false, has) // non-unique queues don't check for duplicates
|
||||
} else { |
||||
assert.EqualValues(t, 1, cnt) |
||||
assert.EqualValues(t, true, has) |
||||
} |
||||
|
||||
// push another item
|
||||
err = q.PushItem(ctx, []byte("bar")) |
||||
assert.NoError(t, err) |
||||
|
||||
// pop the first item (and the duplicate if non-unique)
|
||||
it, err := q.PopItem(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, "foo", string(it)) |
||||
|
||||
if !isUnique { |
||||
it, err = q.PopItem(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, "foo", string(it)) |
||||
} |
||||
|
||||
// pop another item
|
||||
it, err = q.PopItem(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, "bar", string(it)) |
||||
|
||||
// pop an empty queue (timeout, cancel)
|
||||
ctxTimed, cancel := context.WithTimeout(ctx, 10*time.Millisecond) |
||||
it, err = q.PopItem(ctxTimed) |
||||
assert.ErrorIs(t, err, context.DeadlineExceeded) |
||||
assert.Nil(t, it) |
||||
cancel() |
||||
|
||||
ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) |
||||
cancel() |
||||
it, err = q.PopItem(ctxTimed) |
||||
assert.ErrorIs(t, err, context.Canceled) |
||||
assert.Nil(t, it) |
||||
|
||||
// test blocking push if queue is full
|
||||
for i := 0; i < cfg.Length; i++ { |
||||
err = q.PushItem(ctx, []byte(fmt.Sprintf("item-%d", i))) |
||||
assert.NoError(t, err) |
||||
} |
||||
ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) |
||||
err = q.PushItem(ctxTimed, []byte("item-full")) |
||||
assert.ErrorIs(t, err, context.DeadlineExceeded) |
||||
cancel() |
||||
|
||||
// test blocking push if queue is full (with custom pushBlockTime)
|
||||
oldPushBlockTime := pushBlockTime |
||||
timeStart := time.Now() |
||||
pushBlockTime = 30 * time.Millisecond |
||||
err = q.PushItem(ctx, []byte("item-full")) |
||||
assert.ErrorIs(t, err, context.DeadlineExceeded) |
||||
assert.True(t, time.Since(timeStart) >= pushBlockTime*2/3) |
||||
pushBlockTime = oldPushBlockTime |
||||
|
||||
// remove all
|
||||
cnt, err = q.Len(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, cfg.Length, cnt) |
||||
|
||||
_ = q.RemoveAll(ctx) |
||||
|
||||
cnt, err = q.Len(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 0, cnt) |
||||
}) |
||||
} |
||||
|
||||
func TestBaseDummy(t *testing.T) { |
||||
q, err := newBaseDummy(&BaseConfig{}, true) |
||||
assert.NoError(t, err) |
||||
|
||||
ctx := context.Background() |
||||
assert.NoError(t, q.PushItem(ctx, []byte("foo"))) |
||||
|
||||
cnt, err := q.Len(ctx) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, 0, cnt) |
||||
|
||||
has, err := q.HasItem(ctx, []byte("foo")) |
||||
assert.NoError(t, err) |
||||
assert.False(t, has) |
||||
|
||||
it, err := q.PopItem(ctx) |
||||
assert.NoError(t, err) |
||||
assert.Nil(t, it) |
||||
|
||||
assert.NoError(t, q.RemoveAll(ctx)) |
||||
} |
||||
@ -1,69 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import "context" |
||||
|
||||
// ByteFIFO defines a FIFO that takes a byte array
|
||||
type ByteFIFO interface { |
||||
// Len returns the length of the fifo
|
||||
Len(ctx context.Context) int64 |
||||
// PushFunc pushes data to the end of the fifo and calls the callback if it is added
|
||||
PushFunc(ctx context.Context, data []byte, fn func() error) error |
||||
// Pop pops data from the start of the fifo
|
||||
Pop(ctx context.Context) ([]byte, error) |
||||
// Close this fifo
|
||||
Close() error |
||||
// PushBack pushes data back to the top of the fifo
|
||||
PushBack(ctx context.Context, data []byte) error |
||||
} |
||||
|
||||
// UniqueByteFIFO defines a FIFO that Uniques its contents
|
||||
type UniqueByteFIFO interface { |
||||
ByteFIFO |
||||
// Has returns whether the fifo contains this data
|
||||
Has(ctx context.Context, data []byte) (bool, error) |
||||
} |
||||
|
||||
var _ ByteFIFO = &DummyByteFIFO{} |
||||
|
||||
// DummyByteFIFO represents a dummy fifo
|
||||
type DummyByteFIFO struct{} |
||||
|
||||
// PushFunc returns nil
|
||||
func (*DummyByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { |
||||
return nil |
||||
} |
||||
|
||||
// Pop returns nil
|
||||
func (*DummyByteFIFO) Pop(ctx context.Context) ([]byte, error) { |
||||
return []byte{}, nil |
||||
} |
||||
|
||||
// Close returns nil
|
||||
func (*DummyByteFIFO) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
// Len is always 0
|
||||
func (*DummyByteFIFO) Len(ctx context.Context) int64 { |
||||
return 0 |
||||
} |
||||
|
||||
// PushBack pushes data back to the top of the fifo
|
||||
func (*DummyByteFIFO) PushBack(ctx context.Context, data []byte) error { |
||||
return nil |
||||
} |
||||
|
||||
var _ UniqueByteFIFO = &DummyUniqueByteFIFO{} |
||||
|
||||
// DummyUniqueByteFIFO represents a dummy unique fifo
|
||||
type DummyUniqueByteFIFO struct { |
||||
DummyByteFIFO |
||||
} |
||||
|
||||
// Has always returns false
|
||||
func (*DummyUniqueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { |
||||
return false, nil |
||||
} |
||||
@ -0,0 +1,36 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"code.gitea.io/gitea/modules/setting" |
||||
) |
||||
|
||||
type BaseConfig struct { |
||||
ManagedName string |
||||
DataFullDir string // the caller must prepare an absolute path
|
||||
|
||||
ConnStr string |
||||
Length int |
||||
|
||||
QueueFullName, SetFullName string |
||||
} |
||||
|
||||
func toBaseConfig(managedName string, queueSetting setting.QueueSettings) *BaseConfig { |
||||
baseConfig := &BaseConfig{ |
||||
ManagedName: managedName, |
||||
DataFullDir: queueSetting.Datadir, |
||||
|
||||
ConnStr: queueSetting.ConnStr, |
||||
Length: queueSetting.Length, |
||||
} |
||||
|
||||
// queue name and set name
|
||||
baseConfig.QueueFullName = managedName + queueSetting.QueueName |
||||
baseConfig.SetFullName = baseConfig.QueueFullName + queueSetting.SetName |
||||
if baseConfig.SetFullName == baseConfig.QueueFullName { |
||||
baseConfig.SetFullName += "_unique" |
||||
} |
||||
return baseConfig |
||||
} |
||||
@ -1,91 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"reflect" |
||||
|
||||
"code.gitea.io/gitea/modules/json" |
||||
) |
||||
|
||||
// Mappable represents an interface that can MapTo another interface
|
||||
type Mappable interface { |
||||
MapTo(v interface{}) error |
||||
} |
||||
|
||||
// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
|
||||
//
|
||||
// It will tolerate the cfg being passed as a []byte or string of a json representation of the
|
||||
// exemplar or the correct type of the exemplar itself
|
||||
func toConfig(exemplar, cfg interface{}) (interface{}, error) { |
||||
// First of all check if we've got the same type as the exemplar - if so it's all fine.
|
||||
if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { |
||||
return cfg, nil |
||||
} |
||||
|
||||
// Now if not - does it provide a MapTo function we can try?
|
||||
if mappable, ok := cfg.(Mappable); ok { |
||||
newVal := reflect.New(reflect.TypeOf(exemplar)) |
||||
if err := mappable.MapTo(newVal.Interface()); err == nil { |
||||
return newVal.Elem().Interface(), nil |
||||
} |
||||
// MapTo has failed us ... let's try the json route ...
|
||||
} |
||||
|
||||
// OK we've been passed a byte array right?
|
||||
configBytes, ok := cfg.([]byte) |
||||
if !ok { |
||||
// oh ... it's a string then?
|
||||
var configStr string |
||||
|
||||
configStr, ok = cfg.(string) |
||||
configBytes = []byte(configStr) |
||||
} |
||||
if !ok { |
||||
// hmm ... can we marshal it to json?
|
||||
var err error |
||||
configBytes, err = json.Marshal(cfg) |
||||
ok = err == nil |
||||
} |
||||
if !ok { |
||||
// no ... we've tried hard enough at this point - throw an error!
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg} |
||||
} |
||||
|
||||
// OK unmarshal the byte array into a new copy of the exemplar
|
||||
newVal := reflect.New(reflect.TypeOf(exemplar)) |
||||
if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { |
||||
// If we can't unmarshal it then return an error!
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg, err: err} |
||||
} |
||||
return newVal.Elem().Interface(), nil |
||||
} |
||||
|
||||
// unmarshalAs will attempt to unmarshal provided bytes as the provided exemplar
|
||||
func unmarshalAs(bs []byte, exemplar interface{}) (data Data, err error) { |
||||
if exemplar != nil { |
||||
t := reflect.TypeOf(exemplar) |
||||
n := reflect.New(t) |
||||
ne := n.Elem() |
||||
err = json.Unmarshal(bs, ne.Addr().Interface()) |
||||
data = ne.Interface().(Data) |
||||
} else { |
||||
err = json.Unmarshal(bs, &data) |
||||
} |
||||
return data, err |
||||
} |
||||
|
||||
// assignableTo will check if provided data is assignable to the same type as the exemplar
|
||||
// if the provided exemplar is nil then it will always return true
|
||||
func assignableTo(data Data, exemplar interface{}) bool { |
||||
if exemplar == nil { |
||||
return true |
||||
} |
||||
|
||||
// Assert data is of same type as exemplar
|
||||
t := reflect.TypeOf(data) |
||||
exemplarType := reflect.TypeOf(exemplar) |
||||
|
||||
return t.AssignableTo(exemplarType) && data != nil |
||||
} |
||||
@ -0,0 +1,124 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/modules/setting" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestManager(t *testing.T) { |
||||
oldAppDataPath := setting.AppDataPath |
||||
setting.AppDataPath = t.TempDir() |
||||
defer func() { |
||||
setting.AppDataPath = oldAppDataPath |
||||
}() |
||||
|
||||
newQueueFromConfig := func(name, cfg string) (*WorkerPoolQueue[int], error) { |
||||
cfgProvider, err := setting.NewConfigProviderFromData(cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
qs, err := setting.GetQueueSettings(cfgProvider, name) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return NewWorkerPoolQueueBySetting(name, qs, func(s ...int) (unhandled []int) { return nil }, false) |
||||
} |
||||
|
||||
// test invalid CONN_STR
|
||||
_, err := newQueueFromConfig("default", ` |
||||
[queue] |
||||
DATADIR = temp-dir |
||||
CONN_STR = redis://
|
||||
`) |
||||
assert.ErrorContains(t, err, "invalid leveldb connection string") |
||||
|
||||
// test default config
|
||||
q, err := newQueueFromConfig("default", "") |
||||
assert.NoError(t, err) |
||||
assert.Equal(t, "default", q.GetName()) |
||||
assert.Equal(t, "level", q.GetType()) |
||||
assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/common"), q.baseConfig.DataFullDir) |
||||
assert.Equal(t, 100, q.baseConfig.Length) |
||||
assert.Equal(t, 20, q.batchLength) |
||||
assert.Equal(t, "", q.baseConfig.ConnStr) |
||||
assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) |
||||
assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) |
||||
assert.Equal(t, 10, q.GetWorkerMaxNumber()) |
||||
assert.Equal(t, 0, q.GetWorkerNumber()) |
||||
assert.Equal(t, 0, q.GetWorkerActiveNumber()) |
||||
assert.Equal(t, 0, q.GetQueueItemNumber()) |
||||
assert.Equal(t, "int", q.GetItemTypeName()) |
||||
|
||||
// test inherited config
|
||||
cfgProvider, err := setting.NewConfigProviderFromData(` |
||||
[queue] |
||||
TYPE = channel |
||||
DATADIR = queues/dir1 |
||||
LENGTH = 100 |
||||
BATCH_LENGTH = 20 |
||||
CONN_STR = "addrs=127.0.0.1:6379 db=0" |
||||
QUEUE_NAME = _queue1 |
||||
|
||||
[queue.sub] |
||||
TYPE = level |
||||
DATADIR = queues/dir2 |
||||
LENGTH = 102 |
||||
BATCH_LENGTH = 22 |
||||
CONN_STR = |
||||
QUEUE_NAME = _q2 |
||||
SET_NAME = _u2 |
||||
MAX_WORKERS = 2 |
||||
`) |
||||
|
||||
assert.NoError(t, err) |
||||
|
||||
q1 := createWorkerPoolQueue[string]("no-such", cfgProvider, nil, false) |
||||
assert.Equal(t, "no-such", q1.GetName()) |
||||
assert.Equal(t, "dummy", q1.GetType()) // no handler, so it becomes dummy
|
||||
assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir1"), q1.baseConfig.DataFullDir) |
||||
assert.Equal(t, 100, q1.baseConfig.Length) |
||||
assert.Equal(t, 20, q1.batchLength) |
||||
assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr) |
||||
assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName) |
||||
assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName) |
||||
assert.Equal(t, 10, q1.GetWorkerMaxNumber()) |
||||
assert.Equal(t, 0, q1.GetWorkerNumber()) |
||||
assert.Equal(t, 0, q1.GetWorkerActiveNumber()) |
||||
assert.Equal(t, 0, q1.GetQueueItemNumber()) |
||||
assert.Equal(t, "string", q1.GetItemTypeName()) |
||||
qid1 := GetManager().qidCounter |
||||
|
||||
q2 := createWorkerPoolQueue("sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) |
||||
assert.Equal(t, "sub", q2.GetName()) |
||||
assert.Equal(t, "level", q2.GetType()) |
||||
assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir2"), q2.baseConfig.DataFullDir) |
||||
assert.Equal(t, 102, q2.baseConfig.Length) |
||||
assert.Equal(t, 22, q2.batchLength) |
||||
assert.Equal(t, "", q2.baseConfig.ConnStr) |
||||
assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) |
||||
assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) |
||||
assert.Equal(t, 2, q2.GetWorkerMaxNumber()) |
||||
assert.Equal(t, 0, q2.GetWorkerNumber()) |
||||
assert.Equal(t, 0, q2.GetWorkerActiveNumber()) |
||||
assert.Equal(t, 0, q2.GetQueueItemNumber()) |
||||
assert.Equal(t, "int", q2.GetItemTypeName()) |
||||
qid2 := GetManager().qidCounter |
||||
|
||||
assert.Equal(t, q1, GetManager().ManagedQueues()[qid1]) |
||||
|
||||
GetManager().GetManagedQueue(qid1).SetWorkerMaxNumber(120) |
||||
assert.Equal(t, 120, q1.workerMaxNum) |
||||
|
||||
stop := runWorkerPoolQueue(q2) |
||||
assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(context.Background(), 0)) |
||||
assert.NoError(t, GetManager().FlushAll(context.Background(), 0)) |
||||
stop() |
||||
} |
||||
@ -1,201 +1,31 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"time" |
||||
) |
||||
|
||||
// ErrInvalidConfiguration is called when there is invalid configuration for a queue
|
||||
type ErrInvalidConfiguration struct { |
||||
cfg interface{} |
||||
err error |
||||
} |
||||
|
||||
func (err ErrInvalidConfiguration) Error() string { |
||||
if err.err != nil { |
||||
return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err) |
||||
} |
||||
return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg) |
||||
} |
||||
|
||||
// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
|
||||
func IsErrInvalidConfiguration(err error) bool { |
||||
_, ok := err.(ErrInvalidConfiguration) |
||||
return ok |
||||
} |
||||
|
||||
// Type is a type of Queue
|
||||
type Type string |
||||
|
||||
// Data defines an type of queuable data
|
||||
type Data interface{} |
||||
|
||||
// HandlerFunc is a function that takes a variable amount of data and processes it
|
||||
type HandlerFunc func(...Data) (unhandled []Data) |
||||
|
||||
// NewQueueFunc is a function that creates a queue
|
||||
type NewQueueFunc func(handler HandlerFunc, config, exemplar interface{}) (Queue, error) |
||||
|
||||
// Shutdownable represents a queue that can be shutdown
|
||||
type Shutdownable interface { |
||||
Shutdown() |
||||
Terminate() |
||||
} |
||||
|
||||
// Named represents a queue with a name
|
||||
type Named interface { |
||||
Name() string |
||||
} |
||||
|
||||
// Queue defines an interface of a queue-like item
|
||||
// Package queue implements a specialized queue system for Gitea.
|
||||
//
|
||||
// Queues will handle their own contents in the Run method
|
||||
type Queue interface { |
||||
Flushable |
||||
Run(atShutdown, atTerminate func(func())) |
||||
Push(Data) error |
||||
} |
||||
|
||||
// PushBackable queues can be pushed back to
|
||||
type PushBackable interface { |
||||
// PushBack pushes data back to the top of the fifo
|
||||
PushBack(Data) error |
||||
} |
||||
|
||||
// DummyQueueType is the type for the dummy queue
|
||||
const DummyQueueType Type = "dummy" |
||||
|
||||
// NewDummyQueue creates a new DummyQueue
|
||||
func NewDummyQueue(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) { |
||||
return &DummyQueue{}, nil |
||||
} |
||||
|
||||
// DummyQueue represents an empty queue
|
||||
type DummyQueue struct{} |
||||
|
||||
// Run does nothing
|
||||
func (*DummyQueue) Run(_, _ func(func())) {} |
||||
|
||||
// Push fakes a push of data to the queue
|
||||
func (*DummyQueue) Push(Data) error { |
||||
return nil |
||||
} |
||||
|
||||
// PushFunc fakes a push of data to the queue with a function. The function is never run.
|
||||
func (*DummyQueue) PushFunc(Data, func() error) error { |
||||
return nil |
||||
} |
||||
|
||||
// Has always returns false as this queue never does anything
|
||||
func (*DummyQueue) Has(Data) (bool, error) { |
||||
return false, nil |
||||
} |
||||
|
||||
// Flush always returns nil
|
||||
func (*DummyQueue) Flush(time.Duration) error { |
||||
return nil |
||||
} |
||||
|
||||
// FlushWithContext always returns nil
|
||||
func (*DummyQueue) FlushWithContext(context.Context) error { |
||||
return nil |
||||
} |
||||
|
||||
// IsEmpty asserts that the queue is empty
|
||||
func (*DummyQueue) IsEmpty() bool { |
||||
return true |
||||
} |
||||
|
||||
// ImmediateType is the type to execute the function when push
|
||||
const ImmediateType Type = "immediate" |
||||
|
||||
// NewImmediate creates a new false queue to execute the function when push
|
||||
func NewImmediate(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) { |
||||
return &Immediate{ |
||||
handler: handler, |
||||
}, nil |
||||
} |
||||
|
||||
// Immediate represents an direct execution queue
|
||||
type Immediate struct { |
||||
handler HandlerFunc |
||||
} |
||||
|
||||
// Run does nothing
|
||||
func (*Immediate) Run(_, _ func(func())) {} |
||||
|
||||
// Push fakes a push of data to the queue
|
||||
func (q *Immediate) Push(data Data) error { |
||||
return q.PushFunc(data, nil) |
||||
} |
||||
|
||||
// PushFunc fakes a push of data to the queue with a function. The function is never run.
|
||||
func (q *Immediate) PushFunc(data Data, f func() error) error { |
||||
if f != nil { |
||||
if err := f(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
q.handler(data) |
||||
return nil |
||||
} |
||||
|
||||
// Has always returns false as this queue never does anything
|
||||
func (*Immediate) Has(Data) (bool, error) { |
||||
return false, nil |
||||
} |
||||
|
||||
// Flush always returns nil
|
||||
func (*Immediate) Flush(time.Duration) error { |
||||
return nil |
||||
} |
||||
|
||||
// FlushWithContext always returns nil
|
||||
func (*Immediate) FlushWithContext(context.Context) error { |
||||
return nil |
||||
} |
||||
|
||||
// IsEmpty asserts that the queue is empty
|
||||
func (*Immediate) IsEmpty() bool { |
||||
return true |
||||
} |
||||
|
||||
var queuesMap = map[Type]NewQueueFunc{ |
||||
DummyQueueType: NewDummyQueue, |
||||
ImmediateType: NewImmediate, |
||||
} |
||||
// There are two major kinds of concepts:
|
||||
//
|
||||
// * The "base queue": channel, level, redis:
|
||||
// - They have the same abstraction, the same interface, and they are tested by the same testing code.
|
||||
// - The dummy(immediate) queue is special, it's not a real queue, it's only used as a no-op queue or a testing queue.
|
||||
//
|
||||
// * The WorkerPoolQueue: it uses the "base queue" to provide "worker pool" function.
|
||||
// - It calls the "handler" to process the data in the base queue.
|
||||
// - Its "Push" function doesn't block forever,
|
||||
// it will return an error if the queue is full after the timeout.
|
||||
//
|
||||
// A queue can be "simple" or "unique". A unique queue will try to avoid duplicate items.
|
||||
// Unique queue's "Has" function can be used to check whether an item is already in the queue,
|
||||
// although it's not 100% reliable due to there is no proper transaction support.
|
||||
// Simple queue's "Has" function always returns "has=false".
|
||||
//
|
||||
// The HandlerFuncT function is called by the WorkerPoolQueue to process the data in the base queue.
|
||||
// If the handler returns "unhandled" items, they will be re-queued to the base queue after a slight delay,
|
||||
// in case the item processor (eg: document indexer) is not available.
|
||||
package queue |
||||
|
||||
// RegisteredTypes provides the list of requested types of queues
|
||||
func RegisteredTypes() []Type { |
||||
types := make([]Type, len(queuesMap)) |
||||
i := 0 |
||||
for key := range queuesMap { |
||||
types[i] = key |
||||
i++ |
||||
} |
||||
return types |
||||
} |
||||
import "code.gitea.io/gitea/modules/util" |
||||
|
||||
// RegisteredTypesAsString provides the list of requested types of queues
|
||||
func RegisteredTypesAsString() []string { |
||||
types := make([]string, len(queuesMap)) |
||||
i := 0 |
||||
for key := range queuesMap { |
||||
types[i] = string(key) |
||||
i++ |
||||
} |
||||
return types |
||||
} |
||||
type HandlerFuncT[T any] func(...T) (unhandled []T) |
||||
|
||||
// NewQueue takes a queue Type, HandlerFunc, some options and possibly an exemplar and returns a Queue or an error
|
||||
func NewQueue(queueType Type, handlerFunc HandlerFunc, opts, exemplar interface{}) (Queue, error) { |
||||
newFn, ok := queuesMap[queueType] |
||||
if !ok { |
||||
return nil, fmt.Errorf("unsupported queue type: %v", queueType) |
||||
} |
||||
return newFn(handlerFunc, opts, exemplar) |
||||
} |
||||
var ErrAlreadyInQueue = util.NewAlreadyExistErrorf("already in queue") |
||||
|
||||
@ -1,419 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"runtime/pprof" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/json" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/util" |
||||
) |
||||
|
||||
// ByteFIFOQueueConfiguration is the configuration for a ByteFIFOQueue
|
||||
type ByteFIFOQueueConfiguration struct { |
||||
WorkerPoolConfiguration |
||||
Workers int |
||||
WaitOnEmpty bool |
||||
} |
||||
|
||||
var _ Queue = &ByteFIFOQueue{} |
||||
|
||||
// ByteFIFOQueue is a Queue formed from a ByteFIFO and WorkerPool
|
||||
type ByteFIFOQueue struct { |
||||
*WorkerPool |
||||
byteFIFO ByteFIFO |
||||
typ Type |
||||
shutdownCtx context.Context |
||||
shutdownCtxCancel context.CancelFunc |
||||
terminateCtx context.Context |
||||
terminateCtxCancel context.CancelFunc |
||||
exemplar interface{} |
||||
workers int |
||||
name string |
||||
lock sync.Mutex |
||||
waitOnEmpty bool |
||||
pushed chan struct{} |
||||
} |
||||
|
||||
// NewByteFIFOQueue creates a new ByteFIFOQueue
|
||||
func NewByteFIFOQueue(typ Type, byteFIFO ByteFIFO, handle HandlerFunc, cfg, exemplar interface{}) (*ByteFIFOQueue, error) { |
||||
configInterface, err := toConfig(ByteFIFOQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(ByteFIFOQueueConfiguration) |
||||
|
||||
terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) |
||||
shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) |
||||
|
||||
q := &ByteFIFOQueue{ |
||||
byteFIFO: byteFIFO, |
||||
typ: typ, |
||||
shutdownCtx: shutdownCtx, |
||||
shutdownCtxCancel: shutdownCtxCancel, |
||||
terminateCtx: terminateCtx, |
||||
terminateCtxCancel: terminateCtxCancel, |
||||
exemplar: exemplar, |
||||
workers: config.Workers, |
||||
name: config.Name, |
||||
waitOnEmpty: config.WaitOnEmpty, |
||||
pushed: make(chan struct{}, 1), |
||||
} |
||||
q.WorkerPool = NewWorkerPool(func(data ...Data) (failed []Data) { |
||||
for _, unhandled := range handle(data...) { |
||||
if fail := q.PushBack(unhandled); fail != nil { |
||||
failed = append(failed, fail) |
||||
} |
||||
} |
||||
return failed |
||||
}, config.WorkerPoolConfiguration) |
||||
|
||||
return q, nil |
||||
} |
||||
|
||||
// Name returns the name of this queue
|
||||
func (q *ByteFIFOQueue) Name() string { |
||||
return q.name |
||||
} |
||||
|
||||
// Push pushes data to the fifo
|
||||
func (q *ByteFIFOQueue) Push(data Data) error { |
||||
return q.PushFunc(data, nil) |
||||
} |
||||
|
||||
// PushBack pushes data to the fifo
|
||||
func (q *ByteFIFOQueue) PushBack(data Data) error { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) |
||||
} |
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer func() { |
||||
select { |
||||
case q.pushed <- struct{}{}: |
||||
default: |
||||
} |
||||
}() |
||||
return q.byteFIFO.PushBack(q.terminateCtx, bs) |
||||
} |
||||
|
||||
// PushFunc pushes data to the fifo
|
||||
func (q *ByteFIFOQueue) PushFunc(data Data, fn func() error) error { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) |
||||
} |
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer func() { |
||||
select { |
||||
case q.pushed <- struct{}{}: |
||||
default: |
||||
} |
||||
}() |
||||
return q.byteFIFO.PushFunc(q.terminateCtx, bs, fn) |
||||
} |
||||
|
||||
// IsEmpty checks if the queue is empty
|
||||
func (q *ByteFIFOQueue) IsEmpty() bool { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if !q.WorkerPool.IsEmpty() { |
||||
return false |
||||
} |
||||
return q.byteFIFO.Len(q.terminateCtx) == 0 |
||||
} |
||||
|
||||
// NumberInQueue returns the number in the queue
|
||||
func (q *ByteFIFOQueue) NumberInQueue() int64 { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
return q.byteFIFO.Len(q.terminateCtx) + q.WorkerPool.NumberInQueue() |
||||
} |
||||
|
||||
// Flush flushes the ByteFIFOQueue
|
||||
func (q *ByteFIFOQueue) Flush(timeout time.Duration) error { |
||||
select { |
||||
case q.pushed <- struct{}{}: |
||||
default: |
||||
} |
||||
return q.WorkerPool.Flush(timeout) |
||||
} |
||||
|
||||
// Run runs the bytefifo queue
|
||||
func (q *ByteFIFOQueue) Run(atShutdown, atTerminate func(func())) { |
||||
pprof.SetGoroutineLabels(q.baseCtx) |
||||
atShutdown(q.Shutdown) |
||||
atTerminate(q.Terminate) |
||||
log.Debug("%s: %s Starting", q.typ, q.name) |
||||
|
||||
_ = q.AddWorkers(q.workers, 0) |
||||
|
||||
log.Trace("%s: %s Now running", q.typ, q.name) |
||||
q.readToChan() |
||||
|
||||
<-q.shutdownCtx.Done() |
||||
log.Trace("%s: %s Waiting til done", q.typ, q.name) |
||||
q.Wait() |
||||
|
||||
log.Trace("%s: %s Waiting til cleaned", q.typ, q.name) |
||||
q.CleanUp(q.terminateCtx) |
||||
q.terminateCtxCancel() |
||||
} |
||||
|
||||
const maxBackOffTime = time.Second * 3 |
||||
|
||||
func (q *ByteFIFOQueue) readToChan() { |
||||
// handle quick cancels
|
||||
select { |
||||
case <-q.shutdownCtx.Done(): |
||||
// tell the pool to shutdown.
|
||||
q.baseCtxCancel() |
||||
return |
||||
default: |
||||
} |
||||
|
||||
// Default backoff values
|
||||
backOffTime := time.Millisecond * 100 |
||||
backOffTimer := time.NewTimer(0) |
||||
util.StopTimer(backOffTimer) |
||||
|
||||
paused, _ := q.IsPausedIsResumed() |
||||
|
||||
loop: |
||||
for { |
||||
select { |
||||
case <-paused: |
||||
log.Trace("Queue %s pausing", q.name) |
||||
_, resumed := q.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
paused, _ = q.IsPausedIsResumed() |
||||
log.Trace("Queue %s resuming", q.name) |
||||
if q.HasNoWorkerScaling() { |
||||
log.Warn( |
||||
"Queue: %s is configured to be non-scaling and has no workers - this configuration is likely incorrect.\n"+ |
||||
"The queue will be paused to prevent data-loss with the assumption that you will add workers and unpause as required.", q.name) |
||||
q.Pause() |
||||
continue loop |
||||
} |
||||
case <-q.shutdownCtx.Done(): |
||||
// tell the pool to shutdown.
|
||||
q.baseCtxCancel() |
||||
return |
||||
case data, ok := <-q.dataChan: |
||||
if !ok { |
||||
return |
||||
} |
||||
if err := q.PushBack(data); err != nil { |
||||
log.Error("Unable to push back data into queue %s", q.name) |
||||
} |
||||
atomic.AddInt64(&q.numInQueue, -1) |
||||
} |
||||
default: |
||||
} |
||||
|
||||
// empty the pushed channel
|
||||
select { |
||||
case <-q.pushed: |
||||
default: |
||||
} |
||||
|
||||
err := q.doPop() |
||||
|
||||
util.StopTimer(backOffTimer) |
||||
|
||||
if err != nil { |
||||
if err == errQueueEmpty && q.waitOnEmpty { |
||||
log.Trace("%s: %s Waiting on Empty", q.typ, q.name) |
||||
|
||||
// reset the backoff time but don't set the timer
|
||||
backOffTime = 100 * time.Millisecond |
||||
} else if err == errUnmarshal { |
||||
// reset the timer and backoff
|
||||
backOffTime = 100 * time.Millisecond |
||||
backOffTimer.Reset(backOffTime) |
||||
} else { |
||||
// backoff
|
||||
backOffTimer.Reset(backOffTime) |
||||
} |
||||
|
||||
// Need to Backoff
|
||||
select { |
||||
case <-q.shutdownCtx.Done(): |
||||
// Oops we've been shutdown whilst backing off
|
||||
// Make sure the worker pool is shutdown too
|
||||
q.baseCtxCancel() |
||||
return |
||||
case <-q.pushed: |
||||
// Data has been pushed to the fifo (or flush has been called)
|
||||
// reset the backoff time
|
||||
backOffTime = 100 * time.Millisecond |
||||
continue loop |
||||
case <-backOffTimer.C: |
||||
// Calculate the next backoff time
|
||||
backOffTime += backOffTime / 2 |
||||
if backOffTime > maxBackOffTime { |
||||
backOffTime = maxBackOffTime |
||||
} |
||||
continue loop |
||||
} |
||||
} |
||||
|
||||
// Reset the backoff time
|
||||
backOffTime = 100 * time.Millisecond |
||||
|
||||
select { |
||||
case <-q.shutdownCtx.Done(): |
||||
// Oops we've been shutdown
|
||||
// Make sure the worker pool is shutdown too
|
||||
q.baseCtxCancel() |
||||
return |
||||
default: |
||||
continue loop |
||||
} |
||||
} |
||||
} |
||||
|
||||
var ( |
||||
errQueueEmpty = fmt.Errorf("empty queue") |
||||
errEmptyBytes = fmt.Errorf("empty bytes") |
||||
errUnmarshal = fmt.Errorf("failed to unmarshal") |
||||
) |
||||
|
||||
func (q *ByteFIFOQueue) doPop() error { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
bs, err := q.byteFIFO.Pop(q.shutdownCtx) |
||||
if err != nil { |
||||
if err == context.Canceled { |
||||
q.baseCtxCancel() |
||||
return err |
||||
} |
||||
log.Error("%s: %s Error on Pop: %v", q.typ, q.name, err) |
||||
return err |
||||
} |
||||
if len(bs) == 0 { |
||||
if q.waitOnEmpty && q.byteFIFO.Len(q.shutdownCtx) == 0 { |
||||
return errQueueEmpty |
||||
} |
||||
return errEmptyBytes |
||||
} |
||||
|
||||
data, err := unmarshalAs(bs, q.exemplar) |
||||
if err != nil { |
||||
log.Error("%s: %s Failed to unmarshal with error: %v", q.typ, q.name, err) |
||||
return errUnmarshal |
||||
} |
||||
|
||||
log.Trace("%s %s: Task found: %#v", q.typ, q.name, data) |
||||
q.WorkerPool.Push(data) |
||||
return nil |
||||
} |
||||
|
||||
// Shutdown processing from this queue
|
||||
func (q *ByteFIFOQueue) Shutdown() { |
||||
log.Trace("%s: %s Shutting down", q.typ, q.name) |
||||
select { |
||||
case <-q.shutdownCtx.Done(): |
||||
return |
||||
default: |
||||
} |
||||
q.shutdownCtxCancel() |
||||
log.Debug("%s: %s Shutdown", q.typ, q.name) |
||||
} |
||||
|
||||
// IsShutdown returns a channel which is closed when this Queue is shutdown
|
||||
func (q *ByteFIFOQueue) IsShutdown() <-chan struct{} { |
||||
return q.shutdownCtx.Done() |
||||
} |
||||
|
||||
// Terminate this queue and close the queue
|
||||
func (q *ByteFIFOQueue) Terminate() { |
||||
log.Trace("%s: %s Terminating", q.typ, q.name) |
||||
q.Shutdown() |
||||
select { |
||||
case <-q.terminateCtx.Done(): |
||||
return |
||||
default: |
||||
} |
||||
if log.IsDebug() { |
||||
log.Debug("%s: %s Closing with %d tasks left in queue", q.typ, q.name, q.byteFIFO.Len(q.terminateCtx)) |
||||
} |
||||
q.terminateCtxCancel() |
||||
if err := q.byteFIFO.Close(); err != nil { |
||||
log.Error("Error whilst closing internal byte fifo in %s: %s: %v", q.typ, q.name, err) |
||||
} |
||||
q.baseCtxFinished() |
||||
log.Debug("%s: %s Terminated", q.typ, q.name) |
||||
} |
||||
|
||||
// IsTerminated returns a channel which is closed when this Queue is terminated
|
||||
func (q *ByteFIFOQueue) IsTerminated() <-chan struct{} { |
||||
return q.terminateCtx.Done() |
||||
} |
||||
|
||||
var _ UniqueQueue = &ByteFIFOUniqueQueue{} |
||||
|
||||
// ByteFIFOUniqueQueue represents a UniqueQueue formed from a UniqueByteFifo
|
||||
type ByteFIFOUniqueQueue struct { |
||||
ByteFIFOQueue |
||||
} |
||||
|
||||
// NewByteFIFOUniqueQueue creates a new ByteFIFOUniqueQueue
|
||||
func NewByteFIFOUniqueQueue(typ Type, byteFIFO UniqueByteFIFO, handle HandlerFunc, cfg, exemplar interface{}) (*ByteFIFOUniqueQueue, error) { |
||||
configInterface, err := toConfig(ByteFIFOQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(ByteFIFOQueueConfiguration) |
||||
terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) |
||||
shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) |
||||
|
||||
q := &ByteFIFOUniqueQueue{ |
||||
ByteFIFOQueue: ByteFIFOQueue{ |
||||
byteFIFO: byteFIFO, |
||||
typ: typ, |
||||
shutdownCtx: shutdownCtx, |
||||
shutdownCtxCancel: shutdownCtxCancel, |
||||
terminateCtx: terminateCtx, |
||||
terminateCtxCancel: terminateCtxCancel, |
||||
exemplar: exemplar, |
||||
workers: config.Workers, |
||||
name: config.Name, |
||||
}, |
||||
} |
||||
q.WorkerPool = NewWorkerPool(func(data ...Data) (failed []Data) { |
||||
for _, unhandled := range handle(data...) { |
||||
if fail := q.PushBack(unhandled); fail != nil { |
||||
failed = append(failed, fail) |
||||
} |
||||
} |
||||
return failed |
||||
}, config.WorkerPoolConfiguration) |
||||
|
||||
return q, nil |
||||
} |
||||
|
||||
// Has checks if the provided data is in the queue
|
||||
func (q *ByteFIFOUniqueQueue) Has(data Data) (bool, error) { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return false, fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) |
||||
} |
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return q.byteFIFO.(UniqueByteFIFO).Has(q.terminateCtx, bs) |
||||
} |
||||
@ -1,160 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"runtime/pprof" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// ChannelQueueType is the type for channel queue
|
||||
const ChannelQueueType Type = "channel" |
||||
|
||||
// ChannelQueueConfiguration is the configuration for a ChannelQueue
|
||||
type ChannelQueueConfiguration struct { |
||||
WorkerPoolConfiguration |
||||
Workers int |
||||
} |
||||
|
||||
// ChannelQueue implements Queue
|
||||
//
|
||||
// A channel queue is not persistable and does not shutdown or terminate cleanly
|
||||
// It is basically a very thin wrapper around a WorkerPool
|
||||
type ChannelQueue struct { |
||||
*WorkerPool |
||||
shutdownCtx context.Context |
||||
shutdownCtxCancel context.CancelFunc |
||||
terminateCtx context.Context |
||||
terminateCtxCancel context.CancelFunc |
||||
exemplar interface{} |
||||
workers int |
||||
name string |
||||
} |
||||
|
||||
// NewChannelQueue creates a memory channel queue
|
||||
func NewChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(ChannelQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(ChannelQueueConfiguration) |
||||
if config.BatchLength == 0 { |
||||
config.BatchLength = 1 |
||||
} |
||||
|
||||
terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) |
||||
shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) |
||||
|
||||
queue := &ChannelQueue{ |
||||
shutdownCtx: shutdownCtx, |
||||
shutdownCtxCancel: shutdownCtxCancel, |
||||
terminateCtx: terminateCtx, |
||||
terminateCtxCancel: terminateCtxCancel, |
||||
exemplar: exemplar, |
||||
workers: config.Workers, |
||||
name: config.Name, |
||||
} |
||||
queue.WorkerPool = NewWorkerPool(func(data ...Data) []Data { |
||||
unhandled := handle(data...) |
||||
if len(unhandled) > 0 { |
||||
// We can only pushback to the channel if we're paused.
|
||||
if queue.IsPaused() { |
||||
atomic.AddInt64(&queue.numInQueue, int64(len(unhandled))) |
||||
go func() { |
||||
for _, datum := range data { |
||||
queue.dataChan <- datum |
||||
} |
||||
}() |
||||
return nil |
||||
} |
||||
} |
||||
return unhandled |
||||
}, config.WorkerPoolConfiguration) |
||||
|
||||
queue.qid = GetManager().Add(queue, ChannelQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
// Run starts to run the queue
|
||||
func (q *ChannelQueue) Run(atShutdown, atTerminate func(func())) { |
||||
pprof.SetGoroutineLabels(q.baseCtx) |
||||
atShutdown(q.Shutdown) |
||||
atTerminate(q.Terminate) |
||||
log.Debug("ChannelQueue: %s Starting", q.name) |
||||
_ = q.AddWorkers(q.workers, 0) |
||||
} |
||||
|
||||
// Push will push data into the queue
|
||||
func (q *ChannelQueue) Push(data Data) error { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name) |
||||
} |
||||
q.WorkerPool.Push(data) |
||||
return nil |
||||
} |
||||
|
||||
// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager
|
||||
func (q *ChannelQueue) Flush(timeout time.Duration) error { |
||||
if q.IsPaused() { |
||||
return nil |
||||
} |
||||
ctx, cancel := q.commonRegisterWorkers(1, timeout, true) |
||||
defer cancel() |
||||
return q.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// Shutdown processing from this queue
|
||||
func (q *ChannelQueue) Shutdown() { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
select { |
||||
case <-q.shutdownCtx.Done(): |
||||
log.Trace("ChannelQueue: %s Already Shutting down", q.name) |
||||
return |
||||
default: |
||||
} |
||||
log.Trace("ChannelQueue: %s Shutting down", q.name) |
||||
go func() { |
||||
log.Trace("ChannelQueue: %s Flushing", q.name) |
||||
// We can't use Cleanup here because that will close the channel
|
||||
if err := q.FlushWithContext(q.terminateCtx); err != nil { |
||||
count := atomic.LoadInt64(&q.numInQueue) |
||||
if count > 0 { |
||||
log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name) |
||||
} |
||||
return |
||||
} |
||||
log.Debug("ChannelQueue: %s Flushed", q.name) |
||||
}() |
||||
q.shutdownCtxCancel() |
||||
log.Debug("ChannelQueue: %s Shutdown", q.name) |
||||
} |
||||
|
||||
// Terminate this queue and close the queue
|
||||
func (q *ChannelQueue) Terminate() { |
||||
log.Trace("ChannelQueue: %s Terminating", q.name) |
||||
q.Shutdown() |
||||
select { |
||||
case <-q.terminateCtx.Done(): |
||||
return |
||||
default: |
||||
} |
||||
q.terminateCtxCancel() |
||||
q.baseCtxFinished() |
||||
log.Debug("ChannelQueue: %s Terminated", q.name) |
||||
} |
||||
|
||||
// Name returns the name of this queue
|
||||
func (q *ChannelQueue) Name() string { |
||||
return q.name |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[ChannelQueueType] = NewChannelQueue |
||||
} |
||||
@ -1,315 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"os" |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestChannelQueue(t *testing.T) { |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
nilFn := func(_ func()) {} |
||||
|
||||
queue, err := NewChannelQueue(handle, |
||||
ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 0, |
||||
MaxWorkers: 10, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 5, |
||||
Name: "TestChannelQueue", |
||||
}, |
||||
Workers: 0, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
assert.Equal(t, 5, queue.(*ChannelQueue).WorkerPool.boostWorkers) |
||||
|
||||
go queue.Run(nilFn, nilFn) |
||||
|
||||
test1 := testData{"A", 1} |
||||
go queue.Push(&test1) |
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
err = queue.Push(test1) |
||||
assert.Error(t, err) |
||||
} |
||||
|
||||
func TestChannelQueue_Batch(t *testing.T) { |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
assert.True(t, len(data) == 2) |
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
nilFn := func(_ func()) {} |
||||
|
||||
queue, err := NewChannelQueue(handle, |
||||
ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 20, |
||||
BatchLength: 2, |
||||
BlockTimeout: 0, |
||||
BoostTimeout: 0, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
}, |
||||
Workers: 1, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go queue.Run(nilFn, nilFn) |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
|
||||
queue.Push(&test1) |
||||
go queue.Push(&test2) |
||||
|
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
result2 := <-handleChan |
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
err = queue.Push(test1) |
||||
assert.Error(t, err) |
||||
} |
||||
|
||||
func TestChannelQueue_Pause(t *testing.T) { |
||||
if os.Getenv("CI") != "" { |
||||
t.Skip("Skipping because test is flaky on CI") |
||||
} |
||||
lock := sync.Mutex{} |
||||
var queue Queue |
||||
var err error |
||||
pushBack := false |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
lock.Lock() |
||||
if pushBack { |
||||
if pausable, ok := queue.(Pausable); ok { |
||||
pausable.Pause() |
||||
} |
||||
lock.Unlock() |
||||
return data |
||||
} |
||||
lock.Unlock() |
||||
|
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
queueShutdown := []func(){} |
||||
queueTerminate := []func(){} |
||||
|
||||
terminated := make(chan struct{}) |
||||
|
||||
queue, err = NewChannelQueue(handle, |
||||
ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 20, |
||||
BatchLength: 1, |
||||
BlockTimeout: 0, |
||||
BoostTimeout: 0, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
}, |
||||
Workers: 1, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go func() { |
||||
queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
}) |
||||
close(terminated) |
||||
}() |
||||
|
||||
// Shutdown and Terminate in defer
|
||||
defer func() { |
||||
lock.Lock() |
||||
callbacks := make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
lock.Lock() |
||||
log.Info("Finally terminating") |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
}() |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
queue.Push(&test1) |
||||
|
||||
pausable, ok := queue.(Pausable) |
||||
if !assert.True(t, ok) { |
||||
return |
||||
} |
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
pausable.Pause() |
||||
|
||||
paused, _ := pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue is not paused") |
||||
return |
||||
} |
||||
|
||||
queue.Push(&test2) |
||||
|
||||
var result2 *testData |
||||
select { |
||||
case result2 = <-handleChan: |
||||
assert.Fail(t, "handler chan should be empty") |
||||
case <-time.After(100 * time.Millisecond): |
||||
} |
||||
|
||||
assert.Nil(t, result2) |
||||
|
||||
pausable.Resume() |
||||
_, resumed := pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue should be resumed") |
||||
} |
||||
|
||||
select { |
||||
case result2 = <-handleChan: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "handler chan should contain test2") |
||||
} |
||||
|
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
lock.Lock() |
||||
pushBack = true |
||||
lock.Unlock() |
||||
|
||||
_, resumed = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue is not resumed") |
||||
return |
||||
} |
||||
|
||||
queue.Push(&test1) |
||||
paused, _ = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-handleChan: |
||||
assert.Fail(t, "handler chan should not contain test1") |
||||
return |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "queue should be paused") |
||||
return |
||||
} |
||||
|
||||
lock.Lock() |
||||
pushBack = false |
||||
lock.Unlock() |
||||
|
||||
paused, _ = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue is not paused") |
||||
return |
||||
} |
||||
|
||||
pausable.Resume() |
||||
_, resumed = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue should be resumed") |
||||
} |
||||
|
||||
select { |
||||
case result1 = <-handleChan: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "handler chan should contain test1") |
||||
} |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
lock.Lock() |
||||
callbacks := make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
queueShutdown = queueShutdown[:0] |
||||
lock.Unlock() |
||||
// Now shutdown the queue
|
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
// terminate the queue
|
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
queueShutdown = queueTerminate[:0] |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
select { |
||||
case <-terminated: |
||||
case <-time.After(10 * time.Second): |
||||
assert.Fail(t, "Queue should have terminated") |
||||
return |
||||
} |
||||
} |
||||
@ -1,124 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"gitea.com/lunny/levelqueue" |
||||
) |
||||
|
||||
// LevelQueueType is the type for level queue
|
||||
const LevelQueueType Type = "level" |
||||
|
||||
// LevelQueueConfiguration is the configuration for a LevelQueue
|
||||
type LevelQueueConfiguration struct { |
||||
ByteFIFOQueueConfiguration |
||||
DataDir string |
||||
ConnectionString string |
||||
QueueName string |
||||
} |
||||
|
||||
// LevelQueue implements a disk library queue
|
||||
type LevelQueue struct { |
||||
*ByteFIFOQueue |
||||
} |
||||
|
||||
// NewLevelQueue creates a ledis local queue
|
||||
func NewLevelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(LevelQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(LevelQueueConfiguration) |
||||
|
||||
if len(config.ConnectionString) == 0 { |
||||
config.ConnectionString = config.DataDir |
||||
} |
||||
config.WaitOnEmpty = true |
||||
|
||||
byteFIFO, err := NewLevelQueueByteFIFO(config.ConnectionString, config.QueueName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
byteFIFOQueue, err := NewByteFIFOQueue(LevelQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
queue := &LevelQueue{ |
||||
ByteFIFOQueue: byteFIFOQueue, |
||||
} |
||||
queue.qid = GetManager().Add(queue, LevelQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
var _ ByteFIFO = &LevelQueueByteFIFO{} |
||||
|
||||
// LevelQueueByteFIFO represents a ByteFIFO formed from a LevelQueue
|
||||
type LevelQueueByteFIFO struct { |
||||
internal *levelqueue.Queue |
||||
connection string |
||||
} |
||||
|
||||
// NewLevelQueueByteFIFO creates a ByteFIFO formed from a LevelQueue
|
||||
func NewLevelQueueByteFIFO(connection, prefix string) (*LevelQueueByteFIFO, error) { |
||||
db, err := nosql.GetManager().GetLevelDB(connection) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
internal, err := levelqueue.NewQueue(db, []byte(prefix), false) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &LevelQueueByteFIFO{ |
||||
connection: connection, |
||||
internal: internal, |
||||
}, nil |
||||
} |
||||
|
||||
// PushFunc will push data into the fifo
|
||||
func (fifo *LevelQueueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { |
||||
if fn != nil { |
||||
if err := fn(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return fifo.internal.LPush(data) |
||||
} |
||||
|
||||
// PushBack pushes data to the top of the fifo
|
||||
func (fifo *LevelQueueByteFIFO) PushBack(ctx context.Context, data []byte) error { |
||||
return fifo.internal.RPush(data) |
||||
} |
||||
|
||||
// Pop pops data from the start of the fifo
|
||||
func (fifo *LevelQueueByteFIFO) Pop(ctx context.Context) ([]byte, error) { |
||||
data, err := fifo.internal.RPop() |
||||
if err != nil && err != levelqueue.ErrNotFound { |
||||
return nil, err |
||||
} |
||||
return data, nil |
||||
} |
||||
|
||||
// Close this fifo
|
||||
func (fifo *LevelQueueByteFIFO) Close() error { |
||||
err := fifo.internal.Close() |
||||
_ = nosql.GetManager().CloseLevelDB(fifo.connection) |
||||
return err |
||||
} |
||||
|
||||
// Len returns the length of the fifo
|
||||
func (fifo *LevelQueueByteFIFO) Len(ctx context.Context) int64 { |
||||
return fifo.internal.Len() |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[LevelQueueType] = NewLevelQueue |
||||
} |
||||
@ -1,358 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"runtime/pprof" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// PersistableChannelQueueType is the type for persistable queue
|
||||
const PersistableChannelQueueType Type = "persistable-channel" |
||||
|
||||
// PersistableChannelQueueConfiguration is the configuration for a PersistableChannelQueue
|
||||
type PersistableChannelQueueConfiguration struct { |
||||
Name string |
||||
DataDir string |
||||
BatchLength int |
||||
QueueLength int |
||||
Timeout time.Duration |
||||
MaxAttempts int |
||||
Workers int |
||||
MaxWorkers int |
||||
BlockTimeout time.Duration |
||||
BoostTimeout time.Duration |
||||
BoostWorkers int |
||||
} |
||||
|
||||
// PersistableChannelQueue wraps a channel queue and level queue together
|
||||
// The disk level queue will be used to store data at shutdown and terminate - and will be restored
|
||||
// on start up.
|
||||
type PersistableChannelQueue struct { |
||||
channelQueue *ChannelQueue |
||||
delayedStarter |
||||
lock sync.Mutex |
||||
closed chan struct{} |
||||
} |
||||
|
||||
// NewPersistableChannelQueue creates a wrapped batched channel queue with persistable level queue backend when shutting down
|
||||
// This differs from a wrapped queue in that the persistent queue is only used to persist at shutdown/terminate
|
||||
func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(PersistableChannelQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(PersistableChannelQueueConfiguration) |
||||
|
||||
queue := &PersistableChannelQueue{ |
||||
closed: make(chan struct{}), |
||||
} |
||||
|
||||
wrappedHandle := func(data ...Data) (failed []Data) { |
||||
for _, unhandled := range handle(data...) { |
||||
if fail := queue.PushBack(unhandled); fail != nil { |
||||
failed = append(failed, fail) |
||||
} |
||||
} |
||||
return failed |
||||
} |
||||
|
||||
channelQueue, err := NewChannelQueue(wrappedHandle, ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: config.QueueLength, |
||||
BatchLength: config.BatchLength, |
||||
BlockTimeout: config.BlockTimeout, |
||||
BoostTimeout: config.BoostTimeout, |
||||
BoostWorkers: config.BoostWorkers, |
||||
MaxWorkers: config.MaxWorkers, |
||||
Name: config.Name + "-channel", |
||||
}, |
||||
Workers: config.Workers, |
||||
}, exemplar) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// the level backend only needs temporary workers to catch up with the previously dropped work
|
||||
levelCfg := LevelQueueConfiguration{ |
||||
ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: config.QueueLength, |
||||
BatchLength: config.BatchLength, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 1, |
||||
MaxWorkers: 5, |
||||
Name: config.Name + "-level", |
||||
}, |
||||
Workers: 0, |
||||
}, |
||||
DataDir: config.DataDir, |
||||
QueueName: config.Name + "-level", |
||||
} |
||||
|
||||
levelQueue, err := NewLevelQueue(wrappedHandle, levelCfg, exemplar) |
||||
if err == nil { |
||||
queue.channelQueue = channelQueue.(*ChannelQueue) |
||||
queue.delayedStarter = delayedStarter{ |
||||
internal: levelQueue.(*LevelQueue), |
||||
name: config.Name, |
||||
} |
||||
_ = GetManager().Add(queue, PersistableChannelQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
if IsErrInvalidConfiguration(err) { |
||||
// Retrying ain't gonna make this any better...
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg} |
||||
} |
||||
|
||||
queue.channelQueue = channelQueue.(*ChannelQueue) |
||||
queue.delayedStarter = delayedStarter{ |
||||
cfg: levelCfg, |
||||
underlying: LevelQueueType, |
||||
timeout: config.Timeout, |
||||
maxAttempts: config.MaxAttempts, |
||||
name: config.Name, |
||||
} |
||||
_ = GetManager().Add(queue, PersistableChannelQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
// Name returns the name of this queue
|
||||
func (q *PersistableChannelQueue) Name() string { |
||||
return q.delayedStarter.name |
||||
} |
||||
|
||||
// Push will push the indexer data to queue
|
||||
func (q *PersistableChannelQueue) Push(data Data) error { |
||||
select { |
||||
case <-q.closed: |
||||
return q.internal.Push(data) |
||||
default: |
||||
return q.channelQueue.Push(data) |
||||
} |
||||
} |
||||
|
||||
// PushBack will push the indexer data to queue
|
||||
func (q *PersistableChannelQueue) PushBack(data Data) error { |
||||
select { |
||||
case <-q.closed: |
||||
if pbr, ok := q.internal.(PushBackable); ok { |
||||
return pbr.PushBack(data) |
||||
} |
||||
return q.internal.Push(data) |
||||
default: |
||||
return q.channelQueue.Push(data) |
||||
} |
||||
} |
||||
|
||||
// Run starts to run the queue
|
||||
func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) { |
||||
pprof.SetGoroutineLabels(q.channelQueue.baseCtx) |
||||
log.Debug("PersistableChannelQueue: %s Starting", q.delayedStarter.name) |
||||
_ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) |
||||
|
||||
q.lock.Lock() |
||||
if q.internal == nil { |
||||
err := q.setInternal(atShutdown, q.channelQueue.handle, q.channelQueue.exemplar) |
||||
q.lock.Unlock() |
||||
if err != nil { |
||||
log.Fatal("Unable to create internal queue for %s Error: %v", q.Name(), err) |
||||
return |
||||
} |
||||
} else { |
||||
q.lock.Unlock() |
||||
} |
||||
atShutdown(q.Shutdown) |
||||
atTerminate(q.Terminate) |
||||
|
||||
if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.terminateCtx) != 0 { |
||||
// Just run the level queue - we shut it down once it's flushed
|
||||
go q.internal.Run(func(_ func()) {}, func(_ func()) {}) |
||||
go func() { |
||||
for !lq.IsEmpty() { |
||||
_ = lq.Flush(0) |
||||
select { |
||||
case <-time.After(100 * time.Millisecond): |
||||
case <-lq.shutdownCtx.Done(): |
||||
if lq.byteFIFO.Len(lq.terminateCtx) > 0 { |
||||
log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name()) |
||||
} |
||||
return |
||||
} |
||||
} |
||||
log.Debug("LevelQueue: %s flushed so shutting down", q.internal.(*LevelQueue).Name()) |
||||
q.internal.(*LevelQueue).Shutdown() |
||||
GetManager().Remove(q.internal.(*LevelQueue).qid) |
||||
}() |
||||
} else { |
||||
log.Debug("PersistableChannelQueue: %s Skipping running the empty level queue", q.delayedStarter.name) |
||||
q.internal.(*LevelQueue).Shutdown() |
||||
GetManager().Remove(q.internal.(*LevelQueue).qid) |
||||
} |
||||
} |
||||
|
||||
// Flush flushes the queue and blocks till the queue is empty
|
||||
func (q *PersistableChannelQueue) Flush(timeout time.Duration) error { |
||||
var ctx context.Context |
||||
var cancel context.CancelFunc |
||||
if timeout > 0 { |
||||
ctx, cancel = context.WithTimeout(context.Background(), timeout) |
||||
} else { |
||||
ctx, cancel = context.WithCancel(context.Background()) |
||||
} |
||||
defer cancel() |
||||
return q.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// FlushWithContext flushes the queue and blocks till the queue is empty
|
||||
func (q *PersistableChannelQueue) FlushWithContext(ctx context.Context) error { |
||||
errChan := make(chan error, 1) |
||||
go func() { |
||||
errChan <- q.channelQueue.FlushWithContext(ctx) |
||||
}() |
||||
go func() { |
||||
q.lock.Lock() |
||||
if q.internal == nil { |
||||
q.lock.Unlock() |
||||
errChan <- fmt.Errorf("not ready to flush internal queue %s yet", q.Name()) |
||||
return |
||||
} |
||||
q.lock.Unlock() |
||||
errChan <- q.internal.FlushWithContext(ctx) |
||||
}() |
||||
err1 := <-errChan |
||||
err2 := <-errChan |
||||
|
||||
if err1 != nil { |
||||
return err1 |
||||
} |
||||
return err2 |
||||
} |
||||
|
||||
// IsEmpty checks if a queue is empty
|
||||
func (q *PersistableChannelQueue) IsEmpty() bool { |
||||
if !q.channelQueue.IsEmpty() { |
||||
return false |
||||
} |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return false |
||||
} |
||||
return q.internal.IsEmpty() |
||||
} |
||||
|
||||
// IsPaused returns if the pool is paused
|
||||
func (q *PersistableChannelQueue) IsPaused() bool { |
||||
return q.channelQueue.IsPaused() |
||||
} |
||||
|
||||
// IsPausedIsResumed returns if the pool is paused and a channel that is closed when it is resumed
|
||||
func (q *PersistableChannelQueue) IsPausedIsResumed() (<-chan struct{}, <-chan struct{}) { |
||||
return q.channelQueue.IsPausedIsResumed() |
||||
} |
||||
|
||||
// Pause pauses the WorkerPool
|
||||
func (q *PersistableChannelQueue) Pause() { |
||||
q.channelQueue.Pause() |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return |
||||
} |
||||
|
||||
pausable, ok := q.internal.(Pausable) |
||||
if !ok { |
||||
return |
||||
} |
||||
pausable.Pause() |
||||
} |
||||
|
||||
// Resume resumes the WorkerPool
|
||||
func (q *PersistableChannelQueue) Resume() { |
||||
q.channelQueue.Resume() |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return |
||||
} |
||||
|
||||
pausable, ok := q.internal.(Pausable) |
||||
if !ok { |
||||
return |
||||
} |
||||
pausable.Resume() |
||||
} |
||||
|
||||
// Shutdown processing this queue
|
||||
func (q *PersistableChannelQueue) Shutdown() { |
||||
log.Trace("PersistableChannelQueue: %s Shutting down", q.delayedStarter.name) |
||||
q.lock.Lock() |
||||
|
||||
select { |
||||
case <-q.closed: |
||||
q.lock.Unlock() |
||||
return |
||||
default: |
||||
} |
||||
q.channelQueue.Shutdown() |
||||
if q.internal != nil { |
||||
q.internal.(*LevelQueue).Shutdown() |
||||
} |
||||
close(q.closed) |
||||
q.lock.Unlock() |
||||
|
||||
log.Trace("PersistableChannelQueue: %s Cancelling pools", q.delayedStarter.name) |
||||
q.channelQueue.baseCtxCancel() |
||||
q.internal.(*LevelQueue).baseCtxCancel() |
||||
log.Trace("PersistableChannelQueue: %s Waiting til done", q.delayedStarter.name) |
||||
q.channelQueue.Wait() |
||||
q.internal.(*LevelQueue).Wait() |
||||
// Redirect all remaining data in the chan to the internal channel
|
||||
log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name) |
||||
close(q.channelQueue.dataChan) |
||||
countOK, countLost := 0, 0 |
||||
for data := range q.channelQueue.dataChan { |
||||
err := q.internal.Push(data) |
||||
if err != nil { |
||||
log.Error("PersistableChannelQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) |
||||
countLost++ |
||||
} else { |
||||
countOK++ |
||||
} |
||||
atomic.AddInt64(&q.channelQueue.numInQueue, -1) |
||||
} |
||||
if countLost > 0 { |
||||
log.Warn("PersistableChannelQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) |
||||
} else if countOK > 0 { |
||||
log.Warn("PersistableChannelQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) |
||||
} |
||||
log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name) |
||||
|
||||
log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name) |
||||
} |
||||
|
||||
// Terminate this queue and close the queue
|
||||
func (q *PersistableChannelQueue) Terminate() { |
||||
log.Trace("PersistableChannelQueue: %s Terminating", q.delayedStarter.name) |
||||
q.Shutdown() |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
q.channelQueue.Terminate() |
||||
if q.internal != nil { |
||||
q.internal.(*LevelQueue).Terminate() |
||||
} |
||||
log.Debug("PersistableChannelQueue: %s Terminated", q.delayedStarter.name) |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[PersistableChannelQueueType] = NewPersistableChannelQueue |
||||
} |
||||
@ -1,544 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestPersistableChannelQueue(t *testing.T) { |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
for _, datum := range data { |
||||
if datum == nil { |
||||
continue |
||||
} |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
lock := sync.Mutex{} |
||||
queueShutdown := []func(){} |
||||
queueTerminate := []func(){} |
||||
|
||||
tmpDir := t.TempDir() |
||||
|
||||
queue, err := NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ |
||||
DataDir: tmpDir, |
||||
BatchLength: 2, |
||||
QueueLength: 20, |
||||
Workers: 1, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
Name: "test-queue", |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
readyForShutdown := make(chan struct{}) |
||||
readyForTerminate := make(chan struct{}) |
||||
|
||||
go queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
select { |
||||
case <-readyForShutdown: |
||||
default: |
||||
close(readyForShutdown) |
||||
} |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
select { |
||||
case <-readyForTerminate: |
||||
default: |
||||
close(readyForTerminate) |
||||
} |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
}) |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
|
||||
err = queue.Push(&test1) |
||||
assert.NoError(t, err) |
||||
go func() { |
||||
err := queue.Push(&test2) |
||||
assert.NoError(t, err) |
||||
}() |
||||
|
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
result2 := <-handleChan |
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
// test1 is a testData not a *testData so will be rejected
|
||||
err = queue.Push(test1) |
||||
assert.Error(t, err) |
||||
|
||||
<-readyForShutdown |
||||
// Now shutdown the queue
|
||||
lock.Lock() |
||||
callbacks := make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
// Wait til it is closed
|
||||
<-queue.(*PersistableChannelQueue).closed |
||||
|
||||
err = queue.Push(&test1) |
||||
assert.NoError(t, err) |
||||
err = queue.Push(&test2) |
||||
assert.NoError(t, err) |
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
default: |
||||
} |
||||
|
||||
// terminate the queue
|
||||
<-readyForTerminate |
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
default: |
||||
} |
||||
|
||||
// Reopen queue
|
||||
queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ |
||||
DataDir: tmpDir, |
||||
BatchLength: 2, |
||||
QueueLength: 20, |
||||
Workers: 1, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
Name: "test-queue", |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
readyForShutdown = make(chan struct{}) |
||||
readyForTerminate = make(chan struct{}) |
||||
|
||||
go queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
select { |
||||
case <-readyForShutdown: |
||||
default: |
||||
close(readyForShutdown) |
||||
} |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
select { |
||||
case <-readyForTerminate: |
||||
default: |
||||
close(readyForTerminate) |
||||
} |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
}) |
||||
|
||||
result3 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result3.TestString) |
||||
assert.Equal(t, test1.TestInt, result3.TestInt) |
||||
|
||||
result4 := <-handleChan |
||||
assert.Equal(t, test2.TestString, result4.TestString) |
||||
assert.Equal(t, test2.TestInt, result4.TestInt) |
||||
|
||||
<-readyForShutdown |
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
<-readyForTerminate |
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
} |
||||
|
||||
func TestPersistableChannelQueue_Pause(t *testing.T) { |
||||
lock := sync.Mutex{} |
||||
var queue Queue |
||||
var err error |
||||
pushBack := false |
||||
|
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
lock.Lock() |
||||
if pushBack { |
||||
if pausable, ok := queue.(Pausable); ok { |
||||
log.Info("pausing") |
||||
pausable.Pause() |
||||
} |
||||
lock.Unlock() |
||||
return data |
||||
} |
||||
lock.Unlock() |
||||
|
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
queueShutdown := []func(){} |
||||
queueTerminate := []func(){} |
||||
terminated := make(chan struct{}) |
||||
|
||||
tmpDir := t.TempDir() |
||||
|
||||
queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ |
||||
DataDir: tmpDir, |
||||
BatchLength: 2, |
||||
QueueLength: 20, |
||||
Workers: 1, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
Name: "test-queue", |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go func() { |
||||
queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
}) |
||||
close(terminated) |
||||
}() |
||||
|
||||
// Shutdown and Terminate in defer
|
||||
defer func() { |
||||
lock.Lock() |
||||
callbacks := make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
lock.Lock() |
||||
log.Info("Finally terminating") |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
}() |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
|
||||
err = queue.Push(&test1) |
||||
assert.NoError(t, err) |
||||
|
||||
pausable, ok := queue.(Pausable) |
||||
if !assert.True(t, ok) { |
||||
return |
||||
} |
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
pausable.Pause() |
||||
paused, _ := pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue is not paused") |
||||
return |
||||
} |
||||
|
||||
queue.Push(&test2) |
||||
|
||||
var result2 *testData |
||||
select { |
||||
case result2 = <-handleChan: |
||||
assert.Fail(t, "handler chan should be empty") |
||||
case <-time.After(100 * time.Millisecond): |
||||
} |
||||
|
||||
assert.Nil(t, result2) |
||||
|
||||
pausable.Resume() |
||||
_, resumed := pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue should be resumed") |
||||
return |
||||
} |
||||
|
||||
select { |
||||
case result2 = <-handleChan: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "handler chan should contain test2") |
||||
} |
||||
|
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
// Set pushBack to so that the next handle will result in a Pause
|
||||
lock.Lock() |
||||
pushBack = true |
||||
lock.Unlock() |
||||
|
||||
// Ensure that we're still resumed
|
||||
_, resumed = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(100 * time.Millisecond): |
||||
assert.Fail(t, "Queue is not resumed") |
||||
return |
||||
} |
||||
|
||||
// push test1
|
||||
queue.Push(&test1) |
||||
|
||||
// Now as this is handled it should pause
|
||||
paused, _ = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-handleChan: |
||||
assert.Fail(t, "handler chan should not contain test1") |
||||
return |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "queue should be paused") |
||||
return |
||||
} |
||||
|
||||
lock.Lock() |
||||
pushBack = false |
||||
lock.Unlock() |
||||
|
||||
pausable.Resume() |
||||
|
||||
_, resumed = pausable.IsPausedIsResumed() |
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "Queue should be resumed") |
||||
return |
||||
} |
||||
|
||||
select { |
||||
case result1 = <-handleChan: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "handler chan should contain test1") |
||||
return |
||||
} |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
lock.Lock() |
||||
callbacks := make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
queueShutdown = queueShutdown[:0] |
||||
lock.Unlock() |
||||
// Now shutdown the queue
|
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
// Wait til it is closed
|
||||
select { |
||||
case <-queue.(*PersistableChannelQueue).closed: |
||||
case <-time.After(5 * time.Second): |
||||
assert.Fail(t, "queue should close") |
||||
return |
||||
} |
||||
|
||||
err = queue.Push(&test1) |
||||
assert.NoError(t, err) |
||||
err = queue.Push(&test2) |
||||
assert.NoError(t, err) |
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
return |
||||
default: |
||||
} |
||||
|
||||
// terminate the queue
|
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
queueShutdown = queueTerminate[:0] |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
return |
||||
case <-terminated: |
||||
case <-time.After(10 * time.Second): |
||||
assert.Fail(t, "Queue should have terminated") |
||||
return |
||||
} |
||||
|
||||
lock.Lock() |
||||
pushBack = true |
||||
lock.Unlock() |
||||
|
||||
// Reopen queue
|
||||
terminated = make(chan struct{}) |
||||
queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ |
||||
DataDir: tmpDir, |
||||
BatchLength: 1, |
||||
QueueLength: 20, |
||||
Workers: 1, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
Name: "test-queue", |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
pausable, ok = queue.(Pausable) |
||||
if !assert.True(t, ok) { |
||||
return |
||||
} |
||||
|
||||
paused, _ = pausable.IsPausedIsResumed() |
||||
|
||||
go func() { |
||||
queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
}) |
||||
close(terminated) |
||||
}() |
||||
|
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
return |
||||
case <-paused: |
||||
} |
||||
|
||||
paused, _ = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "Queue is not paused") |
||||
return |
||||
} |
||||
|
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
return |
||||
default: |
||||
} |
||||
|
||||
lock.Lock() |
||||
pushBack = false |
||||
lock.Unlock() |
||||
|
||||
pausable.Resume() |
||||
_, resumed = pausable.IsPausedIsResumed() |
||||
select { |
||||
case <-resumed: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "Queue should be resumed") |
||||
return |
||||
} |
||||
|
||||
var result3, result4 *testData |
||||
|
||||
select { |
||||
case result3 = <-handleChan: |
||||
case <-time.After(1 * time.Second): |
||||
assert.Fail(t, "Handler processing should have resumed") |
||||
return |
||||
} |
||||
select { |
||||
case result4 = <-handleChan: |
||||
case <-time.After(1 * time.Second): |
||||
assert.Fail(t, "Handler processing should have resumed") |
||||
return |
||||
} |
||||
if result4.TestString == test1.TestString { |
||||
result3, result4 = result4, result3 |
||||
} |
||||
assert.Equal(t, test1.TestString, result3.TestString) |
||||
assert.Equal(t, test1.TestInt, result3.TestInt) |
||||
|
||||
assert.Equal(t, test2.TestString, result4.TestString) |
||||
assert.Equal(t, test2.TestInt, result4.TestInt) |
||||
|
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueShutdown)) |
||||
copy(callbacks, queueShutdown) |
||||
queueShutdown = queueShutdown[:0] |
||||
lock.Unlock() |
||||
// Now shutdown the queue
|
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
// terminate the queue
|
||||
lock.Lock() |
||||
callbacks = make([]func(), len(queueTerminate)) |
||||
copy(callbacks, queueTerminate) |
||||
queueShutdown = queueTerminate[:0] |
||||
lock.Unlock() |
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
|
||||
select { |
||||
case <-time.After(10 * time.Second): |
||||
assert.Fail(t, "Queue should have terminated") |
||||
return |
||||
case <-terminated: |
||||
} |
||||
} |
||||
@ -1,147 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestLevelQueue(t *testing.T) { |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
assert.True(t, len(data) == 2) |
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
var lock sync.Mutex |
||||
queueShutdown := []func(){} |
||||
queueTerminate := []func(){} |
||||
|
||||
tmpDir := t.TempDir() |
||||
|
||||
queue, err := NewLevelQueue(handle, LevelQueueConfiguration{ |
||||
ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 20, |
||||
BatchLength: 2, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 5, |
||||
MaxWorkers: 10, |
||||
}, |
||||
Workers: 1, |
||||
}, |
||||
DataDir: tmpDir, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
lock.Unlock() |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
lock.Unlock() |
||||
}) |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
|
||||
err = queue.Push(&test1) |
||||
assert.NoError(t, err) |
||||
go func() { |
||||
err := queue.Push(&test2) |
||||
assert.NoError(t, err) |
||||
}() |
||||
|
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
result2 := <-handleChan |
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
err = queue.Push(test1) |
||||
assert.Error(t, err) |
||||
|
||||
lock.Lock() |
||||
for _, callback := range queueShutdown { |
||||
callback() |
||||
} |
||||
lock.Unlock() |
||||
|
||||
time.Sleep(200 * time.Millisecond) |
||||
err = queue.Push(&test1) |
||||
assert.NoError(t, err) |
||||
err = queue.Push(&test2) |
||||
assert.NoError(t, err) |
||||
select { |
||||
case <-handleChan: |
||||
assert.Fail(t, "Handler processing should have stopped") |
||||
default: |
||||
} |
||||
lock.Lock() |
||||
for _, callback := range queueTerminate { |
||||
callback() |
||||
} |
||||
lock.Unlock() |
||||
|
||||
// Reopen queue
|
||||
queue, err = NewWrappedQueue(handle, |
||||
WrappedQueueConfiguration{ |
||||
Underlying: LevelQueueType, |
||||
Config: LevelQueueConfiguration{ |
||||
ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 20, |
||||
BatchLength: 2, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 5, |
||||
MaxWorkers: 10, |
||||
}, |
||||
Workers: 1, |
||||
}, |
||||
DataDir: tmpDir, |
||||
}, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go queue.Run(func(shutdown func()) { |
||||
lock.Lock() |
||||
queueShutdown = append(queueShutdown, shutdown) |
||||
lock.Unlock() |
||||
}, func(terminate func()) { |
||||
lock.Lock() |
||||
queueTerminate = append(queueTerminate, terminate) |
||||
lock.Unlock() |
||||
}) |
||||
|
||||
result3 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result3.TestString) |
||||
assert.Equal(t, test1.TestInt, result3.TestInt) |
||||
|
||||
result4 := <-handleChan |
||||
assert.Equal(t, test2.TestString, result4.TestString) |
||||
assert.Equal(t, test2.TestInt, result4.TestInt) |
||||
|
||||
lock.Lock() |
||||
for _, callback := range queueShutdown { |
||||
callback() |
||||
} |
||||
for _, callback := range queueTerminate { |
||||
callback() |
||||
} |
||||
lock.Unlock() |
||||
} |
||||
@ -1,137 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"code.gitea.io/gitea/modules/graceful" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"github.com/redis/go-redis/v9" |
||||
) |
||||
|
||||
// RedisQueueType is the type for redis queue
|
||||
const RedisQueueType Type = "redis" |
||||
|
||||
// RedisQueueConfiguration is the configuration for the redis queue
|
||||
type RedisQueueConfiguration struct { |
||||
ByteFIFOQueueConfiguration |
||||
RedisByteFIFOConfiguration |
||||
} |
||||
|
||||
// RedisQueue redis queue
|
||||
type RedisQueue struct { |
||||
*ByteFIFOQueue |
||||
} |
||||
|
||||
// NewRedisQueue creates single redis or cluster redis queue
|
||||
func NewRedisQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(RedisQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(RedisQueueConfiguration) |
||||
|
||||
byteFIFO, err := NewRedisByteFIFO(config.RedisByteFIFOConfiguration) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
byteFIFOQueue, err := NewByteFIFOQueue(RedisQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
queue := &RedisQueue{ |
||||
ByteFIFOQueue: byteFIFOQueue, |
||||
} |
||||
|
||||
queue.qid = GetManager().Add(queue, RedisQueueType, config, exemplar) |
||||
|
||||
return queue, nil |
||||
} |
||||
|
||||
type redisClient interface { |
||||
RPush(ctx context.Context, key string, args ...interface{}) *redis.IntCmd |
||||
LPush(ctx context.Context, key string, args ...interface{}) *redis.IntCmd |
||||
LPop(ctx context.Context, key string) *redis.StringCmd |
||||
LLen(ctx context.Context, key string) *redis.IntCmd |
||||
SAdd(ctx context.Context, key string, members ...interface{}) *redis.IntCmd |
||||
SRem(ctx context.Context, key string, members ...interface{}) *redis.IntCmd |
||||
SIsMember(ctx context.Context, key string, member interface{}) *redis.BoolCmd |
||||
Ping(ctx context.Context) *redis.StatusCmd |
||||
Close() error |
||||
} |
||||
|
||||
var _ ByteFIFO = &RedisByteFIFO{} |
||||
|
||||
// RedisByteFIFO represents a ByteFIFO formed from a redisClient
|
||||
type RedisByteFIFO struct { |
||||
client redisClient |
||||
|
||||
queueName string |
||||
} |
||||
|
||||
// RedisByteFIFOConfiguration is the configuration for the RedisByteFIFO
|
||||
type RedisByteFIFOConfiguration struct { |
||||
ConnectionString string |
||||
QueueName string |
||||
} |
||||
|
||||
// NewRedisByteFIFO creates a ByteFIFO formed from a redisClient
|
||||
func NewRedisByteFIFO(config RedisByteFIFOConfiguration) (*RedisByteFIFO, error) { |
||||
fifo := &RedisByteFIFO{ |
||||
queueName: config.QueueName, |
||||
} |
||||
fifo.client = nosql.GetManager().GetRedisClient(config.ConnectionString) |
||||
if err := fifo.client.Ping(graceful.GetManager().ShutdownContext()).Err(); err != nil { |
||||
return nil, err |
||||
} |
||||
return fifo, nil |
||||
} |
||||
|
||||
// PushFunc pushes data to the end of the fifo and calls the callback if it is added
|
||||
func (fifo *RedisByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { |
||||
if fn != nil { |
||||
if err := fn(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return fifo.client.RPush(ctx, fifo.queueName, data).Err() |
||||
} |
||||
|
||||
// PushBack pushes data to the top of the fifo
|
||||
func (fifo *RedisByteFIFO) PushBack(ctx context.Context, data []byte) error { |
||||
return fifo.client.LPush(ctx, fifo.queueName, data).Err() |
||||
} |
||||
|
||||
// Pop pops data from the start of the fifo
|
||||
func (fifo *RedisByteFIFO) Pop(ctx context.Context) ([]byte, error) { |
||||
data, err := fifo.client.LPop(ctx, fifo.queueName).Bytes() |
||||
if err == nil || err == redis.Nil { |
||||
return data, nil |
||||
} |
||||
return data, err |
||||
} |
||||
|
||||
// Close this fifo
|
||||
func (fifo *RedisByteFIFO) Close() error { |
||||
return fifo.client.Close() |
||||
} |
||||
|
||||
// Len returns the length of the fifo
|
||||
func (fifo *RedisByteFIFO) Len(ctx context.Context) int64 { |
||||
val, err := fifo.client.LLen(ctx, fifo.queueName).Result() |
||||
if err != nil { |
||||
log.Error("Error whilst getting length of redis queue %s: Error: %v", fifo.queueName, err) |
||||
return -1 |
||||
} |
||||
return val |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[RedisQueueType] = NewRedisQueue |
||||
} |
||||
@ -1,42 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/modules/json" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
type testData struct { |
||||
TestString string |
||||
TestInt int |
||||
} |
||||
|
||||
func TestToConfig(t *testing.T) { |
||||
cfg := testData{ |
||||
TestString: "Config", |
||||
TestInt: 10, |
||||
} |
||||
exemplar := testData{} |
||||
|
||||
cfg2I, err := toConfig(exemplar, cfg) |
||||
assert.NoError(t, err) |
||||
cfg2, ok := (cfg2I).(testData) |
||||
assert.True(t, ok) |
||||
assert.NotEqual(t, cfg2, exemplar) |
||||
assert.Equal(t, &cfg, &cfg2) |
||||
cfgString, err := json.Marshal(cfg) |
||||
assert.NoError(t, err) |
||||
|
||||
cfg3I, err := toConfig(exemplar, cfgString) |
||||
assert.NoError(t, err) |
||||
cfg3, ok := (cfg3I).(testData) |
||||
assert.True(t, ok) |
||||
assert.Equal(t, cfg.TestString, cfg3.TestString) |
||||
assert.Equal(t, cfg.TestInt, cfg3.TestInt) |
||||
assert.NotEqual(t, cfg3, exemplar) |
||||
} |
||||
@ -1,315 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/util" |
||||
) |
||||
|
||||
// WrappedQueueType is the type for a wrapped delayed starting queue
|
||||
const WrappedQueueType Type = "wrapped" |
||||
|
||||
// WrappedQueueConfiguration is the configuration for a WrappedQueue
|
||||
type WrappedQueueConfiguration struct { |
||||
Underlying Type |
||||
Timeout time.Duration |
||||
MaxAttempts int |
||||
Config interface{} |
||||
QueueLength int |
||||
Name string |
||||
} |
||||
|
||||
type delayedStarter struct { |
||||
internal Queue |
||||
underlying Type |
||||
cfg interface{} |
||||
timeout time.Duration |
||||
maxAttempts int |
||||
name string |
||||
} |
||||
|
||||
// setInternal must be called with the lock locked.
|
||||
func (q *delayedStarter) setInternal(atShutdown func(func()), handle HandlerFunc, exemplar interface{}) error { |
||||
var ctx context.Context |
||||
var cancel context.CancelFunc |
||||
if q.timeout > 0 { |
||||
ctx, cancel = context.WithTimeout(context.Background(), q.timeout) |
||||
} else { |
||||
ctx, cancel = context.WithCancel(context.Background()) |
||||
} |
||||
|
||||
defer cancel() |
||||
// Ensure we also stop at shutdown
|
||||
atShutdown(cancel) |
||||
|
||||
i := 1 |
||||
for q.internal == nil { |
||||
select { |
||||
case <-ctx.Done(): |
||||
cfg := q.cfg |
||||
if s, ok := cfg.([]byte); ok { |
||||
cfg = string(s) |
||||
} |
||||
return fmt.Errorf("timedout creating queue %v with cfg %#v in %s", q.underlying, cfg, q.name) |
||||
default: |
||||
queue, err := NewQueue(q.underlying, handle, q.cfg, exemplar) |
||||
if err == nil { |
||||
q.internal = queue |
||||
break |
||||
} |
||||
if err.Error() != "resource temporarily unavailable" { |
||||
if bs, ok := q.cfg.([]byte); ok { |
||||
log.Warn("[Attempt: %d] Failed to create queue: %v for %s cfg: %s error: %v", i, q.underlying, q.name, string(bs), err) |
||||
} else { |
||||
log.Warn("[Attempt: %d] Failed to create queue: %v for %s cfg: %#v error: %v", i, q.underlying, q.name, q.cfg, err) |
||||
} |
||||
} |
||||
i++ |
||||
if q.maxAttempts > 0 && i > q.maxAttempts { |
||||
if bs, ok := q.cfg.([]byte); ok { |
||||
return fmt.Errorf("unable to create queue %v for %s with cfg %s by max attempts: error: %w", q.underlying, q.name, string(bs), err) |
||||
} |
||||
return fmt.Errorf("unable to create queue %v for %s with cfg %#v by max attempts: error: %w", q.underlying, q.name, q.cfg, err) |
||||
} |
||||
sleepTime := 100 * time.Millisecond |
||||
if q.timeout > 0 && q.maxAttempts > 0 { |
||||
sleepTime = (q.timeout - 200*time.Millisecond) / time.Duration(q.maxAttempts) |
||||
} |
||||
t := time.NewTimer(sleepTime) |
||||
select { |
||||
case <-ctx.Done(): |
||||
util.StopTimer(t) |
||||
case <-t.C: |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// WrappedQueue wraps a delayed starting queue
|
||||
type WrappedQueue struct { |
||||
delayedStarter |
||||
lock sync.Mutex |
||||
handle HandlerFunc |
||||
exemplar interface{} |
||||
channel chan Data |
||||
numInQueue int64 |
||||
} |
||||
|
||||
// NewWrappedQueue will attempt to create a queue of the provided type,
|
||||
// but if there is a problem creating this queue it will instead create
|
||||
// a WrappedQueue with delayed startup of the queue instead and a
|
||||
// channel which will be redirected to the queue
|
||||
func NewWrappedQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(WrappedQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(WrappedQueueConfiguration) |
||||
|
||||
queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar) |
||||
if err == nil { |
||||
// Just return the queue there is no need to wrap
|
||||
return queue, nil |
||||
} |
||||
if IsErrInvalidConfiguration(err) { |
||||
// Retrying ain't gonna make this any better...
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg} |
||||
} |
||||
|
||||
queue = &WrappedQueue{ |
||||
handle: handle, |
||||
channel: make(chan Data, config.QueueLength), |
||||
exemplar: exemplar, |
||||
delayedStarter: delayedStarter{ |
||||
cfg: config.Config, |
||||
underlying: config.Underlying, |
||||
timeout: config.Timeout, |
||||
maxAttempts: config.MaxAttempts, |
||||
name: config.Name, |
||||
}, |
||||
} |
||||
_ = GetManager().Add(queue, WrappedQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
// Name returns the name of the queue
|
||||
func (q *WrappedQueue) Name() string { |
||||
return q.name + "-wrapper" |
||||
} |
||||
|
||||
// Push will push the data to the internal channel checking it against the exemplar
|
||||
func (q *WrappedQueue) Push(data Data) error { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) |
||||
} |
||||
atomic.AddInt64(&q.numInQueue, 1) |
||||
q.channel <- data |
||||
return nil |
||||
} |
||||
|
||||
func (q *WrappedQueue) flushInternalWithContext(ctx context.Context) error { |
||||
q.lock.Lock() |
||||
if q.internal == nil { |
||||
q.lock.Unlock() |
||||
return fmt.Errorf("not ready to flush wrapped queue %s yet", q.Name()) |
||||
} |
||||
q.lock.Unlock() |
||||
select { |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
default: |
||||
} |
||||
return q.internal.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// Flush flushes the queue and blocks till the queue is empty
|
||||
func (q *WrappedQueue) Flush(timeout time.Duration) error { |
||||
var ctx context.Context |
||||
var cancel context.CancelFunc |
||||
if timeout > 0 { |
||||
ctx, cancel = context.WithTimeout(context.Background(), timeout) |
||||
} else { |
||||
ctx, cancel = context.WithCancel(context.Background()) |
||||
} |
||||
defer cancel() |
||||
return q.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// FlushWithContext implements the final part of Flushable
|
||||
func (q *WrappedQueue) FlushWithContext(ctx context.Context) error { |
||||
log.Trace("WrappedQueue: %s FlushWithContext", q.Name()) |
||||
errChan := make(chan error, 1) |
||||
go func() { |
||||
errChan <- q.flushInternalWithContext(ctx) |
||||
close(errChan) |
||||
}() |
||||
|
||||
select { |
||||
case err := <-errChan: |
||||
return err |
||||
case <-ctx.Done(): |
||||
go func() { |
||||
<-errChan |
||||
}() |
||||
return ctx.Err() |
||||
} |
||||
} |
||||
|
||||
// IsEmpty checks whether the queue is empty
|
||||
func (q *WrappedQueue) IsEmpty() bool { |
||||
if atomic.LoadInt64(&q.numInQueue) != 0 { |
||||
return false |
||||
} |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return false |
||||
} |
||||
return q.internal.IsEmpty() |
||||
} |
||||
|
||||
// Run starts to run the queue and attempts to create the internal queue
|
||||
func (q *WrappedQueue) Run(atShutdown, atTerminate func(func())) { |
||||
log.Debug("WrappedQueue: %s Starting", q.name) |
||||
q.lock.Lock() |
||||
if q.internal == nil { |
||||
err := q.setInternal(atShutdown, q.handle, q.exemplar) |
||||
q.lock.Unlock() |
||||
if err != nil { |
||||
log.Fatal("Unable to set the internal queue for %s Error: %v", q.Name(), err) |
||||
return |
||||
} |
||||
go func() { |
||||
for data := range q.channel { |
||||
_ = q.internal.Push(data) |
||||
atomic.AddInt64(&q.numInQueue, -1) |
||||
} |
||||
}() |
||||
} else { |
||||
q.lock.Unlock() |
||||
} |
||||
|
||||
q.internal.Run(atShutdown, atTerminate) |
||||
log.Trace("WrappedQueue: %s Done", q.name) |
||||
} |
||||
|
||||
// Shutdown this queue and stop processing
|
||||
func (q *WrappedQueue) Shutdown() { |
||||
log.Trace("WrappedQueue: %s Shutting down", q.name) |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return |
||||
} |
||||
if shutdownable, ok := q.internal.(Shutdownable); ok { |
||||
shutdownable.Shutdown() |
||||
} |
||||
log.Debug("WrappedQueue: %s Shutdown", q.name) |
||||
} |
||||
|
||||
// Terminate this queue and close the queue
|
||||
func (q *WrappedQueue) Terminate() { |
||||
log.Trace("WrappedQueue: %s Terminating", q.name) |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return |
||||
} |
||||
if shutdownable, ok := q.internal.(Shutdownable); ok { |
||||
shutdownable.Terminate() |
||||
} |
||||
log.Debug("WrappedQueue: %s Terminated", q.name) |
||||
} |
||||
|
||||
// IsPaused will return if the pool or queue is paused
|
||||
func (q *WrappedQueue) IsPaused() bool { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
pausable, ok := q.internal.(Pausable) |
||||
return ok && pausable.IsPaused() |
||||
} |
||||
|
||||
// Pause will pause the pool or queue
|
||||
func (q *WrappedQueue) Pause() { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if pausable, ok := q.internal.(Pausable); ok { |
||||
pausable.Pause() |
||||
} |
||||
} |
||||
|
||||
// Resume will resume the pool or queue
|
||||
func (q *WrappedQueue) Resume() { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if pausable, ok := q.internal.(Pausable); ok { |
||||
pausable.Resume() |
||||
} |
||||
} |
||||
|
||||
// IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed
|
||||
func (q *WrappedQueue) IsPausedIsResumed() (paused, resumed <-chan struct{}) { |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if pausable, ok := q.internal.(Pausable); ok { |
||||
return pausable.IsPausedIsResumed() |
||||
} |
||||
return context.Background().Done(), closedChan |
||||
} |
||||
|
||||
var closedChan chan struct{} |
||||
|
||||
func init() { |
||||
queuesMap[WrappedQueueType] = NewWrappedQueue |
||||
closedChan = make(chan struct{}) |
||||
close(closedChan) |
||||
} |
||||
@ -1,126 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"fmt" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/json" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
) |
||||
|
||||
func validType(t string) (Type, error) { |
||||
if len(t) == 0 { |
||||
return PersistableChannelQueueType, nil |
||||
} |
||||
for _, typ := range RegisteredTypes() { |
||||
if t == string(typ) { |
||||
return typ, nil |
||||
} |
||||
} |
||||
return PersistableChannelQueueType, fmt.Errorf("unknown queue type: %s defaulting to %s", t, string(PersistableChannelQueueType)) |
||||
} |
||||
|
||||
func getQueueSettings(name string) (setting.QueueSettings, []byte) { |
||||
q := setting.GetQueueSettings(name) |
||||
cfg, err := json.Marshal(q) |
||||
if err != nil { |
||||
log.Error("Unable to marshall generic options: %v Error: %v", q, err) |
||||
log.Error("Unable to create queue for %s", name, err) |
||||
return q, []byte{} |
||||
} |
||||
return q, cfg |
||||
} |
||||
|
||||
// CreateQueue for name with provided handler and exemplar
|
||||
func CreateQueue(name string, handle HandlerFunc, exemplar interface{}) Queue { |
||||
q, cfg := getQueueSettings(name) |
||||
if len(cfg) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
typ, err := validType(q.Type) |
||||
if err != nil { |
||||
log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ)) |
||||
} |
||||
|
||||
returnable, err := NewQueue(typ, handle, cfg, exemplar) |
||||
if q.WrapIfNecessary && err != nil { |
||||
log.Warn("Unable to create queue for %s: %v", name, err) |
||||
log.Warn("Attempting to create wrapped queue") |
||||
returnable, err = NewQueue(WrappedQueueType, handle, WrappedQueueConfiguration{ |
||||
Underlying: typ, |
||||
Timeout: q.Timeout, |
||||
MaxAttempts: q.MaxAttempts, |
||||
Config: cfg, |
||||
QueueLength: q.QueueLength, |
||||
Name: name, |
||||
}, exemplar) |
||||
} |
||||
if err != nil { |
||||
log.Error("Unable to create queue for %s: %v", name, err) |
||||
return nil |
||||
} |
||||
|
||||
// Sanity check configuration
|
||||
if q.Workers == 0 && (q.BoostTimeout == 0 || q.BoostWorkers == 0 || q.MaxWorkers == 0) { |
||||
log.Warn("Queue: %s is configured to be non-scaling and have no workers\n - this configuration is likely incorrect and could cause Gitea to block", q.Name) |
||||
if pausable, ok := returnable.(Pausable); ok { |
||||
log.Warn("Queue: %s is being paused to prevent data-loss, add workers manually and unpause.", q.Name) |
||||
pausable.Pause() |
||||
} |
||||
} |
||||
|
||||
return returnable |
||||
} |
||||
|
||||
// CreateUniqueQueue for name with provided handler and exemplar
|
||||
func CreateUniqueQueue(name string, handle HandlerFunc, exemplar interface{}) UniqueQueue { |
||||
q, cfg := getQueueSettings(name) |
||||
if len(cfg) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
if len(q.Type) > 0 && q.Type != "dummy" && q.Type != "immediate" && !strings.HasPrefix(q.Type, "unique-") { |
||||
q.Type = "unique-" + q.Type |
||||
} |
||||
|
||||
typ, err := validType(q.Type) |
||||
if err != nil || typ == PersistableChannelQueueType { |
||||
typ = PersistableChannelUniqueQueueType |
||||
if err != nil { |
||||
log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ)) |
||||
} |
||||
} |
||||
|
||||
returnable, err := NewQueue(typ, handle, cfg, exemplar) |
||||
if q.WrapIfNecessary && err != nil { |
||||
log.Warn("Unable to create unique queue for %s: %v", name, err) |
||||
log.Warn("Attempting to create wrapped queue") |
||||
returnable, err = NewQueue(WrappedUniqueQueueType, handle, WrappedUniqueQueueConfiguration{ |
||||
Underlying: typ, |
||||
Timeout: q.Timeout, |
||||
MaxAttempts: q.MaxAttempts, |
||||
Config: cfg, |
||||
QueueLength: q.QueueLength, |
||||
}, exemplar) |
||||
} |
||||
if err != nil { |
||||
log.Error("Unable to create unique queue for %s: %v", name, err) |
||||
return nil |
||||
} |
||||
|
||||
// Sanity check configuration
|
||||
if q.Workers == 0 && (q.BoostTimeout == 0 || q.BoostWorkers == 0 || q.MaxWorkers == 0) { |
||||
log.Warn("Queue: %s is configured to be non-scaling and have no workers\n - this configuration is likely incorrect and could cause Gitea to block", q.Name) |
||||
if pausable, ok := returnable.(Pausable); ok { |
||||
log.Warn("Queue: %s is being paused to prevent data-loss, add workers manually and unpause.", q.Name) |
||||
pausable.Pause() |
||||
} |
||||
} |
||||
|
||||
return returnable.(UniqueQueue) |
||||
} |
||||
@ -0,0 +1,40 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
) |
||||
|
||||
// testStateRecorder is used to record state changes for testing, to help debug async behaviors
|
||||
type testStateRecorder struct { |
||||
records []string |
||||
mu sync.Mutex |
||||
} |
||||
|
||||
var testRecorder = &testStateRecorder{} |
||||
|
||||
func (t *testStateRecorder) Record(format string, args ...any) { |
||||
t.mu.Lock() |
||||
t.records = append(t.records, fmt.Sprintf(format, args...)) |
||||
if len(t.records) > 1000 { |
||||
t.records = t.records[len(t.records)-1000:] |
||||
} |
||||
t.mu.Unlock() |
||||
} |
||||
|
||||
func (t *testStateRecorder) Records() []string { |
||||
t.mu.Lock() |
||||
r := make([]string, len(t.records)) |
||||
copy(r, t.records) |
||||
t.mu.Unlock() |
||||
return r |
||||
} |
||||
|
||||
func (t *testStateRecorder) Reset() { |
||||
t.mu.Lock() |
||||
t.records = nil |
||||
t.mu.Unlock() |
||||
} |
||||
@ -1,28 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"fmt" |
||||
) |
||||
|
||||
// UniqueQueue defines a queue which guarantees only one instance of same
|
||||
// data is in the queue. Instances with same identity will be
|
||||
// discarded if there is already one in the line.
|
||||
//
|
||||
// This queue is particularly useful for preventing duplicated task
|
||||
// of same purpose - please note that this does not guarantee that a particular
|
||||
// task cannot be processed twice or more at the same time. Uniqueness is
|
||||
// only guaranteed whilst the task is waiting in the queue.
|
||||
//
|
||||
// Users of this queue should be careful to push only the identifier of the
|
||||
// data
|
||||
type UniqueQueue interface { |
||||
Queue |
||||
PushFunc(Data, func() error) error |
||||
Has(Data) (bool, error) |
||||
} |
||||
|
||||
// ErrAlreadyInQueue is returned when trying to push data to the queue that is already in the queue
|
||||
var ErrAlreadyInQueue = fmt.Errorf("already in queue") |
||||
@ -1,212 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"runtime/pprof" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/container" |
||||
"code.gitea.io/gitea/modules/json" |
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// ChannelUniqueQueueType is the type for channel queue
|
||||
const ChannelUniqueQueueType Type = "unique-channel" |
||||
|
||||
// ChannelUniqueQueueConfiguration is the configuration for a ChannelUniqueQueue
|
||||
type ChannelUniqueQueueConfiguration ChannelQueueConfiguration |
||||
|
||||
// ChannelUniqueQueue implements UniqueQueue
|
||||
//
|
||||
// It is basically a thin wrapper around a WorkerPool but keeps a store of
|
||||
// what has been pushed within a table.
|
||||
//
|
||||
// Please note that this Queue does not guarantee that a particular
|
||||
// task cannot be processed twice or more at the same time. Uniqueness is
|
||||
// only guaranteed whilst the task is waiting in the queue.
|
||||
type ChannelUniqueQueue struct { |
||||
*WorkerPool |
||||
lock sync.Mutex |
||||
table container.Set[string] |
||||
shutdownCtx context.Context |
||||
shutdownCtxCancel context.CancelFunc |
||||
terminateCtx context.Context |
||||
terminateCtxCancel context.CancelFunc |
||||
exemplar interface{} |
||||
workers int |
||||
name string |
||||
} |
||||
|
||||
// NewChannelUniqueQueue create a memory channel queue
|
||||
func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(ChannelUniqueQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(ChannelUniqueQueueConfiguration) |
||||
if config.BatchLength == 0 { |
||||
config.BatchLength = 1 |
||||
} |
||||
|
||||
terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) |
||||
shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) |
||||
|
||||
queue := &ChannelUniqueQueue{ |
||||
table: make(container.Set[string]), |
||||
shutdownCtx: shutdownCtx, |
||||
shutdownCtxCancel: shutdownCtxCancel, |
||||
terminateCtx: terminateCtx, |
||||
terminateCtxCancel: terminateCtxCancel, |
||||
exemplar: exemplar, |
||||
workers: config.Workers, |
||||
name: config.Name, |
||||
} |
||||
queue.WorkerPool = NewWorkerPool(func(data ...Data) (unhandled []Data) { |
||||
for _, datum := range data { |
||||
// No error is possible here because PushFunc ensures that this can be marshalled
|
||||
bs, _ := json.Marshal(datum) |
||||
|
||||
queue.lock.Lock() |
||||
queue.table.Remove(string(bs)) |
||||
queue.lock.Unlock() |
||||
|
||||
if u := handle(datum); u != nil { |
||||
if queue.IsPaused() { |
||||
// We can only pushback to the channel if we're paused.
|
||||
go func() { |
||||
if err := queue.Push(u[0]); err != nil { |
||||
log.Error("Unable to push back to queue %d. Error: %v", queue.qid, err) |
||||
} |
||||
}() |
||||
} else { |
||||
unhandled = append(unhandled, u...) |
||||
} |
||||
} |
||||
} |
||||
return unhandled |
||||
}, config.WorkerPoolConfiguration) |
||||
|
||||
queue.qid = GetManager().Add(queue, ChannelUniqueQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
// Run starts to run the queue
|
||||
func (q *ChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { |
||||
pprof.SetGoroutineLabels(q.baseCtx) |
||||
atShutdown(q.Shutdown) |
||||
atTerminate(q.Terminate) |
||||
log.Debug("ChannelUniqueQueue: %s Starting", q.name) |
||||
_ = q.AddWorkers(q.workers, 0) |
||||
} |
||||
|
||||
// Push will push data into the queue if the data is not already in the queue
|
||||
func (q *ChannelUniqueQueue) Push(data Data) error { |
||||
return q.PushFunc(data, nil) |
||||
} |
||||
|
||||
// PushFunc will push data into the queue
|
||||
func (q *ChannelUniqueQueue) PushFunc(data Data, fn func() error) error { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name) |
||||
} |
||||
|
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
q.lock.Lock() |
||||
locked := true |
||||
defer func() { |
||||
if locked { |
||||
q.lock.Unlock() |
||||
} |
||||
}() |
||||
if !q.table.Add(string(bs)) { |
||||
return ErrAlreadyInQueue |
||||
} |
||||
// FIXME: We probably need to implement some sort of limit here
|
||||
// If the downstream queue blocks this table will grow without limit
|
||||
if fn != nil { |
||||
err := fn() |
||||
if err != nil { |
||||
q.table.Remove(string(bs)) |
||||
return err |
||||
} |
||||
} |
||||
locked = false |
||||
q.lock.Unlock() |
||||
q.WorkerPool.Push(data) |
||||
return nil |
||||
} |
||||
|
||||
// Has checks if the data is in the queue
|
||||
func (q *ChannelUniqueQueue) Has(data Data) (bool, error) { |
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
|
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
return q.table.Contains(string(bs)), nil |
||||
} |
||||
|
||||
// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager
|
||||
func (q *ChannelUniqueQueue) Flush(timeout time.Duration) error { |
||||
if q.IsPaused() { |
||||
return nil |
||||
} |
||||
ctx, cancel := q.commonRegisterWorkers(1, timeout, true) |
||||
defer cancel() |
||||
return q.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// Shutdown processing from this queue
|
||||
func (q *ChannelUniqueQueue) Shutdown() { |
||||
log.Trace("ChannelUniqueQueue: %s Shutting down", q.name) |
||||
select { |
||||
case <-q.shutdownCtx.Done(): |
||||
return |
||||
default: |
||||
} |
||||
go func() { |
||||
log.Trace("ChannelUniqueQueue: %s Flushing", q.name) |
||||
if err := q.FlushWithContext(q.terminateCtx); err != nil { |
||||
if !q.IsEmpty() { |
||||
log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name) |
||||
} |
||||
return |
||||
} |
||||
log.Debug("ChannelUniqueQueue: %s Flushed", q.name) |
||||
}() |
||||
q.shutdownCtxCancel() |
||||
log.Debug("ChannelUniqueQueue: %s Shutdown", q.name) |
||||
} |
||||
|
||||
// Terminate this queue and close the queue
|
||||
func (q *ChannelUniqueQueue) Terminate() { |
||||
log.Trace("ChannelUniqueQueue: %s Terminating", q.name) |
||||
q.Shutdown() |
||||
select { |
||||
case <-q.terminateCtx.Done(): |
||||
return |
||||
default: |
||||
} |
||||
q.terminateCtxCancel() |
||||
q.baseCtxFinished() |
||||
log.Debug("ChannelUniqueQueue: %s Terminated", q.name) |
||||
} |
||||
|
||||
// Name returns the name of this queue
|
||||
func (q *ChannelUniqueQueue) Name() string { |
||||
return q.name |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[ChannelUniqueQueueType] = NewChannelUniqueQueue |
||||
} |
||||
@ -1,258 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestChannelUniqueQueue(t *testing.T) { |
||||
_ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
nilFn := func(_ func()) {} |
||||
|
||||
queue, err := NewChannelUniqueQueue(handle, |
||||
ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 0, |
||||
MaxWorkers: 10, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 5, |
||||
Name: "TestChannelQueue", |
||||
}, |
||||
Workers: 0, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
assert.Equal(t, queue.(*ChannelUniqueQueue).WorkerPool.boostWorkers, 5) |
||||
|
||||
go queue.Run(nilFn, nilFn) |
||||
|
||||
test1 := testData{"A", 1} |
||||
go queue.Push(&test1) |
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
err = queue.Push(test1) |
||||
assert.Error(t, err) |
||||
} |
||||
|
||||
func TestChannelUniqueQueue_Batch(t *testing.T) { |
||||
_ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) |
||||
|
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
nilFn := func(_ func()) {} |
||||
|
||||
queue, err := NewChannelUniqueQueue(handle, |
||||
ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 20, |
||||
BatchLength: 2, |
||||
BlockTimeout: 0, |
||||
BoostTimeout: 0, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
}, |
||||
Workers: 1, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go queue.Run(nilFn, nilFn) |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
|
||||
queue.Push(&test1) |
||||
go queue.Push(&test2) |
||||
|
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
result2 := <-handleChan |
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
err = queue.Push(test1) |
||||
assert.Error(t, err) |
||||
} |
||||
|
||||
func TestChannelUniqueQueue_Pause(t *testing.T) { |
||||
_ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) |
||||
|
||||
lock := sync.Mutex{} |
||||
var queue Queue |
||||
var err error |
||||
pushBack := false |
||||
handleChan := make(chan *testData) |
||||
handle := func(data ...Data) []Data { |
||||
lock.Lock() |
||||
if pushBack { |
||||
if pausable, ok := queue.(Pausable); ok { |
||||
pausable.Pause() |
||||
} |
||||
pushBack = false |
||||
lock.Unlock() |
||||
return data |
||||
} |
||||
lock.Unlock() |
||||
|
||||
for _, datum := range data { |
||||
testDatum := datum.(*testData) |
||||
handleChan <- testDatum |
||||
} |
||||
return nil |
||||
} |
||||
nilFn := func(_ func()) {} |
||||
|
||||
queue, err = NewChannelUniqueQueue(handle, |
||||
ChannelQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: 20, |
||||
BatchLength: 1, |
||||
BlockTimeout: 0, |
||||
BoostTimeout: 0, |
||||
BoostWorkers: 0, |
||||
MaxWorkers: 10, |
||||
}, |
||||
Workers: 1, |
||||
}, &testData{}) |
||||
assert.NoError(t, err) |
||||
|
||||
go queue.Run(nilFn, nilFn) |
||||
|
||||
test1 := testData{"A", 1} |
||||
test2 := testData{"B", 2} |
||||
queue.Push(&test1) |
||||
|
||||
pausable, ok := queue.(Pausable) |
||||
if !assert.True(t, ok) { |
||||
return |
||||
} |
||||
result1 := <-handleChan |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
|
||||
pausable.Pause() |
||||
|
||||
paused, resumed := pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-resumed: |
||||
assert.Fail(t, "Queue should not be resumed") |
||||
return |
||||
default: |
||||
assert.Fail(t, "Queue is not paused") |
||||
return |
||||
} |
||||
|
||||
queue.Push(&test2) |
||||
|
||||
var result2 *testData |
||||
select { |
||||
case result2 = <-handleChan: |
||||
assert.Fail(t, "handler chan should be empty") |
||||
case <-time.After(100 * time.Millisecond): |
||||
} |
||||
|
||||
assert.Nil(t, result2) |
||||
|
||||
pausable.Resume() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
default: |
||||
assert.Fail(t, "Queue should be resumed") |
||||
} |
||||
|
||||
select { |
||||
case result2 = <-handleChan: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "handler chan should contain test2") |
||||
} |
||||
|
||||
assert.Equal(t, test2.TestString, result2.TestString) |
||||
assert.Equal(t, test2.TestInt, result2.TestInt) |
||||
|
||||
lock.Lock() |
||||
pushBack = true |
||||
lock.Unlock() |
||||
|
||||
paused, resumed = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
assert.Fail(t, "Queue should not be paused") |
||||
return |
||||
case <-resumed: |
||||
default: |
||||
assert.Fail(t, "Queue is not resumed") |
||||
return |
||||
} |
||||
|
||||
queue.Push(&test1) |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-handleChan: |
||||
assert.Fail(t, "handler chan should not contain test1") |
||||
return |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "queue should be paused") |
||||
return |
||||
} |
||||
|
||||
paused, resumed = pausable.IsPausedIsResumed() |
||||
|
||||
select { |
||||
case <-paused: |
||||
case <-resumed: |
||||
assert.Fail(t, "Queue should not be resumed") |
||||
return |
||||
default: |
||||
assert.Fail(t, "Queue is not paused") |
||||
return |
||||
} |
||||
|
||||
pausable.Resume() |
||||
|
||||
select { |
||||
case <-resumed: |
||||
default: |
||||
assert.Fail(t, "Queue should be resumed") |
||||
} |
||||
|
||||
select { |
||||
case result1 = <-handleChan: |
||||
case <-time.After(500 * time.Millisecond): |
||||
assert.Fail(t, "handler chan should contain test1") |
||||
} |
||||
assert.Equal(t, test1.TestString, result1.TestString) |
||||
assert.Equal(t, test1.TestInt, result1.TestInt) |
||||
} |
||||
@ -1,128 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"code.gitea.io/gitea/modules/nosql" |
||||
|
||||
"gitea.com/lunny/levelqueue" |
||||
) |
||||
|
||||
// LevelUniqueQueueType is the type for level queue
|
||||
const LevelUniqueQueueType Type = "unique-level" |
||||
|
||||
// LevelUniqueQueueConfiguration is the configuration for a LevelUniqueQueue
|
||||
type LevelUniqueQueueConfiguration struct { |
||||
ByteFIFOQueueConfiguration |
||||
DataDir string |
||||
ConnectionString string |
||||
QueueName string |
||||
} |
||||
|
||||
// LevelUniqueQueue implements a disk library queue
|
||||
type LevelUniqueQueue struct { |
||||
*ByteFIFOUniqueQueue |
||||
} |
||||
|
||||
// NewLevelUniqueQueue creates a ledis local queue
|
||||
//
|
||||
// Please note that this Queue does not guarantee that a particular
|
||||
// task cannot be processed twice or more at the same time. Uniqueness is
|
||||
// only guaranteed whilst the task is waiting in the queue.
|
||||
func NewLevelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(LevelUniqueQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(LevelUniqueQueueConfiguration) |
||||
|
||||
if len(config.ConnectionString) == 0 { |
||||
config.ConnectionString = config.DataDir |
||||
} |
||||
config.WaitOnEmpty = true |
||||
|
||||
byteFIFO, err := NewLevelUniqueQueueByteFIFO(config.ConnectionString, config.QueueName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
byteFIFOQueue, err := NewByteFIFOUniqueQueue(LevelUniqueQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
queue := &LevelUniqueQueue{ |
||||
ByteFIFOUniqueQueue: byteFIFOQueue, |
||||
} |
||||
queue.qid = GetManager().Add(queue, LevelUniqueQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
var _ UniqueByteFIFO = &LevelUniqueQueueByteFIFO{} |
||||
|
||||
// LevelUniqueQueueByteFIFO represents a ByteFIFO formed from a LevelUniqueQueue
|
||||
type LevelUniqueQueueByteFIFO struct { |
||||
internal *levelqueue.UniqueQueue |
||||
connection string |
||||
} |
||||
|
||||
// NewLevelUniqueQueueByteFIFO creates a new ByteFIFO formed from a LevelUniqueQueue
|
||||
func NewLevelUniqueQueueByteFIFO(connection, prefix string) (*LevelUniqueQueueByteFIFO, error) { |
||||
db, err := nosql.GetManager().GetLevelDB(connection) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
internal, err := levelqueue.NewUniqueQueue(db, []byte(prefix), []byte(prefix+"-unique"), false) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &LevelUniqueQueueByteFIFO{ |
||||
connection: connection, |
||||
internal: internal, |
||||
}, nil |
||||
} |
||||
|
||||
// PushFunc pushes data to the end of the fifo and calls the callback if it is added
|
||||
func (fifo *LevelUniqueQueueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { |
||||
return fifo.internal.LPushFunc(data, fn) |
||||
} |
||||
|
||||
// PushBack pushes data to the top of the fifo
|
||||
func (fifo *LevelUniqueQueueByteFIFO) PushBack(ctx context.Context, data []byte) error { |
||||
return fifo.internal.RPush(data) |
||||
} |
||||
|
||||
// Pop pops data from the start of the fifo
|
||||
func (fifo *LevelUniqueQueueByteFIFO) Pop(ctx context.Context) ([]byte, error) { |
||||
data, err := fifo.internal.RPop() |
||||
if err != nil && err != levelqueue.ErrNotFound { |
||||
return nil, err |
||||
} |
||||
return data, nil |
||||
} |
||||
|
||||
// Len returns the length of the fifo
|
||||
func (fifo *LevelUniqueQueueByteFIFO) Len(ctx context.Context) int64 { |
||||
return fifo.internal.Len() |
||||
} |
||||
|
||||
// Has returns whether the fifo contains this data
|
||||
func (fifo *LevelUniqueQueueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { |
||||
return fifo.internal.Has(data) |
||||
} |
||||
|
||||
// Close this fifo
|
||||
func (fifo *LevelUniqueQueueByteFIFO) Close() error { |
||||
err := fifo.internal.Close() |
||||
_ = nosql.GetManager().CloseLevelDB(fifo.connection) |
||||
return err |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[LevelUniqueQueueType] = NewLevelUniqueQueue |
||||
} |
||||
@ -1,336 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"runtime/pprof" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// PersistableChannelUniqueQueueType is the type for persistable queue
|
||||
const PersistableChannelUniqueQueueType Type = "unique-persistable-channel" |
||||
|
||||
// PersistableChannelUniqueQueueConfiguration is the configuration for a PersistableChannelUniqueQueue
|
||||
type PersistableChannelUniqueQueueConfiguration struct { |
||||
Name string |
||||
DataDir string |
||||
BatchLength int |
||||
QueueLength int |
||||
Timeout time.Duration |
||||
MaxAttempts int |
||||
Workers int |
||||
MaxWorkers int |
||||
BlockTimeout time.Duration |
||||
BoostTimeout time.Duration |
||||
BoostWorkers int |
||||
} |
||||
|
||||
// PersistableChannelUniqueQueue wraps a channel queue and level queue together
|
||||
//
|
||||
// Please note that this Queue does not guarantee that a particular
|
||||
// task cannot be processed twice or more at the same time. Uniqueness is
|
||||
// only guaranteed whilst the task is waiting in the queue.
|
||||
type PersistableChannelUniqueQueue struct { |
||||
channelQueue *ChannelUniqueQueue |
||||
delayedStarter |
||||
lock sync.Mutex |
||||
closed chan struct{} |
||||
} |
||||
|
||||
// NewPersistableChannelUniqueQueue creates a wrapped batched channel queue with persistable level queue backend when shutting down
|
||||
// This differs from a wrapped queue in that the persistent queue is only used to persist at shutdown/terminate
|
||||
func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(PersistableChannelUniqueQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(PersistableChannelUniqueQueueConfiguration) |
||||
|
||||
queue := &PersistableChannelUniqueQueue{ |
||||
closed: make(chan struct{}), |
||||
} |
||||
|
||||
wrappedHandle := func(data ...Data) (failed []Data) { |
||||
for _, unhandled := range handle(data...) { |
||||
if fail := queue.PushBack(unhandled); fail != nil { |
||||
failed = append(failed, fail) |
||||
} |
||||
} |
||||
return failed |
||||
} |
||||
|
||||
channelUniqueQueue, err := NewChannelUniqueQueue(wrappedHandle, ChannelUniqueQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: config.QueueLength, |
||||
BatchLength: config.BatchLength, |
||||
BlockTimeout: config.BlockTimeout, |
||||
BoostTimeout: config.BoostTimeout, |
||||
BoostWorkers: config.BoostWorkers, |
||||
MaxWorkers: config.MaxWorkers, |
||||
Name: config.Name + "-channel", |
||||
}, |
||||
Workers: config.Workers, |
||||
}, exemplar) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// the level backend only needs temporary workers to catch up with the previously dropped work
|
||||
levelCfg := LevelUniqueQueueConfiguration{ |
||||
ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ |
||||
WorkerPoolConfiguration: WorkerPoolConfiguration{ |
||||
QueueLength: config.QueueLength, |
||||
BatchLength: config.BatchLength, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 1, |
||||
MaxWorkers: 5, |
||||
Name: config.Name + "-level", |
||||
}, |
||||
Workers: 0, |
||||
}, |
||||
DataDir: config.DataDir, |
||||
QueueName: config.Name + "-level", |
||||
} |
||||
|
||||
queue.channelQueue = channelUniqueQueue.(*ChannelUniqueQueue) |
||||
|
||||
levelQueue, err := NewLevelUniqueQueue(func(data ...Data) []Data { |
||||
for _, datum := range data { |
||||
err := queue.Push(datum) |
||||
if err != nil && err != ErrAlreadyInQueue { |
||||
log.Error("Unable push to channelled queue: %v", err) |
||||
} |
||||
} |
||||
return nil |
||||
}, levelCfg, exemplar) |
||||
if err == nil { |
||||
queue.delayedStarter = delayedStarter{ |
||||
internal: levelQueue.(*LevelUniqueQueue), |
||||
name: config.Name, |
||||
} |
||||
|
||||
_ = GetManager().Add(queue, PersistableChannelUniqueQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
if IsErrInvalidConfiguration(err) { |
||||
// Retrying ain't gonna make this any better...
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg} |
||||
} |
||||
|
||||
queue.delayedStarter = delayedStarter{ |
||||
cfg: levelCfg, |
||||
underlying: LevelUniqueQueueType, |
||||
timeout: config.Timeout, |
||||
maxAttempts: config.MaxAttempts, |
||||
name: config.Name, |
||||
} |
||||
_ = GetManager().Add(queue, PersistableChannelUniqueQueueType, config, exemplar) |
||||
return queue, nil |
||||
} |
||||
|
||||
// Name returns the name of this queue
|
||||
func (q *PersistableChannelUniqueQueue) Name() string { |
||||
return q.delayedStarter.name |
||||
} |
||||
|
||||
// Push will push the indexer data to queue
|
||||
func (q *PersistableChannelUniqueQueue) Push(data Data) error { |
||||
return q.PushFunc(data, nil) |
||||
} |
||||
|
||||
// PushFunc will push the indexer data to queue
|
||||
func (q *PersistableChannelUniqueQueue) PushFunc(data Data, fn func() error) error { |
||||
select { |
||||
case <-q.closed: |
||||
return q.internal.(UniqueQueue).PushFunc(data, fn) |
||||
default: |
||||
return q.channelQueue.PushFunc(data, fn) |
||||
} |
||||
} |
||||
|
||||
// PushBack will push the indexer data to queue
|
||||
func (q *PersistableChannelUniqueQueue) PushBack(data Data) error { |
||||
select { |
||||
case <-q.closed: |
||||
if pbr, ok := q.internal.(PushBackable); ok { |
||||
return pbr.PushBack(data) |
||||
} |
||||
return q.internal.Push(data) |
||||
default: |
||||
return q.channelQueue.Push(data) |
||||
} |
||||
} |
||||
|
||||
// Has will test if the queue has the data
|
||||
func (q *PersistableChannelUniqueQueue) Has(data Data) (bool, error) { |
||||
// This is more difficult...
|
||||
has, err := q.channelQueue.Has(data) |
||||
if err != nil || has { |
||||
return has, err |
||||
} |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal == nil { |
||||
return false, nil |
||||
} |
||||
return q.internal.(UniqueQueue).Has(data) |
||||
} |
||||
|
||||
// Run starts to run the queue
|
||||
func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { |
||||
pprof.SetGoroutineLabels(q.channelQueue.baseCtx) |
||||
log.Debug("PersistableChannelUniqueQueue: %s Starting", q.delayedStarter.name) |
||||
|
||||
q.lock.Lock() |
||||
if q.internal == nil { |
||||
err := q.setInternal(atShutdown, func(data ...Data) []Data { |
||||
for _, datum := range data { |
||||
err := q.Push(datum) |
||||
if err != nil && err != ErrAlreadyInQueue { |
||||
log.Error("Unable push to channelled queue: %v", err) |
||||
} |
||||
} |
||||
return nil |
||||
}, q.channelQueue.exemplar) |
||||
q.lock.Unlock() |
||||
if err != nil { |
||||
log.Fatal("Unable to create internal queue for %s Error: %v", q.Name(), err) |
||||
return |
||||
} |
||||
} else { |
||||
q.lock.Unlock() |
||||
} |
||||
atShutdown(q.Shutdown) |
||||
atTerminate(q.Terminate) |
||||
_ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) |
||||
|
||||
if luq, ok := q.internal.(*LevelUniqueQueue); ok && !luq.IsEmpty() { |
||||
// Just run the level queue - we shut it down once it's flushed
|
||||
go luq.Run(func(_ func()) {}, func(_ func()) {}) |
||||
go func() { |
||||
_ = luq.Flush(0) |
||||
for !luq.IsEmpty() { |
||||
_ = luq.Flush(0) |
||||
select { |
||||
case <-time.After(100 * time.Millisecond): |
||||
case <-luq.shutdownCtx.Done(): |
||||
if luq.byteFIFO.Len(luq.terminateCtx) > 0 { |
||||
log.Warn("LevelUniqueQueue: %s shut down before completely flushed", luq.Name()) |
||||
} |
||||
return |
||||
} |
||||
} |
||||
log.Debug("LevelUniqueQueue: %s flushed so shutting down", luq.Name()) |
||||
luq.Shutdown() |
||||
GetManager().Remove(luq.qid) |
||||
}() |
||||
} else { |
||||
log.Debug("PersistableChannelUniqueQueue: %s Skipping running the empty level queue", q.delayedStarter.name) |
||||
_ = q.internal.Flush(0) |
||||
q.internal.(*LevelUniqueQueue).Shutdown() |
||||
GetManager().Remove(q.internal.(*LevelUniqueQueue).qid) |
||||
} |
||||
} |
||||
|
||||
// Flush flushes the queue
|
||||
func (q *PersistableChannelUniqueQueue) Flush(timeout time.Duration) error { |
||||
return q.channelQueue.Flush(timeout) |
||||
} |
||||
|
||||
// FlushWithContext flushes the queue
|
||||
func (q *PersistableChannelUniqueQueue) FlushWithContext(ctx context.Context) error { |
||||
return q.channelQueue.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// IsEmpty checks if a queue is empty
|
||||
func (q *PersistableChannelUniqueQueue) IsEmpty() bool { |
||||
return q.channelQueue.IsEmpty() |
||||
} |
||||
|
||||
// IsPaused will return if the pool or queue is paused
|
||||
func (q *PersistableChannelUniqueQueue) IsPaused() bool { |
||||
return q.channelQueue.IsPaused() |
||||
} |
||||
|
||||
// Pause will pause the pool or queue
|
||||
func (q *PersistableChannelUniqueQueue) Pause() { |
||||
q.channelQueue.Pause() |
||||
} |
||||
|
||||
// Resume will resume the pool or queue
|
||||
func (q *PersistableChannelUniqueQueue) Resume() { |
||||
q.channelQueue.Resume() |
||||
} |
||||
|
||||
// IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed
|
||||
func (q *PersistableChannelUniqueQueue) IsPausedIsResumed() (paused, resumed <-chan struct{}) { |
||||
return q.channelQueue.IsPausedIsResumed() |
||||
} |
||||
|
||||
// Shutdown processing this queue
|
||||
func (q *PersistableChannelUniqueQueue) Shutdown() { |
||||
log.Trace("PersistableChannelUniqueQueue: %s Shutting down", q.delayedStarter.name) |
||||
q.lock.Lock() |
||||
select { |
||||
case <-q.closed: |
||||
q.lock.Unlock() |
||||
return |
||||
default: |
||||
if q.internal != nil { |
||||
q.internal.(*LevelUniqueQueue).Shutdown() |
||||
} |
||||
close(q.closed) |
||||
q.lock.Unlock() |
||||
} |
||||
|
||||
log.Trace("PersistableChannelUniqueQueue: %s Cancelling pools", q.delayedStarter.name) |
||||
q.internal.(*LevelUniqueQueue).baseCtxCancel() |
||||
q.channelQueue.baseCtxCancel() |
||||
log.Trace("PersistableChannelUniqueQueue: %s Waiting til done", q.delayedStarter.name) |
||||
q.channelQueue.Wait() |
||||
q.internal.(*LevelUniqueQueue).Wait() |
||||
// Redirect all remaining data in the chan to the internal channel
|
||||
close(q.channelQueue.dataChan) |
||||
log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name) |
||||
countOK, countLost := 0, 0 |
||||
for data := range q.channelQueue.dataChan { |
||||
err := q.internal.(*LevelUniqueQueue).Push(data) |
||||
if err != nil { |
||||
log.Error("PersistableChannelUniqueQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) |
||||
countLost++ |
||||
} else { |
||||
countOK++ |
||||
} |
||||
} |
||||
if countLost > 0 { |
||||
log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) |
||||
} else if countOK > 0 { |
||||
log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) |
||||
} |
||||
log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name) |
||||
|
||||
log.Debug("PersistableChannelUniqueQueue: %s Shutdown", q.delayedStarter.name) |
||||
} |
||||
|
||||
// Terminate this queue and close the queue
|
||||
func (q *PersistableChannelUniqueQueue) Terminate() { |
||||
log.Trace("PersistableChannelUniqueQueue: %s Terminating", q.delayedStarter.name) |
||||
q.Shutdown() |
||||
q.lock.Lock() |
||||
defer q.lock.Unlock() |
||||
if q.internal != nil { |
||||
q.internal.(*LevelUniqueQueue).Terminate() |
||||
} |
||||
q.channelQueue.baseCtxFinished() |
||||
log.Debug("PersistableChannelUniqueQueue: %s Terminated", q.delayedStarter.name) |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[PersistableChannelUniqueQueueType] = NewPersistableChannelUniqueQueue |
||||
} |
||||
@ -1,265 +0,0 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"strconv" |
||||
"sync" |
||||
"sync/atomic" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestPersistableChannelUniqueQueue(t *testing.T) { |
||||
// Create a temporary directory for the queue
|
||||
tmpDir := t.TempDir() |
||||
_ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) |
||||
|
||||
// Common function to create the Queue
|
||||
newQueue := func(name string, handle func(data ...Data) []Data) Queue { |
||||
q, err := NewPersistableChannelUniqueQueue(handle, |
||||
PersistableChannelUniqueQueueConfiguration{ |
||||
Name: name, |
||||
DataDir: tmpDir, |
||||
QueueLength: 200, |
||||
MaxWorkers: 1, |
||||
BlockTimeout: 1 * time.Second, |
||||
BoostTimeout: 5 * time.Minute, |
||||
BoostWorkers: 1, |
||||
Workers: 0, |
||||
}, "task-0") |
||||
assert.NoError(t, err) |
||||
return q |
||||
} |
||||
|
||||
// runs the provided queue and provides some timer function
|
||||
type channels struct { |
||||
readyForShutdown chan struct{} // closed when shutdown functions have been assigned
|
||||
readyForTerminate chan struct{} // closed when terminate functions have been assigned
|
||||
signalShutdown chan struct{} // Should close to signal shutdown
|
||||
doneShutdown chan struct{} // closed when shutdown function is done
|
||||
queueTerminate []func() // list of atTerminate functions to call atTerminate - need to be accessed with lock
|
||||
} |
||||
runQueue := func(q Queue, lock *sync.Mutex) *channels { |
||||
chans := &channels{ |
||||
readyForShutdown: make(chan struct{}), |
||||
readyForTerminate: make(chan struct{}), |
||||
signalShutdown: make(chan struct{}), |
||||
doneShutdown: make(chan struct{}), |
||||
} |
||||
go q.Run(func(atShutdown func()) { |
||||
go func() { |
||||
lock.Lock() |
||||
select { |
||||
case <-chans.readyForShutdown: |
||||
default: |
||||
close(chans.readyForShutdown) |
||||
} |
||||
lock.Unlock() |
||||
<-chans.signalShutdown |
||||
atShutdown() |
||||
close(chans.doneShutdown) |
||||
}() |
||||
}, func(atTerminate func()) { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
select { |
||||
case <-chans.readyForTerminate: |
||||
default: |
||||
close(chans.readyForTerminate) |
||||
} |
||||
chans.queueTerminate = append(chans.queueTerminate, atTerminate) |
||||
}) |
||||
|
||||
return chans |
||||
} |
||||
|
||||
// call to shutdown and terminate the queue associated with the channels
|
||||
doTerminate := func(chans *channels, lock *sync.Mutex) { |
||||
<-chans.readyForTerminate |
||||
|
||||
lock.Lock() |
||||
callbacks := []func(){} |
||||
callbacks = append(callbacks, chans.queueTerminate...) |
||||
lock.Unlock() |
||||
|
||||
for _, callback := range callbacks { |
||||
callback() |
||||
} |
||||
} |
||||
|
||||
mapLock := sync.Mutex{} |
||||
executedInitial := map[string][]string{} |
||||
hasInitial := map[string][]string{} |
||||
|
||||
fillQueue := func(name string, done chan int64) { |
||||
t.Run("Initial Filling: "+name, func(t *testing.T) { |
||||
lock := sync.Mutex{} |
||||
|
||||
startAt100Queued := make(chan struct{}) |
||||
stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item
|
||||
|
||||
handle := func(data ...Data) []Data { |
||||
<-startAt100Queued |
||||
for _, datum := range data { |
||||
s := datum.(string) |
||||
mapLock.Lock() |
||||
executedInitial[name] = append(executedInitial[name], s) |
||||
mapLock.Unlock() |
||||
if s == "task-20" { |
||||
close(stopAt20Shutdown) |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
q := newQueue(name, handle) |
||||
|
||||
// add 100 tasks to the queue
|
||||
for i := 0; i < 100; i++ { |
||||
_ = q.Push("task-" + strconv.Itoa(i)) |
||||
} |
||||
close(startAt100Queued) |
||||
|
||||
chans := runQueue(q, &lock) |
||||
|
||||
<-chans.readyForShutdown |
||||
<-stopAt20Shutdown |
||||
close(chans.signalShutdown) |
||||
<-chans.doneShutdown |
||||
_ = q.Push("final") |
||||
|
||||
// check which tasks are still in the queue
|
||||
for i := 0; i < 100; i++ { |
||||
if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { |
||||
mapLock.Lock() |
||||
hasInitial[name] = append(hasInitial[name], "task-"+strconv.Itoa(i)) |
||||
mapLock.Unlock() |
||||
} |
||||
} |
||||
if has, _ := q.(UniqueQueue).Has("final"); has { |
||||
mapLock.Lock() |
||||
hasInitial[name] = append(hasInitial[name], "final") |
||||
mapLock.Unlock() |
||||
} else { |
||||
assert.Fail(t, "UnqueQueue %s should have \"final\"", name) |
||||
} |
||||
doTerminate(chans, &lock) |
||||
mapLock.Lock() |
||||
assert.Equal(t, 101, len(executedInitial[name])+len(hasInitial[name])) |
||||
mapLock.Unlock() |
||||
}) |
||||
mapLock.Lock() |
||||
count := int64(len(hasInitial[name])) |
||||
mapLock.Unlock() |
||||
done <- count |
||||
close(done) |
||||
} |
||||
|
||||
hasQueueAChan := make(chan int64) |
||||
hasQueueBChan := make(chan int64) |
||||
|
||||
go fillQueue("QueueA", hasQueueAChan) |
||||
go fillQueue("QueueB", hasQueueBChan) |
||||
|
||||
hasA := <-hasQueueAChan |
||||
hasB := <-hasQueueBChan |
||||
|
||||
executedEmpty := map[string][]string{} |
||||
hasEmpty := map[string][]string{} |
||||
emptyQueue := func(name string, numInQueue int64, done chan struct{}) { |
||||
t.Run("Empty Queue: "+name, func(t *testing.T) { |
||||
lock := sync.Mutex{} |
||||
stop := make(chan struct{}) |
||||
|
||||
// collect the tasks that have been executed
|
||||
atomicCount := int64(0) |
||||
handle := func(data ...Data) []Data { |
||||
lock.Lock() |
||||
for _, datum := range data { |
||||
mapLock.Lock() |
||||
executedEmpty[name] = append(executedEmpty[name], datum.(string)) |
||||
mapLock.Unlock() |
||||
count := atomic.AddInt64(&atomicCount, 1) |
||||
if count >= numInQueue { |
||||
close(stop) |
||||
} |
||||
} |
||||
lock.Unlock() |
||||
return nil |
||||
} |
||||
|
||||
q := newQueue(name, handle) |
||||
chans := runQueue(q, &lock) |
||||
|
||||
<-chans.readyForShutdown |
||||
<-stop |
||||
close(chans.signalShutdown) |
||||
<-chans.doneShutdown |
||||
|
||||
// check which tasks are still in the queue
|
||||
for i := 0; i < 100; i++ { |
||||
if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { |
||||
mapLock.Lock() |
||||
hasEmpty[name] = append(hasEmpty[name], "task-"+strconv.Itoa(i)) |
||||
mapLock.Unlock() |
||||
} |
||||
} |
||||
doTerminate(chans, &lock) |
||||
|
||||
mapLock.Lock() |
||||
assert.Equal(t, 101, len(executedInitial[name])+len(executedEmpty[name])) |
||||
assert.Empty(t, hasEmpty[name]) |
||||
mapLock.Unlock() |
||||
}) |
||||
close(done) |
||||
} |
||||
|
||||
doneA := make(chan struct{}) |
||||
doneB := make(chan struct{}) |
||||
|
||||
go emptyQueue("QueueA", hasA, doneA) |
||||
go emptyQueue("QueueB", hasB, doneB) |
||||
|
||||
<-doneA |
||||
<-doneB |
||||
|
||||
mapLock.Lock() |
||||
t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", |
||||
len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) |
||||
|
||||
// reset and rerun
|
||||
executedInitial = map[string][]string{} |
||||
hasInitial = map[string][]string{} |
||||
executedEmpty = map[string][]string{} |
||||
hasEmpty = map[string][]string{} |
||||
mapLock.Unlock() |
||||
|
||||
hasQueueAChan = make(chan int64) |
||||
hasQueueBChan = make(chan int64) |
||||
|
||||
go fillQueue("QueueA", hasQueueAChan) |
||||
go fillQueue("QueueB", hasQueueBChan) |
||||
|
||||
hasA = <-hasQueueAChan |
||||
hasB = <-hasQueueBChan |
||||
|
||||
doneA = make(chan struct{}) |
||||
doneB = make(chan struct{}) |
||||
|
||||
go emptyQueue("QueueA", hasA, doneA) |
||||
go emptyQueue("QueueB", hasB, doneB) |
||||
|
||||
<-doneA |
||||
<-doneB |
||||
|
||||
mapLock.Lock() |
||||
t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", |
||||
len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) |
||||
mapLock.Unlock() |
||||
} |
||||
@ -1,141 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"github.com/redis/go-redis/v9" |
||||
) |
||||
|
||||
// RedisUniqueQueueType is the type for redis queue
|
||||
const RedisUniqueQueueType Type = "unique-redis" |
||||
|
||||
// RedisUniqueQueue redis queue
|
||||
type RedisUniqueQueue struct { |
||||
*ByteFIFOUniqueQueue |
||||
} |
||||
|
||||
// RedisUniqueQueueConfiguration is the configuration for the redis queue
|
||||
type RedisUniqueQueueConfiguration struct { |
||||
ByteFIFOQueueConfiguration |
||||
RedisUniqueByteFIFOConfiguration |
||||
} |
||||
|
||||
// NewRedisUniqueQueue creates single redis or cluster redis queue.
|
||||
//
|
||||
// Please note that this Queue does not guarantee that a particular
|
||||
// task cannot be processed twice or more at the same time. Uniqueness is
|
||||
// only guaranteed whilst the task is waiting in the queue.
|
||||
func NewRedisUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(RedisUniqueQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(RedisUniqueQueueConfiguration) |
||||
|
||||
byteFIFO, err := NewRedisUniqueByteFIFO(config.RedisUniqueByteFIFOConfiguration) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if len(byteFIFO.setName) == 0 { |
||||
byteFIFO.setName = byteFIFO.queueName + "_unique" |
||||
} |
||||
|
||||
byteFIFOQueue, err := NewByteFIFOUniqueQueue(RedisUniqueQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
queue := &RedisUniqueQueue{ |
||||
ByteFIFOUniqueQueue: byteFIFOQueue, |
||||
} |
||||
|
||||
queue.qid = GetManager().Add(queue, RedisUniqueQueueType, config, exemplar) |
||||
|
||||
return queue, nil |
||||
} |
||||
|
||||
var _ UniqueByteFIFO = &RedisUniqueByteFIFO{} |
||||
|
||||
// RedisUniqueByteFIFO represents a UniqueByteFIFO formed from a redisClient
|
||||
type RedisUniqueByteFIFO struct { |
||||
RedisByteFIFO |
||||
setName string |
||||
} |
||||
|
||||
// RedisUniqueByteFIFOConfiguration is the configuration for the RedisUniqueByteFIFO
|
||||
type RedisUniqueByteFIFOConfiguration struct { |
||||
RedisByteFIFOConfiguration |
||||
SetName string |
||||
} |
||||
|
||||
// NewRedisUniqueByteFIFO creates a UniqueByteFIFO formed from a redisClient
|
||||
func NewRedisUniqueByteFIFO(config RedisUniqueByteFIFOConfiguration) (*RedisUniqueByteFIFO, error) { |
||||
internal, err := NewRedisByteFIFO(config.RedisByteFIFOConfiguration) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
fifo := &RedisUniqueByteFIFO{ |
||||
RedisByteFIFO: *internal, |
||||
setName: config.SetName, |
||||
} |
||||
|
||||
return fifo, nil |
||||
} |
||||
|
||||
// PushFunc pushes data to the end of the fifo and calls the callback if it is added
|
||||
func (fifo *RedisUniqueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { |
||||
added, err := fifo.client.SAdd(ctx, fifo.setName, data).Result() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if added == 0 { |
||||
return ErrAlreadyInQueue |
||||
} |
||||
if fn != nil { |
||||
if err := fn(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return fifo.client.RPush(ctx, fifo.queueName, data).Err() |
||||
} |
||||
|
||||
// PushBack pushes data to the top of the fifo
|
||||
func (fifo *RedisUniqueByteFIFO) PushBack(ctx context.Context, data []byte) error { |
||||
added, err := fifo.client.SAdd(ctx, fifo.setName, data).Result() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if added == 0 { |
||||
return ErrAlreadyInQueue |
||||
} |
||||
return fifo.client.LPush(ctx, fifo.queueName, data).Err() |
||||
} |
||||
|
||||
// Pop pops data from the start of the fifo
|
||||
func (fifo *RedisUniqueByteFIFO) Pop(ctx context.Context) ([]byte, error) { |
||||
data, err := fifo.client.LPop(ctx, fifo.queueName).Bytes() |
||||
if err != nil && err != redis.Nil { |
||||
return data, err |
||||
} |
||||
|
||||
if len(data) == 0 { |
||||
return data, nil |
||||
} |
||||
|
||||
err = fifo.client.SRem(ctx, fifo.setName, data).Err() |
||||
return data, err |
||||
} |
||||
|
||||
// Has returns whether the fifo contains this data
|
||||
func (fifo *RedisUniqueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { |
||||
return fifo.client.SIsMember(ctx, fifo.setName, data).Result() |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[RedisUniqueQueueType] = NewRedisUniqueQueue |
||||
} |
||||
@ -1,174 +0,0 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
"time" |
||||
) |
||||
|
||||
// WrappedUniqueQueueType is the type for a wrapped delayed starting queue
|
||||
const WrappedUniqueQueueType Type = "unique-wrapped" |
||||
|
||||
// WrappedUniqueQueueConfiguration is the configuration for a WrappedUniqueQueue
|
||||
type WrappedUniqueQueueConfiguration struct { |
||||
Underlying Type |
||||
Timeout time.Duration |
||||
MaxAttempts int |
||||
Config interface{} |
||||
QueueLength int |
||||
Name string |
||||
} |
||||
|
||||
// WrappedUniqueQueue wraps a delayed starting unique queue
|
||||
type WrappedUniqueQueue struct { |
||||
*WrappedQueue |
||||
table map[Data]bool |
||||
tlock sync.Mutex |
||||
ready bool |
||||
} |
||||
|
||||
// NewWrappedUniqueQueue will attempt to create a unique queue of the provided type,
|
||||
// but if there is a problem creating this queue it will instead create
|
||||
// a WrappedUniqueQueue with delayed startup of the queue instead and a
|
||||
// channel which will be redirected to the queue
|
||||
//
|
||||
// Please note that this Queue does not guarantee that a particular
|
||||
// task cannot be processed twice or more at the same time. Uniqueness is
|
||||
// only guaranteed whilst the task is waiting in the queue.
|
||||
func NewWrappedUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { |
||||
configInterface, err := toConfig(WrappedUniqueQueueConfiguration{}, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
config := configInterface.(WrappedUniqueQueueConfiguration) |
||||
|
||||
queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar) |
||||
if err == nil { |
||||
// Just return the queue there is no need to wrap
|
||||
return queue, nil |
||||
} |
||||
if IsErrInvalidConfiguration(err) { |
||||
// Retrying ain't gonna make this any better...
|
||||
return nil, ErrInvalidConfiguration{cfg: cfg} |
||||
} |
||||
|
||||
wrapped := &WrappedUniqueQueue{ |
||||
WrappedQueue: &WrappedQueue{ |
||||
channel: make(chan Data, config.QueueLength), |
||||
exemplar: exemplar, |
||||
delayedStarter: delayedStarter{ |
||||
cfg: config.Config, |
||||
underlying: config.Underlying, |
||||
timeout: config.Timeout, |
||||
maxAttempts: config.MaxAttempts, |
||||
name: config.Name, |
||||
}, |
||||
}, |
||||
table: map[Data]bool{}, |
||||
} |
||||
|
||||
// wrapped.handle is passed to the delayedStarting internal queue and is run to handle
|
||||
// data passed to
|
||||
wrapped.handle = func(data ...Data) (unhandled []Data) { |
||||
for _, datum := range data { |
||||
wrapped.tlock.Lock() |
||||
if !wrapped.ready { |
||||
delete(wrapped.table, data) |
||||
// If our table is empty all of the requests we have buffered between the
|
||||
// wrapper queue starting and the internal queue starting have been handled.
|
||||
// We can stop buffering requests in our local table and just pass Push
|
||||
// direct to the internal queue
|
||||
if len(wrapped.table) == 0 { |
||||
wrapped.ready = true |
||||
} |
||||
} |
||||
wrapped.tlock.Unlock() |
||||
if u := handle(datum); u != nil { |
||||
unhandled = append(unhandled, u...) |
||||
} |
||||
} |
||||
return unhandled |
||||
} |
||||
_ = GetManager().Add(queue, WrappedUniqueQueueType, config, exemplar) |
||||
return wrapped, nil |
||||
} |
||||
|
||||
// Push will push the data to the internal channel checking it against the exemplar
|
||||
func (q *WrappedUniqueQueue) Push(data Data) error { |
||||
return q.PushFunc(data, nil) |
||||
} |
||||
|
||||
// PushFunc will push the data to the internal channel checking it against the exemplar
|
||||
func (q *WrappedUniqueQueue) PushFunc(data Data, fn func() error) error { |
||||
if !assignableTo(data, q.exemplar) { |
||||
return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) |
||||
} |
||||
|
||||
q.tlock.Lock() |
||||
if q.ready { |
||||
// ready means our table is empty and all of the requests we have buffered between the
|
||||
// wrapper queue starting and the internal queue starting have been handled.
|
||||
// We can stop buffering requests in our local table and just pass Push
|
||||
// direct to the internal queue
|
||||
q.tlock.Unlock() |
||||
return q.internal.(UniqueQueue).PushFunc(data, fn) |
||||
} |
||||
|
||||
locked := true |
||||
defer func() { |
||||
if locked { |
||||
q.tlock.Unlock() |
||||
} |
||||
}() |
||||
if _, ok := q.table[data]; ok { |
||||
return ErrAlreadyInQueue |
||||
} |
||||
// FIXME: We probably need to implement some sort of limit here
|
||||
// If the downstream queue blocks this table will grow without limit
|
||||
q.table[data] = true |
||||
if fn != nil { |
||||
err := fn() |
||||
if err != nil { |
||||
delete(q.table, data) |
||||
return err |
||||
} |
||||
} |
||||
locked = false |
||||
q.tlock.Unlock() |
||||
|
||||
q.channel <- data |
||||
return nil |
||||
} |
||||
|
||||
// Has checks if the data is in the queue
|
||||
func (q *WrappedUniqueQueue) Has(data Data) (bool, error) { |
||||
q.tlock.Lock() |
||||
defer q.tlock.Unlock() |
||||
if q.ready { |
||||
return q.internal.(UniqueQueue).Has(data) |
||||
} |
||||
_, has := q.table[data] |
||||
return has, nil |
||||
} |
||||
|
||||
// IsEmpty checks whether the queue is empty
|
||||
func (q *WrappedUniqueQueue) IsEmpty() bool { |
||||
q.tlock.Lock() |
||||
if len(q.table) > 0 { |
||||
q.tlock.Unlock() |
||||
return false |
||||
} |
||||
if q.ready { |
||||
q.tlock.Unlock() |
||||
return q.internal.IsEmpty() |
||||
} |
||||
q.tlock.Unlock() |
||||
return false |
||||
} |
||||
|
||||
func init() { |
||||
queuesMap[WrappedUniqueQueueType] = NewWrappedUniqueQueue |
||||
} |
||||
@ -0,0 +1,331 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
var ( |
||||
infiniteTimerC = make(chan time.Time) |
||||
batchDebounceDuration = 100 * time.Millisecond |
||||
workerIdleDuration = 1 * time.Second |
||||
|
||||
unhandledItemRequeueDuration atomic.Int64 // to avoid data race during test
|
||||
) |
||||
|
||||
func init() { |
||||
unhandledItemRequeueDuration.Store(int64(5 * time.Second)) |
||||
} |
||||
|
||||
// workerGroup is a group of workers to work with a WorkerPoolQueue
|
||||
type workerGroup[T any] struct { |
||||
q *WorkerPoolQueue[T] |
||||
wg sync.WaitGroup |
||||
|
||||
ctxWorker context.Context |
||||
ctxWorkerCancel context.CancelFunc |
||||
|
||||
batchBuffer []T |
||||
popItemChan chan []byte |
||||
popItemErr chan error |
||||
} |
||||
|
||||
func (wg *workerGroup[T]) doPrepareWorkerContext() { |
||||
wg.ctxWorker, wg.ctxWorkerCancel = context.WithCancel(wg.q.ctxRun) |
||||
} |
||||
|
||||
// doDispatchBatchToWorker dispatches a batch of items to worker's channel.
|
||||
// If the channel is full, it tries to start a new worker if possible.
|
||||
func (q *WorkerPoolQueue[T]) doDispatchBatchToWorker(wg *workerGroup[T], flushChan chan flushType) { |
||||
batch := wg.batchBuffer |
||||
wg.batchBuffer = nil |
||||
|
||||
if len(batch) == 0 { |
||||
return |
||||
} |
||||
|
||||
full := false |
||||
select { |
||||
case q.batchChan <- batch: |
||||
default: |
||||
full = true |
||||
} |
||||
|
||||
q.workerNumMu.Lock() |
||||
noWorker := q.workerNum == 0 |
||||
if full || noWorker { |
||||
if q.workerNum < q.workerMaxNum || noWorker && q.workerMaxNum <= 0 { |
||||
q.workerNum++ |
||||
q.doStartNewWorker(wg) |
||||
} |
||||
} |
||||
q.workerNumMu.Unlock() |
||||
|
||||
if full { |
||||
select { |
||||
case q.batchChan <- batch: |
||||
case flush := <-flushChan: |
||||
q.doWorkerHandle(batch) |
||||
q.doFlush(wg, flush) |
||||
case <-q.ctxRun.Done(): |
||||
wg.batchBuffer = batch // return the batch to buffer, the "doRun" function will handle it
|
||||
} |
||||
} |
||||
} |
||||
|
||||
// doWorkerHandle calls the safeHandler to handle a batch of items, and it increases/decreases the active worker number.
|
||||
// If the context has been canceled, it should not be caller because the "Push" still needs the context, in such case, call q.safeHandler directly
|
||||
func (q *WorkerPoolQueue[T]) doWorkerHandle(batch []T) { |
||||
q.workerNumMu.Lock() |
||||
q.workerActiveNum++ |
||||
q.workerNumMu.Unlock() |
||||
|
||||
defer func() { |
||||
q.workerNumMu.Lock() |
||||
q.workerActiveNum-- |
||||
q.workerNumMu.Unlock() |
||||
}() |
||||
|
||||
unhandled := q.safeHandler(batch...) |
||||
// if none of the items were handled, it should back-off for a few seconds
|
||||
// in this case the handler (eg: document indexer) may have encountered some errors/failures
|
||||
if len(unhandled) == len(batch) && unhandledItemRequeueDuration.Load() != 0 { |
||||
log.Error("Queue %q failed to handle batch of %d items, backoff for a few seconds", q.GetName(), len(batch)) |
||||
select { |
||||
case <-q.ctxRun.Done(): |
||||
case <-time.After(time.Duration(unhandledItemRequeueDuration.Load())): |
||||
} |
||||
} |
||||
for _, item := range unhandled { |
||||
if err := q.Push(item); err != nil { |
||||
if !q.basePushForShutdown(item) { |
||||
log.Error("Failed to requeue item for queue %q when calling handler: %v", q.GetName(), err) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// basePushForShutdown tries to requeue items into the base queue when the WorkerPoolQueue is shutting down.
|
||||
// If the queue is shutting down, it returns true and try to push the items
|
||||
// Otherwise it does nothing and returns false
|
||||
func (q *WorkerPoolQueue[T]) basePushForShutdown(items ...T) bool { |
||||
ctxShutdown := q.ctxShutdown.Load() |
||||
if ctxShutdown == nil { |
||||
return false |
||||
} |
||||
for _, item := range items { |
||||
// if there is still any error, the queue can do nothing instead of losing the items
|
||||
if err := q.baseQueue.PushItem(*ctxShutdown, q.marshal(item)); err != nil { |
||||
log.Error("Failed to requeue item for queue %q when shutting down: %v", q.GetName(), err) |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
|
||||
// doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items.
|
||||
func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { |
||||
wp.wg.Add(1) |
||||
|
||||
go func() { |
||||
defer wp.wg.Done() |
||||
|
||||
log.Debug("Queue %q starts new worker", q.GetName()) |
||||
defer log.Debug("Queue %q stops idle worker", q.GetName()) |
||||
|
||||
t := time.NewTicker(workerIdleDuration) |
||||
keepWorking := true |
||||
stopWorking := func() { |
||||
q.workerNumMu.Lock() |
||||
keepWorking = false |
||||
q.workerNum-- |
||||
q.workerNumMu.Unlock() |
||||
} |
||||
for keepWorking { |
||||
select { |
||||
case <-wp.ctxWorker.Done(): |
||||
stopWorking() |
||||
case batch, ok := <-q.batchChan: |
||||
if !ok { |
||||
stopWorking() |
||||
} else { |
||||
q.doWorkerHandle(batch) |
||||
t.Reset(workerIdleDuration) |
||||
} |
||||
case <-t.C: |
||||
q.workerNumMu.Lock() |
||||
keepWorking = q.workerNum <= 1 |
||||
if !keepWorking { |
||||
q.workerNum-- |
||||
} |
||||
q.workerNumMu.Unlock() |
||||
} |
||||
} |
||||
}() |
||||
} |
||||
|
||||
// doFlush flushes the queue: it tries to read all items from the queue and handles them.
|
||||
// It is for testing purpose only. It's not designed to work for a cluster.
|
||||
func (q *WorkerPoolQueue[T]) doFlush(wg *workerGroup[T], flush flushType) { |
||||
log.Debug("Queue %q starts flushing", q.GetName()) |
||||
defer log.Debug("Queue %q finishes flushing", q.GetName()) |
||||
|
||||
// stop all workers, and prepare a new worker context to start new workers
|
||||
|
||||
wg.ctxWorkerCancel() |
||||
wg.wg.Wait() |
||||
|
||||
defer func() { |
||||
close(flush) |
||||
wg.doPrepareWorkerContext() |
||||
}() |
||||
|
||||
// drain the batch channel first
|
||||
loop: |
||||
for { |
||||
select { |
||||
case batch := <-q.batchChan: |
||||
q.doWorkerHandle(batch) |
||||
default: |
||||
break loop |
||||
} |
||||
} |
||||
|
||||
// drain the popItem channel
|
||||
emptyCounter := 0 |
||||
for { |
||||
select { |
||||
case data, dataOk := <-wg.popItemChan: |
||||
if !dataOk { |
||||
return |
||||
} |
||||
emptyCounter = 0 |
||||
if v, jsonOk := q.unmarshal(data); !jsonOk { |
||||
continue |
||||
} else { |
||||
q.doWorkerHandle([]T{v}) |
||||
} |
||||
case err := <-wg.popItemErr: |
||||
if !q.isCtxRunCanceled() { |
||||
log.Error("Failed to pop item from queue %q (doFlush): %v", q.GetName(), err) |
||||
} |
||||
return |
||||
case <-q.ctxRun.Done(): |
||||
log.Debug("Queue %q is shutting down", q.GetName()) |
||||
return |
||||
case <-time.After(20 * time.Millisecond): |
||||
// There is no reliable way to make sure all queue items are consumed by the Flush, there always might be some items stored in some buffers/temp variables.
|
||||
// If we run Gitea in a cluster, we can even not guarantee all items are consumed in a deterministic instance.
|
||||
// Luckily, the "Flush" trick is only used in tests, so far so good.
|
||||
if cnt, _ := q.baseQueue.Len(q.ctxRun); cnt == 0 && len(wg.popItemChan) == 0 { |
||||
emptyCounter++ |
||||
} |
||||
if emptyCounter >= 2 { |
||||
return |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) isCtxRunCanceled() bool { |
||||
select { |
||||
case <-q.ctxRun.Done(): |
||||
return true |
||||
default: |
||||
return false |
||||
} |
||||
} |
||||
|
||||
var skipFlushChan = make(chan flushType) // an empty flush chan, used to skip reading other flush requests
|
||||
|
||||
// doRun is the main loop of the queue. All related "doXxx" functions are executed in its context.
|
||||
func (q *WorkerPoolQueue[T]) doRun() { |
||||
log.Debug("Queue %q starts running", q.GetName()) |
||||
defer log.Debug("Queue %q stops running", q.GetName()) |
||||
|
||||
wg := &workerGroup[T]{q: q} |
||||
wg.doPrepareWorkerContext() |
||||
wg.popItemChan, wg.popItemErr = popItemByChan(q.ctxRun, q.baseQueue.PopItem) |
||||
|
||||
defer func() { |
||||
q.ctxRunCancel() |
||||
|
||||
// drain all data on the fly
|
||||
// since the queue is shutting down, the items can't be dispatched to workers because the context is canceled
|
||||
// it can't call doWorkerHandle either, because there is no chance to push unhandled items back to the queue
|
||||
var unhandled []T |
||||
close(q.batchChan) |
||||
for batch := range q.batchChan { |
||||
unhandled = append(unhandled, batch...) |
||||
} |
||||
unhandled = append(unhandled, wg.batchBuffer...) |
||||
for data := range wg.popItemChan { |
||||
if v, ok := q.unmarshal(data); ok { |
||||
unhandled = append(unhandled, v) |
||||
} |
||||
} |
||||
|
||||
ctxShutdownPtr := q.ctxShutdown.Load() |
||||
if ctxShutdownPtr != nil { |
||||
// if there is a shutdown context, try to push the items back to the base queue
|
||||
q.basePushForShutdown(unhandled...) |
||||
workerDone := make(chan struct{}) |
||||
// the only way to wait for the workers, because the handlers do not have context to wait for
|
||||
go func() { wg.wg.Wait(); close(workerDone) }() |
||||
select { |
||||
case <-workerDone: |
||||
case <-(*ctxShutdownPtr).Done(): |
||||
log.Error("Queue %q is shutting down, but workers are still running after timeout", q.GetName()) |
||||
} |
||||
} else { |
||||
// if there is no shutdown context, just call the handler to try to handle the items. if the handler fails again, the items are lost
|
||||
q.safeHandler(unhandled...) |
||||
} |
||||
|
||||
close(q.shutdownDone) |
||||
}() |
||||
|
||||
var batchDispatchC <-chan time.Time = infiniteTimerC |
||||
for { |
||||
select { |
||||
case data, dataOk := <-wg.popItemChan: |
||||
if !dataOk { |
||||
return |
||||
} |
||||
if v, jsonOk := q.unmarshal(data); !jsonOk { |
||||
testRecorder.Record("pop:corrupted:%s", data) // in rare cases the levelqueue(leveldb) might be corrupted
|
||||
continue |
||||
} else { |
||||
wg.batchBuffer = append(wg.batchBuffer, v) |
||||
} |
||||
if len(wg.batchBuffer) >= q.batchLength { |
||||
q.doDispatchBatchToWorker(wg, q.flushChan) |
||||
} else if batchDispatchC == infiniteTimerC { |
||||
batchDispatchC = time.After(batchDebounceDuration) |
||||
} // else: batchDispatchC is already a debounce timer, it will be triggered soon
|
||||
case <-batchDispatchC: |
||||
batchDispatchC = infiniteTimerC |
||||
q.doDispatchBatchToWorker(wg, q.flushChan) |
||||
case flush := <-q.flushChan: |
||||
// before flushing, it needs to try to dispatch the batch to worker first, in case there is no worker running
|
||||
// after the flushing, there is at least one worker running, so "doFlush" could wait for workers to finish
|
||||
// since we are already in a "flush" operation, so the dispatching function shouldn't read the flush chan.
|
||||
q.doDispatchBatchToWorker(wg, skipFlushChan) |
||||
q.doFlush(wg, flush) |
||||
case err := <-wg.popItemErr: |
||||
if !q.isCtxRunCanceled() { |
||||
log.Error("Failed to pop item from queue %q (doRun): %v", q.GetName(), err) |
||||
} |
||||
return |
||||
case <-q.ctxRun.Done(): |
||||
log.Debug("Queue %q is shutting down", q.GetName()) |
||||
return |
||||
} |
||||
} |
||||
} |
||||
@ -1,613 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"runtime/pprof" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/process" |
||||
"code.gitea.io/gitea/modules/util" |
||||
) |
||||
|
||||
// WorkerPool represent a dynamically growable worker pool for a
|
||||
// provided handler function. They have an internal channel which
|
||||
// they use to detect if there is a block and will grow and shrink in
|
||||
// response to demand as per configuration.
|
||||
type WorkerPool struct { |
||||
// This field requires to be the first one in the struct.
|
||||
// This is to allow 64 bit atomic operations on 32-bit machines.
|
||||
// See: https://pkg.go.dev/sync/atomic#pkg-note-BUG & Gitea issue 19518
|
||||
numInQueue int64 |
||||
lock sync.Mutex |
||||
baseCtx context.Context |
||||
baseCtxCancel context.CancelFunc |
||||
baseCtxFinished process.FinishedFunc |
||||
paused chan struct{} |
||||
resumed chan struct{} |
||||
cond *sync.Cond |
||||
qid int64 |
||||
maxNumberOfWorkers int |
||||
numberOfWorkers int |
||||
batchLength int |
||||
handle HandlerFunc |
||||
dataChan chan Data |
||||
blockTimeout time.Duration |
||||
boostTimeout time.Duration |
||||
boostWorkers int |
||||
} |
||||
|
||||
var ( |
||||
_ Flushable = &WorkerPool{} |
||||
_ ManagedPool = &WorkerPool{} |
||||
) |
||||
|
||||
// WorkerPoolConfiguration is the basic configuration for a WorkerPool
|
||||
type WorkerPoolConfiguration struct { |
||||
Name string |
||||
QueueLength int |
||||
BatchLength int |
||||
BlockTimeout time.Duration |
||||
BoostTimeout time.Duration |
||||
BoostWorkers int |
||||
MaxWorkers int |
||||
} |
||||
|
||||
// NewWorkerPool creates a new worker pool
|
||||
func NewWorkerPool(handle HandlerFunc, config WorkerPoolConfiguration) *WorkerPool { |
||||
ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Queue: %s", config.Name), process.SystemProcessType, false) |
||||
|
||||
dataChan := make(chan Data, config.QueueLength) |
||||
pool := &WorkerPool{ |
||||
baseCtx: ctx, |
||||
baseCtxCancel: cancel, |
||||
baseCtxFinished: finished, |
||||
batchLength: config.BatchLength, |
||||
dataChan: dataChan, |
||||
resumed: closedChan, |
||||
paused: make(chan struct{}), |
||||
handle: handle, |
||||
blockTimeout: config.BlockTimeout, |
||||
boostTimeout: config.BoostTimeout, |
||||
boostWorkers: config.BoostWorkers, |
||||
maxNumberOfWorkers: config.MaxWorkers, |
||||
} |
||||
|
||||
return pool |
||||
} |
||||
|
||||
// Done returns when this worker pool's base context has been cancelled
|
||||
func (p *WorkerPool) Done() <-chan struct{} { |
||||
return p.baseCtx.Done() |
||||
} |
||||
|
||||
// Push pushes the data to the internal channel
|
||||
func (p *WorkerPool) Push(data Data) { |
||||
atomic.AddInt64(&p.numInQueue, 1) |
||||
p.lock.Lock() |
||||
select { |
||||
case <-p.paused: |
||||
p.lock.Unlock() |
||||
p.dataChan <- data |
||||
return |
||||
default: |
||||
} |
||||
|
||||
if p.blockTimeout > 0 && p.boostTimeout > 0 && (p.numberOfWorkers <= p.maxNumberOfWorkers || p.maxNumberOfWorkers < 0) { |
||||
if p.numberOfWorkers == 0 { |
||||
p.zeroBoost() |
||||
} else { |
||||
p.lock.Unlock() |
||||
} |
||||
p.pushBoost(data) |
||||
} else { |
||||
p.lock.Unlock() |
||||
p.dataChan <- data |
||||
} |
||||
} |
||||
|
||||
// HasNoWorkerScaling will return true if the queue has no workers, and has no worker boosting
|
||||
func (p *WorkerPool) HasNoWorkerScaling() bool { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.hasNoWorkerScaling() |
||||
} |
||||
|
||||
func (p *WorkerPool) hasNoWorkerScaling() bool { |
||||
return p.numberOfWorkers == 0 && (p.boostTimeout == 0 || p.boostWorkers == 0 || p.maxNumberOfWorkers == 0) |
||||
} |
||||
|
||||
// zeroBoost will add a temporary boost worker for a no worker queue
|
||||
// p.lock must be locked at the start of this function BUT it will be unlocked by the end of this function
|
||||
// (This is because addWorkers has to be called whilst unlocked)
|
||||
func (p *WorkerPool) zeroBoost() { |
||||
ctx, cancel := context.WithTimeout(p.baseCtx, p.boostTimeout) |
||||
mq := GetManager().GetManagedQueue(p.qid) |
||||
boost := p.boostWorkers |
||||
if (boost+p.numberOfWorkers) > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0 { |
||||
boost = p.maxNumberOfWorkers - p.numberOfWorkers |
||||
} |
||||
if mq != nil { |
||||
log.Debug("WorkerPool: %d (for %s) has zero workers - adding %d temporary workers for %s", p.qid, mq.Name, boost, p.boostTimeout) |
||||
|
||||
start := time.Now() |
||||
pid := mq.RegisterWorkers(boost, start, true, start.Add(p.boostTimeout), cancel, false) |
||||
cancel = func() { |
||||
mq.RemoveWorkers(pid) |
||||
} |
||||
} else { |
||||
log.Debug("WorkerPool: %d has zero workers - adding %d temporary workers for %s", p.qid, p.boostWorkers, p.boostTimeout) |
||||
} |
||||
p.lock.Unlock() |
||||
p.addWorkers(ctx, cancel, boost) |
||||
} |
||||
|
||||
func (p *WorkerPool) pushBoost(data Data) { |
||||
select { |
||||
case p.dataChan <- data: |
||||
default: |
||||
p.lock.Lock() |
||||
if p.blockTimeout <= 0 { |
||||
p.lock.Unlock() |
||||
p.dataChan <- data |
||||
return |
||||
} |
||||
ourTimeout := p.blockTimeout |
||||
timer := time.NewTimer(p.blockTimeout) |
||||
p.lock.Unlock() |
||||
select { |
||||
case p.dataChan <- data: |
||||
util.StopTimer(timer) |
||||
case <-timer.C: |
||||
p.lock.Lock() |
||||
if p.blockTimeout > ourTimeout || (p.numberOfWorkers > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0) { |
||||
p.lock.Unlock() |
||||
p.dataChan <- data |
||||
return |
||||
} |
||||
p.blockTimeout *= 2 |
||||
boostCtx, boostCtxCancel := context.WithCancel(p.baseCtx) |
||||
mq := GetManager().GetManagedQueue(p.qid) |
||||
boost := p.boostWorkers |
||||
if (boost+p.numberOfWorkers) > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0 { |
||||
boost = p.maxNumberOfWorkers - p.numberOfWorkers |
||||
} |
||||
if mq != nil { |
||||
log.Debug("WorkerPool: %d (for %s) Channel blocked for %v - adding %d temporary workers for %s, block timeout now %v", p.qid, mq.Name, ourTimeout, boost, p.boostTimeout, p.blockTimeout) |
||||
|
||||
start := time.Now() |
||||
pid := mq.RegisterWorkers(boost, start, true, start.Add(p.boostTimeout), boostCtxCancel, false) |
||||
go func() { |
||||
<-boostCtx.Done() |
||||
mq.RemoveWorkers(pid) |
||||
boostCtxCancel() |
||||
}() |
||||
} else { |
||||
log.Debug("WorkerPool: %d Channel blocked for %v - adding %d temporary workers for %s, block timeout now %v", p.qid, ourTimeout, p.boostWorkers, p.boostTimeout, p.blockTimeout) |
||||
} |
||||
go func() { |
||||
<-time.After(p.boostTimeout) |
||||
boostCtxCancel() |
||||
p.lock.Lock() |
||||
p.blockTimeout /= 2 |
||||
p.lock.Unlock() |
||||
}() |
||||
p.lock.Unlock() |
||||
p.addWorkers(boostCtx, boostCtxCancel, boost) |
||||
p.dataChan <- data |
||||
} |
||||
} |
||||
} |
||||
|
||||
// NumberOfWorkers returns the number of current workers in the pool
|
||||
func (p *WorkerPool) NumberOfWorkers() int { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.numberOfWorkers |
||||
} |
||||
|
||||
// NumberInQueue returns the number of items in the queue
|
||||
func (p *WorkerPool) NumberInQueue() int64 { |
||||
return atomic.LoadInt64(&p.numInQueue) |
||||
} |
||||
|
||||
// MaxNumberOfWorkers returns the maximum number of workers automatically added to the pool
|
||||
func (p *WorkerPool) MaxNumberOfWorkers() int { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.maxNumberOfWorkers |
||||
} |
||||
|
||||
// BoostWorkers returns the number of workers for a boost
|
||||
func (p *WorkerPool) BoostWorkers() int { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.boostWorkers |
||||
} |
||||
|
||||
// BoostTimeout returns the timeout of the next boost
|
||||
func (p *WorkerPool) BoostTimeout() time.Duration { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.boostTimeout |
||||
} |
||||
|
||||
// BlockTimeout returns the timeout til the next boost
|
||||
func (p *WorkerPool) BlockTimeout() time.Duration { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.blockTimeout |
||||
} |
||||
|
||||
// SetPoolSettings sets the setable boost values
|
||||
func (p *WorkerPool) SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
p.maxNumberOfWorkers = maxNumberOfWorkers |
||||
p.boostWorkers = boostWorkers |
||||
p.boostTimeout = timeout |
||||
} |
||||
|
||||
// SetMaxNumberOfWorkers sets the maximum number of workers automatically added to the pool
|
||||
// Changing this number will not change the number of current workers but will change the limit
|
||||
// for future additions
|
||||
func (p *WorkerPool) SetMaxNumberOfWorkers(newMax int) { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
p.maxNumberOfWorkers = newMax |
||||
} |
||||
|
||||
func (p *WorkerPool) commonRegisterWorkers(number int, timeout time.Duration, isFlusher bool) (context.Context, context.CancelFunc) { |
||||
var ctx context.Context |
||||
var cancel context.CancelFunc |
||||
start := time.Now() |
||||
end := start |
||||
hasTimeout := false |
||||
if timeout > 0 { |
||||
ctx, cancel = context.WithTimeout(p.baseCtx, timeout) |
||||
end = start.Add(timeout) |
||||
hasTimeout = true |
||||
} else { |
||||
ctx, cancel = context.WithCancel(p.baseCtx) |
||||
} |
||||
|
||||
mq := GetManager().GetManagedQueue(p.qid) |
||||
if mq != nil { |
||||
pid := mq.RegisterWorkers(number, start, hasTimeout, end, cancel, isFlusher) |
||||
log.Trace("WorkerPool: %d (for %s) adding %d workers with group id: %d", p.qid, mq.Name, number, pid) |
||||
return ctx, func() { |
||||
mq.RemoveWorkers(pid) |
||||
} |
||||
} |
||||
log.Trace("WorkerPool: %d adding %d workers (no group id)", p.qid, number) |
||||
|
||||
return ctx, cancel |
||||
} |
||||
|
||||
// AddWorkers adds workers to the pool - this allows the number of workers to go above the limit
|
||||
func (p *WorkerPool) AddWorkers(number int, timeout time.Duration) context.CancelFunc { |
||||
ctx, cancel := p.commonRegisterWorkers(number, timeout, false) |
||||
p.addWorkers(ctx, cancel, number) |
||||
return cancel |
||||
} |
||||
|
||||
// addWorkers adds workers to the pool
|
||||
func (p *WorkerPool) addWorkers(ctx context.Context, cancel context.CancelFunc, number int) { |
||||
for i := 0; i < number; i++ { |
||||
p.lock.Lock() |
||||
if p.cond == nil { |
||||
p.cond = sync.NewCond(&p.lock) |
||||
} |
||||
p.numberOfWorkers++ |
||||
p.lock.Unlock() |
||||
go func() { |
||||
pprof.SetGoroutineLabels(ctx) |
||||
p.doWork(ctx) |
||||
|
||||
p.lock.Lock() |
||||
p.numberOfWorkers-- |
||||
if p.numberOfWorkers == 0 { |
||||
p.cond.Broadcast() |
||||
cancel() |
||||
} else if p.numberOfWorkers < 0 { |
||||
// numberOfWorkers can't go negative but...
|
||||
log.Warn("Number of Workers < 0 for QID %d - this shouldn't happen", p.qid) |
||||
p.numberOfWorkers = 0 |
||||
p.cond.Broadcast() |
||||
cancel() |
||||
} |
||||
select { |
||||
case <-p.baseCtx.Done(): |
||||
// Don't warn or check for ongoing work if the baseCtx is shutdown
|
||||
case <-p.paused: |
||||
// Don't warn or check for ongoing work if the pool is paused
|
||||
default: |
||||
if p.hasNoWorkerScaling() { |
||||
log.Warn( |
||||
"Queue: %d is configured to be non-scaling and has no workers - this configuration is likely incorrect.\n"+ |
||||
"The queue will be paused to prevent data-loss with the assumption that you will add workers and unpause as required.", p.qid) |
||||
p.pause() |
||||
} else if p.numberOfWorkers == 0 && atomic.LoadInt64(&p.numInQueue) > 0 { |
||||
// OK there are no workers but... there's still work to be done -> Reboost
|
||||
p.zeroBoost() |
||||
// p.lock will be unlocked by zeroBoost
|
||||
return |
||||
} |
||||
} |
||||
p.lock.Unlock() |
||||
}() |
||||
} |
||||
} |
||||
|
||||
// Wait for WorkerPool to finish
|
||||
func (p *WorkerPool) Wait() { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
if p.cond == nil { |
||||
p.cond = sync.NewCond(&p.lock) |
||||
} |
||||
if p.numberOfWorkers <= 0 { |
||||
return |
||||
} |
||||
p.cond.Wait() |
||||
} |
||||
|
||||
// IsPaused returns if the pool is paused
|
||||
func (p *WorkerPool) IsPaused() bool { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
select { |
||||
case <-p.paused: |
||||
return true |
||||
default: |
||||
return false |
||||
} |
||||
} |
||||
|
||||
// IsPausedIsResumed returns if the pool is paused and a channel that is closed when it is resumed
|
||||
func (p *WorkerPool) IsPausedIsResumed() (<-chan struct{}, <-chan struct{}) { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
return p.paused, p.resumed |
||||
} |
||||
|
||||
// Pause pauses the WorkerPool
|
||||
func (p *WorkerPool) Pause() { |
||||
p.lock.Lock() |
||||
defer p.lock.Unlock() |
||||
p.pause() |
||||
} |
||||
|
||||
func (p *WorkerPool) pause() { |
||||
select { |
||||
case <-p.paused: |
||||
default: |
||||
p.resumed = make(chan struct{}) |
||||
close(p.paused) |
||||
} |
||||
} |
||||
|
||||
// Resume resumes the WorkerPool
|
||||
func (p *WorkerPool) Resume() { |
||||
p.lock.Lock() // can't defer unlock because of the zeroBoost at the end
|
||||
select { |
||||
case <-p.resumed: |
||||
// already resumed - there's nothing to do
|
||||
p.lock.Unlock() |
||||
return |
||||
default: |
||||
} |
||||
|
||||
p.paused = make(chan struct{}) |
||||
close(p.resumed) |
||||
|
||||
// OK now we need to check if we need to add some workers...
|
||||
if p.numberOfWorkers > 0 || p.hasNoWorkerScaling() || atomic.LoadInt64(&p.numInQueue) == 0 { |
||||
// We either have workers, can't scale or there's no work to be done -> so just resume
|
||||
p.lock.Unlock() |
||||
return |
||||
} |
||||
|
||||
// OK we got some work but no workers we need to think about boosting
|
||||
select { |
||||
case <-p.baseCtx.Done(): |
||||
// don't bother boosting if the baseCtx is done
|
||||
p.lock.Unlock() |
||||
return |
||||
default: |
||||
} |
||||
|
||||
// OK we'd better add some boost workers!
|
||||
p.zeroBoost() |
||||
// p.zeroBoost will unlock the lock
|
||||
} |
||||
|
||||
// CleanUp will drain the remaining contents of the channel
|
||||
// This should be called after AddWorkers context is closed
|
||||
func (p *WorkerPool) CleanUp(ctx context.Context) { |
||||
log.Trace("WorkerPool: %d CleanUp", p.qid) |
||||
close(p.dataChan) |
||||
for data := range p.dataChan { |
||||
if unhandled := p.handle(data); unhandled != nil { |
||||
if unhandled != nil { |
||||
log.Error("Unhandled Data in clean-up of queue %d", p.qid) |
||||
} |
||||
} |
||||
|
||||
atomic.AddInt64(&p.numInQueue, -1) |
||||
select { |
||||
case <-ctx.Done(): |
||||
log.Warn("WorkerPool: %d Cleanup context closed before finishing clean-up", p.qid) |
||||
return |
||||
default: |
||||
} |
||||
} |
||||
log.Trace("WorkerPool: %d CleanUp Done", p.qid) |
||||
} |
||||
|
||||
// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager
|
||||
func (p *WorkerPool) Flush(timeout time.Duration) error { |
||||
ctx, cancel := p.commonRegisterWorkers(1, timeout, true) |
||||
defer cancel() |
||||
return p.FlushWithContext(ctx) |
||||
} |
||||
|
||||
// IsEmpty returns if true if the worker queue is empty
|
||||
func (p *WorkerPool) IsEmpty() bool { |
||||
return atomic.LoadInt64(&p.numInQueue) == 0 |
||||
} |
||||
|
||||
// contextError returns either ctx.Done(), the base context's error or nil
|
||||
func (p *WorkerPool) contextError(ctx context.Context) error { |
||||
select { |
||||
case <-p.baseCtx.Done(): |
||||
return p.baseCtx.Err() |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
|
||||
// FlushWithContext is very similar to CleanUp but it will return as soon as the dataChan is empty
|
||||
// NB: The worker will not be registered with the manager.
|
||||
func (p *WorkerPool) FlushWithContext(ctx context.Context) error { |
||||
log.Trace("WorkerPool: %d Flush", p.qid) |
||||
paused, _ := p.IsPausedIsResumed() |
||||
for { |
||||
// Because select will return any case that is satisified at random we precheck here before looking at dataChan.
|
||||
select { |
||||
case <-paused: |
||||
// Ensure that even if paused that the cancelled error is still sent
|
||||
return p.contextError(ctx) |
||||
case <-p.baseCtx.Done(): |
||||
return p.baseCtx.Err() |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
default: |
||||
} |
||||
|
||||
select { |
||||
case <-paused: |
||||
return p.contextError(ctx) |
||||
case data, ok := <-p.dataChan: |
||||
if !ok { |
||||
return nil |
||||
} |
||||
if unhandled := p.handle(data); unhandled != nil { |
||||
log.Error("Unhandled Data whilst flushing queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1) |
||||
case <-p.baseCtx.Done(): |
||||
return p.baseCtx.Err() |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
default: |
||||
return nil |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (p *WorkerPool) doWork(ctx context.Context) { |
||||
pprof.SetGoroutineLabels(ctx) |
||||
delay := time.Millisecond * 300 |
||||
|
||||
// Create a common timer - we will use this elsewhere
|
||||
timer := time.NewTimer(0) |
||||
util.StopTimer(timer) |
||||
|
||||
paused, _ := p.IsPausedIsResumed() |
||||
data := make([]Data, 0, p.batchLength) |
||||
for { |
||||
// Because select will return any case that is satisified at random we precheck here before looking at dataChan.
|
||||
select { |
||||
case <-paused: |
||||
log.Trace("Worker for Queue %d Pausing", p.qid) |
||||
if len(data) > 0 { |
||||
log.Trace("Handling: %d data, %v", len(data), data) |
||||
if unhandled := p.handle(data...); unhandled != nil { |
||||
log.Error("Unhandled Data in queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) |
||||
} |
||||
_, resumed := p.IsPausedIsResumed() |
||||
select { |
||||
case <-resumed: |
||||
paused, _ = p.IsPausedIsResumed() |
||||
log.Trace("Worker for Queue %d Resuming", p.qid) |
||||
util.StopTimer(timer) |
||||
case <-ctx.Done(): |
||||
log.Trace("Worker shutting down") |
||||
return |
||||
} |
||||
case <-ctx.Done(): |
||||
if len(data) > 0 { |
||||
log.Trace("Handling: %d data, %v", len(data), data) |
||||
if unhandled := p.handle(data...); unhandled != nil { |
||||
log.Error("Unhandled Data in queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) |
||||
} |
||||
log.Trace("Worker shutting down") |
||||
return |
||||
default: |
||||
} |
||||
|
||||
select { |
||||
case <-paused: |
||||
// go back around
|
||||
case <-ctx.Done(): |
||||
if len(data) > 0 { |
||||
log.Trace("Handling: %d data, %v", len(data), data) |
||||
if unhandled := p.handle(data...); unhandled != nil { |
||||
log.Error("Unhandled Data in queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) |
||||
} |
||||
log.Trace("Worker shutting down") |
||||
return |
||||
case datum, ok := <-p.dataChan: |
||||
if !ok { |
||||
// the dataChan has been closed - we should finish up:
|
||||
if len(data) > 0 { |
||||
log.Trace("Handling: %d data, %v", len(data), data) |
||||
if unhandled := p.handle(data...); unhandled != nil { |
||||
log.Error("Unhandled Data in queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) |
||||
} |
||||
log.Trace("Worker shutting down") |
||||
return |
||||
} |
||||
data = append(data, datum) |
||||
util.StopTimer(timer) |
||||
|
||||
if len(data) >= p.batchLength { |
||||
log.Trace("Handling: %d data, %v", len(data), data) |
||||
if unhandled := p.handle(data...); unhandled != nil { |
||||
log.Error("Unhandled Data in queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) |
||||
data = make([]Data, 0, p.batchLength) |
||||
} else { |
||||
timer.Reset(delay) |
||||
} |
||||
case <-timer.C: |
||||
delay = time.Millisecond * 100 |
||||
if len(data) > 0 { |
||||
log.Trace("Handling: %d data, %v", len(data), data) |
||||
if unhandled := p.handle(data...); unhandled != nil { |
||||
log.Error("Unhandled Data in queue %d", p.qid) |
||||
} |
||||
atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) |
||||
data = make([]Data, 0, p.batchLength) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,241 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"sync" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/graceful" |
||||
"code.gitea.io/gitea/modules/json" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
) |
||||
|
||||
// WorkerPoolQueue is a queue that uses a pool of workers to process items
|
||||
// It can use different underlying (base) queue types
|
||||
type WorkerPoolQueue[T any] struct { |
||||
ctxRun context.Context |
||||
ctxRunCancel context.CancelFunc |
||||
ctxShutdown atomic.Pointer[context.Context] |
||||
shutdownDone chan struct{} |
||||
|
||||
origHandler HandlerFuncT[T] |
||||
safeHandler HandlerFuncT[T] |
||||
|
||||
baseQueueType string |
||||
baseConfig *BaseConfig |
||||
baseQueue baseQueue |
||||
|
||||
batchChan chan []T |
||||
flushChan chan flushType |
||||
|
||||
batchLength int |
||||
workerNum int |
||||
workerMaxNum int |
||||
workerActiveNum int |
||||
workerNumMu sync.Mutex |
||||
} |
||||
|
||||
type flushType chan struct{} |
||||
|
||||
var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil) |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetName() string { |
||||
return q.baseConfig.ManagedName |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetType() string { |
||||
return q.baseQueueType |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetItemTypeName() string { |
||||
var t T |
||||
return fmt.Sprintf("%T", t) |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetWorkerNumber() int { |
||||
q.workerNumMu.Lock() |
||||
defer q.workerNumMu.Unlock() |
||||
return q.workerNum |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetWorkerActiveNumber() int { |
||||
q.workerNumMu.Lock() |
||||
defer q.workerNumMu.Unlock() |
||||
return q.workerActiveNum |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetWorkerMaxNumber() int { |
||||
q.workerNumMu.Lock() |
||||
defer q.workerNumMu.Unlock() |
||||
return q.workerMaxNum |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) SetWorkerMaxNumber(num int) { |
||||
q.workerNumMu.Lock() |
||||
defer q.workerNumMu.Unlock() |
||||
q.workerMaxNum = num |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) GetQueueItemNumber() int { |
||||
cnt, err := q.baseQueue.Len(q.ctxRun) |
||||
if err != nil { |
||||
log.Error("Failed to get number of items in queue %q: %v", q.GetName(), err) |
||||
} |
||||
return cnt |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.Duration) (err error) { |
||||
if q.isBaseQueueDummy() { |
||||
return |
||||
} |
||||
|
||||
log.Debug("Try to flush queue %q with timeout %v", q.GetName(), timeout) |
||||
defer log.Debug("Finish flushing queue %q, err: %v", q.GetName(), err) |
||||
|
||||
var after <-chan time.Time |
||||
after = infiniteTimerC |
||||
if timeout > 0 { |
||||
after = time.After(timeout) |
||||
} |
||||
c := make(flushType) |
||||
|
||||
// send flush request
|
||||
// if it blocks, it means that there is a flush in progress or the queue hasn't been started yet
|
||||
select { |
||||
case q.flushChan <- c: |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
case <-q.ctxRun.Done(): |
||||
return q.ctxRun.Err() |
||||
case <-after: |
||||
return context.DeadlineExceeded |
||||
} |
||||
|
||||
// wait for flush to finish
|
||||
select { |
||||
case <-c: |
||||
return nil |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
case <-q.ctxRun.Done(): |
||||
return q.ctxRun.Err() |
||||
case <-after: |
||||
return context.DeadlineExceeded |
||||
} |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) marshal(data T) []byte { |
||||
bs, err := json.Marshal(data) |
||||
if err != nil { |
||||
log.Error("Failed to marshal item for queue %q: %v", q.GetName(), err) |
||||
return nil |
||||
} |
||||
return bs |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) unmarshal(data []byte) (t T, ok bool) { |
||||
if err := json.Unmarshal(data, &t); err != nil { |
||||
log.Error("Failed to unmarshal item from queue %q: %v", q.GetName(), err) |
||||
return t, false |
||||
} |
||||
return t, true |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) isBaseQueueDummy() bool { |
||||
_, isDummy := q.baseQueue.(*baseDummy) |
||||
return isDummy |
||||
} |
||||
|
||||
// Push adds an item to the queue, it may block for a while and then returns an error if the queue is full
|
||||
func (q *WorkerPoolQueue[T]) Push(data T) error { |
||||
if q.isBaseQueueDummy() && q.safeHandler != nil { |
||||
// FIXME: the "immediate" queue is only for testing, but it really causes problems because its behavior is different from a real queue.
|
||||
// Even if tests pass, it doesn't mean that there is no bug in code.
|
||||
if data, ok := q.unmarshal(q.marshal(data)); ok { |
||||
q.safeHandler(data) |
||||
} |
||||
} |
||||
return q.baseQueue.PushItem(q.ctxRun, q.marshal(data)) |
||||
} |
||||
|
||||
// Has only works for unique queues. Keep in mind that this check may not be reliable (due to lacking of proper transaction support)
|
||||
// There could be a small chance that duplicate items appear in the queue
|
||||
func (q *WorkerPoolQueue[T]) Has(data T) (bool, error) { |
||||
return q.baseQueue.HasItem(q.ctxRun, q.marshal(data)) |
||||
} |
||||
|
||||
func (q *WorkerPoolQueue[T]) Run(atShutdown, atTerminate func(func())) { |
||||
atShutdown(func() { |
||||
// in case some queue handlers are slow or have hanging bugs, at most wait for a short time
|
||||
q.ShutdownWait(1 * time.Second) |
||||
}) |
||||
q.doRun() |
||||
} |
||||
|
||||
// ShutdownWait shuts down the queue, waits for all workers to finish their jobs, and pushes the unhandled items back to the base queue
|
||||
// It waits for all workers (handlers) to finish their jobs, in case some buggy handlers would hang forever, a reasonable timeout is needed
|
||||
func (q *WorkerPoolQueue[T]) ShutdownWait(timeout time.Duration) { |
||||
shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), timeout) |
||||
defer shutdownCtxCancel() |
||||
if q.ctxShutdown.CompareAndSwap(nil, &shutdownCtx) { |
||||
q.ctxRunCancel() |
||||
} |
||||
<-q.shutdownDone |
||||
} |
||||
|
||||
func getNewQueueFn(t string) (string, func(cfg *BaseConfig, unique bool) (baseQueue, error)) { |
||||
switch t { |
||||
case "dummy", "immediate": |
||||
return t, newBaseDummy |
||||
case "channel": |
||||
return t, newBaseChannelGeneric |
||||
case "redis": |
||||
return t, newBaseRedisGeneric |
||||
default: // level(leveldb,levelqueue,persistable-channel)
|
||||
return "level", newBaseLevelQueueGeneric |
||||
} |
||||
} |
||||
|
||||
func NewWorkerPoolQueueBySetting[T any](name string, queueSetting setting.QueueSettings, handler HandlerFuncT[T], unique bool) (*WorkerPoolQueue[T], error) { |
||||
if handler == nil { |
||||
log.Debug("Use dummy queue for %q because handler is nil and caller doesn't want to process the queue items", name) |
||||
queueSetting.Type = "dummy" |
||||
} |
||||
|
||||
var w WorkerPoolQueue[T] |
||||
var err error |
||||
queueType, newQueueFn := getNewQueueFn(queueSetting.Type) |
||||
w.baseQueueType = queueType |
||||
w.baseConfig = toBaseConfig(name, queueSetting) |
||||
w.baseQueue, err = newQueueFn(w.baseConfig, unique) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
log.Trace("Created queue %q of type %q", name, queueType) |
||||
|
||||
w.ctxRun, w.ctxRunCancel = context.WithCancel(graceful.GetManager().ShutdownContext()) |
||||
w.batchChan = make(chan []T) |
||||
w.flushChan = make(chan flushType) |
||||
w.shutdownDone = make(chan struct{}) |
||||
w.workerMaxNum = queueSetting.MaxWorkers |
||||
w.batchLength = queueSetting.BatchLength |
||||
|
||||
w.origHandler = handler |
||||
w.safeHandler = func(t ...T) (unhandled []T) { |
||||
defer func() { |
||||
err := recover() |
||||
if err != nil { |
||||
log.Error("Recovered from panic in queue %q handler: %v\n%s", name, err, log.Stack(2)) |
||||
} |
||||
}() |
||||
return w.origHandler(t...) |
||||
} |
||||
|
||||
return &w, nil |
||||
} |
||||
@ -0,0 +1,260 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue |
||||
|
||||
import ( |
||||
"context" |
||||
"strconv" |
||||
"sync" |
||||
"testing" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/setting" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func runWorkerPoolQueue[T any](q *WorkerPoolQueue[T]) func() { |
||||
var stop func() |
||||
started := make(chan struct{}) |
||||
stopped := make(chan struct{}) |
||||
go func() { |
||||
q.Run(func(f func()) { stop = f; close(started) }, nil) |
||||
close(stopped) |
||||
}() |
||||
<-started |
||||
return func() { |
||||
stop() |
||||
<-stopped |
||||
} |
||||
} |
||||
|
||||
func TestWorkerPoolQueueUnhandled(t *testing.T) { |
||||
oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load() |
||||
unhandledItemRequeueDuration.Store(0) |
||||
defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration) |
||||
|
||||
mu := sync.Mutex{} |
||||
|
||||
test := func(t *testing.T, queueSetting setting.QueueSettings) { |
||||
queueSetting.Length = 100 |
||||
queueSetting.Type = "channel" |
||||
queueSetting.Datadir = t.TempDir() + "/test-queue" |
||||
m := map[int]int{} |
||||
|
||||
// odds are handled once, evens are handled twice
|
||||
handler := func(items ...int) (unhandled []int) { |
||||
testRecorder.Record("handle:%v", items) |
||||
for _, item := range items { |
||||
mu.Lock() |
||||
if item%2 == 0 && m[item] == 0 { |
||||
unhandled = append(unhandled, item) |
||||
} |
||||
m[item]++ |
||||
mu.Unlock() |
||||
} |
||||
return unhandled |
||||
} |
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", queueSetting, handler, false) |
||||
stop := runWorkerPoolQueue(q) |
||||
for i := 0; i < queueSetting.Length; i++ { |
||||
testRecorder.Record("push:%v", i) |
||||
assert.NoError(t, q.Push(i)) |
||||
} |
||||
assert.NoError(t, q.FlushWithContext(context.Background(), 0)) |
||||
stop() |
||||
|
||||
ok := true |
||||
for i := 0; i < queueSetting.Length; i++ { |
||||
if i%2 == 0 { |
||||
ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i) |
||||
} else { |
||||
ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i) |
||||
} |
||||
} |
||||
if !ok { |
||||
t.Logf("m: %v", m) |
||||
t.Logf("records: %v", testRecorder.Records()) |
||||
} |
||||
testRecorder.Reset() |
||||
} |
||||
|
||||
runCount := 2 // we can run these tests even hundreds times to see its stability
|
||||
t.Run("1/1", func(t *testing.T) { |
||||
for i := 0; i < runCount; i++ { |
||||
test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1}) |
||||
} |
||||
}) |
||||
t.Run("3/1", func(t *testing.T) { |
||||
for i := 0; i < runCount; i++ { |
||||
test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1}) |
||||
} |
||||
}) |
||||
t.Run("4/5", func(t *testing.T) { |
||||
for i := 0; i < runCount; i++ { |
||||
test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5}) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
func TestWorkerPoolQueuePersistence(t *testing.T) { |
||||
runCount := 2 // we can run these tests even hundreds times to see its stability
|
||||
t.Run("1/1", func(t *testing.T) { |
||||
for i := 0; i < runCount; i++ { |
||||
testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100}) |
||||
} |
||||
}) |
||||
t.Run("3/1", func(t *testing.T) { |
||||
for i := 0; i < runCount; i++ { |
||||
testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100}) |
||||
} |
||||
}) |
||||
t.Run("4/5", func(t *testing.T) { |
||||
for i := 0; i < runCount; i++ { |
||||
testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100}) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSettings) { |
||||
testCount := queueSetting.Length |
||||
queueSetting.Type = "level" |
||||
queueSetting.Datadir = t.TempDir() + "/test-queue" |
||||
|
||||
mu := sync.Mutex{} |
||||
|
||||
var tasksQ1, tasksQ2 []string |
||||
q1 := func() { |
||||
startWhenAllReady := make(chan struct{}) // only start data consuming when the "testCount" tasks are all pushed into queue
|
||||
stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item
|
||||
|
||||
testHandler := func(data ...string) []string { |
||||
<-startWhenAllReady |
||||
time.Sleep(10 * time.Millisecond) |
||||
for _, s := range data { |
||||
mu.Lock() |
||||
tasksQ1 = append(tasksQ1, s) |
||||
mu.Unlock() |
||||
|
||||
if s == "task-20" { |
||||
close(stopAt20Shutdown) |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true) |
||||
stop := runWorkerPoolQueue(q) |
||||
for i := 0; i < testCount; i++ { |
||||
_ = q.Push("task-" + strconv.Itoa(i)) |
||||
} |
||||
close(startWhenAllReady) |
||||
<-stopAt20Shutdown // it's possible to have more than 20 tasks executed
|
||||
stop() |
||||
} |
||||
|
||||
q1() // run some tasks and shutdown at an intermediate point
|
||||
|
||||
time.Sleep(100 * time.Millisecond) // because the handler in q1 has a slight delay, we need to wait for it to finish
|
||||
|
||||
q2 := func() { |
||||
testHandler := func(data ...string) []string { |
||||
for _, s := range data { |
||||
mu.Lock() |
||||
tasksQ2 = append(tasksQ2, s) |
||||
mu.Unlock() |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true) |
||||
stop := runWorkerPoolQueue(q) |
||||
assert.NoError(t, q.FlushWithContext(context.Background(), 0)) |
||||
stop() |
||||
} |
||||
|
||||
q2() // restart the queue to continue to execute the tasks in it
|
||||
|
||||
assert.NotZero(t, len(tasksQ1)) |
||||
assert.NotZero(t, len(tasksQ2)) |
||||
assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2)) |
||||
} |
||||
|
||||
func TestWorkerPoolQueueActiveWorkers(t *testing.T) { |
||||
oldWorkerIdleDuration := workerIdleDuration |
||||
workerIdleDuration = 300 * time.Millisecond |
||||
defer func() { |
||||
workerIdleDuration = oldWorkerIdleDuration |
||||
}() |
||||
|
||||
handler := func(items ...int) (unhandled []int) { |
||||
time.Sleep(100 * time.Millisecond) |
||||
return nil |
||||
} |
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false) |
||||
stop := runWorkerPoolQueue(q) |
||||
for i := 0; i < 5; i++ { |
||||
assert.NoError(t, q.Push(i)) |
||||
} |
||||
|
||||
time.Sleep(50 * time.Millisecond) |
||||
assert.EqualValues(t, 1, q.GetWorkerNumber()) |
||||
assert.EqualValues(t, 1, q.GetWorkerActiveNumber()) |
||||
time.Sleep(500 * time.Millisecond) |
||||
assert.EqualValues(t, 1, q.GetWorkerNumber()) |
||||
assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) |
||||
time.Sleep(workerIdleDuration) |
||||
assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working
|
||||
stop() |
||||
|
||||
q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false) |
||||
stop = runWorkerPoolQueue(q) |
||||
for i := 0; i < 15; i++ { |
||||
assert.NoError(t, q.Push(i)) |
||||
} |
||||
|
||||
time.Sleep(50 * time.Millisecond) |
||||
assert.EqualValues(t, 3, q.GetWorkerNumber()) |
||||
assert.EqualValues(t, 3, q.GetWorkerActiveNumber()) |
||||
time.Sleep(500 * time.Millisecond) |
||||
assert.EqualValues(t, 3, q.GetWorkerNumber()) |
||||
assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) |
||||
time.Sleep(workerIdleDuration) |
||||
assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working
|
||||
stop() |
||||
} |
||||
|
||||
func TestWorkerPoolQueueShutdown(t *testing.T) { |
||||
oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load() |
||||
unhandledItemRequeueDuration.Store(int64(100 * time.Millisecond)) |
||||
defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration) |
||||
|
||||
// simulate a slow handler, it doesn't handle any item (all items will be pushed back to the queue)
|
||||
handlerCalled := make(chan struct{}) |
||||
handler := func(items ...int) (unhandled []int) { |
||||
if items[0] == 0 { |
||||
close(handlerCalled) |
||||
} |
||||
time.Sleep(100 * time.Millisecond) |
||||
return items |
||||
} |
||||
|
||||
qs := setting.QueueSettings{Type: "level", Datadir: t.TempDir() + "/queue", BatchLength: 3, MaxWorkers: 4, Length: 20} |
||||
q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false) |
||||
stop := runWorkerPoolQueue(q) |
||||
for i := 0; i < qs.Length; i++ { |
||||
assert.NoError(t, q.Push(i)) |
||||
} |
||||
<-handlerCalled |
||||
time.Sleep(50 * time.Millisecond) // wait for a while to make sure all workers are active
|
||||
assert.EqualValues(t, 4, q.GetWorkerActiveNumber()) |
||||
stop() // stop triggers shutdown
|
||||
assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) |
||||
|
||||
// no item was ever handled, so we still get all of them again
|
||||
q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false) |
||||
assert.EqualValues(t, 20, q.GetQueueItemNumber()) |
||||
} |
||||
@ -0,0 +1,59 @@ |
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package admin |
||||
|
||||
import ( |
||||
"net/http" |
||||
"strconv" |
||||
|
||||
"code.gitea.io/gitea/modules/context" |
||||
"code.gitea.io/gitea/modules/queue" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
) |
||||
|
||||
// Queue shows details for a specific queue
|
||||
func Queue(ctx *context.Context) { |
||||
qid := ctx.ParamsInt64("qid") |
||||
mq := queue.GetManager().GetManagedQueue(qid) |
||||
if mq == nil { |
||||
ctx.Status(http.StatusNotFound) |
||||
return |
||||
} |
||||
ctx.Data["Title"] = ctx.Tr("admin.monitor.queue", mq.GetName()) |
||||
ctx.Data["PageIsAdminMonitor"] = true |
||||
ctx.Data["Queue"] = mq |
||||
ctx.HTML(http.StatusOK, tplQueueManage) |
||||
} |
||||
|
||||
// QueueSet sets the maximum number of workers and other settings for this queue
|
||||
func QueueSet(ctx *context.Context) { |
||||
qid := ctx.ParamsInt64("qid") |
||||
mq := queue.GetManager().GetManagedQueue(qid) |
||||
if mq == nil { |
||||
ctx.Status(http.StatusNotFound) |
||||
return |
||||
} |
||||
|
||||
maxNumberStr := ctx.FormString("max-number") |
||||
|
||||
var err error |
||||
var maxNumber int |
||||
if len(maxNumberStr) > 0 { |
||||
maxNumber, err = strconv.Atoi(maxNumberStr) |
||||
if err != nil { |
||||
ctx.Flash.Error(ctx.Tr("admin.monitor.queue.settings.maxnumberworkers.error")) |
||||
ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) |
||||
return |
||||
} |
||||
if maxNumber < -1 { |
||||
maxNumber = -1 |
||||
} |
||||
} else { |
||||
maxNumber = mq.GetWorkerMaxNumber() |
||||
} |
||||
|
||||
mq.SetWorkerMaxNumber(maxNumber) |
||||
ctx.Flash.Success(ctx.Tr("admin.monitor.queue.settings.changed")) |
||||
ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) |
||||
} |
||||
@ -0,0 +1,48 @@ |
||||
{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}} |
||||
<div class="admin-setting-content"> |
||||
<h4 class="ui top attached header"> |
||||
{{.locale.Tr "admin.monitor.queue" .Queue.GetName}} |
||||
</h4> |
||||
<div class="ui attached table segment"> |
||||
<table class="ui very basic striped table"> |
||||
<thead> |
||||
<tr> |
||||
<th>{{.locale.Tr "admin.monitor.queue.name"}}</th> |
||||
<th>{{.locale.Tr "admin.monitor.queue.type"}}</th> |
||||
<th>{{.locale.Tr "admin.monitor.queue.exemplar"}}</th> |
||||
<th>{{.locale.Tr "admin.monitor.queue.numberworkers"}}</th> |
||||
<th>{{.locale.Tr "admin.monitor.queue.maxnumberworkers"}}</th> |
||||
<th>{{.locale.Tr "admin.monitor.queue.numberinqueue"}}</th> |
||||
</tr> |
||||
</thead> |
||||
<tbody> |
||||
<tr> |
||||
<td>{{.Queue.GetName}}</td> |
||||
<td>{{.Queue.GetType}}</td> |
||||
<td>{{.Queue.GetItemTypeName}}</td> |
||||
<td>{{$sum := .Queue.GetWorkerNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td> |
||||
<td>{{if lt $sum 0}}-{{else}}{{.Queue.GetWorkerMaxNumber}}{{end}}</td> |
||||
<td>{{$sum = .Queue.GetQueueItemNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}</td> |
||||
</tr> |
||||
</tbody> |
||||
</table> |
||||
</div> |
||||
|
||||
<h4 class="ui top attached header"> |
||||
{{.locale.Tr "admin.monitor.queue.settings.title"}} |
||||
</h4> |
||||
<div class="ui attached segment"> |
||||
<p>{{.locale.Tr "admin.monitor.queue.settings.desc"}}</p> |
||||
<form method="POST" action="{{.Link}}/set"> |
||||
{{$.CsrfTokenHtml}} |
||||
<div class="ui form"> |
||||
<div class="inline field"> |
||||
<label for="max-number">{{.locale.Tr "admin.monitor.queue.settings.maxnumberworkers"}}</label> |
||||
<input name="max-number" type="text" placeholder="{{.locale.Tr "admin.monitor.queue.settings.maxnumberworkers.placeholder" .Queue.GetWorkerMaxNumber}}"> |
||||
</div> |
||||
<button class="ui submit button">{{.locale.Tr "admin.monitor.queue.settings.submit"}}</button> |
||||
</div> |
||||
</form> |
||||
</div> |
||||
</div> |
||||
{{template "admin/layout_footer" .}} |
||||
Loading…
Reference in new issue