mirror of
https://github.com/absmach/supermq.git
synced 2026-06-23 06:30:22 +00:00
Split lifecycle from queue root
Signed-off-by: dusan <borovcanindusan1@gmail.com>
This commit is contained in:
@@ -13,7 +13,7 @@ import (
|
||||
paho "github.com/eclipse/paho.mqtt.golang"
|
||||
)
|
||||
|
||||
// BenchmarkConnectionEstablishment measures connection throughput
|
||||
// BenchmarkConnectionEstablishment measures connection throughput.
|
||||
func BenchmarkConnectionEstablishment(b *testing.B) {
|
||||
server := startTestBroker(b)
|
||||
defer server.Stop()
|
||||
@@ -33,7 +33,7 @@ func BenchmarkConnectionEstablishment(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkConnectionEstablishment_Parallel measures concurrent connection throughput
|
||||
// BenchmarkConnectionEstablishment_Parallel measures concurrent connection throughput.
|
||||
func BenchmarkConnectionEstablishment_Parallel(b *testing.B) {
|
||||
server := startTestBroker(b)
|
||||
defer server.Stop()
|
||||
@@ -59,7 +59,7 @@ func BenchmarkConnectionEstablishment_Parallel(b *testing.B) {
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkConcurrentClients measures steady-state performance with N clients
|
||||
// BenchmarkConcurrentClients measures steady-state performance with N clients.
|
||||
func BenchmarkConcurrentClients(b *testing.B) {
|
||||
clientCounts := []int{100, 1000, 5000, 10000}
|
||||
|
||||
@@ -108,7 +108,7 @@ func BenchmarkConcurrentClients(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessageThroughput_EndToEnd measures end-to-end message throughput
|
||||
// BenchmarkMessageThroughput_EndToEnd measures end-to-end message throughput.
|
||||
func BenchmarkMessageThroughput_EndToEnd(b *testing.B) {
|
||||
payloadSizes := []int{100, 1024, 10240, 65536}
|
||||
|
||||
@@ -158,7 +158,7 @@ func BenchmarkMessageThroughput_EndToEnd(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessageThroughput_QoS measures throughput at different QoS levels
|
||||
// BenchmarkMessageThroughput_QoS measures throughput at different QoS levels.
|
||||
func BenchmarkMessageThroughput_QoS(b *testing.B) {
|
||||
qosLevels := []byte{0, 1, 2}
|
||||
|
||||
@@ -205,7 +205,7 @@ func BenchmarkMessageThroughput_QoS(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkFanOut measures 1:N message distribution
|
||||
// BenchmarkFanOut measures 1:N message distribution.
|
||||
func BenchmarkFanOut(b *testing.B) {
|
||||
fanoutCounts := []int{10, 100, 500, 1000}
|
||||
|
||||
@@ -261,7 +261,7 @@ func BenchmarkFanOut(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRetainedMessages measures retained message performance
|
||||
// BenchmarkRetainedMessages measures retained message performance.
|
||||
func BenchmarkRetainedMessages(b *testing.B) {
|
||||
server := startTestBroker(b)
|
||||
defer server.Stop()
|
||||
@@ -286,7 +286,7 @@ func BenchmarkRetainedMessages(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkWildcardSubscriptions measures wildcard routing performance
|
||||
// BenchmarkWildcardSubscriptions measures wildcard routing performance.
|
||||
func BenchmarkWildcardSubscriptions(b *testing.B) {
|
||||
server := startTestBroker(b)
|
||||
defer server.Stop()
|
||||
@@ -335,7 +335,7 @@ func BenchmarkWildcardSubscriptions(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkSessionPersistence measures persistent session overhead
|
||||
// BenchmarkSessionPersistence measures persistent session overhead.
|
||||
func BenchmarkSessionPersistence(b *testing.B) {
|
||||
server := startTestBroker(b)
|
||||
defer server.Stop()
|
||||
@@ -370,7 +370,7 @@ func BenchmarkSessionPersistence(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkKeepAlive measures keep-alive ping/pong overhead
|
||||
// BenchmarkKeepAlive measures keep-alive ping/pong overhead.
|
||||
func BenchmarkKeepAlive(b *testing.B) {
|
||||
server := startTestBroker(b)
|
||||
defer server.Stop()
|
||||
@@ -422,7 +422,7 @@ func createMQTTClient(tb testing.TB, addr, clientID string) paho.Client {
|
||||
return paho.NewClient(opts)
|
||||
}
|
||||
|
||||
// TestServer wraps broker for testing
|
||||
// TestServer wraps broker for testing.
|
||||
type TestServer struct {
|
||||
// TODO: Add actual broker instance
|
||||
addr string
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"github.com/absmach/fluxmq/storage"
|
||||
)
|
||||
|
||||
// BenchmarkMessagePublish_SingleSubscriber benchmarks publishing a message to a single subscriber
|
||||
// BenchmarkMessagePublish_SingleSubscriber benchmarks publishing a message to a single subscriber.
|
||||
func BenchmarkMessagePublish_SingleSubscriber(b *testing.B) {
|
||||
sizes := []int{
|
||||
100, // Small message
|
||||
@@ -53,7 +53,7 @@ func BenchmarkMessagePublish_SingleSubscriber(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_MultipleSubscribers benchmarks publishing to multiple subscribers
|
||||
// BenchmarkMessagePublish_MultipleSubscribers benchmarks publishing to multiple subscribers.
|
||||
func BenchmarkMessagePublish_MultipleSubscribers(b *testing.B) {
|
||||
subscriberCounts := []int{1, 10, 100, 1000}
|
||||
|
||||
@@ -88,7 +88,7 @@ func BenchmarkMessagePublish_MultipleSubscribers(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_QoS1 benchmarks QoS 1 message publishing
|
||||
// BenchmarkMessagePublish_QoS1 benchmarks QoS 1 message publishing.
|
||||
func BenchmarkMessagePublish_QoS1(b *testing.B) {
|
||||
broker := createBenchBroker(b)
|
||||
defer broker.Close()
|
||||
@@ -113,7 +113,7 @@ func BenchmarkMessagePublish_QoS1(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_QoS2 benchmarks QoS 2 message publishing
|
||||
// BenchmarkMessagePublish_QoS2 benchmarks QoS 2 message publishing.
|
||||
func BenchmarkMessagePublish_QoS2(b *testing.B) {
|
||||
broker := createBenchBroker(b)
|
||||
defer broker.Close()
|
||||
@@ -138,7 +138,7 @@ func BenchmarkMessagePublish_QoS2(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_SharedSubscription benchmarks shared subscription routing
|
||||
// BenchmarkMessagePublish_SharedSubscription benchmarks shared subscription routing.
|
||||
func BenchmarkMessagePublish_SharedSubscription(b *testing.B) {
|
||||
subscriberCounts := []int{2, 5, 10}
|
||||
|
||||
@@ -173,7 +173,7 @@ func BenchmarkMessagePublish_SharedSubscription(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_MixedSizes benchmarks realistic mixed message sizes
|
||||
// BenchmarkMessagePublish_MixedSizes benchmarks realistic mixed message sizes.
|
||||
func BenchmarkMessagePublish_MixedSizes(b *testing.B) {
|
||||
broker := createBenchBroker(b)
|
||||
defer broker.Close()
|
||||
@@ -214,7 +214,7 @@ func BenchmarkMessagePublish_MixedSizes(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_FanOut benchmarks 1:N fanout pattern
|
||||
// BenchmarkMessagePublish_FanOut benchmarks 1:N fanout pattern.
|
||||
func BenchmarkMessagePublish_FanOut(b *testing.B) {
|
||||
fanoutSizes := []int{10, 100, 500, 1000}
|
||||
|
||||
@@ -249,7 +249,7 @@ func BenchmarkMessagePublish_FanOut(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessagePublish_TopicVariety benchmarks with different topics
|
||||
// BenchmarkMessagePublish_TopicVariety benchmarks with different topics.
|
||||
func BenchmarkMessagePublish_TopicVariety(b *testing.B) {
|
||||
broker := createBenchBroker(b)
|
||||
defer broker.Close()
|
||||
@@ -289,7 +289,7 @@ func BenchmarkMessagePublish_TopicVariety(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessageDistribute benchmarks the distribute function directly
|
||||
// BenchmarkMessageDistribute benchmarks the distribute function directly.
|
||||
func BenchmarkMessageDistribute(b *testing.B) {
|
||||
broker := createBenchBroker(b)
|
||||
defer broker.Close()
|
||||
@@ -318,7 +318,7 @@ func BenchmarkMessageDistribute(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkBufferPooling benchmarks the buffer pool performance
|
||||
// BenchmarkBufferPooling benchmarks the buffer pool performance.
|
||||
func BenchmarkBufferPooling(b *testing.B) {
|
||||
pool := core.NewBufferPool()
|
||||
payload := make([]byte, 1024)
|
||||
@@ -331,7 +331,7 @@ func BenchmarkBufferPooling(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkBufferPooling_Parallel benchmarks parallel buffer pool usage
|
||||
// BenchmarkBufferPooling_Parallel benchmarks parallel buffer pool usage.
|
||||
func BenchmarkBufferPooling_Parallel(b *testing.B) {
|
||||
pool := core.NewBufferPool()
|
||||
payload := make([]byte, 1024)
|
||||
@@ -347,7 +347,7 @@ func BenchmarkBufferPooling_Parallel(b *testing.B) {
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkMessageCopy_Legacy simulates the old copy-based approach
|
||||
// BenchmarkMessageCopy_Legacy simulates the old copy-based approach.
|
||||
func BenchmarkMessageCopy_Legacy(b *testing.B) {
|
||||
sizes := []int{100, 1024, 10240, 65536}
|
||||
|
||||
@@ -378,7 +378,7 @@ func BenchmarkMessageCopy_Legacy(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMessageCopy_ZeroCopy benchmarks the new zero-copy approach
|
||||
// BenchmarkMessageCopy_ZeroCopy benchmarks the new zero-copy approach.
|
||||
func BenchmarkMessageCopy_ZeroCopy(b *testing.B) {
|
||||
sizes := []int{100, 1024, 10240, 65536}
|
||||
|
||||
@@ -439,13 +439,13 @@ func createBenchSession(tb testing.TB, broker *Broker, clientID string) *session
|
||||
return s
|
||||
}
|
||||
|
||||
// benchAddr implements net.Addr for benchmarks
|
||||
// benchAddr implements net.Addr for benchmarks.
|
||||
type benchAddr struct{}
|
||||
|
||||
func (b *benchAddr) Network() string { return "tcp" }
|
||||
func (b *benchAddr) String() string { return "127.0.0.1:1883" }
|
||||
|
||||
// mockBenchConn is a minimal mock connection for benchmarks
|
||||
// mockBenchConn is a minimal mock connection for benchmarks.
|
||||
type mockBenchConn struct {
|
||||
net.Conn
|
||||
clientID string
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestStress_HighThroughputPublish tests sustained high-throughput publishing
|
||||
// TestStress_HighThroughputPublish tests sustained high-throughput publishing.
|
||||
func TestStress_HighThroughputPublish(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
@@ -92,7 +92,7 @@ func TestStress_HighThroughputPublish(t *testing.T) {
|
||||
assert.Less(t, memIncreasePerMsg, 1000.0, "Memory usage per message should be minimal")
|
||||
}
|
||||
|
||||
// TestStress_ConcurrentPublishers tests multiple concurrent publishers
|
||||
// TestStress_ConcurrentPublishers tests multiple concurrent publishers.
|
||||
func TestStress_ConcurrentPublishers(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
@@ -160,7 +160,7 @@ func TestStress_ConcurrentPublishers(t *testing.T) {
|
||||
assert.Equal(t, uint64(0), errors.Load())
|
||||
}
|
||||
|
||||
// TestStress_MemoryPressure tests behavior under memory pressure with large messages
|
||||
// TestStress_MemoryPressure tests behavior under memory pressure with large messages.
|
||||
func TestStress_MemoryPressure(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
@@ -222,7 +222,7 @@ func TestStress_MemoryPressure(t *testing.T) {
|
||||
assert.Less(t, memIncrease, int64(100*1024*1024), "Memory increase should be reasonable")
|
||||
}
|
||||
|
||||
// TestStress_SustainedLoad tests broker under sustained mixed load
|
||||
// TestStress_SustainedLoad tests broker under sustained mixed load.
|
||||
func TestStress_SustainedLoad(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
@@ -318,7 +318,7 @@ func TestStress_SustainedLoad(t *testing.T) {
|
||||
assert.Less(t, errors.Load(), totalMessages.Load()/1000) // Less than 0.1% errors
|
||||
}
|
||||
|
||||
// TestStress_BufferPoolExhaustion tests buffer pool behavior under extreme load
|
||||
// TestStress_BufferPoolExhaustion tests buffer pool behavior under extreme load.
|
||||
func TestStress_BufferPoolExhaustion(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
@@ -378,7 +378,7 @@ func TestStress_BufferPoolExhaustion(t *testing.T) {
|
||||
assert.Greater(t, totalHits, uint64(0), "Pool should have some hits")
|
||||
}
|
||||
|
||||
// TestStress_FanOutExtreme tests extreme fanout (1 publisher to many subscribers)
|
||||
// TestStress_FanOutExtreme tests extreme fanout (1 publisher to many subscribers).
|
||||
func TestStress_FanOutExtreme(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
@@ -448,7 +448,7 @@ func TestStress_FanOutExtreme(t *testing.T) {
|
||||
assert.Less(t, bytesPerDelivery, 100.0, "Zero-copy should keep per-delivery memory low")
|
||||
}
|
||||
|
||||
// TestStress_RapidSubscribeUnsubscribe tests stability with subscription churn
|
||||
// TestStress_RapidSubscribeUnsubscribe tests stability with subscription churn.
|
||||
func TestStress_RapidSubscribeUnsubscribe(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping stress test in short mode")
|
||||
|
||||
@@ -22,7 +22,7 @@ func isQueueAckTopic(topic string) bool {
|
||||
}
|
||||
|
||||
// extractQueueTopicFromAck extracts the queue topic from an ack topic.
|
||||
// Example: "$queue/tasks/image/$ack" -> "$queue/tasks/image"
|
||||
// Example: "$queue/tasks/image/$ack" -> "$queue/tasks/image".
|
||||
func extractQueueTopicFromAck(ackTopic string) string {
|
||||
if strings.HasSuffix(ackTopic, "/$ack") {
|
||||
return strings.TrimSuffix(ackTopic, "/$ack")
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/absmach/fluxmq/storage"
|
||||
)
|
||||
|
||||
// Pool for subscription slices to reduce allocations in Match()
|
||||
// Pool for subscription slices to reduce allocations in Match().
|
||||
var subscriptionSlicePool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
// Pre-allocate with reasonable capacity for most use cases
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
// - Match operations are lock-free (only atomic loads)
|
||||
// - Subscribe/Unsubscribe use copy-on-write with CAS
|
||||
// - All data structures (maps, slices) are immutable
|
||||
// - Contention on writes is handled by CAS retry loop
|
||||
// - Contention on writes is handled by CAS retry loop.
|
||||
type LockFreeRouter struct {
|
||||
root atomic.Pointer[lockFreeNode]
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
// - Match operations lock only the nodes they traverse (RLock)
|
||||
// - Subscribe/Unsubscribe lock only the affected path (Lock on final node)
|
||||
// - Different branches can be accessed concurrently
|
||||
// - Much better performance than both global mutex and pure lock-free CAS
|
||||
// - Much better performance than both global mutex and pure lock-free CAS.
|
||||
type OptimizedRouter struct {
|
||||
root *optimizedNode
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
"github.com/absmach/fluxmq/config"
|
||||
)
|
||||
|
||||
// mockSender implements Sender interface for testing
|
||||
// mockSender implements Sender interface for testing.
|
||||
type mockSender struct {
|
||||
mu sync.Mutex
|
||||
sendCount int32
|
||||
|
||||
@@ -218,7 +218,6 @@ func (c *Client) sendConnect(conn net.Conn) error {
|
||||
c.opts.Will.MessageExpiry > 0 || c.opts.Will.ContentType != "" ||
|
||||
c.opts.Will.ResponseTopic != "" || len(c.opts.Will.CorrelationData) > 0 ||
|
||||
len(c.opts.Will.UserProperties) > 0 {
|
||||
|
||||
pkt.WillProperties = &v5.WillProperties{}
|
||||
|
||||
if c.opts.Will.WillDelayInterval > 0 {
|
||||
|
||||
+9
-9
@@ -7,19 +7,19 @@ import "errors"
|
||||
|
||||
// Client errors.
|
||||
var (
|
||||
// Configuration errors
|
||||
// Configuration errors.
|
||||
ErrNoServers = errors.New("no servers configured")
|
||||
ErrEmptyClientID = errors.New("client ID cannot be empty")
|
||||
ErrInvalidProtocol = errors.New("invalid protocol version (must be 4 or 5)")
|
||||
|
||||
// Connection errors
|
||||
// Connection errors.
|
||||
ErrNotConnected = errors.New("client not connected")
|
||||
ErrAlreadyConnected = errors.New("client already connected")
|
||||
ErrConnectFailed = errors.New("connection failed")
|
||||
ErrConnectRejected = errors.New("connection rejected by broker")
|
||||
ErrConnectTimeout = errors.New("connection timeout")
|
||||
|
||||
// Operation errors
|
||||
// Operation errors.
|
||||
ErrTimeout = errors.New("operation timed out")
|
||||
ErrMaxInflight = errors.New("maximum inflight messages exceeded")
|
||||
ErrConnectionLost = errors.New("connection lost")
|
||||
@@ -28,7 +28,7 @@ var (
|
||||
ErrInvalidTopic = errors.New("invalid topic")
|
||||
ErrSubscribeFailed = errors.New("subscription failed")
|
||||
|
||||
// Protocol errors
|
||||
// Protocol errors.
|
||||
ErrUnexpectedPacket = errors.New("unexpected packet type")
|
||||
ErrMalformedPacket = errors.New("malformed packet")
|
||||
)
|
||||
@@ -38,12 +38,12 @@ type ConnAckCode byte
|
||||
|
||||
// MQTT 3.1.1 CONNACK return codes.
|
||||
const (
|
||||
ConnAccepted ConnAckCode = 0x00
|
||||
ConnRefusedProtocol ConnAckCode = 0x01
|
||||
ConnRefusedIDRejected ConnAckCode = 0x02
|
||||
ConnAccepted ConnAckCode = 0x00
|
||||
ConnRefusedProtocol ConnAckCode = 0x01
|
||||
ConnRefusedIDRejected ConnAckCode = 0x02
|
||||
ConnRefusedUnavailable ConnAckCode = 0x03
|
||||
ConnRefusedBadAuth ConnAckCode = 0x04
|
||||
ConnRefusedNotAuth ConnAckCode = 0x05
|
||||
ConnRefusedBadAuth ConnAckCode = 0x04
|
||||
ConnRefusedNotAuth ConnAckCode = 0x05
|
||||
)
|
||||
|
||||
// String returns a human-readable description of the CONNACK code.
|
||||
|
||||
+7
-7
@@ -16,13 +16,13 @@ type Message struct {
|
||||
Timestamp time.Time
|
||||
|
||||
// MQTT 5.0 properties
|
||||
PayloadFormat *byte
|
||||
MessageExpiry *uint32
|
||||
ContentType string
|
||||
ResponseTopic string
|
||||
CorrelationData []byte
|
||||
UserProperties map[string]string
|
||||
SubscriptionIDs []uint32
|
||||
PayloadFormat *byte
|
||||
MessageExpiry *uint32
|
||||
ContentType string
|
||||
ResponseTopic string
|
||||
CorrelationData []byte
|
||||
UserProperties map[string]string
|
||||
SubscriptionIDs []uint32
|
||||
}
|
||||
|
||||
// NewMessage creates a new message with the given parameters.
|
||||
|
||||
+19
-19
@@ -41,15 +41,15 @@ type WillMessage struct {
|
||||
// Options configures the MQTT client.
|
||||
type Options struct {
|
||||
// Connection
|
||||
Servers []string // List of broker addresses (host:port)
|
||||
ClientID string // Client identifier
|
||||
Username string // Optional username
|
||||
Password string // Optional password
|
||||
TLSConfig *tls.Config // TLS configuration (nil for plain TCP)
|
||||
ConnectTimeout time.Duration // Timeout for connection attempts
|
||||
WriteTimeout time.Duration // Timeout for write operations
|
||||
KeepAlive time.Duration // Keep-alive interval (0 to disable)
|
||||
PingTimeout time.Duration // Timeout waiting for PINGRESP
|
||||
Servers []string // List of broker addresses (host:port)
|
||||
ClientID string // Client identifier
|
||||
Username string // Optional username
|
||||
Password string // Optional password
|
||||
TLSConfig *tls.Config // TLS configuration (nil for plain TCP)
|
||||
ConnectTimeout time.Duration // Timeout for connection attempts
|
||||
WriteTimeout time.Duration // Timeout for write operations
|
||||
KeepAlive time.Duration // Keep-alive interval (0 to disable)
|
||||
PingTimeout time.Duration // Timeout waiting for PINGRESP
|
||||
|
||||
// Session
|
||||
CleanSession bool // Start with clean session
|
||||
@@ -57,11 +57,11 @@ type Options struct {
|
||||
ProtocolVersion byte // 4 for MQTT 3.1.1, 5 for MQTT 5.0
|
||||
|
||||
// MQTT 5.0 Connect Properties
|
||||
ReceiveMaximum uint16 // Maximum inflight messages client accepts (0 = use default 65535)
|
||||
MaximumPacketSize uint32 // Maximum packet size client accepts (0 = no limit)
|
||||
TopicAliasMaximum uint16 // Maximum topic aliases client accepts (0 = disabled)
|
||||
RequestResponseInfo bool // Request server to send response information in CONNACK
|
||||
RequestProblemInfo bool // Request detailed error information (default true)
|
||||
ReceiveMaximum uint16 // Maximum inflight messages client accepts (0 = use default 65535)
|
||||
MaximumPacketSize uint32 // Maximum packet size client accepts (0 = no limit)
|
||||
TopicAliasMaximum uint16 // Maximum topic aliases client accepts (0 = disabled)
|
||||
RequestResponseInfo bool // Request server to send response information in CONNACK
|
||||
RequestProblemInfo bool // Request detailed error information (default true)
|
||||
|
||||
// Will
|
||||
Will *WillMessage // Last will and testament
|
||||
@@ -76,12 +76,12 @@ type Options struct {
|
||||
MaxReconnectWait time.Duration // Maximum reconnect delay
|
||||
|
||||
// Callbacks
|
||||
OnConnect func() // Called on successful connection
|
||||
OnConnectionLost func(error) // Called when connection is lost
|
||||
OnReconnecting func(attempt int) // Called before each reconnect attempt
|
||||
OnConnect func() // Called on successful connection
|
||||
OnConnectionLost func(error) // Called when connection is lost
|
||||
OnReconnecting func(attempt int) // Called before each reconnect attempt
|
||||
OnMessage func(topic string, payload []byte, qos byte) // Called for incoming messages (basic)
|
||||
OnMessageV2 func(msg *Message) // Called for incoming messages (full context, takes precedence over OnMessage)
|
||||
OnServerCapabilities func(*ServerCapabilities) // Called when server capabilities received (MQTT 5.0)
|
||||
OnMessageV2 func(msg *Message) // Called for incoming messages (full context, takes precedence over OnMessage)
|
||||
OnServerCapabilities func(*ServerCapabilities) // Called when server capabilities received (MQTT 5.0)
|
||||
|
||||
// Advanced
|
||||
MessageChanSize int // Size of internal message channel
|
||||
|
||||
@@ -46,9 +46,9 @@ func TestOptionsBuilder(t *testing.T) {
|
||||
SetCredentials("user", "pass").
|
||||
SetTLSConfig(tlsConfig).
|
||||
SetCleanSession(false).
|
||||
SetKeepAlive(30 * time.Second).
|
||||
SetConnectTimeout(5 * time.Second).
|
||||
SetAckTimeout(20 * time.Second).
|
||||
SetKeepAlive(30*time.Second).
|
||||
SetConnectTimeout(5*time.Second).
|
||||
SetAckTimeout(20*time.Second).
|
||||
SetProtocolVersion(5).
|
||||
SetWill("will/topic", []byte("goodbye"), 1, true).
|
||||
SetAutoReconnect(false).
|
||||
|
||||
+8
-8
@@ -19,14 +19,14 @@ const (
|
||||
|
||||
// pendingOp represents a pending operation waiting for acknowledgment.
|
||||
type pendingOp struct {
|
||||
id uint16
|
||||
opType pendingType
|
||||
done chan struct{}
|
||||
err error
|
||||
result interface{} // For SUBACK return codes, etc.
|
||||
created time.Time
|
||||
message *Message // For QoS 1/2 retransmission
|
||||
qos2State int // 0: waiting PUBREC, 1: waiting PUBCOMP
|
||||
id uint16
|
||||
opType pendingType
|
||||
done chan struct{}
|
||||
err error
|
||||
result interface{} // For SUBACK return codes, etc.
|
||||
created time.Time
|
||||
message *Message // For QoS 1/2 retransmission
|
||||
qos2State int // 0: waiting PUBREC, 1: waiting PUBCOMP
|
||||
}
|
||||
|
||||
// pendingStore manages pending operations.
|
||||
|
||||
@@ -36,7 +36,6 @@ func TestPendingStoreAdd(t *testing.T) {
|
||||
|
||||
msg := NewMessage("test/topic", []byte("payload"), 1, false)
|
||||
op, err := ps.add(1, pendingPublish, msg)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("add failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func newTopicAliasManager(clientMax, serverMax uint16) *topicAliasManager {
|
||||
// Returns (alias, isNew, ok).
|
||||
// - If topic already has an alias: (alias, false, true)
|
||||
// - If topic can be assigned new alias: (newAlias, true, true)
|
||||
// - If aliases disabled or limit reached: (0, false, false)
|
||||
// - If aliases disabled or limit reached: (0, false, false).
|
||||
func (m *topicAliasManager) getOrAssignOutbound(topic string) (uint16, bool, bool) {
|
||||
if m.outboundMaximum == 0 {
|
||||
return 0, false, false // Aliases disabled
|
||||
|
||||
+1
-1
@@ -159,7 +159,7 @@ type NodeInfo struct {
|
||||
// - Delivering messages routed from other nodes
|
||||
// - Providing session state during takeover
|
||||
// - Fetching retained messages from local storage
|
||||
// - Fetching will messages from local storage
|
||||
// - Fetching will messages from local storage.
|
||||
type MessageHandler interface {
|
||||
// DeliverToClient delivers a message to a local MQTT client.
|
||||
// This is called when a message is routed from another broker node.
|
||||
|
||||
@@ -322,7 +322,7 @@ func TestCrossNode_SubscriptionPropagation(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestCrossNode_UnsubscribePropagation verifies unsubscribe propagates and stops delivery.
|
||||
// NOTE: Requires Unsubscribe() method in testutil.TestMQTTClient (see plan.md Task #7)
|
||||
// NOTE: Requires Unsubscribe() method in testutil.TestMQTTClient (see plan.md Task #7).
|
||||
func TestCrossNode_UnsubscribePropagation(t *testing.T) {
|
||||
t.Skip("Requires Unsubscribe() implementation in test client (plan.md Task #7)")
|
||||
|
||||
@@ -365,7 +365,7 @@ func TestCrossNode_UnsubscribePropagation(t *testing.T) {
|
||||
|
||||
// TestCrossNode_RetainedMessages verifies retained messages work across nodes.
|
||||
// NOTE: Retained messages currently only work within single node (local storage).
|
||||
// Cross-node delivery requires centralized storage (plan.md Task #8)
|
||||
// Cross-node delivery requires centralized storage (plan.md Task #8).
|
||||
func TestCrossNode_RetainedMessages(t *testing.T) {
|
||||
t.Skip("Retained message cross-node delivery requires centralized storage (plan.md Task #8)")
|
||||
|
||||
@@ -407,7 +407,7 @@ func TestCrossNode_RetainedMessages(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestCrossNode_HighThroughput verifies cluster can handle high message throughput.
|
||||
// NOTE: Performance test - can be slow, enable manually when needed
|
||||
// NOTE: Performance test - can be slow, enable manually when needed.
|
||||
func TestCrossNode_HighThroughput(t *testing.T) {
|
||||
t.Skip("Performance test - enable manually when needed")
|
||||
|
||||
@@ -462,7 +462,7 @@ func TestCrossNode_HighThroughput(t *testing.T) {
|
||||
}
|
||||
|
||||
// BenchmarkCrossNode_MessageLatency measures cross-node message delivery latency.
|
||||
// NOTE: Benchmarks are disabled for now - enable when needed for performance testing
|
||||
// NOTE: Benchmarks are disabled for now - enable when needed for performance testing.
|
||||
func BenchmarkCrossNode_MessageLatency(b *testing.B) {
|
||||
b.Skip("Benchmarks disabled - enable manually when needed")
|
||||
|
||||
@@ -518,7 +518,7 @@ func BenchmarkCrossNode_MessageLatency(b *testing.B) {
|
||||
}
|
||||
|
||||
// BenchmarkCrossNode_Throughput measures maximum message throughput across nodes.
|
||||
// NOTE: Benchmarks are disabled for now - enable when needed for performance testing
|
||||
// NOTE: Benchmarks are disabled for now - enable when needed for performance testing.
|
||||
func BenchmarkCrossNode_Throughput(b *testing.B) {
|
||||
b.Skip("Benchmarks disabled - enable manually when needed")
|
||||
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ type RetainedDataEntry struct {
|
||||
|
||||
// RetainedStore implements storage.RetainedStore using hybrid storage strategy:
|
||||
// - Small messages (<1KB): Replicated to all nodes via etcd
|
||||
// - Large messages (≥1KB): Stored on owner node, fetched on-demand via gRPC
|
||||
// - Large messages (≥1KB): Stored on owner node, fetched on-demand via gRPC.
|
||||
type RetainedStore struct {
|
||||
nodeID string
|
||||
localStore storage.RetainedStore // BadgerDB for local payload storage
|
||||
|
||||
@@ -235,7 +235,7 @@ func TestRetainedDataEntry_JSON(t *testing.T) {
|
||||
assert.Equal(t, entry.Properties, unmarshaled.Properties)
|
||||
}
|
||||
|
||||
// Benchmark tests
|
||||
// Benchmark tests.
|
||||
func BenchmarkRetainedStore_SetSmallMessage(b *testing.B) {
|
||||
b.Skip("Requires etcd instance")
|
||||
// Benchmark Set() for small messages
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package cluster
|
||||
|
||||
import (
|
||||
|
||||
@@ -149,7 +149,6 @@ func main() {
|
||||
var cl cluster.Cluster
|
||||
var etcdCluster *cluster.EtcdCluster
|
||||
if cfg.Cluster.Enabled {
|
||||
|
||||
// Build transport TLS config if enabled
|
||||
var transportTLS *cluster.TransportTLSConfig
|
||||
if cfg.Cluster.Transport.TLSEnabled {
|
||||
@@ -193,14 +192,12 @@ func main() {
|
||||
"etcd_data_dir", cfg.Cluster.Etcd.DataDir,
|
||||
"etcd_bind", cfg.Cluster.Etcd.BindAddr)
|
||||
} else {
|
||||
|
||||
cl = cluster.NewNoopCluster(cfg.Cluster.NodeID)
|
||||
slog.Info("Running in single-node mode", "node_id", cfg.Cluster.NodeID)
|
||||
}
|
||||
|
||||
var webhooks broker.Notifier
|
||||
if cfg.Webhook.Enabled {
|
||||
|
||||
sender := webhook.NewHTTPSender()
|
||||
|
||||
wh, err := webhook.NewNotifier(cfg.Webhook, cfg.Cluster.NodeID, sender, logger)
|
||||
@@ -223,7 +220,6 @@ func main() {
|
||||
var tracer trace.Tracer
|
||||
|
||||
if cfg.Server.MetricsEnabled {
|
||||
|
||||
shutdown, err := otel.InitProvider(cfg.Server, cfg.Cluster.NodeID)
|
||||
if err != nil {
|
||||
slog.Error("Failed to initialize OpenTelemetry", "error", err)
|
||||
|
||||
+43
-43
@@ -25,7 +25,7 @@ type Config struct {
|
||||
|
||||
// RateLimitConfig holds rate limiting configuration.
|
||||
type RateLimitConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Connection ConnectionRateLimitConfig `yaml:"connection"`
|
||||
Message MessageRateLimitConfig `yaml:"message"`
|
||||
Subscribe SubscribeRateLimitConfig `yaml:"subscribe"`
|
||||
@@ -55,33 +55,33 @@ type SubscribeRateLimitConfig struct {
|
||||
|
||||
// ServerConfig holds server-related configuration.
|
||||
type ServerConfig struct {
|
||||
TCPAddr string `yaml:"tcp_addr"`
|
||||
TLSCertFile string `yaml:"tls_cert_file"`
|
||||
TLSKeyFile string `yaml:"tls_key_file"`
|
||||
TLSCAFile string `yaml:"tls_ca_file"` // CA certificate for client verification
|
||||
TLSClientAuth string `yaml:"tls_client_auth"` // "none", "request", or "require"
|
||||
HTTPAddr string `yaml:"http_addr"`
|
||||
WSAddr string `yaml:"ws_addr"`
|
||||
WSPath string `yaml:"ws_path"`
|
||||
WSAllowedOrigins []string `yaml:"ws_allowed_origins"` // Allowed origins for WebSocket (empty = allow all in dev mode)
|
||||
CoAPAddr string `yaml:"coap_addr"`
|
||||
CoAPDTLSEnabled bool `yaml:"coap_dtls_enabled"`
|
||||
CoAPDTLSCertFile string `yaml:"coap_dtls_cert_file"`
|
||||
CoAPDTLSKeyFile string `yaml:"coap_dtls_key_file"`
|
||||
CoAPDTLSCAFile string `yaml:"coap_dtls_ca_file"` // For mDTLS client verification
|
||||
CoAPDTLSClientAuth string `yaml:"coap_dtls_client_auth"` // "none", "request", "require"
|
||||
HealthAddr string `yaml:"health_addr"`
|
||||
MetricsAddr string `yaml:"metrics_addr"` // Now used for OTLP endpoint
|
||||
TCPMaxConn int `yaml:"tcp_max_connections"`
|
||||
TCPReadTimeout time.Duration `yaml:"tcp_read_timeout"`
|
||||
TCPWriteTimeout time.Duration `yaml:"tcp_write_timeout"`
|
||||
ShutdownTimeout time.Duration `yaml:"shutdown_timeout"`
|
||||
TLSEnabled bool `yaml:"tls_enabled"`
|
||||
HTTPEnabled bool `yaml:"http_enabled"`
|
||||
WSEnabled bool `yaml:"ws_enabled"`
|
||||
CoAPEnabled bool `yaml:"coap_enabled"`
|
||||
HealthEnabled bool `yaml:"health_enabled"`
|
||||
MetricsEnabled bool `yaml:"metrics_enabled"` // Now enables OTel
|
||||
TCPAddr string `yaml:"tcp_addr"`
|
||||
TLSCertFile string `yaml:"tls_cert_file"`
|
||||
TLSKeyFile string `yaml:"tls_key_file"`
|
||||
TLSCAFile string `yaml:"tls_ca_file"` // CA certificate for client verification
|
||||
TLSClientAuth string `yaml:"tls_client_auth"` // "none", "request", or "require"
|
||||
HTTPAddr string `yaml:"http_addr"`
|
||||
WSAddr string `yaml:"ws_addr"`
|
||||
WSPath string `yaml:"ws_path"`
|
||||
WSAllowedOrigins []string `yaml:"ws_allowed_origins"` // Allowed origins for WebSocket (empty = allow all in dev mode)
|
||||
CoAPAddr string `yaml:"coap_addr"`
|
||||
CoAPDTLSEnabled bool `yaml:"coap_dtls_enabled"`
|
||||
CoAPDTLSCertFile string `yaml:"coap_dtls_cert_file"`
|
||||
CoAPDTLSKeyFile string `yaml:"coap_dtls_key_file"`
|
||||
CoAPDTLSCAFile string `yaml:"coap_dtls_ca_file"` // For mDTLS client verification
|
||||
CoAPDTLSClientAuth string `yaml:"coap_dtls_client_auth"` // "none", "request", "require"
|
||||
HealthAddr string `yaml:"health_addr"`
|
||||
MetricsAddr string `yaml:"metrics_addr"` // Now used for OTLP endpoint
|
||||
TCPMaxConn int `yaml:"tcp_max_connections"`
|
||||
TCPReadTimeout time.Duration `yaml:"tcp_read_timeout"`
|
||||
TCPWriteTimeout time.Duration `yaml:"tcp_write_timeout"`
|
||||
ShutdownTimeout time.Duration `yaml:"shutdown_timeout"`
|
||||
TLSEnabled bool `yaml:"tls_enabled"`
|
||||
HTTPEnabled bool `yaml:"http_enabled"`
|
||||
WSEnabled bool `yaml:"ws_enabled"`
|
||||
CoAPEnabled bool `yaml:"coap_enabled"`
|
||||
HealthEnabled bool `yaml:"health_enabled"`
|
||||
MetricsEnabled bool `yaml:"metrics_enabled"` // Now enables OTel
|
||||
|
||||
// OpenTelemetry configuration
|
||||
OtelServiceName string `yaml:"otel_service_name"`
|
||||
@@ -225,26 +225,26 @@ type WebhookEndpoint struct {
|
||||
func Default() *Config {
|
||||
return &Config{
|
||||
Server: ServerConfig{
|
||||
TCPAddr: ":1883",
|
||||
TCPMaxConn: 10000,
|
||||
TCPReadTimeout: 60 * time.Second,
|
||||
TCPWriteTimeout: 60 * time.Second,
|
||||
TLSEnabled: false,
|
||||
TLSClientAuth: "none",
|
||||
HTTPAddr: ":8080",
|
||||
HTTPEnabled: false,
|
||||
WSAddr: ":8083",
|
||||
WSPath: "/mqtt",
|
||||
WSEnabled: true,
|
||||
TCPAddr: ":1883",
|
||||
TCPMaxConn: 10000,
|
||||
TCPReadTimeout: 60 * time.Second,
|
||||
TCPWriteTimeout: 60 * time.Second,
|
||||
TLSEnabled: false,
|
||||
TLSClientAuth: "none",
|
||||
HTTPAddr: ":8080",
|
||||
HTTPEnabled: false,
|
||||
WSAddr: ":8083",
|
||||
WSPath: "/mqtt",
|
||||
WSEnabled: true,
|
||||
CoAPAddr: ":5683",
|
||||
CoAPEnabled: false,
|
||||
CoAPDTLSEnabled: false,
|
||||
CoAPDTLSClientAuth: "none",
|
||||
HealthAddr: ":8081",
|
||||
HealthEnabled: true,
|
||||
MetricsAddr: "localhost:4317",
|
||||
MetricsEnabled: false,
|
||||
ShutdownTimeout: 30 * time.Second,
|
||||
HealthEnabled: true,
|
||||
MetricsAddr: "localhost:4317",
|
||||
MetricsEnabled: false,
|
||||
ShutdownTimeout: 30 * time.Second,
|
||||
|
||||
// OpenTelemetry defaults
|
||||
OtelServiceName: "mqtt-broker",
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Publish packet pool to reduce allocations during message delivery
|
||||
// Publish packet pool to reduce allocations during message delivery.
|
||||
var publishPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &Publish{}
|
||||
|
||||
@@ -347,7 +347,7 @@ func (pkt *ConnAck) Unpack(r io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reason returns a string representation of the meaning of the ReasonCode
|
||||
// Reason returns a string representation of the meaning of the ReasonCode.
|
||||
func (c *ConnAck) Reason() string {
|
||||
switch c.ReasonCode {
|
||||
case 0:
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Publish packet pool to reduce allocations during message delivery
|
||||
// Publish packet pool to reduce allocations during message delivery.
|
||||
var publishPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &Publish{
|
||||
|
||||
@@ -115,7 +115,7 @@ func (pkt *PubAck) Unpack(r io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reason returns a string representation of the meaning of the ReasonCode
|
||||
// Reason returns a string representation of the meaning of the ReasonCode.
|
||||
func (p *PubAck) Reason() string {
|
||||
switch *p.ReasonCode {
|
||||
case 0:
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/absmach/fluxmq/core/packets"
|
||||
)
|
||||
|
||||
// The List of valid PubRec reason codes
|
||||
// The List of valid PubRec reason codes.
|
||||
const (
|
||||
PubRecSuccess = 0x00
|
||||
PubRecNoMatchingSubscribers = 0x10
|
||||
|
||||
@@ -110,7 +110,7 @@ func (pkt *SubAck) Unpack(r io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reason returns a string representation of the meaning of the ReasonCode
|
||||
// Reason returns a string representation of the meaning of the ReasonCode.
|
||||
func (s *SubAck) Reason(index int) string {
|
||||
if index >= 0 && index < len(*s.ReasonCodes) {
|
||||
switch (*s.ReasonCodes)[index] {
|
||||
|
||||
@@ -108,7 +108,7 @@ func (pkt *UnsubAck) Details() Details {
|
||||
return Details{Type: UnsubAckType, ID: pkt.ID, QoS: 0}
|
||||
}
|
||||
|
||||
// Reason returns a string representation of the meaning of the ReasonCode
|
||||
// Reason returns a string representation of the meaning of the ReasonCode.
|
||||
func (u *UnsubAck) Reason(index int) string {
|
||||
if index >= 0 && index < len(*u.ReasonCodes) {
|
||||
switch (*u.ReasonCodes)[index] {
|
||||
|
||||
@@ -72,8 +72,8 @@ func TestRefCountedBuffer_SizeClasses(t *testing.T) {
|
||||
pool := NewBufferPool()
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
size int
|
||||
name string
|
||||
size int
|
||||
expectedCap int
|
||||
}{
|
||||
{"small", 512, 1024},
|
||||
|
||||
+11
-10
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// Partition is an interface for partition operations needed by consumer groups.
|
||||
@@ -63,7 +64,7 @@ func (cgm *GroupManager) AddConsumer(ctx context.Context, groupID, consumerID, c
|
||||
cgm.groups[groupID] = group
|
||||
}
|
||||
|
||||
consumer := &queueStorage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: consumerID,
|
||||
ClientID: clientID,
|
||||
GroupID: groupID,
|
||||
@@ -134,7 +135,7 @@ func (cgm *GroupManager) ListGroups() []*Group {
|
||||
|
||||
// RestoreConsumer restores a consumer from persistent storage without re-persisting.
|
||||
// This is used during startup to restore consumers that were saved before shutdown.
|
||||
func (cgm *GroupManager) RestoreConsumer(consumer *queueStorage.Consumer) {
|
||||
func (cgm *GroupManager) RestoreConsumer(consumer *types.Consumer) {
|
||||
cgm.mu.Lock()
|
||||
defer cgm.mu.Unlock()
|
||||
|
||||
@@ -238,7 +239,7 @@ func (cgm *GroupManager) Stop() {
|
||||
type Group struct {
|
||||
id string
|
||||
queueName string
|
||||
consumers map[string]*queueStorage.Consumer
|
||||
consumers map[string]*types.Consumer
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
@@ -247,7 +248,7 @@ func NewGroup(id, queueName string) *Group {
|
||||
return &Group{
|
||||
id: id,
|
||||
queueName: queueName,
|
||||
consumers: make(map[string]*queueStorage.Consumer),
|
||||
consumers: make(map[string]*types.Consumer),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,7 +258,7 @@ func (cg *Group) ID() string {
|
||||
}
|
||||
|
||||
// AddConsumer adds a consumer to the group.
|
||||
func (cg *Group) AddConsumer(consumer *queueStorage.Consumer) {
|
||||
func (cg *Group) AddConsumer(consumer *types.Consumer) {
|
||||
cg.mu.Lock()
|
||||
defer cg.mu.Unlock()
|
||||
|
||||
@@ -273,7 +274,7 @@ func (cg *Group) RemoveConsumer(consumerID string) {
|
||||
}
|
||||
|
||||
// GetConsumer returns a consumer by ID.
|
||||
func (cg *Group) GetConsumer(consumerID string) (*queueStorage.Consumer, bool) {
|
||||
func (cg *Group) GetConsumer(consumerID string) (*types.Consumer, bool) {
|
||||
cg.mu.RLock()
|
||||
defer cg.mu.RUnlock()
|
||||
|
||||
@@ -282,11 +283,11 @@ func (cg *Group) GetConsumer(consumerID string) (*queueStorage.Consumer, bool) {
|
||||
}
|
||||
|
||||
// ListConsumers returns all consumers in the group.
|
||||
func (cg *Group) ListConsumers() []*queueStorage.Consumer {
|
||||
func (cg *Group) ListConsumers() []*types.Consumer {
|
||||
cg.mu.RLock()
|
||||
defer cg.mu.RUnlock()
|
||||
|
||||
consumers := make([]*queueStorage.Consumer, 0, len(cg.consumers))
|
||||
consumers := make([]*types.Consumer, 0, len(cg.consumers))
|
||||
for _, consumer := range cg.consumers {
|
||||
consumers = append(consumers, consumer)
|
||||
}
|
||||
@@ -320,7 +321,7 @@ func (cg *Group) Rebalance(partitions []Partition) {
|
||||
}
|
||||
|
||||
// Convert to slice for indexing
|
||||
consumers := make([]*queueStorage.Consumer, 0, len(cg.consumers))
|
||||
consumers := make([]*types.Consumer, 0, len(cg.consumers))
|
||||
for _, consumer := range cg.consumers {
|
||||
consumers = append(consumers, consumer)
|
||||
}
|
||||
@@ -347,7 +348,7 @@ func (cg *Group) Rebalance(partitions []Partition) {
|
||||
}
|
||||
|
||||
// GetConsumerForPartition returns the consumer assigned to a partition.
|
||||
func (cg *Group) GetConsumerForPartition(partitionID int) (*queueStorage.Consumer, bool) {
|
||||
func (cg *Group) GetConsumerForPartition(partitionID int) (*types.Consumer, bool) {
|
||||
cg.mu.RLock()
|
||||
defer cg.mu.RUnlock()
|
||||
|
||||
|
||||
@@ -8,13 +8,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// MockPartition implements Partition interface for testing
|
||||
// MockPartition implements Partition interface for testing.
|
||||
type MockPartition struct {
|
||||
id int
|
||||
assignedTo string
|
||||
@@ -61,7 +61,7 @@ func TestNewGroupManager(t *testing.T) {
|
||||
func TestGroupManager_AddConsumer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -85,7 +85,7 @@ func TestGroupManager_AddConsumer(t *testing.T) {
|
||||
func TestGroupManager_AddConsumer_Duplicate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -102,7 +102,7 @@ func TestGroupManager_AddConsumer_Duplicate(t *testing.T) {
|
||||
func TestGroupManager_RemoveConsumer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -132,7 +132,7 @@ func TestGroupManager_RemoveConsumer_NonExistent(t *testing.T) {
|
||||
func TestGroupManager_GetGroup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -157,7 +157,7 @@ func TestGroupManager_GetGroup(t *testing.T) {
|
||||
func TestGroupManager_ListGroups(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -180,7 +180,7 @@ func TestGroupManager_ListGroups(t *testing.T) {
|
||||
func TestGroupManager_Rebalance(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 6
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -229,7 +229,7 @@ func TestGroupManager_Rebalance(t *testing.T) {
|
||||
func TestGroupManager_Rebalance_UnevenDistribution(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 5
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -268,7 +268,7 @@ func TestGroupManager_Rebalance_UnevenDistribution(t *testing.T) {
|
||||
func TestGroup_Consumers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -291,7 +291,7 @@ func TestGroup_Consumers(t *testing.T) {
|
||||
func TestGroup_Size(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -312,7 +312,7 @@ func TestGroup_Size(t *testing.T) {
|
||||
func TestGroup_GetConsumerForPartition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 4
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/consumer"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// DeliverFn defines the function signature for delivering messages to clients.
|
||||
@@ -26,7 +26,7 @@ const (
|
||||
// QueueSource abstract the queue source to avoid circular dependency.
|
||||
type QueueSource interface {
|
||||
Name() string
|
||||
Config() queueStorage.QueueConfig
|
||||
Config() types.QueueConfig
|
||||
OrderingEnforcer() OrderingEnforcer
|
||||
ConsumerGroups() *consumer.GroupManager
|
||||
}
|
||||
@@ -34,8 +34,8 @@ type QueueSource interface {
|
||||
// OrderingEnforcer abstracts the ordering enforcement logic.
|
||||
// Group-aware: each consumer group tracks its own ordering independently.
|
||||
type OrderingEnforcer interface {
|
||||
CanDeliver(msg *queueStorage.Message, groupID string) (bool, error)
|
||||
MarkDelivered(msg *queueStorage.Message, groupID string)
|
||||
CanDeliver(msg *types.Message, groupID string) (bool, error)
|
||||
MarkDelivered(msg *types.Message, groupID string)
|
||||
}
|
||||
|
||||
// RaftManager abstracts the Raft consensus manager.
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/absmach/fluxmq/cluster"
|
||||
"github.com/absmach/fluxmq/core"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
brokerStorage "github.com/absmach/fluxmq/storage"
|
||||
)
|
||||
|
||||
@@ -196,9 +197,9 @@ func (pw *PartitionWorker) ProcessMessages(ctx context.Context) {
|
||||
// deliverMessage delivers a single message to a consumer.
|
||||
func (pw *PartitionWorker) deliverMessage(
|
||||
ctx context.Context,
|
||||
msg *queueStorage.Message,
|
||||
consumer *queueStorage.Consumer,
|
||||
config queueStorage.QueueConfig,
|
||||
msg *types.Message,
|
||||
consumer *types.Consumer,
|
||||
config types.QueueConfig,
|
||||
) error {
|
||||
// Check ordering constraints (per-group)
|
||||
if pw.queue.OrderingEnforcer() != nil {
|
||||
@@ -212,7 +213,7 @@ func (pw *PartitionWorker) deliverMessage(
|
||||
}
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: msg.ID,
|
||||
QueueName: pw.queueName,
|
||||
PartitionID: pw.partitionID,
|
||||
@@ -226,8 +227,7 @@ func (pw *PartitionWorker) deliverMessage(
|
||||
return fmt.Errorf("failed to mark inflight: %w", err)
|
||||
}
|
||||
|
||||
// Update message state
|
||||
msg.State = queueStorage.StateDelivered
|
||||
msg.State = types.StateDelivered
|
||||
msg.DeliveredAt = time.Now()
|
||||
|
||||
if err := pw.messageStore.UpdateMessage(ctx, pw.queueName, msg); err != nil {
|
||||
@@ -274,7 +274,7 @@ func (pw *PartitionWorker) deliverMessage(
|
||||
}
|
||||
|
||||
// ToStorageMessage converts a queue message to a broker storage message for MQTT delivery.
|
||||
func ToStorageMessage(msg *queueStorage.Message, queueTopic string) interface{} {
|
||||
func ToStorageMessage(msg *types.Message, queueTopic string) interface{} {
|
||||
storageMsg := &brokerStorage.Message{
|
||||
Topic: queueTopic,
|
||||
QoS: 1, // Queue messages always use QoS 1 for delivery confirmation
|
||||
|
||||
@@ -12,16 +12,16 @@ import (
|
||||
"github.com/absmach/fluxmq/cluster"
|
||||
"github.com/absmach/fluxmq/cluster/grpc"
|
||||
"github.com/absmach/fluxmq/queue/consumer"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
brokerStorage "github.com/absmach/fluxmq/storage"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// MockQueue implements QueueSource interface
|
||||
// MockQueue implements QueueSource interface.
|
||||
type MockQueue struct {
|
||||
config queueStorage.QueueConfig
|
||||
config types.QueueConfig
|
||||
consumerGroups *consumer.GroupManager
|
||||
orderingEnforcer OrderingEnforcer
|
||||
}
|
||||
@@ -30,7 +30,7 @@ func (m *MockQueue) Name() string {
|
||||
return m.config.Name
|
||||
}
|
||||
|
||||
func (m *MockQueue) Config() queueStorage.QueueConfig {
|
||||
func (m *MockQueue) Config() types.QueueConfig {
|
||||
return m.config
|
||||
}
|
||||
|
||||
@@ -42,16 +42,16 @@ func (m *MockQueue) ConsumerGroups() *consumer.GroupManager {
|
||||
return m.consumerGroups
|
||||
}
|
||||
|
||||
// MockOrderingEnforcer implements OrderingEnforcer interface
|
||||
// MockOrderingEnforcer implements OrderingEnforcer interface.
|
||||
type MockOrderingEnforcer struct{}
|
||||
|
||||
func (m *MockOrderingEnforcer) CanDeliver(msg *queueStorage.Message, groupID string) (bool, error) {
|
||||
func (m *MockOrderingEnforcer) CanDeliver(msg *types.Message, groupID string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *MockOrderingEnforcer) MarkDelivered(msg *queueStorage.Message, groupID string) {}
|
||||
func (m *MockOrderingEnforcer) MarkDelivered(msg *types.Message, groupID string) {}
|
||||
|
||||
// MockBroker is a simple mock for broker delivery
|
||||
// MockBroker is a simple mock for broker delivery.
|
||||
type MockBroker struct {
|
||||
deliveries map[string][]any
|
||||
mu sync.Mutex
|
||||
@@ -76,7 +76,7 @@ func (m *MockBroker) GetDeliveries(clientID string) []any {
|
||||
return m.deliveries[clientID]
|
||||
}
|
||||
|
||||
// MockCluster implements cluster.Cluster interface
|
||||
// MockCluster implements cluster.Cluster interface.
|
||||
type MockCluster struct {
|
||||
localNodeID string
|
||||
routeQueueMsgCalls []RouteQueueMsgCall
|
||||
@@ -100,7 +100,7 @@ func NewMockCluster(nodeID string) *MockCluster {
|
||||
}
|
||||
}
|
||||
|
||||
// Implement required methods of cluster.Cluster interface (stubbed)
|
||||
// Implement required methods of cluster.Cluster interface (stubbed).
|
||||
func (c *MockCluster) NodeID() string { return c.localNodeID }
|
||||
func (c *MockCluster) IsLeader() bool { return true }
|
||||
func (c *MockCluster) WaitForLeader(context.Context) error { return nil }
|
||||
@@ -178,7 +178,7 @@ func (c *MockCluster) RouteQueueMessage(ctx context.Context, nodeID, clientID, q
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper to create consumer partitions
|
||||
// Helper to create consumer partitions.
|
||||
type mockPartition struct {
|
||||
id int
|
||||
}
|
||||
@@ -207,7 +207,7 @@ func TestPartitionWorker_RouteQueueMessage_ProxyMode(t *testing.T) {
|
||||
parts := toConsumerPartitions(1)
|
||||
consumerMgr := consumer.NewGroupManager(queueName, consumerStore, time.Second, parts)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
mockQueue := &MockQueue{
|
||||
config: config,
|
||||
consumerGroups: consumerMgr,
|
||||
@@ -243,12 +243,12 @@ func TestPartitionWorker_RouteQueueMessage_ProxyMode(t *testing.T) {
|
||||
consumerMgr.Rebalance("group-1", parts)
|
||||
|
||||
// Create and store a message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: queueName,
|
||||
Payload: []byte("test payload"),
|
||||
Properties: map[string]string{"key": "value"},
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
}
|
||||
@@ -290,7 +290,7 @@ func TestPartitionWorker_LocalDelivery_ProxyMode(t *testing.T) {
|
||||
parts := toConsumerPartitions(1)
|
||||
consumerMgr := consumer.NewGroupManager(queueName, consumerStore, time.Second, parts)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
mockQueue := &MockQueue{
|
||||
config: config,
|
||||
consumerGroups: consumerMgr,
|
||||
@@ -322,12 +322,12 @@ func TestPartitionWorker_LocalDelivery_ProxyMode(t *testing.T) {
|
||||
consumerMgr.Rebalance("group-1", parts)
|
||||
|
||||
// Create and store a message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: queueName,
|
||||
Payload: []byte("test payload"),
|
||||
Properties: map[string]string{"key": "value"},
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
}
|
||||
@@ -362,7 +362,7 @@ func TestPartitionWorker_DirectMode_LocalDelivery(t *testing.T) {
|
||||
parts := toConsumerPartitions(1)
|
||||
consumerMgr := consumer.NewGroupManager(queueName, consumerStore, time.Second, parts)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
mockQueue := &MockQueue{
|
||||
config: config,
|
||||
consumerGroups: consumerMgr,
|
||||
@@ -392,12 +392,12 @@ func TestPartitionWorker_DirectMode_LocalDelivery(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
consumerMgr.Rebalance("group-1", parts)
|
||||
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: queueName,
|
||||
Payload: []byte("test payload"),
|
||||
Properties: map[string]string{"key": "value"},
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
}
|
||||
|
||||
+36
-36
@@ -11,14 +11,14 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/delivery"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
brokerStorage "github.com/absmach/fluxmq/storage"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// MockBrokerWithError extends MockBroker to simulate delivery errors
|
||||
// MockBrokerWithError extends MockBroker to simulate delivery errors.
|
||||
type MockBrokerWithError struct {
|
||||
deliveries map[string][]interface{}
|
||||
deliveryError error
|
||||
@@ -63,7 +63,7 @@ func (m *MockBrokerWithError) ClearDeliveries() {
|
||||
|
||||
func TestNewDeliveryWorker(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
queue := NewQueue(config, store, store)
|
||||
broker := NewMockBrokerWithError()
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestNewDeliveryWorker(t *testing.T) {
|
||||
|
||||
func TestDeliveryWorker_StartStop(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
queue := NewQueue(config, store, store)
|
||||
broker := NewMockBrokerWithError()
|
||||
|
||||
@@ -108,7 +108,7 @@ func TestDeliveryWorker_StartStop(t *testing.T) {
|
||||
|
||||
func TestDeliveryWorker_StartStopViaContext(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
queue := NewQueue(config, store, store)
|
||||
broker := NewMockBrokerWithError()
|
||||
|
||||
@@ -141,7 +141,7 @@ func TestDeliveryWorker_StartStopViaContext(t *testing.T) {
|
||||
func TestDeliveryWorker_DeliverMessages(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 2
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -156,14 +156,14 @@ func TestDeliveryWorker_DeliverMessages(t *testing.T) {
|
||||
// Enqueue message with partition key
|
||||
props := map[string]string{"partition-key": "test-key"}
|
||||
partitionID := queue.GetPartitionForMessage("test-key")
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload"),
|
||||
Properties: props,
|
||||
PartitionID: partitionID,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -180,7 +180,7 @@ func TestDeliveryWorker_DeliverMessages(t *testing.T) {
|
||||
// Verify message state changed to delivered
|
||||
retrieved, err := store.GetMessage(ctx, "$queue/test", "msg-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateDelivered, retrieved.State)
|
||||
assert.Equal(t, types.StateDelivered, retrieved.State)
|
||||
|
||||
// Verify marked as inflight
|
||||
inflight, err := store.GetInflight(ctx, "$queue/test")
|
||||
@@ -191,7 +191,7 @@ func TestDeliveryWorker_DeliverMessages(t *testing.T) {
|
||||
func TestDeliveryWorker_DeliverMessages_MultiplePartitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 3
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -205,13 +205,13 @@ func TestDeliveryWorker_DeliverMessages_MultiplePartitions(t *testing.T) {
|
||||
|
||||
// Enqueue messages to different partitions
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-" + string(rune('0'+i)),
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload " + string(rune('0'+i))),
|
||||
PartitionID: i,
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -230,7 +230,7 @@ func TestDeliveryWorker_DeliverMessages_MultiplePartitions(t *testing.T) {
|
||||
func TestDeliveryWorker_DeliverMessages_MultipleGroups(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 2
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -249,14 +249,14 @@ func TestDeliveryWorker_DeliverMessages_MultipleGroups(t *testing.T) {
|
||||
partitionID := queue.GetPartitionForMessage("test-key")
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-" + string(rune('1'+i)),
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload " + string(rune('1'+i))),
|
||||
Properties: props,
|
||||
PartitionID: partitionID,
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -279,7 +279,7 @@ func TestDeliveryWorker_DeliverMessages_MultipleGroups(t *testing.T) {
|
||||
func TestDeliveryWorker_DeliverMessages_UnassignedPartition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 2
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -288,13 +288,13 @@ func TestDeliveryWorker_DeliverMessages_UnassignedPartition(t *testing.T) {
|
||||
broker := NewMockBrokerWithError()
|
||||
|
||||
// Enqueue message but don't assign any consumer
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -311,13 +311,13 @@ func TestDeliveryWorker_DeliverMessages_UnassignedPartition(t *testing.T) {
|
||||
// Message should still be queued
|
||||
retrieved, err := store.GetMessage(ctx, "$queue/test", "msg-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateQueued, retrieved.State)
|
||||
assert.Equal(t, types.StateQueued, retrieved.State)
|
||||
}
|
||||
|
||||
func TestDeliveryWorker_DeliverNext_BrokerDeliveryError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 1
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -330,13 +330,13 @@ func TestDeliveryWorker_DeliverNext_BrokerDeliveryError(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -357,13 +357,13 @@ func TestDeliveryWorker_DeliverNext_BrokerDeliveryError(t *testing.T) {
|
||||
// Message state should be delivered (state updated before broker delivery)
|
||||
retrieved, err := store.GetMessage(ctx, "$queue/test", "msg-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateDelivered, retrieved.State)
|
||||
assert.Equal(t, types.StateDelivered, retrieved.State)
|
||||
}
|
||||
|
||||
func TestDeliveryWorker_DeliverNext_NoMessages(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 1
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -387,9 +387,9 @@ func TestDeliveryWorker_DeliverNext_NoMessages(t *testing.T) {
|
||||
func TestDeliveryWorker_OrderingEnforcement(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 1
|
||||
config.Ordering = queueStorage.OrderingStrict
|
||||
config.Ordering = types.OrderingStrict
|
||||
config.BatchSize = 1 // Use small batch size to test single-message delivery
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -403,13 +403,13 @@ func TestDeliveryWorker_OrderingEnforcement(t *testing.T) {
|
||||
|
||||
// Enqueue messages in order
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-" + string(rune('0'+i)),
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload " + string(rune('0'+i))),
|
||||
PartitionID: 0,
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -439,7 +439,7 @@ func TestDeliveryWorker_OrderingEnforcement(t *testing.T) {
|
||||
func TestDeliveryWorker_IntegrationWithRetry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 1
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -452,13 +452,13 @@ func TestDeliveryWorker_IntegrationWithRetry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
RetryCount: 2,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -481,7 +481,7 @@ func TestDeliveryWorker_IntegrationWithRetry(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestToStorageMessage(t *testing.T) {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload"),
|
||||
@@ -489,7 +489,7 @@ func TestToStorageMessage(t *testing.T) {
|
||||
"key1": "value1",
|
||||
"key2": "value2",
|
||||
},
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -510,7 +510,7 @@ func TestToStorageMessage(t *testing.T) {
|
||||
func TestDeliveryWorker_ConcurrentDelivery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 4
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -527,13 +527,13 @@ func TestDeliveryWorker_ConcurrentDelivery(t *testing.T) {
|
||||
// Enqueue messages to different partitions
|
||||
for i := 0; i < 10; i++ {
|
||||
partitionID := i % 4
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-" + string(rune('0'+i)),
|
||||
Topic: "$queue/test",
|
||||
Payload: []byte("test payload " + string(rune('0'+i))),
|
||||
PartitionID: partitionID,
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/delivery"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// setupQueueWithMessages creates a queue and enqueues N messages
|
||||
// setupQueueWithMessages creates a queue and enqueues N messages.
|
||||
func setupQueueWithMessages(b *testing.B, queueName string, partitions, messageCount int) (*Manager, *memory.Store, *MockBroker) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -31,7 +31,7 @@ func setupQueueWithMessages(b *testing.B, queueName string, partitions, messageC
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.Partitions = partitions
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
@@ -51,7 +51,7 @@ func setupQueueWithMessages(b *testing.B, queueName string, partitions, messageC
|
||||
return mgr, store, broker
|
||||
}
|
||||
|
||||
// BenchmarkDequeue_SingleConsumer measures dequeue with one consumer
|
||||
// BenchmarkDequeue_SingleConsumer measures dequeue with one consumer.
|
||||
func BenchmarkDequeue_SingleConsumer(b *testing.B) {
|
||||
queueName := "$queue/bench-dequeue-single"
|
||||
mgr, store, broker := setupQueueWithMessages(b, queueName, 1, b.N)
|
||||
@@ -80,7 +80,7 @@ func BenchmarkDequeue_SingleConsumer(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkDequeue_MultipleConsumers measures dequeue with multiple consumers
|
||||
// BenchmarkDequeue_MultipleConsumers measures dequeue with multiple consumers.
|
||||
func BenchmarkDequeue_MultipleConsumers(b *testing.B) {
|
||||
consumerCounts := []int{1, 2, 5, 10}
|
||||
|
||||
@@ -117,7 +117,7 @@ func BenchmarkDequeue_MultipleConsumers(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkDequeue_WithAck measures dequeue + ack cycle
|
||||
// BenchmarkDequeue_WithAck measures dequeue + ack cycle.
|
||||
func BenchmarkDequeue_WithAck(b *testing.B) {
|
||||
queueName := "$queue/bench-dequeue-ack"
|
||||
mgr, store, broker := setupQueueWithMessages(b, queueName, 1, b.N)
|
||||
@@ -159,7 +159,7 @@ func BenchmarkDequeue_WithAck(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkDequeue_PartitionScanning measures partition scan performance
|
||||
// BenchmarkDequeue_PartitionScanning measures partition scan performance.
|
||||
func BenchmarkDequeue_PartitionScanning(b *testing.B) {
|
||||
partitionCounts := []int{1, 5, 10, 20}
|
||||
|
||||
@@ -191,7 +191,7 @@ func BenchmarkDequeue_PartitionScanning(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkDequeue_Throughput measures sustained dequeue throughput
|
||||
// BenchmarkDequeue_Throughput measures sustained dequeue throughput.
|
||||
func BenchmarkDequeue_Throughput(b *testing.B) {
|
||||
queueName := "$queue/bench-throughput"
|
||||
messageCount := 100000
|
||||
|
||||
+12
-12
@@ -12,11 +12,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/delivery"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// BenchmarkE2E_PublishToAck measures full message lifecycle
|
||||
// BenchmarkE2E_PublishToAck measures full message lifecycle.
|
||||
func BenchmarkE2E_PublishToAck(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -34,7 +34,7 @@ func BenchmarkE2E_PublishToAck(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-e2e"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
b.Fatal(err)
|
||||
@@ -81,7 +81,7 @@ func BenchmarkE2E_PublishToAck(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkE2E_ConcurrentProducerConsumer measures concurrent pub/sub
|
||||
// BenchmarkE2E_ConcurrentProducerConsumer measures concurrent pub/sub.
|
||||
func BenchmarkE2E_ConcurrentProducerConsumer(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -99,7 +99,7 @@ func BenchmarkE2E_ConcurrentProducerConsumer(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-concurrent"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.Partitions = 10
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
@@ -171,7 +171,7 @@ func BenchmarkE2E_ConcurrentProducerConsumer(b *testing.B) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// BenchmarkE2E_Latency measures message latency distribution
|
||||
// BenchmarkE2E_Latency measures message latency distribution.
|
||||
func BenchmarkE2E_Latency(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -189,7 +189,7 @@ func BenchmarkE2E_Latency(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-latency"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
b.Fatal(err)
|
||||
@@ -253,7 +253,7 @@ func BenchmarkE2E_Latency(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkE2E_Sustained measures sustained throughput over time
|
||||
// BenchmarkE2E_Sustained measures sustained throughput over time.
|
||||
func BenchmarkE2E_Sustained(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -271,7 +271,7 @@ func BenchmarkE2E_Sustained(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-sustained"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.Partitions = 10
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
@@ -351,7 +351,7 @@ func BenchmarkE2E_Sustained(b *testing.B) {
|
||||
b.ReportMetric(throughput, "msgs/sec")
|
||||
}
|
||||
|
||||
// BenchmarkAllocs_Enqueue measures allocations during enqueue
|
||||
// BenchmarkAllocs_Enqueue measures allocations during enqueue.
|
||||
func BenchmarkAllocs_Enqueue(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -369,7 +369,7 @@ func BenchmarkAllocs_Enqueue(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-allocs-enq"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
b.Fatal(err)
|
||||
@@ -388,7 +388,7 @@ func BenchmarkAllocs_Enqueue(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to calculate percentiles
|
||||
// Helper function to calculate percentiles.
|
||||
func calculatePercentile(latencies []time.Duration, percentile float64) time.Duration {
|
||||
if len(latencies) == 0 {
|
||||
return 0
|
||||
|
||||
+15
-15
@@ -8,11 +8,11 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// BenchmarkEnqueue_SinglePartition measures enqueue performance with one partition
|
||||
// BenchmarkEnqueue_SinglePartition measures enqueue performance with one partition.
|
||||
func BenchmarkEnqueue_SinglePartition(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -30,7 +30,7 @@ func BenchmarkEnqueue_SinglePartition(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-single"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
config.Partitions = 1
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
@@ -52,7 +52,7 @@ func BenchmarkEnqueue_SinglePartition(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEnqueue_MultiplePartitions measures enqueue with multiple partitions
|
||||
// BenchmarkEnqueue_MultiplePartitions measures enqueue with multiple partitions.
|
||||
func BenchmarkEnqueue_MultiplePartitions(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -70,7 +70,7 @@ func BenchmarkEnqueue_MultiplePartitions(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-multi"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
config.Partitions = 10
|
||||
|
||||
@@ -91,7 +91,7 @@ func BenchmarkEnqueue_MultiplePartitions(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEnqueue_SmallPayload measures enqueue with small messages
|
||||
// BenchmarkEnqueue_SmallPayload measures enqueue with small messages.
|
||||
func BenchmarkEnqueue_SmallPayload(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -109,7 +109,7 @@ func BenchmarkEnqueue_SmallPayload(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-small"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
@@ -129,7 +129,7 @@ func BenchmarkEnqueue_SmallPayload(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEnqueue_LargePayload measures enqueue with large messages
|
||||
// BenchmarkEnqueue_LargePayload measures enqueue with large messages.
|
||||
func BenchmarkEnqueue_LargePayload(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -147,7 +147,7 @@ func BenchmarkEnqueue_LargePayload(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-large"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
@@ -171,7 +171,7 @@ func BenchmarkEnqueue_LargePayload(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEnqueue_WithPartitionKey measures enqueue with partition key routing
|
||||
// BenchmarkEnqueue_WithPartitionKey measures enqueue with partition key routing.
|
||||
func BenchmarkEnqueue_WithPartitionKey(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -189,7 +189,7 @@ func BenchmarkEnqueue_WithPartitionKey(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-partkey"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
config.Partitions = 10
|
||||
|
||||
@@ -210,7 +210,7 @@ func BenchmarkEnqueue_WithPartitionKey(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEnqueue_Parallel measures concurrent enqueue performance
|
||||
// BenchmarkEnqueue_Parallel measures concurrent enqueue performance.
|
||||
func BenchmarkEnqueue_Parallel(b *testing.B) {
|
||||
store := memory.New()
|
||||
broker := NewMockBroker()
|
||||
@@ -228,7 +228,7 @@ func BenchmarkEnqueue_Parallel(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/bench-parallel"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
config.Partitions = 10
|
||||
|
||||
@@ -251,7 +251,7 @@ func BenchmarkEnqueue_Parallel(b *testing.B) {
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkEnqueue_BatchSize measures different batch sizes
|
||||
// BenchmarkEnqueue_BatchSize measures different batch sizes.
|
||||
func BenchmarkEnqueue_BatchSize(b *testing.B) {
|
||||
batchSizes := []int{1, 10, 100, 1000}
|
||||
|
||||
@@ -273,7 +273,7 @@ func BenchmarkEnqueue_BatchSize(b *testing.B) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := fmt.Sprintf("$queue/bench-batch-%d", size)
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 10000000 // Large limit for benchmarks
|
||||
|
||||
if err := mgr.CreateQueue(ctx, config); err != nil {
|
||||
|
||||
@@ -12,8 +12,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
badgerstore "github.com/absmach/fluxmq/queue/storage/badger"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -510,19 +510,19 @@ func TestFailover_GracefulShutdown(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create replicated queue on all nodes
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: 3,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: 2,
|
||||
AckTimeout: 5 * time.Second,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
|
||||
+11
-11
@@ -10,8 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/delivery"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -35,7 +35,7 @@ func TestIntegration_CompleteMessageLifecycle(t *testing.T) {
|
||||
|
||||
// Create queue
|
||||
queueName := "$queue/lifecycle-test"
|
||||
err = mgr.CreateQueue(ctx, queueStorage.DefaultQueueConfig(queueName))
|
||||
err = mgr.CreateQueue(ctx, types.DefaultQueueConfig(queueName))
|
||||
require.NoError(t, err)
|
||||
|
||||
// Subscribe consumer
|
||||
@@ -94,7 +94,7 @@ func TestIntegration_RetryFlow(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create queue with retry policy
|
||||
config := queueStorage.DefaultQueueConfig("$queue/retry-test")
|
||||
config := types.DefaultQueueConfig("$queue/retry-test")
|
||||
config.RetryPolicy.MaxRetries = 3
|
||||
config.RetryPolicy.InitialBackoff = 100 * time.Millisecond
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
@@ -137,7 +137,7 @@ func TestIntegration_RetryFlow(t *testing.T) {
|
||||
// Verify message state changed to retry
|
||||
msg, err := store.GetMessage(ctx, "$queue/retry-test", msgID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateRetry, msg.State)
|
||||
assert.Equal(t, types.StateRetry, msg.State)
|
||||
assert.Equal(t, 1, msg.RetryCount)
|
||||
assert.False(t, msg.NextRetryAt.IsZero())
|
||||
|
||||
@@ -162,7 +162,7 @@ func TestIntegration_DLQFlow(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create queue with DLQ enabled
|
||||
config := queueStorage.DefaultQueueConfig("$queue/dlq-test")
|
||||
config := types.DefaultQueueConfig("$queue/dlq-test")
|
||||
config.DLQConfig.Enabled = true
|
||||
config.DLQConfig.Topic = "$queue/dlq-test-dlq"
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
@@ -221,7 +221,7 @@ func TestIntegration_MultipleConsumersWithRebalancing(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create queue with multiple partitions
|
||||
config := queueStorage.DefaultQueueConfig("$queue/rebalance-test")
|
||||
config := types.DefaultQueueConfig("$queue/rebalance-test")
|
||||
config.Partitions = 6
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -287,9 +287,9 @@ func TestIntegration_OrderingGuarantees(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create queue with strict ordering
|
||||
config := queueStorage.DefaultQueueConfig("$queue/ordering-test")
|
||||
config := types.DefaultQueueConfig("$queue/ordering-test")
|
||||
config.Partitions = 1
|
||||
config.Ordering = queueStorage.OrderingStrict
|
||||
config.Ordering = types.OrderingStrict
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -350,7 +350,7 @@ func TestIntegration_ConcurrentOperations(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/concurrent-test")
|
||||
config := types.DefaultQueueConfig("$queue/concurrent-test")
|
||||
config.Partitions = 4
|
||||
config.BatchSize = 1 // Use small batch size to avoid race conditions in concurrent test
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
@@ -436,7 +436,7 @@ func TestIntegration_StatsCollection(t *testing.T) {
|
||||
|
||||
// Create queue
|
||||
queueName := "$queue/stats-test"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.Partitions = 5
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -480,7 +480,7 @@ func TestIntegration_MultipleGroups_IndependentConsumption(t *testing.T) {
|
||||
|
||||
// Create queue
|
||||
queueName := "$queue/multi-group-test"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.Partitions = 2
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -12,17 +12,18 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// DLQManager manages dead letter queue operations.
|
||||
type DLQManager struct {
|
||||
messageStore queueStorage.MessageStore
|
||||
messageStore storage.MessageStore
|
||||
alertHandler AlertHandler
|
||||
}
|
||||
|
||||
// NewDLQManager creates a new DLQ manager.
|
||||
func NewDLQManager(messageStore queueStorage.MessageStore, alertHandler AlertHandler) *DLQManager {
|
||||
func NewDLQManager(messageStore storage.MessageStore, alertHandler AlertHandler) *DLQManager {
|
||||
return &DLQManager{
|
||||
messageStore: messageStore,
|
||||
alertHandler: alertHandler,
|
||||
@@ -30,7 +31,7 @@ func NewDLQManager(messageStore queueStorage.MessageStore, alertHandler AlertHan
|
||||
}
|
||||
|
||||
// MoveToDLQ moves a failed message to the dead letter queue.
|
||||
func (d *DLQManager) MoveToDLQ(ctx context.Context, queue *Queue, msg *queueStorage.Message, reason string) error {
|
||||
func (d *DLQManager) MoveToDLQ(ctx context.Context, queue QueueInfo, msg *types.Message, reason string) error {
|
||||
config := queue.Config()
|
||||
|
||||
// Only move to DLQ if enabled
|
||||
@@ -40,7 +41,7 @@ func (d *DLQManager) MoveToDLQ(ctx context.Context, queue *Queue, msg *queueStor
|
||||
}
|
||||
|
||||
// Update message metadata
|
||||
msg.State = queueStorage.StateDLQ
|
||||
msg.State = types.StateDLQ
|
||||
msg.FailureReason = reason
|
||||
msg.MovedToDLQAt = time.Now()
|
||||
|
||||
@@ -96,7 +97,7 @@ func (d *DLQManager) RetryFromDLQ(ctx context.Context, queueName, messageID stri
|
||||
return fmt.Errorf("failed to list DLQ messages: %w", err)
|
||||
}
|
||||
|
||||
var msg *queueStorage.Message
|
||||
var msg *types.Message
|
||||
for _, m := range dlqMessages {
|
||||
if m.ID == messageID {
|
||||
msg = m
|
||||
@@ -105,11 +106,11 @@ func (d *DLQManager) RetryFromDLQ(ctx context.Context, queueName, messageID stri
|
||||
}
|
||||
|
||||
if msg == nil {
|
||||
return queueStorage.ErrMessageNotFound
|
||||
return storage.ErrMessageNotFound
|
||||
}
|
||||
|
||||
// Reset message state for retry
|
||||
msg.State = queueStorage.StateQueued
|
||||
msg.State = types.StateQueued
|
||||
msg.RetryCount = 0
|
||||
msg.NextRetryAt = time.Time{}
|
||||
msg.FailureReason = ""
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -12,35 +12,57 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mockQueueInfo is a mock implementation of QueueInfo for testing.
|
||||
type mockQueueInfo struct {
|
||||
name string
|
||||
config types.QueueConfig
|
||||
}
|
||||
|
||||
func (m *mockQueueInfo) Name() string {
|
||||
return m.name
|
||||
}
|
||||
|
||||
func (m *mockQueueInfo) Config() types.QueueConfig {
|
||||
return m.config
|
||||
}
|
||||
|
||||
func newMockQueueInfo(name string, config types.QueueConfig) *mockQueueInfo {
|
||||
return &mockQueueInfo{
|
||||
name: name,
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
func TestDLQManager_MoveToDLQ(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
// Create queue config
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.DLQConfig.Enabled = true
|
||||
config.DLQConfig.Topic = "$queue/dlq/test"
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
|
||||
// Create message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateDelivered,
|
||||
State: types.StateDelivered,
|
||||
RetryCount: 5,
|
||||
CreatedAt: time.Now().Add(-1 * time.Hour),
|
||||
}
|
||||
@@ -56,7 +78,7 @@ func TestDLQManager_MoveToDLQ(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, dlqMessages, 1)
|
||||
assert.Equal(t, "msg-1", dlqMessages[0].ID)
|
||||
assert.Equal(t, queueStorage.StateDLQ, dlqMessages[0].State)
|
||||
assert.Equal(t, types.StateDLQ, dlqMessages[0].State)
|
||||
assert.Equal(t, "max retries exceeded", dlqMessages[0].FailureReason)
|
||||
assert.False(t, dlqMessages[0].MovedToDLQAt.IsZero())
|
||||
|
||||
@@ -71,22 +93,22 @@ func TestDLQManager_MoveToDLQ_DLQDisabled(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
|
||||
// Create queue with DLQ disabled
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.DLQConfig.Enabled = false
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
|
||||
// Create message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateDelivered,
|
||||
State: types.StateDelivered,
|
||||
RetryCount: 5,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -113,23 +135,21 @@ func TestDLQManager_RetryFromDLQ(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.DLQConfig.Enabled = true
|
||||
config.DLQConfig.Topic = "$queue/dlq/test"
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
// Create message and move to DLQ
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
RetryCount: 10,
|
||||
FailureReason: "max retries exceeded",
|
||||
CreatedAt: time.Now().Add(-2 * time.Hour),
|
||||
@@ -139,13 +159,13 @@ func TestDLQManager_RetryFromDLQ(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Retry from DLQ
|
||||
err = dlqManager.RetryFromDLQ(ctx, queue.Name(), msg.ID)
|
||||
err = dlqManager.RetryFromDLQ(ctx, "$queue/test", msg.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify message back in original queue with reset state
|
||||
retrieved, err := store.GetMessage(ctx, queue.Name(), msg.ID)
|
||||
retrieved, err := store.GetMessage(ctx, "$queue/test", msg.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateQueued, retrieved.State)
|
||||
assert.Equal(t, types.StateQueued, retrieved.State)
|
||||
assert.Equal(t, 0, retrieved.RetryCount)
|
||||
assert.True(t, retrieved.NextRetryAt.IsZero())
|
||||
assert.Equal(t, "", retrieved.FailureReason)
|
||||
@@ -162,13 +182,13 @@ func TestDLQManager_RetryFromDLQ_NotFound(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to retry non-existent message
|
||||
err = dlqManager.RetryFromDLQ(ctx, "$queue/test", "non-existent")
|
||||
assert.ErrorIs(t, err, queueStorage.ErrMessageNotFound)
|
||||
assert.ErrorIs(t, err, storage.ErrMessageNotFound)
|
||||
}
|
||||
|
||||
func TestDLQManager_PurgeDLQ(t *testing.T) {
|
||||
@@ -177,18 +197,18 @@ func TestDLQManager_PurgeDLQ(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.DLQConfig.Topic = "$queue/dlq/test"
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add multiple messages to DLQ
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
State: queueStorage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
MovedToDLQAt: time.Now(),
|
||||
}
|
||||
err = store.EnqueueDLQ(ctx, config.DLQConfig.Topic, msg)
|
||||
@@ -217,7 +237,7 @@ func TestDLQManager_GetDLQStats(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.DLQConfig.Topic = "$queue/dlq/test"
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -225,11 +245,11 @@ func TestDLQManager_GetDLQStats(t *testing.T) {
|
||||
// Add messages with different failure reasons
|
||||
reasons := []string{"max retries", "max retries", "timeout", "timeout", "timeout"}
|
||||
for i, reason := range reasons {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
State: queueStorage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: reason,
|
||||
MovedToDLQAt: time.Now(),
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// QueueInfo provides queue metadata needed by lifecycle managers.
|
||||
type QueueInfo interface {
|
||||
Name() string
|
||||
Config() types.QueueConfig
|
||||
}
|
||||
|
||||
// RaftCoordinator provides Raft operations needed by retention management.
|
||||
type RaftCoordinator interface {
|
||||
IsLeader(partitionID int) bool
|
||||
ApplyRetentionDelete(ctx context.Context, partitionID int, messageIDs []string) error
|
||||
}
|
||||
@@ -1,25 +1,25 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// OrderingEnforcer enforces message ordering within partitions.
|
||||
// Group-aware: each consumer group tracks its own ordering independently.
|
||||
type OrderingEnforcer struct {
|
||||
mode queueStorage.OrderingMode
|
||||
mode types.OrderingMode
|
||||
lastDelivered map[string]map[int]uint64 // groupID -> partitionID -> last delivered sequence
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewOrderingEnforcer creates a new ordering enforcer.
|
||||
func NewOrderingEnforcer(mode queueStorage.OrderingMode) *OrderingEnforcer {
|
||||
func NewOrderingEnforcer(mode types.OrderingMode) *OrderingEnforcer {
|
||||
return &OrderingEnforcer{
|
||||
mode: mode,
|
||||
lastDelivered: make(map[string]map[int]uint64),
|
||||
@@ -28,17 +28,17 @@ func NewOrderingEnforcer(mode queueStorage.OrderingMode) *OrderingEnforcer {
|
||||
|
||||
// CanDeliver checks if a message can be delivered according to ordering rules.
|
||||
// groupID identifies the consumer group - each group has independent ordering tracking.
|
||||
func (oe *OrderingEnforcer) CanDeliver(msg *queueStorage.Message, groupID string) (bool, error) {
|
||||
func (oe *OrderingEnforcer) CanDeliver(msg *types.Message, groupID string) (bool, error) {
|
||||
switch oe.mode {
|
||||
case queueStorage.OrderingNone:
|
||||
case types.OrderingNone:
|
||||
// No ordering requirements
|
||||
return true, nil
|
||||
|
||||
case queueStorage.OrderingPartition:
|
||||
case types.OrderingPartition:
|
||||
// Partition-based ordering: messages within same partition must be delivered in order
|
||||
return oe.checkPartitionOrder(msg, groupID)
|
||||
|
||||
case queueStorage.OrderingStrict:
|
||||
case types.OrderingStrict:
|
||||
// Strict global ordering: all messages must be delivered in global sequence order
|
||||
// This requires single partition (partition 0)
|
||||
if msg.PartitionID != 0 {
|
||||
@@ -52,7 +52,7 @@ func (oe *OrderingEnforcer) CanDeliver(msg *queueStorage.Message, groupID string
|
||||
}
|
||||
|
||||
// checkPartitionOrder checks if message is next in sequence for its partition within a group.
|
||||
func (oe *OrderingEnforcer) checkPartitionOrder(msg *queueStorage.Message, groupID string) (bool, error) {
|
||||
func (oe *OrderingEnforcer) checkPartitionOrder(msg *types.Message, groupID string) (bool, error) {
|
||||
oe.mu.RLock()
|
||||
groupMap, groupExists := oe.lastDelivered[groupID]
|
||||
if !groupExists {
|
||||
@@ -78,8 +78,8 @@ func (oe *OrderingEnforcer) checkPartitionOrder(msg *queueStorage.Message, group
|
||||
}
|
||||
|
||||
// MarkDelivered records that a message was delivered to a consumer group.
|
||||
func (oe *OrderingEnforcer) MarkDelivered(msg *queueStorage.Message, groupID string) {
|
||||
if oe.mode == queueStorage.OrderingNone {
|
||||
func (oe *OrderingEnforcer) MarkDelivered(msg *types.Message, groupID string) {
|
||||
if oe.mode == types.OrderingNone {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ func (oe *OrderingEnforcer) Stats() OrderingStats {
|
||||
|
||||
// OrderingStats holds ordering statistics.
|
||||
type OrderingStats struct {
|
||||
Mode queueStorage.OrderingMode
|
||||
Mode types.OrderingMode
|
||||
GroupCount int
|
||||
PartitionCount int
|
||||
GroupStats map[string]map[int]uint64
|
||||
@@ -1,13 +1,13 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -15,15 +15,15 @@ import (
|
||||
const testGroupID = "test-group"
|
||||
|
||||
func TestOrderingEnforcer_OrderingNone(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingNone)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingNone)
|
||||
|
||||
// With no ordering, all messages can be delivered
|
||||
msg1 := &queueStorage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 5,
|
||||
}
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 0,
|
||||
Sequence: 3, // Out of order
|
||||
@@ -41,9 +41,9 @@ func TestOrderingEnforcer_OrderingNone(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_PartitionOrdering_InOrder(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
messages := []*queueStorage.Message{
|
||||
messages := []*types.Message{
|
||||
{ID: "msg-1", PartitionID: 0, Sequence: 1},
|
||||
{ID: "msg-2", PartitionID: 0, Sequence: 2},
|
||||
{ID: "msg-3", PartitionID: 0, Sequence: 3},
|
||||
@@ -59,10 +59,10 @@ func TestOrderingEnforcer_PartitionOrdering_InOrder(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_PartitionOrdering_OutOfOrder(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
// Deliver message with sequence 3
|
||||
msg1 := &queueStorage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 3,
|
||||
@@ -73,7 +73,7 @@ func TestOrderingEnforcer_PartitionOrdering_OutOfOrder(t *testing.T) {
|
||||
enforcer.MarkDelivered(msg1, testGroupID)
|
||||
|
||||
// Try to deliver message with sequence 2 (out of order)
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
@@ -85,10 +85,10 @@ func TestOrderingEnforcer_PartitionOrdering_OutOfOrder(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_PartitionOrdering_MultiplePartitions(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
// Partition 0
|
||||
msg1 := &queueStorage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
@@ -99,7 +99,7 @@ func TestOrderingEnforcer_PartitionOrdering_MultiplePartitions(t *testing.T) {
|
||||
enforcer.MarkDelivered(msg1, testGroupID)
|
||||
|
||||
// Partition 1 (independent ordering)
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 1,
|
||||
Sequence: 1,
|
||||
@@ -110,7 +110,7 @@ func TestOrderingEnforcer_PartitionOrdering_MultiplePartitions(t *testing.T) {
|
||||
enforcer.MarkDelivered(msg2, testGroupID)
|
||||
|
||||
// Partition 0 continues
|
||||
msg3 := &queueStorage.Message{
|
||||
msg3 := &types.Message{
|
||||
ID: "msg-3",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
@@ -121,10 +121,10 @@ func TestOrderingEnforcer_PartitionOrdering_MultiplePartitions(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_PartitionOrdering_Gaps(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
// Deliver message with sequence 1
|
||||
msg1 := &queueStorage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
@@ -135,7 +135,7 @@ func TestOrderingEnforcer_PartitionOrdering_Gaps(t *testing.T) {
|
||||
enforcer.MarkDelivered(msg1, testGroupID)
|
||||
|
||||
// Deliver message with sequence 5 (gap is allowed - message 2-4 might have been acked/failed)
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 0,
|
||||
Sequence: 5,
|
||||
@@ -147,10 +147,10 @@ func TestOrderingEnforcer_PartitionOrdering_Gaps(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_StrictOrdering_SinglePartition(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingStrict)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingStrict)
|
||||
|
||||
// Message in partition 0 is allowed
|
||||
msg1 := &queueStorage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
@@ -160,7 +160,7 @@ func TestOrderingEnforcer_StrictOrdering_SinglePartition(t *testing.T) {
|
||||
assert.True(t, canDeliver)
|
||||
|
||||
// Message in partition 1 is NOT allowed for strict ordering
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 1,
|
||||
Sequence: 1,
|
||||
@@ -172,9 +172,9 @@ func TestOrderingEnforcer_StrictOrdering_SinglePartition(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_Reset(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
msg1 := &queueStorage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 5,
|
||||
@@ -194,7 +194,7 @@ func TestOrderingEnforcer_Reset(t *testing.T) {
|
||||
assert.False(t, exists)
|
||||
|
||||
// Can now deliver message with lower sequence
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
@@ -205,11 +205,11 @@ func TestOrderingEnforcer_Reset(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_ResetAll(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
// Mark several partitions as delivered
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
PartitionID: i,
|
||||
Sequence: 10,
|
||||
@@ -230,14 +230,14 @@ func TestOrderingEnforcer_ResetAll(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_Stats(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
// Deliver messages to different partitions
|
||||
partitions := []int{0, 0, 1, 1, 2}
|
||||
sequences := []uint64{1, 5, 3, 7, 2}
|
||||
|
||||
for i, partID := range partitions {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
PartitionID: partID,
|
||||
Sequence: sequences[i],
|
||||
@@ -246,7 +246,7 @@ func TestOrderingEnforcer_Stats(t *testing.T) {
|
||||
}
|
||||
|
||||
stats := enforcer.Stats()
|
||||
assert.Equal(t, queueStorage.OrderingPartition, stats.Mode)
|
||||
assert.Equal(t, types.OrderingPartition, stats.Mode)
|
||||
assert.Equal(t, 3, stats.PartitionCount)
|
||||
assert.Equal(t, uint64(5), stats.GroupStats[testGroupID][0])
|
||||
assert.Equal(t, uint64(7), stats.GroupStats[testGroupID][1])
|
||||
@@ -254,10 +254,10 @@ func TestOrderingEnforcer_Stats(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrderingEnforcer_MultipleGroups(t *testing.T) {
|
||||
enforcer := NewOrderingEnforcer(queueStorage.OrderingPartition)
|
||||
enforcer := NewOrderingEnforcer(types.OrderingPartition)
|
||||
|
||||
// Same message delivered to multiple groups should work
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
@@ -276,7 +276,7 @@ func TestOrderingEnforcer_MultipleGroups(t *testing.T) {
|
||||
enforcer.MarkDelivered(msg, "group2")
|
||||
|
||||
// Next message in group1
|
||||
msg2 := &queueStorage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
@@ -286,7 +286,7 @@ func TestOrderingEnforcer_MultipleGroups(t *testing.T) {
|
||||
assert.True(t, canDeliver)
|
||||
|
||||
// Out of order message in group1 should fail
|
||||
msgOld := &queueStorage.Message{
|
||||
msgOld := &types.Message{
|
||||
ID: "msg-0",
|
||||
PartitionID: 0,
|
||||
Sequence: 0,
|
||||
@@ -300,42 +300,3 @@ func TestOrderingEnforcer_MultipleGroups(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
assert.False(t, canDeliver)
|
||||
}
|
||||
|
||||
func TestHashPartitionStrategy(t *testing.T) {
|
||||
strategy := &HashPartitionStrategy{}
|
||||
|
||||
// Same key should always go to same partition
|
||||
partition1 := strategy.GetPartition("user-123", 10)
|
||||
partition2 := strategy.GetPartition("user-123", 10)
|
||||
assert.Equal(t, partition1, partition2)
|
||||
|
||||
// Different keys should distribute across partitions
|
||||
partitions := make(map[int]int)
|
||||
for i := 0; i < 100; i++ {
|
||||
key := fmt.Sprintf("key-%d", i)
|
||||
partition := strategy.GetPartition(key, 10)
|
||||
assert.GreaterOrEqual(t, partition, 0)
|
||||
assert.Less(t, partition, 10)
|
||||
partitions[partition]++
|
||||
}
|
||||
|
||||
// Should have reasonable distribution (at least 5 different partitions used)
|
||||
assert.GreaterOrEqual(t, len(partitions), 5)
|
||||
}
|
||||
|
||||
func TestHashPartitionStrategy_EmptyKey(t *testing.T) {
|
||||
strategy := &HashPartitionStrategy{}
|
||||
|
||||
// Empty key should return random partition
|
||||
// Call multiple times and expect variation
|
||||
partitions := make(map[int]bool)
|
||||
for i := 0; i < 50; i++ {
|
||||
partition := strategy.GetPartition("", 10)
|
||||
assert.GreaterOrEqual(t, partition, 0)
|
||||
assert.Less(t, partition, 10)
|
||||
partitions[partition] = true
|
||||
}
|
||||
|
||||
// Should have some variation (probabilistically very likely)
|
||||
assert.Greater(t, len(partitions), 1)
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,17 +11,18 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// RetentionManager handles automatic message cleanup based on retention policies.
|
||||
// It implements both time-based (background) and size-based (active) retention strategies.
|
||||
type RetentionManager struct {
|
||||
queueName string
|
||||
policy storage.RetentionPolicy
|
||||
policy types.RetentionPolicy
|
||||
store storage.MessageStore
|
||||
|
||||
// Raft integration (optional)
|
||||
raftManager *RaftManager
|
||||
raftManager RaftCoordinator
|
||||
|
||||
// Size-based retention tracking
|
||||
enqueueCounter int
|
||||
@@ -43,7 +44,7 @@ type RetentionStats struct {
|
||||
}
|
||||
|
||||
// NewRetentionManager creates a new retention manager for a queue.
|
||||
func NewRetentionManager(queueName string, policy storage.RetentionPolicy, store storage.MessageStore, raftManager *RaftManager, logger *slog.Logger) *RetentionManager {
|
||||
func NewRetentionManager(queueName string, policy types.RetentionPolicy, store storage.MessageStore, raftManager RaftCoordinator, logger *slog.Logger) *RetentionManager {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
@@ -301,7 +302,7 @@ func (rm *RetentionManager) compactionLoop(ctx context.Context, partitionID int)
|
||||
continue
|
||||
}
|
||||
|
||||
stats, err := rm.runCompaction(ctx, partitionID)
|
||||
stats, err := rm.RunCompaction(ctx, partitionID)
|
||||
if err != nil {
|
||||
rm.logger.Error("compaction failed",
|
||||
slog.String("queue", rm.queueName),
|
||||
@@ -322,8 +323,9 @@ func (rm *RetentionManager) compactionLoop(ctx context.Context, partitionID int)
|
||||
}
|
||||
}
|
||||
|
||||
// runCompaction performs log compaction by keeping only the latest message per compaction key.
|
||||
func (rm *RetentionManager) runCompaction(ctx context.Context, partitionID int) (*RetentionStats, error) {
|
||||
// RunCompaction performs log compaction by keeping only the latest message per compaction key.
|
||||
// This method is exported for testing purposes.
|
||||
func (rm *RetentionManager) RunCompaction(ctx context.Context, partitionID int) (*RetentionStats, error) {
|
||||
startTime := time.Now()
|
||||
stats := &RetentionStats{
|
||||
LastRunTime: startTime,
|
||||
@@ -351,7 +353,7 @@ func (rm *RetentionManager) runCompaction(ctx context.Context, partitionID int)
|
||||
}
|
||||
|
||||
// Group messages by compaction key
|
||||
keyGroups := make(map[string][]*storage.Message)
|
||||
keyGroups := make(map[string][]*types.Message)
|
||||
cutoffTime := time.Now().Add(-rm.policy.CompactionLag)
|
||||
|
||||
for _, msg := range messages {
|
||||
@@ -424,7 +426,7 @@ func (rm *RetentionManager) runCompaction(ctx context.Context, partitionID int)
|
||||
}
|
||||
|
||||
// extractCompactionKey extracts the compaction key from a message's properties.
|
||||
func (rm *RetentionManager) extractCompactionKey(msg *storage.Message) string {
|
||||
func (rm *RetentionManager) extractCompactionKey(msg *types.Message) string {
|
||||
if msg.Properties == nil {
|
||||
return ""
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -22,12 +22,12 @@ func TestRetentionManager_SizeBasedRetention_Messages(t *testing.T) {
|
||||
queueName := "$queue/test-retention"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure retention: max 5 messages
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
RetentionMessages: 5,
|
||||
SizeCheckEvery: 1, // Check on every enqueue for testing
|
||||
}
|
||||
@@ -36,12 +36,12 @@ func TestRetentionManager_SizeBasedRetention_Messages(t *testing.T) {
|
||||
|
||||
// Enqueue 10 messages
|
||||
for i := 0; i < 10; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("test message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -79,12 +79,12 @@ func TestRetentionManager_SizeBasedRetention_Bytes(t *testing.T) {
|
||||
queueName := "$queue/test-retention-bytes"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure retention: max 100 bytes
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
RetentionBytes: 100,
|
||||
SizeCheckEvery: 1, // Check on every enqueue for testing
|
||||
}
|
||||
@@ -93,12 +93,12 @@ func TestRetentionManager_SizeBasedRetention_Bytes(t *testing.T) {
|
||||
|
||||
// Enqueue messages with 30-byte payloads (total: 150 bytes)
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: make([]byte, 30), // 30 bytes each
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -134,12 +134,12 @@ func TestRetentionManager_TimeBasedRetention(t *testing.T) {
|
||||
queueName := "$queue/test-time-retention"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure retention: 100ms time retention
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
RetentionTime: 100 * time.Millisecond,
|
||||
TimeCheckInterval: 50 * time.Millisecond,
|
||||
}
|
||||
@@ -149,12 +149,12 @@ func TestRetentionManager_TimeBasedRetention(t *testing.T) {
|
||||
// Enqueue old messages (created 200ms ago)
|
||||
oldTime := time.Now().Add(-200 * time.Millisecond)
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("old-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("old message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: oldTime,
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -163,12 +163,12 @@ func TestRetentionManager_TimeBasedRetention(t *testing.T) {
|
||||
|
||||
// Enqueue new messages (just created)
|
||||
for i := 3; i < 6; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("new-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("new message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -204,22 +204,22 @@ func TestRetentionManager_NoRetentionConfigured(t *testing.T) {
|
||||
queueName := "$queue/test-no-retention"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// No retention policy configured
|
||||
policy := storage.RetentionPolicy{}
|
||||
policy := types.RetentionPolicy{}
|
||||
rm := NewRetentionManager(queueName, policy, store, nil, slog.Default())
|
||||
|
||||
// Enqueue messages
|
||||
for i := 0; i < 10; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("test message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -244,12 +244,12 @@ func TestRetentionManager_SizeCheckOptimization(t *testing.T) {
|
||||
queueName := "$queue/test-check-optimization"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure retention with specific check interval
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
RetentionMessages: 5,
|
||||
SizeCheckEvery: 5, // Only check every 5 enqueues
|
||||
}
|
||||
@@ -258,12 +258,12 @@ func TestRetentionManager_SizeCheckOptimization(t *testing.T) {
|
||||
|
||||
// Enqueue 10 messages
|
||||
for i := 0; i < 10; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("test"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -298,12 +298,12 @@ func TestRetentionManager_BothSizeLimits(t *testing.T) {
|
||||
queueName := "$queue/test-both-limits"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure both byte and message limits
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
RetentionBytes: 100, // 100 bytes
|
||||
RetentionMessages: 10, // 10 messages
|
||||
SizeCheckEvery: 1,
|
||||
@@ -313,12 +313,12 @@ func TestRetentionManager_BothSizeLimits(t *testing.T) {
|
||||
|
||||
// Enqueue 5 messages with 30 bytes each (150 bytes total, but only 5 messages)
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: make([]byte, 30),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err := store.Enqueue(ctx, queueName, msg)
|
||||
@@ -349,12 +349,12 @@ func TestRetentionManager_Compaction_Basic(t *testing.T) {
|
||||
queueName := "$queue/test-compaction"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure compaction with a key
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
CompactionEnabled: true,
|
||||
CompactionKey: "entity_id", // Compact by entity_id property
|
||||
CompactionLag: 0, // No lag for testing
|
||||
@@ -366,12 +366,12 @@ func TestRetentionManager_Compaction_Basic(t *testing.T) {
|
||||
// Enqueue multiple messages for the same entity_id (should compact to 1)
|
||||
// Entity A: 3 messages (only latest should survive)
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("entity-a-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte(fmt.Sprintf("entity A update %d", i)),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now().Add(-time.Hour), // Old messages
|
||||
Properties: map[string]string{
|
||||
"entity_id": "entity-A",
|
||||
@@ -383,12 +383,12 @@ func TestRetentionManager_Compaction_Basic(t *testing.T) {
|
||||
|
||||
// Entity B: 2 messages
|
||||
for i := 3; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("entity-b-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte(fmt.Sprintf("entity B update %d", i-3)),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now().Add(-time.Hour),
|
||||
Properties: map[string]string{
|
||||
"entity_id": "entity-B",
|
||||
@@ -404,7 +404,7 @@ func TestRetentionManager_Compaction_Basic(t *testing.T) {
|
||||
assert.Equal(t, int64(5), count)
|
||||
|
||||
// Run compaction
|
||||
stats, err := rm.runCompaction(ctx, 0)
|
||||
stats, err := rm.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should delete 3 messages (2 from entity-A, 1 from entity-B)
|
||||
@@ -435,12 +435,12 @@ func TestRetentionManager_Compaction_RespectLag(t *testing.T) {
|
||||
queueName := "$queue/test-compaction-lag"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure compaction with 1-hour lag
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
CompactionEnabled: true,
|
||||
CompactionKey: "entity_id",
|
||||
CompactionLag: 1 * time.Hour, // Messages within the last hour won't be compacted
|
||||
@@ -451,12 +451,12 @@ func TestRetentionManager_Compaction_RespectLag(t *testing.T) {
|
||||
|
||||
// Enqueue old messages (should be compacted)
|
||||
for i := 0; i < 2; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("old-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("old message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now().Add(-2 * time.Hour), // 2 hours ago
|
||||
Properties: map[string]string{
|
||||
"entity_id": "entity-X",
|
||||
@@ -468,12 +468,12 @@ func TestRetentionManager_Compaction_RespectLag(t *testing.T) {
|
||||
|
||||
// Enqueue recent messages (should NOT be compacted due to lag)
|
||||
for i := 2; i < 4; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("new-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("new message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now().Add(-10 * time.Minute), // 10 minutes ago (within lag)
|
||||
Properties: map[string]string{
|
||||
"entity_id": "entity-X",
|
||||
@@ -489,7 +489,7 @@ func TestRetentionManager_Compaction_RespectLag(t *testing.T) {
|
||||
assert.Equal(t, int64(4), count)
|
||||
|
||||
// Run compaction
|
||||
stats, err := rm.runCompaction(ctx, 0)
|
||||
stats, err := rm.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should only delete 1 old message (keep latest old message, all new messages untouched)
|
||||
@@ -507,12 +507,12 @@ func TestRetentionManager_Compaction_NoKeyProperty(t *testing.T) {
|
||||
queueName := "$queue/test-compaction-no-key"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure compaction with a key
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
CompactionEnabled: true,
|
||||
CompactionKey: "entity_id",
|
||||
CompactionLag: 0,
|
||||
@@ -523,12 +523,12 @@ func TestRetentionManager_Compaction_NoKeyProperty(t *testing.T) {
|
||||
|
||||
// Enqueue messages WITHOUT the compaction key property
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("message without entity_id"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now().Add(-time.Hour),
|
||||
Properties: map[string]string{
|
||||
"other_key": "some_value", // No entity_id
|
||||
@@ -544,7 +544,7 @@ func TestRetentionManager_Compaction_NoKeyProperty(t *testing.T) {
|
||||
assert.Equal(t, int64(5), count)
|
||||
|
||||
// Run compaction
|
||||
stats, err := rm.runCompaction(ctx, 0)
|
||||
stats, err := rm.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// No messages should be deleted (no compaction key)
|
||||
@@ -562,12 +562,12 @@ func TestRetentionManager_Compaction_NotConfigured(t *testing.T) {
|
||||
queueName := "$queue/test-no-compaction"
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// No compaction configured (empty key)
|
||||
policy := storage.RetentionPolicy{
|
||||
policy := types.RetentionPolicy{
|
||||
CompactionEnabled: true,
|
||||
CompactionKey: "", // No key = no compaction
|
||||
CompactionInterval: 100 * time.Millisecond,
|
||||
@@ -577,12 +577,12 @@ func TestRetentionManager_Compaction_NotConfigured(t *testing.T) {
|
||||
|
||||
// Enqueue messages
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("message"),
|
||||
Sequence: uint64(i),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now().Add(-time.Hour),
|
||||
Properties: map[string]string{
|
||||
"entity_id": "entity-A",
|
||||
@@ -593,7 +593,7 @@ func TestRetentionManager_Compaction_NotConfigured(t *testing.T) {
|
||||
}
|
||||
|
||||
// Run compaction
|
||||
stats, err := rm.runCompaction(ctx, 0)
|
||||
stats, err := rm.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// No messages should be deleted (no compaction key configured)
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,13 +10,14 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// RetryManager monitors inflight messages and handles retry logic.
|
||||
type RetryManager struct {
|
||||
queues map[string]*Queue
|
||||
messageStore queueStorage.MessageStore
|
||||
queues map[string]QueueInfo
|
||||
messageStore storage.MessageStore
|
||||
dlqManager *DLQManager
|
||||
checkInterval time.Duration
|
||||
stopCh chan struct{}
|
||||
@@ -25,9 +26,9 @@ type RetryManager struct {
|
||||
}
|
||||
|
||||
// NewRetryManager creates a new retry manager.
|
||||
func NewRetryManager(messageStore queueStorage.MessageStore, dlqManager *DLQManager) *RetryManager {
|
||||
func NewRetryManager(messageStore storage.MessageStore, dlqManager *DLQManager) *RetryManager {
|
||||
return &RetryManager{
|
||||
queues: make(map[string]*Queue),
|
||||
queues: make(map[string]QueueInfo),
|
||||
messageStore: messageStore,
|
||||
dlqManager: dlqManager,
|
||||
checkInterval: 1 * time.Second, // Check every second
|
||||
@@ -36,7 +37,7 @@ func NewRetryManager(messageStore queueStorage.MessageStore, dlqManager *DLQMana
|
||||
}
|
||||
|
||||
// RegisterQueue adds a queue to be monitored for retry timeouts.
|
||||
func (rm *RetryManager) RegisterQueue(queue *Queue) {
|
||||
func (rm *RetryManager) RegisterQueue(queue QueueInfo) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
rm.queues[queue.Name()] = queue
|
||||
@@ -85,7 +86,7 @@ func (rm *RetryManager) run(ctx context.Context) {
|
||||
// checkInflightTimeouts checks all inflight messages for delivery timeout.
|
||||
func (rm *RetryManager) checkInflightTimeouts(ctx context.Context) {
|
||||
rm.mu.RLock()
|
||||
queues := make([]*Queue, 0, len(rm.queues))
|
||||
queues := make([]QueueInfo, 0, len(rm.queues))
|
||||
for _, queue := range rm.queues {
|
||||
queues = append(queues, queue)
|
||||
}
|
||||
@@ -108,7 +109,7 @@ func (rm *RetryManager) checkInflightTimeouts(ctx context.Context) {
|
||||
// checkRetrySchedule checks for messages scheduled for retry.
|
||||
func (rm *RetryManager) checkRetrySchedule(ctx context.Context) {
|
||||
rm.mu.RLock()
|
||||
queues := make([]*Queue, 0, len(rm.queues))
|
||||
queues := make([]QueueInfo, 0, len(rm.queues))
|
||||
for _, queue := range rm.queues {
|
||||
queues = append(queues, queue)
|
||||
}
|
||||
@@ -134,7 +135,7 @@ func (rm *RetryManager) checkRetrySchedule(ctx context.Context) {
|
||||
}
|
||||
|
||||
// handleInflightTimeout handles a message that has timed out while inflight.
|
||||
func (rm *RetryManager) handleInflightTimeout(ctx context.Context, queue *Queue, deliveryState *queueStorage.DeliveryState) {
|
||||
func (rm *RetryManager) handleInflightTimeout(ctx context.Context, queue QueueInfo, deliveryState *types.DeliveryState) {
|
||||
// Get the actual message
|
||||
msg, err := rm.messageStore.GetMessage(ctx, queue.Name(), deliveryState.MessageID)
|
||||
if err != nil {
|
||||
@@ -151,7 +152,7 @@ func (rm *RetryManager) handleInflightTimeout(ctx context.Context, queue *Queue,
|
||||
}
|
||||
|
||||
// processRetry determines if a message should be retried or moved to DLQ.
|
||||
func (rm *RetryManager) processRetry(ctx context.Context, queue *Queue, msg *queueStorage.Message) {
|
||||
func (rm *RetryManager) processRetry(ctx context.Context, queue QueueInfo, msg *types.Message) {
|
||||
config := queue.Config()
|
||||
|
||||
// Check if max retries exceeded
|
||||
@@ -169,7 +170,7 @@ func (rm *RetryManager) processRetry(ctx context.Context, queue *Queue, msg *que
|
||||
|
||||
// Schedule retry
|
||||
msg.RetryCount++
|
||||
msg.State = queueStorage.StateRetry
|
||||
msg.State = types.StateRetry
|
||||
msg.NextRetryAt = time.Now().Add(rm.calculateBackoff(msg.RetryCount, config.RetryPolicy))
|
||||
|
||||
if err := rm.messageStore.UpdateMessage(ctx, queue.Name(), msg); err != nil {
|
||||
@@ -179,7 +180,7 @@ func (rm *RetryManager) processRetry(ctx context.Context, queue *Queue, msg *que
|
||||
}
|
||||
|
||||
// moveToDLQ moves a message to the dead letter queue.
|
||||
func (rm *RetryManager) moveToDLQ(ctx context.Context, queue *Queue, msg *queueStorage.Message, reason string) {
|
||||
func (rm *RetryManager) moveToDLQ(ctx context.Context, queue QueueInfo, msg *types.Message, reason string) {
|
||||
if rm.dlqManager != nil {
|
||||
if err := rm.dlqManager.MoveToDLQ(ctx, queue, msg, reason); err != nil {
|
||||
// Failed to move to DLQ, log but continue
|
||||
@@ -192,7 +193,7 @@ func (rm *RetryManager) moveToDLQ(ctx context.Context, queue *Queue, msg *queueS
|
||||
}
|
||||
|
||||
// calculateBackoff calculates exponential backoff duration for a retry.
|
||||
func (rm *RetryManager) calculateBackoff(retryCount int, policy queueStorage.RetryPolicy) time.Duration {
|
||||
func (rm *RetryManager) calculateBackoff(retryCount int, policy types.RetryPolicy) time.Duration {
|
||||
// Calculate exponential backoff: initialBackoff * (multiplier ^ (retryCount - 1))
|
||||
backoff := float64(policy.InitialBackoff) * math.Pow(policy.BackoffMultiplier, float64(retryCount-1))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -20,7 +20,7 @@ func TestRetryManager_CalculateBackoff(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
rm := NewRetryManager(store, dlqManager)
|
||||
|
||||
policy := queueStorage.RetryPolicy{
|
||||
policy := types.RetryPolicy{
|
||||
InitialBackoff: 5 * time.Second,
|
||||
MaxBackoff: 5 * time.Minute,
|
||||
BackoffMultiplier: 2.0,
|
||||
@@ -57,31 +57,31 @@ func TestRetryManager_InflightTimeout(t *testing.T) {
|
||||
rm.checkInterval = 100 * time.Millisecond
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.RetryPolicy.InitialBackoff = 1 * time.Second
|
||||
config.DeliveryTimeout = 200 * time.Millisecond // Short timeout for testing
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
rm.RegisterQueue(queue)
|
||||
|
||||
// Enqueue a message
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, queue.Name(), msg)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as inflight with short timeout
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: msg.ID,
|
||||
QueueName: queue.Name(),
|
||||
PartitionID: 0,
|
||||
@@ -102,7 +102,7 @@ func TestRetryManager_InflightTimeout(t *testing.T) {
|
||||
// Message should be in retry state
|
||||
retrieved, err := store.GetMessage(ctx, queue.Name(), msg.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateRetry, retrieved.State)
|
||||
assert.Equal(t, types.StateRetry, retrieved.State)
|
||||
assert.Equal(t, 1, retrieved.RetryCount)
|
||||
assert.True(t, retrieved.NextRetryAt.After(time.Now()))
|
||||
|
||||
@@ -119,24 +119,24 @@ func TestRetryManager_MaxRetriesExceeded(t *testing.T) {
|
||||
rm := NewRetryManager(store, dlqManager)
|
||||
|
||||
// Create queue with low max retries
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.RetryPolicy.MaxRetries = 3
|
||||
config.DLQConfig.Enabled = true
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
rm.RegisterQueue(queue)
|
||||
|
||||
// Create message that has already been retried max times
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateDelivered,
|
||||
State: types.StateDelivered,
|
||||
RetryCount: 3, // Already at max
|
||||
CreatedAt: time.Now().Add(-1 * time.Hour),
|
||||
}
|
||||
@@ -144,7 +144,7 @@ func TestRetryManager_MaxRetriesExceeded(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: msg.ID,
|
||||
QueueName: queue.Name(),
|
||||
PartitionID: 0,
|
||||
@@ -165,7 +165,7 @@ func TestRetryManager_MaxRetriesExceeded(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, dlqMessages, 1)
|
||||
assert.Equal(t, "msg-1", dlqMessages[0].ID)
|
||||
assert.Equal(t, queueStorage.StateDLQ, dlqMessages[0].State)
|
||||
assert.Equal(t, types.StateDLQ, dlqMessages[0].State)
|
||||
assert.Equal(t, "max retries exceeded", dlqMessages[0].FailureReason)
|
||||
|
||||
// Should be removed from original queue
|
||||
@@ -180,7 +180,7 @@ func TestRetryManager_TotalTimeoutExceeded(t *testing.T) {
|
||||
rm := NewRetryManager(store, dlqManager)
|
||||
|
||||
// Create queue with short total timeout
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.RetryPolicy.MaxRetries = 10
|
||||
config.RetryPolicy.TotalTimeout = 1 * time.Hour
|
||||
config.DLQConfig.Enabled = true
|
||||
@@ -188,17 +188,17 @@ func TestRetryManager_TotalTimeoutExceeded(t *testing.T) {
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
rm.RegisterQueue(queue)
|
||||
|
||||
// Create message created 2 hours ago (exceeds total timeout)
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateDelivered,
|
||||
State: types.StateDelivered,
|
||||
RetryCount: 2,
|
||||
CreatedAt: time.Now().Add(-2 * time.Hour),
|
||||
}
|
||||
@@ -224,23 +224,23 @@ func TestRetryManager_RetryScheduling(t *testing.T) {
|
||||
rm.checkInterval = 100 * time.Millisecond
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.RetryPolicy.InitialBackoff = 200 * time.Millisecond
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
rm.RegisterQueue(queue)
|
||||
|
||||
// Create message in retry state scheduled for near future
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: queueStorage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
RetryCount: 1,
|
||||
CreatedAt: time.Now(),
|
||||
NextRetryAt: time.Now().Add(150 * time.Millisecond),
|
||||
@@ -254,7 +254,7 @@ func TestRetryManager_RetryScheduling(t *testing.T) {
|
||||
// Message should still be in retry state (not ready yet)
|
||||
retrieved, err := store.GetMessage(ctx, queue.Name(), msg.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateRetry, retrieved.State)
|
||||
assert.Equal(t, types.StateRetry, retrieved.State)
|
||||
assert.Equal(t, 1, retrieved.RetryCount) // Still 1
|
||||
|
||||
// Wait for retry time
|
||||
@@ -276,16 +276,16 @@ func TestRetryManager_GetStats(t *testing.T) {
|
||||
rm := NewRetryManager(store, dlqManager)
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
rm.RegisterQueue(queue)
|
||||
|
||||
// Add some inflight messages
|
||||
for i := 0; i < 3; i++ {
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: fmt.Sprintf("msg-%d", i),
|
||||
QueueName: queue.Name(),
|
||||
PartitionID: 0,
|
||||
@@ -299,13 +299,13 @@ func TestRetryManager_GetStats(t *testing.T) {
|
||||
|
||||
// Add some retry messages
|
||||
for i := 0; i < 2; i++ {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("retry-msg-%d", i),
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: uint64(i + 10),
|
||||
State: queueStorage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
RetryCount: 1,
|
||||
CreatedAt: time.Now(),
|
||||
NextRetryAt: time.Now().Add(5 * time.Second),
|
||||
@@ -327,8 +327,8 @@ func TestRetryManager_RegisterUnregisterQueue(t *testing.T) {
|
||||
dlqManager := NewDLQManager(store, &NoOpAlertHandler{})
|
||||
rm := NewRetryManager(store, dlqManager)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
queue := NewQueue(config, store, store)
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
queue := newMockQueueInfo(config.Name, config)
|
||||
|
||||
// Register queue
|
||||
rm.RegisterQueue(queue)
|
||||
+23
-21
@@ -14,7 +14,9 @@ import (
|
||||
"github.com/absmach/fluxmq/cluster"
|
||||
"github.com/absmach/fluxmq/queue/consumer"
|
||||
"github.com/absmach/fluxmq/queue/delivery"
|
||||
"github.com/absmach/fluxmq/queue/lifecycle"
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
@@ -48,14 +50,14 @@ const (
|
||||
type Manager struct {
|
||||
queues map[string]*Queue
|
||||
deliveryWorkers map[string]*delivery.Worker
|
||||
raftManagers map[string]*RaftManager // Per-queue Raft managers
|
||||
retentionManagers map[string]*RetentionManager // Per-queue retention managers
|
||||
raftManagers map[string]*RaftManager // Per-queue Raft managers
|
||||
retentionManagers map[string]*lifecycle.RetentionManager // Per-queue retention managers
|
||||
queueStore storage.QueueStore
|
||||
messageStore storage.MessageStore
|
||||
consumerStore storage.ConsumerStore
|
||||
broker DeliverFn
|
||||
retryManager *RetryManager
|
||||
dlqManager *DLQManager
|
||||
retryManager *lifecycle.RetryManager
|
||||
dlqManager *lifecycle.DLQManager
|
||||
mu sync.RWMutex
|
||||
stopCh chan struct{}
|
||||
stopOnce sync.Once
|
||||
@@ -93,11 +95,11 @@ func NewManager(cfg Config) (*Manager, error) {
|
||||
}
|
||||
|
||||
// Create DLQ manager with HTTP alert handler
|
||||
alertHandler := NewHTTPAlertHandler(10 * time.Second)
|
||||
dlqManager := NewDLQManager(cfg.MessageStore, alertHandler)
|
||||
alertHandler := lifecycle.NewHTTPAlertHandler(10 * time.Second)
|
||||
dlqManager := lifecycle.NewDLQManager(cfg.MessageStore, alertHandler)
|
||||
|
||||
// Create retry manager
|
||||
retryManager := NewRetryManager(cfg.MessageStore, dlqManager)
|
||||
retryManager := lifecycle.NewRetryManager(cfg.MessageStore, dlqManager)
|
||||
|
||||
// Default to single-node if no cluster or node ID provided
|
||||
localNodeID := cfg.LocalNodeID
|
||||
@@ -115,7 +117,7 @@ func NewManager(cfg Config) (*Manager, error) {
|
||||
queues: make(map[string]*Queue),
|
||||
deliveryWorkers: make(map[string]*delivery.Worker),
|
||||
raftManagers: make(map[string]*RaftManager),
|
||||
retentionManagers: make(map[string]*RetentionManager),
|
||||
retentionManagers: make(map[string]*lifecycle.RetentionManager),
|
||||
queueStore: cfg.QueueStore,
|
||||
messageStore: cfg.MessageStore,
|
||||
consumerStore: cfg.ConsumerStore,
|
||||
@@ -195,7 +197,7 @@ func (m *Manager) Stop() error {
|
||||
}
|
||||
|
||||
// CreateQueue creates a new queue with the given configuration.
|
||||
func (m *Manager) CreateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (m *Manager) CreateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
// Set defaults
|
||||
if config.DLQConfig.Topic == "" && config.DLQConfig.Enabled {
|
||||
config.DLQConfig.Topic = "$queue/dlq/" + strings.TrimPrefix(config.Name, "$queue/")
|
||||
@@ -216,7 +218,7 @@ func (m *Manager) CreateQueue(ctx context.Context, config storage.QueueConfig) e
|
||||
}
|
||||
|
||||
// createQueueInstance creates a queue instance in memory.
|
||||
func (m *Manager) createQueueInstance(config storage.QueueConfig) error {
|
||||
func (m *Manager) createQueueInstance(config types.QueueConfig) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
@@ -272,7 +274,7 @@ func (m *Manager) createQueueInstance(config storage.QueueConfig) error {
|
||||
|
||||
// Create RetentionManager if retention policy is configured
|
||||
if config.Retention.RetentionTime > 0 || config.Retention.RetentionBytes > 0 || config.Retention.RetentionMessages > 0 || config.Retention.CompactionEnabled {
|
||||
retentionMgr := NewRetentionManager(config.Name, config.Retention, m.messageStore, m.raftManagers[config.Name], slog.Default())
|
||||
retentionMgr := lifecycle.NewRetentionManager(config.Name, config.Retention, m.messageStore, m.raftManagers[config.Name], slog.Default())
|
||||
m.retentionManagers[config.Name] = retentionMgr
|
||||
// Note: RetentionManager will be started by partition workers (only leaders should run retention)
|
||||
}
|
||||
@@ -336,7 +338,7 @@ func (m *Manager) GetOrCreateQueue(ctx context.Context, queueName string) (*Queu
|
||||
}
|
||||
|
||||
// Create with default config
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
if err := m.CreateQueue(ctx, config); err != nil && err != storage.ErrQueueAlreadyExists {
|
||||
return nil, err
|
||||
}
|
||||
@@ -424,7 +426,7 @@ func (m *Manager) Enqueue(ctx context.Context, queueTopic string, payload []byte
|
||||
msg.PartitionID = partitionID
|
||||
msg.Sequence = sequence
|
||||
msg.Properties = msgProps
|
||||
msg.State = storage.StateQueued
|
||||
msg.State = types.StateQueued
|
||||
msg.CreatedAt = time.Now()
|
||||
msg.ExpiresAt = msg.CreatedAt.Add(config.MessageTTL)
|
||||
|
||||
@@ -478,7 +480,7 @@ func (m *Manager) Enqueue(ctx context.Context, queueTopic string, payload []byte
|
||||
}
|
||||
|
||||
// enqueueReplicated routes an enqueue through Raft for replication.
|
||||
func (m *Manager) enqueueReplicated(ctx context.Context, queueTopic string, partitionID int, payload []byte, properties map[string]string, config storage.QueueConfig) error {
|
||||
func (m *Manager) enqueueReplicated(ctx context.Context, queueTopic string, partitionID int, payload []byte, properties map[string]string, config types.QueueConfig) error {
|
||||
// Get Raft manager for this queue
|
||||
m.mu.RLock()
|
||||
raftMgr, exists := m.raftManagers[queueTopic]
|
||||
@@ -539,13 +541,13 @@ func (m *Manager) enqueueReplicated(ctx context.Context, queueTopic string, part
|
||||
msg.PartitionID = partitionID
|
||||
msg.Sequence = sequence
|
||||
msg.Properties = msgProps
|
||||
msg.State = storage.StateQueued
|
||||
msg.State = types.StateQueued
|
||||
msg.CreatedAt = time.Now()
|
||||
msg.ExpiresAt = msg.CreatedAt.Add(config.MessageTTL)
|
||||
|
||||
// Create a copy for Raft (FSM needs the data to persist)
|
||||
// We can't return msg to pool until FSM applies it
|
||||
raftMsg := &storage.Message{
|
||||
raftMsg := &types.Message{
|
||||
ID: msg.ID,
|
||||
Payload: append([]byte(nil), msg.Payload...), // Copy payload
|
||||
Topic: msg.Topic,
|
||||
@@ -663,7 +665,7 @@ func (m *Manager) Nack(ctx context.Context, queueTopic, messageID string) error
|
||||
}
|
||||
|
||||
// Update message state for retry
|
||||
msg.State = storage.StateRetry
|
||||
msg.State = types.StateRetry
|
||||
msg.RetryCount++
|
||||
msg.NextRetryAt = time.Now() // Immediate retry
|
||||
|
||||
@@ -691,7 +693,7 @@ func (m *Manager) Reject(ctx context.Context, queueTopic, messageID string, reas
|
||||
// Move to DLQ
|
||||
config := queue.Config()
|
||||
if config.DLQConfig.Enabled {
|
||||
msg.State = storage.StateDLQ
|
||||
msg.State = types.StateDLQ
|
||||
msg.FailureReason = reason
|
||||
msg.MovedToDLQAt = time.Now()
|
||||
|
||||
@@ -786,10 +788,10 @@ func (m *Manager) getPartitionOwner(ctx context.Context, queueName string, parti
|
||||
}
|
||||
|
||||
// getPartitionAssigner returns the partition assigner based on queue configuration.
|
||||
func (m *Manager) getPartitionAssigner(config storage.QueueConfig) consumer.PartitionAssigner {
|
||||
func (m *Manager) getPartitionAssigner(config types.QueueConfig) consumer.PartitionAssigner {
|
||||
// Check if a partition strategy is configured
|
||||
// For now, we'll use hash by default until we add PartitionStrategy to QueueConfig
|
||||
// In Phase 2, we'll add this field to storage.QueueConfig
|
||||
// In Phase 2, we'll add this field to types.QueueConfig
|
||||
|
||||
// Default to hash-based assignment
|
||||
if m.cluster == nil {
|
||||
@@ -904,7 +906,7 @@ func (m *Manager) EnqueueLocal(ctx context.Context, queueName string, payload []
|
||||
msg.PartitionID = partitionID
|
||||
msg.Sequence = sequence
|
||||
msg.Properties = msgProps
|
||||
msg.State = storage.StateQueued
|
||||
msg.State = types.StateQueued
|
||||
msg.CreatedAt = time.Now()
|
||||
msg.ExpiresAt = msg.CreatedAt.Add(config.MessageTTL)
|
||||
|
||||
|
||||
+16
-15
@@ -13,12 +13,13 @@ import (
|
||||
"github.com/absmach/fluxmq/cluster/grpc"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
brokerStorage "github.com/absmach/fluxmq/storage"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// MockBroker implements BrokerInterface for testing
|
||||
// MockBroker implements BrokerInterface for testing.
|
||||
type MockBroker struct {
|
||||
mu sync.Mutex
|
||||
deliveries map[string][]interface{}
|
||||
@@ -101,7 +102,7 @@ func TestManager_CreateQueue(t *testing.T) {
|
||||
mgr, err := NewManager(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -125,7 +126,7 @@ func TestManager_CreateQueue_Duplicate(t *testing.T) {
|
||||
mgr, err := NewManager(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -148,7 +149,7 @@ func TestManager_CreateQueue_InvalidConfig(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Invalid config (empty name)
|
||||
config := queueStorage.QueueConfig{
|
||||
config := types.QueueConfig{
|
||||
Name: "",
|
||||
}
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
@@ -174,7 +175,7 @@ func TestManager_GetQueue(t *testing.T) {
|
||||
assert.Nil(t, queue)
|
||||
|
||||
// Create and get queue
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -389,7 +390,7 @@ func TestManager_Ack(t *testing.T) {
|
||||
msg := messages[0]
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: msg.ID,
|
||||
QueueName: queue.Name(),
|
||||
PartitionID: partitionID,
|
||||
@@ -440,7 +441,7 @@ func TestManager_Nack(t *testing.T) {
|
||||
msg := messages[0]
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: msg.ID,
|
||||
QueueName: queue.Name(),
|
||||
PartitionID: partitionID,
|
||||
@@ -458,7 +459,7 @@ func TestManager_Nack(t *testing.T) {
|
||||
// Verify message state changed to retry
|
||||
retrieved, err := store.GetMessage(ctx, queue.Name(), msg.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, queueStorage.StateRetry, retrieved.State)
|
||||
assert.Equal(t, types.StateRetry, retrieved.State)
|
||||
assert.Equal(t, 1, retrieved.RetryCount)
|
||||
}
|
||||
|
||||
@@ -476,7 +477,7 @@ func TestManager_Reject(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create queue with DLQ enabled
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.DLQConfig.Enabled = true
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -494,7 +495,7 @@ func TestManager_Reject(t *testing.T) {
|
||||
msg := messages[0]
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &queueStorage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: msg.ID,
|
||||
QueueName: queue.Name(),
|
||||
PartitionID: partitionID,
|
||||
@@ -535,7 +536,7 @@ func TestManager_GetStats(t *testing.T) {
|
||||
mgr, err := NewManager(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -558,7 +559,7 @@ func TestManager_GetStats(t *testing.T) {
|
||||
assert.GreaterOrEqual(t, stats.TotalMessages, int64(5))
|
||||
}
|
||||
|
||||
// MockCluster implements cluster.Cluster for testing
|
||||
// MockCluster implements cluster.Cluster for testing.
|
||||
type MockCluster struct {
|
||||
nodeID string
|
||||
nodes []string // List of node IDs in the cluster
|
||||
@@ -664,7 +665,7 @@ func (m *MockCluster) SetPartitionOwner(queueName string, partitionID int, nodeI
|
||||
m.partitionOwners[queueName][partitionID] = nodeID
|
||||
}
|
||||
|
||||
// Stub methods for cluster.Cluster interface
|
||||
// Stub methods for cluster.Cluster interface.
|
||||
func (m *MockCluster) AcquireSession(ctx context.Context, clientID, nodeID string) error { return nil }
|
||||
func (m *MockCluster) ReleaseSession(ctx context.Context, clientID string) error { return nil }
|
||||
func (m *MockCluster) GetSessionOwner(ctx context.Context, clientID string) (string, bool, error) {
|
||||
@@ -757,7 +758,7 @@ func TestManager_EnqueueRemote(t *testing.T) {
|
||||
queueName := "$queue/test"
|
||||
|
||||
// Create queue
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -806,7 +807,7 @@ func TestManager_EnqueueRemote_NoCluster(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
queueName := "$queue/test"
|
||||
config := queueStorage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
err = mgr.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Hash pool for partition selection
|
||||
// Hash pool for partition selection.
|
||||
var hashPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return fnv.New32a()
|
||||
|
||||
+5
-5
@@ -6,13 +6,13 @@ package queue
|
||||
import (
|
||||
"sync"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
var (
|
||||
messagePool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &queueStorage.Message{
|
||||
return &types.Message{
|
||||
Properties: make(map[string]string, 8),
|
||||
}
|
||||
},
|
||||
@@ -26,12 +26,12 @@ var (
|
||||
)
|
||||
|
||||
// getMessageFromPool retrieves a QueueMessage from the pool.
|
||||
func getMessageFromPool() *queueStorage.Message {
|
||||
return messagePool.Get().(*queueStorage.Message)
|
||||
func getMessageFromPool() *types.Message {
|
||||
return messagePool.Get().(*types.Message)
|
||||
}
|
||||
|
||||
// putMessageToPool returns a QueueMessage to the pool.
|
||||
func putMessageToPool(msg *queueStorage.Message) {
|
||||
func putMessageToPool(msg *types.Message) {
|
||||
if msg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
+9
-7
@@ -9,23 +9,25 @@ import (
|
||||
|
||||
"github.com/absmach/fluxmq/queue/consumer"
|
||||
"github.com/absmach/fluxmq/queue/delivery"
|
||||
"github.com/absmach/fluxmq/queue/lifecycle"
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// Queue represents a single durable queue with partitions and consumer groups.
|
||||
type Queue struct {
|
||||
name string
|
||||
config queueStorage.QueueConfig
|
||||
config types.QueueConfig
|
||||
partitions []*Partition
|
||||
strategy PartitionStrategy
|
||||
consumerGroups *consumer.GroupManager
|
||||
messageStore queueStorage.MessageStore
|
||||
orderingEnforcer *OrderingEnforcer
|
||||
orderingEnforcer *lifecycle.OrderingEnforcer
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewQueue creates a new queue instance.
|
||||
func NewQueue(config queueStorage.QueueConfig, messageStore queueStorage.MessageStore, consumerStore queueStorage.ConsumerStore) *Queue {
|
||||
func NewQueue(config types.QueueConfig, messageStore queueStorage.MessageStore, consumerStore queueStorage.ConsumerStore) *Queue {
|
||||
partitions := make([]*Partition, config.Partitions)
|
||||
for i := 0; i < config.Partitions; i++ {
|
||||
partitions[i] = NewPartition(i)
|
||||
@@ -44,7 +46,7 @@ func NewQueue(config queueStorage.QueueConfig, messageStore queueStorage.Message
|
||||
strategy: &HashPartitionStrategy{},
|
||||
consumerGroups: consumer.NewGroupManager(config.Name, consumerStore, config.HeartbeatTimeout, consumerPartitions),
|
||||
messageStore: messageStore,
|
||||
orderingEnforcer: NewOrderingEnforcer(config.Ordering),
|
||||
orderingEnforcer: lifecycle.NewOrderingEnforcer(config.Ordering),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +56,7 @@ func (q *Queue) Name() string {
|
||||
}
|
||||
|
||||
// Config returns the queue configuration.
|
||||
func (q *Queue) Config() queueStorage.QueueConfig {
|
||||
func (q *Queue) Config() types.QueueConfig {
|
||||
q.mu.RLock()
|
||||
defer q.mu.RUnlock()
|
||||
|
||||
@@ -62,7 +64,7 @@ func (q *Queue) Config() queueStorage.QueueConfig {
|
||||
}
|
||||
|
||||
// UpdateConfig updates the queue configuration.
|
||||
func (q *Queue) UpdateConfig(config queueStorage.QueueConfig) {
|
||||
func (q *Queue) UpdateConfig(config types.QueueConfig) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
|
||||
@@ -127,7 +129,7 @@ func (q *Queue) RemoveConsumer(ctx context.Context, groupID, consumerID string)
|
||||
}
|
||||
|
||||
// GetConsumerForPartition returns the consumer assigned to a partition in a specific group.
|
||||
func (q *Queue) GetConsumerForPartition(groupID string, partitionID int) (*queueStorage.Consumer, bool) {
|
||||
func (q *Queue) GetConsumerForPartition(groupID string, partitionID int) (*types.Consumer, bool) {
|
||||
group, exists := q.consumerGroups.GetGroup(groupID)
|
||||
if !exists {
|
||||
return nil, false
|
||||
|
||||
+19
-18
@@ -7,15 +7,16 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/lifecycle"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewQueue(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
@@ -28,7 +29,7 @@ func TestNewQueue(t *testing.T) {
|
||||
|
||||
func TestQueue_Name(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/myqueue")
|
||||
config := types.DefaultQueueConfig("$queue/myqueue")
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
assert.Equal(t, "$queue/myqueue", queue.Name())
|
||||
@@ -36,7 +37,7 @@ func TestQueue_Name(t *testing.T) {
|
||||
|
||||
func TestQueue_Config(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 5
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
@@ -47,7 +48,7 @@ func TestQueue_Config(t *testing.T) {
|
||||
|
||||
func TestQueue_UpdateConfig(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
newConfig := config
|
||||
@@ -60,7 +61,7 @@ func TestQueue_UpdateConfig(t *testing.T) {
|
||||
|
||||
func TestQueue_Partitions(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 5
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
@@ -73,7 +74,7 @@ func TestQueue_Partitions(t *testing.T) {
|
||||
|
||||
func TestQueue_GetPartition(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 3
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
@@ -96,7 +97,7 @@ func TestQueue_GetPartition(t *testing.T) {
|
||||
|
||||
func TestQueue_GetPartitionForMessage(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 10
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
@@ -117,7 +118,7 @@ func TestQueue_GetPartitionForMessage(t *testing.T) {
|
||||
func TestQueue_AddConsumer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -139,7 +140,7 @@ func TestQueue_AddConsumer(t *testing.T) {
|
||||
func TestQueue_AddConsumer_MultipleConsumers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 4
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -167,7 +168,7 @@ func TestQueue_AddConsumer_MultipleConsumers(t *testing.T) {
|
||||
func TestQueue_RemoveConsumer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -192,7 +193,7 @@ func TestQueue_RemoveConsumer(t *testing.T) {
|
||||
func TestQueue_RemoveConsumer_RebalancesPartitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 6
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -223,7 +224,7 @@ func TestQueue_RemoveConsumer_RebalancesPartitions(t *testing.T) {
|
||||
func TestQueue_GetConsumerForPartition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 2
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -248,21 +249,21 @@ func TestQueue_GetConsumerForPartition(t *testing.T) {
|
||||
|
||||
func TestQueue_OrderingEnforcer(t *testing.T) {
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config.Ordering = queueStorage.OrderingPartition
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Ordering = types.OrderingPartition
|
||||
queue := NewQueue(config, store, store)
|
||||
|
||||
enforcer := queue.OrderingEnforcer()
|
||||
assert.NotNil(t, enforcer)
|
||||
|
||||
stats := enforcer.(*OrderingEnforcer).Stats()
|
||||
assert.Equal(t, queueStorage.OrderingPartition, stats.Mode)
|
||||
stats := enforcer.(*lifecycle.OrderingEnforcer).Stats()
|
||||
assert.Equal(t, types.OrderingPartition, stats.Mode)
|
||||
}
|
||||
|
||||
func TestQueue_MultipleGroups(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := memory.New()
|
||||
config := queueStorage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 4
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
+8
-7
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/hashicorp/raft"
|
||||
)
|
||||
|
||||
@@ -34,7 +35,7 @@ type Operation struct {
|
||||
Timestamp time.Time
|
||||
|
||||
// For OpEnqueue
|
||||
Message *storage.Message
|
||||
Message *types.Message
|
||||
|
||||
// For OpAck, OpNack, OpReject
|
||||
MessageID string
|
||||
@@ -106,7 +107,7 @@ func (f *PartitionFSM) Apply(l *raft.Log) interface{} {
|
||||
}
|
||||
|
||||
// applyEnqueue applies an enqueue operation.
|
||||
func (f *PartitionFSM) applyEnqueue(ctx context.Context, msg *storage.Message) error {
|
||||
func (f *PartitionFSM) applyEnqueue(ctx context.Context, msg *types.Message) error {
|
||||
if msg == nil {
|
||||
return fmt.Errorf("nil message in enqueue operation")
|
||||
}
|
||||
@@ -182,7 +183,7 @@ func (f *PartitionFSM) applyNack(ctx context.Context, messageID string, reason s
|
||||
}
|
||||
|
||||
// Update state to retry
|
||||
msg.State = storage.StateRetry
|
||||
msg.State = types.StateRetry
|
||||
msg.RetryCount++
|
||||
msg.FailureReason = reason
|
||||
msg.LastAttempt = time.Now()
|
||||
@@ -230,7 +231,7 @@ func (f *PartitionFSM) applyReject(ctx context.Context, messageID string, reason
|
||||
}
|
||||
|
||||
// Move to DLQ
|
||||
msg.State = storage.StateDLQ
|
||||
msg.State = types.StateDLQ
|
||||
msg.FailureReason = reason
|
||||
msg.MovedToDLQAt = time.Now()
|
||||
|
||||
@@ -257,7 +258,7 @@ func (f *PartitionFSM) applyReject(ctx context.Context, messageID string, reason
|
||||
}
|
||||
|
||||
// applyUpdateMessage applies a message update operation.
|
||||
func (f *PartitionFSM) applyUpdateMessage(ctx context.Context, msg *storage.Message) error {
|
||||
func (f *PartitionFSM) applyUpdateMessage(ctx context.Context, msg *types.Message) error {
|
||||
if msg == nil {
|
||||
return fmt.Errorf("nil message in update operation")
|
||||
}
|
||||
@@ -400,7 +401,7 @@ type PartitionSnapshot struct {
|
||||
type SnapshotData struct {
|
||||
QueueName string
|
||||
PartitionID int
|
||||
Messages []*storage.Message
|
||||
Messages []*types.Message
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
@@ -409,7 +410,7 @@ func (s *PartitionSnapshot) Persist(sink raft.SnapshotSink) error {
|
||||
// Get all messages from partition
|
||||
// Note: This is a simplified implementation
|
||||
// In production, you'd want to page through messages
|
||||
messages := make([]*storage.Message, 0)
|
||||
messages := make([]*types.Message, 0)
|
||||
|
||||
// Get first and last sequences to determine range
|
||||
// This would need to be implemented in the message store
|
||||
|
||||
+14
-13
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
badgerstore "github.com/absmach/fluxmq/queue/storage/badger"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/hashicorp/raft"
|
||||
)
|
||||
@@ -51,11 +52,11 @@ func TestFSM_ApplyEnqueue(t *testing.T) {
|
||||
fsm, store, cleanup := setupTestFSM(t)
|
||||
defer cleanup()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Sequence: 1,
|
||||
Payload: []byte("test payload"),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
|
||||
op := Operation{
|
||||
@@ -99,11 +100,11 @@ func TestFSM_ApplyAck(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// First enqueue a message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Sequence: 1,
|
||||
Payload: []byte("test payload"),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
|
||||
if err := store.Enqueue(ctx, "test-queue", msg); err != nil {
|
||||
@@ -111,7 +112,7 @@ func TestFSM_ApplyAck(t *testing.T) {
|
||||
}
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &storage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "test-queue",
|
||||
PartitionID: 0,
|
||||
@@ -160,11 +161,11 @@ func TestFSM_ApplyNack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// First enqueue a message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Sequence: 1,
|
||||
Payload: []byte("test payload"),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
|
||||
if err := store.Enqueue(ctx, "test-queue", msg); err != nil {
|
||||
@@ -172,7 +173,7 @@ func TestFSM_ApplyNack(t *testing.T) {
|
||||
}
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &storage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "test-queue",
|
||||
PartitionID: 0,
|
||||
@@ -214,8 +215,8 @@ func TestFSM_ApplyNack(t *testing.T) {
|
||||
t.Fatalf("failed to get message: %v", err)
|
||||
}
|
||||
|
||||
if retrieved.State != storage.StateRetry {
|
||||
t.Errorf("expected state %v, got %v", storage.StateRetry, retrieved.State)
|
||||
if retrieved.State != types.StateRetry {
|
||||
t.Errorf("expected state %v, got %v", types.StateRetry, retrieved.State)
|
||||
}
|
||||
|
||||
if retrieved.RetryCount != 1 {
|
||||
@@ -234,11 +235,11 @@ func TestFSM_ApplyReject(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue a message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Sequence: 1,
|
||||
Payload: []byte("test payload"),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
|
||||
if err := store.Enqueue(ctx, "test-queue", msg); err != nil {
|
||||
@@ -246,7 +247,7 @@ func TestFSM_ApplyReject(t *testing.T) {
|
||||
}
|
||||
|
||||
// Mark as inflight
|
||||
deliveryState := &storage.DeliveryState{
|
||||
deliveryState := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "test-queue",
|
||||
PartitionID: 0,
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/hashicorp/raft"
|
||||
)
|
||||
|
||||
|
||||
// BadgerLogStore implements raft.LogStore using BadgerDB.
|
||||
// It stores Raft log entries with efficient sequential writes.
|
||||
type BadgerLogStore struct {
|
||||
|
||||
@@ -14,13 +14,14 @@ import (
|
||||
|
||||
"github.com/absmach/fluxmq/queue/raft"
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
raftlib "github.com/hashicorp/raft"
|
||||
)
|
||||
|
||||
// RaftManager manages Raft groups for all partitions of a queue.
|
||||
type RaftManager struct {
|
||||
queueName string
|
||||
config storage.ReplicationConfig
|
||||
config types.ReplicationConfig
|
||||
nodeID string
|
||||
dataDir string
|
||||
messageStore storage.MessageStore
|
||||
@@ -40,7 +41,7 @@ type RaftManager struct {
|
||||
// RaftManagerConfig contains configuration for creating a RaftManager.
|
||||
type RaftManagerConfig struct {
|
||||
QueueName string
|
||||
Config storage.ReplicationConfig
|
||||
Config types.ReplicationConfig
|
||||
NodeID string
|
||||
DataDir string
|
||||
MessageStore storage.MessageStore
|
||||
@@ -61,9 +62,9 @@ func NewRaftManager(cfg RaftManagerConfig) (*RaftManager, error) {
|
||||
// Create placement strategy
|
||||
var placement PlacementStrategy
|
||||
switch cfg.Config.Placement {
|
||||
case storage.PlacementRoundRobin:
|
||||
case types.PlacementRoundRobin:
|
||||
placement = NewRoundRobinPlacement(cfg.NodeAddresses)
|
||||
case storage.PlacementManual:
|
||||
case types.PlacementManual:
|
||||
placement = NewManualPlacement(cfg.Config.ManualReplicas)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown placement strategy: %s", cfg.Config.Placement)
|
||||
@@ -103,7 +104,7 @@ func (rm *RaftManager) StartPartition(ctx context.Context, partitionID int, part
|
||||
NodeID: rm.nodeID,
|
||||
BindAddr: rm.getPartitionBindAddress(rm.nodeID, partitionID),
|
||||
DataDir: filepath.Join(rm.dataDir, rm.queueName),
|
||||
SyncMode: rm.config.Mode == storage.ReplicationSync,
|
||||
SyncMode: rm.config.Mode == types.ReplicationSync,
|
||||
AckTimeout: rm.config.AckTimeout,
|
||||
|
||||
HeartbeatTimeout: rm.config.HeartbeatTimeout,
|
||||
@@ -242,7 +243,7 @@ func (rm *RaftManager) IsLeader(partitionID int) bool {
|
||||
}
|
||||
|
||||
// ApplyEnqueue replicates an enqueue operation via Raft.
|
||||
func (rm *RaftManager) ApplyEnqueue(ctx context.Context, partitionID int, msg *storage.Message) error {
|
||||
func (rm *RaftManager) ApplyEnqueue(ctx context.Context, partitionID int, msg *types.Message) error {
|
||||
group, err := rm.GetPartitionGroup(partitionID)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -12,17 +12,17 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
badgerstore "github.com/absmach/fluxmq/queue/storage/badger"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// BenchmarkReplication_SyncMode benchmarks sync replication throughput.
|
||||
// Target: >5K enqueues/sec per partition
|
||||
// Target: >5K enqueues/sec per partition.
|
||||
func BenchmarkReplication_SyncMode(b *testing.B) {
|
||||
queueName := "$queue/bench-sync"
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 3, queueStorage.ReplicationSync)
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 3, types.ReplicationSync)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -48,10 +48,10 @@ func BenchmarkReplication_SyncMode(b *testing.B) {
|
||||
}
|
||||
|
||||
// BenchmarkReplication_AsyncMode benchmarks async replication throughput.
|
||||
// Target: >50K enqueues/sec per partition
|
||||
// Target: >50K enqueues/sec per partition.
|
||||
func BenchmarkReplication_AsyncMode(b *testing.B) {
|
||||
queueName := "$queue/bench-async"
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 3, queueStorage.ReplicationAsync)
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 3, types.ReplicationAsync)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -77,10 +77,10 @@ func BenchmarkReplication_AsyncMode(b *testing.B) {
|
||||
}
|
||||
|
||||
// BenchmarkReplication_Latency measures enqueue latency with sync replication.
|
||||
// Target P99: <50ms (sync)
|
||||
// Target P99: <50ms (sync).
|
||||
func BenchmarkReplication_Latency(b *testing.B) {
|
||||
queueName := "$queue/bench-latency"
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 1, queueStorage.ReplicationSync)
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 1, types.ReplicationSync)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -123,7 +123,7 @@ func BenchmarkReplication_Latency(b *testing.B) {
|
||||
// BenchmarkReplication_Concurrent benchmarks concurrent enqueues from multiple goroutines.
|
||||
func BenchmarkReplication_Concurrent(b *testing.B) {
|
||||
queueName := "$queue/bench-concurrent"
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 3, queueStorage.ReplicationSync)
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 3, types.ReplicationSync)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -170,7 +170,7 @@ func BenchmarkReplication_Concurrent(b *testing.B) {
|
||||
// BenchmarkReplication_MessageSizes benchmarks different message sizes.
|
||||
func BenchmarkReplication_MessageSizes(b *testing.B) {
|
||||
queueName := "$queue/bench-sizes"
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 1, queueStorage.ReplicationSync)
|
||||
managers, _, cleanup := setupBenchCluster(b, 3, queueName, 1, types.ReplicationSync)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -210,7 +210,7 @@ func BenchmarkReplication_MessageSizes(b *testing.B) {
|
||||
}
|
||||
|
||||
// setupBenchCluster creates a 3-node cluster for benchmarking.
|
||||
func setupBenchCluster(b *testing.B, nodeCount int, queueName string, partitions int, mode queueStorage.ReplicationMode) ([]*Manager, []*badgerstore.Store, func()) {
|
||||
func setupBenchCluster(b *testing.B, nodeCount int, queueName string, partitions int, mode types.ReplicationMode) ([]*Manager, []*badgerstore.Store, func()) {
|
||||
b.Helper()
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "bench-*")
|
||||
@@ -259,21 +259,21 @@ func setupBenchCluster(b *testing.B, nodeCount int, queueName string, partitions
|
||||
}
|
||||
|
||||
// Create replicated queue
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: nodeCount,
|
||||
Mode: mode,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: (nodeCount / 2) + 1,
|
||||
AckTimeout: 5 * time.Second,
|
||||
HeartbeatTimeout: 1 * time.Second,
|
||||
ElectionTimeout: 3 * time.Second,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
|
||||
+32
-32
@@ -13,9 +13,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
badgerstore "github.com/absmach/fluxmq/queue/storage/badger"
|
||||
memstore "github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -96,21 +96,21 @@ func setupReplicatedTest(t *testing.T, nodeCount int, queueName string, partitio
|
||||
}
|
||||
|
||||
// Create replicated queue on all nodes
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: nodeCount,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: (nodeCount / 2) + 1, // Quorum
|
||||
AckTimeout: 5 * time.Second,
|
||||
HeartbeatTimeout: 1 * time.Second,
|
||||
ElectionTimeout: 3 * time.Second,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -222,21 +222,21 @@ func TestReplication_BasicEnqueueDequeue(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create replicated queue on all nodes
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: nodeCount,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: (nodeCount / 2) + 1,
|
||||
AckTimeout: 5 * time.Second,
|
||||
HeartbeatTimeout: 1 * time.Second,
|
||||
ElectionTimeout: 3 * time.Second,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -392,7 +392,7 @@ func TestReplication_MessageDurability(t *testing.T) {
|
||||
// Verifies that:
|
||||
// - All replicas start in-sync (quorum maintained)
|
||||
// - Cluster tolerates follower failure (continues with quorum)
|
||||
// - Failed follower catches up when it rejoins
|
||||
// - Failed follower catches up when it rejoins.
|
||||
func TestReplication_ISRTracking(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping ISR tracking test in short mode")
|
||||
@@ -576,26 +576,26 @@ func TestReplication_ConfigValidation(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
config queueStorage.QueueConfig
|
||||
config types.QueueConfig
|
||||
expectError bool
|
||||
errorMsg string
|
||||
skip bool // Skip tests that require full cluster infrastructure
|
||||
}{
|
||||
{
|
||||
name: "valid sync replication",
|
||||
config: queueStorage.QueueConfig{
|
||||
config: types.QueueConfig{
|
||||
Name: "$queue/valid-sync",
|
||||
Partitions: 3,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: 3,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: 2,
|
||||
AckTimeout: 5 * time.Second,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -612,19 +612,19 @@ func TestReplication_ConfigValidation(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "valid async replication",
|
||||
config: queueStorage.QueueConfig{
|
||||
config: types.QueueConfig{
|
||||
Name: "$queue/valid-async",
|
||||
Partitions: 3,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: 3,
|
||||
Mode: queueStorage.ReplicationAsync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationAsync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: 2,
|
||||
AckTimeout: 5 * time.Second,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -641,14 +641,14 @@ func TestReplication_ConfigValidation(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "replication disabled",
|
||||
config: queueStorage.QueueConfig{
|
||||
config: types.QueueConfig{
|
||||
Name: "$queue/no-replication",
|
||||
Partitions: 3,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: false,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -698,7 +698,7 @@ func TestReplication_BackwardCompatibility(t *testing.T) {
|
||||
|
||||
// Create queue WITHOUT replication
|
||||
queueName := "$queue/legacy"
|
||||
queueConfig := queueStorage.DefaultQueueConfig(queueName)
|
||||
queueConfig := types.DefaultQueueConfig(queueName)
|
||||
queueConfig.Replication.Enabled = false // Explicitly disabled
|
||||
|
||||
err = mgr.CreateQueue(ctx, queueConfig)
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
queueStorage "github.com/absmach/fluxmq/queue/storage"
|
||||
badgerstore "github.com/absmach/fluxmq/queue/storage/badger"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -197,12 +197,12 @@ func TestRetention_TimeBasedReplication(t *testing.T) {
|
||||
oldTime := time.Now().Add(-200 * time.Millisecond)
|
||||
for i := 0; i < 3; i++ {
|
||||
for _, store := range stores {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("old-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("old message"),
|
||||
Sequence: uint64(i + 1), // Sequences start at 1
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: oldTime,
|
||||
PartitionID: 0,
|
||||
}
|
||||
@@ -214,12 +214,12 @@ func TestRetention_TimeBasedReplication(t *testing.T) {
|
||||
// Enqueue new messages on all stores
|
||||
for i := 3; i < 6; i++ {
|
||||
for _, store := range stores {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("new-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("new message"),
|
||||
Sequence: uint64(i + 1), // Sequences start at 1
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
PartitionID: 0,
|
||||
}
|
||||
@@ -317,28 +317,28 @@ func setupReplicatedTestWithRetention(t *testing.T, nodeCount int, queueName str
|
||||
}
|
||||
|
||||
// Create replicated queue with retention policy
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: nodeCount,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: (nodeCount / 2) + 1,
|
||||
AckTimeout: 5 * time.Second,
|
||||
HeartbeatTimeout: 1 * time.Second,
|
||||
ElectionTimeout: 3 * time.Second,
|
||||
},
|
||||
Retention: queueStorage.RetentionPolicy{
|
||||
Retention: types.RetentionPolicy{
|
||||
RetentionMessages: 5, // Keep max 5 messages
|
||||
SizeCheckEvery: 5, // Check every 5 enqueues
|
||||
RetentionBytes: 0, // No byte limit
|
||||
RetentionTime: 0, // No time limit
|
||||
TimeCheckInterval: 5 * time.Minute, // Not used for size-based
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -438,28 +438,28 @@ func setupReplicatedTestWithTimeRetention(t *testing.T, nodeCount int, queueName
|
||||
managers[i] = mgr
|
||||
}
|
||||
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: nodeCount,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: (nodeCount / 2) + 1,
|
||||
AckTimeout: 5 * time.Second,
|
||||
HeartbeatTimeout: 1 * time.Second,
|
||||
ElectionTimeout: 3 * time.Second,
|
||||
},
|
||||
Retention: queueStorage.RetentionPolicy{
|
||||
Retention: types.RetentionPolicy{
|
||||
RetentionMessages: 0, // No message limit
|
||||
SizeCheckEvery: 100, // Not used for time-based
|
||||
RetentionBytes: 0, // No byte limit
|
||||
RetentionTime: 100 * time.Millisecond, // Messages older than 100ms are deleted
|
||||
TimeCheckInterval: 50 * time.Millisecond, // Check every 50ms
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
@@ -537,12 +537,12 @@ func TestCompaction_ReplicationBasic(t *testing.T) {
|
||||
oldTime := time.Now().Add(-time.Hour) // Old enough to be compacted
|
||||
for i := 0; i < 3; i++ {
|
||||
for _, store := range stores {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("entity-a-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte(fmt.Sprintf("entity A update %d", i)),
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: oldTime,
|
||||
PartitionID: 0,
|
||||
Properties: map[string]string{
|
||||
@@ -557,12 +557,12 @@ func TestCompaction_ReplicationBasic(t *testing.T) {
|
||||
// Entity B: 2 messages
|
||||
for i := 3; i < 5; i++ {
|
||||
for _, store := range stores {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("entity-b-msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte(fmt.Sprintf("entity B update %d", i-3)),
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: oldTime,
|
||||
PartitionID: 0,
|
||||
Properties: map[string]string{
|
||||
@@ -585,7 +585,7 @@ func TestCompaction_ReplicationBasic(t *testing.T) {
|
||||
retentionMgr := leaderMgr.retentionManagers[queueName]
|
||||
require.NotNil(t, retentionMgr, "retention manager not found")
|
||||
|
||||
stats, err := retentionMgr.runCompaction(ctx, 0)
|
||||
stats, err := retentionMgr.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should delete 3 messages (2 from entity-A, 1 from entity-B)
|
||||
@@ -648,12 +648,12 @@ func TestCompaction_LeaderOnly(t *testing.T) {
|
||||
oldTime := time.Now().Add(-time.Hour)
|
||||
for i := 0; i < 3; i++ {
|
||||
for _, store := range stores {
|
||||
msg := &queueStorage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Topic: queueName,
|
||||
Payload: []byte("message"),
|
||||
Sequence: uint64(i + 1),
|
||||
State: queueStorage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: oldTime,
|
||||
PartitionID: 0,
|
||||
Properties: map[string]string{
|
||||
@@ -676,7 +676,7 @@ func TestCompaction_LeaderOnly(t *testing.T) {
|
||||
followerRetentionMgr := followerMgrs[0].retentionManagers[queueName]
|
||||
require.NotNil(t, followerRetentionMgr)
|
||||
|
||||
stats, err := followerRetentionMgr.runCompaction(ctx, 0)
|
||||
stats, err := followerRetentionMgr.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), stats.MessagesDeleted, "follower should not compact messages")
|
||||
|
||||
@@ -689,7 +689,7 @@ func TestCompaction_LeaderOnly(t *testing.T) {
|
||||
|
||||
// Now run compaction on leader
|
||||
leaderRetentionMgr := leaderMgr.retentionManagers[queueName]
|
||||
stats, err = leaderRetentionMgr.runCompaction(ctx, 0)
|
||||
stats, err = leaderRetentionMgr.RunCompaction(ctx, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(2), stats.MessagesDeleted, "leader should compact 2 messages")
|
||||
|
||||
@@ -752,27 +752,27 @@ func setupReplicatedTestWithCompaction(t *testing.T, nodeCount int, queueName st
|
||||
managers[i] = mgr
|
||||
}
|
||||
|
||||
queueConfig := queueStorage.QueueConfig{
|
||||
queueConfig := types.QueueConfig{
|
||||
Name: queueName,
|
||||
Partitions: partitions,
|
||||
Ordering: queueStorage.OrderingPartition,
|
||||
Replication: queueStorage.ReplicationConfig{
|
||||
Ordering: types.OrderingPartition,
|
||||
Replication: types.ReplicationConfig{
|
||||
Enabled: true,
|
||||
ReplicationFactor: nodeCount,
|
||||
Mode: queueStorage.ReplicationSync,
|
||||
Placement: queueStorage.PlacementRoundRobin,
|
||||
Mode: types.ReplicationSync,
|
||||
Placement: types.PlacementRoundRobin,
|
||||
MinInSyncReplicas: (nodeCount / 2) + 1,
|
||||
AckTimeout: 5 * time.Second,
|
||||
HeartbeatTimeout: 1 * time.Second,
|
||||
ElectionTimeout: 3 * time.Second,
|
||||
},
|
||||
Retention: queueStorage.RetentionPolicy{
|
||||
Retention: types.RetentionPolicy{
|
||||
CompactionEnabled: true,
|
||||
CompactionKey: "entity_id",
|
||||
CompactionLag: 0, // No lag for testing
|
||||
CompactionInterval: 10 * time.Minute,
|
||||
},
|
||||
RetryPolicy: queueStorage.RetryPolicy{
|
||||
RetryPolicy: types.RetryPolicy{
|
||||
MaxRetries: 3,
|
||||
InitialBackoff: 100 * time.Millisecond,
|
||||
MaxBackoff: 1 * time.Second,
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package queue_test
|
||||
|
||||
import (
|
||||
@@ -6,8 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue"
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -27,7 +30,7 @@ func TestRetentionPolicy(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/retention-test"
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MaxQueueDepth = 5 // Small limit for testing
|
||||
config.MessageTTL = 100 * time.Millisecond // Short TTL
|
||||
|
||||
@@ -68,7 +71,7 @@ func TestMessageTTL(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
queueName := "$queue/ttl-test"
|
||||
config := storage.DefaultQueueConfig(queueName)
|
||||
config := types.DefaultQueueConfig(queueName)
|
||||
config.MessageTTL = 100 * time.Millisecond
|
||||
|
||||
err = manager.CreateQueue(ctx, config)
|
||||
@@ -95,7 +98,7 @@ func TestMessageTTL(t *testing.T) {
|
||||
// Since we use random partition if no key, might be in any partition.
|
||||
// But DefaultQueueConfig uses 10 partitions.
|
||||
// Let's find it.
|
||||
var found *storage.Message
|
||||
var found *types.Message
|
||||
for i := 0; i < 10; i++ {
|
||||
msgs, err := memStore.ListQueued(ctx, queueName, i, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
@@ -39,7 +40,7 @@ func New(db *badger.DB) *Store {
|
||||
|
||||
// QueueStore implementation
|
||||
|
||||
func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) CreateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -71,9 +72,9 @@ func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) err
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueConfig, error) {
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*types.QueueConfig, error) {
|
||||
key := queueMetaPrefix + queueName
|
||||
var config storage.QueueConfig
|
||||
var config types.QueueConfig
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
item, err := txn.Get([]byte(key))
|
||||
@@ -95,7 +96,7 @@ func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueC
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -127,8 +128,8 @@ func (s *Store) DeleteQueue(ctx context.Context, queueName string) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
configs := make([]storage.QueueConfig, 0)
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]types.QueueConfig, error) {
|
||||
configs := make([]types.QueueConfig, 0)
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
@@ -140,7 +141,7 @@ func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var config storage.QueueConfig
|
||||
var config types.QueueConfig
|
||||
if err := json.Unmarshal(val, &config); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -157,8 +158,8 @@ func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
return configs, err
|
||||
}
|
||||
|
||||
// MessageStore implementation
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
// MessageStore implementation.
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
key := makeMessageKey(queueName, msg.PartitionID, msg.Sequence)
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
@@ -217,9 +218,9 @@ func (s *Store) Count(ctx context.Context, queueName string) (int64, error) {
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*storage.Message, error) {
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*types.Message, error) {
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
var msg *storage.Message
|
||||
var msg *types.Message
|
||||
|
||||
err := s.db.Update(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
@@ -232,7 +233,7 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
|
||||
var qm storage.Message
|
||||
var qm types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &qm)
|
||||
})
|
||||
@@ -241,8 +242,8 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
}
|
||||
|
||||
// Only return queued or retry messages that are ready
|
||||
if qm.State == storage.StateQueued ||
|
||||
(qm.State == storage.StateRetry && time.Now().After(qm.NextRetryAt)) {
|
||||
if qm.State == types.StateQueued ||
|
||||
(qm.State == types.StateRetry && time.Now().After(qm.NextRetryAt)) {
|
||||
msg = &qm
|
||||
return nil
|
||||
}
|
||||
@@ -253,13 +254,13 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
return msg, err
|
||||
}
|
||||
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
var messages []*storage.Message
|
||||
var messages []*types.Message
|
||||
|
||||
// Note: Using View (not Update) because we don't delete on dequeue - just read
|
||||
// Messages are marked as delivered and tracked via inflight state
|
||||
@@ -275,7 +276,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
for it.Rewind(); it.Valid() && len(messages) < limit; it.Next() {
|
||||
item := it.Item()
|
||||
|
||||
var qm storage.Message
|
||||
var qm types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &qm)
|
||||
})
|
||||
@@ -284,8 +285,8 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
}
|
||||
|
||||
// Only return queued or retry messages that are ready
|
||||
if qm.State == storage.StateQueued ||
|
||||
(qm.State == storage.StateRetry && time.Now().After(qm.NextRetryAt)) {
|
||||
if qm.State == types.StateQueued ||
|
||||
(qm.State == types.StateRetry && time.Now().After(qm.NextRetryAt)) {
|
||||
messages = append(messages, &qm)
|
||||
}
|
||||
}
|
||||
@@ -302,7 +303,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
key := makeMessageKey(queueName, msg.PartitionID, msg.Sequence)
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
@@ -334,7 +335,7 @@ func (s *Store) DeleteMessage(ctx context.Context, queueName string, messageID s
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
@@ -358,8 +359,8 @@ func (s *Store) DeleteMessage(ctx context.Context, queueName string, messageID s
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*storage.Message, error) {
|
||||
var result *storage.Message
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*types.Message, error) {
|
||||
var result *types.Message
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
prefix := queueMessagePrefix + queueName + ":"
|
||||
@@ -372,7 +373,7 @@ func (s *Store) GetMessage(ctx context.Context, queueName string, messageID stri
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
@@ -392,7 +393,7 @@ func (s *Store) GetMessage(ctx context.Context, queueName string, messageID stri
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *storage.DeliveryState) error {
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *types.DeliveryState) error {
|
||||
key := makeInflightKey(state.QueueName, state.MessageID)
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
@@ -404,8 +405,8 @@ func (s *Store) MarkInflight(ctx context.Context, state *storage.DeliveryState)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.DeliveryState, error) {
|
||||
states := make([]*storage.DeliveryState, 0)
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*types.DeliveryState, error) {
|
||||
states := make([]*types.DeliveryState, 0)
|
||||
prefix := queueInflightPrefix + queueName + ":"
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
@@ -418,7 +419,7 @@ func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.D
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var state storage.DeliveryState
|
||||
var state types.DeliveryState
|
||||
if err := json.Unmarshal(val, &state); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -435,9 +436,9 @@ func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.D
|
||||
return states, err
|
||||
}
|
||||
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*types.DeliveryState, error) {
|
||||
key := makeInflightKey(queueName, messageID)
|
||||
var state storage.DeliveryState
|
||||
var state types.DeliveryState
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
item, err := txn.Get([]byte(key))
|
||||
@@ -466,7 +467,7 @@ func (s *Store) RemoveInflight(ctx context.Context, queueName, messageID string)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *storage.Message) error {
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *types.Message) error {
|
||||
key := makeDLQKey(dlqTopic, msg.ID)
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
@@ -478,8 +479,8 @@ func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *storage.Me
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*storage.Message, error) {
|
||||
messages := make([]*storage.Message, 0, limit)
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*types.Message, error) {
|
||||
messages := make([]*types.Message, 0, limit)
|
||||
// Remove $queue/dlq/ prefix if present (consistent with makeDLQKey)
|
||||
topic := strings.TrimPrefix(dlqTopic, "$queue/dlq/")
|
||||
prefix := queueDLQPrefix + topic + ":"
|
||||
@@ -495,7 +496,7 @@ func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*sto
|
||||
for it.Rewind(); it.Valid() && (limit == 0 || count < limit); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
if err := json.Unmarshal(val, &msg); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -520,8 +521,8 @@ func (s *Store) DeleteDLQMessage(ctx context.Context, dlqTopic, messageID string
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*storage.Message, error) {
|
||||
messages := make([]*storage.Message, 0)
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error) {
|
||||
messages := make([]*types.Message, 0)
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
@@ -534,11 +535,11 @@ func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
if err := json.Unmarshal(val, &msg); err != nil {
|
||||
return err
|
||||
}
|
||||
if msg.State == storage.StateRetry {
|
||||
if msg.State == types.StateRetry {
|
||||
messages = append(messages, &msg)
|
||||
}
|
||||
return nil
|
||||
@@ -618,8 +619,8 @@ func (s *Store) GetOffset(ctx context.Context, queueName string, partitionID int
|
||||
return offset, err
|
||||
}
|
||||
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
messages := make([]*storage.Message, 0, limit)
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
messages := make([]*types.Message, 0, limit)
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
@@ -633,11 +634,11 @@ func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID in
|
||||
for it.Rewind(); it.Valid() && (limit == 0 || count < limit); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
if err := json.Unmarshal(val, &msg); err != nil {
|
||||
return err
|
||||
}
|
||||
if msg.State == storage.StateQueued || msg.State == storage.StateRetry {
|
||||
if msg.State == types.StateQueued || msg.State == types.StateRetry {
|
||||
messages = append(messages, &msg)
|
||||
count++
|
||||
}
|
||||
@@ -655,7 +656,7 @@ func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID in
|
||||
|
||||
// ConsumerStore implementation
|
||||
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *storage.Consumer) error {
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *types.Consumer) error {
|
||||
key := makeConsumerKey(consumer.QueueName, consumer.GroupID, consumer.ID)
|
||||
data, err := json.Marshal(consumer)
|
||||
if err != nil {
|
||||
@@ -674,9 +675,9 @@ func (s *Store) UnregisterConsumer(ctx context.Context, queueName, groupID, cons
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*storage.Consumer, error) {
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*types.Consumer, error) {
|
||||
key := makeConsumerKey(queueName, groupID, consumerID)
|
||||
var consumer storage.Consumer
|
||||
var consumer types.Consumer
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
item, err := txn.Get([]byte(key))
|
||||
@@ -698,8 +699,8 @@ func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID
|
||||
return &consumer, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*storage.Consumer, error) {
|
||||
consumers := make([]*storage.Consumer, 0)
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*types.Consumer, error) {
|
||||
consumers := make([]*types.Consumer, 0)
|
||||
prefix := queueConsumerPrefix + queueName + ":" + groupID + ":"
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
@@ -712,7 +713,7 @@ func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var consumer storage.Consumer
|
||||
var consumer types.Consumer
|
||||
if err := json.Unmarshal(val, &consumer); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -743,7 +744,7 @@ func (s *Store) ListGroups(ctx context.Context, queueName string) ([]string, err
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
var consumer storage.Consumer
|
||||
var consumer types.Consumer
|
||||
if err := json.Unmarshal(val, &consumer); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -841,9 +842,9 @@ func makeOffsetKey(queueName string, partitionID int) string {
|
||||
|
||||
// Retention operations
|
||||
|
||||
func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
var messages []*storage.Message
|
||||
var messages []*types.Message
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
@@ -860,7 +861,7 @@ func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partit
|
||||
}
|
||||
|
||||
item := it.Item()
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
@@ -879,9 +880,9 @@ func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partit
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListMessagesBefore(ctx context.Context, queueName string, partitionID int, cutoffTime time.Time, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListMessagesBefore(ctx context.Context, queueName string, partitionID int, cutoffTime time.Time, limit int) ([]*types.Message, error) {
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
var messages []*storage.Message
|
||||
var messages []*types.Message
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
@@ -897,7 +898,7 @@ func (s *Store) ListMessagesBefore(ctx context.Context, queueName string, partit
|
||||
}
|
||||
|
||||
item := it.Item()
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
@@ -953,7 +954,7 @@ func (s *Store) DeleteMessageBatch(ctx context.Context, queueName string, messag
|
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
@@ -1014,7 +1015,7 @@ func (s *Store) GetQueueSize(ctx context.Context, queueName string) (int64, erro
|
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
@@ -1034,8 +1035,8 @@ func (s *Store) GetQueueSize(ctx context.Context, queueName string) (int64, erro
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListAllMessages(ctx context.Context, queueName string, partitionID int) ([]*storage.Message, error) {
|
||||
messages := make([]*storage.Message, 0)
|
||||
func (s *Store) ListAllMessages(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error) {
|
||||
messages := make([]*types.Message, 0)
|
||||
prefix := makePartitionPrefix(queueName, partitionID)
|
||||
|
||||
err := s.db.View(func(txn *badger.Txn) error {
|
||||
@@ -1048,7 +1049,7 @@ func (s *Store) ListAllMessages(ctx context.Context, queueName string, partition
|
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
var msg storage.Message
|
||||
var msg types.Message
|
||||
err := item.Value(func(val []byte) error {
|
||||
return json.Unmarshal(val, &msg)
|
||||
})
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -47,7 +48,7 @@ func TestBadgerQueueStore_CreateQueue(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -64,7 +65,7 @@ func TestBadgerQueueStore_CreateQueue_Duplicate(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -81,7 +82,7 @@ func TestBadgerQueueStore_CreateQueue_InvalidConfig(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Invalid config (empty name)
|
||||
config := storage.QueueConfig{Name: ""}
|
||||
config := types.QueueConfig{Name: ""}
|
||||
err := store.CreateQueue(ctx, config)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -97,7 +98,7 @@ func TestBadgerQueueStore_GetQueue(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
|
||||
// Create and get queue
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -113,7 +114,7 @@ func TestBadgerQueueStore_UpdateQueue(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Update non-existent queue
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.UpdateQueue(ctx, config)
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
|
||||
@@ -138,7 +139,7 @@ func TestBadgerQueueStore_DeleteQueue(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -163,9 +164,9 @@ func TestBadgerQueueStore_ListQueues(t *testing.T) {
|
||||
assert.Len(t, queues, 0)
|
||||
|
||||
// Create multiple queues
|
||||
config1 := storage.DefaultQueueConfig("$queue/test1")
|
||||
config2 := storage.DefaultQueueConfig("$queue/test2")
|
||||
config3 := storage.DefaultQueueConfig("$queue/test3")
|
||||
config1 := types.DefaultQueueConfig("$queue/test1")
|
||||
config2 := types.DefaultQueueConfig("$queue/test2")
|
||||
config3 := types.DefaultQueueConfig("$queue/test3")
|
||||
|
||||
require.NoError(t, store.CreateQueue(ctx, config1))
|
||||
require.NoError(t, store.CreateQueue(ctx, config2))
|
||||
@@ -194,13 +195,13 @@ func TestBadgerMessageStore_Enqueue(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -221,22 +222,22 @@ func TestBadgerMessageStore_Dequeue(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue multiple messages
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("payload-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("payload-2"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -261,13 +262,13 @@ func TestBadgerMessageStore_Dequeue_RetryReady(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Message ready for retry (NextRetryAt in the past)
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
NextRetryAt: time.Now().Add(-1 * time.Second), // Past
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -288,13 +289,13 @@ func TestBadgerMessageStore_Dequeue_RetryNotReady(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Message not ready for retry (NextRetryAt in the future)
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
NextRetryAt: time.Now().Add(1 * time.Hour), // Future
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -313,13 +314,13 @@ func TestBadgerMessageStore_UpdateMessage(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("original"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
RetryCount: 0,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -327,7 +328,7 @@ func TestBadgerMessageStore_UpdateMessage(t *testing.T) {
|
||||
require.NoError(t, store.Enqueue(ctx, "$queue/test", msg))
|
||||
|
||||
// Update message
|
||||
msg.State = storage.StateRetry
|
||||
msg.State = types.StateRetry
|
||||
msg.RetryCount = 1
|
||||
msg.Payload = []byte("updated")
|
||||
|
||||
@@ -337,7 +338,7 @@ func TestBadgerMessageStore_UpdateMessage(t *testing.T) {
|
||||
// Verify update
|
||||
retrieved, err := store.GetMessage(ctx, "$queue/test", "msg-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, storage.StateRetry, retrieved.State)
|
||||
assert.Equal(t, types.StateRetry, retrieved.State)
|
||||
assert.Equal(t, 1, retrieved.RetryCount)
|
||||
assert.Equal(t, []byte("updated"), retrieved.Payload)
|
||||
}
|
||||
@@ -348,13 +349,13 @@ func TestBadgerMessageStore_DeleteMessage(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -390,13 +391,13 @@ func TestBadgerMessageStore_GetMessage(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrMessageNotFound)
|
||||
|
||||
// Create and get message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -436,31 +437,31 @@ func TestBadgerMessageStore_ListQueued(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue messages in different states
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("test-2"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg3 := &storage.Message{
|
||||
msg3 := &types.Message{
|
||||
ID: "msg-3",
|
||||
Payload: []byte("test-3"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 3,
|
||||
State: storage.StateDLQ, // Should not be listed
|
||||
State: types.StateDLQ, // Should not be listed
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -486,22 +487,22 @@ func TestBadgerMessageStore_ListRetry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue messages with different states
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("test-2"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -513,7 +514,7 @@ func TestBadgerMessageStore_ListRetry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, messages, 1)
|
||||
assert.Equal(t, "msg-2", messages[0].ID)
|
||||
assert.Equal(t, storage.StateRetry, messages[0].State)
|
||||
assert.Equal(t, types.StateRetry, messages[0].State)
|
||||
}
|
||||
|
||||
// Inflight Tests
|
||||
@@ -524,7 +525,7 @@ func TestBadgerMessageStore_MarkInflight(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -550,7 +551,7 @@ func TestBadgerMessageStore_GetInflight(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Mark multiple messages inflight
|
||||
state1 := &storage.DeliveryState{
|
||||
state1 := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -558,7 +559,7 @@ func TestBadgerMessageStore_GetInflight(t *testing.T) {
|
||||
DeliveredAt: time.Now(),
|
||||
Timeout: time.Now().Add(30 * time.Second),
|
||||
}
|
||||
state2 := &storage.DeliveryState{
|
||||
state2 := &types.DeliveryState{
|
||||
MessageID: "msg-2",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 1,
|
||||
@@ -587,7 +588,7 @@ func TestBadgerMessageStore_GetInflightMessage(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrMessageNotFound)
|
||||
|
||||
// Mark inflight
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -610,7 +611,7 @@ func TestBadgerMessageStore_RemoveInflight(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -638,13 +639,13 @@ func TestBadgerMessageStore_EnqueueDLQ(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed message"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "max retries exceeded",
|
||||
CreatedAt: time.Now(),
|
||||
MovedToDLQAt: time.Now(),
|
||||
@@ -668,19 +669,19 @@ func TestBadgerMessageStore_ListDLQ(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue multiple DLQ messages
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed-1"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "reason-1",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("failed-2"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "reason-2",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -705,11 +706,11 @@ func TestBadgerMessageStore_DeleteDLQMessage(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "test failure",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -771,7 +772,7 @@ func TestBadgerConsumerStore_RegisterConsumer(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -797,7 +798,7 @@ func TestBadgerConsumerStore_UnregisterConsumer(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -828,7 +829,7 @@ func TestBadgerConsumerStore_GetConsumer(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrConsumerNotFound)
|
||||
|
||||
// Register and get consumer
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -856,7 +857,7 @@ func TestBadgerConsumerStore_ListConsumers(t *testing.T) {
|
||||
assert.Len(t, consumers, 0)
|
||||
|
||||
// Register multiple consumers
|
||||
consumer1 := &storage.Consumer{
|
||||
consumer1 := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -864,7 +865,7 @@ func TestBadgerConsumerStore_ListConsumers(t *testing.T) {
|
||||
AssignedParts: []int{0},
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
consumer2 := &storage.Consumer{
|
||||
consumer2 := &types.Consumer{
|
||||
ID: "consumer-2",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -889,7 +890,7 @@ func TestBadgerConsumerStore_ListGroups(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Register consumers in different groups
|
||||
consumer1 := &storage.Consumer{
|
||||
consumer1 := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -897,7 +898,7 @@ func TestBadgerConsumerStore_ListGroups(t *testing.T) {
|
||||
AssignedParts: []int{0},
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
consumer2 := &storage.Consumer{
|
||||
consumer2 := &types.Consumer{
|
||||
ID: "consumer-2",
|
||||
GroupID: "group-2",
|
||||
QueueName: "$queue/test",
|
||||
@@ -929,7 +930,7 @@ func TestBadgerConsumerStore_UpdateHeartbeat(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -968,13 +969,13 @@ func TestBadgerStore_ConcurrentEnqueue(t *testing.T) {
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
go func(routineID int) {
|
||||
for j := 0; j < messagesPerGoroutine; j++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: filepath.Join("msg", string(rune(routineID)), string(rune(j))),
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: routineID % 3,
|
||||
Sequence: uint64(routineID*messagesPerGoroutine + j),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -1006,7 +1007,7 @@ func TestBadgerStore_ConcurrentConsumerRegistration(t *testing.T) {
|
||||
|
||||
for i := 0; i < numConsumers; i++ {
|
||||
go func(id int) {
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: filepath.Join("consumer", string(rune(id))),
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -1039,22 +1040,22 @@ func TestBadgerStore_MessageKeyFormat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Test that messages with different partition IDs don't collide
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("partition-0"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("partition-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 1,
|
||||
Sequence: 1, // Same sequence, different partition
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -1077,11 +1078,11 @@ func TestBadgerStore_DLQTopicPrefixHandling(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "test",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/memory/lockfree"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// PersistentStore combines all storage interfaces for the persistent layer.
|
||||
@@ -27,7 +28,7 @@ type PersistentStore interface {
|
||||
// - Overflow: BadgerDB when ring buffers are full
|
||||
// - Persistence: All messages written to BadgerDB for durability
|
||||
// - Recovery: Load recent messages from BadgerDB on startup
|
||||
// - Metadata: Inflight, DLQ, consumers stored in BadgerDB
|
||||
// - Metadata: Inflight, DLQ, consumers stored in BadgerDB.
|
||||
type Store struct {
|
||||
// Hot path: lock-free ring buffers
|
||||
lockfree *lockfree.Store
|
||||
@@ -43,7 +44,7 @@ type Store struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
// Async batch persistence
|
||||
batchBuffer []*storage.Message
|
||||
batchBuffer []*types.Message
|
||||
batchMu sync.Mutex
|
||||
flushTicker *time.Ticker
|
||||
stopCh chan struct{}
|
||||
@@ -121,7 +122,7 @@ func NewWithConfig(persistent PersistentStore, cfg Config) *Store {
|
||||
lockfree: lockfree.NewWithConfig(lockfreeConfig),
|
||||
persistent: persistent,
|
||||
config: cfg,
|
||||
batchBuffer: make([]*storage.Message, 0, cfg.PersistBatchSize),
|
||||
batchBuffer: make([]*types.Message, 0, cfg.PersistBatchSize),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
@@ -140,7 +141,7 @@ func NewWithConfig(persistent PersistentStore, cfg Config) *Store {
|
||||
}
|
||||
|
||||
// CreateQueue creates a new queue in both stores.
|
||||
func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) CreateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
// Create in persistent store first (source of truth)
|
||||
if err := s.persistent.CreateQueue(ctx, config); err != nil {
|
||||
return err
|
||||
@@ -157,12 +158,12 @@ func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) err
|
||||
}
|
||||
|
||||
// GetQueue retrieves queue configuration from persistent store.
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueConfig, error) {
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*types.QueueConfig, error) {
|
||||
return s.persistent.GetQueue(ctx, queueName)
|
||||
}
|
||||
|
||||
// UpdateQueue updates queue configuration in both stores.
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := s.persistent.UpdateQueue(ctx, config); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -176,7 +177,7 @@ func (s *Store) DeleteQueue(ctx context.Context, queueName string) error {
|
||||
}
|
||||
|
||||
// ListQueues lists queues from persistent store.
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]types.QueueConfig, error) {
|
||||
return s.persistent.ListQueues(ctx)
|
||||
}
|
||||
|
||||
@@ -184,8 +185,8 @@ func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
// 1. Validate queue exists
|
||||
// 2. Persist to BadgerDB (async batch if enabled, sync otherwise)
|
||||
// 3. Try lock-free ring buffer (best effort hot path)
|
||||
// 4. Track metrics for hit/miss rate
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
// 4. Track metrics for hit/miss rate.
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
// Validate queue exists
|
||||
_, err := s.GetQueue(ctx, queueName)
|
||||
if err != nil {
|
||||
@@ -193,7 +194,7 @@ func (s *Store) Enqueue(ctx context.Context, queueName string, msg *storage.Mess
|
||||
}
|
||||
|
||||
// Make a deep copy for storage (original might be reused by caller)
|
||||
msgCopy := &storage.Message{
|
||||
msgCopy := &types.Message{
|
||||
ID: msg.ID,
|
||||
Payload: make([]byte, len(msg.Payload)),
|
||||
Topic: msg.Topic,
|
||||
@@ -262,8 +263,8 @@ func (s *Store) Count(ctx context.Context, queueName string) (int64, error) {
|
||||
// 1. Validate queue exists
|
||||
// 2. Try lock-free ring buffer first (hot path)
|
||||
// 3. Mark as delivered in BadgerDB to avoid duplicates
|
||||
// 4. If ring buffer empty, read from BadgerDB (cold path)
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*storage.Message, error) {
|
||||
// 4. If ring buffer empty, read from BadgerDB (cold path).
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*types.Message, error) {
|
||||
// Validate queue exists
|
||||
_, err := s.GetQueue(ctx, queueName)
|
||||
if err != nil {
|
||||
@@ -274,7 +275,7 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
msg, err := s.lockfree.Dequeue(ctx, queueName, partitionID)
|
||||
if err == nil && msg != nil {
|
||||
// Ring buffer hit - mark as delivered in BadgerDB to avoid returning it again
|
||||
msg.State = storage.StateDelivered
|
||||
msg.State = types.StateDelivered
|
||||
s.persistent.UpdateMessage(ctx, queueName, msg)
|
||||
|
||||
if s.metrics != nil {
|
||||
@@ -293,7 +294,7 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
}
|
||||
|
||||
// Mark as delivered to avoid returning it again
|
||||
msg.State = storage.StateDelivered
|
||||
msg.State = types.StateDelivered
|
||||
s.persistent.UpdateMessage(ctx, queueName, msg)
|
||||
|
||||
if s.metrics != nil {
|
||||
@@ -313,9 +314,9 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
// 1. Drain lock-free ring buffer first
|
||||
// 2. Mark ring buffer messages as delivered in BadgerDB
|
||||
// 3. If needed, fetch remaining from BadgerDB
|
||||
// 4. Combine results
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
var messages []*storage.Message
|
||||
// 4. Combine results.
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
var messages []*types.Message
|
||||
|
||||
// Drain lock-free ring buffer first (hot path)
|
||||
ringMessages, err := s.lockfree.DequeueBatch(ctx, queueName, partitionID, limit)
|
||||
@@ -324,7 +325,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
|
||||
// Mark ring buffer messages as delivered in BadgerDB
|
||||
for _, msg := range ringMessages {
|
||||
msg.State = storage.StateDelivered
|
||||
msg.State = types.StateDelivered
|
||||
s.persistent.UpdateMessage(ctx, queueName, msg)
|
||||
}
|
||||
|
||||
@@ -357,7 +358,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
|
||||
// Mark BadgerDB messages as delivered
|
||||
for _, msg := range diskMessages {
|
||||
msg.State = storage.StateDelivered
|
||||
msg.State = types.StateDelivered
|
||||
s.persistent.UpdateMessage(ctx, queueName, msg)
|
||||
}
|
||||
|
||||
@@ -382,7 +383,7 @@ func (s *Store) GetNextSequence(ctx context.Context, queueName string, partition
|
||||
|
||||
// All metadata operations delegate to persistent store
|
||||
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
return s.persistent.UpdateMessage(ctx, queueName, msg)
|
||||
}
|
||||
|
||||
@@ -390,19 +391,19 @@ func (s *Store) DeleteMessage(ctx context.Context, queueName string, messageID s
|
||||
return s.persistent.DeleteMessage(ctx, queueName, messageID)
|
||||
}
|
||||
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*storage.Message, error) {
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*types.Message, error) {
|
||||
return s.persistent.GetMessage(ctx, queueName, messageID)
|
||||
}
|
||||
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *storage.DeliveryState) error {
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *types.DeliveryState) error {
|
||||
return s.persistent.MarkInflight(ctx, state)
|
||||
}
|
||||
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*types.DeliveryState, error) {
|
||||
return s.persistent.GetInflight(ctx, queueName)
|
||||
}
|
||||
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*types.DeliveryState, error) {
|
||||
return s.persistent.GetInflightMessage(ctx, queueName, messageID)
|
||||
}
|
||||
|
||||
@@ -410,11 +411,11 @@ func (s *Store) RemoveInflight(ctx context.Context, queueName, messageID string)
|
||||
return s.persistent.RemoveInflight(ctx, queueName, messageID)
|
||||
}
|
||||
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *storage.Message) error {
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *types.Message) error {
|
||||
return s.persistent.EnqueueDLQ(ctx, dlqTopic, msg)
|
||||
}
|
||||
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*types.Message, error) {
|
||||
return s.persistent.ListDLQ(ctx, dlqTopic, limit)
|
||||
}
|
||||
|
||||
@@ -422,7 +423,7 @@ func (s *Store) DeleteDLQMessage(ctx context.Context, dlqTopic, messageID string
|
||||
return s.persistent.DeleteDLQMessage(ctx, dlqTopic, messageID)
|
||||
}
|
||||
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error) {
|
||||
return s.persistent.ListRetry(ctx, queueName, partitionID)
|
||||
}
|
||||
|
||||
@@ -434,11 +435,11 @@ func (s *Store) GetOffset(ctx context.Context, queueName string, partitionID int
|
||||
return s.persistent.GetOffset(ctx, queueName, partitionID)
|
||||
}
|
||||
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
return s.persistent.ListQueued(ctx, queueName, partitionID, limit)
|
||||
}
|
||||
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *storage.Consumer) error {
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *types.Consumer) error {
|
||||
return s.persistent.RegisterConsumer(ctx, consumer)
|
||||
}
|
||||
|
||||
@@ -446,11 +447,11 @@ func (s *Store) UnregisterConsumer(ctx context.Context, queueName, groupID, cons
|
||||
return s.persistent.UnregisterConsumer(ctx, queueName, groupID, consumerID)
|
||||
}
|
||||
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*storage.Consumer, error) {
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*types.Consumer, error) {
|
||||
return s.persistent.GetConsumer(ctx, queueName, groupID, consumerID)
|
||||
}
|
||||
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*storage.Consumer, error) {
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*types.Consumer, error) {
|
||||
return s.persistent.ListConsumers(ctx, queueName, groupID)
|
||||
}
|
||||
|
||||
@@ -535,7 +536,7 @@ func (s *Store) flushBatch() {
|
||||
|
||||
// Swap buffer (minimize lock time)
|
||||
batch := s.batchBuffer
|
||||
s.batchBuffer = make([]*storage.Message, 0, s.config.PersistBatchSize)
|
||||
s.batchBuffer = make([]*types.Message, 0, s.config.PersistBatchSize)
|
||||
s.batchMu.Unlock()
|
||||
|
||||
// Write batch to BadgerDB (cross-partition batching)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/storage/badger"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
badgerdb "github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -61,7 +62,7 @@ func TestHybridStore_CreateQueue(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -78,7 +79,7 @@ func TestHybridStore_CreateQueue_Duplicate(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -93,18 +94,18 @@ func TestHybridStore_EnqueueDequeue(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue a message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -125,19 +126,19 @@ func TestHybridStore_HotPath_RingBufferHit(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue messages (should go to both stores)
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Payload: []byte(fmt.Sprintf("payload-%d", i)),
|
||||
PartitionID: 0,
|
||||
Sequence: uint64(i + 1),
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -169,18 +170,18 @@ func TestHybridStore_ColdPath_RingBufferEmpty(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue and immediately dequeue to empty ring buffer
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -190,13 +191,13 @@ func TestHybridStore_ColdPath_RingBufferEmpty(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue another message
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("test2"),
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg2)
|
||||
require.NoError(t, err)
|
||||
@@ -219,7 +220,7 @@ func TestHybridStore_Overflow_RingBufferFull(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -228,13 +229,13 @@ func TestHybridStore_Overflow_RingBufferFull(t *testing.T) {
|
||||
numMessages := 20 // Exceed capacity
|
||||
|
||||
for i := 0; i < numMessages; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Payload: []byte(fmt.Sprintf("payload-%d", i)),
|
||||
PartitionID: 0,
|
||||
Sequence: uint64(i + 1),
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -262,19 +263,19 @@ func TestHybridStore_DequeueBatch_FromRingBuffer(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue 10 messages
|
||||
for i := 0; i < 10; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Payload: []byte(fmt.Sprintf("payload-%d", i)),
|
||||
PartitionID: 0,
|
||||
Sequence: uint64(i + 1),
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -300,19 +301,19 @@ func TestHybridStore_DequeueBatch_Mixed(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enqueue messages to fill ring buffer
|
||||
for i := 0; i < 20; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("msg-%d", i),
|
||||
Payload: []byte(fmt.Sprintf("payload-%d", i)),
|
||||
PartitionID: 0,
|
||||
Sequence: uint64(i + 1),
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -320,7 +321,7 @@ func TestHybridStore_DequeueBatch_Mixed(t *testing.T) {
|
||||
|
||||
// Batch dequeue all messages
|
||||
// Should get some from ring buffer, rest from BadgerDB
|
||||
var allMessages []*storage.Message
|
||||
var allMessages []*types.Message
|
||||
for {
|
||||
messages, err := store.DequeueBatch(ctx, "$queue/test", 0, 5)
|
||||
require.NoError(t, err)
|
||||
@@ -346,7 +347,7 @@ func TestHybridStore_Metrics_HitRate(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -355,13 +356,13 @@ func TestHybridStore_Metrics_HitRate(t *testing.T) {
|
||||
assert.Equal(t, 0.0, hitRate)
|
||||
|
||||
// Enqueue and dequeue (hit)
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -379,18 +380,18 @@ func TestHybridStore_Metrics_ResetMetrics(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Generate some metrics
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -418,7 +419,7 @@ func TestHybridStore_MultiPartition(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
config.Partitions = 3
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
@@ -426,13 +427,13 @@ func TestHybridStore_MultiPartition(t *testing.T) {
|
||||
// Enqueue to different partitions
|
||||
for partition := 0; partition < 3; partition++ {
|
||||
for i := 0; i < 5; i++ {
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: fmt.Sprintf("p%d-msg-%d", partition, i),
|
||||
Payload: []byte(fmt.Sprintf("partition-%d-payload-%d", partition, i)),
|
||||
PartitionID: partition,
|
||||
Sequence: uint64(i + 1),
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
@@ -457,7 +458,7 @@ func TestHybridStore_EmptyQueue(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -479,13 +480,13 @@ func TestHybridStore_NonExistentQueue(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue to non-existent queue
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err := store.Enqueue(ctx, "$queue/nonexistent", msg)
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
@@ -502,24 +503,24 @@ func TestHybridStore_InflightTracking(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// First enqueue a message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
CreatedAt: time.Now(),
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/test", msg)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Mark message as inflight
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -558,12 +559,12 @@ func TestHybridStore_ConsumerManagement(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Register consumer
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
ClientID: "client-1",
|
||||
GroupID: "group-1",
|
||||
|
||||
@@ -6,7 +6,7 @@ package lockfree
|
||||
import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// RingBuffer is a lock-free SPSC (Single-Producer-Single-Consumer) ring buffer.
|
||||
@@ -18,9 +18,9 @@ import (
|
||||
// - Single consumer (dequeue operations)
|
||||
// - Fixed capacity (power of 2 for fast modulo)
|
||||
// - Cache-line padding to prevent false sharing
|
||||
// - Zero-copy (stores messages by value)
|
||||
// - Zero-copy (stores messages by value).
|
||||
type RingBuffer struct {
|
||||
buffer []storage.Message
|
||||
buffer []types.Message
|
||||
capacity uint64
|
||||
mask uint64 // capacity - 1, for fast modulo
|
||||
|
||||
@@ -45,7 +45,7 @@ func NewRingBuffer(capacity uint64) *RingBuffer {
|
||||
}
|
||||
|
||||
return &RingBuffer{
|
||||
buffer: make([]storage.Message, capacity),
|
||||
buffer: make([]types.Message, capacity),
|
||||
capacity: capacity,
|
||||
mask: capacity - 1,
|
||||
}
|
||||
@@ -57,8 +57,8 @@ func NewRingBuffer(capacity uint64) *RingBuffer {
|
||||
// This is lock-free because:
|
||||
// - Only one producer calls this
|
||||
// - Uses atomic loads/stores for synchronization
|
||||
// - No CAS loops needed (SPSC guarantees)
|
||||
func (rb *RingBuffer) Enqueue(msg storage.Message) bool {
|
||||
// - No CAS loops needed (SPSC guarantees).
|
||||
func (rb *RingBuffer) Enqueue(msg types.Message) bool {
|
||||
// Load current positions
|
||||
tail := atomic.LoadUint64(&rb.tail)
|
||||
head := atomic.LoadUint64(&rb.head)
|
||||
@@ -86,15 +86,15 @@ func (rb *RingBuffer) Enqueue(msg storage.Message) bool {
|
||||
// This is lock-free because:
|
||||
// - Only one consumer calls this
|
||||
// - Uses atomic loads/stores for synchronization
|
||||
// - No CAS loops needed (SPSC guarantees)
|
||||
func (rb *RingBuffer) Dequeue() (storage.Message, bool) {
|
||||
// - No CAS loops needed (SPSC guarantees).
|
||||
func (rb *RingBuffer) Dequeue() (types.Message, bool) {
|
||||
// Load current positions
|
||||
head := atomic.LoadUint64(&rb.head)
|
||||
tail := atomic.LoadUint64(&rb.tail)
|
||||
|
||||
// Check if buffer is empty
|
||||
if head >= tail {
|
||||
return storage.Message{}, false
|
||||
return types.Message{}, false
|
||||
}
|
||||
|
||||
// Read message from buffer
|
||||
@@ -113,8 +113,8 @@ func (rb *RingBuffer) Dequeue() (storage.Message, bool) {
|
||||
// This provides better performance than calling Dequeue() in a loop because:
|
||||
// - Fewer atomic operations (one head update for entire batch)
|
||||
// - Better cache utilization
|
||||
// - Reduced function call overhead
|
||||
func (rb *RingBuffer) DequeueBatch(limit int) []storage.Message {
|
||||
// - Reduced function call overhead.
|
||||
func (rb *RingBuffer) DequeueBatch(limit int) []types.Message {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -136,7 +136,7 @@ func (rb *RingBuffer) DequeueBatch(limit int) []storage.Message {
|
||||
}
|
||||
|
||||
// Pre-allocate result slice
|
||||
messages := make([]storage.Message, count)
|
||||
messages := make([]types.Message, count)
|
||||
|
||||
// Copy messages from ring buffer
|
||||
for i := uint64(0); i < count; i++ {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -41,7 +41,7 @@ func TestRingBuffer_EnqueueDequeue(t *testing.T) {
|
||||
rb := NewRingBuffer(4)
|
||||
|
||||
// Enqueue a message
|
||||
msg1 := storage.Message{
|
||||
msg1 := types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
Sequence: 1,
|
||||
@@ -64,7 +64,7 @@ func TestRingBuffer_FIFO(t *testing.T) {
|
||||
|
||||
// Enqueue multiple messages
|
||||
for i := 1; i <= 5; i++ {
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: string(rune('0' + i)),
|
||||
Sequence: uint64(i),
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func TestRingBuffer_Full(t *testing.T) {
|
||||
|
||||
// Fill the buffer (capacity - 1 because we keep one slot empty)
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := storage.Message{Sequence: uint64(i)}
|
||||
msg := types.Message{Sequence: uint64(i)}
|
||||
ok := rb.Enqueue(msg)
|
||||
require.True(t, ok)
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func TestRingBuffer_Full(t *testing.T) {
|
||||
assert.True(t, rb.IsFull())
|
||||
|
||||
// Enqueue should fail
|
||||
msg := storage.Message{Sequence: 999}
|
||||
msg := types.Message{Sequence: 999}
|
||||
ok := rb.Enqueue(msg)
|
||||
assert.False(t, ok)
|
||||
|
||||
@@ -119,7 +119,7 @@ func TestRingBuffer_Empty(t *testing.T) {
|
||||
assert.False(t, ok)
|
||||
|
||||
// Enqueue and dequeue
|
||||
msg := storage.Message{ID: "test"}
|
||||
msg := types.Message{ID: "test"}
|
||||
rb.Enqueue(msg)
|
||||
rb.Dequeue()
|
||||
|
||||
@@ -135,7 +135,7 @@ func TestRingBuffer_Wraparound(t *testing.T) {
|
||||
for round := 0; round < 10; round++ {
|
||||
// Fill buffer
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
Sequence: uint64(round*10 + i),
|
||||
}
|
||||
ok := rb.Enqueue(msg)
|
||||
@@ -158,7 +158,7 @@ func TestRingBuffer_DequeueBatch(t *testing.T) {
|
||||
|
||||
// Enqueue 10 messages
|
||||
for i := 1; i <= 10; i++ {
|
||||
msg := storage.Message{Sequence: uint64(i)}
|
||||
msg := types.Message{Sequence: uint64(i)}
|
||||
ok := rb.Enqueue(msg)
|
||||
require.True(t, ok)
|
||||
}
|
||||
@@ -189,7 +189,7 @@ func TestRingBuffer_DequeueBatch_Empty(t *testing.T) {
|
||||
|
||||
func TestRingBuffer_DequeueBatch_Zero(t *testing.T) {
|
||||
rb := NewRingBuffer(8)
|
||||
rb.Enqueue(storage.Message{ID: "test"})
|
||||
rb.Enqueue(types.Message{ID: "test"})
|
||||
|
||||
messages := rb.DequeueBatch(0)
|
||||
assert.Nil(t, messages)
|
||||
@@ -200,7 +200,7 @@ func TestRingBuffer_Reset(t *testing.T) {
|
||||
|
||||
// Add some messages
|
||||
for i := 0; i < 5; i++ {
|
||||
rb.Enqueue(storage.Message{Sequence: uint64(i)})
|
||||
rb.Enqueue(types.Message{Sequence: uint64(i)})
|
||||
}
|
||||
assert.Equal(t, 5, rb.Len())
|
||||
|
||||
@@ -210,7 +210,7 @@ func TestRingBuffer_Reset(t *testing.T) {
|
||||
assert.Equal(t, 0, rb.Len())
|
||||
|
||||
// Should be able to use after reset
|
||||
msg := storage.Message{ID: "after-reset"}
|
||||
msg := types.Message{ID: "after-reset"}
|
||||
ok := rb.Enqueue(msg)
|
||||
assert.True(t, ok)
|
||||
}
|
||||
@@ -226,7 +226,7 @@ func TestRingBuffer_ConcurrentSPSC(t *testing.T) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < numMessages; i++ {
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: "msg",
|
||||
Sequence: uint64(i),
|
||||
}
|
||||
@@ -272,7 +272,7 @@ func TestRingBuffer_ConcurrentBatchSPSC(t *testing.T) {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < numMessages; i++ {
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: "msg",
|
||||
Sequence: uint64(i),
|
||||
}
|
||||
@@ -307,10 +307,10 @@ func TestRingBuffer_ConcurrentBatchSPSC(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark tests
|
||||
// Benchmark tests.
|
||||
func BenchmarkRingBuffer_Enqueue(b *testing.B) {
|
||||
rb := NewRingBuffer(uint64(b.N))
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: "bench-msg",
|
||||
Payload: make([]byte, 100),
|
||||
Sequence: 1,
|
||||
@@ -324,7 +324,7 @@ func BenchmarkRingBuffer_Enqueue(b *testing.B) {
|
||||
|
||||
func BenchmarkRingBuffer_Dequeue(b *testing.B) {
|
||||
rb := NewRingBuffer(uint64(b.N))
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: "bench-msg",
|
||||
Payload: make([]byte, 100),
|
||||
Sequence: 1,
|
||||
@@ -343,7 +343,7 @@ func BenchmarkRingBuffer_Dequeue(b *testing.B) {
|
||||
|
||||
func BenchmarkRingBuffer_DequeueBatch(b *testing.B) {
|
||||
rb := NewRingBuffer(102400)
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: "bench-msg",
|
||||
Payload: make([]byte, 100),
|
||||
Sequence: 1,
|
||||
@@ -362,7 +362,7 @@ func BenchmarkRingBuffer_DequeueBatch(b *testing.B) {
|
||||
|
||||
func BenchmarkRingBuffer_SPSC_Parallel(b *testing.B) {
|
||||
rb := NewRingBuffer(4096)
|
||||
msg := storage.Message{
|
||||
msg := types.Message{
|
||||
ID: "bench-msg",
|
||||
Payload: make([]byte, 100),
|
||||
Sequence: 1,
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// Store is a lock-free message store using SPSC ring buffers per partition.
|
||||
@@ -21,7 +22,7 @@ type Store struct {
|
||||
|
||||
// queueBuffers holds all partition buffers for a single queue.
|
||||
type queueBuffers struct {
|
||||
config storage.QueueConfig
|
||||
config types.QueueConfig
|
||||
partitions []*partitionBuffer
|
||||
}
|
||||
|
||||
@@ -60,7 +61,7 @@ func NewWithConfig(cfg Config) *Store {
|
||||
}
|
||||
|
||||
// CreateQueue creates a new queue with the specified configuration.
|
||||
func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) CreateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -89,7 +90,7 @@ func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) err
|
||||
}
|
||||
|
||||
// GetQueue retrieves queue configuration.
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueConfig, error) {
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*types.QueueConfig, error) {
|
||||
val, exists := s.queues.Load(queueName)
|
||||
if !exists {
|
||||
return nil, storage.ErrQueueNotFound
|
||||
@@ -100,7 +101,7 @@ func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueC
|
||||
}
|
||||
|
||||
// UpdateQueue updates queue configuration.
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -122,8 +123,8 @@ func (s *Store) DeleteQueue(ctx context.Context, queueName string) error {
|
||||
}
|
||||
|
||||
// ListQueues returns all queue configurations.
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
var configs []storage.QueueConfig
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]types.QueueConfig, error) {
|
||||
var configs []types.QueueConfig
|
||||
|
||||
s.queues.Range(func(key, value interface{}) bool {
|
||||
qb := value.(*queueBuffers)
|
||||
@@ -135,7 +136,7 @@ func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
}
|
||||
|
||||
// Enqueue adds a message to a partition's ring buffer (lock-free).
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
val, exists := s.queues.Load(queueName)
|
||||
if !exists {
|
||||
return storage.ErrQueueNotFound
|
||||
@@ -157,7 +158,7 @@ func (s *Store) Enqueue(ctx context.Context, queueName string, msg *storage.Mess
|
||||
}
|
||||
|
||||
// Dequeue removes the next message from a partition (lock-free).
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*storage.Message, error) {
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*types.Message, error) {
|
||||
val, exists := s.queues.Load(queueName)
|
||||
if !exists {
|
||||
return nil, storage.ErrQueueNotFound
|
||||
@@ -180,7 +181,7 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
}
|
||||
|
||||
// DequeueBatch removes up to 'limit' messages from a partition (lock-free).
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
val, exists := s.queues.Load(queueName)
|
||||
if !exists {
|
||||
return nil, storage.ErrQueueNotFound
|
||||
@@ -200,7 +201,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
}
|
||||
|
||||
// Convert to pointers
|
||||
result := make([]*storage.Message, len(messages))
|
||||
result := make([]*types.Message, len(messages))
|
||||
for i := range messages {
|
||||
result[i] = &messages[i]
|
||||
}
|
||||
@@ -231,7 +232,7 @@ func (s *Store) GetNextSequence(ctx context.Context, queueName string, partition
|
||||
// They're part of the MessageStore interface but not in the hot path.
|
||||
|
||||
// UpdateMessage updates a message (not used in lock-free hot path).
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
// Lock-free ring buffers don't support in-place updates
|
||||
// Messages are consumed on dequeue
|
||||
return nil
|
||||
@@ -245,23 +246,23 @@ func (s *Store) DeleteMessage(ctx context.Context, queueName string, messageID s
|
||||
}
|
||||
|
||||
// GetMessage retrieves a specific message (not supported in lock-free store).
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*storage.Message, error) {
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*types.Message, error) {
|
||||
return nil, fmt.Errorf("GetMessage not supported in lock-free store")
|
||||
}
|
||||
|
||||
// MarkInflight tracks inflight messages (delegated to separate store).
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *storage.DeliveryState) error {
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *types.DeliveryState) error {
|
||||
// Inflight tracking delegated to hybrid store
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetInflight returns all inflight messages.
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*types.DeliveryState, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// GetInflightMessage retrieves a specific inflight message.
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*types.DeliveryState, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -271,13 +272,13 @@ func (s *Store) RemoveInflight(ctx context.Context, queueName, messageID string)
|
||||
}
|
||||
|
||||
// EnqueueDLQ adds a message to the dead-letter queue.
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *storage.Message) error {
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *types.Message) error {
|
||||
// DLQ delegated to hybrid store
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListDLQ lists messages in the dead-letter queue.
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*types.Message, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -287,7 +288,7 @@ func (s *Store) DeleteDLQMessage(ctx context.Context, dlqTopic, messageID string
|
||||
}
|
||||
|
||||
// ListRetry lists retry messages.
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -302,13 +303,13 @@ func (s *Store) GetOffset(ctx context.Context, queueName string, partitionID int
|
||||
}
|
||||
|
||||
// ListQueued lists queued messages.
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
// Could implement by peeking ring buffer without dequeuing
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// RegisterConsumer registers a consumer (delegated to separate store).
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *storage.Consumer) error {
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *types.Consumer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -318,12 +319,12 @@ func (s *Store) UnregisterConsumer(ctx context.Context, queueName, groupID, cons
|
||||
}
|
||||
|
||||
// GetConsumer retrieves consumer information.
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*storage.Consumer, error) {
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*types.Consumer, error) {
|
||||
return nil, storage.ErrConsumerNotFound
|
||||
}
|
||||
|
||||
// ListConsumers lists all consumers in a group.
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*storage.Consumer, error) {
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*types.Consumer, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -11,30 +11,31 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
// Store implements all queue storage interfaces using in-memory maps.
|
||||
// This implementation is primarily for testing and development.
|
||||
type Store struct {
|
||||
queues map[string]storage.QueueConfig
|
||||
messages map[string]map[int]map[uint64]*storage.Message // queueName -> partitionID -> sequence -> message
|
||||
inflight map[string]map[string]*storage.DeliveryState // queueName -> messageID -> state
|
||||
dlq map[string]map[string]*storage.Message // dlqTopic -> messageID -> message
|
||||
consumers map[string]map[string]map[string]*storage.Consumer // queueName -> groupID -> consumerID -> consumer
|
||||
sequences map[string]map[int]uint64 // queueName -> partitionID -> nextSeq
|
||||
offsets map[string]map[int]uint64 // queueName -> partitionID -> offset
|
||||
counts map[string]int64 // queueName -> message count (for O(1) Count())
|
||||
queues map[string]types.QueueConfig
|
||||
messages map[string]map[int]map[uint64]*types.Message // queueName -> partitionID -> sequence -> message
|
||||
inflight map[string]map[string]*types.DeliveryState // queueName -> messageID -> state
|
||||
dlq map[string]map[string]*types.Message // dlqTopic -> messageID -> message
|
||||
consumers map[string]map[string]map[string]*types.Consumer // queueName -> groupID -> consumerID -> consumer
|
||||
sequences map[string]map[int]uint64 // queueName -> partitionID -> nextSeq
|
||||
offsets map[string]map[int]uint64 // queueName -> partitionID -> offset
|
||||
counts map[string]int64 // queueName -> message count (for O(1) Count())
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// New creates a new in-memory queue store.
|
||||
func New() *Store {
|
||||
return &Store{
|
||||
queues: make(map[string]storage.QueueConfig),
|
||||
messages: make(map[string]map[int]map[uint64]*storage.Message),
|
||||
inflight: make(map[string]map[string]*storage.DeliveryState),
|
||||
dlq: make(map[string]map[string]*storage.Message),
|
||||
consumers: make(map[string]map[string]map[string]*storage.Consumer),
|
||||
queues: make(map[string]types.QueueConfig),
|
||||
messages: make(map[string]map[int]map[uint64]*types.Message),
|
||||
inflight: make(map[string]map[string]*types.DeliveryState),
|
||||
dlq: make(map[string]map[string]*types.Message),
|
||||
consumers: make(map[string]map[string]map[string]*types.Consumer),
|
||||
sequences: make(map[string]map[int]uint64),
|
||||
offsets: make(map[string]map[int]uint64),
|
||||
counts: make(map[string]int64),
|
||||
@@ -43,7 +44,7 @@ func New() *Store {
|
||||
|
||||
// QueueStore implementation
|
||||
|
||||
func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) CreateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -56,16 +57,16 @@ func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) err
|
||||
}
|
||||
|
||||
s.queues[config.Name] = config
|
||||
s.messages[config.Name] = make(map[int]map[uint64]*storage.Message)
|
||||
s.inflight[config.Name] = make(map[string]*storage.DeliveryState)
|
||||
s.consumers[config.Name] = make(map[string]map[string]*storage.Consumer)
|
||||
s.messages[config.Name] = make(map[int]map[uint64]*types.Message)
|
||||
s.inflight[config.Name] = make(map[string]*types.DeliveryState)
|
||||
s.consumers[config.Name] = make(map[string]map[string]*types.Consumer)
|
||||
s.sequences[config.Name] = make(map[int]uint64)
|
||||
s.offsets[config.Name] = make(map[int]uint64)
|
||||
s.counts[config.Name] = 0 // Initialize message counter
|
||||
|
||||
// Initialize partitions
|
||||
for i := 0; i < config.Partitions; i++ {
|
||||
s.messages[config.Name][i] = make(map[uint64]*storage.Message)
|
||||
s.messages[config.Name][i] = make(map[uint64]*types.Message)
|
||||
s.sequences[config.Name][i] = 0
|
||||
s.offsets[config.Name][i] = 0
|
||||
}
|
||||
@@ -73,7 +74,7 @@ func (s *Store) CreateQueue(ctx context.Context, config storage.QueueConfig) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueConfig, error) {
|
||||
func (s *Store) GetQueue(ctx context.Context, queueName string) (*types.QueueConfig, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -86,7 +87,7 @@ func (s *Store) GetQueue(ctx context.Context, queueName string) (*storage.QueueC
|
||||
return &configCopy, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config storage.QueueConfig) error {
|
||||
func (s *Store) UpdateQueue(ctx context.Context, config types.QueueConfig) error {
|
||||
if err := config.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -120,11 +121,11 @@ func (s *Store) DeleteQueue(ctx context.Context, queueName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
func (s *Store) ListQueues(ctx context.Context) ([]types.QueueConfig, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
configs := make([]storage.QueueConfig, 0, len(s.queues))
|
||||
configs := make([]types.QueueConfig, 0, len(s.queues))
|
||||
for _, config := range s.queues {
|
||||
configs = append(configs, config)
|
||||
}
|
||||
@@ -134,7 +135,7 @@ func (s *Store) ListQueues(ctx context.Context) ([]storage.QueueConfig, error) {
|
||||
|
||||
// MessageStore implementation
|
||||
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
func (s *Store) Enqueue(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -169,14 +170,14 @@ func (s *Store) Count(ctx context.Context, queueName string) (int64, error) {
|
||||
}
|
||||
|
||||
// deepCopyMessage creates a deep copy of a message including Properties map.
|
||||
func (s *Store) deepCopyMessage(msg *storage.Message) *storage.Message {
|
||||
func (s *Store) deepCopyMessage(msg *types.Message) *types.Message {
|
||||
// Copy properties map
|
||||
props := make(map[string]string, len(msg.Properties))
|
||||
for k, v := range msg.Properties {
|
||||
props[k] = v
|
||||
}
|
||||
|
||||
return &storage.Message{
|
||||
return &types.Message{
|
||||
ID: msg.ID,
|
||||
Payload: msg.Payload,
|
||||
Topic: msg.Topic,
|
||||
@@ -197,7 +198,7 @@ func (s *Store) deepCopyMessage(msg *storage.Message) *storage.Message {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*storage.Message, error) {
|
||||
func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int) (*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -213,12 +214,12 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
|
||||
// Find first queued or ready-to-retry message (ordered by sequence)
|
||||
var minSeq uint64 = ^uint64(0)
|
||||
var msg *storage.Message
|
||||
var msg *types.Message
|
||||
|
||||
for seq, m := range partition {
|
||||
if seq < minSeq {
|
||||
if m.State == storage.StateQueued ||
|
||||
(m.State == storage.StateRetry && time.Now().After(m.NextRetryAt)) {
|
||||
if m.State == types.StateQueued ||
|
||||
(m.State == types.StateRetry && time.Now().After(m.NextRetryAt)) {
|
||||
minSeq = seq
|
||||
msg = m
|
||||
}
|
||||
@@ -233,7 +234,7 @@ func (s *Store) Dequeue(ctx context.Context, queueName string, partitionID int)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -250,14 +251,14 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
// Collect all sequences for available messages
|
||||
type seqMsg struct {
|
||||
seq uint64
|
||||
msg *storage.Message
|
||||
msg *types.Message
|
||||
}
|
||||
var available []seqMsg
|
||||
|
||||
now := time.Now()
|
||||
for seq, m := range partition {
|
||||
if m.State == storage.StateQueued ||
|
||||
(m.State == storage.StateRetry && now.After(m.NextRetryAt)) {
|
||||
if m.State == types.StateQueued ||
|
||||
(m.State == types.StateRetry && now.After(m.NextRetryAt)) {
|
||||
available = append(available, seqMsg{seq, m})
|
||||
}
|
||||
}
|
||||
@@ -277,7 +278,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
count = limit
|
||||
}
|
||||
|
||||
messages := make([]*storage.Message, 0, count)
|
||||
messages := make([]*types.Message, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
msgCopy := *available[i].msg
|
||||
messages = append(messages, &msgCopy)
|
||||
@@ -286,7 +287,7 @@ func (s *Store) DequeueBatch(ctx context.Context, queueName string, partitionID
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *storage.Message) error {
|
||||
func (s *Store) UpdateMessage(ctx context.Context, queueName string, msg *types.Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -328,7 +329,7 @@ func (s *Store) DeleteMessage(ctx context.Context, queueName string, messageID s
|
||||
return storage.ErrMessageNotFound
|
||||
}
|
||||
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*storage.Message, error) {
|
||||
func (s *Store) GetMessage(ctx context.Context, queueName string, messageID string) (*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -350,7 +351,7 @@ func (s *Store) GetMessage(ctx context.Context, queueName string, messageID stri
|
||||
return nil, storage.ErrMessageNotFound
|
||||
}
|
||||
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *storage.DeliveryState) error {
|
||||
func (s *Store) MarkInflight(ctx context.Context, state *types.DeliveryState) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -364,7 +365,7 @@ func (s *Store) MarkInflight(ctx context.Context, state *storage.DeliveryState)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*types.DeliveryState, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -373,7 +374,7 @@ func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.D
|
||||
return nil, storage.ErrQueueNotFound
|
||||
}
|
||||
|
||||
states := make([]*storage.DeliveryState, 0, len(inflight))
|
||||
states := make([]*types.DeliveryState, 0, len(inflight))
|
||||
for _, state := range inflight {
|
||||
stateCopy := *state
|
||||
states = append(states, &stateCopy)
|
||||
@@ -382,7 +383,7 @@ func (s *Store) GetInflight(ctx context.Context, queueName string) ([]*storage.D
|
||||
return states, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*storage.DeliveryState, error) {
|
||||
func (s *Store) GetInflightMessage(ctx context.Context, queueName, messageID string) (*types.DeliveryState, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -413,12 +414,12 @@ func (s *Store) RemoveInflight(ctx context.Context, queueName, messageID string)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *storage.Message) error {
|
||||
func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *types.Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.dlq[dlqTopic] == nil {
|
||||
s.dlq[dlqTopic] = make(map[string]*storage.Message)
|
||||
s.dlq[dlqTopic] = make(map[string]*types.Message)
|
||||
}
|
||||
|
||||
msgCopy := *msg
|
||||
@@ -426,11 +427,11 @@ func (s *Store) EnqueueDLQ(ctx context.Context, dlqTopic string, msg *storage.Me
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
messages := make([]*storage.Message, 0)
|
||||
messages := make([]*types.Message, 0)
|
||||
dlqMessages, exists := s.dlq[dlqTopic]
|
||||
if !exists {
|
||||
return messages, nil
|
||||
@@ -466,11 +467,11 @@ func (s *Store) DeleteDLQMessage(ctx context.Context, dlqTopic, messageID string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
messages := make([]*storage.Message, 0)
|
||||
messages := make([]*types.Message, 0)
|
||||
partitions, exists := s.messages[queueName]
|
||||
if !exists {
|
||||
return messages, nil
|
||||
@@ -483,7 +484,7 @@ func (s *Store) ListRetry(ctx context.Context, queueName string, partitionID int
|
||||
|
||||
// Iterate through partition messages and collect those in retry state
|
||||
for _, msg := range partition {
|
||||
if msg.State == storage.StateRetry {
|
||||
if msg.State == types.StateRetry {
|
||||
msgCopy := *msg
|
||||
messages = append(messages, &msgCopy)
|
||||
}
|
||||
@@ -532,7 +533,7 @@ func (s *Store) GetOffset(ctx context.Context, queueName string, partitionID int
|
||||
return offsets[partitionID], nil
|
||||
}
|
||||
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -546,13 +547,13 @@ func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID in
|
||||
return nil, fmt.Errorf("partition %d not found", partitionID)
|
||||
}
|
||||
|
||||
messages := make([]*storage.Message, 0)
|
||||
messages := make([]*types.Message, 0)
|
||||
count := 0
|
||||
for _, msg := range partition {
|
||||
if limit > 0 && count >= limit {
|
||||
break
|
||||
}
|
||||
if msg.State == storage.StateQueued || msg.State == storage.StateRetry {
|
||||
if msg.State == types.StateQueued || msg.State == types.StateRetry {
|
||||
msgCopy := *msg
|
||||
messages = append(messages, &msgCopy)
|
||||
count++
|
||||
@@ -564,19 +565,19 @@ func (s *Store) ListQueued(ctx context.Context, queueName string, partitionID in
|
||||
|
||||
// ConsumerStore implementation
|
||||
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *storage.Consumer) error {
|
||||
func (s *Store) RegisterConsumer(ctx context.Context, consumer *types.Consumer) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
groups, exists := s.consumers[consumer.QueueName]
|
||||
if !exists {
|
||||
groups = make(map[string]map[string]*storage.Consumer)
|
||||
groups = make(map[string]map[string]*types.Consumer)
|
||||
s.consumers[consumer.QueueName] = groups
|
||||
}
|
||||
|
||||
group, exists := groups[consumer.GroupID]
|
||||
if !exists {
|
||||
group = make(map[string]*storage.Consumer)
|
||||
group = make(map[string]*types.Consumer)
|
||||
groups[consumer.GroupID] = group
|
||||
}
|
||||
|
||||
@@ -603,7 +604,7 @@ func (s *Store) UnregisterConsumer(ctx context.Context, queueName, groupID, cons
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*storage.Consumer, error) {
|
||||
func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*types.Consumer, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -626,7 +627,7 @@ func (s *Store) GetConsumer(ctx context.Context, queueName, groupID, consumerID
|
||||
return &consumerCopy, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*storage.Consumer, error) {
|
||||
func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([]*types.Consumer, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -637,10 +638,10 @@ func (s *Store) ListConsumers(ctx context.Context, queueName, groupID string) ([
|
||||
|
||||
group, exists := groups[groupID]
|
||||
if !exists {
|
||||
return []*storage.Consumer{}, nil
|
||||
return []*types.Consumer{}, nil
|
||||
}
|
||||
|
||||
consumers := make([]*storage.Consumer, 0, len(group))
|
||||
consumers := make([]*types.Consumer, 0, len(group))
|
||||
for _, consumer := range group {
|
||||
consumerCopy := *consumer
|
||||
consumers = append(consumers, &consumerCopy)
|
||||
@@ -691,7 +692,7 @@ func (s *Store) UpdateHeartbeat(ctx context.Context, queueName, groupID, consume
|
||||
|
||||
// Retention operations
|
||||
|
||||
func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partitionID int, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -702,11 +703,11 @@ func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partit
|
||||
|
||||
partition, exists := partitions[partitionID]
|
||||
if !exists {
|
||||
return []*storage.Message{}, nil
|
||||
return []*types.Message{}, nil
|
||||
}
|
||||
|
||||
// Collect all messages from partition
|
||||
messages := make([]*storage.Message, 0, len(partition))
|
||||
messages := make([]*types.Message, 0, len(partition))
|
||||
for _, msg := range partition {
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
@@ -724,7 +725,7 @@ func (s *Store) ListOldestMessages(ctx context.Context, queueName string, partit
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListMessagesBefore(ctx context.Context, queueName string, partitionID int, cutoffTime time.Time, limit int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListMessagesBefore(ctx context.Context, queueName string, partitionID int, cutoffTime time.Time, limit int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -735,11 +736,11 @@ func (s *Store) ListMessagesBefore(ctx context.Context, queueName string, partit
|
||||
|
||||
partition, exists := partitions[partitionID]
|
||||
if !exists {
|
||||
return []*storage.Message{}, nil
|
||||
return []*types.Message{}, nil
|
||||
}
|
||||
|
||||
// Collect messages older than cutoff time
|
||||
messages := make([]*storage.Message, 0)
|
||||
messages := make([]*types.Message, 0)
|
||||
for _, msg := range partition {
|
||||
if msg.CreatedAt.Before(cutoffTime) {
|
||||
messages = append(messages, msg)
|
||||
@@ -817,7 +818,7 @@ func (s *Store) GetQueueSize(ctx context.Context, queueName string) (int64, erro
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListAllMessages(ctx context.Context, queueName string, partitionID int) ([]*storage.Message, error) {
|
||||
func (s *Store) ListAllMessages(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
@@ -828,11 +829,11 @@ func (s *Store) ListAllMessages(ctx context.Context, queueName string, partition
|
||||
|
||||
partition, exists := partitions[partitionID]
|
||||
if !exists {
|
||||
return []*storage.Message{}, nil
|
||||
return []*types.Message{}, nil
|
||||
}
|
||||
|
||||
// Collect all messages from partition
|
||||
messages := make([]*storage.Message, 0, len(partition))
|
||||
messages := make([]*types.Message, 0, len(partition))
|
||||
for _, msg := range partition {
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/queue/storage"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -20,7 +21,7 @@ func TestMemoryQueueStore_CreateQueue(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -35,7 +36,7 @@ func TestMemoryQueueStore_CreateQueue_Duplicate(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -49,7 +50,7 @@ func TestMemoryQueueStore_CreateQueue_InvalidConfig(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Invalid config (empty name)
|
||||
config := storage.QueueConfig{Name: ""}
|
||||
config := types.QueueConfig{Name: ""}
|
||||
err := store.CreateQueue(ctx, config)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -63,7 +64,7 @@ func TestMemoryQueueStore_GetQueue(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
|
||||
// Create and get queue
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -77,7 +78,7 @@ func TestMemoryQueueStore_UpdateQueue(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Update non-existent queue
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err := store.UpdateQueue(ctx, config)
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
|
||||
@@ -104,7 +105,7 @@ func TestMemoryQueueStore_DeleteQueue(t *testing.T) {
|
||||
err := store.DeleteQueue(ctx, "$queue/nonexistent")
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
err = store.CreateQueue(ctx, config)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -127,9 +128,9 @@ func TestMemoryQueueStore_ListQueues(t *testing.T) {
|
||||
assert.Len(t, queues, 0)
|
||||
|
||||
// Create multiple queues
|
||||
config1 := storage.DefaultQueueConfig("$queue/test1")
|
||||
config2 := storage.DefaultQueueConfig("$queue/test2")
|
||||
config3 := storage.DefaultQueueConfig("$queue/test3")
|
||||
config1 := types.DefaultQueueConfig("$queue/test1")
|
||||
config2 := types.DefaultQueueConfig("$queue/test2")
|
||||
config3 := types.DefaultQueueConfig("$queue/test3")
|
||||
|
||||
require.NoError(t, store.CreateQueue(ctx, config1))
|
||||
require.NoError(t, store.CreateQueue(ctx, config2))
|
||||
@@ -157,16 +158,16 @@ func TestMemoryMessageStore_Enqueue(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create queue first
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test payload"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -184,13 +185,13 @@ func TestMemoryMessageStore_Enqueue_InvalidQueue(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/nonexistent",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -203,26 +204,26 @@ func TestMemoryMessageStore_Dequeue(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create queue
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Enqueue multiple messages
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("payload-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("payload-2"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -244,17 +245,17 @@ func TestMemoryMessageStore_Dequeue_RetryReady(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Message ready for retry (NextRetryAt in the past)
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
NextRetryAt: time.Now().Add(-1 * time.Second), // Past
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -272,17 +273,17 @@ func TestMemoryMessageStore_Dequeue_RetryNotReady(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Message not ready for retry (NextRetryAt in the future)
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
NextRetryAt: time.Now().Add(1 * time.Hour), // Future
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -299,16 +300,16 @@ func TestMemoryMessageStore_UpdateMessage(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("original"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
RetryCount: 0,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -316,7 +317,7 @@ func TestMemoryMessageStore_UpdateMessage(t *testing.T) {
|
||||
require.NoError(t, store.Enqueue(ctx, "$queue/test", msg))
|
||||
|
||||
// Update message
|
||||
msg.State = storage.StateRetry
|
||||
msg.State = types.StateRetry
|
||||
msg.RetryCount = 1
|
||||
msg.Payload = []byte("updated")
|
||||
|
||||
@@ -326,7 +327,7 @@ func TestMemoryMessageStore_UpdateMessage(t *testing.T) {
|
||||
// Verify update
|
||||
retrieved, err := store.GetMessage(ctx, "$queue/test", "msg-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, storage.StateRetry, retrieved.State)
|
||||
assert.Equal(t, types.StateRetry, retrieved.State)
|
||||
assert.Equal(t, 1, retrieved.RetryCount)
|
||||
assert.Equal(t, []byte("updated"), retrieved.Payload)
|
||||
}
|
||||
@@ -335,16 +336,16 @@ func TestMemoryMessageStore_DeleteMessage(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -363,7 +364,7 @@ func TestMemoryMessageStore_DeleteMessage_NotFound(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
err := store.DeleteMessage(ctx, "$queue/test", "nonexistent")
|
||||
@@ -374,7 +375,7 @@ func TestMemoryMessageStore_GetMessage(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Get non-existent message
|
||||
@@ -382,13 +383,13 @@ func TestMemoryMessageStore_GetMessage(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrMessageNotFound)
|
||||
|
||||
// Create and get message
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -403,7 +404,7 @@ func TestMemoryMessageStore_GetNextSequence(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// First sequence should be 1
|
||||
@@ -426,35 +427,35 @@ func TestMemoryMessageStore_ListQueued(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Enqueue messages in different states
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("test-2"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg3 := &storage.Message{
|
||||
msg3 := &types.Message{
|
||||
ID: "msg-3",
|
||||
Payload: []byte("test-3"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 3,
|
||||
State: storage.StateDLQ, // Should not be listed
|
||||
State: types.StateDLQ, // Should not be listed
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -477,26 +478,26 @@ func TestMemoryMessageStore_ListRetry(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Enqueue messages with different states
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test-1"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("test-2"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 2,
|
||||
State: storage.StateRetry,
|
||||
State: types.StateRetry,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
@@ -508,7 +509,7 @@ func TestMemoryMessageStore_ListRetry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, messages, 1)
|
||||
assert.Equal(t, "msg-2", messages[0].ID)
|
||||
assert.Equal(t, storage.StateRetry, messages[0].State)
|
||||
assert.Equal(t, types.StateRetry, messages[0].State)
|
||||
}
|
||||
|
||||
// Inflight Tests
|
||||
@@ -517,10 +518,10 @@ func TestMemoryMessageStore_MarkInflight(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -543,11 +544,11 @@ func TestMemoryMessageStore_GetInflight(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Mark multiple messages inflight
|
||||
state1 := &storage.DeliveryState{
|
||||
state1 := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -555,7 +556,7 @@ func TestMemoryMessageStore_GetInflight(t *testing.T) {
|
||||
DeliveredAt: time.Now(),
|
||||
Timeout: time.Now().Add(30 * time.Second),
|
||||
}
|
||||
state2 := &storage.DeliveryState{
|
||||
state2 := &types.DeliveryState{
|
||||
MessageID: "msg-2",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 1,
|
||||
@@ -577,7 +578,7 @@ func TestMemoryMessageStore_GetInflightMessage(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Get non-existent inflight message
|
||||
@@ -585,7 +586,7 @@ func TestMemoryMessageStore_GetInflightMessage(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrMessageNotFound)
|
||||
|
||||
// Mark inflight
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -606,10 +607,10 @@ func TestMemoryMessageStore_RemoveInflight(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
state := &storage.DeliveryState{
|
||||
state := &types.DeliveryState{
|
||||
MessageID: "msg-1",
|
||||
QueueName: "$queue/test",
|
||||
PartitionID: 0,
|
||||
@@ -635,13 +636,13 @@ func TestMemoryMessageStore_EnqueueDLQ(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed message"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "max retries exceeded",
|
||||
CreatedAt: time.Now(),
|
||||
MovedToDLQAt: time.Now(),
|
||||
@@ -663,19 +664,19 @@ func TestMemoryMessageStore_ListDLQ(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Enqueue multiple DLQ messages
|
||||
msg1 := &storage.Message{
|
||||
msg1 := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed-1"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "reason-1",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
msg2 := &storage.Message{
|
||||
msg2 := &types.Message{
|
||||
ID: "msg-2",
|
||||
Payload: []byte("failed-2"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "reason-2",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -698,11 +699,11 @@ func TestMemoryMessageStore_DeleteDLQMessage(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("failed"),
|
||||
Topic: "$queue/test",
|
||||
State: storage.StateDLQ,
|
||||
State: types.StateDLQ,
|
||||
FailureReason: "test failure",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
@@ -725,7 +726,7 @@ func TestMemoryMessageStore_UpdateOffset(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
err := store.UpdateOffset(ctx, "$queue/test", 0, 100)
|
||||
@@ -741,7 +742,7 @@ func TestMemoryMessageStore_GetOffset(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Get offset for partition (should return 0 initially)
|
||||
@@ -764,10 +765,10 @@ func TestMemoryConsumerStore_RegisterConsumer(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -791,7 +792,7 @@ func TestMemoryConsumerStore_RegisterConsumer_NoQueue(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/nonexistent",
|
||||
@@ -809,10 +810,10 @@ func TestMemoryConsumerStore_UnregisterConsumer(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -836,7 +837,7 @@ func TestMemoryConsumerStore_GetConsumer(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Get non-existent consumer
|
||||
@@ -844,7 +845,7 @@ func TestMemoryConsumerStore_GetConsumer(t *testing.T) {
|
||||
assert.ErrorIs(t, err, storage.ErrConsumerNotFound)
|
||||
|
||||
// Register and get consumer
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -864,7 +865,7 @@ func TestMemoryConsumerStore_ListConsumers(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Empty list
|
||||
@@ -873,7 +874,7 @@ func TestMemoryConsumerStore_ListConsumers(t *testing.T) {
|
||||
assert.Len(t, consumers, 0)
|
||||
|
||||
// Register multiple consumers
|
||||
consumer1 := &storage.Consumer{
|
||||
consumer1 := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -881,7 +882,7 @@ func TestMemoryConsumerStore_ListConsumers(t *testing.T) {
|
||||
AssignedParts: []int{0},
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
consumer2 := &storage.Consumer{
|
||||
consumer2 := &types.Consumer{
|
||||
ID: "consumer-2",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -903,11 +904,11 @@ func TestMemoryConsumerStore_ListGroups(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Register consumers in different groups
|
||||
consumer1 := &storage.Consumer{
|
||||
consumer1 := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -915,7 +916,7 @@ func TestMemoryConsumerStore_ListGroups(t *testing.T) {
|
||||
AssignedParts: []int{0},
|
||||
LastHeartbeat: time.Now(),
|
||||
}
|
||||
consumer2 := &storage.Consumer{
|
||||
consumer2 := &types.Consumer{
|
||||
ID: "consumer-2",
|
||||
GroupID: "group-2",
|
||||
QueueName: "$queue/test",
|
||||
@@ -945,10 +946,10 @@ func TestMemoryConsumerStore_UpdateHeartbeat(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: "consumer-1",
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -976,7 +977,7 @@ func TestMemoryStore_ConcurrentEnqueue(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Enqueue messages concurrently
|
||||
@@ -990,13 +991,13 @@ func TestMemoryStore_ConcurrentEnqueue(t *testing.T) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < messagesPerGoroutine; j++ {
|
||||
seq, _ := store.GetNextSequence(ctx, "$queue/test", routineID%3)
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: string(rune(routineID*1000 + j)),
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/test",
|
||||
PartitionID: routineID % 3,
|
||||
Sequence: seq,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
store.Enqueue(ctx, "$queue/test", msg)
|
||||
@@ -1016,7 +1017,7 @@ func TestMemoryStore_ConcurrentConsumerRegistration(t *testing.T) {
|
||||
store := New()
|
||||
ctx := context.Background()
|
||||
|
||||
config := storage.DefaultQueueConfig("$queue/test")
|
||||
config := types.DefaultQueueConfig("$queue/test")
|
||||
require.NoError(t, store.CreateQueue(ctx, config))
|
||||
|
||||
// Register consumers concurrently
|
||||
@@ -1027,7 +1028,7 @@ func TestMemoryStore_ConcurrentConsumerRegistration(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
consumer := &storage.Consumer{
|
||||
consumer := &types.Consumer{
|
||||
ID: string(rune(id)),
|
||||
GroupID: "group-1",
|
||||
QueueName: "$queue/test",
|
||||
@@ -1060,13 +1061,13 @@ func TestMemoryStore_ErrorHandling_NonExistentQueue(t *testing.T) {
|
||||
err = store.DeleteQueue(ctx, "$queue/nonexistent")
|
||||
assert.ErrorIs(t, err, storage.ErrQueueNotFound)
|
||||
|
||||
msg := &storage.Message{
|
||||
msg := &types.Message{
|
||||
ID: "msg-1",
|
||||
Payload: []byte("test"),
|
||||
Topic: "$queue/nonexistent",
|
||||
PartitionID: 0,
|
||||
Sequence: 1,
|
||||
State: storage.StateQueued,
|
||||
State: types.StateQueued,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
err = store.Enqueue(ctx, "$queue/nonexistent", msg)
|
||||
|
||||
+23
-341
@@ -6,10 +6,9 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/core"
|
||||
"github.com/absmach/fluxmq/queue/types"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -17,251 +16,42 @@ var (
|
||||
ErrMessageNotFound = errors.New("message not found")
|
||||
ErrConsumerNotFound = errors.New("consumer not found")
|
||||
ErrQueueAlreadyExists = errors.New("queue already exists")
|
||||
ErrInvalidConfig = errors.New("invalid queue configuration")
|
||||
)
|
||||
|
||||
// OrderingMode defines message ordering guarantees.
|
||||
type OrderingMode string
|
||||
|
||||
const (
|
||||
OrderingNone OrderingMode = "none" // No ordering guarantees
|
||||
OrderingPartition OrderingMode = "partition" // FIFO per partition key
|
||||
OrderingStrict OrderingMode = "strict" // Global FIFO (single partition)
|
||||
)
|
||||
|
||||
// MessageState represents the lifecycle state of a queue message.
|
||||
type MessageState string
|
||||
|
||||
const (
|
||||
StateQueued MessageState = "queued"
|
||||
StateDelivered MessageState = "delivered"
|
||||
StateAcked MessageState = "acked"
|
||||
StateRetry MessageState = "retry"
|
||||
StateDLQ MessageState = "dlq"
|
||||
)
|
||||
|
||||
// ReplicationMode defines the replication behavior for queue messages.
|
||||
type ReplicationMode string
|
||||
|
||||
const (
|
||||
ReplicationSync ReplicationMode = "sync" // Wait for quorum ACK before returning
|
||||
ReplicationAsync ReplicationMode = "async" // Return immediately after leader accepts
|
||||
)
|
||||
|
||||
// PlacementStrategy defines how replicas are assigned to nodes.
|
||||
type PlacementStrategy string
|
||||
|
||||
const (
|
||||
PlacementRoundRobin PlacementStrategy = "round-robin" // Distribute replicas evenly
|
||||
PlacementManual PlacementStrategy = "manual" // Operator-specified placement
|
||||
)
|
||||
|
||||
// QueueConfig defines configuration for a queue.
|
||||
type QueueConfig struct {
|
||||
Name string
|
||||
Partitions int
|
||||
Ordering OrderingMode
|
||||
|
||||
RetryPolicy RetryPolicy
|
||||
DLQConfig DLQConfig
|
||||
Replication ReplicationConfig
|
||||
Retention RetentionPolicy
|
||||
|
||||
// Limits
|
||||
MaxMessageSize int64
|
||||
MaxQueueDepth int64
|
||||
MessageTTL time.Duration
|
||||
|
||||
// Performance
|
||||
DeliveryTimeout time.Duration
|
||||
BatchSize int
|
||||
HeartbeatTimeout time.Duration
|
||||
}
|
||||
|
||||
// RetryPolicy defines retry behavior for failed messages.
|
||||
type RetryPolicy struct {
|
||||
MaxRetries int
|
||||
InitialBackoff time.Duration
|
||||
MaxBackoff time.Duration
|
||||
BackoffMultiplier float64
|
||||
TotalTimeout time.Duration
|
||||
}
|
||||
|
||||
// DLQConfig defines dead-letter queue configuration.
|
||||
type DLQConfig struct {
|
||||
Enabled bool
|
||||
Topic string
|
||||
AlertWebhook string
|
||||
}
|
||||
|
||||
// ReplicationConfig defines Raft-based replication for queue partitions.
|
||||
type ReplicationConfig struct {
|
||||
Enabled bool
|
||||
ReplicationFactor int // Number of replicas per partition (default: 3)
|
||||
Mode ReplicationMode // sync or async
|
||||
Placement PlacementStrategy // How to assign replicas to nodes
|
||||
ManualReplicas map[int][]string // For manual placement: partitionID -> []nodeID
|
||||
MinInSyncReplicas int // Min replicas that must ACK (default: 2)
|
||||
AckTimeout time.Duration // Timeout for sync mode operations (default: 5s)
|
||||
|
||||
// Raft tuning (optional, uses defaults if zero)
|
||||
HeartbeatTimeout time.Duration // Raft heartbeat interval (default: 1s)
|
||||
ElectionTimeout time.Duration // Raft election timeout (default: 3s)
|
||||
SnapshotInterval time.Duration // Snapshot frequency (default: 5m)
|
||||
SnapshotThreshold uint64 // Snapshot after N log entries (default: 8192)
|
||||
}
|
||||
|
||||
// RetentionPolicy defines Kafka-style retention policies for automatic message cleanup.
|
||||
type RetentionPolicy struct {
|
||||
// Time-based retention (background cleanup)
|
||||
RetentionTime time.Duration // Delete messages older than this (0 = disabled)
|
||||
TimeCheckInterval time.Duration // How often to run cleanup (default: 5m)
|
||||
|
||||
// Size-based retention (active check on enqueue)
|
||||
RetentionBytes int64 // Max total queue size in bytes (0 = unlimited)
|
||||
RetentionMessages int64 // Max message count (0 = unlimited)
|
||||
SizeCheckEvery int // Check size every N enqueues (default: 100, optimization)
|
||||
|
||||
// Log compaction (Kafka-style)
|
||||
CompactionEnabled bool // Enable log compaction
|
||||
CompactionKey string // Message property to use as compaction key
|
||||
CompactionLag time.Duration // Wait before compacting new messages (default: 5m)
|
||||
CompactionInterval time.Duration // How often to run compaction (default: 10m)
|
||||
}
|
||||
|
||||
// Message represents a message in the queue system.
|
||||
type Message struct {
|
||||
ID string
|
||||
Payload []byte // Deprecated: Use PayloadBuf for zero-copy
|
||||
PayloadBuf *core.RefCountedBuffer // Zero-copy payload buffer (preferred)
|
||||
Topic string
|
||||
PartitionKey string
|
||||
PartitionID int
|
||||
Sequence uint64
|
||||
Properties map[string]string
|
||||
|
||||
// Lifecycle tracking
|
||||
State MessageState
|
||||
CreatedAt time.Time
|
||||
DeliveredAt time.Time
|
||||
NextRetryAt time.Time
|
||||
RetryCount int
|
||||
|
||||
// DLQ metadata
|
||||
FailureReason string
|
||||
FirstAttempt time.Time
|
||||
LastAttempt time.Time
|
||||
MovedToDLQAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// GetPayload returns the message payload, preferring PayloadBuf if available.
|
||||
// This provides backward compatibility during migration to zero-copy.
|
||||
func (m *Message) GetPayload() []byte {
|
||||
if m.PayloadBuf != nil {
|
||||
return m.PayloadBuf.Bytes()
|
||||
}
|
||||
return m.Payload
|
||||
}
|
||||
|
||||
// SetPayloadFromBuffer sets the payload from a RefCountedBuffer.
|
||||
// The message takes ownership of one reference.
|
||||
func (m *Message) SetPayloadFromBuffer(buf *core.RefCountedBuffer) {
|
||||
if m.PayloadBuf != nil {
|
||||
m.PayloadBuf.Release() // Release previous buffer
|
||||
}
|
||||
m.PayloadBuf = buf
|
||||
m.Payload = nil // Clear legacy field
|
||||
}
|
||||
|
||||
// SetPayloadFromBytes creates a new buffer from bytes (for backward compatibility).
|
||||
// This will eventually be phased out in favor of direct buffer creation.
|
||||
func (m *Message) SetPayloadFromBytes(data []byte) {
|
||||
if m.PayloadBuf != nil {
|
||||
m.PayloadBuf.Release()
|
||||
}
|
||||
if len(data) > 0 {
|
||||
m.PayloadBuf = core.GetBufferWithData(data)
|
||||
} else {
|
||||
m.PayloadBuf = nil
|
||||
}
|
||||
m.Payload = nil
|
||||
}
|
||||
|
||||
// ReleasePayload releases the buffer reference if PayloadBuf is set.
|
||||
// This should be called when the message is no longer needed.
|
||||
func (m *Message) ReleasePayload() {
|
||||
if m.PayloadBuf != nil {
|
||||
m.PayloadBuf.Release()
|
||||
m.PayloadBuf = nil
|
||||
}
|
||||
m.Payload = nil
|
||||
}
|
||||
|
||||
// DeliveryState tracks inflight message delivery.
|
||||
type DeliveryState struct {
|
||||
MessageID string
|
||||
QueueName string
|
||||
PartitionID int
|
||||
ConsumerID string
|
||||
DeliveredAt time.Time
|
||||
Timeout time.Time
|
||||
RetryCount int
|
||||
}
|
||||
|
||||
// Consumer represents a queue consumer.
|
||||
type Consumer struct {
|
||||
ID string
|
||||
ClientID string
|
||||
GroupID string
|
||||
QueueName string
|
||||
AssignedParts []int
|
||||
RegisteredAt time.Time
|
||||
LastHeartbeat time.Time
|
||||
ProxyNodeID string // For cluster routing
|
||||
}
|
||||
|
||||
// ConsumerGroup represents a group of consumers.
|
||||
type ConsumerGroup struct {
|
||||
ID string
|
||||
QueueName string
|
||||
Consumers map[string]*Consumer
|
||||
}
|
||||
|
||||
// QueueStore manages queue metadata and configuration.
|
||||
type QueueStore interface {
|
||||
CreateQueue(ctx context.Context, config QueueConfig) error
|
||||
GetQueue(ctx context.Context, queueName string) (*QueueConfig, error)
|
||||
UpdateQueue(ctx context.Context, config QueueConfig) error
|
||||
CreateQueue(ctx context.Context, config types.QueueConfig) error
|
||||
GetQueue(ctx context.Context, queueName string) (*types.QueueConfig, error)
|
||||
UpdateQueue(ctx context.Context, config types.QueueConfig) error
|
||||
DeleteQueue(ctx context.Context, queueName string) error
|
||||
ListQueues(ctx context.Context) ([]QueueConfig, error)
|
||||
ListQueues(ctx context.Context) ([]types.QueueConfig, error)
|
||||
}
|
||||
|
||||
// MessageStore manages queue messages and delivery state.
|
||||
type MessageStore interface {
|
||||
// Message operations
|
||||
Enqueue(ctx context.Context, queueName string, msg *Message) error
|
||||
Enqueue(ctx context.Context, queueName string, msg *types.Message) error
|
||||
// Count returns the number of messages in the queue (across all partitions).
|
||||
Count(ctx context.Context, queueName string) (int64, error)
|
||||
Dequeue(ctx context.Context, queueName string, partitionID int) (*Message, error)
|
||||
DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*Message, error)
|
||||
UpdateMessage(ctx context.Context, queueName string, msg *Message) error
|
||||
Dequeue(ctx context.Context, queueName string, partitionID int) (*types.Message, error)
|
||||
DequeueBatch(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error)
|
||||
UpdateMessage(ctx context.Context, queueName string, msg *types.Message) error
|
||||
DeleteMessage(ctx context.Context, queueName string, messageID string) error
|
||||
GetMessage(ctx context.Context, queueName string, messageID string) (*Message, error)
|
||||
GetMessage(ctx context.Context, queueName string, messageID string) (*types.Message, error)
|
||||
|
||||
// Inflight tracking
|
||||
MarkInflight(ctx context.Context, state *DeliveryState) error
|
||||
GetInflight(ctx context.Context, queueName string) ([]*DeliveryState, error)
|
||||
GetInflightMessage(ctx context.Context, queueName, messageID string) (*DeliveryState, error)
|
||||
MarkInflight(ctx context.Context, state *types.DeliveryState) error
|
||||
GetInflight(ctx context.Context, queueName string) ([]*types.DeliveryState, error)
|
||||
GetInflightMessage(ctx context.Context, queueName, messageID string) (*types.DeliveryState, error)
|
||||
RemoveInflight(ctx context.Context, queueName, messageID string) error
|
||||
|
||||
// DLQ operations
|
||||
EnqueueDLQ(ctx context.Context, dlqTopic string, msg *Message) error
|
||||
ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*Message, error)
|
||||
EnqueueDLQ(ctx context.Context, dlqTopic string, msg *types.Message) error
|
||||
ListDLQ(ctx context.Context, dlqTopic string, limit int) ([]*types.Message, error)
|
||||
DeleteDLQMessage(ctx context.Context, dlqTopic, messageID string) error
|
||||
|
||||
// Retry operations
|
||||
ListRetry(ctx context.Context, queueName string, partitionID int) ([]*Message, error)
|
||||
ListRetry(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error)
|
||||
|
||||
// Partition operations
|
||||
GetNextSequence(ctx context.Context, queueName string, partitionID int) (uint64, error)
|
||||
@@ -269,132 +59,24 @@ type MessageStore interface {
|
||||
GetOffset(ctx context.Context, queueName string, partitionID int) (uint64, error)
|
||||
|
||||
// Batch operations
|
||||
ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*Message, error)
|
||||
ListQueued(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error)
|
||||
|
||||
// Retention operations
|
||||
ListOldestMessages(ctx context.Context, queueName string, partitionID int, limit int) ([]*Message, error)
|
||||
ListMessagesBefore(ctx context.Context, queueName string, partitionID int, cutoffTime time.Time, limit int) ([]*Message, error)
|
||||
ListOldestMessages(ctx context.Context, queueName string, partitionID int, limit int) ([]*types.Message, error)
|
||||
ListMessagesBefore(ctx context.Context, queueName string, partitionID int, cutoffTime time.Time, limit int) ([]*types.Message, error)
|
||||
DeleteMessageBatch(ctx context.Context, queueName string, messageIDs []string) (int64, error)
|
||||
GetQueueSize(ctx context.Context, queueName string) (int64, error) // Total size in bytes
|
||||
|
||||
// Compaction operations
|
||||
ListAllMessages(ctx context.Context, queueName string, partitionID int) ([]*Message, error)
|
||||
ListAllMessages(ctx context.Context, queueName string, partitionID int) ([]*types.Message, error)
|
||||
}
|
||||
|
||||
// ConsumerStore manages consumer group state.
|
||||
type ConsumerStore interface {
|
||||
RegisterConsumer(ctx context.Context, consumer *Consumer) error
|
||||
RegisterConsumer(ctx context.Context, consumer *types.Consumer) error
|
||||
UnregisterConsumer(ctx context.Context, queueName, groupID, consumerID string) error
|
||||
GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*Consumer, error)
|
||||
ListConsumers(ctx context.Context, queueName, groupID string) ([]*Consumer, error)
|
||||
GetConsumer(ctx context.Context, queueName, groupID, consumerID string) (*types.Consumer, error)
|
||||
ListConsumers(ctx context.Context, queueName, groupID string) ([]*types.Consumer, error)
|
||||
ListGroups(ctx context.Context, queueName string) ([]string, error)
|
||||
UpdateHeartbeat(ctx context.Context, queueName, groupID, consumerID string, timestamp time.Time) error
|
||||
}
|
||||
|
||||
// DefaultQueueConfig returns default queue configuration.
|
||||
func DefaultQueueConfig(name string) QueueConfig {
|
||||
return QueueConfig{
|
||||
Name: name,
|
||||
Partitions: 10,
|
||||
Ordering: OrderingPartition,
|
||||
MaxMessageSize: 1024 * 1024, // 1MB
|
||||
MaxQueueDepth: 100000,
|
||||
MessageTTL: 7 * 24 * time.Hour,
|
||||
DeliveryTimeout: 30 * time.Second,
|
||||
BatchSize: 100,
|
||||
HeartbeatTimeout: 30 * time.Second,
|
||||
|
||||
RetryPolicy: RetryPolicy{
|
||||
MaxRetries: 10,
|
||||
InitialBackoff: 5 * time.Second,
|
||||
MaxBackoff: 5 * time.Minute,
|
||||
BackoffMultiplier: 2.0,
|
||||
TotalTimeout: 3 * time.Hour,
|
||||
},
|
||||
|
||||
DLQConfig: DLQConfig{
|
||||
Enabled: true,
|
||||
Topic: "", // Auto-generated
|
||||
},
|
||||
|
||||
Replication: ReplicationConfig{
|
||||
Enabled: false, // Disabled by default for backward compatibility
|
||||
ReplicationFactor: 3,
|
||||
Mode: ReplicationSync,
|
||||
Placement: PlacementRoundRobin,
|
||||
MinInSyncReplicas: 2,
|
||||
AckTimeout: 5 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates queue configuration.
|
||||
func (c *QueueConfig) Validate() error {
|
||||
switch {
|
||||
|
||||
case c.Name == "":
|
||||
return ErrInvalidConfig
|
||||
case !strings.HasPrefix(c.Name, "$queue/"):
|
||||
// Queue name must start with $queue/
|
||||
return ErrInvalidConfig
|
||||
case c.Partitions < 1:
|
||||
return ErrInvalidConfig
|
||||
case c.Partitions > 1000:
|
||||
// Reasonable upper limit
|
||||
return ErrInvalidConfig
|
||||
case c.Ordering != OrderingNone && c.Ordering != OrderingPartition && c.Ordering != OrderingStrict:
|
||||
return ErrInvalidConfig
|
||||
case c.Ordering == OrderingStrict && c.Partitions != 1:
|
||||
return ErrInvalidConfig
|
||||
case c.MaxMessageSize <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.MaxQueueDepth <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.DeliveryTimeout <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.BatchSize <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.HeartbeatTimeout <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.MaxRetries < 0:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.InitialBackoff < 0 || c.RetryPolicy.MaxBackoff < c.RetryPolicy.InitialBackoff:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.BackoffMultiplier < 1.0:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.TotalTimeout < 0:
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
|
||||
// Validate replication config if enabled
|
||||
if c.Replication.Enabled {
|
||||
switch {
|
||||
case c.Replication.ReplicationFactor < 1 || c.Replication.ReplicationFactor > 10:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.MinInSyncReplicas < 1 || c.Replication.MinInSyncReplicas > c.Replication.ReplicationFactor:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.Mode != ReplicationSync && c.Replication.Mode != ReplicationAsync:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.Placement != PlacementRoundRobin && c.Replication.Placement != PlacementManual:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.Placement == PlacementManual && len(c.Replication.ManualReplicas) != c.Partitions:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.AckTimeout <= 0:
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
|
||||
// Validate manual replica assignments
|
||||
if c.Replication.Placement == PlacementManual {
|
||||
for partID, replicas := range c.Replication.ManualReplicas {
|
||||
if partID < 0 || partID >= c.Partitions {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if len(replicas) != c.Replication.ReplicationFactor {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig indicates an invalid queue configuration.
|
||||
var ErrInvalidConfig = errors.New("invalid queue configuration")
|
||||
|
||||
// OrderingMode defines message ordering guarantees.
|
||||
type OrderingMode string
|
||||
|
||||
const (
|
||||
OrderingNone OrderingMode = "none" // No ordering guarantees
|
||||
OrderingPartition OrderingMode = "partition" // FIFO per partition key
|
||||
OrderingStrict OrderingMode = "strict" // Global FIFO (single partition)
|
||||
)
|
||||
|
||||
// ReplicationMode defines the replication behavior for queue messages.
|
||||
type ReplicationMode string
|
||||
|
||||
const (
|
||||
ReplicationSync ReplicationMode = "sync" // Wait for quorum ACK before returning
|
||||
ReplicationAsync ReplicationMode = "async" // Return immediately after leader accepts
|
||||
)
|
||||
|
||||
// PlacementStrategy defines how replicas are assigned to nodes.
|
||||
type PlacementStrategy string
|
||||
|
||||
const (
|
||||
PlacementRoundRobin PlacementStrategy = "round-robin" // Distribute replicas evenly
|
||||
PlacementManual PlacementStrategy = "manual" // Operator-specified placement
|
||||
)
|
||||
|
||||
// QueueConfig defines configuration for a queue.
|
||||
type QueueConfig struct {
|
||||
Name string
|
||||
Partitions int
|
||||
Ordering OrderingMode
|
||||
|
||||
RetryPolicy RetryPolicy
|
||||
DLQConfig DLQConfig
|
||||
Replication ReplicationConfig
|
||||
Retention RetentionPolicy
|
||||
|
||||
// Limits
|
||||
MaxMessageSize int64
|
||||
MaxQueueDepth int64
|
||||
MessageTTL time.Duration
|
||||
|
||||
// Performance
|
||||
DeliveryTimeout time.Duration
|
||||
BatchSize int
|
||||
HeartbeatTimeout time.Duration
|
||||
}
|
||||
|
||||
// RetryPolicy defines retry behavior for failed messages.
|
||||
type RetryPolicy struct {
|
||||
MaxRetries int
|
||||
InitialBackoff time.Duration
|
||||
MaxBackoff time.Duration
|
||||
BackoffMultiplier float64
|
||||
TotalTimeout time.Duration
|
||||
}
|
||||
|
||||
// DLQConfig defines dead-letter queue configuration.
|
||||
type DLQConfig struct {
|
||||
Enabled bool
|
||||
Topic string
|
||||
AlertWebhook string
|
||||
}
|
||||
|
||||
// ReplicationConfig defines Raft-based replication for queue partitions.
|
||||
type ReplicationConfig struct {
|
||||
Enabled bool
|
||||
ReplicationFactor int // Number of replicas per partition (default: 3)
|
||||
Mode ReplicationMode // sync or async
|
||||
Placement PlacementStrategy // How to assign replicas to nodes
|
||||
ManualReplicas map[int][]string // For manual placement: partitionID -> []nodeID
|
||||
MinInSyncReplicas int // Min replicas that must ACK (default: 2)
|
||||
AckTimeout time.Duration // Timeout for sync mode operations (default: 5s)
|
||||
|
||||
// Raft tuning (optional, uses defaults if zero)
|
||||
HeartbeatTimeout time.Duration // Raft heartbeat interval (default: 1s)
|
||||
ElectionTimeout time.Duration // Raft election timeout (default: 3s)
|
||||
SnapshotInterval time.Duration // Snapshot frequency (default: 5m)
|
||||
SnapshotThreshold uint64 // Snapshot after N log entries (default: 8192)
|
||||
}
|
||||
|
||||
// RetentionPolicy defines Kafka-style retention policies for automatic message cleanup.
|
||||
type RetentionPolicy struct {
|
||||
// Time-based retention (background cleanup)
|
||||
RetentionTime time.Duration // Delete messages older than this (0 = disabled)
|
||||
TimeCheckInterval time.Duration // How often to run cleanup (default: 5m)
|
||||
|
||||
// Size-based retention (active check on enqueue)
|
||||
RetentionBytes int64 // Max total queue size in bytes (0 = unlimited)
|
||||
RetentionMessages int64 // Max message count (0 = unlimited)
|
||||
SizeCheckEvery int // Check size every N enqueues (default: 100, optimization)
|
||||
|
||||
// Log compaction (Kafka-style)
|
||||
CompactionEnabled bool // Enable log compaction
|
||||
CompactionKey string // Message property to use as compaction key
|
||||
CompactionLag time.Duration // Wait before compacting new messages (default: 5m)
|
||||
CompactionInterval time.Duration // How often to run compaction (default: 10m)
|
||||
}
|
||||
|
||||
// DefaultQueueConfig returns default queue configuration.
|
||||
func DefaultQueueConfig(name string) QueueConfig {
|
||||
return QueueConfig{
|
||||
Name: name,
|
||||
Partitions: 10,
|
||||
Ordering: OrderingPartition,
|
||||
MaxMessageSize: 1024 * 1024, // 1MB
|
||||
MaxQueueDepth: 100000,
|
||||
MessageTTL: 7 * 24 * time.Hour,
|
||||
DeliveryTimeout: 30 * time.Second,
|
||||
BatchSize: 100,
|
||||
HeartbeatTimeout: 30 * time.Second,
|
||||
|
||||
RetryPolicy: RetryPolicy{
|
||||
MaxRetries: 10,
|
||||
InitialBackoff: 5 * time.Second,
|
||||
MaxBackoff: 5 * time.Minute,
|
||||
BackoffMultiplier: 2.0,
|
||||
TotalTimeout: 3 * time.Hour,
|
||||
},
|
||||
|
||||
DLQConfig: DLQConfig{
|
||||
Enabled: true,
|
||||
Topic: "", // Auto-generated
|
||||
},
|
||||
|
||||
Replication: ReplicationConfig{
|
||||
Enabled: false, // Disabled by default for backward compatibility
|
||||
ReplicationFactor: 3,
|
||||
Mode: ReplicationSync,
|
||||
Placement: PlacementRoundRobin,
|
||||
MinInSyncReplicas: 2,
|
||||
AckTimeout: 5 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates queue configuration.
|
||||
func (c *QueueConfig) Validate() error {
|
||||
switch {
|
||||
case c.Name == "":
|
||||
return ErrInvalidConfig
|
||||
case !strings.HasPrefix(c.Name, "$queue/"):
|
||||
// Queue name must start with $queue/
|
||||
return ErrInvalidConfig
|
||||
case c.Partitions < 1:
|
||||
return ErrInvalidConfig
|
||||
case c.Partitions > 1000:
|
||||
// Reasonable upper limit
|
||||
return ErrInvalidConfig
|
||||
case c.Ordering != OrderingNone && c.Ordering != OrderingPartition && c.Ordering != OrderingStrict:
|
||||
return ErrInvalidConfig
|
||||
case c.Ordering == OrderingStrict && c.Partitions != 1:
|
||||
return ErrInvalidConfig
|
||||
case c.MaxMessageSize <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.MaxQueueDepth <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.DeliveryTimeout <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.BatchSize <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.HeartbeatTimeout <= 0:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.MaxRetries < 0:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.InitialBackoff < 0 || c.RetryPolicy.MaxBackoff < c.RetryPolicy.InitialBackoff:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.BackoffMultiplier < 1.0:
|
||||
return ErrInvalidConfig
|
||||
case c.RetryPolicy.TotalTimeout < 0:
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
|
||||
// Validate replication config if enabled
|
||||
if c.Replication.Enabled {
|
||||
switch {
|
||||
case c.Replication.ReplicationFactor < 1 || c.Replication.ReplicationFactor > 10:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.MinInSyncReplicas < 1 || c.Replication.MinInSyncReplicas > c.Replication.ReplicationFactor:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.Mode != ReplicationSync && c.Replication.Mode != ReplicationAsync:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.Placement != PlacementRoundRobin && c.Replication.Placement != PlacementManual:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.Placement == PlacementManual && len(c.Replication.ManualReplicas) != c.Partitions:
|
||||
return ErrInvalidConfig
|
||||
case c.Replication.AckTimeout <= 0:
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
|
||||
// Validate manual replica assignments
|
||||
if c.Replication.Placement == PlacementManual {
|
||||
for partID, replicas := range c.Replication.ManualReplicas {
|
||||
if partID < 0 || partID >= c.Partitions {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
if len(replicas) != c.Replication.ReplicationFactor {
|
||||
return ErrInvalidConfig
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package storage
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -0,0 +1,25 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package types
|
||||
|
||||
import "time"
|
||||
|
||||
// Consumer represents a queue consumer.
|
||||
type Consumer struct {
|
||||
ID string
|
||||
ClientID string
|
||||
GroupID string
|
||||
QueueName string
|
||||
AssignedParts []int
|
||||
RegisteredAt time.Time
|
||||
LastHeartbeat time.Time
|
||||
ProxyNodeID string // For cluster routing
|
||||
}
|
||||
|
||||
// ConsumerGroup represents a group of consumers.
|
||||
type ConsumerGroup struct {
|
||||
ID string
|
||||
QueueName string
|
||||
Consumers map[string]*Consumer
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package types
|
||||
|
||||
import "time"
|
||||
|
||||
// DeliveryState tracks inflight message delivery.
|
||||
type DeliveryState struct {
|
||||
MessageID string
|
||||
QueueName string
|
||||
PartitionID int
|
||||
ConsumerID string
|
||||
DeliveredAt time.Time
|
||||
Timeout time.Time
|
||||
RetryCount int
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/absmach/fluxmq/core"
|
||||
)
|
||||
|
||||
// MessageState represents the lifecycle state of a queue message.
|
||||
type MessageState string
|
||||
|
||||
const (
|
||||
StateQueued MessageState = "queued"
|
||||
StateDelivered MessageState = "delivered"
|
||||
StateAcked MessageState = "acked"
|
||||
StateRetry MessageState = "retry"
|
||||
StateDLQ MessageState = "dlq"
|
||||
)
|
||||
|
||||
// Message represents a message in the queue system.
|
||||
type Message struct {
|
||||
ID string
|
||||
Payload []byte // Deprecated: Use PayloadBuf for zero-copy
|
||||
PayloadBuf *core.RefCountedBuffer // Zero-copy payload buffer (preferred)
|
||||
Topic string
|
||||
PartitionKey string
|
||||
PartitionID int
|
||||
Sequence uint64
|
||||
Properties map[string]string
|
||||
|
||||
// Lifecycle tracking
|
||||
State MessageState
|
||||
CreatedAt time.Time
|
||||
DeliveredAt time.Time
|
||||
NextRetryAt time.Time
|
||||
RetryCount int
|
||||
|
||||
// DLQ metadata
|
||||
FailureReason string
|
||||
FirstAttempt time.Time
|
||||
LastAttempt time.Time
|
||||
MovedToDLQAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// GetPayload returns the message payload, preferring PayloadBuf if available.
|
||||
// This provides backward compatibility during migration to zero-copy.
|
||||
func (m *Message) GetPayload() []byte {
|
||||
if m.PayloadBuf != nil {
|
||||
return m.PayloadBuf.Bytes()
|
||||
}
|
||||
return m.Payload
|
||||
}
|
||||
|
||||
// SetPayloadFromBuffer sets the payload from a RefCountedBuffer.
|
||||
// The message takes ownership of one reference.
|
||||
func (m *Message) SetPayloadFromBuffer(buf *core.RefCountedBuffer) {
|
||||
if m.PayloadBuf != nil {
|
||||
m.PayloadBuf.Release() // Release previous buffer
|
||||
}
|
||||
m.PayloadBuf = buf
|
||||
m.Payload = nil // Clear legacy field
|
||||
}
|
||||
|
||||
// SetPayloadFromBytes creates a new buffer from bytes (for backward compatibility).
|
||||
// This will eventually be phased out in favor of direct buffer creation.
|
||||
func (m *Message) SetPayloadFromBytes(data []byte) {
|
||||
if m.PayloadBuf != nil {
|
||||
m.PayloadBuf.Release()
|
||||
}
|
||||
if len(data) > 0 {
|
||||
m.PayloadBuf = core.GetBufferWithData(data)
|
||||
} else {
|
||||
m.PayloadBuf = nil
|
||||
}
|
||||
m.Payload = nil
|
||||
}
|
||||
|
||||
// ReleasePayload releases the buffer reference if PayloadBuf is set.
|
||||
// This should be called when the message is no longer needed.
|
||||
func (m *Message) ReleasePayload() {
|
||||
if m.PayloadBuf != nil {
|
||||
m.PayloadBuf.Release()
|
||||
m.PayloadBuf = nil
|
||||
}
|
||||
m.Payload = nil
|
||||
}
|
||||
@@ -180,9 +180,9 @@ func TestManager_Enabled(t *testing.T) {
|
||||
cfg := Config{
|
||||
Enabled: true,
|
||||
Connection: ConnectionConfig{
|
||||
Enabled: true,
|
||||
Rate: 1,
|
||||
Burst: 1,
|
||||
Enabled: true,
|
||||
Rate: 1,
|
||||
Burst: 1,
|
||||
CleanupInterval: time.Minute,
|
||||
},
|
||||
Message: MessageConfig{
|
||||
@@ -230,9 +230,9 @@ func TestManager_SelectiveEnable(t *testing.T) {
|
||||
cfg := Config{
|
||||
Enabled: true,
|
||||
Connection: ConnectionConfig{
|
||||
Enabled: true,
|
||||
Rate: 1,
|
||||
Burst: 1,
|
||||
Enabled: true,
|
||||
Rate: 1,
|
||||
Burst: 1,
|
||||
CleanupInterval: time.Minute,
|
||||
},
|
||||
Message: MessageConfig{
|
||||
|
||||
@@ -129,9 +129,9 @@ func TestServer_ListenDTLS_MissingCert(t *testing.T) {
|
||||
|
||||
func TestBuildDTLSConfig_ClientAuthModes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
clientAuth string
|
||||
expectedAuth string
|
||||
name string
|
||||
clientAuth string
|
||||
expectedAuth string
|
||||
}{
|
||||
{"none", "none", "none"},
|
||||
{"request", "request", "request"},
|
||||
|
||||
@@ -26,10 +26,10 @@ type Metrics struct {
|
||||
errorsTotal metric.Int64Counter
|
||||
|
||||
// UpDownCounters (Gauges)
|
||||
connectionsCurrent metric.Int64UpDownCounter
|
||||
subscriptionsActive metric.Int64UpDownCounter
|
||||
retainedMessages metric.Int64UpDownCounter
|
||||
sessionsActive metric.Int64UpDownCounter
|
||||
connectionsCurrent metric.Int64UpDownCounter
|
||||
subscriptionsActive metric.Int64UpDownCounter
|
||||
retainedMessages metric.Int64UpDownCounter
|
||||
sessionsActive metric.Int64UpDownCounter
|
||||
|
||||
// Histograms
|
||||
messageSize metric.Int64Histogram
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package badger
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package badger
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package badger
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package badger
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package badger
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (c) Abstract Machines
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package badger
|
||||
|
||||
import (
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Message pool to reduce allocations during message distribution
|
||||
// Message pool to reduce allocations during message distribution.
|
||||
var messagePool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &Message{}
|
||||
|
||||
@@ -33,11 +33,11 @@ const (
|
||||
|
||||
// Errors.
|
||||
var (
|
||||
ErrNotConnected = errors.New("client not connected")
|
||||
ErrNotConnected = errors.New("client not connected")
|
||||
ErrAlreadyConnected = errors.New("client already connected")
|
||||
ErrTimeout = errors.New("operation timed out")
|
||||
ErrConnectionLost = errors.New("connection lost")
|
||||
ErrInvalidPacket = errors.New("invalid packet received")
|
||||
ErrTimeout = errors.New("operation timed out")
|
||||
ErrConnectionLost = errors.New("connection lost")
|
||||
ErrInvalidPacket = errors.New("invalid packet received")
|
||||
)
|
||||
|
||||
// Message represents a received MQTT message.
|
||||
@@ -166,11 +166,11 @@ type TestMQTTClient struct {
|
||||
qos2IncomingMu sync.Mutex
|
||||
|
||||
// Lifecycle
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
errCh chan error
|
||||
mu sync.Mutex
|
||||
connMu sync.RWMutex
|
||||
stopCh chan struct{}
|
||||
doneCh chan struct{}
|
||||
errCh chan error
|
||||
mu sync.Mutex
|
||||
connMu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewTestMQTTClient creates a new test MQTT client.
|
||||
|
||||
@@ -7,11 +7,11 @@ import "testing"
|
||||
|
||||
func TestParseShared(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filter string
|
||||
expectedShare string
|
||||
expectedTopic string
|
||||
expectedIsShared bool
|
||||
name string
|
||||
filter string
|
||||
expectedShare string
|
||||
expectedTopic string
|
||||
expectedIsShared bool
|
||||
}{
|
||||
{
|
||||
name: "Valid shared subscription",
|
||||
|
||||
Reference in New Issue
Block a user