enhance: enable write ahead buffer for streaming service

- Make a timetick-commit-based write ahead buffer at write side. - Add a switchable scanner at read side to transfer the state between catchup and tailing read Signed-off-by: chyezh <[email protected]>
milvus-io · Jan 23, 2025 · b5aa2c5 · b5aa2c5
1 parent 44e5e19
commit b5aa2c5
Show file tree

Hide file tree

Showing 33 changed files with 1,413 additions and 390 deletions.
diff --git a/internal/.mockery.yaml b/internal/.mockery.yaml
@@ -54,6 +54,9 @@ packages:
   github.com/milvus-io/milvus/internal/streamingnode/server/wal/interceptors/timetick/inspector:
     interfaces:
       TimeTickSyncOperator:
+  github.com/milvus-io/milvus/internal/streamingnode/server/wal/interceptors/wab:
+    interfaces:
+      ROWriteAheadBuffer:
   google.golang.org/grpc:
     interfaces:
       ClientStream:

diff --git a/internal/mocks/distributed/mock_streaming/mock_WALAccesser.go b/internal/mocks/distributed/mock_streaming/mock_WALAccesser.go
diff --git a/internal/mocks/streamingnode/server/wal/interceptors/mock_wab/mock_ROWriteAheadBuffer.go b/internal/mocks/streamingnode/server/wal/interceptors/mock_wab/mock_ROWriteAheadBuffer.go
diff --git a/...treamingnode/server/wal/interceptors/timetick/mock_inspector/mock_TimeTickSyncOperator.go b/...treamingnode/server/wal/interceptors/timetick/mock_inspector/mock_TimeTickSyncOperator.go
diff --git a/internal/streamingnode/server/flusher/flusherimpl/util.go b/internal/streamingnode/server/flusher/flusherimpl/util.go
@@ -4,9 +4,9 @@ import (
 	"context"
 	"math"
 
+	"github.com/cockroachdb/errors"
 	"go.uber.org/zap"
 
-	"github.com/cockroachdb/errors"
 	"github.com/milvus-io/milvus/internal/streamingnode/server/resource"
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/proto/datapb"