feat: Export all user Memos as a .zip of Markdown files

api/v2/acl.go

@@ -27,6 +27,40 @@ const (
 	usernameContextKey ContextKey = iota
 )
 
+// Used to set modified context of ServerStream.
+type WrappedStream struct {
+	ctx    context.Context
+	stream grpc.ServerStream
+}
+
+func (w *WrappedStream) RecvMsg(m any) error {
+	return w.stream.RecvMsg(m)
+}
+
+func (w *WrappedStream) SendMsg(m any) error {
+	return w.stream.SendMsg(m)
+}
+
+func (w *WrappedStream) SendHeader(md metadata.MD) error {
+	return w.stream.SendHeader(md)
+}
+
+func (w *WrappedStream) SetHeader(md metadata.MD) error {
+	return w.stream.SetHeader(md)
+}
+
+func (w *WrappedStream) SetTrailer(md metadata.MD) {
+	w.stream.SetTrailer(md)
+}
+
+func (w *WrappedStream) Context() context.Context {
+	return w.ctx
+}
+
+func newWrappedStream(ctx context.Context, stream grpc.ServerStream) grpc.ServerStream {
+	return &WrappedStream{ctx, stream}
+}
+
 // GRPCAuthInterceptor is the auth interceptor for gRPC server.
 type GRPCAuthInterceptor struct {
 	Store  *store.Store
@@ -80,6 +114,45 @@ func (in *GRPCAuthInterceptor) AuthenticationInterceptor(ctx context.Context, re
 	return handler(childCtx, request)
 }
 
+func (in *GRPCAuthInterceptor) StreamAuthenticationInterceptor(srv any, stream grpc.ServerStream, serverInfo *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	md, ok := metadata.FromIncomingContext(stream.Context())
+	if !ok {
+		return status.Errorf(codes.Unauthenticated, "failed to parse metadata from incoming context")
+	}
+	accessToken, err := getTokenFromMetadata(md)
+	if err != nil {
+		return status.Errorf(codes.Unauthenticated, err.Error())
+	}
+
+	username, err := in.authenticate(stream.Context(), accessToken)
+	if err != nil {
+		if isUnauthorizeAllowedMethod(serverInfo.FullMethod) {
+			return handler(stream.Context(), stream)
+		}
+		return err
+	}
+	user, err := in.Store.GetUser(stream.Context(), &store.FindUser{
+		Username: &username,
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to get user")
+	}
+	if user == nil {
+		return errors.Errorf("user %q not exists", username)
+	}
+	if user.RowStatus == store.Archived {
+		return errors.Errorf("user %q is archived", username)
+	}
+	if isOnlyForAdminAllowedMethod(serverInfo.FullMethod) && user.Role != store.RoleHost && user.Role != store.RoleAdmin {
+		return errors.Errorf("user %q is not admin", username)
+	}
+
+	// Stores userID into context.
+	childCtx := context.WithValue(stream.Context(), usernameContextKey, username)
+
+	return handler(srv, newWrappedStream(childCtx, stream))
+}
+
 func (in *GRPCAuthInterceptor) authenticate(ctx context.Context, accessToken string) (string, error) {
 	if accessToken == "" {
 		return "", status.Errorf(codes.Unauthenticated, "access token not found")

api/v2/memo_service.go

@@ -1,6 +1,8 @@
 package v2
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -32,6 +34,7 @@ import (
 const (
 	DefaultPageSize  = 10
 	MaxContentLength = 8 * 1024
+	ChunkSize        = 64 * 1024 // 64 KiB
 )
 
 func (s *APIV2Service) CreateMemo(ctx context.Context, request *apiv2pb.CreateMemoRequest) (*apiv2pb.CreateMemoResponse, error) {
@@ -100,84 +103,9 @@ func (s *APIV2Service) CreateMemo(ctx context.Context, request *apiv2pb.CreateMe
 }
 
 func (s *APIV2Service) ListMemos(ctx context.Context, request *apiv2pb.ListMemosRequest) (*apiv2pb.ListMemosResponse, error) {
-	memoFind := &store.FindMemo{
-		// Exclude comments by default.
-		ExcludeComments: true,
-	}
-	if request.Filter != "" {
-		filter, err := parseListMemosFilter(request.Filter)
-		if err != nil {
-			return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
-		}
-		if len(filter.ContentSearch) > 0 {
-			memoFind.ContentSearch = filter.ContentSearch
-		}
-		if len(filter.Visibilities) > 0 {
-			memoFind.VisibilityList = filter.Visibilities
-		}
-		if filter.OrderByPinned {
-			memoFind.OrderByPinned = filter.OrderByPinned
-		}
-		if filter.DisplayTimeAfter != nil {
-			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
-			}
-			if displayWithUpdatedTs {
-				memoFind.UpdatedTsAfter = filter.DisplayTimeAfter
-			} else {
-				memoFind.CreatedTsAfter = filter.DisplayTimeAfter
-			}
-		}
-		if filter.DisplayTimeBefore != nil {
-			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
-			}
-			if displayWithUpdatedTs {
-				memoFind.UpdatedTsBefore = filter.DisplayTimeBefore
-			} else {
-				memoFind.CreatedTsBefore = filter.DisplayTimeBefore
-			}
-		}
-		if filter.Creator != nil {
-			username, err := ExtractUsernameFromName(*filter.Creator)
-			if err != nil {
-				return nil, status.Errorf(codes.InvalidArgument, "invalid creator name")
-			}
-			user, err := s.Store.GetUser(ctx, &store.FindUser{
-				Username: &username,
-			})
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, "failed to get user")
-			}
-			if user == nil {
-				return nil, status.Errorf(codes.NotFound, "user not found")
-			}
-			memoFind.CreatorID = &user.ID
-		}
-		if filter.RowStatus != nil {
-			memoFind.RowStatus = filter.RowStatus
-		}
-	} else {
-		return nil, status.Errorf(codes.InvalidArgument, "filter is required")
-	}
-
-	user, _ := getCurrentUser(ctx, s.Store)
-	// If the user is not authenticated, only public memos are visible.
-	if user == nil {
-		memoFind.VisibilityList = []store.Visibility{store.Public}
-	}
-	if user != nil && memoFind.CreatorID != nil && *memoFind.CreatorID != user.ID {
-		memoFind.VisibilityList = []store.Visibility{store.Public, store.Protected}
-	}
-
-	displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+	memoFind, err := s.buildFindMemosWithFilter(ctx, request.Filter, true)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
-	}
-	if displayWithUpdatedTs {
-		memoFind.OrderByUpdatedTs = true
+		return nil, err
 	}
 
 	var limit, offset int
@@ -621,6 +549,61 @@ func (s *APIV2Service) GetUserMemosStats(ctx context.Context, request *apiv2pb.G
 	return response, nil
 }
 
+func (s *APIV2Service) ExportMemos(request *apiv2pb.ExportMemosRequest, srv apiv2pb.MemoService_ExportMemosServer) error {
+	ctx := srv.Context()
+	fmt.Printf("%+v\n", ctx)
+	memoFind, err := s.buildFindMemosWithFilter(ctx, request.Filter, true)
+	if err != nil {
+		return err
+	}
+
+	memos, err := s.Store.ListMemos(ctx, memoFind)
+	if err != nil {
+		return err
+	}
+
+	buf := new(bytes.Buffer)
+	writer := zip.NewWriter(buf)
+
+	for _, memo := range memos {
+		memoMessage, err := s.convertMemoFromStore(ctx, memo)
+		log.Info(memoMessage.Content)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert memo")
+		}
+		file, err := writer.Create(time.Unix(memo.CreatedTs, 0).Format(time.RFC3339) + ".md")
+		if err != nil {
+			return status.Errorf(codes.Internal, "Failed to create memo file")
+		}
+		_, err = file.Write([]byte(memoMessage.Content))
+		if err != nil {
+			return status.Errorf(codes.Internal, "Failed to write to memo file")
+		}
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return status.Errorf(codes.Internal, "Failed to close zip file writer")
+	}
+
+	exportChunk := &apiv2pb.ExportMemosResponse{}
+	sizeOfFile := len(buf.Bytes())
+	for currentByte := 0; currentByte < sizeOfFile; currentByte += ChunkSize {
+		if currentByte+ChunkSize > sizeOfFile {
+			exportChunk.File = buf.Bytes()[currentByte:sizeOfFile]
+		} else {
+			exportChunk.File = buf.Bytes()[currentByte : currentByte+ChunkSize]
+		}
+
+		err := srv.Send(exportChunk)
+		if err != nil {
+			return status.Error(codes.Internal, "Unable to stream ExportMemosResponse chunk")
+		}
+	}
+
+	return nil
+}
+
 func (s *APIV2Service) convertMemoFromStore(ctx context.Context, memo *store.Memo) (*apiv2pb.Memo, error) {
 	rawNodes, err := parser.Parse(tokenizer.Tokenize(memo.Content))
 	if err != nil {
@@ -847,6 +830,90 @@ func (s *APIV2Service) dispatchMemoRelatedWebhook(ctx context.Context, memo *api
 	return nil
 }
 
+func (s *APIV2Service) buildFindMemosWithFilter(ctx context.Context, filter string, excludeComments bool) (*store.FindMemo, error) {
+	memoFind := &store.FindMemo{
+		// Exclude comments by default.
+		ExcludeComments: excludeComments,
+	}
+	if filter != "" {
+		filter, err := parseListMemosFilter(filter)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
+		}
+		if len(filter.ContentSearch) > 0 {
+			memoFind.ContentSearch = filter.ContentSearch
+		}
+		if len(filter.Visibilities) > 0 {
+			memoFind.VisibilityList = filter.Visibilities
+		}
+		if filter.OrderByPinned {
+			memoFind.OrderByPinned = filter.OrderByPinned
+		}
+		if filter.DisplayTimeAfter != nil {
+			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
+			}
+			if displayWithUpdatedTs {
+				memoFind.UpdatedTsAfter = filter.DisplayTimeAfter
+			} else {
+				memoFind.CreatedTsAfter = filter.DisplayTimeAfter
+			}
+		}
+		if filter.DisplayTimeBefore != nil {
+			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
+			}
+			if displayWithUpdatedTs {
+				memoFind.UpdatedTsBefore = filter.DisplayTimeBefore
+			} else {
+				memoFind.CreatedTsBefore = filter.DisplayTimeBefore
+			}
+		}
+		if filter.Creator != nil {
+			username, err := ExtractUsernameFromName(*filter.Creator)
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "invalid creator name")
+			}
+			user, err := s.Store.GetUser(ctx, &store.FindUser{
+				Username: &username,
+			})
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to get user")
+			}
+			if user == nil {
+				return nil, status.Errorf(codes.NotFound, "user not found")
+			}
+			memoFind.CreatorID = &user.ID
+		}
+		if filter.RowStatus != nil {
+			memoFind.RowStatus = filter.RowStatus
+		}
+	} else {
+		return nil, status.Errorf(codes.InvalidArgument, "filter is required")
+	}
+
+	user, _ := getCurrentUser(ctx, s.Store)
+	// If the user is not authenticated, only public memos are visible.
+	if user == nil {
+		memoFind.VisibilityList = []store.Visibility{store.Public}
+	}
+	if user != nil && memoFind.CreatorID != nil && *memoFind.CreatorID != user.ID {
+		memoFind.VisibilityList = []store.Visibility{store.Public, store.Protected}
+	}
+
+	displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
+	}
+	if displayWithUpdatedTs {
+		memoFind.OrderByUpdatedTs = true
+	}
+
+	return memoFind, nil
+}
+
 func convertMemoToWebhookPayload(memo *apiv2pb.Memo) *webhook.WebhookPayload {
 	return &webhook.WebhookPayload{
 		CreatorID: memo.CreatorId,

api/v2/v2.go

@@ -47,6 +47,9 @@ func NewAPIV2Service(secret string, profile *profile.Profile, store *store.Store
 		grpc.ChainUnaryInterceptor(
 			authProvider.AuthenticationInterceptor,
 		),
+		grpc.ChainStreamInterceptor(
+			authProvider.StreamAuthenticationInterceptor,
+		),
 	)
 	apiv2Service := &APIV2Service{
 		Secret:         secret,

proto/api/v2/memo_service.proto

@@ -90,6 +90,10 @@ service MemoService {
     option (google.api.http) = {get: "/api/v2/memos/stats"};
     option (google.api.method_signature) = "username";
   }
+
+  rpc ExportMemos(ExportMemosRequest) returns (stream ExportMemosResponse) {
+    option (google.api.http) = {get: "/api/v2/memos/export"};
+  }
 }
 
 enum Visibility {
@@ -273,3 +277,12 @@ message GetUserMemosStatsResponse {
   // key is the year-month-day string. e.g. "2020-01-01".
   map<string, int32> stats = 1;
 }
+
+message ExportMemosRequest {
+  // Same as ListMemosRequest.filter
+  string filter = 1;
+}
+
+message ExportMemosResponse {
+  bytes file = 1;
+}