feat: Export all user Memos as a .zip of Markdown files

api/v2/acl.go

@@ -27,6 +27,40 @@ const (
 	usernameContextKey ContextKey = iota
 )
 
+// Used to set modified context of ServerStream.
+type WrappedStream struct {
+	ctx    context.Context
+	stream grpc.ServerStream
+}
+
+func (w *WrappedStream) RecvMsg(m any) error {
+	return w.stream.RecvMsg(m)
+}
+
+func (w *WrappedStream) SendMsg(m any) error {
+	return w.stream.SendMsg(m)
+}
+
+func (w *WrappedStream) SendHeader(md metadata.MD) error {
+	return w.stream.SendHeader(md)
+}
+
+func (w *WrappedStream) SetHeader(md metadata.MD) error {
+	return w.stream.SetHeader(md)
+}
+
+func (w *WrappedStream) SetTrailer(md metadata.MD) {
+	w.stream.SetTrailer(md)
+}
+
+func (w *WrappedStream) Context() context.Context {
+	return w.ctx
+}
+
+func newWrappedStream(ctx context.Context, stream grpc.ServerStream) grpc.ServerStream {
+	return &WrappedStream{ctx, stream}
+}
+
 // GRPCAuthInterceptor is the auth interceptor for gRPC server.
 type GRPCAuthInterceptor struct {
 	Store  *store.Store
@@ -80,6 +114,45 @@ func (in *GRPCAuthInterceptor) AuthenticationInterceptor(ctx context.Context, re
 	return handler(childCtx, request)
 }
 
+func (in *GRPCAuthInterceptor) StreamAuthenticationInterceptor(srv any, stream grpc.ServerStream, serverInfo *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	md, ok := metadata.FromIncomingContext(stream.Context())
+	if !ok {
+		return status.Errorf(codes.Unauthenticated, "failed to parse metadata from incoming context")
+	}
+	accessToken, err := getTokenFromMetadata(md)
+	if err != nil {
+		return status.Errorf(codes.Unauthenticated, err.Error())
+	}
+
+	username, err := in.authenticate(stream.Context(), accessToken)
+	if err != nil {
+		if isUnauthorizeAllowedMethod(serverInfo.FullMethod) {
+			return handler(stream.Context(), stream)
+		}
+		return err
+	}
+	user, err := in.Store.GetUser(stream.Context(), &store.FindUser{
+		Username: &username,
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to get user")
+	}
+	if user == nil {
+		return errors.Errorf("user %q not exists", username)
+	}
+	if user.RowStatus == store.Archived {
+		return errors.Errorf("user %q is archived", username)
+	}
+	if isOnlyForAdminAllowedMethod(serverInfo.FullMethod) && user.Role != store.RoleHost && user.Role != store.RoleAdmin {
+		return errors.Errorf("user %q is not admin", username)
+	}
+
+	// Stores userID into context.
+	childCtx := context.WithValue(stream.Context(), usernameContextKey, username)
+
+	return handler(srv, newWrappedStream(childCtx, stream))
+}
+
 func (in *GRPCAuthInterceptor) authenticate(ctx context.Context, accessToken string) (string, error) {
 	if accessToken == "" {
 		return "", status.Errorf(codes.Unauthenticated, "access token not found")

api/v2/memo_service.go

@@ -1,6 +1,8 @@
 package v2
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -32,6 +34,7 @@ import (
 const (
 	DefaultPageSize  = 10
 	MaxContentLength = 8 * 1024
+	ChunkSize        = 64 * 1024 // 64 KiB
 )
 
 func (s *APIV2Service) CreateMemo(ctx context.Context, request *apiv2pb.CreateMemoRequest) (*apiv2pb.CreateMemoResponse, error) {
@@ -100,84 +103,9 @@ func (s *APIV2Service) CreateMemo(ctx context.Context, request *apiv2pb.CreateMe
 }
 
 func (s *APIV2Service) ListMemos(ctx context.Context, request *apiv2pb.ListMemosRequest) (*apiv2pb.ListMemosResponse, error) {
-	memoFind := &store.FindMemo{
-		// Exclude comments by default.
-		ExcludeComments: true,
-	}
-	if request.Filter != "" {
-		filter, err := parseListMemosFilter(request.Filter)
-		if err != nil {
-			return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
-		}
-		if len(filter.ContentSearch) > 0 {
-			memoFind.ContentSearch = filter.ContentSearch
-		}
-		if len(filter.Visibilities) > 0 {
-			memoFind.VisibilityList = filter.Visibilities
-		}
-		if filter.OrderByPinned {
-			memoFind.OrderByPinned = filter.OrderByPinned
-		}
-		if filter.DisplayTimeAfter != nil {
-			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
-			}
-			if displayWithUpdatedTs {
-				memoFind.UpdatedTsAfter = filter.DisplayTimeAfter
-			} else {
-				memoFind.CreatedTsAfter = filter.DisplayTimeAfter
-			}
-		}
-		if filter.DisplayTimeBefore != nil {
-			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
-			}
-			if displayWithUpdatedTs {
-				memoFind.UpdatedTsBefore = filter.DisplayTimeBefore
-			} else {
-				memoFind.CreatedTsBefore = filter.DisplayTimeBefore
-			}
-		}
-		if filter.Creator != nil {
-			username, err := ExtractUsernameFromName(*filter.Creator)
-			if err != nil {
-				return nil, status.Errorf(codes.InvalidArgument, "invalid creator name")
-			}
-			user, err := s.Store.GetUser(ctx, &store.FindUser{
-				Username: &username,
-			})
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, "failed to get user")
-			}
-			if user == nil {
-				return nil, status.Errorf(codes.NotFound, "user not found")
-			}
-			memoFind.CreatorID = &user.ID
-		}
-		if filter.RowStatus != nil {
-			memoFind.RowStatus = filter.RowStatus
-		}
-	} else {
-		return nil, status.Errorf(codes.InvalidArgument, "filter is required")
-	}
-
-	user, _ := getCurrentUser(ctx, s.Store)
-	// If the user is not authenticated, only public memos are visible.
-	if user == nil {
-		memoFind.VisibilityList = []store.Visibility{store.Public}
-	}
-	if user != nil && memoFind.CreatorID != nil && *memoFind.CreatorID != user.ID {
-		memoFind.VisibilityList = []store.Visibility{store.Public, store.Protected}
-	}
-
-	displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+	memoFind, err := s.buildFindMemosWithFilter(ctx, request.Filter, true)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
-	}
-	if displayWithUpdatedTs {
-		memoFind.OrderByUpdatedTs = true
+		return nil, err
 	}
 
 	var limit, offset int
@@ -621,6 +549,61 @@ func (s *APIV2Service) GetUserMemosStats(ctx context.Context, request *apiv2pb.G
 	return response, nil
 }
 
+func (s *APIV2Service) ExportMemos(request *apiv2pb.ExportMemosRequest, srv apiv2pb.MemoService_ExportMemosServer) error {
+	ctx := srv.Context()
+	fmt.Printf("%+v\n", ctx)
+	memoFind, err := s.buildFindMemosWithFilter(ctx, request.Filter, true)
+	if err != nil {
+		return err
+	}
+
+	memos, err := s.Store.ListMemos(ctx, memoFind)
+	if err != nil {
+		return err
+	}
+
+	buf := new(bytes.Buffer)
+	writer := zip.NewWriter(buf)
+
+	for _, memo := range memos {
+		memoMessage, err := s.convertMemoFromStore(ctx, memo)
+		log.Info(memoMessage.Content)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert memo")
+		}
+		file, err := writer.Create(time.Unix(memo.CreatedTs, 0).Format(time.RFC3339) + ".md")
+		if err != nil {
+			return status.Errorf(codes.Internal, "Failed to create memo file")
+		}
+		_, err = file.Write([]byte(memoMessage.Content))
+		if err != nil {
+			return status.Errorf(codes.Internal, "Failed to write to memo file")
+		}
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return status.Errorf(codes.Internal, "Failed to close zip file writer")
+	}
+
+	exportChunk := &apiv2pb.ExportMemosResponse{}
+	sizeOfFile := len(buf.Bytes())
+	for currentByte := 0; currentByte < sizeOfFile; currentByte += ChunkSize {
+		if currentByte+ChunkSize > sizeOfFile {
+			exportChunk.File = buf.Bytes()[currentByte:sizeOfFile]
+		} else {
+			exportChunk.File = buf.Bytes()[currentByte : currentByte+ChunkSize]
+		}
+
+		err := srv.Send(exportChunk)
+		if err != nil {
+			return status.Error(codes.Internal, "Unable to stream ExportMemosResponse chunk")
+		}
+	}
+
+	return nil
+}
+
 func (s *APIV2Service) convertMemoFromStore(ctx context.Context, memo *store.Memo) (*apiv2pb.Memo, error) {
 	rawNodes, err := parser.Parse(tokenizer.Tokenize(memo.Content))
 	if err != nil {
@@ -847,6 +830,90 @@ func (s *APIV2Service) dispatchMemoRelatedWebhook(ctx context.Context, memo *api
 	return nil
 }
 
+func (s *APIV2Service) buildFindMemosWithFilter(ctx context.Context, filter string, excludeComments bool) (*store.FindMemo, error) {
+	memoFind := &store.FindMemo{
+		// Exclude comments by default.
+		ExcludeComments: excludeComments,
+	}
+	if filter != "" {
+		filter, err := parseListMemosFilter(filter)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
+		}
+		if len(filter.ContentSearch) > 0 {
+			memoFind.ContentSearch = filter.ContentSearch
+		}
+		if len(filter.Visibilities) > 0 {
+			memoFind.VisibilityList = filter.Visibilities
+		}
+		if filter.OrderByPinned {
+			memoFind.OrderByPinned = filter.OrderByPinned
+		}
+		if filter.DisplayTimeAfter != nil {
+			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
+			}
+			if displayWithUpdatedTs {
+				memoFind.UpdatedTsAfter = filter.DisplayTimeAfter
+			} else {
+				memoFind.CreatedTsAfter = filter.DisplayTimeAfter
+			}
+		}
+		if filter.DisplayTimeBefore != nil {
+			displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
+			}
+			if displayWithUpdatedTs {
+				memoFind.UpdatedTsBefore = filter.DisplayTimeBefore
+			} else {
+				memoFind.CreatedTsBefore = filter.DisplayTimeBefore
+			}
+		}
+		if filter.Creator != nil {
+			username, err := ExtractUsernameFromName(*filter.Creator)
+			if err != nil {
+				return nil, status.Errorf(codes.InvalidArgument, "invalid creator name")
+			}
+			user, err := s.Store.GetUser(ctx, &store.FindUser{
+				Username: &username,
+			})
+			if err != nil {
+				return nil, status.Errorf(codes.Internal, "failed to get user")
+			}
+			if user == nil {
+				return nil, status.Errorf(codes.NotFound, "user not found")
+			}
+			memoFind.CreatorID = &user.ID
+		}
+		if filter.RowStatus != nil {
+			memoFind.RowStatus = filter.RowStatus
+		}
+	} else {
+		return nil, status.Errorf(codes.InvalidArgument, "filter is required")
+	}
+
+	user, _ := getCurrentUser(ctx, s.Store)
+	// If the user is not authenticated, only public memos are visible.
+	if user == nil {
+		memoFind.VisibilityList = []store.Visibility{store.Public}
+	}
+	if user != nil && memoFind.CreatorID != nil && *memoFind.CreatorID != user.ID {
+		memoFind.VisibilityList = []store.Visibility{store.Public, store.Protected}
+	}
+
+	displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
+	}
+	if displayWithUpdatedTs {
+		memoFind.OrderByUpdatedTs = true
+	}
+
+	return memoFind, nil
+}
+
 func convertMemoToWebhookPayload(memo *apiv2pb.Memo) *webhook.WebhookPayload {
 	return &webhook.WebhookPayload{
 		CreatorID: memo.CreatorId,

api/v2/v2.go

@@ -47,6 +47,9 @@ func NewAPIV2Service(secret string, profile *profile.Profile, store *store.Store
 		grpc.ChainUnaryInterceptor(
 			authProvider.AuthenticationInterceptor,
 		),
+		grpc.ChainStreamInterceptor(
+			authProvider.StreamAuthenticationInterceptor,
+		),
 	)
 	apiv2Service := &APIV2Service{
 		Secret:         secret,

proto/api/v2/memo_service.proto

@@ -90,6 +90,10 @@ service MemoService {
     option (google.api.http) = {get: "/api/v2/memos/stats"};
     option (google.api.method_signature) = "username";
   }
+
+  rpc ExportMemos(ExportMemosRequest) returns (stream ExportMemosResponse) {
+    option (google.api.http) = {get: "/api/v2/memos/export"};
+  }
 }
 
 enum Visibility {
@@ -273,3 +277,12 @@ message GetUserMemosStatsResponse {
   // key is the year-month-day string. e.g. "2020-01-01".
   map<string, int32> stats = 1;
 }
+
+message ExportMemosRequest {
+  // Same as ListMemosRequest.filter
+  string filter = 1;
+}
+
+message ExportMemosResponse {
+  bytes file = 1;
+}

proto/gen/api/v2/README.md

@@ -134,6 +134,8 @@
     - [CreateMemoResponse](#memos-api-v2-CreateMemoResponse)
     - [DeleteMemoRequest](#memos-api-v2-DeleteMemoRequest)
     - [DeleteMemoResponse](#memos-api-v2-DeleteMemoResponse)
+    - [ExportMemosRequest](#memos-api-v2-ExportMemosRequest)
+    - [ExportMemosResponse](#memos-api-v2-ExportMemosResponse)
     - [GetMemoByNameRequest](#memos-api-v2-GetMemoByNameRequest)
     - [GetMemoByNameResponse](#memos-api-v2-GetMemoByNameResponse)
     - [GetMemoRequest](#memos-api-v2-GetMemoRequest)
@@ -1939,6 +1941,36 @@ Used internally for obfuscating the page token.
 
 
 
+<a name="memos-api-v2-ExportMemosRequest"></a>
+
+### ExportMemosRequest
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| filter | [string](#string) |  | Same as ListMemosRequest.filter |
+
+
+
+
+
+
+<a name="memos-api-v2-ExportMemosResponse"></a>
+
+### ExportMemosResponse
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| file | [bytes](#bytes) |  |  |
+
+
+
+
+
+
 <a name="memos-api-v2-GetMemoByNameRequest"></a>
 
 ### GetMemoByNameRequest
@@ -2323,6 +2355,7 @@ Used internally for obfuscating the page token.
 | CreateMemoComment | [CreateMemoCommentRequest](#memos-api-v2-CreateMemoCommentRequest) | [CreateMemoCommentResponse](#memos-api-v2-CreateMemoCommentResponse) | CreateMemoComment creates a comment for a memo. |
 | ListMemoComments | [ListMemoCommentsRequest](#memos-api-v2-ListMemoCommentsRequest) | [ListMemoCommentsResponse](#memos-api-v2-ListMemoCommentsResponse) | ListMemoComments lists comments for a memo. |
 | GetUserMemosStats | [GetUserMemosStatsRequest](#memos-api-v2-GetUserMemosStatsRequest) | [GetUserMemosStatsResponse](#memos-api-v2-GetUserMemosStatsResponse) | GetUserMemosStats gets stats of memos for a user. |
+| ExportMemos | [ExportMemosRequest](#memos-api-v2-ExportMemosRequest) | [ExportMemosResponse](#memos-api-v2-ExportMemosResponse) stream |  |
 
 
 

web/src/components/Settings/MyAccountSection.tsx

@@ -1,4 +1,6 @@
 import { Button } from "@mui/joy";
+import { memoServiceClient } from "@/grpcweb";
+import { downloadFileFromUrl } from "@/helpers/utils";
 import useCurrentUser from "@/hooks/useCurrentUser";
 import { useTranslate } from "@/utils/i18n";
 import showChangePasswordDialog from "../ChangePasswordDialog";
@@ -10,6 +12,17 @@ const MyAccountSection = () => {
   const t = useTranslate();
   const user = useCurrentUser();
 
+  const downloadExportedMemos = async (user: any) => {
+    const chunks = [];
+    for await (const response of memoServiceClient.exportMemos({ filter: `creator == "${user.name}"` })) {
+      chunks.push(response.file.buffer);
+    }
+    const blob = new Blob(chunks);
+    const downloadUrl = window.URL.createObjectURL(blob);
+    downloadFileFromUrl(downloadUrl, "memos-export.zip");
+    URL.revokeObjectURL(downloadUrl);
+  };
+
   return (
     <div className="w-full gap-2 pt-2 pb-4">
       <p className="font-medium text-gray-700 dark:text-gray-500">{t("setting.account-section.title")}</p>
@@ -27,6 +40,9 @@ const MyAccountSection = () => {
         <Button variant="outlined" onClick={showChangePasswordDialog}>
           {t("setting.account-section.change-password")}
         </Button>
+        <Button variant="outlined" onClick={() => downloadExportedMemos(user)}>
+          {t("setting.account-section.export-memos")}
+        </Button>
       </div>
 
       <AccessTokenSection />

web/src/components/ShareMemoDialog.tsx

@@ -3,6 +3,7 @@ import copy from "copy-to-clipboard";
 import React, { useEffect, useRef } from "react";
 import { toast } from "react-hot-toast";
 import { getDateTimeString } from "@/helpers/datetime";
+import { downloadFileFromUrl } from "@/helpers/utils";
 import useLoading from "@/hooks/useLoading";
 import toImage from "@/labs/html2image";
 import { useUserStore, extractUsernameFromName } from "@/store/v1";
@@ -51,6 +52,7 @@ const ShareMemoDialog: React.FC<Props> = (props: Props) => {
       .then((url) => {
         downloadFileFromUrl(url, `memos-${getDateTimeString(Date.now())}.png`);
         downloadingImageState.setFinish();
+        URL.revokeObjectURL(url);
       })
       .catch((err) => {
         console.error(err);
@@ -59,14 +61,9 @@ const ShareMemoDialog: React.FC<Props> = (props: Props) => {
 
   const handleDownloadTextFileBtnClick = () => {
     const blob = new Blob([memo.content], { type: "text/plain;charset=utf-8" });
-    downloadFileFromUrl(URL.createObjectURL(blob), `memos-${getDateTimeString(Date.now())}.md`);
-  };
-
-  const downloadFileFromUrl = (url: string, filename: string) => {
-    const a = document.createElement("a");
-    a.href = url;
-    a.download = filename;
-    a.click();
+    const url = URL.createObjectURL(blob);
+    downloadFileFromUrl(url, `memos-${getDateTimeString(Date.now())}.md`);
+    URL.revokeObjectURL(url);
   };
 
   const handleCopyLinkBtnClick = () => {

web/src/helpers/utils.ts

@@ -92,3 +92,11 @@ export const isValidUrl = (url: string): boolean => {
     return false;
   }
 };
+
+export const downloadFileFromUrl = (url: string, filename: string) => {
+  const a = document.createElement("a");
+  a.href = url;
+  a.download = filename;
+  a.click();
+  a.remove();
+};

web/src/locales/en.json

@@ -177,6 +177,7 @@
       "email-note": "Optional",
       "update-information": "Update Information",
       "change-password": "Change password",
+      "export-memos": "Export Memos",
       "reset-api": "Reset API",
       "openapi-title": "OpenAPI",
       "openapi-reset": "Reset OpenAPI Key",

web/src/locales/es.json

@@ -92,7 +92,8 @@
     "account-section": {
       "title": "Información de la Cuenta",
       "update-information": "Actualizar Información",
-      "change-password": "Cambiar Contraseña"
+      "change-password": "Cambiar Contraseña",
+      "export-memos": "Exportar Notas"
     },
     "preference-section": {
       "theme": "Tema",