Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Export all user Memos as a .zip of Markdown files #2854

Merged
merged 5 commits into from
Jan 30, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions api/v2/acl.go
Original file line number Diff line number Diff line change
@@ -27,6 +27,40 @@ const (
usernameContextKey ContextKey = iota
)

// Used to set modified context of ServerStream.
type WrappedStream struct {
ctx context.Context
stream grpc.ServerStream
}

func (w *WrappedStream) RecvMsg(m any) error {
return w.stream.RecvMsg(m)
}

func (w *WrappedStream) SendMsg(m any) error {
return w.stream.SendMsg(m)
}

func (w *WrappedStream) SendHeader(md metadata.MD) error {
return w.stream.SendHeader(md)
}

func (w *WrappedStream) SetHeader(md metadata.MD) error {
return w.stream.SetHeader(md)
}

func (w *WrappedStream) SetTrailer(md metadata.MD) {
w.stream.SetTrailer(md)
}

func (w *WrappedStream) Context() context.Context {
return w.ctx
}

func newWrappedStream(ctx context.Context, stream grpc.ServerStream) grpc.ServerStream {
return &WrappedStream{ctx, stream}
}

// GRPCAuthInterceptor is the auth interceptor for gRPC server.
type GRPCAuthInterceptor struct {
Store *store.Store
@@ -80,6 +114,45 @@ func (in *GRPCAuthInterceptor) AuthenticationInterceptor(ctx context.Context, re
return handler(childCtx, request)
}

func (in *GRPCAuthInterceptor) StreamAuthenticationInterceptor(srv any, stream grpc.ServerStream, serverInfo *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
md, ok := metadata.FromIncomingContext(stream.Context())
if !ok {
return status.Errorf(codes.Unauthenticated, "failed to parse metadata from incoming context")
}
accessToken, err := getTokenFromMetadata(md)
if err != nil {
return status.Errorf(codes.Unauthenticated, err.Error())
}

username, err := in.authenticate(stream.Context(), accessToken)
if err != nil {
if isUnauthorizeAllowedMethod(serverInfo.FullMethod) {
return handler(stream.Context(), stream)
}
return err
}
user, err := in.Store.GetUser(stream.Context(), &store.FindUser{
Username: &username,
})
if err != nil {
return errors.Wrap(err, "failed to get user")
}
if user == nil {
return errors.Errorf("user %q not exists", username)
}
if user.RowStatus == store.Archived {
return errors.Errorf("user %q is archived", username)
}
if isOnlyForAdminAllowedMethod(serverInfo.FullMethod) && user.Role != store.RoleHost && user.Role != store.RoleAdmin {
return errors.Errorf("user %q is not admin", username)
}

// Stores userID into context.
childCtx := context.WithValue(stream.Context(), usernameContextKey, username)

return handler(srv, newWrappedStream(childCtx, stream))
}

func (in *GRPCAuthInterceptor) authenticate(ctx context.Context, accessToken string) (string, error) {
if accessToken == "" {
return "", status.Errorf(codes.Unauthenticated, "access token not found")
221 changes: 144 additions & 77 deletions api/v2/memo_service.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package v2

import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"fmt"
@@ -32,6 +34,7 @@ import (
const (
DefaultPageSize = 10
MaxContentLength = 8 * 1024
ChunkSize = 64 * 1024 // 64 KiB
)

func (s *APIV2Service) CreateMemo(ctx context.Context, request *apiv2pb.CreateMemoRequest) (*apiv2pb.CreateMemoResponse, error) {
@@ -100,84 +103,9 @@ func (s *APIV2Service) CreateMemo(ctx context.Context, request *apiv2pb.CreateMe
}

func (s *APIV2Service) ListMemos(ctx context.Context, request *apiv2pb.ListMemosRequest) (*apiv2pb.ListMemosResponse, error) {
memoFind := &store.FindMemo{
// Exclude comments by default.
ExcludeComments: true,
}
if request.Filter != "" {
filter, err := parseListMemosFilter(request.Filter)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
}
if len(filter.ContentSearch) > 0 {
memoFind.ContentSearch = filter.ContentSearch
}
if len(filter.Visibilities) > 0 {
memoFind.VisibilityList = filter.Visibilities
}
if filter.OrderByPinned {
memoFind.OrderByPinned = filter.OrderByPinned
}
if filter.DisplayTimeAfter != nil {
displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
}
if displayWithUpdatedTs {
memoFind.UpdatedTsAfter = filter.DisplayTimeAfter
} else {
memoFind.CreatedTsAfter = filter.DisplayTimeAfter
}
}
if filter.DisplayTimeBefore != nil {
displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
}
if displayWithUpdatedTs {
memoFind.UpdatedTsBefore = filter.DisplayTimeBefore
} else {
memoFind.CreatedTsBefore = filter.DisplayTimeBefore
}
}
if filter.Creator != nil {
username, err := ExtractUsernameFromName(*filter.Creator)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid creator name")
}
user, err := s.Store.GetUser(ctx, &store.FindUser{
Username: &username,
})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get user")
}
if user == nil {
return nil, status.Errorf(codes.NotFound, "user not found")
}
memoFind.CreatorID = &user.ID
}
if filter.RowStatus != nil {
memoFind.RowStatus = filter.RowStatus
}
} else {
return nil, status.Errorf(codes.InvalidArgument, "filter is required")
}

user, _ := getCurrentUser(ctx, s.Store)
// If the user is not authenticated, only public memos are visible.
if user == nil {
memoFind.VisibilityList = []store.Visibility{store.Public}
}
if user != nil && memoFind.CreatorID != nil && *memoFind.CreatorID != user.ID {
memoFind.VisibilityList = []store.Visibility{store.Public, store.Protected}
}

displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
memoFind, err := s.buildFindMemosWithFilter(ctx, request.Filter, true)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
}
if displayWithUpdatedTs {
memoFind.OrderByUpdatedTs = true
return nil, err
}

var limit, offset int
@@ -621,6 +549,61 @@ func (s *APIV2Service) GetUserMemosStats(ctx context.Context, request *apiv2pb.G
return response, nil
}

func (s *APIV2Service) ExportMemos(request *apiv2pb.ExportMemosRequest, srv apiv2pb.MemoService_ExportMemosServer) error {
ctx := srv.Context()
fmt.Printf("%+v\n", ctx)
memoFind, err := s.buildFindMemosWithFilter(ctx, request.Filter, true)
if err != nil {
return err
}

memos, err := s.Store.ListMemos(ctx, memoFind)
if err != nil {
return err
}

buf := new(bytes.Buffer)
writer := zip.NewWriter(buf)

for _, memo := range memos {
memoMessage, err := s.convertMemoFromStore(ctx, memo)
log.Info(memoMessage.Content)
if err != nil {
return errors.Wrap(err, "failed to convert memo")
}
file, err := writer.Create(time.Unix(memo.CreatedTs, 0).Format(time.RFC3339) + ".md")
if err != nil {
return status.Errorf(codes.Internal, "Failed to create memo file")
}
_, err = file.Write([]byte(memoMessage.Content))
if err != nil {
return status.Errorf(codes.Internal, "Failed to write to memo file")
}
}

err = writer.Close()
if err != nil {
return status.Errorf(codes.Internal, "Failed to close zip file writer")
}

exportChunk := &apiv2pb.ExportMemosResponse{}
sizeOfFile := len(buf.Bytes())
for currentByte := 0; currentByte < sizeOfFile; currentByte += ChunkSize {
if currentByte+ChunkSize > sizeOfFile {
exportChunk.File = buf.Bytes()[currentByte:sizeOfFile]
} else {
exportChunk.File = buf.Bytes()[currentByte : currentByte+ChunkSize]
}

err := srv.Send(exportChunk)
if err != nil {
return status.Error(codes.Internal, "Unable to stream ExportMemosResponse chunk")
}
}

return nil
}

func (s *APIV2Service) convertMemoFromStore(ctx context.Context, memo *store.Memo) (*apiv2pb.Memo, error) {
rawNodes, err := parser.Parse(tokenizer.Tokenize(memo.Content))
if err != nil {
@@ -847,6 +830,90 @@ func (s *APIV2Service) dispatchMemoRelatedWebhook(ctx context.Context, memo *api
return nil
}

func (s *APIV2Service) buildFindMemosWithFilter(ctx context.Context, filter string, excludeComments bool) (*store.FindMemo, error) {
memoFind := &store.FindMemo{
// Exclude comments by default.
ExcludeComments: excludeComments,
}
if filter != "" {
filter, err := parseListMemosFilter(filter)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)
}
if len(filter.ContentSearch) > 0 {
memoFind.ContentSearch = filter.ContentSearch
}
if len(filter.Visibilities) > 0 {
memoFind.VisibilityList = filter.Visibilities
}
if filter.OrderByPinned {
memoFind.OrderByPinned = filter.OrderByPinned
}
if filter.DisplayTimeAfter != nil {
displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
}
if displayWithUpdatedTs {
memoFind.UpdatedTsAfter = filter.DisplayTimeAfter
} else {
memoFind.CreatedTsAfter = filter.DisplayTimeAfter
}
}
if filter.DisplayTimeBefore != nil {
displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
}
if displayWithUpdatedTs {
memoFind.UpdatedTsBefore = filter.DisplayTimeBefore
} else {
memoFind.CreatedTsBefore = filter.DisplayTimeBefore
}
}
if filter.Creator != nil {
username, err := ExtractUsernameFromName(*filter.Creator)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "invalid creator name")
}
user, err := s.Store.GetUser(ctx, &store.FindUser{
Username: &username,
})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get user")
}
if user == nil {
return nil, status.Errorf(codes.NotFound, "user not found")
}
memoFind.CreatorID = &user.ID
}
if filter.RowStatus != nil {
memoFind.RowStatus = filter.RowStatus
}
} else {
return nil, status.Errorf(codes.InvalidArgument, "filter is required")
}

user, _ := getCurrentUser(ctx, s.Store)
// If the user is not authenticated, only public memos are visible.
if user == nil {
memoFind.VisibilityList = []store.Visibility{store.Public}
}
if user != nil && memoFind.CreatorID != nil && *memoFind.CreatorID != user.ID {
memoFind.VisibilityList = []store.Visibility{store.Public, store.Protected}
}

displayWithUpdatedTs, err := s.getMemoDisplayWithUpdatedTsSettingValue(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get memo display with updated ts setting value")
}
if displayWithUpdatedTs {
memoFind.OrderByUpdatedTs = true
}

return memoFind, nil
}

func convertMemoToWebhookPayload(memo *apiv2pb.Memo) *webhook.WebhookPayload {
return &webhook.WebhookPayload{
CreatorID: memo.CreatorId,
3 changes: 3 additions & 0 deletions api/v2/v2.go
Original file line number Diff line number Diff line change
@@ -47,6 +47,9 @@ func NewAPIV2Service(secret string, profile *profile.Profile, store *store.Store
grpc.ChainUnaryInterceptor(
authProvider.AuthenticationInterceptor,
),
grpc.ChainStreamInterceptor(
authProvider.StreamAuthenticationInterceptor,
),
)
apiv2Service := &APIV2Service{
Secret: secret,
13 changes: 13 additions & 0 deletions proto/api/v2/memo_service.proto
Original file line number Diff line number Diff line change
@@ -90,6 +90,10 @@ service MemoService {
option (google.api.http) = {get: "/api/v2/memos/stats"};
option (google.api.method_signature) = "username";
}

rpc ExportMemos(ExportMemosRequest) returns (stream ExportMemosResponse) {
option (google.api.http) = {get: "/api/v2/memos/export"};
}
}

enum Visibility {
@@ -273,3 +277,12 @@ message GetUserMemosStatsResponse {
// key is the year-month-day string. e.g. "2020-01-01".
map<string, int32> stats = 1;
}

message ExportMemosRequest {
// Same as ListMemosRequest.filter
string filter = 1;
}

message ExportMemosResponse {
bytes file = 1;
}
33 changes: 33 additions & 0 deletions proto/gen/api/v2/README.md
Original file line number Diff line number Diff line change
@@ -134,6 +134,8 @@
- [CreateMemoResponse](#memos-api-v2-CreateMemoResponse)
- [DeleteMemoRequest](#memos-api-v2-DeleteMemoRequest)
- [DeleteMemoResponse](#memos-api-v2-DeleteMemoResponse)
- [ExportMemosRequest](#memos-api-v2-ExportMemosRequest)
- [ExportMemosResponse](#memos-api-v2-ExportMemosResponse)
- [GetMemoByNameRequest](#memos-api-v2-GetMemoByNameRequest)
- [GetMemoByNameResponse](#memos-api-v2-GetMemoByNameResponse)
- [GetMemoRequest](#memos-api-v2-GetMemoRequest)
@@ -1939,6 +1941,36 @@ Used internally for obfuscating the page token.



<a name="memos-api-v2-ExportMemosRequest"></a>

### ExportMemosRequest



| Field | Type | Label | Description |
| ----- | ---- | ----- | ----------- |
| filter | [string](#string) | | Same as ListMemosRequest.filter |






<a name="memos-api-v2-ExportMemosResponse"></a>

### ExportMemosResponse



| Field | Type | Label | Description |
| ----- | ---- | ----- | ----------- |
| file | [bytes](#bytes) | | |






<a name="memos-api-v2-GetMemoByNameRequest"></a>

### GetMemoByNameRequest
@@ -2323,6 +2355,7 @@ Used internally for obfuscating the page token.
| CreateMemoComment | [CreateMemoCommentRequest](#memos-api-v2-CreateMemoCommentRequest) | [CreateMemoCommentResponse](#memos-api-v2-CreateMemoCommentResponse) | CreateMemoComment creates a comment for a memo. |
| ListMemoComments | [ListMemoCommentsRequest](#memos-api-v2-ListMemoCommentsRequest) | [ListMemoCommentsResponse](#memos-api-v2-ListMemoCommentsResponse) | ListMemoComments lists comments for a memo. |
| GetUserMemosStats | [GetUserMemosStatsRequest](#memos-api-v2-GetUserMemosStatsRequest) | [GetUserMemosStatsResponse](#memos-api-v2-GetUserMemosStatsResponse) | GetUserMemosStats gets stats of memos for a user. |
| ExportMemos | [ExportMemosRequest](#memos-api-v2-ExportMemosRequest) | [ExportMemosResponse](#memos-api-v2-ExportMemosResponse) stream | |



446 changes: 291 additions & 155 deletions proto/gen/api/v2/memo_service.pb.go

Large diffs are not rendered by default.

61 changes: 61 additions & 0 deletions proto/gen/api/v2/memo_service.pb.gw.go
67 changes: 66 additions & 1 deletion proto/gen/api/v2/memo_service_grpc.pb.go
16 changes: 16 additions & 0 deletions web/src/components/Settings/MyAccountSection.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { Button } from "@mui/joy";
import { memoServiceClient } from "@/grpcweb";
import { downloadFileFromUrl } from "@/helpers/utils";
import useCurrentUser from "@/hooks/useCurrentUser";
import { useTranslate } from "@/utils/i18n";
import showChangePasswordDialog from "../ChangePasswordDialog";
@@ -10,6 +12,17 @@ const MyAccountSection = () => {
const t = useTranslate();
const user = useCurrentUser();

const downloadExportedMemos = async (user: any) => {
const chunks = [];
for await (const response of memoServiceClient.exportMemos({ filter: `creator == "${user.name}"` })) {
chunks.push(response.file.buffer);
}
const blob = new Blob(chunks);
const downloadUrl = window.URL.createObjectURL(blob);
downloadFileFromUrl(downloadUrl, "memos-export.zip");
URL.revokeObjectURL(downloadUrl);
};

return (
<div className="w-full gap-2 pt-2 pb-4">
<p className="font-medium text-gray-700 dark:text-gray-500">{t("setting.account-section.title")}</p>
@@ -27,6 +40,9 @@ const MyAccountSection = () => {
<Button variant="outlined" onClick={showChangePasswordDialog}>
{t("setting.account-section.change-password")}
</Button>
<Button variant="outlined" onClick={() => downloadExportedMemos(user)}>
{t("setting.account-section.export-memos")}
</Button>
</div>

<AccessTokenSection />
13 changes: 5 additions & 8 deletions web/src/components/ShareMemoDialog.tsx
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ import copy from "copy-to-clipboard";
import React, { useEffect, useRef } from "react";
import { toast } from "react-hot-toast";
import { getDateTimeString } from "@/helpers/datetime";
import { downloadFileFromUrl } from "@/helpers/utils";
import useLoading from "@/hooks/useLoading";
import toImage from "@/labs/html2image";
import { useUserStore, extractUsernameFromName } from "@/store/v1";
@@ -51,6 +52,7 @@ const ShareMemoDialog: React.FC<Props> = (props: Props) => {
.then((url) => {
downloadFileFromUrl(url, `memos-${getDateTimeString(Date.now())}.png`);
downloadingImageState.setFinish();
URL.revokeObjectURL(url);
})
.catch((err) => {
console.error(err);
@@ -59,14 +61,9 @@ const ShareMemoDialog: React.FC<Props> = (props: Props) => {

const handleDownloadTextFileBtnClick = () => {
const blob = new Blob([memo.content], { type: "text/plain;charset=utf-8" });
downloadFileFromUrl(URL.createObjectURL(blob), `memos-${getDateTimeString(Date.now())}.md`);
};

const downloadFileFromUrl = (url: string, filename: string) => {
const a = document.createElement("a");
a.href = url;
a.download = filename;
a.click();
const url = URL.createObjectURL(blob);
downloadFileFromUrl(url, `memos-${getDateTimeString(Date.now())}.md`);
URL.revokeObjectURL(url);
};

const handleCopyLinkBtnClick = () => {
8 changes: 8 additions & 0 deletions web/src/helpers/utils.ts
Original file line number Diff line number Diff line change
@@ -92,3 +92,11 @@ export const isValidUrl = (url: string): boolean => {
return false;
}
};

export const downloadFileFromUrl = (url: string, filename: string) => {
const a = document.createElement("a");
a.href = url;
a.download = filename;
a.click();
a.remove();
};
1 change: 1 addition & 0 deletions web/src/locales/en.json
Original file line number Diff line number Diff line change
@@ -177,6 +177,7 @@
"email-note": "Optional",
"update-information": "Update Information",
"change-password": "Change password",
"export-memos": "Export Memos",
"reset-api": "Reset API",
"openapi-title": "OpenAPI",
"openapi-reset": "Reset OpenAPI Key",
3 changes: 2 additions & 1 deletion web/src/locales/es.json
Original file line number Diff line number Diff line change
@@ -92,7 +92,8 @@
"account-section": {
"title": "Información de la Cuenta",
"update-information": "Actualizar Información",
"change-password": "Cambiar Contraseña"
"change-password": "Cambiar Contraseña",
"export-memos": "Exportar Notas"
},
"preference-section": {
"theme": "Tema",