diff --git a/packages/global/common/file/s3TTL/type.d.ts b/packages/global/common/file/s3TTL/type.d.ts new file mode 100644 index 000000000000..f9442368d0ca --- /dev/null +++ b/packages/global/common/file/s3TTL/type.d.ts @@ -0,0 +1,6 @@ +export type S3TtlSchemaType = { + _id: string; + bucketName: string; + minioKey: string; + expiredTime: Date; +}; diff --git a/packages/global/common/tsRest/fastgpt/contracts/core/chat/setting.ts b/packages/global/common/tsRest/fastgpt/contracts/core/chat/setting.ts index a63f21c9a509..0e2057acce6c 100644 --- a/packages/global/common/tsRest/fastgpt/contracts/core/chat/setting.ts +++ b/packages/global/common/tsRest/fastgpt/contracts/core/chat/setting.ts @@ -8,7 +8,7 @@ import { ObjectIdSchema } from '../../../../../type'; import { initContract } from '@ts-rest/core'; const c = initContract(); -const favouriteContract = c.router({ +export const favouriteContract = c.router({ list: { path: '/proApi/core/chat/setting/favourite/list', method: 'GET', diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index a661a9b77d82..196829c759ab 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -109,23 +109,6 @@ const getIdFromPath = (path?: string) => { return id; }; -// 删除旧的头像,新的头像去除过期时间 -export const refreshSourceAvatar = async ( - path?: string, - oldPath?: string, - session?: ClientSession -) => { - const newId = getIdFromPath(path); - const oldId = getIdFromPath(oldPath); - - if (!newId || newId === oldId) return; - - await MongoImage.updateOne({ _id: newId }, { $unset: { expiredTime: 1 } }, { session }); - - if (oldId) { - await MongoImage.deleteOne({ _id: oldId }, { session }); - } -}; export const removeImageByPath = (path?: string, session?: ClientSession) => { if (!path) return; diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts new file mode 100644 index 000000000000..abb319d96178 --- /dev/null +++ b/packages/service/common/s3/buckets/base.ts @@ -0,0 +1,132 @@ +import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio'; +import { + type ExtensionType, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3BucketName, + type S3OptionsType +} from '../type'; +import { defaultS3Options, Mimes } from '../constants'; +import path from 'node:path'; +import { MongoS3TTL } from '../schema'; +import { UserError } from '@fastgpt/global/common/error/utils'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import { addHours } from 'date-fns'; + +export class S3BaseBucket { + private _client: Client; + private _externalClient: Client | undefined; + + /** + * + * @param _bucket the bucket you want to operate + * @param options the options for the s3 client + */ + constructor( + private readonly _bucket: S3BucketName, + public options: Partial = defaultS3Options + ) { + options = { ...defaultS3Options, ...options }; + this.options = options; + this._client = new Client(options as S3OptionsType); + + if (this.options.externalBaseURL) { + const externalBaseURL = new URL(this.options.externalBaseURL); + const endpoint = externalBaseURL.hostname; + const useSSL = externalBaseURL.protocol === 'https'; + + this._externalClient = new Client({ + useSSL: useSSL, + endPoint: endpoint, + port: options.port, + accessKey: options.accessKey, + secretKey: options.secretKey, + transportAgent: options.transportAgent + }); + } + + const init = async () => { + if (!(await this.exist())) { + await this.client.makeBucket(this._bucket); + } + await this.options.afterInit?.(); + }; + init(); + } + + get name(): string { + return this._bucket; + } + + protected get client(): Client { + return this._externalClient ?? this._client; + } + + move(src: string, dst: string, options?: CopyConditions): Promise { + const bucket = this.name; + this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options); + return this.delete(src); + } + + copy(src: string, dst: string, options?: CopyConditions): ReturnType { + return this.client.copyObject(this.name, src, dst, options); + } + + exist(): Promise { + return this.client.bucketExists(this.name); + } + + delete(objectKey: string, options?: RemoveOptions): Promise { + return this.client.removeObject(this.name, objectKey, options); + } + + async createPostPresignedUrl( + params: CreatePostPresignedUrlParams, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + try { + const { expiredHours } = options; + const filename = params.filename; + const ext = path.extname(filename).toLowerCase() as ExtensionType; + const contentType = Mimes[ext] ?? 'application/octet-stream'; + const maxFileSize = this.options.maxFileSize as number; + + const key = (() => { + if ('rawKey' in params) return params.rawKey; + + return `${params.source}/${params.teamId}/${getNanoid(6)}-${filename}`; + })(); + + const policy = this.client.newPostPolicy(); + policy.setKey(key); + policy.setBucket(this.name); + policy.setContentType(contentType); + policy.setContentLengthRange(1, maxFileSize); + policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); + policy.setUserMetaData({ + 'content-type': contentType, + 'content-disposition': `attachment; filename="${encodeURIComponent(filename)}"`, + 'origin-filename': encodeURIComponent(filename), + 'upload-time': new Date().toISOString() + }); + + const { formData, postURL } = await this.client.presignedPostPolicy(policy); + + if (expiredHours) { + await MongoS3TTL.create({ + minioKey: key, + bucketName: this.name, + expiredTime: addHours(new Date(), expiredHours) + }); + } + + return { + url: postURL, + fields: formData + }; + } catch (error) { + return Promise.reject(error); + } + } +} diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts new file mode 100644 index 000000000000..1d85712f7191 --- /dev/null +++ b/packages/service/common/s3/buckets/private.ts @@ -0,0 +1,9 @@ +import { S3BaseBucket } from './base'; +import { S3Buckets } from '../constants'; +import { type S3OptionsType } from '../type'; + +export class S3PrivateBucket extends S3BaseBucket { + constructor(options?: Partial) { + super(S3Buckets.private, options); + } +} diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts new file mode 100644 index 000000000000..99a8b4cc9fc4 --- /dev/null +++ b/packages/service/common/s3/buckets/public.ts @@ -0,0 +1,51 @@ +import { S3BaseBucket } from './base'; +import { S3Buckets } from '../constants'; +import { type S3OptionsType } from '../type'; + +export class S3PublicBucket extends S3BaseBucket { + constructor(options?: Partial) { + super(S3Buckets.public, { + ...options, + afterInit: async () => { + const bucket = this.name; + const policy = JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucket}/*` + } + ] + }); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // NOTE: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + console.error('Failed to set bucket policy:', error); + } + } + }); + } + + createPublicUrl(objectKey: string): string { + const protocol = this.options.useSSL ? 'https' : 'http'; + const hostname = this.options.endPoint; + const port = this.options.port; + const bucket = this.name; + + const url = new URL(`${protocol}://${hostname}:${port}/${bucket}/${objectKey}`); + + if (this.options.externalBaseURL) { + const externalBaseURL = new URL(this.options.externalBaseURL); + + url.port = externalBaseURL.port; + url.hostname = externalBaseURL.hostname; + url.protocol = externalBaseURL.protocol; + } + + return url.toString(); + } +} diff --git a/packages/service/common/s3/config.ts b/packages/service/common/s3/config.ts deleted file mode 100644 index 501d61be4b1a..000000000000 --- a/packages/service/common/s3/config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { S3ServiceConfig } from './type'; - -export const defualtS3Config: Omit = { - endPoint: process.env.S3_ENDPOINT || 'localhost', - port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, - useSSL: process.env.S3_USE_SSL === 'true', - accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', - secretKey: process.env.S3_SECRET_KEY || 'minioadmin', - externalBaseURL: process.env.S3_EXTERNAL_BASE_URL -}; diff --git a/packages/service/common/s3/const.ts b/packages/service/common/s3/const.ts deleted file mode 100644 index c91043dc1f96..000000000000 --- a/packages/service/common/s3/const.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const mimeMap: Record = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.txt': 'text/plain', - '.json': 'application/json', - '.csv': 'text/csv', - '.zip': 'application/zip', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.doc': 'application/msword', - '.xls': 'application/vnd.ms-excel', - '.ppt': 'application/vnd.ms-powerpoint', - '.js': 'application/javascript' -}; diff --git a/packages/service/common/s3/constants.ts b/packages/service/common/s3/constants.ts new file mode 100644 index 000000000000..579b86e85773 --- /dev/null +++ b/packages/service/common/s3/constants.ts @@ -0,0 +1,58 @@ +import type { S3PrivateBucket } from './buckets/private'; +import type { S3PublicBucket } from './buckets/public'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import type { ClientOptions } from 'minio'; + +export const Mimes = { + '.gif': 'image/gif', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + + '.csv': 'text/csv', + '.txt': 'text/plain', + + '.pdf': 'application/pdf', + '.zip': 'application/zip', + '.json': 'application/json', + '.doc': 'application/msword', + '.js': 'application/javascript', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' +} as const; + +export const defaultS3Options: { + externalBaseURL?: string; + maxFileSize?: number; + afterInit?: () => Promise | void; +} & ClientOptions = { + maxFileSize: 1024 ** 3, // 1GB + + useSSL: process.env.S3_USE_SSL === 'true', + endPoint: process.env.S3_ENDPOINT || 'localhost', + externalBaseURL: process.env.S3_EXTERNAL_BASE_URL, + accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', + secretKey: process.env.S3_SECRET_KEY || 'minioadmin', + port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined +}; + +export const S3Buckets = { + public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public', + private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private' +} as const; + +export const S3BucketMap = { + public: null as unknown as S3PublicBucket, + private: null as unknown as S3PrivateBucket +}; diff --git a/packages/service/common/s3/controller.ts b/packages/service/common/s3/controller.ts index faaae7cdc91e..cbeb209b4766 100644 --- a/packages/service/common/s3/controller.ts +++ b/packages/service/common/s3/controller.ts @@ -1,168 +1,62 @@ -import { Client } from 'minio'; -import { - type FileMetadataType, - type PresignedUrlInput as UploadPresignedURLProps, - type UploadPresignedURLResponse, - type S3ServiceConfig -} from './type'; -import { defualtS3Config } from './config'; -import { randomBytes } from 'crypto'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { extname } from 'path'; -import { addLog } from '../../common/system/log'; -import { getErrText } from '@fastgpt/global/common/error/utils'; -import { mimeMap } from './const'; - -export class S3Service { - private client: Client; - private config: S3ServiceConfig; - private initialized: boolean = false; - initFunction?: () => Promise; - - constructor(config?: Partial) { - this.config = { ...defualtS3Config, ...config } as S3ServiceConfig; - - this.client = new Client({ - endPoint: this.config.endPoint, - port: this.config.port, - useSSL: this.config.useSSL, - accessKey: this.config.accessKey, - secretKey: this.config.secretKey, - transportAgent: process.env.HTTP_PROXY - ? new HttpProxyAgent(process.env.HTTP_PROXY) - : process.env.HTTPS_PROXY - ? new HttpsProxyAgent(process.env.HTTPS_PROXY) - : undefined - }); - - this.initFunction = config?.initFunction; - } - - public async init() { - if (!this.initialized) { - if (!(await this.client.bucketExists(this.config.bucket))) { - addLog.debug(`Creating bucket: ${this.config.bucket}`); - await this.client.makeBucket(this.config.bucket); - } - - await this.initFunction?.(); - this.initialized = true; - } - } - - private generateFileId(): string { - return randomBytes(16).toString('hex'); - } - - private generateAccessUrl(filename: string): string { - const protocol = this.config.useSSL ? 'https' : 'http'; - const port = - this.config.port && this.config.port !== (this.config.useSSL ? 443 : 80) - ? `:${this.config.port}` - : ''; - - const externalBaseURL = this.config.externalBaseURL; - return externalBaseURL - ? `${externalBaseURL}/${this.config.bucket}/${encodeURIComponent(filename)}` - : `${protocol}://${this.config.endPoint}${port}/${this.config.bucket}/${encodeURIComponent(filename)}`; - } - - uploadFile = async (fileBuffer: Buffer, originalFilename: string): Promise => { - await this.init(); - const inferContentType = (filename: string) => { - const ext = extname(filename).toLowerCase(); - return mimeMap[ext] || 'application/octet-stream'; - }; - - if (this.config.maxFileSize && fileBuffer.length > this.config.maxFileSize) { - return Promise.reject( - `File size ${fileBuffer.length} exceeds limit ${this.config.maxFileSize}` - ); +import { MongoS3TTL } from './schema'; +import { addLog } from '../system/log'; +import { setCron } from '../system/cron'; +import { checkTimerLock } from '../system/timerLock/utils'; +import { TimerIdEnum } from '../system/timerLock/constants'; +import { S3BucketMap } from './constants'; + +export async function clearExpiredMinioFiles() { + try { + const now = new Date(); + + const expiredFiles = await MongoS3TTL.find({ expiredTime: { $lte: now } }).lean(); + if (expiredFiles.length === 0) { + addLog.info('No expired minio files to clean'); + return; } - const fileId = this.generateFileId(); - const objectName = `${fileId}-${originalFilename}`; - const uploadTime = new Date(); - - const contentType = inferContentType(originalFilename); - await this.client.putObject(this.config.bucket, objectName, fileBuffer, fileBuffer.length, { - 'Content-Type': contentType, - 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, - 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), - 'x-amz-meta-upload-time': uploadTime.toISOString() - }); - - const metadata: FileMetadataType = { - fileId, - originalFilename, - contentType, - size: fileBuffer.length, - uploadTime, - accessUrl: this.generateAccessUrl(objectName) - }; - - return metadata; - }; - - generateUploadPresignedURL = async ({ - filepath, - contentType, - metadata, - filename - }: UploadPresignedURLProps): Promise => { - await this.init(); - const objectName = `${filepath}/${filename}`; - - try { - const policy = this.client.newPostPolicy(); - - policy.setBucket(this.config.bucket); - policy.setKey(objectName); - if (contentType) { - policy.setContentType(contentType); - } - if (this.config.maxFileSize) { - policy.setContentLengthRange(1, this.config.maxFileSize); + addLog.info(`Found ${expiredFiles.length} expired minio files to clean`); + + let success = 0; + let fail = 0; + + for (const file of expiredFiles) { + try { + const bucketName = file.bucketName as keyof typeof S3BucketMap; + const bucket = S3BucketMap[bucketName]; + if (bucket) { + await bucket.delete(file.minioKey); + await MongoS3TTL.deleteOne({ _id: file._id }); + + success++; + addLog.info( + `Deleted expired minio file: ${file.minioKey} from bucket: ${file.bucketName}` + ); + } else { + addLog.warn(`Bucket not found: ${file.bucketName}`); + } + } catch (error) { + fail++; + addLog.error(`Failed to delete minio file: ${file.minioKey}`, error); } - policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); // 10 mins - - policy.setUserMetaData({ - 'original-filename': encodeURIComponent(filename), - 'upload-time': new Date().toISOString(), - ...metadata - }); - - const { postURL, formData } = await this.client.presignedPostPolicy(policy); - - const response: UploadPresignedURLResponse = { - objectName, - uploadUrl: postURL, - formData - }; - - return response; - } catch (error) { - addLog.error('Failed to generate Upload Presigned URL', error); - return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); } - }; - generateDownloadUrl = (objectName: string): string => { - const pathParts = objectName.split('/'); - const encodedParts = pathParts.map((part) => encodeURIComponent(part)); - const encodedObjectName = encodedParts.join('/'); - return `${this.config.bucket}/${encodedObjectName}`; - }; - - getFile = async (objectName: string): Promise => { - const stat = await this.client.statObject(this.config.bucket, objectName); + addLog.info(`Minio TTL cleanup completed. Success: ${success}, Failed: ${fail}`); + } catch (error) { + addLog.error('Error in clearExpiredMinioFiles', error); + } +} - if (stat.size > 0) { - const accessUrl = this.generateDownloadUrl(objectName); - return accessUrl; +export function clearExpiredMinioFilesCron() { + // 每小时执行一次 + setCron('0 */1 * * *', async () => { + if ( + await checkTimerLock({ + timerId: TimerIdEnum.clearExpiredMinioFiles, + lockMinuted: 59 + }) + ) { + await clearExpiredMinioFiles(); } - - return Promise.reject(`File ${objectName} not found`); - }; + }); } diff --git a/packages/service/common/s3/index.ts b/packages/service/common/s3/index.ts index 761bd564097e..aeff13d813d9 100644 --- a/packages/service/common/s3/index.ts +++ b/packages/service/common/s3/index.ts @@ -1,16 +1,8 @@ -import { S3Service } from './controller'; +import { S3PublicBucket } from './buckets/public'; +import { S3BucketMap } from './constants'; +import { S3PrivateBucket } from './buckets/private'; -export const PluginS3Service = (() => { - if (!global.pluginS3Service) { - global.pluginS3Service = new S3Service({ - bucket: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', - maxFileSize: 50 * 1024 * 1024 // 50MB - }); - } - - return global.pluginS3Service; -})(); - -declare global { - var pluginS3Service: S3Service; +export function initS3Buckets() { + S3BucketMap.public = new S3PublicBucket(); + S3BucketMap.private = new S3PrivateBucket(); } diff --git a/packages/service/common/s3/schema.ts b/packages/service/common/s3/schema.ts new file mode 100644 index 000000000000..169e2a1a519b --- /dev/null +++ b/packages/service/common/s3/schema.ts @@ -0,0 +1,24 @@ +import { Schema, getMongoModel } from '../mongo'; +import { type S3TtlSchemaType } from '@fastgpt/global/common/file/s3TTL/type'; + +const collectionName = 's3_ttls'; + +const S3TTLSchema = new Schema({ + bucketName: { + type: String, + required: true + }, + minioKey: { + type: String, + required: true + }, + expiredTime: { + type: Date, + required: true + } +}); + +S3TTLSchema.index({ expiredTime: 1 }); +S3TTLSchema.index({ bucketName: 1, minioKey: 1 }); + +export const MongoS3TTL = getMongoModel(collectionName, S3TTLSchema); diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts new file mode 100644 index 000000000000..9d1614da58f2 --- /dev/null +++ b/packages/service/common/s3/sources/avatar.ts @@ -0,0 +1,70 @@ +import { S3Sources } from '../type'; +import { MongoS3TTL } from '../schema'; +import { S3PublicBucket } from '../buckets/public'; +import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; +import type { ClientSession } from 'mongoose'; + +class S3AvatarSource { + private bucket: S3PublicBucket; + private static instance: S3AvatarSource; + + constructor() { + this.bucket = new S3PublicBucket(); + } + + static getInstance() { + return (this.instance ??= new S3AvatarSource()); + } + + get prefix(): string { + return imageBaseUrl; + } + + async createUploadAvatarURL({ + filename, + teamId, + autoExpired = true + }: { + filename: string; + teamId: string; + autoExpired?: boolean; + }) { + return this.bucket.createPostPresignedUrl( + { filename, teamId, source: S3Sources.avatar }, + { expiredHours: autoExpired ? 1 : undefined } // 1 Hourse + ); + } + + createPublicUrl(objectKey: string): string { + return this.bucket.createPublicUrl(objectKey); + } + + async removeAvatarTTL(avatar: string, session?: ClientSession): Promise { + const key = avatar.slice(this.prefix.length); + await MongoS3TTL.deleteOne({ minioKey: key, bucketName: this.bucket.name }, session); + } + + async deleteAvatar(avatar: string, session?: ClientSession): Promise { + const key = avatar.slice(this.prefix.length); + await MongoS3TTL.deleteOne({ minioKey: key, bucketName: this.bucket.name }, session); + await this.bucket.delete(key); + } + + async refreshAvatar(newAvatar?: string, oldAvatar?: string, session?: ClientSession) { + if (!newAvatar || newAvatar === oldAvatar) return; + + // remove the TTL for the new avatar + await this.removeAvatarTTL(newAvatar, session); + + if (oldAvatar) { + // delete the old avatar + // 1. delete the TTL record if it exists + // 2. delete the avatar in S3 + await this.deleteAvatar(oldAvatar, session); + } + } +} + +export function getS3AvatarSource() { + return S3AvatarSource.getInstance(); +} diff --git a/packages/service/common/s3/type.ts b/packages/service/common/s3/type.ts index a480530493e8..4250da401c9d 100644 --- a/packages/service/common/s3/type.ts +++ b/packages/service/common/s3/type.ts @@ -1,49 +1,49 @@ -import type { ClientOptions } from 'minio'; - -export type S3ServiceConfig = { - bucket: string; - externalBaseURL?: string; - /** - * Unit: Byte - */ - maxFileSize?: number; - /** - * for executing some init function for the s3 service - */ - initFunction?: () => Promise; -} & ClientOptions; - -export type FileMetadataType = { - fileId: string; - originalFilename: string; - contentType: string; - size: number; - uploadTime: Date; - accessUrl: string; -}; - -export type PresignedUrlInput = { - filepath: string; - filename: string; - contentType?: string; - metadata?: Record; -}; - -export type UploadPresignedURLResponse = { - objectName: string; - uploadUrl: string; - formData: Record; -}; - -export type FileUploadInput = { - buffer: Buffer; - filename: string; -}; - -export enum PluginTypeEnum { - tool = 'tool' -} - -export const PluginFilePath = { - [PluginTypeEnum.tool]: 'plugin/tools' -}; +import { z } from 'zod'; +import type { defaultS3Options, Mimes, S3Buckets } from './constants'; + +export const S3MetadataSchema = z.object({ + filename: z.string(), + uploadedAt: z.date(), + accessUrl: z.string(), + contentType: z.string(), + id: z.string().length(32), + size: z.number().positive() +}); +export type S3Metadata = z.infer; + +export type ContentType = (typeof Mimes)[keyof typeof Mimes]; +export type ExtensionType = keyof typeof Mimes; + +export type S3OptionsType = typeof defaultS3Options; + +export type S3BucketName = (typeof S3Buckets)[keyof typeof S3Buckets]; + +export const S3SourcesSchema = z.enum(['avatar']); +export const S3Sources = S3SourcesSchema.enum; +export type S3SourceType = z.infer; + +export const CreatePostPresignedUrlParamsSchema = z.union([ + // Option 1: Only rawKey + z.object({ + filename: z.string().min(1), + rawKey: z.string().min(1) + }), + // Option 2: filename with optional source and teamId + z.object({ + filename: z.string().min(1), + source: S3SourcesSchema.optional(), + teamId: z.string().length(16).optional() + }) +]); +export type CreatePostPresignedUrlParams = z.infer; + +export const CreatePostPresignedUrlOptionsSchema = z.object({ + expiredHours: z.number().positive().optional() // TTL in Hours, default 7 * 24 +}); +export type CreatePostPresignedUrlOptions = z.infer; + +export const CreatePostPresignedUrlResultSchema = z.object({ + url: z.string().min(1), + fields: z.record(z.string(), z.string()) +}); +export type CreatePostPresignedUrlResult = z.infer; diff --git a/packages/service/common/system/timerLock/constants.ts b/packages/service/common/system/timerLock/constants.ts index 76189686c775..2768d0085a19 100644 --- a/packages/service/common/system/timerLock/constants.ts +++ b/packages/service/common/system/timerLock/constants.ts @@ -8,7 +8,8 @@ export enum TimerIdEnum { notification = 'notification', clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer', - clearExpiredDatasetImage = 'clearExpiredDatasetImage' + clearExpiredDatasetImage = 'clearExpiredDatasetImage', + clearExpiredMinioFiles = 'clearExpiredMinioFiles' } export enum LockNotificationEnum { diff --git a/packages/service/package.json b/packages/service/package.json index eb367ff8ed38..41438d10f7af 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -54,7 +54,8 @@ "tiktoken": "1.0.17", "tunnel": "^0.0.6", "turndown": "^7.1.2", - "winston": "^3.17.0" + "winston": "^3.17.0", + "zod": "^3.24.2" }, "devDependencies": { "@types/cookie": "^0.5.2", diff --git a/packages/service/support/user/team/controller.ts b/packages/service/support/user/team/controller.ts index fa8ed440b390..ecae87871b71 100644 --- a/packages/service/support/user/team/controller.ts +++ b/packages/service/support/user/team/controller.ts @@ -17,7 +17,7 @@ import { mongoSessionRun } from '../../../common/mongo/sessionRun'; import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant'; import { getAIApi } from '../../../core/ai/config'; import { createRootOrg } from '../../permission/org/controllers'; -import { refreshSourceAvatar } from '../../../common/file/image/controller'; +import { getS3AvatarSource } from '../../../common/s3/sources/avatar'; async function getTeamMember(match: Record): Promise { const tmb = await MongoTeamMember.findOne(match).populate<{ team: TeamSchema }>('team').lean(); @@ -244,7 +244,7 @@ export async function updateTeam({ { session } ); - await refreshSourceAvatar(avatar, team?.avatar, session); + await getS3AvatarSource().refreshAvatar(avatar, team?.avatar, session); } }); } diff --git a/packages/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx new file mode 100644 index 000000000000..b6df4c00883f --- /dev/null +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -0,0 +1,91 @@ +import { base64ToFile, fileToBase64 } from '../utils'; +import { compressBase64Img } from '../img'; +import { useToast } from '../../../hooks/useToast'; +import { useCallback, useRef, useTransition } from 'react'; +import { useTranslation } from 'next-i18next'; +import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/type'; +import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; + +export const useUploadAvatar = ( + api: (params: { filename: string }) => Promise, + { onSuccess }: { onSuccess?: (avatar: string) => void } = {} +) => { + const { toast } = useToast(); + const { t } = useTranslation(); + const [uploading, startUpload] = useTransition(); + const uploadAvatarRef = useRef(null); + + const handleFileSelectorOpen = useCallback(() => { + if (!uploadAvatarRef.current) return; + uploadAvatarRef.current.click(); + }, []); + + // manually upload avatar + const handleUploadAvatar = useCallback( + async (file: File) => { + if (!file.name.match(/\.(jpg|png|jpeg)$/)) { + toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + return; + } + + startUpload(async () => { + const compressed = base64ToFile( + await compressBase64Img({ + base64Img: await fileToBase64(file), + maxW: 300, + maxH: 300 + }), + file.name + ); + const { url, fields } = await api({ filename: file.name }); + const formData = new FormData(); + Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); + formData.set('file', compressed); + const res = await fetch(url, { method: 'POST', body: formData }); // 204 + if (res.ok && res.status === 204) { + onSuccess?.(`${imageBaseUrl}${fields.key}`); + } + }); + }, + [t, toast, api, onSuccess] + ); + + const onUploadAvatarChange = useCallback( + async (e: React.ChangeEvent) => { + const files = e.target.files; + + if (!files || files.length === 0) { + e.target.value = ''; + return; + } + if (files.length > 1) { + toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + e.target.value = ''; + return; + } + const file = files[0]!; + handleUploadAvatar(file); + }, + [t, toast, handleUploadAvatar] + ); + + const Component = useCallback(() => { + return ( + + ); + }, [onUploadAvatarChange]); + + return { + uploading, + Component, + handleFileSelectorOpen, + handleUploadAvatar + }; +}; diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts index 7fd10744530f..f4107301d585 100644 --- a/packages/web/common/file/utils.ts +++ b/packages/web/common/file/utils.ts @@ -99,3 +99,24 @@ async function detectFileEncoding(file: File): Promise { return encoding || 'utf-8'; } + +export const fileToBase64 = (file: File) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); +}; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; diff --git a/packages/web/i18n/en/account_info.json b/packages/web/i18n/en/account_info.json index 15a520732bef..c420e057d285 100644 --- a/packages/web/i18n/en/account_info.json +++ b/packages/web/i18n/en/account_info.json @@ -9,6 +9,8 @@ "app_amount": "App amount", "avatar": "Avatar", "avatar_selection_exception": "Abnormal avatar selection", + "avatar_can_only_select_one": "Avatar can only select one picture", + "avatar_can_only_select_jpg_png": "Avatar can only select jpg or png format", "balance": "balance", "billing_standard": "Standards", "cancel": "Cancel", diff --git a/packages/web/i18n/zh-CN/account_info.json b/packages/web/i18n/zh-CN/account_info.json index cd20076cdcc6..7822d8765995 100644 --- a/packages/web/i18n/zh-CN/account_info.json +++ b/packages/web/i18n/zh-CN/account_info.json @@ -9,6 +9,8 @@ "app_amount": "应用数量", "avatar": "头像", "avatar_selection_exception": "头像选择异常", + "avatar_can_only_select_one": "头像只能选择一张图片", + "avatar_can_only_select_jpg_png": "头像只能选择 jpg 或 png 格式", "balance": "余额", "billing_standard": "计费标准", "cancel": "取消", diff --git a/packages/web/i18n/zh-Hant/account_info.json b/packages/web/i18n/zh-Hant/account_info.json index 9c5560d48d91..5646df649802 100644 --- a/packages/web/i18n/zh-Hant/account_info.json +++ b/packages/web/i18n/zh-Hant/account_info.json @@ -9,6 +9,8 @@ "app_amount": "應用數量", "avatar": "頭像", "avatar_selection_exception": "頭像選擇異常", + "avatar_can_only_select_one": "頭像只能選擇一張圖片", + "avatar_can_only_select_jpg_png": "頭像只能選擇 jpg 或 png 格式", "balance": "餘額", "billing_standard": "計費標準", "cancel": "取消", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5da78fe0a6c..5b12c66af728 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -297,6 +297,9 @@ importers: winston: specifier: ^3.17.0 version: 3.17.0 + zod: + specifier: ^3.24.2 + version: 3.25.51 devDependencies: '@types/cookie': specifier: ^0.5.2 diff --git a/projects/app/.env.template b/projects/app/.env.template index fd51a63b3852..eb383699b3cf 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -41,7 +41,10 @@ S3_PORT=9000 S3_USE_SSL=false S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin -S3_PLUGIN_BUCKET=fastgpt-plugin # 插件文件存储bucket +S3_PLUGIN_BUCKET=fastgpt-plugins # 插件文件存储bucket +S3_PUBLIC_BUCKET=fastgpt-public # 插件文件存储公开桶 +S3_PRIVATE_BUCKET=fastgpt-private # 插件文件存储公开桶 + # Redis URL REDIS_URL=redis://default:mypassword@127.0.0.1:6379 # mongo 数据库连接参数,本地开发连接远程数据库时,可能需要增加 directConnection=true 参数,才能连接上。 diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 85c8d06d2c49..f805b28d91e0 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -1,13 +1,14 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { ModalFooter, ModalBody, Input, Button, Box, Textarea, HStack } from '@chakra-ui/react'; import MyModal from '@fastgpt/web/components/common/MyModal/index'; import { useTranslation } from 'next-i18next'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import { useForm } from 'react-hook-form'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type EditResourceInfoFormType = { id: string; @@ -41,14 +42,14 @@ const EditResourceModal = ({ } ); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); + const afterUploadAvatar = useCallback( + (avatar: string) => { + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } = + useUploadAvatar(getUploadAvatarPresignedUrl, { onSuccess: afterUploadAvatar }); return ( @@ -64,7 +65,7 @@ const EditResourceModal = ({ h={'2rem'} cursor={'pointer'} borderRadius={'sm'} - onClick={onOpenSelectFile} + onClick={handleAvatarSelectorOpen} /> - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/instrumentation.ts b/projects/app/src/instrumentation.ts index e7de610b1ecb..0de13fd5cea6 100644 --- a/projects/app/src/instrumentation.ts +++ b/projects/app/src/instrumentation.ts @@ -21,7 +21,8 @@ export async function register() { { loadSystemModels }, { connectSignoz }, { getSystemTools }, - { trackTimerProcess } + { trackTimerProcess }, + { initS3Buckets } ] = await Promise.all([ import('@fastgpt/service/common/mongo/init'), import('@fastgpt/service/common/mongo/index'), @@ -36,7 +37,8 @@ export async function register() { import('@fastgpt/service/core/ai/config/utils'), import('@fastgpt/service/common/otel/trace/register'), import('@fastgpt/service/core/app/plugin/controller'), - import('@fastgpt/service/common/middle/tracks/processor') + import('@fastgpt/service/common/middle/tracks/processor'), + import('@fastgpt/service/common/s3') ]); // connect to signoz @@ -46,6 +48,9 @@ export async function register() { systemStartCb(); initGlobalVariables(); + // init s3 buckets + initS3Buckets(); + // Connect to MongoDB await connectMongo({ db: connectionMongo, diff --git a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx index db6f4574b403..0e0a0e95a114 100644 --- a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx +++ b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx @@ -1,7 +1,6 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { useForm } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { @@ -21,6 +20,8 @@ import { type CreateTeamProps } from '@fastgpt/global/support/user/team/controll import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants'; import Icon from '@fastgpt/web/components/common/Icon'; import dynamic from 'next/dynamic'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; const UpdateContact = dynamic(() => import('@/components/support/user/inform/UpdateContactModal')); export type EditTeamFormDataType = CreateTeamProps & { @@ -50,15 +51,6 @@ function EditModal({ const avatar = watch('avatar'); const notificationAccount = watch('notificationAccount'); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png,.svg', - multiple: false - }); - const { mutate: onclickCreate, isLoading: creating } = useRequest({ mutationFn: async (data: CreateTeamProps) => { return postCreateTeam(data); @@ -88,6 +80,19 @@ function EditModal({ const { isOpen: isOpenContact, onClose: onCloseContact, onOpen: onOpenContact } = useDisclosure(); + const afterUploadAvatar = useCallback( + (avatar: string) => { + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + return ( {t('account_team:set_name_avatar')} + )} - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> {isOpenContact && ( { + setValue('avatar', avatar); + } }); const { register, handleSubmit, getValues, setValue } = useForm({ @@ -43,20 +45,6 @@ function GroupInfoModal({ } }); - const { loading: uploadingAvatar, run: onSelectAvatar } = useRequest2( - async (file: File[]) => { - return onSelectImage(file, { - maxW: 300, - maxH: 300 - }); - }, - { - onSuccess: (src: string) => { - setValue('avatar', src); - } - } - ); - const { runAsync: onCreate, loading: isLoadingCreate } = useRequest2( (data: GroupFormType) => { return postCreateGroup({ @@ -96,7 +84,7 @@ function GroupInfoModal({ @@ -121,7 +109,8 @@ function GroupInfoModal({ {editGroup ? t('common:Save') : t('common:new_create')} - + {/* */} + ); } diff --git a/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx b/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx index 091bfa1c84f3..62e756179192 100644 --- a/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx +++ b/projects/app/src/pageComponents/account/team/OrgManage/OrgInfoModal.tsx @@ -1,7 +1,8 @@ -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; import { postCreateOrg, putUpdateOrg } from '@/web/support/user/team/org/api'; import { Button, HStack, Input, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react'; import { DEFAULT_ORG_AVATAR } from '@fastgpt/global/common/system/constants'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; import Avatar from '@fastgpt/web/components/common/Avatar'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import MyModal from '@fastgpt/web/components/common/MyModal'; @@ -90,26 +91,14 @@ function OrgInfoModal({ ); const { - File: AvatarSelect, - onOpen: onOpenSelectAvatar, - onSelectImage - } = useSelectFile({ - fileType: '.jpg, .jpeg, .png', - multiple: false - }); - const { loading: uploadingAvatar, run: onSelectAvatar } = useRequest2( - async (file: File[]) => { - return onSelectImage(file, { - maxW: 300, - maxH: 300 - }); - }, - { - onSuccess: (src: string) => { - setValue('avatar', src); - } + Component: AvatarUploader, + uploading: uploadingAvatar, + handleFileSelectorOpen: handleAvatarSelectorOpen + } = useUploadAvatar(getUploadAvatarPresignedUrl, { + onSuccess: (avatar) => { + setValue('avatar', avatar); } - ); + }); const isLoading = uploadingAvatar || isLoadingUpdate || isLoadingCreate; @@ -125,7 +114,7 @@ function OrgInfoModal({ @@ -158,7 +147,7 @@ function OrgInfoModal({ {isEdit ? t('common:Save') : t('common:new_create')} - + ); } diff --git a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx index d2470cce6513..0a1e4fe9c0ae 100644 --- a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx +++ b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx @@ -1,10 +1,10 @@ -import { useState, useRef } from 'react'; +import { useState, useRef, useCallback } from 'react'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { useTranslation } from 'next-i18next'; import { useMemoizedFn } from 'ahooks'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; -import { formatFileSize } from '@fastgpt/global/common/file/tools'; import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type UploadedFileItem = { url: string; @@ -32,47 +32,13 @@ export const useImageUpload = ({ maxSize, onFileSelect }: UseImageUploadProps) = const finalMaxSize = Math.min(configMaxSize, clientLimitMB); const maxSizeBytes = finalMaxSize * 1024 * 1024; + const afterUploadAvatar = useCallback((avatar: string) => onFileSelect(avatar), [onFileSelect]); const { - File: SelectFileComponent, - onOpen: onOpenSelectFile, - onSelectImage, - loading - } = useSelectFile({ - fileType: 'image/*', - multiple: false, - maxCount: 1 - }); - - // validate file size - const validateFile = useMemoizedFn((file: File): string | null => { - if (file.size > maxSizeBytes) { - return t('chat:setting.copyright.file_size_exceeds_limit', { - maxSize: formatFileSize(maxSizeBytes) - }); - } - return null; - }); - - // handle file select - immediate upload if enabled - const handleFileSelect = useMemoizedFn(async (files: File[]) => { - const file = files[0]; - - const validationError = validateFile(file); - if (validationError) { - toast({ - status: 'warning', - title: validationError - }); - } - - try { - // 立即上传文件,带TTL - const url = await onSelectImage([file], { maxW: 1000, maxH: 1000 }); - onFileSelect(url); - } catch (error) { - console.error('Failed to upload file:', error); - } - }); + Component: SelectFileComponent, + uploading: loading, + handleFileSelectorOpen: onOpenSelectFile, + handleUploadAvatar: handleFileSelect + } = useUploadAvatar(getUploadAvatarPresignedUrl, { onSuccess: afterUploadAvatar }); // 拖拽处理 const handleDragEnter = useMemoizedFn((e: React.DragEvent) => { @@ -106,7 +72,7 @@ export const useImageUpload = ({ maxSize, onFileSelect }: UseImageUploadProps) = if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { const files = Array.from(e.dataTransfer.files); - await handleFileSelect(files); + await handleFileSelect(files[0]); } }); diff --git a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx index bf652689ac71..d89119b22304 100644 --- a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx +++ b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx @@ -80,7 +80,7 @@ const ImageUpload = ({ return ( - + void }) => { } ); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: 'image/*', - multiple: false - }); + const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } = + useUploadAvatar(getUploadAvatarPresignedUrl, { + onSuccess: (avatar: string) => { + setValue('avatar', avatar); + } + }); return ( <> @@ -123,7 +122,7 @@ const MCPToolsEditModal = ({ onClose }: { onClose: () => void }) => { h={['28px', '32px']} cursor={'pointer'} borderRadius={'md'} - onClick={onOpenSelectFile} + onClick={handleAvatarSelectorOpen} /> void }) => { - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/pageComponents/dataset/list/CreateModal.tsx b/projects/app/src/pageComponents/dataset/list/CreateModal.tsx index 6f1137a0473e..f3f07f66766b 100644 --- a/projects/app/src/pageComponents/dataset/list/CreateModal.tsx +++ b/projects/app/src/pageComponents/dataset/list/CreateModal.tsx @@ -1,6 +1,5 @@ import React, { useMemo } from 'react'; import { Box, Flex, Button, ModalFooter, ModalBody, Input, HStack } from '@chakra-ui/react'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useForm } from 'react-hook-form'; import { useRouter } from 'next/router'; import { useSystemStore } from '@/web/common/system/useSystemStore'; @@ -20,6 +19,8 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import { getDocPath } from '@/web/common/system/doc'; import ApiDatasetForm from '../ApiDatasetForm'; import { getWebDefaultEmbeddingModel, getWebDefaultLLMModel } from '@/web/common/system/utils'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type CreateDatasetType = | DatasetTypeEnum.dataset @@ -66,14 +67,12 @@ const CreateModal = ({ const agentModel = watch('agentModel'); const vlmModel = watch('vlmModel'); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: 'image/*', - multiple: false - }); + const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } = + useUploadAvatar(getUploadAvatarPresignedUrl, { + onSuccess: (avatar: string) => { + setValue('avatar', avatar); + } + }); /* create a new kb and router to it */ const { runAsync: onclickCreate, loading: creating } = useRequest2( @@ -135,7 +134,7 @@ const CreateModal = ({ h={['28px', '32px']} cursor={'pointer'} borderRadius={'md'} - onClick={onOpenSelectFile} + onClick={handleAvatarSelectorOpen} /> - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx index 783cdfa85290..7dfe2fbebf50 100644 --- a/projects/app/src/pages/account/info/index.tsx +++ b/projects/app/src/pages/account/info/index.tsx @@ -18,7 +18,6 @@ import { useToast } from '@fastgpt/web/hooks/useToast'; import { useUserStore } from '@/web/support/user/useUserStore'; import type { UserType } from '@fastgpt/global/support/user/type.d'; import dynamic from 'next/dynamic'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useTranslation } from 'next-i18next'; import Avatar from '@fastgpt/web/components/common/Avatar'; @@ -46,6 +45,8 @@ import { getWorkorderURL } from '@/web/common/workorder/api'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { useMount } from 'ahooks'; import MyDivider from '@fastgpt/web/components/common/MyDivider'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; const RedeemCouponModal = dynamic(() => import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -140,14 +141,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { onClose: onCloseUpdateContact, onOpen: onOpenUpdateContact } = useDisclosure(); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); const onclickSave = useCallback( async (data: UserType) => { @@ -164,6 +157,20 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); + const afterUploadAvatar = useCallback( + (avatar: string) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }, + [onclickSave, userInfo] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + const labelStyles: BoxProps = { flex: '0 0 80px', color: 'var(--light-general-on-surface-lowest, var(--Gray-Modern-500, #667085))', @@ -241,6 +248,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} + {isPc ? ( {t('account_info:avatar')}  @@ -253,7 +261,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} > @@ -264,7 +272,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} > void }) => { )} + {feConfigs?.isPlus && ( {t('account_info:member_name')}  @@ -334,21 +343,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} {isOpenUpdatePsw && } {isOpenUpdateContact && } - - onSelectImage(e, { - maxW: 300, - maxH: 300, - callback: (src) => { - if (!userInfo) return; - onclickSave({ - ...userInfo, - avatar: src - }); - } - }) - } - /> ); }; diff --git a/projects/app/src/pages/api/common/file/updateAvatar.ts b/projects/app/src/pages/api/common/file/updateAvatar.ts new file mode 100644 index 000000000000..eeb88368dff0 --- /dev/null +++ b/projects/app/src/pages/api/common/file/updateAvatar.ts @@ -0,0 +1,25 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middleware/entry'; +import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; + +export type updateAvatarQuery = {}; + +export type updateAvatarBody = { + filename: string; + autoExpired?: boolean; +}; + +export type updateAvatarResponse = CreatePostPresignedUrlResult; + +async function handler( + req: ApiRequestProps, + _: ApiResponseType +): Promise { + const { filename, autoExpired } = req.body; + const { teamId } = await authCert({ req, authToken: true }); + return await getS3AvatarSource().createUploadAvatarURL({ teamId, filename, autoExpired }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/app/create.ts b/projects/app/src/pages/api/core/app/create.ts index 842d9108cc74..8690b0844718 100644 --- a/projects/app/src/pages/api/core/app/create.ts +++ b/projects/app/src/pages/api/core/app/create.ts @@ -12,7 +12,6 @@ import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { TeamAppCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant'; -import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils'; import { type ClientSession } from '@fastgpt/service/common/mongo'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; @@ -29,6 +28,7 @@ import { getI18nAppType } from '@fastgpt/service/support/user/audit/util'; import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; import { getMyModels } from '@fastgpt/service/support/permission/model/controller'; import { removeUnauthModels } from '@fastgpt/global/core/workflow/utils'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type CreateAppBody = { parentId?: ParentIdType; @@ -201,7 +201,7 @@ export const onCreateApp = async ({ }); })(); - await refreshSourceAvatar(avatar, undefined, session); + await getS3AvatarSource().refreshAvatar(avatar, undefined, session); return appId; }; diff --git a/projects/app/src/pages/api/core/app/update.ts b/projects/app/src/pages/api/core/app/update.ts index 5709f8ccc7f9..a084d0dbe9dc 100644 --- a/projects/app/src/pages/api/core/app/update.ts +++ b/projects/app/src/pages/api/core/app/update.ts @@ -22,11 +22,11 @@ import { getResourceOwnedClbs } from '@fastgpt/service/support/permission/contro import { authUserPer } from '@fastgpt/service/support/permission/user/auth'; import { TeamAppCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; -import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nAppType } from '@fastgpt/service/support/user/audit/util'; import { i18nT } from '@fastgpt/web/i18n/utils'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type AppUpdateQuery = { appId: string; @@ -111,18 +111,12 @@ async function handler(req: ApiRequestProps) { nodes }); - await refreshSourceAvatar(avatar, app.avatar, session); - if (app.type === AppTypeEnum.toolSet && avatar) { - await MongoApp.updateMany( - { parentId: appId, teamId: app.teamId }, - { - avatar - }, - { session } - ); + await MongoApp.updateMany({ parentId: appId, teamId: app.teamId }, { avatar }, { session }); } + await getS3AvatarSource().refreshAvatar(avatar, app.avatar, session); + return MongoApp.findByIdAndUpdate( appId, { diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index 482d3f623d35..5b087dfc25a0 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -8,7 +8,6 @@ import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { TeamDatasetCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant'; -import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { @@ -26,6 +25,7 @@ import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type DatasetCreateQuery = {}; export type DatasetCreateBody = CreateDatasetParams; @@ -103,7 +103,7 @@ async function handler( resourceType: PerResourceTypeEnum.dataset }); - await refreshSourceAvatar(avatar, undefined, session); + await getS3AvatarSource().refreshAvatar(avatar, undefined, session); return dataset._id; }); diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index a2b0611ad353..3334dcd684da 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -25,7 +25,6 @@ import { authUserPer } from '@fastgpt/service/support/permission/user/auth'; import { TeamDatasetCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; -import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { type DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { removeDatasetSyncJobScheduler, @@ -39,6 +38,7 @@ import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; import { computedCollectionChunkSettings } from '@fastgpt/global/core/dataset/training/utils'; import { getResourceOwnedClbs } from '@fastgpt/service/support/permission/controller'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type DatasetUpdateQuery = {}; export type DatasetUpdateResponse = any; @@ -220,12 +220,13 @@ async function handler( }, { session } ); + await updateSyncSchedule({ dataset, autoSync }); - await refreshSourceAvatar(avatar, dataset.avatar, session); + await getS3AvatarSource().refreshAvatar(avatar, dataset.avatar, session); }; await mongoSessionRun(async (session) => { diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index dfa451e29028..33cb8f85bacb 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -6,8 +6,8 @@ import { type UserUpdateParams } from '@/types/user'; import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; import { NextAPI } from '@/service/middleware/entry'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; -import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type UserAccountUpdateQuery = {}; export type UserAccountUpdateBody = UserUpdateParams; @@ -37,15 +37,9 @@ async function handler( } // if avatar, update team member avatar if (avatar) { - await MongoTeamMember.updateOne( - { - _id: tmbId - }, - { - avatar - } - ).session(session); - await refreshSourceAvatar(avatar, tmb?.avatar, session); + await MongoTeamMember.updateOne({ _id: tmbId }, { avatar }).session(session); + + await getS3AvatarSource().refreshAvatar(avatar, tmb?.avatar, session); } }); diff --git a/projects/app/src/pages/api/system/img/[...id].ts b/projects/app/src/pages/api/system/img/[...id].ts new file mode 100644 index 000000000000..ce57e98fd62b --- /dev/null +++ b/projects/app/src/pages/api/system/img/[...id].ts @@ -0,0 +1,32 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import path from 'path'; + +import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; +import { Types } from '@fastgpt/service/common/mongo'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; + +// get the models available to the system +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { id } = req.query as { id: string[] }; + + const joined = id.join('/'); + const parsed = path.parse(joined); + const keys = path.format({ dir: parsed.dir, name: parsed.name, ext: '' }); + + if (Types.ObjectId.isValid(keys)) { + const { binary, mime } = await readMongoImg({ id: joined }); + res.setHeader('Content-Type', mime); + res.send(binary); + return; + } + + res.redirect(301, getS3AvatarSource().createPublicUrl(joined)); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/projects/app/src/pages/api/system/img/[id].ts b/projects/app/src/pages/api/system/img/[id].ts deleted file mode 100644 index 882d21ab3c87..000000000000 --- a/projects/app/src/pages/api/system/img/[id].ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; - -import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; - -// get the models available to the system -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { id } = req.query as { id: string }; - - const { binary, mime } = await readMongoImg({ id }); - - res.setHeader('Content-Type', mime); - res.send(binary); - } catch (error) { - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts index 70127a39523c..eb0d0e1f143e 100644 --- a/projects/app/src/service/common/system/cron.ts +++ b/projects/app/src/service/common/system/cron.ts @@ -14,6 +14,7 @@ import { getScheduleTriggerApp } from '@/service/core/app/utils'; import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller'; import { clearExpiredDatasetImageCron } from '@fastgpt/service/core/dataset/image/controller'; import { cronRefreshModels } from '@fastgpt/service/core/ai/config/utils'; +import { clearExpiredMinioFilesCron } from '@fastgpt/service/common/s3/controller'; // Try to run train every minute const setTrainingQueueCron = () => { @@ -90,4 +91,5 @@ export const startCron = () => { clearExpiredRawTextBufferCron(); clearExpiredDatasetImageCron(); cronRefreshModels(); + clearExpiredMinioFilesCron(); }; diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index c73dad821095..98c15553d53f 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -1,6 +1,6 @@ -import { DELETE, GET, POST } from '@/web/common/api/request'; +import { POST } from '@/web/common/api/request'; import type { UploadImgProps } from '@fastgpt/global/common/file/api.d'; -import type { UploadPresignedURLResponse } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; import { type AxiosProgressEvent } from 'axios'; export const postUploadImg = (e: UploadImgProps) => POST('/common/file/uploadImage', e); @@ -32,3 +32,10 @@ export const postS3UploadFile = ( }, onUploadProgress }); + +export const getUploadAvatarPresignedUrl = (params: { + filename: string; + autoExpired?: boolean; +}) => { + return POST('/common/file/updateAvatar', params); +}; diff --git a/test/mocks/index.ts b/test/mocks/index.ts index 9ce08e29f64f..6ba3abc9f26b 100644 --- a/test/mocks/index.ts +++ b/test/mocks/index.ts @@ -56,3 +56,72 @@ vi.mock('@fastgpt/service/common/bullmq', async (importOriginal) => { getWorker: vi.fn(() => mockWorker) }; }); + +vi.mock('@fastgpt/service/common/s3/buckets/base', async (importOriginal) => { + const actual = (await importOriginal()) as any; + + class MockS3BaseBucket { + private _bucket: string; + public options: any; + + constructor(bucket: string, afterInits?: any, options?: any) { + this._bucket = bucket; + this.options = options || {}; + } + + get name(): string { + return this._bucket; + } + + get client(): any { + return { + bucketExists: vi.fn().mockResolvedValue(true), + makeBucket: vi.fn().mockResolvedValue(undefined), + setBucketPolicy: vi.fn().mockResolvedValue(undefined), + copyObject: vi.fn().mockResolvedValue(undefined), + removeObject: vi.fn().mockResolvedValue(undefined), + presignedPostPolicy: vi.fn().mockResolvedValue({ + postURL: 'http://localhost:9000/mock-bucket', + formData: { key: 'mock-key' } + }), + newPostPolicy: vi.fn(() => ({ + setKey: vi.fn(), + setBucket: vi.fn(), + setContentType: vi.fn(), + setContentLengthRange: vi.fn(), + setExpires: vi.fn(), + setUserMetaData: vi.fn() + })) + }; + } + + move(src: string, dst: string, options?: any): Promise { + return Promise.resolve(); + } + + copy(src: string, dst: string, options?: any): any { + return Promise.resolve(); + } + + exist(): Promise { + return Promise.resolve(true); + } + + delete(objectKey: string, options?: any): Promise { + return Promise.resolve(); + } + + async createPostPresignedUrl(params: any, options?: any): Promise { + const key = `mock/${params.teamId}/${params.filename}`; + return { + url: 'http://localhost:9000/mock-bucket', + fields: { key } + }; + } + } + + return { + ...actual, + S3BaseBucket: MockS3BaseBucket + }; +});