refactor: s3 class

Author: xqvvu

packages/global/common/file/minioTtl/type.d.ts renamed to packages/global/common/file/s3TTL/type.d.ts

@@ -8,6 +8,7 @@ import { addHours } from 'date-fns';
 import { imageFileType } from '@fastgpt/global/common/file/constants';
 import { retryFn } from '@fastgpt/global/common/system/utils';
 import { UserError } from '@fastgpt/global/common/error/utils';
+import { getS3AvatarSource } from '../../s3/sources/avatar';
 
 export const maxImgSize = 1024 * 1024 * 12;
 const base64MimeRegex = /data:image\/([^\)]+);base64/;
@@ -109,6 +110,15 @@ const getIdFromPath = (path?: string) => {
 
   return id;
 };
+export const refreshSourceAvatarS3 = async (
+  path?: string,
+  oldPath?: string,
+  session?: ClientSession
+) => {
+  const s3AvatarSource = getS3AvatarSource();
+  await s3AvatarSource.deleteAvatar(oldPath || '', session);
+  await s3AvatarSource.removeAvatarTTL(path || '', session);
+};
 // 删除旧的头像，新的头像去除过期时间
 export const refreshSourceAvatar = async (
   path?: string,

packages/service/common/file/image/controller.ts

@@ -1,9 +1,10 @@
 import { MongoS3TTL } from './schema';
-import { S3BucketManager } from '../../s3/buckets/manager';
 import { addLog } from '../../system/log';
 import { setCron } from '../../system/cron';
 import { checkTimerLock } from '../../system/timerLock/utils';
 import { TimerIdEnum } from '../../system/timerLock/constants';
+import { S3PublicBucket } from '../../s3/buckets/public';
+import { S3PrivateBucket } from '../../s3/buckets/private';
 
 export async function clearExpiredMinioFiles() {
   try {
@@ -17,18 +18,17 @@ export async function clearExpiredMinioFiles() {
 
     addLog.info(`Found ${expiredFiles.length} expired minio files to clean`);
 
-    const s3Manager = S3BucketManager.getInstance();
     let success = 0;
     let fail = 0;
 
     for (const file of expiredFiles) {
       try {
         const bucket = (() => {
           if (file.bucketName === process.env.S3_PUBLIC_BUCKET) {
-            return s3Manager.getPublicBucket();
+            return new S3PublicBucket();
           }
           if (file.bucketName === process.env.S3_PRIVATE_BUCKET) {
-            return s3Manager.getPrivateBucket();
+            return new S3PrivateBucket();
           }
           throw new Error(`Unknown bucket name: ${file.bucketName}`);
         })();

packages/service/common/file/s3Ttl/controller.ts

@@ -1,9 +1,9 @@
 import { Schema, getMongoModel } from '../../mongo';
-import { type S3TtlSchemaType } from '@fastgpt/global/common/file/minioTtl/type';
+import { type S3TtlSchemaType } from '@fastgpt/global/common/file/s3Ttl/type';
 
 const collectionName = 's3_ttl_files';
 
-const S3TtlSchema = new Schema({
+const S3TTLSchema = new Schema({
   bucketName: {
     type: String,
     required: true
@@ -18,7 +18,7 @@ const S3TtlSchema = new Schema({
   }
 });
 
-S3TtlSchema.index({ expiredTime: 1 });
-S3TtlSchema.index({ bucketName: 1, minioKey: 1 });
+S3TTLSchema.index({ expiredTime: 1 });
+S3TTLSchema.index({ bucketName: 1, minioKey: 1 });
 
-export const MongoS3TTL = getMongoModel<S3TtlSchemaType>(collectionName, S3TtlSchema);
+export const MongoS3TTL = getMongoModel<S3TtlSchemaType>(collectionName, S3TTLSchema);

packages/service/common/file/s3Ttl/schema.ts

@@ -1,21 +1,20 @@
 import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio';
 import {
-  defaultS3Options,
   type ExtensionType,
-  Mimes,
   type CreatePostPresignedUrlOptions,
   type CreatePostPresignedUrlParams,
   type CreatePostPresignedUrlResult,
   type S3BucketName,
-  type S3Options
+  type S3OptionsType
 } from '../type';
-import type { BucketBasicOperationsType } from '../interface';
-import { createObjectKey, createTempObjectKey } from '../helpers';
+import { defaultS3Options, Mimes } from '../constants';
+import { createObjectKey } from '../helpers';
 import path from 'node:path';
-import { MongoS3TTL } from 'common/file/s3TTL/schema';
+import { MongoS3TTL } from '../../file/s3Ttl/schema';
 
-export class S3BaseBucket implements BucketBasicOperationsType {
-  public client: Client;
+export class S3BaseBucket {
+  private _client: Client;
+  private _externalClient: Client | undefined;
 
   /**
    *
@@ -26,17 +25,32 @@ export class S3BaseBucket implements BucketBasicOperationsType {
   constructor(
     private readonly _bucket: S3BucketName,
     private readonly afterInits?: (() => Promise<void> | void)[],
-    public options: Partial<S3Options> = defaultS3Options
+    public options: Partial<S3OptionsType> = defaultS3Options
   ) {
     options = { ...defaultS3Options, ...options };
-    this.options = options as S3Options;
-    this.client = new Client(options as S3Options);
+    this.options = options;
+    this._client = new Client(options as S3OptionsType);
+
+    if (this.options.externalBaseURL) {
+      const externalBaseURL = new URL(this.options.externalBaseURL);
+      const endpoint = externalBaseURL.hostname;
+      const useSSL = externalBaseURL.protocol === 'https';
+
+      this._externalClient = new Client({
+        useSSL: useSSL,
+        endPoint: endpoint,
+        port: options.port,
+        accessKey: options.accessKey,
+        secretKey: options.secretKey,
+        transportAgent: options.transportAgent
+      });
+    }
 
     const init = async () => {
       if (!(await this.exist())) {
         await this.client.makeBucket(this._bucket);
       }
-      await Promise.all(this.afterInits?.map((afterInit) => afterInit()) ?? []);
+      await Promise.all(afterInits?.map((afterInit) => afterInit()) ?? []);
     };
     init();
   }
@@ -45,10 +59,14 @@ export class S3BaseBucket implements BucketBasicOperationsType {
     return this._bucket;
   }
 
-  async move(src: string, dst: string, options?: CopyConditions): Promise<void> {
+  protected get client(): Client {
+    return this._externalClient ?? this._client;
+  }
+
+  move(src: string, dst: string, options?: CopyConditions): Promise<void> {
     const bucket = this.name;
-    await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options);
-    await this.delete(src);
+    this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options);
+    return this.delete(src);
   }
 
   copy(src: string, dst: string, options?: CopyConditions): ReturnType<Client['copyObject']> {
@@ -59,27 +77,19 @@ export class S3BaseBucket implements BucketBasicOperationsType {
     return this.client.bucketExists(this.name);
   }
 
-  async delete(objectKey: string, options?: RemoveOptions): Promise<void> {
-    await this.client.removeObject(this.name, objectKey, options);
-  }
-
-  get(): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
-
-  getLifecycle(): Promise<LifecycleConfig | null> {
-    return this.client.getBucketLifecycle(this.name);
+  delete(objectKey: string, options?: RemoveOptions): Promise<void> {
+    return this.client.removeObject(this.name, objectKey, options);
   }
 
   async createPostPresignedUrl(
     params: CreatePostPresignedUrlParams,
     options: CreatePostPresignedUrlOptions = {}
   ): Promise<CreatePostPresignedUrlResult> {
-    const { temporary, ttl = 7 * 24 } = options;
+    const { expiredHours } = options;
     const ext = path.extname(params.filename).toLowerCase() as ExtensionType;
     const contentType = Mimes[ext] ?? 'application/octet-stream';
     const maxFileSize = this.options.maxFileSize as number;
-    const key = temporary ? createTempObjectKey(params) : createObjectKey(params);
+    const key = createObjectKey(params);
 
     const policy = this.client.newPostPolicy();
     policy.setKey(key);
@@ -88,17 +98,19 @@ export class S3BaseBucket implements BucketBasicOperationsType {
     policy.setContentLengthRange(1, maxFileSize);
     policy.setExpires(new Date(Date.now() + 10 * 60 * 1000));
     policy.setUserMetaData({
-      filename: encodeURIComponent(params.filename),
-      visibility: params.visibility
+      'content-type': contentType,
+      'content-disposition': `attachment; filename="${encodeURIComponent(params.filename)}"`,
+      'origin-filename': encodeURIComponent(params.filename),
+      'upload-time': new Date().toISOString()
     });
 
     const { formData, postURL } = await this.client.presignedPostPolicy(policy);
 
-    if (temporary) {
+    if (expiredHours) {
       await MongoS3TTL.create({
         minioKey: key,
         bucketName: this.name,
-        expiredTime: new Date(Date.now() + ttl * 3.6e6)
+        expiredTime: new Date(Date.now() + expiredHours * 3.6e6)
       });
     }
 

packages/service/common/s3/buckets/base.ts

@@ -1,19 +1,9 @@
 import { S3BaseBucket } from './base';
-import {
-  S3Buckets,
-  type CreatePostPresignedUrlParams,
-  type CreatePostPresignedUrlResult,
-  type S3Options
-} from '../type';
+import { S3Buckets } from '../constants';
+import { type S3OptionsType } from '../type';
 
 export class S3PrivateBucket extends S3BaseBucket {
-  constructor(options?: Partial<S3Options>) {
+  constructor(options?: Partial<S3OptionsType>) {
     super(S3Buckets.private, undefined, options);
   }
-
-  override createPostPresignedUrl(
-    params: Omit<CreatePostPresignedUrlParams, 'visibility'>
-  ): Promise<CreatePostPresignedUrlResult> {
-    return super.createPostPresignedUrl({ ...params, visibility: 'private' });
-  }
 }

packages/service/common/s3/buckets/manager.ts

@@ -1,35 +1,24 @@
 import { S3BaseBucket } from './base';
-import { createBucketPolicy } from '../helpers';
-import {
-  S3Buckets,
-  type CreatePostPresignedUrlOptions,
-  type CreatePostPresignedUrlParams,
-  type CreatePostPresignedUrlResult,
-  type S3Options
-} from '../type';
-import type { IPublicBucketOperations } from '../interface';
-import { lifecycleOfTemporaryAvatars } from '../lifecycle';
+import { createS3PublicBucketPolicy } from '../helpers';
+import { S3Buckets } from '../constants';
+import { type S3OptionsType } from '../type';
 
-export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations {
-  constructor(options?: Partial<S3Options>) {
+export class S3PublicBucket extends S3BaseBucket {
+  constructor(options?: Partial<S3OptionsType>) {
     super(
       S3Buckets.public,
       [
         // set bucket policy
         async () => {
           const bucket = this.name;
-          const policy = createBucketPolicy(bucket);
+          const policy = createS3PublicBucketPolicy(bucket);
           try {
             await this.client.setBucketPolicy(bucket, policy);
           } catch (error) {
-            // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error,
+            // NOTE: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error,
             // maybe we can ignore the error, or we have other plan to handle this.
+            console.error('Failed to set bucket policy:', error);
           }
-        },
-        // set bucket lifecycle
-        async () => {
-          const bucket = this.name;
-          await this.client.setBucketLifecycle(bucket, lifecycleOfTemporaryAvatars);
         }
       ],
       options
@@ -54,11 +43,4 @@ export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperati
 
     return url.toString();
   }
-
-  override createPostPresignedUrl(
-    params: Omit<CreatePostPresignedUrlParams, 'visibility'>,
-    options: CreatePostPresignedUrlOptions = {}
-  ): Promise<CreatePostPresignedUrlResult> {
-    return super.createPostPresignedUrl({ ...params, visibility: 'public' }, options);
-  }
 }

packages/service/common/s3/buckets/private.ts

@@ -0,0 +1,49 @@
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import type { ClientOptions } from 'minio';
+
+export const Mimes = {
+  '.gif': 'image/gif',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.webp': 'image/webp',
+  '.svg': 'image/svg+xml',
+
+  '.csv': 'text/csv',
+  '.txt': 'text/plain',
+
+  '.pdf': 'application/pdf',
+  '.zip': 'application/zip',
+  '.json': 'application/json',
+  '.doc': 'application/msword',
+  '.js': 'application/javascript',
+  '.xls': 'application/vnd.ms-excel',
+  '.ppt': 'application/vnd.ms-powerpoint',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
+} as const;
+
+export const defaultS3Options: { externalBaseURL?: string; maxFileSize?: number } & ClientOptions =
+  {
+    maxFileSize: 1024 ** 3, // 1GB
+
+    useSSL: process.env.S3_USE_SSL === 'true',
+    endPoint: process.env.S3_ENDPOINT || 'localhost',
+    externalBaseURL: process.env.S3_EXTERNAL_BASE_URL,
+    accessKey: process.env.S3_ACCESS_KEY || 'minioadmin',
+    secretKey: process.env.S3_SECRET_KEY || 'minioadmin',
+    port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000,
+    transportAgent: process.env.HTTP_PROXY
+      ? new HttpProxyAgent(process.env.HTTP_PROXY)
+      : process.env.HTTPS_PROXY
+        ? new HttpsProxyAgent(process.env.HTTPS_PROXY)
+        : undefined
+  };
+
+export const S3Buckets = {
+  plugin: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin',
+  public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public',
+  private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private'
+} as const;