Skip to content

Commit

Permalink
Feat: Support AWS file storage support (#343)
Browse files Browse the repository at this point in the history
  • Loading branch information
saimanoj authored Dec 16, 2024
1 parent 35c3b2d commit c8798f3
Show file tree
Hide file tree
Showing 12 changed files with 1,746 additions and 815 deletions.
8 changes: 7 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,15 @@ TYPESENSE_PORT=8108
TYPESENSE_PROTOCOL=http

# Setting these will save the uploaded files in that bucket
GCP_BUCKET_NAME=
STORAGE_PROVIDER=

BUCKET_NAME=
GCP_SERVICE_ACCOUNT_FILE=

AWS_REGION=
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=

# Email settings
# SMTP_HOST=
# SMTP_PORT=
Expand Down
2 changes: 2 additions & 0 deletions apps/server/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
},
"dependencies": {
"@ai-sdk/openai": "^0.0.63",
"@aws-sdk/client-s3": "^3.712.0",
"@aws-sdk/s3-request-presigner": "^3.712.0",
"@devoxa/prisma-relay-cursor-connection": "3.1.1",
"@google-cloud/storage": "^7.10.0",
"@nestjs-modules/mailer": "^2.0.2",
Expand Down
4 changes: 2 additions & 2 deletions apps/server/src/modules/attachments/attachments.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ export class AttachmentController {
@Param() attachementRequestParams: AttachmentRequestParams,
) {
try {
return await this.attachementService.getFileFromGCSSignedUrl(
return await this.attachementService.getFileFromStorageSignedUrl(
attachementRequestParams,
workspaceId,
);
Expand All @@ -96,7 +96,7 @@ export class AttachmentController {
@Res() res: Response,
) {
try {
const file = await this.attachementService.getFileFromGCS(
const file = await this.attachementService.getFileFromStorage(
attachementRequestParams,
workspaceId,
);
Expand Down
3 changes: 2 additions & 1 deletion apps/server/src/modules/attachments/attachments.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { UsersService } from 'modules/users/users.service';

import { AttachmentController } from './attachments.controller';
import { AttachmentService } from './attachments.service';
import { StorageFactory } from './storage.factory';

@Module({
imports: [
Expand All @@ -16,7 +17,7 @@ import { AttachmentService } from './attachments.service';
}),
],
controllers: [AttachmentController],
providers: [AttachmentService, PrismaService, UsersService],
providers: [AttachmentService, PrismaService, UsersService, StorageFactory],
exports: [AttachmentService],
})
export class AttachmentModule {}
179 changes: 60 additions & 119 deletions apps/server/src/modules/attachments/attachments.service.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { Storage } from '@google-cloud/storage';
import {
BadRequestException,
Injectable,
InternalServerErrorException,
} from '@nestjs/common';
import {
Attachment,
AttachmentResponse,
AttachmentStatusEnum,
SignedURLBody,
Expand All @@ -13,27 +13,28 @@ import { PrismaService } from 'nestjs-prisma';

import { LoggerService } from 'modules/logger/logger.service';

import { AttachmentRequestParams, ExternalFile } from './attachments.interface';
import { AttachmentRequestParams } from './attachments.interface';
import { StorageProvider } from './storage-provider.interface';
import { StorageFactory } from './storage.factory';
@Injectable()
export class AttachmentService {
private storage: Storage;
private bucketName = process.env.GCP_BUCKET_NAME;
private readonly logger: LoggerService = new LoggerService(
'AttachmentService',
);
private storageProvider: StorageProvider;

constructor(private prisma: PrismaService) {
this.storage = new Storage({
keyFilename: process.env.GCP_SERVICE_ACCOUNT_FILE,
});
constructor(
private prisma: PrismaService,
private storageFactory: StorageFactory,
) {
this.storageProvider = this.storageFactory.createStorageProvider();
}

async uploadGenerateSignedURL(
file: SignedURLBody,
userId: string,
workspaceId: string,
) {
const bucket = this.storage.bucket(this.bucketName);
const attachment = await this.prisma.attachment.create({
data: {
fileName: file.fileName,
Expand All @@ -51,14 +52,12 @@ export class AttachmentService {
});

try {
const [url] = await bucket
.file(`${workspaceId}/${attachment.id}.${attachment.fileExt}`)
.getSignedUrl({
version: 'v4',
action: 'write',
expires: Date.now() + 15 * 60 * 1000, // 15 minutes
contentType: file.contentType,
});
const filePath = this.getFilePath(workspaceId, attachment);
const url = await this.storageProvider.getSignedUrl(filePath, {
action: 'write',
expires: Date.now() + 15 * 60 * 1000,
contentType: file.contentType,
});

const publicURL = `${process.env.PUBLIC_ATTACHMENT_URL}/v1/attachment/${workspaceId}/${attachment.id}`;

Expand All @@ -85,8 +84,6 @@ export class AttachmentService {
workspaceId: string,
sourceMetadata?: Record<string, string>,
): Promise<AttachmentResponse[]> {
const bucket = this.storage.bucket(this.bucketName);

const attachmentPromises = files.map(async (file) => {
const attachment = await this.prisma.attachment.create({
data: {
Expand All @@ -105,15 +102,11 @@ export class AttachmentService {
},
});

const blob = bucket.file(
`${workspaceId}/${attachment.id}.${attachment.fileExt}`,
);
await blob.save(file.buffer, {
const filePath = this.getFilePath(workspaceId, attachment);
await this.storageProvider.uploadFile(filePath, file.buffer, {
contentType: file.mimetype,
resumable: false,
validation: false,
metadata: {
contentType: file.mimetype,
},
});

const publicURL = `${process.env.PUBLIC_ATTACHMENT_URL}/v1/attachment/${workspaceId}/${attachment.id}`;
Expand All @@ -133,108 +126,46 @@ export class AttachmentService {
return await Promise.all(attachmentPromises);
}

async createExternalAttachment(
files: ExternalFile[],
userId: string,
workspaceId: string,
sourceMetadata?: Record<string, string>,
) {
const attachmentPromises = files.map(async (file) => {
const attachment = await this.prisma.attachment.create({
data: {
fileName: file.filename,
originalName: file.originalname,
fileType: file.mimetype,
size: file.size,
status: AttachmentStatusEnum.External,
fileExt: file.originalname.split('.').pop(),
uploadedById: userId,
workspaceId,
url: file.url,
sourceMetadata,
},
include: {
workspace: true,
},
});

return {
publicURL: file.url,
fileType: attachment.fileType,
originalName: attachment.originalName,
size: attachment.size,
} as AttachmentResponse;
});

return await Promise.all(attachmentPromises);
}

async getFileFromGCS(
async getFileFromStorage(
attachementRequestParams: AttachmentRequestParams,
workspaceId: string,
) {
const { attachmentId } = attachementRequestParams;

const attachment = await this.prisma.attachment.findFirst({
where: { id: attachmentId, workspaceId },
});

if (!attachment) {
throw new BadRequestException(
`No attachment found for this id: ${attachmentId}`,
);
}
const attachment = await this.getAttachment(
attachementRequestParams.attachmentId,
workspaceId,
);
const filePath = this.getFilePath(workspaceId, attachment);

const bucket = this.storage.bucket(this.bucketName);
const filePath = `${workspaceId}/${attachment.id}.${attachment.fileExt}`;
const [fileExists] = await bucket.file(filePath).exists();

if (!fileExists) {
if (!(await this.storageProvider.fileExists(filePath))) {
throw new BadRequestException('File not found');
}

const [buffer] = await bucket.file(filePath).download();

const buffer = await this.storageProvider.downloadFile(filePath);
return {
buffer,
contentType: attachment.fileType,
originalName: attachment.originalName,
};
}

async getFileFromGCSSignedUrl(
async getFileFromStorageSignedUrl(
attachementRequestParams: AttachmentRequestParams,
workspaceId: string,
) {
const { attachmentId } = attachementRequestParams;
const attachment = await this.getAttachment(
attachementRequestParams.attachmentId,
workspaceId,
);
const filePath = this.getFilePath(workspaceId, attachment);

const attachment = await this.prisma.attachment.findFirst({
where: { id: attachmentId, workspaceId },
});

if (!attachment) {
throw new BadRequestException(
`No attachment found for this id: ${attachmentId}`,
);
}

const bucket = this.storage.bucket(this.bucketName);
const filePath = `${workspaceId}/${attachment.id}.${attachment.fileExt}`;
const file = bucket.file(filePath);

const [exists] = await file.exists();
if (!exists) {
if (!(await this.storageProvider.fileExists(filePath))) {
throw new BadRequestException('File not found');
}

// Get file metadata for size
const [metadata] = await file.getMetadata();

const [signedUrl] = await file.getSignedUrl({
version: 'v4',
const metadata = await this.storageProvider.getMetadata(filePath);
const signedUrl = await this.storageProvider.getSignedUrl(filePath, {
action: 'read',
expires: Date.now() + 60 * 60 * 1000, // 1 hour
// Enable range requests and other necessary headers
expires: Date.now() + 60 * 60 * 1000,
responseDisposition: 'inline',
responseType: attachment.fileType,
});
Expand All @@ -251,23 +182,17 @@ export class AttachmentService {
attachementRequestParams: AttachmentRequestParams,
workspaceId: string,
) {
const { attachmentId } = attachementRequestParams;

const attachment = await this.prisma.attachment.findFirst({
where: { id: attachmentId, workspaceId },
});

if (!attachment) {
throw new BadRequestException('Attachment not found');
}

const filePath = `${workspaceId}/${attachment.id}.${attachment.fileExt}`;
const attachment = await this.getAttachment(
attachementRequestParams.attachmentId,
workspaceId,
);
const filePath = this.getFilePath(workspaceId, attachment);

try {
await Promise.all([
this.storage.bucket(this.bucketName).file(filePath).delete(),
this.storageProvider.deleteFile(filePath),
this.prisma.attachment.update({
where: { id: attachmentId },
where: { id: attachementRequestParams.attachmentId },
data: {
deleted: new Date().toISOString(),
status: AttachmentStatusEnum.Deleted,
Expand All @@ -278,4 +203,20 @@ export class AttachmentService {
throw new InternalServerErrorException('Error deleting attachment');
}
}

private async getAttachment(attachmentId: string, workspaceId: string) {
const attachment = await this.prisma.attachment.findFirst({
where: { id: attachmentId, workspaceId },
});

if (!attachment) {
throw new BadRequestException('Attachment not found');
}

return attachment;
}

private getFilePath(workspaceId: string, attachment: Attachment): string {
return `${workspaceId}/${attachment.id}.${attachment.fileExt}`;
}
}
Loading

0 comments on commit c8798f3

Please sign in to comment.