Skip to content

Commit

Permalink
Store file size in DB (#704)
Browse files Browse the repository at this point in the history
  • Loading branch information
Julien Kirch authored Sep 14, 2023
1 parent eedf19a commit ab7ff25
Show file tree
Hide file tree
Showing 29 changed files with 484 additions and 163 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ public File update(File file) {
fileDb.setMimeType(file.getMimeType());
fileDb.setVersionId(file.getVersionId());
fileDb.setLatestVersion(file.isLatestVersion());
fileDb.setSize(file.getSize());

return file;
}
Expand Down Expand Up @@ -224,4 +225,12 @@ public List<File> getByVersionId(String versionId) {
q.setParameter("versionId", versionId);
return q.getResultList();
}

public List<File> getFilesWithUnknownSize(int limit) {
EntityManager em = ThreadLocalContext.get().getEntityManager();
TypedQuery<File> q = em.createQuery("select f from File f where f.size = :size and f.deleteDate is null order by f.order asc", File.class);
q.setParameter("size", File.UNKNOWN_SIZE);
q.setMaxResults(limit);
return q.getResultList();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public class FileDeletedAsyncEvent extends UserEvent {
*/
private String fileId;

private Long fileSize;

public String getFileId() {
return fileId;
}
Expand All @@ -21,10 +23,19 @@ public void setFileId(String fileId) {
this.fileId = fileId;
}

public Long getFileSize() {
return fileSize;
}

public void setFileSize(Long fileSize) {
this.fileSize = fileSize;
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("fileId", fileId)
.add("fileSize", fileSize)
.toString();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,19 @@

import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.event.FileDeletedAsyncEvent;
import com.sismics.docs.core.model.context.AppContext;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Listener on file deleted.
*
*
* @author bgamard
*/
public class FileDeletedAsyncListener {
Expand All @@ -22,7 +25,7 @@ public class FileDeletedAsyncListener {

/**
* File deleted.
*
*
* @param event File deleted event
* @throws Exception e
*/
Expand All @@ -32,6 +35,24 @@ public void on(final FileDeletedAsyncEvent event) throws Exception {
if (log.isInfoEnabled()) {
log.info("File deleted event: " + event.toString());
}
TransactionUtil.handle(() -> {
// Update the user quota
UserDao userDao = new UserDao();
User user = userDao.getById(event.getUserId());
if (user != null) {
Long fileSize = event.getFileSize();

if (fileSize.equals(File.UNKNOWN_SIZE)) {
// The file size was not in the database, in this case we need to get from the unencrypted size.
fileSize = FileUtil.getFileSize(event.getFileId(), user);
}

if (! fileSize.equals(File.UNKNOWN_SIZE)) {
user.setStorageCurrent(user.getStorageCurrent() - fileSize);
userDao.updateQuota(user);
}
}
});

// Delete the file from storage
FileUtil.delete(event.getFileId());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.sismics.docs.core.listener.async.*;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.service.FileService;
import com.sismics.docs.core.service.FileSizeService;
import com.sismics.docs.core.service.InboxService;
import com.sismics.docs.core.util.PdfUtil;
import com.sismics.docs.core.util.indexing.IndexingHandler;
Expand Down Expand Up @@ -65,6 +66,11 @@ public class AppContext {
*/
private FileService fileService;

/**
* File size service.
*/
private FileSizeService fileSizeService;

/**
* Asynchronous executors.
*/
Expand Down Expand Up @@ -102,6 +108,11 @@ private void startUp() {
inboxService.startAsync();
inboxService.awaitRunning();

// Start file size service
fileSizeService = new FileSizeService();
fileSizeService.startAsync();
fileSizeService.awaitRunning();

// Register fonts
PdfUtil.registerFonts();

Expand Down Expand Up @@ -238,6 +249,10 @@ public void shutDown() {
fileService.stopAsync();
}

if (fileSizeService != null) {
fileSizeService.stopAsync();
}

instance = null;
}
}
20 changes: 20 additions & 0 deletions docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,14 @@ public class File implements Loggable {
@Column(name = "FIL_LATESTVERSION_B", nullable = false)
private boolean latestVersion;

public static final Long UNKNOWN_SIZE = -1L;

/**
* Can be {@link File#UNKNOWN_SIZE} if the size has not been stored in the database when the file has been uploaded
*/
@Column(name = "FIL_SIZE_N", nullable = false)
private Long size;

/**
* Private key to decrypt the file.
* Not saved to database, of course.
Expand Down Expand Up @@ -204,6 +212,18 @@ public File setLatestVersion(boolean latestVersion) {
return this;
}

/**
* Can return {@link File#UNKNOWN_SIZE} if the file size is not stored in the database.
*/
public Long getSize() {
return size;
}

public File setSize(Long size) {
this.size = size;
return this;
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package com.sismics.docs.core.service;

import com.google.common.util.concurrent.AbstractScheduledService;
import com.sismics.docs.core.dao.FileDao;
import com.sismics.docs.core.dao.UserDao;
import com.sismics.docs.core.model.jpa.File;
import com.sismics.docs.core.model.jpa.User;
import com.sismics.docs.core.util.FileUtil;
import com.sismics.docs.core.util.TransactionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.concurrent.TimeUnit;

/**
* Service that retrieve files sizes when they are not in the database.
*/
public class FileSizeService extends AbstractScheduledService {
/**
* Logger.
*/
private static final Logger log = LoggerFactory.getLogger(FileSizeService.class);

public FileSizeService() {
}

@Override
protected void startUp() {
log.info("File size service starting up");
}

@Override
protected void shutDown() {
log.info("File size service shutting down");
}

private static final int BATCH_SIZE = 30;

@Override
protected void runOneIteration() {
try {
TransactionUtil.handle(() -> {
FileDao fileDao = new FileDao();
List<File> files = fileDao.getFilesWithUnknownSize(BATCH_SIZE);
for(File file : files) {
processFile(file);
}
if(files.size() < BATCH_SIZE) {
log.info("No more file to process, stopping the service");
stopAsync();
}
});
} catch (Throwable e) {
log.error("Exception during file service iteration", e);
}
}

void processFile(File file) {
UserDao userDao = new UserDao();
User user = userDao.getById(file.getUserId());
if(user == null) {
return;
}

long fileSize = FileUtil.getFileSize(file.getId(), user);
if(fileSize != File.UNKNOWN_SIZE){
FileDao fileDao = new FileDao();
file.setSize(fileSize);
fileDao.update(file);
}
}

@Override
protected Scheduler scheduler() {
return Scheduler.newFixedDelaySchedule(0, 1, TimeUnit.MINUTES);
}
}
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package com.sismics.docs.core.service;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractScheduledService;
import com.sismics.docs.core.constant.ConfigType;
import com.sismics.docs.core.dao.TagDao;
import com.sismics.docs.core.dao.criteria.TagCriteria;
import com.sismics.docs.core.dao.dto.TagDto;
import com.sismics.docs.core.event.DocumentCreatedAsyncEvent;
import com.sismics.docs.core.model.jpa.Config;
import com.sismics.docs.core.model.jpa.Document;
import com.sismics.docs.core.model.jpa.Tag;
import com.sismics.docs.core.util.ConfigUtil;
Expand Down
33 changes: 32 additions & 1 deletion docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
import com.sismics.util.context.ThreadLocalContext;
import com.sismics.util.io.InputStreamReaderThread;
import com.sismics.util.mime.MimeTypeUtil;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CountingInputStream;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -46,7 +49,7 @@ public class FileUtil {
/**
* File ID of files currently being processed.
*/
private static Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());
private static final Set<String> processingFileSet = Collections.synchronizedSet(new HashSet<>());

/**
* Optical character recognition on an image.
Expand Down Expand Up @@ -149,6 +152,7 @@ public static String createFile(String name, String previousFileId, Path unencry
file.setName(StringUtils.abbreviate(name, 200));
file.setMimeType(mimeType);
file.setUserId(userId);
file.setSize(fileSize);

// Get files of this document
FileDao fileDao = new FileDao();
Expand Down Expand Up @@ -240,4 +244,31 @@ public static void endProcessingFile(String fileId) {
public static boolean isProcessingFile(String fileId) {
return processingFileSet.contains(fileId);
}

/**
* Get the size of a file on disk.
*
* @param fileId the file id
* @param user the file owner
* @return the size or -1 if something went wrong
*/
public static long getFileSize(String fileId, User user) {
// To get the size we copy the decrypted content into a null output stream
// and count the copied byte size.
Path storedFile = DirectoryUtil.getStorageDirectory().resolve(fileId);
if (! Files.exists(storedFile)) {
log.debug("File does not exist " + fileId);
return File.UNKNOWN_SIZE;
}
try (InputStream fileInputStream = Files.newInputStream(storedFile);
InputStream inputStream = EncryptionUtil.decryptInputStream(fileInputStream, user.getPrivateKey());
CountingInputStream countingInputStream = new CountingInputStream(inputStream);
) {
IOUtils.copy(countingInputStream, NullOutputStream.NULL_OUTPUT_STREAM);
return countingInputStream.getByteCount();
} catch (Exception e) {
log.debug("Can't find size of file " + fileId, e);
return File.UNKNOWN_SIZE;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import com.google.common.collect.Lists;
import com.sismics.util.ClasspathScanner;

import java.lang.reflect.InvocationTargetException;
import java.util.List;

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.sismics.docs.core.util.format;

import com.google.common.base.Charsets;
import com.google.common.io.Closer;
import com.lowagie.text.*;
import com.lowagie.text.pdf.PdfWriter;
Expand Down
2 changes: 1 addition & 1 deletion docs-core/src/main/resources/config.properties
Original file line number Diff line number Diff line change
@@ -1 +1 @@
db.version=28
db.version=29
2 changes: 2 additions & 0 deletions docs-core/src/main/resources/db/update/dbupdate-029-0.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
alter table T_FILE add column FIL_SIZE_N bigint not null default -1;
update T_CONFIG set CFG_VALUE_C = '29' where CFG_ID_C = 'DB_VERSION';
Loading

0 comments on commit ab7ff25

Please sign in to comment.