Skip to content

Commit

Permalink
Merge pull request #1248 from veraPDF/merge/1.20/oom-fixes
Browse files Browse the repository at this point in the history 
MERGE: Fix returning correct exit codes in case OOM and VeraPDF exceptions
  • Loading branch information
@carlwilson
carlwilson authored May 19, 2022
2 parents 8626cb6 + 25809ee commit 8386bf5
Show file tree
Hide file tree
Showing 7 changed files with 94 additions and 40 deletions.
6 changes: 5 additions & 1 deletion core/src/main/java/org/verapdf/processor/ProcessorFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,8 @@ public static final class BatchSummariser {
private int totalJobs = 0;
private int failedToParse = 0;
private int encrypted = 0;
private int outOfMemory = 0;
private int veraExceptions = 0;
private Components.Timer timer = Components.Timer.start();
private Summarisers.ValidationSummaryBuilder validationBuilder = new Summarisers.ValidationSummaryBuilder();
private Summarisers.FeatureSummaryBuilder featureBuilder = new Summarisers.FeatureSummaryBuilder();
Expand All @@ -174,6 +176,8 @@ public void addProcessingResult(ProcessorResult result) {
this.totalJobs++;
if (!result.isPdf()) this.failedToParse++;
if (result.isEncryptedPdf()) this.encrypted++;
if (result.isOutOfMemory()) this.outOfMemory++;
if (result.hasException()) this.veraExceptions++;
if (this.config.hasTask(TaskType.VALIDATE))
this.validationBuilder.addResult(result);
if (this.config.hasTask(TaskType.EXTRACT_FEATURES))
Expand All @@ -184,7 +188,7 @@ public void addProcessingResult(ProcessorResult result) {

public BatchSummary summarise() {
return Reports.createBatchSummary(this.timer, this.validationBuilder.build(), this.featureBuilder.build(),
this.repairBuilder.build(), this.totalJobs, this.failedToParse, this.encrypted);
this.repairBuilder.build(), this.totalJobs, this.failedToParse, this.encrypted, this.outOfMemory, this.veraExceptions);
}
}
}
32 changes: 16 additions & 16 deletions core/src/main/java/org/verapdf/processor/ProcessorImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,13 @@ public ProcessorResult process(File toProcess) throws VeraPDFException {
}
ItemDetails fileDetails = ItemDetails.fromFile(toProcess);
Components.Timer parseTimer = Components.Timer.start();
TaskType task = null;
try (PDFAParser parser = this.hasCustomProfile()
? foundry.createParser(toProcess, this.processorConfig.getCustomProfile().getPDFAFlavour())
: this.isAuto() ? foundry.createParser(toProcess, PDFAFlavour.NO_FLAVOUR, this.valConf().getDefaultFlavour())
: foundry.createParser(toProcess, this.valConf().getFlavour(), this.valConf().getDefaultFlavour())) {
for (TaskType task : this.getConfig().getTasks()) {
for (TaskType t : this.getConfig().getTasks()) {
task = t;
switch (task) {
case VALIDATE:
validate(parser);
Expand All @@ -127,7 +129,6 @@ public ProcessorResult process(File toProcess) throws VeraPDFException {
break;
default:
break;

}
}
} catch (EncryptedPdfException e) {
Expand All @@ -140,6 +141,11 @@ public ProcessorResult process(File toProcess) throws VeraPDFException {
logger.log(Level.FINE, "Exception details:", e); //$NON-NLS-1$
return ProcessorResultImpl.invalidPdfResult(fileDetails,
TaskResultImpl.fromValues(TaskType.PARSE, parseTimer.stop(), e));
} catch (OutOfMemoryError e) {
logger.log(Level.WARNING, "OutOfMemory caught when validating item"); //$NON-NLS-1$
logger.log(Level.FINE, "Exception details:", e); //$NON-NLS-1$
return ProcessorResultImpl.outOfMemoryResult(fileDetails,
TaskResultImpl.fromValues(task, parseTimer.stop(), new VeraPDFException("OutOfMemory caught when validating item", e)));
} catch (IOException excep) {
logger.log(Level.FINER, "Problem closing PDF Stream", excep); //$NON-NLS-1$
} catch (Exception e) {
Expand All @@ -158,11 +164,13 @@ public ProcessorResult process(ItemDetails fileDetails, InputStream pdfFileStrea
this.initialise();
checkArguments(pdfFileStream, fileDetails, this.processorConfig);
Components.Timer parseTimer = Components.Timer.start();
TaskType task = null;
try (PDFAParser parser = this.hasCustomProfile()
? foundry.createParser(pdfFileStream, this.processorConfig.getCustomProfile().getPDFAFlavour())
: this.isAuto() ? foundry.createParser(pdfFileStream)
: foundry.createParser(pdfFileStream, this.valConf().getFlavour())) {
for (TaskType task : this.getConfig().getTasks()) {
for (TaskType t : this.getConfig().getTasks()) {
task = t;
switch (task) {
case VALIDATE:
validate(parser);
Expand All @@ -188,6 +196,11 @@ public ProcessorResult process(ItemDetails fileDetails, InputStream pdfFileStrea
logger.log(Level.FINE, "Exception details:", e); //$NON-NLS-1$
return ProcessorResultImpl.invalidPdfResult(fileDetails,
TaskResultImpl.fromValues(TaskType.PARSE, parseTimer.stop(), e));
} catch (OutOfMemoryError e) {
logger.log(Level.WARNING, "OutOfMemory caught when validating item"); //$NON-NLS-1$
logger.log(Level.FINE, "Exception details:", e); //$NON-NLS-1$
return ProcessorResultImpl.outOfMemoryResult(fileDetails,
TaskResultImpl.fromValues(task, parseTimer.stop(), new VeraPDFException("OutOfMemory caught when validating item", e)));
} catch (IOException excep) {
logger.log(Level.FINER, "Problem closing PDF Stream", excep); //$NON-NLS-1$
} catch (Exception e) {
Expand Down Expand Up @@ -233,10 +246,6 @@ private void validate(final PDFAParser parser) {
} catch (ValidationException excep) {
logger.log(Level.WARNING, "Exception caught when validating item", excep); //$NON-NLS-1$
this.taskResults.put(type, TaskResultImpl.fromValues(type, timer.stop(), excep));
} catch (OutOfMemoryError excep) {
logger.log(Level.WARNING, "OutOfMemory caught when validating item", excep); //$NON-NLS-1$
VeraPDFException veraExcep = new VeraPDFException("OutOfMemory caught when validating item", excep); //$NON-NLS-1$
this.taskResults.put(type, TaskResultImpl.fromValues(type, timer.stop(), veraExcep));
} catch (IOException excep) {
logger.log(Level.INFO, "IOException closing validator.", excep); //$NON-NLS-1$
}
Expand Down Expand Up @@ -279,10 +288,6 @@ private void fixMetadata(final PDFAParser parser, final String fileName) {
this.fixerResult = fixer.fixMetadata(parser, fxos, this.validationResult);
rpStat = this.fixerResult.getRepairStatus();
this.taskResults.put(type, TaskResultImpl.fromValues(type, timer.stop()));
} catch (OutOfMemoryError excep) {
logger.log(Level.WARNING, "OutOfMemory caught when validating item", excep); //$NON-NLS-1$
VeraPDFException veraExcep = new VeraPDFException("OutOfMemory caught when validating item", excep); //$NON-NLS-1$
this.taskResults.put(type, TaskResultImpl.fromValues(type, timer.stop(), veraExcep));
} catch (IOException excep) {
this.taskResults.put(type, TaskResultImpl.fromValues(type, timer.stop(),
new VeraPDFException("Processing exception in metadata fixer", excep))); //$NON-NLS-1$
Expand All @@ -302,11 +307,6 @@ private void extractFeatures(PDFAParser parser) {
this.featureResult = parser.getFeatures(this.featConf(), this.getPlugins());
this.taskResults.put(TaskType.EXTRACT_FEATURES,
TaskResultImpl.fromValues(TaskType.EXTRACT_FEATURES, timer.stop()));
} catch (OutOfMemoryError excep) {
logger.log(Level.WARNING, "OutOfMemory caught when validating item", excep); //$NON-NLS-1$
VeraPDFException veraExcep = new VeraPDFException("OutOfMemory caught when validating item", excep); //$NON-NLS-1$
this.taskResults.put(TaskType.EXTRACT_FEATURES,
TaskResultImpl.fromValues(TaskType.EXTRACT_FEATURES, timer.stop(), veraExcep));
} catch (Throwable e) {
logger.log(Level.WARNING, "Exception caught when extracting features of item", e); //$NON-NLS-1$
VeraPDFException veraExcep = new VeraPDFException("Exception caught when extracting features of item", e); //$NON-NLS-1$
Expand Down
11 changes: 11 additions & 0 deletions core/src/main/java/org/verapdf/processor/ProcessorResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,4 +91,15 @@ public interface ProcessorResult {
* not continue to process it.
*/
public boolean isEncryptedPdf();

/**
* @return true if the parser detected OutOfMemoryException and could
* not continue to process PDF.
*/
public boolean isOutOfMemory();

/**
* @return true if processed PDF has VeraPDFException.
*/
public boolean hasException();
}
37 changes: 29 additions & 8 deletions core/src/main/java/org/verapdf/processor/ProcessorResultImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ class ProcessorResultImpl implements ProcessorResult {
private final boolean isPdf;
@XmlAttribute
private final boolean isEncryptedPdf;
@XmlAttribute
private final boolean isOutOfMemory;
@XmlAttribute
private final boolean hasException;
@XmlElement
private final ItemDetails itemDetails;
private final EnumMap<TaskType, TaskResult> taskResults;
Expand All @@ -66,33 +70,36 @@ private ProcessorResultImpl() {
}

private ProcessorResultImpl(final ItemDetails details, final TaskResult result) {
this(details, false, false, result);
this(details, false, false, false, result);
}

private ProcessorResultImpl(final ItemDetails details, boolean isEncrypted, final TaskResult result) {
this(details, true, isEncrypted, result);
private ProcessorResultImpl(final ItemDetails details, boolean isEncrypted, boolean isOutOfMemory, final TaskResult result) {
this(details, true, isEncrypted, isOutOfMemory, result);
}

private ProcessorResultImpl(final ItemDetails details, boolean isValidPdf, boolean isEncrypted,
private ProcessorResultImpl(final ItemDetails details, boolean isValidPdf, boolean isEncrypted, boolean isOutOfMemory,
final TaskResult result) {
this(details, isValidPdf, isEncrypted, resMap(result), ValidationResults.defaultResult(),
this(details, isValidPdf, isEncrypted, isOutOfMemory, resMap(result), ValidationResults.defaultResult(),
new FeatureExtractionResult(), FixerFactory.defaultResult());
}

private ProcessorResultImpl(final ItemDetails details, final EnumMap<TaskType, TaskResult> results,
final ValidationResult validationResult, final FeatureExtractionResult featuresResult,
final MetadataFixerResult fixerResult) {
this(details, true, false, results, validationResult, featuresResult, fixerResult);
this(details, true, false, false, results, validationResult, featuresResult, fixerResult);
}

private ProcessorResultImpl(final ItemDetails details, final boolean isPdf, final boolean isEncrypted,
final boolean isOutOfMemory,
final EnumMap<TaskType, TaskResult> results, final ValidationResult validationResult,
final FeatureExtractionResult featuresResult, final MetadataFixerResult fixerResult) {
super();
this.itemDetails = details;
this.isPdf = isPdf;
this.isEncryptedPdf = isEncrypted;
this.isOutOfMemory = isOutOfMemory;
this.taskResults = results;
this.hasException = this.taskResults.values().stream().anyMatch(res -> res.getException() != null);
this.validationResult = validationResult;
this.featuresResult = featuresResult;
this.fixerResult = fixerResult;
Expand Down Expand Up @@ -130,15 +137,19 @@ static ProcessorResult defaultInstance() {
static ProcessorResult fromValues(final ItemDetails details, final EnumMap<TaskType, TaskResult> results,
final ValidationResult validationResult, final FeatureExtractionResult featuresResult,
final MetadataFixerResult fixerResult) {
return new ProcessorResultImpl(details, true, false, results, validationResult, featuresResult, fixerResult);
return new ProcessorResultImpl(details, true, false, false, results, validationResult, featuresResult, fixerResult);
}

static ProcessorResult invalidPdfResult(final ItemDetails details, final TaskResult res) {
return new ProcessorResultImpl(details, res);
}

static ProcessorResult encryptedResult(final ItemDetails details, final TaskResult res) {
return new ProcessorResultImpl(details, true, true, res);
return new ProcessorResultImpl(details, true, false, res);
}

static ProcessorResult outOfMemoryResult(final ItemDetails details, final TaskResult res) {
return new ProcessorResultImpl(details, false, true, res);
}

@Override
Expand Down Expand Up @@ -172,6 +183,16 @@ public boolean isEncryptedPdf() {
return this.isEncryptedPdf;
}

@Override
public boolean isOutOfMemory() {
return this.isOutOfMemory;
}

@Override
public boolean hasException() {
return this.hasException;
}

static class Adapter extends XmlAdapter<ProcessorResultImpl, ProcessorResult> {
@Override
public ProcessorResult unmarshal(ProcessorResultImpl procResultImpl) {
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/verapdf/processor/reports/BatchSummary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,8 @@ public interface BatchSummary {
int getFailedParsingJobs();

int getFailedEncryptedJobs();

int getOutOfMemory();

int getVeraExceptions();
}
37 changes: 24 additions & 13 deletions core/src/main/java/org/verapdf/processor/reports/BatchSummaryImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,20 +47,19 @@ final class BatchSummaryImpl implements BatchSummary {
private final int failedToParse;
@XmlAttribute
private final int encrypted;
@XmlAttribute
private final int outOfMemory;
@XmlAttribute
private final int veraExceptions;

private BatchSummaryImpl() {
this(Components.defaultDuration(), ValidationBatchSummaryImpl.defaultInstance(),
FeaturesBatchSummary.defaultInstance(), MetadataRepairBatchSummary.defaultInstance(), 0, 0, 0);
FeaturesBatchSummary.defaultInstance(), MetadataRepairBatchSummary.defaultInstance(), 0, 0, 0, 0, 0);
}

/**
* @param duration
* @param jobs
* @param failedJobs
*/
private BatchSummaryImpl(final AuditDuration duration, final ValidationBatchSummary validationSummary,
final FeaturesBatchSummary featureSummary, final MetadataRepairBatchSummary repairSummary,
final int totalJobs, final int failedToParse, final int encrypted) {
final FeaturesBatchSummary featureSummary, final MetadataRepairBatchSummary repairSummary,
final int totalJobs, final int failedToParse, final int encrypted, int outOfMemory, int veraExceptions) {
super();
this.duration = duration;
this.validationReports = validationSummary;
Expand All @@ -69,6 +68,8 @@ private BatchSummaryImpl(final AuditDuration duration, final ValidationBatchSumm
this.totalJobs = totalJobs;
this.failedToParse = failedToParse;
this.encrypted = encrypted;
this.outOfMemory = outOfMemory;
this.veraExceptions = veraExceptions;
}

/**
Expand Down Expand Up @@ -114,15 +115,25 @@ public int getFailedEncryptedJobs() {
return this.encrypted;
}

public static final BatchSummary defaultInstance() {
@Override
public int getOutOfMemory() {
return this.outOfMemory;
}

@Override
public int getVeraExceptions() {
return this.veraExceptions;
}

public static BatchSummary defaultInstance() {
return DEFAULT;
}

static final BatchSummary fromValues(final AuditDuration duration, final ValidationBatchSummary validationSummary,
final FeaturesBatchSummary featureSummary, final MetadataRepairBatchSummary repairSummary,
final int totalJobs, final int failedToParse, final int encrypted) {
static BatchSummary fromValues(final AuditDuration duration, final ValidationBatchSummary validationSummary,
final FeaturesBatchSummary featureSummary, final MetadataRepairBatchSummary repairSummary,
final int totalJobs, final int failedToParse, final int encrypted, final int outOfMemory, final int veraExceptions) {
return new BatchSummaryImpl(duration, validationSummary, featureSummary, repairSummary, totalJobs,
failedToParse, encrypted);
failedToParse, encrypted, outOfMemory, veraExceptions);
}

static class Adapter extends XmlAdapter<BatchSummaryImpl, BatchSummary> {
Expand Down
7 changes: 5 additions & 2 deletions core/src/main/java/org/verapdf/processor/reports/Reports.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,15 @@ private Reports() {
* values
*/
public static final BatchSummary createBatchSummary(final Components.Timer timer, final ValidationBatchSummary validationSummary,
final FeaturesBatchSummary featureSummary, final MetadataRepairBatchSummary repairSummary, final int totalJobs, final int failedToParse, final int encrypted) {
final FeaturesBatchSummary featureSummary, final MetadataRepairBatchSummary repairSummary, final int totalJobs,
final int failedToParse, final int encrypted, final int outOfMemory, final int veraExceptions) {
if (totalJobs < 0) throw new IllegalArgumentException("Argument totalJobs must be >= 0"); //$NON-NLS-1$
if (failedToParse < 0) throw new IllegalArgumentException("Argument failedToParse must be >= 0"); //$NON-NLS-1$
if (encrypted < 0) throw new IllegalArgumentException("Argument encrypted must be >= 0"); //$NON-NLS-1$
if (outOfMemory < 0) throw new IllegalArgumentException("Argument outOfMemory must be >= 0"); //$NON-NLS-1$
if (veraExceptions < 0) throw new IllegalArgumentException("Argument veraExceptions must be >= 0"); //$NON-NLS-1$
if ((failedToParse + encrypted) > totalJobs) throw new IllegalArgumentException("Argument totalJobs must be >= arguments (failedJobs + encrypted)"); //$NON-NLS-1$
return BatchSummaryImpl.fromValues(timer.stop(), validationSummary, featureSummary, repairSummary, totalJobs, failedToParse, encrypted);
return BatchSummaryImpl.fromValues(timer.stop(), validationSummary, featureSummary, repairSummary, totalJobs, failedToParse, encrypted, outOfMemory, veraExceptions);
}

/**
Expand Down

0 comments on commit 8386bf5

Please sign in to comment.