diff --git a/README.adoc b/README.adoc index 4e544d1..a64912f 100644 --- a/README.adoc +++ b/README.adoc @@ -14,7 +14,7 @@ In your Maven/Gradle project, first add the corresponding dependency: io.github.vaa25 poiji2 - 1.4.1 + 1.5.0 org.apache.poi @@ -29,7 +29,7 @@ In your Maven/Gradle project, first add the corresponding dependency: [source,groovy] ---- dependencies { - implementation 'io.github.vaa25:poiji2:1.4.1' + implementation 'io.github.vaa25:poiji2:1.5.0' implementation 'org.apache.poi:poi-ooxml:5.2.5' } ---- @@ -52,3 +52,4 @@ Also: - Poiji2 can read and write huge xlsx files (see HugeTest.java) - Poiji2 (since v1.4.0) can work with immutable java classes (see IgnoreTest.java). lombok @Value and java records applicable also. - Poiji2 (since v1.4.1) can work with immutable java classes with many constructors (see ExcelConstructorTest.java). Apply @ExcelConstructor to choose one of. +- Poiji2 (since v1.5.0) can read sheet names (see ReadSheetNamesTest.java). diff --git a/src/main/java/com/poiji/bind/FromExcel.java b/src/main/java/com/poiji/bind/FromExcel.java index d189b41..68240f6 100644 --- a/src/main/java/com/poiji/bind/FromExcel.java +++ b/src/main/java/com/poiji/bind/FromExcel.java @@ -5,6 +5,8 @@ import com.poiji.exception.PoijiExcelType; import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; +import org.apache.poi.ss.usermodel.Sheet; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -13,7 +15,6 @@ import java.util.List; import java.util.function.Consumer; import java.util.stream.Stream; -import org.apache.poi.ss.usermodel.Sheet; public class FromExcel { @@ -42,6 +43,12 @@ public List toList() { return result; } + public List readSheetNames() { + validateSource(); + validateOptions(); + return source.getDeserializer(options).readSheetNames(); + } + public Stream toStream() { validate(); final Stream stream = source.getDeserializer(options).stream(javaType); @@ -53,17 +60,29 @@ public Stream toStream() { } private void validate() { - if (source == null) { - throw new PoijiException("Source must be set"); - } + validateSource(); + validateJavaType(); + validateOptions(); + } + + private void validateJavaType() { if (javaType == null) { throw new PoijiException("Class must be set"); } + } + + private void validateOptions() { if (options == null){ options = PoijiOptions.PoijiOptionsBuilder.settings().build(); } } + private void validateSource() { + if (source == null) { + throw new PoijiException("Source must be set"); + } + } + public FromExcel withConsumer(final Consumer consumer) { this.consumer = consumer; return this; @@ -173,4 +192,4 @@ public Unmarshaller getDeserializer(final PoijiOptions options) { } } -} \ No newline at end of file +} diff --git a/src/main/java/com/poiji/bind/Unmarshaller.java b/src/main/java/com/poiji/bind/Unmarshaller.java index 5c71e73..5baff30 100644 --- a/src/main/java/com/poiji/bind/Unmarshaller.java +++ b/src/main/java/com/poiji/bind/Unmarshaller.java @@ -1,5 +1,6 @@ package com.poiji.bind; +import java.util.List; import java.util.function.Consumer; import java.util.stream.Stream; @@ -11,4 +12,6 @@ public interface Unmarshaller { void unmarshal(Class type, Consumer consumer); Stream stream(Class type); + + List readSheetNames(); } diff --git a/src/main/java/com/poiji/bind/mapping/CsvUnmarshallerStream.java b/src/main/java/com/poiji/bind/mapping/CsvUnmarshallerStream.java index 1e44bf9..e66dc18 100644 --- a/src/main/java/com/poiji/bind/mapping/CsvUnmarshallerStream.java +++ b/src/main/java/com/poiji/bind/mapping/CsvUnmarshallerStream.java @@ -4,10 +4,12 @@ import com.poiji.bind.Unmarshaller; import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import com.poiji.util.ReflectUtil; + +import java.io.*; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; @@ -47,6 +49,20 @@ public Stream stream(final Class type) { } } + /** + * Excel uses file name as sheet name for CSV format. + */ + @Override + public List readSheetNames() { + final InputStream stream = poijiStream.stream(); + if (stream instanceof FileInputStream) { + final String path = ReflectUtil.getFieldData("path", stream); + final String fileName = Paths.get(path).getFileName().toString(); + return Collections.singletonList(fileName.substring(0, fileName.lastIndexOf("."))); + } + return options.preferNullOverDefault() ? null : Collections.emptyList(); + } + private static class BomInputStream extends InputStream{ private final InputStream inner; @@ -73,7 +89,7 @@ public Optional getCharset(){ } @Override - public int read() throws IOException { + public int read() { return 0; } diff --git a/src/main/java/com/poiji/bind/mapping/HSSFUnmarshaller.java b/src/main/java/com/poiji/bind/mapping/HSSFUnmarshaller.java index ee1c13e..40e46f1 100644 --- a/src/main/java/com/poiji/bind/mapping/HSSFUnmarshaller.java +++ b/src/main/java/com/poiji/bind/mapping/HSSFUnmarshaller.java @@ -4,22 +4,16 @@ import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; import com.poiji.save.TransposeUtil; +import org.apache.poi.hssf.usermodel.HSSFFormulaEvaluator; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.formula.BaseFormulaEvaluator; +import org.apache.poi.ss.usermodel.*; + import java.io.IOException; -import java.util.Iterator; -import java.util.Optional; -import java.util.Spliterator; -import java.util.Spliterators; +import java.util.*; import java.util.function.Consumer; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.apache.poi.hssf.usermodel.HSSFFormulaEvaluator; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.ss.formula.BaseFormulaEvaluator; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellType; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; /** * This is the main class that converts the excel sheet fromExcel Java object @@ -47,6 +41,17 @@ public void unmarshal(Class type, Consumer consumer) { } } + @Override + public List readSheetNames() { + try (final HSSFWorkbook workbook = (HSSFWorkbook) workbook()) { + final List result = new ArrayList<>(); + workbook.forEach(sheet-> result.add(sheet.getSheetName())); + return result; + } catch (final IOException e) { + throw new PoijiException("Problem occurred while closing HSSFWorkbook", e); + } + } + private Sheet getSheet(final Class type, final HSSFWorkbook workbook) { if (options.getTransposed()){ TransposeUtil.transpose(workbook); diff --git a/src/main/java/com/poiji/bind/mapping/ReadMappedFields.java b/src/main/java/com/poiji/bind/mapping/ReadMappedFields.java index b782d3d..1e6ac09 100644 --- a/src/main/java/com/poiji/bind/mapping/ReadMappedFields.java +++ b/src/main/java/com/poiji/bind/mapping/ReadMappedFields.java @@ -1,26 +1,15 @@ package com.poiji.bind.mapping; -import com.poiji.annotation.ExcelCell; -import com.poiji.annotation.ExcelCellName; -import com.poiji.annotation.ExcelCellRange; -import com.poiji.annotation.ExcelList; -import com.poiji.annotation.ExcelParseExceptions; -import com.poiji.annotation.ExcelRow; -import com.poiji.annotation.ExcelUnknownCells; -import com.poiji.annotation.ExcelWriteOnly; +import com.poiji.annotation.*; import com.poiji.config.Casting; import com.poiji.exception.ExcelParseException; import com.poiji.option.PoijiOptions; import com.poiji.util.AnnotationUtil; import com.poiji.util.ReflectUtil; + import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static com.poiji.annotation.ExcelCellName.ABSENT_ORDER; import static java.util.Arrays.asList; @@ -127,7 +116,7 @@ public void setCellInData(final int row, final int column, final String content, for (final Field unknownField : unknownFields) { final Object unknownData = data.get(unknownField); if (unknownData == null) { - final Map map = new HashMap<>(); + final Map map = ReflectUtil.newMap(unknownField); data.put(unknownField, map); map.put(unknownColumns.get(column), content); } else { @@ -235,7 +224,7 @@ private List getPossibleFieldNames(final ExcelCellName annotation) { private List parseUnknownCells(final List fields) { final List rest = new ArrayList<>(fields.size()); for (final Field field : fields) { - if (field.getAnnotation(ExcelUnknownCells.class) != null && field.getType().isAssignableFrom(Map.class)) { + if (field.isAnnotationPresent(ExcelUnknownCells.class) && Map.class.isAssignableFrom(field.getType())) { unknownFields.add(field); ReflectUtil.setAccessible(field); } else { diff --git a/src/main/java/com/poiji/bind/mapping/SheetUnmarshaller.java b/src/main/java/com/poiji/bind/mapping/SheetUnmarshaller.java index eb6c049..3f60d6e 100644 --- a/src/main/java/com/poiji/bind/mapping/SheetUnmarshaller.java +++ b/src/main/java/com/poiji/bind/mapping/SheetUnmarshaller.java @@ -2,8 +2,6 @@ import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; -import java.util.function.Consumer; -import java.util.stream.Stream; import org.apache.poi.hssf.usermodel.HSSFFormulaEvaluator; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.ss.usermodel.Sheet; @@ -11,6 +9,11 @@ import org.apache.poi.xssf.usermodel.XSSFFormulaEvaluator; import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Stream; + /** * Created by hakan on 11.10.2020 */ @@ -29,6 +32,11 @@ public void unmarshal(Class type, Consumer consumer) { processRowsToObjects(sheet, type, consumer); } + @Override + public List readSheetNames() { + return Collections.singletonList(sheet.getSheetName()); + } + private void setBaseFormulaEvaluator() { Workbook workbook = workbook(); if (workbook instanceof HSSFWorkbook) { diff --git a/src/main/java/com/poiji/bind/mapping/XSSFUnmarshaller.java b/src/main/java/com/poiji/bind/mapping/XSSFUnmarshaller.java index a24a732..4dd0071 100644 --- a/src/main/java/com/poiji/bind/mapping/XSSFUnmarshaller.java +++ b/src/main/java/com/poiji/bind/mapping/XSSFUnmarshaller.java @@ -4,18 +4,6 @@ import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; import com.poiji.save.TransposeUtil; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.List; -import java.util.Locale; -import java.util.Optional; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.function.Consumer; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import javax.xml.parsers.ParserConfigurationException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.poifs.filesystem.DocumentFactoryHelper; @@ -34,6 +22,15 @@ import org.xml.sax.SAXException; import org.xml.sax.XMLReader; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + import static org.apache.poi.openxml4j.opc.PackageAccess.READ_WRITE; /** @@ -47,78 +44,84 @@ abstract class XSSFUnmarshaller implements Unmarshaller { this.options = options; } - protected void unmarshal0(Class type, Consumer consumer, OPCPackage open) - throws ParserConfigurationException, IOException, SAXException, OpenXML4JException - { - if (options.getTransposed()) { - if (open.getPackageAccess() == READ_WRITE) { - final XSSFWorkbook workbook = new XSSFWorkbook(open); - TransposeUtil.transpose(workbook); - workbook.write(new OutputStream() { - @Override - public void write(final int b) { - } - }); - } else { - throw new UnsupportedOperationException("Can't apply transposition for streamed XLSX source"); - } - } - - ReadOnlySharedStringsTable readOnlySharedStringsTable = new ReadOnlySharedStringsTable(open); - XSSFReader workbookReader = new XSSFReader(open); - StylesTable styles = workbookReader.getStylesTable(); - XMLReader reader = XMLHelper.newXMLReader(); - - InputSource is = new InputSource(workbookReader.getWorkbookData()); + @Override + public void unmarshal(Class type, Consumer consumer) { + openFileAndExecute(opcPackage -> unmarshal0(type, consumer, opcPackage)); + } - reader.setContentHandler(new WorkBookContentHandler(options)); - reader.parse(is); + protected void unmarshal0(Class type, Consumer consumer, OPCPackage open) { + try { + if (options.getTransposed()) { + if (open.getPackageAccess() == READ_WRITE) { + final XSSFWorkbook workbook = new XSSFWorkbook(open); + TransposeUtil.transpose(workbook); + workbook.write(new OutputStream() { + @Override + public void write(final int b) { + } + }); + } else { + throw new UnsupportedOperationException("Can't apply transposition for streamed XLSX source"); + } + } + ReadOnlySharedStringsTable readOnlySharedStringsTable = new ReadOnlySharedStringsTable(open); + XSSFReader workbookReader = new XSSFReader(open); + StylesTable styles = workbookReader.getStylesTable(); + XMLReader reader = XMLHelper.newXMLReader(); - WorkBookContentHandler wbch = (WorkBookContentHandler) reader.getContentHandler(); - List sheets = wbch.getSheets(); - if (sheets.isEmpty()) { - throw new PoijiException("no excel sheets found"); - } - PoijiNumberFormat poijiNumberFormat = options.getPoijiNumberFormat(); - if (poijiNumberFormat != null) { - poijiNumberFormat.overrideExcelNumberFormats(styles); - } + InputSource is = new InputSource(workbookReader.getWorkbookData()); - SheetIterator iter = (SheetIterator) workbookReader.getSheetsData(); - int sheetCounter = 0; + reader.setContentHandler(new WorkBookContentHandler(options)); + reader.parse(is); - Optional maybeSheetName = SheetNameExtractor.getSheetName(type, options); + WorkBookContentHandler wbch = (WorkBookContentHandler) reader.getContentHandler(); + List sheets = wbch.getSheets(); + if (sheets.isEmpty()) { + throw new PoijiException("no excel sheets found"); + } + PoijiNumberFormat poijiNumberFormat = options.getPoijiNumberFormat(); + if (poijiNumberFormat != null) { + poijiNumberFormat.overrideExcelNumberFormats(styles); + } - if (!maybeSheetName.isPresent()) { - int requestedIndex = options.sheetIndex(); - int nonHiddenSheetIndex = 0; - while (iter.hasNext()) { - try (InputStream stream = iter.next()) { - WorkBookSheet wbs = sheets.get(sheetCounter); - if (wbs.getState().equals("visible")) { - if (nonHiddenSheetIndex == requestedIndex) { - processSheet(styles, reader, readOnlySharedStringsTable, type, stream, consumer); - return; + SheetIterator iter = (SheetIterator) workbookReader.getSheetsData(); + int sheetCounter = 0; + + Optional maybeSheetName = SheetNameExtractor.getSheetName(type, options); + + if (!maybeSheetName.isPresent()) { + int requestedIndex = options.sheetIndex(); + int nonHiddenSheetIndex = 0; + while (iter.hasNext()) { + try (InputStream stream = iter.next()) { + WorkBookSheet wbs = sheets.get(sheetCounter); + if (wbs.getState().equals("visible")) { + if (nonHiddenSheetIndex == requestedIndex) { + processSheet(styles, reader, readOnlySharedStringsTable, type, stream, consumer); + return; + } + nonHiddenSheetIndex++; } - nonHiddenSheetIndex++; } + sheetCounter++; } - sheetCounter++; - } - } else { - String sheetName = maybeSheetName.get(); - while (iter.hasNext()) { - try (InputStream stream = iter.next()) { - WorkBookSheet wbs = sheets.get(sheetCounter); - if (wbs.getState().equals("visible")) { - if (iter.getSheetName().equalsIgnoreCase(sheetName)) { - processSheet(styles, reader, readOnlySharedStringsTable, type, stream, consumer); - return; + } else { + String sheetName = maybeSheetName.get(); + while (iter.hasNext()) { + try (InputStream stream = iter.next()) { + WorkBookSheet wbs = sheets.get(sheetCounter); + if (wbs.getState().equals("visible")) { + if (iter.getSheetName().equalsIgnoreCase(sheetName)) { + processSheet(styles, reader, readOnlySharedStringsTable, type, stream, consumer); + return; + } } } + sheetCounter++; } - sheetCounter++; } + } catch (IOException | OpenXML4JException | ParserConfigurationException | SAXException e) { + throw new PoijiException("Problem occurred while reading data: " + e.getMessage(), e); } } @@ -268,15 +271,13 @@ private Stream streamSheet( } - protected final void listOfEncryptedItems(Class type, Consumer consumer, POIFSFileSystem fs) throws IOException { - InputStream stream = DocumentFactoryHelper.getDecryptedStream(fs, options.getPassword()); - - try (OPCPackage open = OPCPackage.open(stream)) { - unmarshal0(type, consumer, open); - - } catch (ParserConfigurationException | SAXException | IOException | OpenXML4JException e) { - IOUtils.closeQuietly(fs); - throw new PoijiException("Problem occurred while reading data", e); + protected final void applyInEncryptedOpcPackage(Consumer process, POIFSFileSystem fileSystem) { + try (InputStream stream = DocumentFactoryHelper.getDecryptedStream(fileSystem, options.getPassword()); + OPCPackage open = OPCPackage.open(stream)) { + process.accept(open); + } catch (IOException | OpenXML4JException e) { + IOUtils.closeQuietly(fileSystem); + throw new PoijiException("Problem occurred while reading data: " + e.getMessage(), e); } } @@ -292,4 +293,22 @@ protected final Stream streamOfEncryptedItems(Class type, POIFSFileSys } } + protected abstract void openFileAndExecute(Consumer process); + + @Override + public List readSheetNames() { + final List result = new ArrayList<>(); + openFileAndExecute(opcPackage -> readSheetNames(opcPackage, result)); + return result; + } + + private void readSheetNames(OPCPackage opcPackage, List result) { + try { + final XSSFWorkbook workbook = new XSSFWorkbook(opcPackage); + workbook.forEach(sheet -> result.add(sheet.getSheetName())); + } catch (IOException e) { + throw new PoijiException("Problem occurred while reading data: " + e.getMessage(), e); + } + } + } diff --git a/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerFile.java b/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerFile.java index 8a1b67c..3ccd364 100644 --- a/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerFile.java +++ b/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerFile.java @@ -3,16 +3,17 @@ import com.poiji.bind.PoijiFile; import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; -import java.io.IOException; -import java.util.function.Consumer; -import java.util.stream.Stream; -import javax.xml.parsers.ParserConfigurationException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.opc.PackageAccess; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.xml.sax.SAXException; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.util.function.Consumer; +import java.util.stream.Stream; + import static org.apache.poi.openxml4j.opc.PackageAccess.READ; import static org.apache.poi.openxml4j.opc.PackageAccess.READ_WRITE; @@ -28,13 +29,13 @@ final class XSSFUnmarshallerFile extends XSSFUnmarshaller { this.poijiFile = poijiFile; } - @Override - public void unmarshal(Class type, Consumer consumer) { + + protected void openFileAndExecute(Consumer process) { if (options.getPassword() != null) { - returnFromEncryptedFile(type, consumer); + applyInFileSystem(fileSystem-> applyInEncryptedOpcPackage(process, fileSystem)); } else { - returnFromExcelFile(type,consumer); + applyInOpcPackage(process); } } @@ -52,27 +53,20 @@ public Stream stream(final Class type) { } catch (ParserConfigurationException | SAXException | IOException | OpenXML4JException e) { throw new PoijiException("Problem occurred while reading data", e); } - } - public void returnFromExcelFile(Class type, Consumer consumer) { + private void applyInOpcPackage(Consumer process) { final PackageAccess packageAccess = options.getTransposed() ? READ_WRITE : READ; - try (OPCPackage open = OPCPackage.open(poijiFile.file(), packageAccess)) { - - unmarshal0(type, consumer, open); - - } catch (ParserConfigurationException | SAXException | IOException | OpenXML4JException e) { - throw new PoijiException("Problem occurred while reading data", e); + process.accept(open); + } catch (IOException | OpenXML4JException e) { + throw new PoijiException("Problem occurred while reading data: " + e.getMessage(), e); } } - private void returnFromEncryptedFile(Class type, Consumer consumer) { - + private void applyInFileSystem(Consumer process) { try (POIFSFileSystem fs = new POIFSFileSystem(poijiFile.file(), true)) { - - listOfEncryptedItems(type, consumer, fs); - + process.accept(fs); } catch (IOException e) { throw new PoijiException("Problem occurred while reading data", e); } diff --git a/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerStream.java b/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerStream.java index 6290be5..2a2597b 100644 --- a/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerStream.java +++ b/src/main/java/com/poiji/bind/mapping/XSSFUnmarshallerStream.java @@ -3,15 +3,16 @@ import com.poiji.bind.PoijiInputStream; import com.poiji.exception.PoijiException; import com.poiji.option.PoijiOptions; -import java.io.IOException; -import java.util.function.Consumer; -import java.util.stream.Stream; -import javax.xml.parsers.ParserConfigurationException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.xml.sax.SAXException; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.util.function.Consumer; +import java.util.stream.Stream; + /** * Created by hakan on 22/10/2017 */ @@ -24,16 +25,6 @@ final class XSSFUnmarshallerStream extends XSSFUnmarshaller { this.poijiInputStream = poijiInputStream; } - @Override - public void unmarshal(Class type, Consumer consumer) { - - if (options.getPassword() != null) { - returnFromEncryptedFile(type, consumer); - } else { - returnFromExcelFile(type, consumer); - } - } - @Override public Stream stream(final Class type) { try { @@ -49,20 +40,28 @@ public Stream stream(final Class type) { } } - private void returnFromExcelFile(Class type, Consumer consumer) { + private void applyInOpcPackage(Consumer process) { try (OPCPackage open = OPCPackage.open(poijiInputStream.stream())) { - unmarshal0(type, consumer, open); - } catch (ParserConfigurationException | SAXException | IOException | OpenXML4JException e) { - throw new PoijiException("Problem occurred while reading data", e); + process.accept(open); + } catch (IOException | OpenXML4JException e) { + throw new PoijiException("Problem occurred while reading data: " + e.getMessage(), e); } } - private void returnFromEncryptedFile(Class type, Consumer consumer) { - try (POIFSFileSystem fs = new POIFSFileSystem(poijiInputStream.stream())) { - listOfEncryptedItems(type, consumer, fs); + private void applyInFileSystem(Consumer process) { + try (POIFSFileSystem fileSystem = new POIFSFileSystem(poijiInputStream.stream())) { + applyInEncryptedOpcPackage(process, fileSystem); } catch (IOException e) { throw new PoijiException("Problem occurred while reading data", e); } } + @Override + protected void openFileAndExecute(Consumer process) { + if (options.getPassword() != null) { + applyInFileSystem(process); + } else { + applyInOpcPackage(process); + } + } } diff --git a/src/main/java/com/poiji/util/ReflectUtil.java b/src/main/java/com/poiji/util/ReflectUtil.java index 4d703c8..e28433f 100644 --- a/src/main/java/com/poiji/util/ReflectUtil.java +++ b/src/main/java/com/poiji/util/ReflectUtil.java @@ -5,14 +5,12 @@ import com.poiji.exception.IllegalCastException; import com.poiji.exception.PoijiException; import com.poiji.exception.PoijiInstantiationException; + import java.lang.annotation.Annotation; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; public class ReflectUtil { public static T newInstanceOf(Class type) { @@ -170,4 +168,24 @@ public static void setAccessible(Field field) { field.setAccessible(true); } } + + public static T getFieldData(String fieldName, Object instance) { + try { + final Field field = instance.getClass().getDeclaredField(fieldName); + setAccessible(field); + return (T) field.get(instance); + } catch (IllegalAccessException e) { + throw new IllegalCastException("Unexpected cast type {"); + } catch (NoSuchFieldException e) { + throw new PoijiException(e.getMessage(), e); + } + } + + public static Map newMap(Field field) { + final Class type = field.getType(); + if (type == Map.class) { + return new LinkedHashMap<>(); + } + return (Map) newInstanceOf(type); + } } diff --git a/src/test/java/com/poiji/deserialize/CaseInsensitiveTest.java b/src/test/java/com/poiji/deserialize/CaseInsensitiveTest.java index f690d80..822ff22 100644 --- a/src/test/java/com/poiji/deserialize/CaseInsensitiveTest.java +++ b/src/test/java/com/poiji/deserialize/CaseInsensitiveTest.java @@ -3,12 +3,13 @@ import com.poiji.bind.Poiji; import com.poiji.deserialize.model.byname.OrgWithUnknownCellsByName; import com.poiji.option.PoijiOptions; -import java.io.File; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.File; +import java.util.List; + import static java.util.Arrays.asList; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; @@ -46,16 +47,16 @@ public void caseInsensitiveColumnNames() { .filter(org -> org.getId().equals("CrEaTe")) .findFirst() .get(); - assertThat(firstRow.getUnknownCells().size(), is(1)); - assertThat(firstRow.getUnknownCells().get("region"), is("EMEA")); + assertThat(firstRow.getSortedUnknownCells().size(), is(1)); + assertThat(firstRow.getSortedUnknownCells().get("region"), is("EMEA")); OrgWithUnknownCellsByName secondRow = organisations.stream() .filter(org -> org.getId().equals("8d9e6430-8626-4556-8004-079085d2df2d")) .findFirst() .get(); - assertThat(secondRow.getUnknownCells().size(), is(1)); - assertThat(secondRow.getUnknownCells().get("region"), is("NA")); + assertThat(secondRow.getSortedUnknownCells().size(), is(1)); + assertThat(secondRow.getSortedUnknownCells().get("region"), is("NA")); } } diff --git a/src/test/java/com/poiji/deserialize/IgnoreWhitespacesTest.java b/src/test/java/com/poiji/deserialize/IgnoreWhitespacesTest.java index 0450f0f..22176ab 100644 --- a/src/test/java/com/poiji/deserialize/IgnoreWhitespacesTest.java +++ b/src/test/java/com/poiji/deserialize/IgnoreWhitespacesTest.java @@ -3,12 +3,13 @@ import com.poiji.bind.Poiji; import com.poiji.deserialize.model.byname.OrgWithUnknownCellsByName; import com.poiji.option.PoijiOptions; -import java.io.File; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.File; +import java.util.List; + import static java.util.Arrays.asList; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; @@ -51,16 +52,16 @@ public void ignoreWhitespaceColumnNames() { .filter(org -> org.getId().equals("CrEaTe")) .findFirst() .get(); - assertThat(firstRow.getUnknownCells().size(), is(1)); - assertThat(firstRow.getUnknownCells().get("region"), is("EMEA")); + assertThat(firstRow.getSortedUnknownCells().size(), is(1)); + assertThat(firstRow.getSortedUnknownCells().get("region"), is("EMEA")); OrgWithUnknownCellsByName secondRow = organisations.stream() .filter(org -> org.getId().equals("8d9e6430-8626-4556-8004-079085d2df2d")) .findFirst() .get(); - assertThat(secondRow.getUnknownCells().size(), is(1)); - assertThat(secondRow.getUnknownCells().get("region"), is("NA")); + assertThat(secondRow.getSortedUnknownCells().size(), is(1)); + assertThat(secondRow.getSortedUnknownCells().get("region"), is("NA")); } } diff --git a/src/test/java/com/poiji/deserialize/ReadExcelBySheetNameTest.java b/src/test/java/com/poiji/deserialize/ReadExcelBySheetNameTest.java index 9ddf384..2ebfbb7 100644 --- a/src/test/java/com/poiji/deserialize/ReadExcelBySheetNameTest.java +++ b/src/test/java/com/poiji/deserialize/ReadExcelBySheetNameTest.java @@ -1,56 +1,47 @@ package com.poiji.deserialize; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -import java.io.File; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.Arrays; -import java.util.List; - +import com.poiji.bind.Poiji; +import com.poiji.deserialize.model.byid.Calculation; +import com.poiji.option.PoijiOptions; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import com.poiji.bind.Poiji; -import com.poiji.deserialize.model.byid.Calculation; -import com.poiji.option.PoijiOptions; +import java.io.File; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; @RunWith(Parameterized.class) public class ReadExcelBySheetNameTest { - - private String path; - - public ReadExcelBySheetNameTest(String path) { - this.path = path; - } - - @Parameterized.Parameters(name = "{index}: ({0})={1}") - public static Iterable queries() { - return Arrays.asList(new Object[][]{ - {"src/test/resources/calculations.xlsx"}, - }); - } - @Test - public void shouldReadExcelBySheetName() { + private String path; - final DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .appendPattern("M/d/") - .appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, LocalDate.now().minusYears(80)).toFormatter(); + public ReadExcelBySheetNameTest(String path) { + this.path = path; + } - PoijiOptions options = PoijiOptions.PoijiOptionsBuilder.settings().sheetName("SI calculations").dateFormatter(formatter).build(); + @Parameterized.Parameters(name = "{index}: ({0})={1}") + public static Iterable queries() { + return asList(new Object[][]{ + {"src/test/resources/calculations.xlsx"}, + {"src/test/resources/calculations.xls"}, + {"src/test/resources/SI calculations.csv"}, + }); + } - List calculations = Poiji.fromExcel(new File(path), Calculation.class, options); + @Test + public void shouldReadExcelBySheetName() { + PoijiOptions options = PoijiOptions.PoijiOptionsBuilder.settings().sheetName("SI calculations").build(); - for (Calculation calculation : calculations) { + List calculations = Poiji.fromExcel(new File(path), Calculation.class, options); - assertThat(calculation.getToDate().toString(), is("2018-06-30")); - assertThat(calculation.getFromDate().toString(), is("2018-01-01")); - } - } + assertThat(calculations.get(0).getIs(), is("ABXXXXXXXXXX")); + assertThat(calculations.get(1).getIs(), is("BCXXXXXXXXXX")); + assertThat(calculations.get(2).getIs(), is("CDXXXXXXXXXX")); + assertThat(calculations.get(3).getIs(), is("DEXXXXXXXXXX")); + } } diff --git a/src/test/java/com/poiji/deserialize/ReadSheetNamesTest.java b/src/test/java/com/poiji/deserialize/ReadSheetNamesTest.java new file mode 100644 index 0000000..f915f18 --- /dev/null +++ b/src/test/java/com/poiji/deserialize/ReadSheetNamesTest.java @@ -0,0 +1,57 @@ +package com.poiji.deserialize; + +import com.poiji.bind.Poiji; +import com.poiji.exception.PoijiExcelType; +import com.poiji.option.PoijiOptions; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.Collections; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class ReadSheetNamesTest { + + @Test + public void readSheetNamesFromExcelFormat() throws IOException { + readFromExcelFormat(new File("src/test/resources/calculations.xlsx")); + readFromExcelFormat(new File("src/test/resources/calculations.xls")); + } + + private void readFromExcelFormat(File file) throws IOException { + final List expected = asList("Explanatory note", "SI calculations"); + + assertEquals(expected, Poiji.fromExcel().withSource(file).readSheetNames()); + + final InputStream inputStream = Files.newInputStream(file.toPath()); + final PoijiExcelType excelType = PoijiExcelType.fromFileName(file.getName()); + assertEquals(expected, Poiji.fromExcel().withSource(inputStream, excelType).readSheetNames()); + } + + @Test + public void readSheetNamesFromCsvFile() { + final List expected = Collections.singletonList("SI calculations"); + + final File file = new File("src/test/resources/SI calculations.csv"); + assertEquals(expected, Poiji.fromExcel().withSource(file).readSheetNames()); + } + + @Test + public void readSheetNamesFromCsvStream() throws IOException { + + final File file = new File("src/test/resources/SI calculations.csv"); + final PoijiExcelType excelType = PoijiExcelType.CSV; + + assertEquals(Collections.emptyList(), Poiji.fromExcel().withSource(Files.newInputStream(file.toPath()), excelType).readSheetNames()); + + final PoijiOptions options = PoijiOptions.PoijiOptionsBuilder.settings().preferNullOverDefault(true).build(); + assertNull(Poiji.fromExcel().withSource(Files.newInputStream(file.toPath()), excelType).withOptions(options).readSheetNames()); + } + +} diff --git a/src/test/java/com/poiji/deserialize/UnknownCellsIdenticalHeadersTest.java b/src/test/java/com/poiji/deserialize/UnknownCellsIdenticalHeadersTest.java index 4bee575..b33a338 100644 --- a/src/test/java/com/poiji/deserialize/UnknownCellsIdenticalHeadersTest.java +++ b/src/test/java/com/poiji/deserialize/UnknownCellsIdenticalHeadersTest.java @@ -4,13 +4,14 @@ import com.poiji.deserialize.model.byid.OrgWithUnknownCells; import com.poiji.deserialize.model.byname.OrgWithUnknownCellsByName; import com.poiji.option.PoijiOptions; -import java.io.File; -import java.util.Arrays; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.File; +import java.util.Arrays; +import java.util.List; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; @@ -50,18 +51,18 @@ public void byName() { .filter(org -> org.getId().equals("CrEaTe")) .findFirst() .get(); - assertThat(firstRow.getUnknownCells().size(), is(2)); - assertThat(firstRow.getUnknownCells().get("Tag"), is("testTag")); - assertThat(firstRow.getUnknownCells().get("Tag@5"), is("rndTag")); + assertThat(firstRow.getSortedUnknownCells().size(), is(2)); + assertThat(firstRow.getSortedUnknownCells().get("Tag"), is("testTag")); + assertThat(firstRow.getSortedUnknownCells().get("Tag@5"), is("rndTag")); OrgWithUnknownCellsByName secondRow = organisations.stream() .filter(org -> org.getId().equals("8d9e6430-8626-4556-8004-079085d2df2d")) .findFirst() .get(); - assertThat(secondRow.getUnknownCells().size(), is(2)); - assertThat(secondRow.getUnknownCells().get("Tag"), is("testTag2")); - assertThat(secondRow.getUnknownCells().get("Tag@5"), is("rndTag2")); + assertThat(secondRow.getSortedUnknownCells().size(), is(2)); + assertThat(secondRow.getSortedUnknownCells().get("Tag"), is("testTag2")); + assertThat(secondRow.getSortedUnknownCells().get("Tag@5"), is("rndTag2")); } @Test diff --git a/src/test/java/com/poiji/deserialize/UnknownCellsTest.java b/src/test/java/com/poiji/deserialize/UnknownCellsTest.java index 536b7c4..416d3fd 100644 --- a/src/test/java/com/poiji/deserialize/UnknownCellsTest.java +++ b/src/test/java/com/poiji/deserialize/UnknownCellsTest.java @@ -4,13 +4,16 @@ import com.poiji.deserialize.model.byid.OrgWithUnknownCells; import com.poiji.deserialize.model.byname.OrgWithUnknownCellsByName; import com.poiji.option.PoijiOptions; -import java.io.File; -import java.util.Arrays; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.File; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; @@ -34,7 +37,7 @@ public static List excel() { } @Test - public void byName() { + public void byNameFromTreeMap() { List organisations = Poiji.fromExcel( new File(path), OrgWithUnknownCellsByName.class, @@ -50,16 +53,88 @@ public void byName() { .filter(org -> org.getId().equals("CrEaTe")) .findFirst() .get(); - assertThat(firstRow.getUnknownCells().size(), is(1)); - assertThat(firstRow.getUnknownCells().get("Region"), is("EMEA")); + final Map unknownCells1 = firstRow.getSortedUnknownCells(); + assertThat(unknownCells1.size(), is(4)); + + assertRow1Values(unknownCells1); + final Iterator iterator1 = unknownCells1.keySet().iterator(); + assertThat(iterator1.next(), is("Region")); + assertThat(iterator1.next(), is("UnknownName1")); + assertThat(iterator1.next(), is("UnknownName2")); + assertThat(iterator1.next(), is("UnknownName3")); OrgWithUnknownCellsByName secondRow = organisations.stream() .filter(org -> org.getId().equals("8d9e6430-8626-4556-8004-079085d2df2d")) .findFirst() .get(); - assertThat(secondRow.getUnknownCells().size(), is(1)); - assertThat(secondRow.getUnknownCells().get("Region"), is("NA")); + final Map unknownCells2 = secondRow.getSortedUnknownCells(); + assertRow2Values(unknownCells2); + + final Iterator iterator2 = unknownCells2.keySet().iterator(); + assertThat(iterator2.next(), is("Region")); + assertThat(iterator2.next(), is("UnknownName1")); + assertThat(iterator2.next(), is("UnknownName2")); + assertThat(iterator2.next(), is("UnknownName3")); + + } + + @Test + public void byNameFromMap() { + List organisations = Poiji.fromExcel( + new File(path), + OrgWithUnknownCellsByName.class, + PoijiOptions.PoijiOptionsBuilder.settings() + .sheetName("Organisation") + .build() + ); + + assertThat(organisations, notNullValue()); + assertThat(organisations.size(), is(2)); + + OrgWithUnknownCellsByName firstRow = organisations.stream() + .filter(org -> org.getId().equals("CrEaTe")) + .findFirst() + .get(); + final Map unknownCells1 = firstRow.getLinkedUnknownCells(); + assertThat(unknownCells1.size(), is(4)); + + assertRow1Values(unknownCells1); + + final Iterator iterator1 = unknownCells1.keySet().iterator(); + assertThat(iterator1.next(), is("Region")); + assertThat(iterator1.next(), is("UnknownName3")); + assertThat(iterator1.next(), is("UnknownName1")); + assertThat(iterator1.next(), is("UnknownName2")); + + OrgWithUnknownCellsByName secondRow = organisations.stream() + .filter(org -> org.getId().equals("8d9e6430-8626-4556-8004-079085d2df2d")) + .findFirst() + .get(); + final Map unknownCells2 = secondRow.getLinkedUnknownCells(); + assertRow2Values(unknownCells2); + + final Iterator iterator2 = unknownCells2.keySet().iterator(); + assertThat(iterator2.next(), is("Region")); + assertThat(iterator2.next(), is("UnknownName3")); + assertThat(iterator2.next(), is("UnknownName1")); + assertThat(iterator2.next(), is("UnknownName2")); + + } + + private void assertRow1Values(Map unknownCells1) { + assertThat(unknownCells1.get("Region"), is("EMEA")); + assertThat(unknownCells1.get("UnknownName1"), is("UnknownValue11")); + assertThat(unknownCells1.get("UnknownName2"), is("UnknownValue21")); + assertThat(unknownCells1.get("UnknownName3"), is("UnknownValue31")); + } + + private void assertRow2Values(Map unknownCells2) { + assertThat(unknownCells2.size(), is(4)); + assertThat(unknownCells2.get("Region"), is("NA")); + assertThat(unknownCells2.get("UnknownName1"), is("UnknownValue12")); + assertThat(unknownCells2.get("UnknownName2"), is("UnknownValue22")); + assertThat(unknownCells2.get("UnknownName3"), is("UnknownValue32")); } @Test @@ -79,7 +154,7 @@ public void byIndex() { .filter(org -> org.getId().equals("CrEaTe")) .findFirst() .get(); - assertThat(firstRow.getUnknownCells().size(), is(1)); + assertThat(firstRow.getUnknownCells().size(), is(4)); assertThat(firstRow.getUnknownCells().get("Region"), is("EMEA")); @@ -87,7 +162,7 @@ public void byIndex() { .filter(org -> org.getId().equals("8d9e6430-8626-4556-8004-079085d2df2d")) .findFirst() .get(); - assertThat(secondRow.getUnknownCells().size(), is(1)); + assertThat(secondRow.getUnknownCells().size(), is(4)); assertThat(secondRow.getUnknownCells().get("Region"), is("NA")); } } diff --git a/src/test/java/com/poiji/deserialize/model/byname/OrgWithUnknownCellsByName.java b/src/test/java/com/poiji/deserialize/model/byname/OrgWithUnknownCellsByName.java index 66464ee..e188f52 100644 --- a/src/test/java/com/poiji/deserialize/model/byname/OrgWithUnknownCellsByName.java +++ b/src/test/java/com/poiji/deserialize/model/byname/OrgWithUnknownCellsByName.java @@ -5,6 +5,7 @@ import com.poiji.annotation.ExcelUnknownCells; import java.util.Map; +import java.util.TreeMap; public class OrgWithUnknownCellsByName { @@ -23,10 +24,13 @@ public class OrgWithUnknownCellsByName { private String externalId; @ExcelUnknownCells - private Map unknownCells; + private TreeMap sortedUnknownCells; - public Map getUnknownCells() { - return unknownCells; + @ExcelUnknownCells + private Map linkedUnknownCells; + + public Map getSortedUnknownCells() { + return sortedUnknownCells; } @ExcelCellName(HEADER_ORGANISATION_NAME) @@ -74,4 +78,8 @@ public String getCustomerExternalId() { public void setCustomerExternalId(String customerExternalId) { this.customerExternalId = customerExternalId; } + + public Map getLinkedUnknownCells() { + return linkedUnknownCells; + } } diff --git a/src/test/resources/SI calculations.csv b/src/test/resources/SI calculations.csv new file mode 100644 index 0000000..571db03 --- /dev/null +++ b/src/test/resources/SI calculations.csv @@ -0,0 +1,5 @@ +Calculation From Date,Calculation To Date,ISIN,Total number of transactions executed in the EU,Total turnover executed in the EU +1/1/2018,6/30/2018,ABXXXXXXXXXX, 123.00 ," 123,456.78 " +1/1/2018,6/30/2018,BCXXXXXXXXXX," 1,234.00 "," 23,456.78 " +1/1/2018,6/30/2018,CDXXXXXXXXXX, 12.00 ," 3,456.78 " +1/1/2018,6/30/2018,DEXXXXXXXXXX, 1.00 , 456.78 diff --git a/src/test/resources/calculations.xls b/src/test/resources/calculations.xls new file mode 100644 index 0000000..b457d91 Binary files /dev/null and b/src/test/resources/calculations.xls differ diff --git a/src/test/resources/unknown-cells.csv b/src/test/resources/unknown-cells.csv index 30cd13f..b8347f5 100644 --- a/src/test/resources/unknown-cells.csv +++ b/src/test/resources/unknown-cells.csv @@ -1,4 +1,4 @@ -"Organisation ID","Customer External ID","Organisation External ID","Organisation Name","Region" -,,,, -"CrEaTe","customerOrgExt","createOrgExt","createOrg","EMEA" -"8d9e6430-8626-4556-8004-079085d2df2d","customerOrgExt","updateOrgExt","updateOrg","NA" +Organisation ID,Customer External ID,Organisation External ID,Organisation Name,Region,UnknownName3,UnknownName1,UnknownName2 +,,,,,,, +CrEaTe,customerOrgExt,createOrgExt,createOrg,EMEA,UnknownValue31,UnknownValue11,UnknownValue21 +8d9e6430-8626-4556-8004-079085d2df2d,customerOrgExt,updateOrgExt,updateOrg,NA,UnknownValue32,UnknownValue12,UnknownValue22 diff --git a/src/test/resources/unknown-cells.xls b/src/test/resources/unknown-cells.xls index 632bbc0..cf8dbfa 100644 Binary files a/src/test/resources/unknown-cells.xls and b/src/test/resources/unknown-cells.xls differ diff --git a/src/test/resources/unknown-cells.xlsx b/src/test/resources/unknown-cells.xlsx index 0a772f0..f134d29 100644 Binary files a/src/test/resources/unknown-cells.xlsx and b/src/test/resources/unknown-cells.xlsx differ