Merge pull request #2 from vaa25/huge

add HugeTest.java

README.adoc

@@ -51,3 +51,4 @@ Also:
 - Poiji2 has usable builders for any case (ex. use `Poiji.<JavaClass>fromExcel().withSource(new File(...)).withJavaType(JavaClass.class).toStream()`)
 - Poiji2 (since v1.2.1) can be used with https://github.com/vaa25/spring-boot-starter-web-excel[spring-boot-starter-web-excel]
 - Poiji2 can read lists in row (use `@ExcelList` on `List`)
+- Poiji2 can read and write huge xlsx files (see HugeTest.java)

src/test/java/com/poiji/deserialize/HugeEntityBuilder.java

@@ -0,0 +1,39 @@
+package com.poiji.deserialize;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class HugeEntityBuilder {
+
+    public static void main(String[] args) throws IOException {
+
+//        final int totalFields = SpreadsheetVersion.EXCEL2007.getLastColumnIndex();
+        final int totalFields = 50;
+        final StringBuilder body = new StringBuilder("package com.poiji.deserialize.model;\n" +
+                "\n" +
+                "import com.poiji.annotation.ExcelCell;\n" +
+                "import com.poiji.annotation.ExcelSheet;\n" +
+                "\n" +
+                "@ExcelSheet(\"test\")\n" +
+                "public class HugeEntity {\n" +
+                "public int totalFields = ")
+                .append(totalFields)
+                .append(";\n");
+
+        for (int i = 0; i < totalFields; i++) {
+            body.append(String.format("@ExcelCell(%d) public String field%d;%n", i, i));
+
+        }
+
+        body.append("}\n");
+
+        System.out.println(body);
+        final Path path = Paths.get(System.getProperty("user.dir") + "/src/test/java/com/poiji/deserialize/model/HugeEntity.java");
+        Files.deleteIfExists(path);
+        Files.createFile(path);
+        Files.write(path, body.toString().getBytes(StandardCharsets.UTF_8));
+    }
+}

src/test/java/com/poiji/deserialize/HugeTest.java

@@ -0,0 +1,124 @@
+package com.poiji.deserialize;
+
+import com.poiji.bind.Poiji;
+import com.poiji.deserialize.model.HugeEntity;
+import com.poiji.option.PoijiOptions;
+import org.apache.poi.ss.SpreadsheetVersion;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+import static java.util.Arrays.asList;
+
+/**
+ * For manual testing only.
+ */
+public class HugeTest {
+
+    /**
+     * Rows: 1048575
+     * Columns: 50
+     * Written in 138303 ms
+     * src/test/resources/concurrent4.xlsx (210273 kB, 1048575 rows) read in 138001 ms
+     * src/test/resources/concurrent3.xlsx (210273 kB, 1048575 rows) read in 138401 ms
+     * src/test/resources/concurrent1.xlsx (210273 kB, 1048575 rows) read in 138672 ms
+     * src/test/resources/concurrent2.xlsx (210273 kB, 1048575 rows) read in 138934 ms
+     * Total read in 138935 ms
+     */
+    @Test
+    @Ignore("Test disabled to prevent huge xlsx files writing in CI")
+    public void writeThenReadStream() {
+
+        final long start = System.nanoTime();
+        final int size = SpreadsheetVersion.EXCEL2007.getLastRowIndex();
+        System.out.println("Rows: " + size);
+        System.out.println("Columns: " + new HugeEntity().totalFields);
+        final Stream<HugeEntity> entities1 = generateEntities(size, "1");
+        final Stream<HugeEntity> entities2 = generateEntities(size, "2");
+        final Stream<HugeEntity> entities3 = generateEntities(size, "3");
+        final Stream<HugeEntity> entities4 = generateEntities(size, "4");
+        final String name1 = "src/test/resources/concurrent1.xlsx";
+        final String name2 = "src/test/resources/concurrent2.xlsx";
+        final String name3 = "src/test/resources/concurrent3.xlsx";
+        final String name4 = "src/test/resources/concurrent4.xlsx";
+        final List<WriteData> writeData = asList(
+                new WriteData(name1, entities1),
+                new WriteData(name2, entities2),
+                new WriteData(name3, entities3),
+                new WriteData(name4, entities4)
+        );
+
+        final PoijiOptions options = PoijiOptions.PoijiOptionsBuilder.settings().preferNullOverDefault(true).build();
+
+        writeData.parallelStream()
+                .forEach(data -> Poiji.toExcel(new File(data.path), HugeEntity.class, data.entities, options));
+
+        final long written = System.nanoTime();
+        System.out.println("Written in " + (written - start) / 1000000 + " ms");
+
+        asList(name1, name2, name3, name4)
+                .parallelStream()
+                .forEach(fileName -> load(fileName, options));
+
+        final long read = System.nanoTime();
+        System.out.println("Total read in " + (read - written) / 1000000 + " ms");
+
+    }
+
+    private Stream<HugeEntity> generateEntities(final int size, final String marker) {
+        final AtomicInteger index = new AtomicInteger();
+        return Stream.generate(() -> generateEntity(index.getAndIncrement(), marker))
+                .limit(size);
+    }
+
+    public HugeEntity generateEntity(int index, final String marker) {
+        final HugeEntity hugeEntity = new HugeEntity();
+        final Field[] declaredFields = hugeEntity.getClass().getDeclaredFields();
+        for (Field declaredField : declaredFields) {
+            if (declaredField.getName().startsWith("field")) {
+                final String fieldIndex = declaredField.getName().substring("field".length());
+                final String value = fieldIndex + "_" + index + "_" + marker;
+                try {
+                    declaredField.set(hugeEntity, value);
+                } catch (IllegalAccessException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+        return hugeEntity;
+    }
+
+    private void load(String fileName, PoijiOptions options) {
+        final long written = System.nanoTime();
+        final File file = new File(fileName);
+        final long count = Poiji.fromExcelToStream(file, HugeEntity.class, options).count();
+        final long size = getSize(file);
+        System.out.printf("%s (%d kB, %d rows) read in %d ms%n", fileName, size / 1024, count, (System.nanoTime() - written) / 1000000);
+    }
+
+    private long getSize(File file) {
+        try {
+            return Files.size(file.toPath());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static class WriteData {
+        private final String path;
+        private final Stream<HugeEntity> entities;
+
+        public WriteData(final String path, final Stream<HugeEntity> entities) {
+            this.path = path;
+            this.entities = entities;
+        }
+    }
+
+}

src/test/java/com/poiji/deserialize/model/HugeEntity.java

@@ -0,0 +1,59 @@
+package com.poiji.deserialize.model;
+
+import com.poiji.annotation.ExcelCell;
+import com.poiji.annotation.ExcelSheet;
+
+@ExcelSheet("test")
+public class HugeEntity {
+public int totalFields = 50;
+@ExcelCell(0) public String field0;
+@ExcelCell(1) public String field1;
+@ExcelCell(2) public String field2;
+@ExcelCell(3) public String field3;
+@ExcelCell(4) public String field4;
+@ExcelCell(5) public String field5;
+@ExcelCell(6) public String field6;
+@ExcelCell(7) public String field7;
+@ExcelCell(8) public String field8;
+@ExcelCell(9) public String field9;
+@ExcelCell(10) public String field10;
+@ExcelCell(11) public String field11;
+@ExcelCell(12) public String field12;
+@ExcelCell(13) public String field13;
+@ExcelCell(14) public String field14;
+@ExcelCell(15) public String field15;
+@ExcelCell(16) public String field16;
+@ExcelCell(17) public String field17;
+@ExcelCell(18) public String field18;
+@ExcelCell(19) public String field19;
+@ExcelCell(20) public String field20;
+@ExcelCell(21) public String field21;
+@ExcelCell(22) public String field22;
+@ExcelCell(23) public String field23;
+@ExcelCell(24) public String field24;
+@ExcelCell(25) public String field25;
+@ExcelCell(26) public String field26;
+@ExcelCell(27) public String field27;
+@ExcelCell(28) public String field28;
+@ExcelCell(29) public String field29;
+@ExcelCell(30) public String field30;
+@ExcelCell(31) public String field31;
+@ExcelCell(32) public String field32;
+@ExcelCell(33) public String field33;
+@ExcelCell(34) public String field34;
+@ExcelCell(35) public String field35;
+@ExcelCell(36) public String field36;
+@ExcelCell(37) public String field37;
+@ExcelCell(38) public String field38;
+@ExcelCell(39) public String field39;
+@ExcelCell(40) public String field40;
+@ExcelCell(41) public String field41;
+@ExcelCell(42) public String field42;
+@ExcelCell(43) public String field43;
+@ExcelCell(44) public String field44;
+@ExcelCell(45) public String field45;
+@ExcelCell(46) public String field46;
+@ExcelCell(47) public String field47;
+@ExcelCell(48) public String field48;
+@ExcelCell(49) public String field49;
+}