From 670ab105db8b3e028ba4f7746bee31f571c12b61 Mon Sep 17 00:00:00 2001 From: vaa25 Date: Thu, 14 Mar 2024 17:00:46 +0200 Subject: [PATCH] add HugeTest.java --- README.adoc | 1 + .../poiji/deserialize/HugeEntityBuilder.java | 39 ++++++ .../java/com/poiji/deserialize/HugeTest.java | 124 ++++++++++++++++++ .../poiji/deserialize/model/HugeEntity.java | 59 +++++++++ 4 files changed, 223 insertions(+) create mode 100644 src/test/java/com/poiji/deserialize/HugeEntityBuilder.java create mode 100644 src/test/java/com/poiji/deserialize/HugeTest.java create mode 100644 src/test/java/com/poiji/deserialize/model/HugeEntity.java diff --git a/README.adoc b/README.adoc index 68a7568..6f80c3e 100644 --- a/README.adoc +++ b/README.adoc @@ -51,3 +51,4 @@ Also: - Poiji2 has usable builders for any case (ex. use `Poiji.fromExcel().withSource(new File(...)).withJavaType(JavaClass.class).toStream()`) - Poiji2 (since v1.2.1) can be used with https://github.com/vaa25/spring-boot-starter-web-excel[spring-boot-starter-web-excel] - Poiji2 can read lists in row (use `@ExcelList` on `List`) +- Poiji2 can read and write huge xlsx files (see HugeTest.java) diff --git a/src/test/java/com/poiji/deserialize/HugeEntityBuilder.java b/src/test/java/com/poiji/deserialize/HugeEntityBuilder.java new file mode 100644 index 0000000..a31db1e --- /dev/null +++ b/src/test/java/com/poiji/deserialize/HugeEntityBuilder.java @@ -0,0 +1,39 @@ +package com.poiji.deserialize; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class HugeEntityBuilder { + + public static void main(String[] args) throws IOException { + +// final int totalFields = SpreadsheetVersion.EXCEL2007.getLastColumnIndex(); + final int totalFields = 50; + final StringBuilder body = new StringBuilder("package com.poiji.deserialize.model;\n" + + "\n" + + "import com.poiji.annotation.ExcelCell;\n" + + "import com.poiji.annotation.ExcelSheet;\n" + + "\n" + + "@ExcelSheet(\"test\")\n" + + "public class HugeEntity {\n" + + "public int totalFields = ") + .append(totalFields) + .append(";\n"); + + for (int i = 0; i < totalFields; i++) { + body.append(String.format("@ExcelCell(%d) public String field%d;%n", i, i)); + + } + + body.append("}\n"); + + System.out.println(body); + final Path path = Paths.get(System.getProperty("user.dir") + "/src/test/java/com/poiji/deserialize/model/HugeEntity.java"); + Files.deleteIfExists(path); + Files.createFile(path); + Files.write(path, body.toString().getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/src/test/java/com/poiji/deserialize/HugeTest.java b/src/test/java/com/poiji/deserialize/HugeTest.java new file mode 100644 index 0000000..dfe23fa --- /dev/null +++ b/src/test/java/com/poiji/deserialize/HugeTest.java @@ -0,0 +1,124 @@ +package com.poiji.deserialize; + +import com.poiji.bind.Poiji; +import com.poiji.deserialize.model.HugeEntity; +import com.poiji.option.PoijiOptions; +import org.apache.poi.ss.SpreadsheetVersion; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static java.util.Arrays.asList; + +/** + * For manual testing only. + */ +public class HugeTest { + + /** + * Rows: 1048575 + * Columns: 50 + * Written in 138303 ms + * src/test/resources/concurrent4.xlsx (210273 kB, 1048575 rows) read in 138001 ms + * src/test/resources/concurrent3.xlsx (210273 kB, 1048575 rows) read in 138401 ms + * src/test/resources/concurrent1.xlsx (210273 kB, 1048575 rows) read in 138672 ms + * src/test/resources/concurrent2.xlsx (210273 kB, 1048575 rows) read in 138934 ms + * Total read in 138935 ms + */ + @Test + @Ignore("Test disabled to prevent huge xlsx files writing in CI") + public void writeThenReadStream() { + + final long start = System.nanoTime(); + final int size = SpreadsheetVersion.EXCEL2007.getLastRowIndex(); + System.out.println("Rows: " + size); + System.out.println("Columns: " + new HugeEntity().totalFields); + final Stream entities1 = generateEntities(size, "1"); + final Stream entities2 = generateEntities(size, "2"); + final Stream entities3 = generateEntities(size, "3"); + final Stream entities4 = generateEntities(size, "4"); + final String name1 = "src/test/resources/concurrent1.xlsx"; + final String name2 = "src/test/resources/concurrent2.xlsx"; + final String name3 = "src/test/resources/concurrent3.xlsx"; + final String name4 = "src/test/resources/concurrent4.xlsx"; + final List writeData = asList( + new WriteData(name1, entities1), + new WriteData(name2, entities2), + new WriteData(name3, entities3), + new WriteData(name4, entities4) + ); + + final PoijiOptions options = PoijiOptions.PoijiOptionsBuilder.settings().preferNullOverDefault(true).build(); + + writeData.parallelStream() + .forEach(data -> Poiji.toExcel(new File(data.path), HugeEntity.class, data.entities, options)); + + final long written = System.nanoTime(); + System.out.println("Written in " + (written - start) / 1000000 + " ms"); + + asList(name1, name2, name3, name4) + .parallelStream() + .forEach(fileName -> load(fileName, options)); + + final long read = System.nanoTime(); + System.out.println("Total read in " + (read - written) / 1000000 + " ms"); + + } + + private Stream generateEntities(final int size, final String marker) { + final AtomicInteger index = new AtomicInteger(); + return Stream.generate(() -> generateEntity(index.getAndIncrement(), marker)) + .limit(size); + } + + public HugeEntity generateEntity(int index, final String marker) { + final HugeEntity hugeEntity = new HugeEntity(); + final Field[] declaredFields = hugeEntity.getClass().getDeclaredFields(); + for (Field declaredField : declaredFields) { + if (declaredField.getName().startsWith("field")) { + final String fieldIndex = declaredField.getName().substring("field".length()); + final String value = fieldIndex + "_" + index + "_" + marker; + try { + declaredField.set(hugeEntity, value); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + return hugeEntity; + } + + private void load(String fileName, PoijiOptions options) { + final long written = System.nanoTime(); + final File file = new File(fileName); + final long count = Poiji.fromExcelToStream(file, HugeEntity.class, options).count(); + final long size = getSize(file); + System.out.printf("%s (%d kB, %d rows) read in %d ms%n", fileName, size / 1024, count, (System.nanoTime() - written) / 1000000); + } + + private long getSize(File file) { + try { + return Files.size(file.toPath()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static class WriteData { + private final String path; + private final Stream entities; + + public WriteData(final String path, final Stream entities) { + this.path = path; + this.entities = entities; + } + } + +} diff --git a/src/test/java/com/poiji/deserialize/model/HugeEntity.java b/src/test/java/com/poiji/deserialize/model/HugeEntity.java new file mode 100644 index 0000000..904975e --- /dev/null +++ b/src/test/java/com/poiji/deserialize/model/HugeEntity.java @@ -0,0 +1,59 @@ +package com.poiji.deserialize.model; + +import com.poiji.annotation.ExcelCell; +import com.poiji.annotation.ExcelSheet; + +@ExcelSheet("test") +public class HugeEntity { +public int totalFields = 50; +@ExcelCell(0) public String field0; +@ExcelCell(1) public String field1; +@ExcelCell(2) public String field2; +@ExcelCell(3) public String field3; +@ExcelCell(4) public String field4; +@ExcelCell(5) public String field5; +@ExcelCell(6) public String field6; +@ExcelCell(7) public String field7; +@ExcelCell(8) public String field8; +@ExcelCell(9) public String field9; +@ExcelCell(10) public String field10; +@ExcelCell(11) public String field11; +@ExcelCell(12) public String field12; +@ExcelCell(13) public String field13; +@ExcelCell(14) public String field14; +@ExcelCell(15) public String field15; +@ExcelCell(16) public String field16; +@ExcelCell(17) public String field17; +@ExcelCell(18) public String field18; +@ExcelCell(19) public String field19; +@ExcelCell(20) public String field20; +@ExcelCell(21) public String field21; +@ExcelCell(22) public String field22; +@ExcelCell(23) public String field23; +@ExcelCell(24) public String field24; +@ExcelCell(25) public String field25; +@ExcelCell(26) public String field26; +@ExcelCell(27) public String field27; +@ExcelCell(28) public String field28; +@ExcelCell(29) public String field29; +@ExcelCell(30) public String field30; +@ExcelCell(31) public String field31; +@ExcelCell(32) public String field32; +@ExcelCell(33) public String field33; +@ExcelCell(34) public String field34; +@ExcelCell(35) public String field35; +@ExcelCell(36) public String field36; +@ExcelCell(37) public String field37; +@ExcelCell(38) public String field38; +@ExcelCell(39) public String field39; +@ExcelCell(40) public String field40; +@ExcelCell(41) public String field41; +@ExcelCell(42) public String field42; +@ExcelCell(43) public String field43; +@ExcelCell(44) public String field44; +@ExcelCell(45) public String field45; +@ExcelCell(46) public String field46; +@ExcelCell(47) public String field47; +@ExcelCell(48) public String field48; +@ExcelCell(49) public String field49; +}