Adding JsonConverter::chunkSize

thephpleague · Oct 10, 2024 · e7276ba · e7276ba
1 parent 8cab815
commit e7276ba
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 16 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,24 @@
 
 All Notable changes to `Csv` will be documented in this file
 
+## [Next](https://github.com/thephpleague/csv/compare/9.17.0...master) - TBD
+
+### Added
+
+- `League\Csv\JsonConverter::chunkSize`
+
+### Deprecated
+
+- None
+
+### Fixed
+
+- None
+
+### Remove
+
+- None
+
 ## [9.17.0](https://github.com/thephpleague/csv/compare/9.16.0...9.17.0) - 2024-10-10
 
 ### Added

diff --git a/docs/9.0/converter/json.md b/docs/9.0/converter/json.md
@@ -110,6 +110,25 @@ structure. The resulting conversion may differ to what you expect. This callback
 specify how each item will be converted. The formatter should return a type that can be handled
 by PHP `json_encode` function.
 
+### Chunk sixze
+
+<p class="message-notice">available since version <code>9.18.0</code></p>
+
+```php
+public JsonConverter::chunkSize(int $chunkSize): self
+```
+
+This method sets the number of rows to buffer before convert into JSON string. This allow
+for faster conversion while retaining the low memory usage. Of course, the default
+chunk size can vary for one scenario to another. The correct size is therefore
+left to the user discretion. By default, the value is `500`. The value can not
+be lower than one otherwise a exception will be thrown.
+
+```php
+$converter = JsonConverter::create()->chunkSize(1_000);
+$converter->chunkSize; //returns the value used
+```
+
 ## Conversion
 
 ```php

diff --git a/src/JsonConverter.php b/src/JsonConverter.php
@@ -113,22 +113,25 @@ final class JsonConverter
     private readonly string $indentation;
     /** @var Closure(string, array-key): string */
     private readonly Closure $internalFormatter;
+    /** @var int<1, max> */
+    public readonly int $chunkSize;
 
     public static function create(): self
     {
-        return new self(flags: 0, depth: 512, indentSize: 4, formatter: null);
+        return new self(flags: 0, depth: 512, indentSize: 4, formatter: null, chunkSize: 500);
     }
 
     /**
      * @param int<1, max> $depth
      * @param int<1, max> $indentSize
      */
-    private function __construct(int $flags, int $depth, int $indentSize, ?Closure $formatter)
+    private function __construct(int $flags, int $depth, int $indentSize, ?Closure $formatter, int $chunkSize)
     {
         json_encode([], $flags & ~JSON_THROW_ON_ERROR, $depth);
 
         JSON_ERROR_NONE === json_last_error() || throw new InvalidArgumentException('The flags or the depth given are not valid JSON encoding parameters in PHP; '.json_last_error_msg());
         1 <= $indentSize || throw new InvalidArgumentException('The indentation space must be greater or equal to 1.');
+        1 <= $chunkSize || throw new InvalidArgumentException('The chunk size must be greater or equal to 1.');
 
         $this->flags = $flags;
         $this->depth = $depth;
@@ -138,6 +141,7 @@ private function __construct(int $flags, int $depth, int $indentSize, ?Closure $
         $this->isPrettyPrint = ($this->flags & JSON_PRETTY_PRINT) === JSON_PRETTY_PRINT;
         $this->isForceObject = ($this->flags & JSON_FORCE_OBJECT) === JSON_FORCE_OBJECT;
         $this->internalFormatter = $this->setInternalFormatter();
+        $this->chunkSize = $chunkSize;
     }
 
     /**
@@ -239,7 +243,7 @@ private function setFlags(int $flags): self
     {
         return match ($flags) {
             $this->flags => $this,
-            default => new self($flags, $this->depth, $this->indentSize, $this->formatter),
+            default => new self($flags, $this->depth, $this->indentSize, $this->formatter, $this->chunkSize),
         };
     }
 
@@ -252,7 +256,7 @@ public function depth(int $depth): self
     {
         return match ($depth) {
             $this->depth => $this,
-            default => new self($this->flags, $depth, $this->indentSize, $this->formatter),
+            default => new self($this->flags, $depth, $this->indentSize, $this->formatter, $this->chunkSize),
         };
     }
 
@@ -265,7 +269,20 @@ public function indentSize(int $indentSize): self
     {
         return match ($indentSize) {
             $this->indentSize => $this,
-            default => new self($this->flags, $this->depth, $indentSize, $this->formatter),
+            default => new self($this->flags, $this->depth, $indentSize, $this->formatter, $this->chunkSize),
+        };
+    }
+
+    /**
+     * Set the indentation size.
+     *
+     * @param int<1, max> $chunkSize
+     */
+    public function chunkSize(int $chunkSize): self
+    {
+        return match ($chunkSize) {
+            $this->chunkSize => $this,
+            default => new self($this->flags, $this->depth, $this->indentSize, $this->formatter, $chunkSize),
         };
     }
 
@@ -274,7 +291,7 @@ public function indentSize(int $indentSize): self
      */
     public function formatter(?Closure $formatter): self
     {
-        return new self($this->flags, $this->depth, $this->indentSize, $formatter);
+        return new self($this->flags, $this->depth, $this->indentSize, $formatter, $this->chunkSize);
     }
 
     /**
@@ -395,30 +412,49 @@ public function convert(iterable $records): Iterator
 
         yield $start;
 
+        $incr = 0;
+        $buffer = [];
         while ($records->valid()) {
-            yield $this->format($current, $offset).$separator;
+            if ($incr === $this->chunkSize) {
+                yield $this->format($buffer, $offset).$separator;
+
+                $incr = 0;
+                $buffer = [];
+            }
+            $incr++;
+            $buffer[] = $current;
 
             $offset++;
             $current = $records->current();
             $records->next();
         }
 
-        yield $this->format($current, $offset).$end;
+        $last = $this->format($buffer, $offset);
+        if ('' !== $last) {
+            yield $last.$separator;
+        }
+
+        yield $this->format([$current], $offset++).$end;
     }
 
     /**
      * @throws JsonException
      */
-    private function format(mixed $value, int|string $offset): string
+    private function format(array $value, int $offset): string
     {
-        return ($this->internalFormatter)(
-            json_encode(
-                value: ($this->formatter)($value, $offset),
-                flags: ($this->flags & ~JSON_PRETTY_PRINT) | JSON_THROW_ON_ERROR,
-                depth: $this->depth
-            ),
-            $offset
+        $data = [];
+        foreach ($value as $item) {
+            $data[] = ($this->formatter)($item, $offset);
+            ++$offset;
+        }
+
+        $json = json_encode(
+            value: $data,
+            flags: ($this->flags & ~JSON_PRETTY_PRINT) | JSON_THROW_ON_ERROR,
+            depth: $this->depth
         );
+
+        return ($this->internalFormatter)(substr($json, 1, -1), $offset);
     }
 
     /**

diff --git a/src/JsonConverterTest.php b/src/JsonConverterTest.php
@@ -66,6 +66,7 @@ public function it_has_default_values(): void
                 ->addFlags(0)
                 ->removeFlags(0)
                 ->depth(512)
+                ->chunkSize(500)
         );
     }
 
@@ -85,6 +86,14 @@ public function it_fails_if_the_indentation_size_is_invalud(): void
         JsonConverter::create()->indentSize(0); /* @phpstan-ignore-line */
     }
 
+    #[Test]
+    public function it_fails_if_the_chunk_size_is_invalud(): void
+    {
+        $this->expectException(InvalidArgumentException::class);
+
+        JsonConverter::create()->chunkSize(0); /* @phpstan-ignore-line */
+    }
+
     #[Test]
     public function it_only_uses_indentation_if_pretty_print_is_present(): void
     {