Merge pull request #1863 from MTES-MCT/feature/1853-import-files-with…

…-signalements [Import] Ajout de la gestion des fichiers lors de l'import de signalements via csv
MTES-MCT · Nov 17, 2023 · 9911440 · 9911440
2 parents eebee3d + 1e4542d
commit 9911440
Show file tree

Hide file tree

Showing 6 changed files with 158 additions and 39 deletions.
diff --git a/scripts/upload-s3.sh b/scripts/upload-s3.sh
@@ -44,6 +44,11 @@ else
       aws s3 cp data/signalement/signalement_${zip}.csv s3://${BUCKET_URL}/csv/ ${debug}
       aws s3 ls s3://${BUCKET_URL}/csv/signalement_${zip}.csv
       ;;
+    "slugify-signalement")
+      echo "Upload signalement_$2.csv to cloud..."
+      aws s3 cp data/images/signalement_${zip}.csv s3://${BUCKET_URL}/csv/ ${debug}
+      aws s3 ls s3://${BUCKET_URL}/csv/signalement_${zip}.csv
+      ;;
     "image")
       echo "Upload image_$zip to cloud"
       aws s3 cp --recursive data/images/import_${zip} s3://${BUCKET_URL}/ ${debug}

diff --git a/src/Command/SlugifyDocumentSignalementCommand.php b/src/Command/SlugifyDocumentSignalementCommand.php
@@ -29,9 +29,17 @@
 )]
 class SlugifyDocumentSignalementCommand extends Command
 {
-    public const PREFIX_FILENAME_STORAGE = 'mapping_doc_signalement_slugged_';
+    public const PREFIX_FILENAME_STORAGE_MAPPING = 'mapping_doc_signalement_';
+    public const PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED = 'mapping_doc_signalement_slugged_';
+    public const PREFIX_FILENAME_STORAGE_SIGNALEMENT = 'signalement_';
+    public const PREFIX_FILENAME_STORAGE_SIGNALEMENT_SLUGGED = 'signalement_slugged_';
     public const BASE_DIRECTORY_CSV = 'csv/';
+    public const IMPORT_SIGNALEMENT_COLUMN_PHOTOS = 'ref des photos';
+    public const IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS = 'ref des documents';
+
     private ?Territory $territory = null;
+    private bool $isMappingFile;
+    private ?string $filename = null;
     private ?string $directoryPath = null;
     private ?string $sourceFile = null;
     private ?string $destinationFile = null;
@@ -54,6 +62,7 @@ public function __construct(
     protected function configure(): void
     {
         $this->addArgument('zip', InputArgument::REQUIRED, 'Territory zip to target');
+        $this->addArgument('mapping', InputArgument::REQUIRED, 'Is it a mapping or a list of signalements');
     }
 
     /**
@@ -78,46 +87,44 @@ public function execute(InputInterface $input, OutputInterface $output): int
         $tmpDirectory = $this->parameterBag->get('uploads_tmp_dir');
         $this->uploadHandlerService->createTmpFileFromBucket($this->sourceFile, $this->destinationFile);
 
-        if ($this->hasMissingColumnLabel($io)) {
+        if ($this->isMappingFile && $this->hasMissingColumnLabel($io)) {
             return Command::FAILURE;
         }
 
         $rows = $this->csvParser->parseAsDict($this->destinationFile);
-        $filename = self::PREFIX_FILENAME_STORAGE.$this->territory->getZip().'.csv';
+        $filename = $this->isMappingFile
+                        ? self::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.$this->territory->getZip().'.csv'
+                        : self::PREFIX_FILENAME_STORAGE_SIGNALEMENT_SLUGGED.$this->territory->getZip().'.csv';
 
         $csvWriter = new CsvWriter(
             $tmpDirectory.$filename,
             $this->csvParser->getHeaders($this->destinationFile)
         );
         $countFileSlugged = 0;
         foreach ($rows as $index => $row) {
-            $fileInfo = pathinfo($row[SignalementImportImageHeader::COLUMN_FILENAME]);
-            $extension = $fileInfo['extension'] ?? null;
-            if (null === $extension) {
-                continue;
-            }
-            $filenameSlugged = $this->slugger->slug($fileInfo['filename'])->toString();
-            if (\strlen($filenameSlugged) > 25) {
-                $filenameSlugged = substr($filenameSlugged, 0, 25);
-            }
-            $filenameSlugged = uniqid().'-'.$filenameSlugged.'.'.$extension;
-
-            try {
-                $this->filesystem->rename(
-                    $this->directoryPath.$row[SignalementImportImageHeader::COLUMN_FILENAME],
-                    $this->directoryPath.$filenameSlugged,
-                    true
-                );
-                $csvWriter->writeRow(
-                    [
-                        $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT_ATTACHMENT],
-                        $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT],
-                        $filenameSlugged,
-                    ]
-                );
-                ++$countFileSlugged;
-            } catch (\Throwable $exception) {
-                $this->logger->error(sprintf('N° %s ligne avec %s', $index, $exception->getMessage()));
+            if ($this->isMappingFile) {
+                if (!empty($row[SignalementImportImageHeader::COLUMN_FILENAME])) {
+                    $rowFilename = $row[SignalementImportImageHeader::COLUMN_FILENAME];
+                    if ($this->makeSlugForMappingFile($csvWriter, $rowFilename, $index, $row)) {
+                        ++$countFileSlugged;
+                    }
+                }
+            } else {
+                if (!empty($row[self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS])) {
+                    $row[self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS]
+                        = $this->makeSlugsForSignalementFile(self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS, $index, $row);
+                    $countFileSlugged += \count(explode('|', $row[self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS]));
+                }
+                if (!empty($row[self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS])) {
+                    $row[self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS]
+                        = $this->makeSlugsForSignalementFile(self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS, $index, $row);
+                    $countFileSlugged += \count(explode('|', $row[self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS]));
+                }
+                try {
+                    $csvWriter->writeRow($row);
+                } catch (\Throwable $exception) {
+                    $this->logger->error(sprintf('CSV Write - row %s - error: %s', $index, $exception->getMessage()));
+                }
             }
         }
 
@@ -128,7 +135,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
         $command = 'make upload action=image zip='.$this->territory->getZip();
         if (\count($file) > 1) {
             $this->uploadHandlerService->moveFromBucketTempFolder($filename, self::BASE_DIRECTORY_CSV);
-            $io->success(sprintf('%s files has been slugify', $countFileSlugged));
+            $io->success(sprintf('%s files have been slugified', $countFileSlugged));
             $io->success(
                 sprintf(
                     '%s has been pushed to S3 bucket storage, please send your images to S3 Bucket `%s`',
@@ -137,8 +144,8 @@ public function execute(InputInterface $input, OutputInterface $output): int
                 )
             );
         } else {
-            $io->warning(sprintf('%s files has been slugify', $countFileSlugged));
-            $io->warning(sprintf('%s is empty, please check if your images has been already slugged', $filename));
+            $io->warning(sprintf('%s files have been slugified', $countFileSlugged));
+            $io->warning(sprintf('%s is empty, please check if your images have been already slugged', $filename));
             $io->warning(sprintf('You should send your images to S3 Bucket with`%s`', $command));
 
             return Command::FAILURE;
@@ -147,6 +154,78 @@ public function execute(InputInterface $input, OutputInterface $output): int
         return Command::SUCCESS;
     }
 
+    private function makeSlugForMappingFile(CsvWriter $csvWriter, $filename, int $index, $row): int
+    {
+        $filenameSlugged = !empty($filename) ? $this->getSluggedFile($filename, $index) : null;
+        if (!empty($filenameSlugged)) {
+            try {
+                $csvWriter->writeRow(
+                    [
+                        $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT_ATTACHMENT],
+                        $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT],
+                        $filenameSlugged,
+                    ]
+                );
+
+                return 1;
+            } catch (\Throwable $exception) {
+                $this->logger->error(sprintf('CSV Write - N° %s ligne avec %s', $index, $exception->getMessage()));
+            }
+        }
+
+        return 0;
+    }
+
+    private function makeSlugsForSignalementFile(string $colName, $index, $row): ?string
+    {
+        $fileListSlugged = [];
+        $fileList = explode('|', $row[$colName]);
+        foreach ($fileList as $filename) {
+            $filenameSlugged = !empty($filename) ? $this->getSluggedFile($filename, $index) : null;
+            if (!empty($filenameSlugged)) {
+                $fileListSlugged[] = $filenameSlugged;
+            }
+        }
+
+        $countFileList = \count($fileList);
+        $countFileSlugged = \count($fileListSlugged);
+        if ($countFileList != $countFileSlugged) {
+            $this->logger->error(sprintf('Different count - row %s col %s - %s // %s', $index, $colName, $countFileSlugged, $countFileList));
+
+            return null;
+        }
+
+        return implode('|', $fileListSlugged);
+    }
+
+    private function getSluggedFile(string $filename, int $index): ?string
+    {
+        $fileInfo = pathinfo($filename);
+        $extension = $fileInfo['extension'] ?? null;
+        if (null === $extension) {
+            return null;
+        }
+        $filenameSlugged = $this->slugger->slug($fileInfo['filename'])->toString();
+        if (\strlen($filenameSlugged) > 25) {
+            $filenameSlugged = substr($filenameSlugged, 0, 25);
+        }
+        $filenameSlugged = uniqid().'-'.$filenameSlugged.'.'.$extension;
+
+        try {
+            $this->filesystem->rename(
+                $this->directoryPath.$filename,
+                $this->directoryPath.$filenameSlugged,
+                true
+            );
+
+            return $filenameSlugged;
+        } catch (\Throwable $exception) {
+            $this->logger->error(sprintf('File rename - row %s - error: %s', $index, $exception->getMessage()));
+        }
+
+        return null;
+    }
+
     /**
      * @throws FilesystemException
      */
@@ -166,8 +245,13 @@ private function validate(InputInterface $input, OutputInterface $output): bool
             return false;
         }
 
-        $fromFile = 'csv/mapping_doc_signalement_'.$zip.'.csv';
-        $toFile = $this->parameterBag->get('uploads_tmp_dir').'mapping_doc_signalement_'.$zip.'.csv';
+        $this->isMappingFile = '1' == $input->getArgument('mapping');
+        $this->filename = $this->isMappingFile
+                            ? self::PREFIX_FILENAME_STORAGE_MAPPING
+                            : self::PREFIX_FILENAME_STORAGE_SIGNALEMENT;
+
+        $fromFile = 'csv/'.$this->filename.$zip.'.csv';
+        $toFile = $this->parameterBag->get('uploads_tmp_dir').$this->filename.$zip.'.csv';
 
         /** @var Territory $territory */
         $territory = $this->territoryManager->findOneBy(['zip' => $zip]);
@@ -179,7 +263,7 @@ private function validate(InputInterface $input, OutputInterface $output): bool
         if ($this->filesystem->exists($directoryPath)) {
             $countFile = \count(scandir($directoryPath)) - 2; // ignore single dot (.) and double dots (..)
             $question = new ConfirmationQuestion(
-                sprintf('Do you want to slugify %s files from your directory %s ? ', $countFile, $directoryPath),
+                sprintf('Do you want to slugify %s files from your directory %s?', $countFile, $directoryPath),
                 false
             );
             if (!$helper->ask($input, $output, $question)) {

diff --git a/src/Command/UpdateSignalementDocumentFieldsCommand.php b/src/Command/UpdateSignalementDocumentFieldsCommand.php
@@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             return Command::FAILURE;
         }
 
-        $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE.$zip.'.csv';
+        $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.$zip.'.csv';
         $toFile = $this->parameterBag->get('uploads_tmp_dir').'mapping_doc_signalement_'.$zip.'.csv';
         if (!$this->fileStorage->fileExists($fromFile)) {
             $io->error('CSV Mapping file '.$fromFile.' does not exist, please execute app:slugify-doc-signalement');

diff --git a/src/Service/Import/Signalement/SignalementImportLoader.php b/src/Service/Import/Signalement/SignalementImportLoader.php
@@ -6,11 +6,13 @@
 use App\Entity\Critere;
 use App\Entity\Criticite;
 use App\Entity\Enum\MotifCloture;
+use App\Entity\File;
 use App\Entity\Partner;
 use App\Entity\Signalement;
 use App\Entity\Territory;
 use App\Entity\User;
 use App\Manager\AffectationManager;
+use App\Manager\FileManager;
 use App\Manager\SignalementManager;
 use App\Manager\SuiviManager;
 use App\Manager\TagManager;
@@ -57,6 +59,7 @@ public function __construct(
         private LoggerInterface $logger,
         private CriticiteCalculator $criticiteCalculator,
         private SignalementQualificationUpdater $signalementQualificationUpdater,
+        private FileManager $fileManager,
     ) {
     }
 
@@ -103,6 +106,9 @@ public function load(Territory $territory, array $data, array $headers, ?OutputI
                     $signalement->addSuivi($suivi);
                 }
 
+                $this->loadFiles($signalement, File::INPUT_NAME_PHOTOS, $dataMapped, File::FILE_TYPE_PHOTO);
+                $this->loadFiles($signalement, File::INPUT_NAME_DOCUMENTS, $dataMapped, File::FILE_TYPE_DOCUMENT);
+
                 $this->metadata['count_signalement'] = $countSignalement;
                 if (0 === $countSignalement % self::FLUSH_COUNT) {
                     $this->logger->info(sprintf('in progress - %s signalements saved', $countSignalement));
@@ -251,4 +257,24 @@ private function loadSuivi(Signalement $signalement, array $dataMapped): ArrayCo
 
         return $suiviCollection;
     }
+
+    private function loadFiles(Signalement $signalement, string $colName, array $dataMapped, string $fileType): void
+    {
+        if (empty($dataMapped[$colName])) {
+            return;
+        }
+
+        $fileList = explode('|', $dataMapped[$colName]);
+        foreach ($fileList as $filename) {
+            $file = $this->fileManager->createOrUpdate(
+                filename: $filename,
+                title: $filename,
+                type: $fileType,
+                signalement: $signalement,
+            );
+            unset($file);
+        }
+
+        $this->fileManager->flush();
+    }
 }
diff --git a/tests/Functional/Service/Import/Signalement/SignalementImportLoaderTest.php b/tests/Functional/Service/Import/Signalement/SignalementImportLoaderTest.php
@@ -5,6 +5,7 @@
 use App\Entity\Territory;
 use App\EventListener\SuiviCreatedListener;
 use App\Manager\AffectationManager;
+use App\Manager\FileManager;
 use App\Manager\SignalementManager;
 use App\Manager\SuiviManager;
 use App\Manager\TagManager;
@@ -32,6 +33,7 @@ class SignalementImportLoaderTest extends KernelTestCase
     private LoggerInterface $logger;
     private CriticiteCalculator $criticiteCalculator;
     private SignalementQualificationUpdater $signalementQualificationUpdater;
+    private FileManager $fileManager;
 
     protected function setUp(): void
     {
@@ -45,6 +47,7 @@ protected function setUp(): void
         $this->logger = self::getContainer()->get(LoggerInterface::class);
         $this->criticiteCalculator = self::getContainer()->get(CriticiteCalculator::class);
         $this->signalementQualificationUpdater = self::getContainer()->get(SignalementQualificationUpdater::class);
+        $this->fileManager = self::getContainer()->get(FileManager::class);
 
         $this->entityManager = $kernel->getContainer()->get('doctrine')->getManager();
     }
@@ -66,6 +69,7 @@ public function testLoadSignalementImport()
             $this->logger,
             $this->criticiteCalculator,
             $this->signalementQualificationUpdater,
+            $this->fileManager,
         );
 
         $territory = $this->entityManager->getRepository(Territory::class)->findOneBy(['zip' => '01']);

diff --git a/tests/Unit/Command/UpdateSignalementDocumentFieldsCommandTest.php b/tests/Unit/Command/UpdateSignalementDocumentFieldsCommandTest.php
@@ -76,7 +76,7 @@ public function testDisplaySuccessfullyMessage(): void
             ->method('getRepository')
             ->willReturn($signalementRepositoryMock);
 
-        $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE.'01.csv';
+        $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.'01.csv';
         $toFile = $this->parameterBag->get('uploads_tmp_dir').'mapping_doc_signalement_01.csv';
 
         $this->fileStorage->expects($this->once())
@@ -174,7 +174,7 @@ public function testDisplayFailedMessageWithMappingFileDoesNotExist(): void
             ->with(['zip' => '99'])
             ->willReturn($this->territory);
 
-        $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE.'99.csv';
+        $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.'99.csv';
 
         $this->fileStorage->expects($this->once())
             ->method('fileExists')