From 85d1631c14ab388a32edd9a1b0fd95558cc08bc3 Mon Sep 17 00:00:00 2001 From: emilschn Date: Thu, 2 Nov 2023 21:00:30 +0100 Subject: [PATCH 1/3] update command to slugify files in import signalement files --- scripts/upload-s3.sh | 5 + .../SlugifyDocumentSignalementCommand.php | 156 ++++++++++++++---- ...UpdateSignalementDocumentFieldsCommand.php | 2 +- ...teSignalementDocumentFieldsCommandTest.php | 4 +- 4 files changed, 128 insertions(+), 39 deletions(-) diff --git a/scripts/upload-s3.sh b/scripts/upload-s3.sh index bfb6f604f..6f9be77e2 100755 --- a/scripts/upload-s3.sh +++ b/scripts/upload-s3.sh @@ -44,6 +44,11 @@ else aws s3 cp data/signalement/signalement_${zip}.csv s3://${BUCKET_URL}/csv/ ${debug} aws s3 ls s3://${BUCKET_URL}/csv/signalement_${zip}.csv ;; + "slugify-signalement") + echo "Upload signalement_$2.csv to cloud..." + aws s3 cp data/images/signalement_${zip}.csv s3://${BUCKET_URL}/csv/ ${debug} + aws s3 ls s3://${BUCKET_URL}/csv/signalement_${zip}.csv + ;; "image") echo "Upload image_$zip to cloud" aws s3 cp --recursive data/images/import_${zip} s3://${BUCKET_URL}/ ${debug} diff --git a/src/Command/SlugifyDocumentSignalementCommand.php b/src/Command/SlugifyDocumentSignalementCommand.php index 664237d42..e59473ba2 100644 --- a/src/Command/SlugifyDocumentSignalementCommand.php +++ b/src/Command/SlugifyDocumentSignalementCommand.php @@ -29,9 +29,17 @@ )] class SlugifyDocumentSignalementCommand extends Command { - public const PREFIX_FILENAME_STORAGE = 'mapping_doc_signalement_slugged_'; + public const PREFIX_FILENAME_STORAGE_MAPPING = 'mapping_doc_signalement_'; + public const PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED = 'mapping_doc_signalement_slugged_'; + public const PREFIX_FILENAME_STORAGE_SIGNALEMENT = 'signalement_'; + public const PREFIX_FILENAME_STORAGE_SIGNALEMENT_SLUGGED = 'signalement_slugged_'; public const BASE_DIRECTORY_CSV = 'csv/'; + public const IMPORT_SIGNALEMENT_COLUMN_PHOTOS = 'ref des photos'; + public const IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS = 'ref des documents'; + private ?Territory $territory = null; + private bool $isMappingFile; + private ?string $filename = null; private ?string $directoryPath = null; private ?string $sourceFile = null; private ?string $destinationFile = null; @@ -54,6 +62,7 @@ public function __construct( protected function configure(): void { $this->addArgument('zip', InputArgument::REQUIRED, 'Territory zip to target'); + $this->addArgument('mapping', InputArgument::REQUIRED, 'Is it a mapping or a list of signalements'); } /** @@ -78,12 +87,14 @@ public function execute(InputInterface $input, OutputInterface $output): int $tmpDirectory = $this->parameterBag->get('uploads_tmp_dir'); $this->uploadHandlerService->createTmpFileFromBucket($this->sourceFile, $this->destinationFile); - if ($this->hasMissingColumnLabel($io)) { + if ($this->isMappingFile && $this->hasMissingColumnLabel($io)) { return Command::FAILURE; } $rows = $this->csvParser->parseAsDict($this->destinationFile); - $filename = self::PREFIX_FILENAME_STORAGE.$this->territory->getZip().'.csv'; + $filename = $this->isMappingFile + ? self::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.$this->territory->getZip().'.csv' + : self::PREFIX_FILENAME_STORAGE_SIGNALEMENT_SLUGGED.$this->territory->getZip().'.csv'; $csvWriter = new CsvWriter( $tmpDirectory.$filename, @@ -91,33 +102,29 @@ public function execute(InputInterface $input, OutputInterface $output): int ); $countFileSlugged = 0; foreach ($rows as $index => $row) { - $fileInfo = pathinfo($row[SignalementImportImageHeader::COLUMN_FILENAME]); - $extension = $fileInfo['extension'] ?? null; - if (null === $extension) { - continue; - } - $filenameSlugged = $this->slugger->slug($fileInfo['filename'])->toString(); - if (\strlen($filenameSlugged) > 25) { - $filenameSlugged = substr($filenameSlugged, 0, 25); - } - $filenameSlugged = uniqid().'-'.$filenameSlugged.'.'.$extension; - - try { - $this->filesystem->rename( - $this->directoryPath.$row[SignalementImportImageHeader::COLUMN_FILENAME], - $this->directoryPath.$filenameSlugged, - true - ); - $csvWriter->writeRow( - [ - $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT_ATTACHMENT], - $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT], - $filenameSlugged, - ] - ); - ++$countFileSlugged; - } catch (\Throwable $exception) { - $this->logger->error(sprintf('N° %s ligne avec %s', $index, $exception->getMessage())); + if ($this->isMappingFile) { + if (!empty($row[SignalementImportImageHeader::COLUMN_FILENAME])) { + $rowFilename = $row[SignalementImportImageHeader::COLUMN_FILENAME]; + if ($this->makeSlugForMappingFile($csvWriter, $rowFilename, $index, $row)) { + ++$countFileSlugged; + } + } + } else { + if (!empty($row[self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS])) { + $row[self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS] + = $this->makeSlugsForSignalementFile(self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS, $index, $row); + $countFileSlugged += \count(explode('|', $row[self::IMPORT_SIGNALEMENT_COLUMN_PHOTOS])); + } + if (!empty($row[self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS])) { + $row[self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS] + = $this->makeSlugsForSignalementFile(self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS, $index, $row); + $countFileSlugged += \count(explode('|', $row[self::IMPORT_SIGNALEMENT_COLUMN_DOCUMENTS])); + } + try { + $csvWriter->writeRow($row); + } catch (\Throwable $exception) { + $this->logger->error(sprintf('CSV Write - row %s - error: %s', $index, $exception->getMessage())); + } } } @@ -128,7 +135,7 @@ public function execute(InputInterface $input, OutputInterface $output): int $command = 'make upload action=image zip='.$this->territory->getZip(); if (\count($file) > 1) { $this->uploadHandlerService->moveFromBucketTempFolder($filename, self::BASE_DIRECTORY_CSV); - $io->success(sprintf('%s files has been slugify', $countFileSlugged)); + $io->success(sprintf('%s files have been slugified', $countFileSlugged)); $io->success( sprintf( '%s has been pushed to S3 bucket storage, please send your images to S3 Bucket `%s`', @@ -137,8 +144,8 @@ public function execute(InputInterface $input, OutputInterface $output): int ) ); } else { - $io->warning(sprintf('%s files has been slugify', $countFileSlugged)); - $io->warning(sprintf('%s is empty, please check if your images has been already slugged', $filename)); + $io->warning(sprintf('%s files have been slugified', $countFileSlugged)); + $io->warning(sprintf('%s is empty, please check if your images have been already slugged', $filename)); $io->warning(sprintf('You should send your images to S3 Bucket with`%s`', $command)); return Command::FAILURE; @@ -147,6 +154,78 @@ public function execute(InputInterface $input, OutputInterface $output): int return Command::SUCCESS; } + private function makeSlugForMappingFile(CsvWriter $csvWriter, $filename, int $index, $row): int + { + $filenameSlugged = !empty($filename) ? $this->getSluggedFile($filename, $index) : null; + if (!empty($filenameSlugged)) { + try { + $csvWriter->writeRow( + [ + $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT_ATTACHMENT], + $row[SignalementImportImageHeader::COLUMN_ID_ENREGISTREMENT], + $filenameSlugged, + ] + ); + + return 1; + } catch (\Throwable $exception) { + $this->logger->error(sprintf('CSV Write - N° %s ligne avec %s', $index, $exception->getMessage())); + } + } + + return 0; + } + + private function makeSlugsForSignalementFile(string $colName, $index, $row): ?string + { + $fileListSlugged = []; + $fileList = explode('|', $row[$colName]); + foreach ($fileList as $filename) { + $filenameSlugged = !empty($filename) ? $this->getSluggedFile($filename, $index) : null; + if (!empty($filenameSlugged)) { + $fileListSlugged[] = $filenameSlugged; + } + } + + $countFileList = \count($fileList); + $countFileSlugged = \count($fileListSlugged); + if ($countFileList != $countFileSlugged) { + $this->logger->error(sprintf('Different count - row %s col %s - %s // %s', $index, $colName, $countFileSlugged, $countFileList)); + + return null; + } + + return implode('|', $fileListSlugged); + } + + private function getSluggedFile(string $filename, int $index): ?string + { + $fileInfo = pathinfo($filename); + $extension = $fileInfo['extension'] ?? null; + if (null === $extension) { + return null; + } + $filenameSlugged = $this->slugger->slug($fileInfo['filename'])->toString(); + if (\strlen($filenameSlugged) > 25) { + $filenameSlugged = substr($filenameSlugged, 0, 25); + } + $filenameSlugged = uniqid().'-'.$filenameSlugged.'.'.$extension; + + try { + $this->filesystem->rename( + $this->directoryPath.$filename, + $this->directoryPath.$filenameSlugged, + true + ); + + return $filenameSlugged; + } catch (\Throwable $exception) { + $this->logger->error(sprintf('File rename - row %s - error: %s', $index, $exception->getMessage())); + } + + return null; + } + /** * @throws FilesystemException */ @@ -166,8 +245,13 @@ private function validate(InputInterface $input, OutputInterface $output): bool return false; } - $fromFile = 'csv/mapping_doc_signalement_'.$zip.'.csv'; - $toFile = $this->parameterBag->get('uploads_tmp_dir').'mapping_doc_signalement_'.$zip.'.csv'; + $this->isMappingFile = '1' == $input->getArgument('mapping'); + $this->filename = $this->isMappingFile + ? self::PREFIX_FILENAME_STORAGE_MAPPING + : self::PREFIX_FILENAME_STORAGE_SIGNALEMENT; + + $fromFile = 'csv/'.$this->filename.$zip.'.csv'; + $toFile = $this->parameterBag->get('uploads_tmp_dir').$this->filename.$zip.'.csv'; /** @var Territory $territory */ $territory = $this->territoryManager->findOneBy(['zip' => $zip]); @@ -179,7 +263,7 @@ private function validate(InputInterface $input, OutputInterface $output): bool if ($this->filesystem->exists($directoryPath)) { $countFile = \count(scandir($directoryPath)) - 2; // ignore single dot (.) and double dots (..) $question = new ConfirmationQuestion( - sprintf('Do you want to slugify %s files from your directory %s ? ', $countFile, $directoryPath), + sprintf('Do you want to slugify %s files from your directory %s?', $countFile, $directoryPath), false ); if (!$helper->ask($input, $output, $question)) { diff --git a/src/Command/UpdateSignalementDocumentFieldsCommand.php b/src/Command/UpdateSignalementDocumentFieldsCommand.php index 9a36f6a84..afa498fd9 100644 --- a/src/Command/UpdateSignalementDocumentFieldsCommand.php +++ b/src/Command/UpdateSignalementDocumentFieldsCommand.php @@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::FAILURE; } - $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE.$zip.'.csv'; + $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.$zip.'.csv'; $toFile = $this->parameterBag->get('uploads_tmp_dir').'mapping_doc_signalement_'.$zip.'.csv'; if (!$this->fileStorage->fileExists($fromFile)) { $io->error('CSV Mapping file '.$fromFile.' does not exist, please execute app:slugify-doc-signalement'); diff --git a/tests/Unit/Command/UpdateSignalementDocumentFieldsCommandTest.php b/tests/Unit/Command/UpdateSignalementDocumentFieldsCommandTest.php index 646879bdd..2a7b353fe 100644 --- a/tests/Unit/Command/UpdateSignalementDocumentFieldsCommandTest.php +++ b/tests/Unit/Command/UpdateSignalementDocumentFieldsCommandTest.php @@ -76,7 +76,7 @@ public function testDisplaySuccessfullyMessage(): void ->method('getRepository') ->willReturn($signalementRepositoryMock); - $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE.'01.csv'; + $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.'01.csv'; $toFile = $this->parameterBag->get('uploads_tmp_dir').'mapping_doc_signalement_01.csv'; $this->fileStorage->expects($this->once()) @@ -174,7 +174,7 @@ public function testDisplayFailedMessageWithMappingFileDoesNotExist(): void ->with(['zip' => '99']) ->willReturn($this->territory); - $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE.'99.csv'; + $fromFile = 'csv/'.SlugifyDocumentSignalementCommand::PREFIX_FILENAME_STORAGE_MAPPING_SLUGGED.'99.csv'; $this->fileStorage->expects($this->once()) ->method('fileExists') From 34c44efe6612c052ea6b22d1eccf13af2a31ed1b Mon Sep 17 00:00:00 2001 From: emilschn Date: Thu, 2 Nov 2023 21:35:47 +0100 Subject: [PATCH 2/3] add file manager to import signalements --- .../Signalement/SignalementImportLoader.php | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Service/Import/Signalement/SignalementImportLoader.php b/src/Service/Import/Signalement/SignalementImportLoader.php index 300f6b83e..3ec7eac2a 100644 --- a/src/Service/Import/Signalement/SignalementImportLoader.php +++ b/src/Service/Import/Signalement/SignalementImportLoader.php @@ -6,11 +6,13 @@ use App\Entity\Critere; use App\Entity\Criticite; use App\Entity\Enum\MotifCloture; +use App\Entity\File; use App\Entity\Partner; use App\Entity\Signalement; use App\Entity\Territory; use App\Entity\User; use App\Manager\AffectationManager; +use App\Manager\FileManager; use App\Manager\SignalementManager; use App\Manager\SuiviManager; use App\Manager\TagManager; @@ -57,6 +59,7 @@ public function __construct( private LoggerInterface $logger, private CriticiteCalculator $criticiteCalculator, private SignalementQualificationUpdater $signalementQualificationUpdater, + private FileManager $fileManager, ) { } @@ -103,6 +106,9 @@ public function load(Territory $territory, array $data, array $headers, ?OutputI $signalement->addSuivi($suivi); } + $this->loadFiles($signalement, File::INPUT_NAME_PHOTOS, $dataMapped, File::FILE_TYPE_PHOTO); + $this->loadFiles($signalement, File::INPUT_NAME_DOCUMENTS, $dataMapped, File::FILE_TYPE_DOCUMENT); + $this->metadata['count_signalement'] = $countSignalement; if (0 === $countSignalement % self::FLUSH_COUNT) { $this->logger->info(sprintf('in progress - %s signalements saved', $countSignalement)); @@ -251,4 +257,24 @@ private function loadSuivi(Signalement $signalement, array $dataMapped): ArrayCo return $suiviCollection; } + + private function loadFiles(Signalement $signalement, string $colName, array $dataMapped, string $fileType): void + { + if (empty($dataMapped[$colName])) { + return; + } + + $fileList = explode('|', $dataMapped[$colName]); + foreach ($fileList as $filename) { + $file = $this->fileManager->createOrUpdate( + filename: $filename, + title: $filename, + type: $fileType, + signalement: $signalement, + ); + unset($file); + } + + $this->fileManager->flush(); + } } From 1e4542d797ffe5b96a95003d7a23142e2850783e Mon Sep 17 00:00:00 2001 From: emilschn Date: Thu, 2 Nov 2023 21:49:58 +0100 Subject: [PATCH 3/3] add test file --- .../Import/Signalement/SignalementImportLoaderTest.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/Functional/Service/Import/Signalement/SignalementImportLoaderTest.php b/tests/Functional/Service/Import/Signalement/SignalementImportLoaderTest.php index 6aaef2dac..9d2cbeac3 100644 --- a/tests/Functional/Service/Import/Signalement/SignalementImportLoaderTest.php +++ b/tests/Functional/Service/Import/Signalement/SignalementImportLoaderTest.php @@ -5,6 +5,7 @@ use App\Entity\Territory; use App\EventListener\SuiviCreatedListener; use App\Manager\AffectationManager; +use App\Manager\FileManager; use App\Manager\SignalementManager; use App\Manager\SuiviManager; use App\Manager\TagManager; @@ -32,6 +33,7 @@ class SignalementImportLoaderTest extends KernelTestCase private LoggerInterface $logger; private CriticiteCalculator $criticiteCalculator; private SignalementQualificationUpdater $signalementQualificationUpdater; + private FileManager $fileManager; protected function setUp(): void { @@ -45,6 +47,7 @@ protected function setUp(): void $this->logger = self::getContainer()->get(LoggerInterface::class); $this->criticiteCalculator = self::getContainer()->get(CriticiteCalculator::class); $this->signalementQualificationUpdater = self::getContainer()->get(SignalementQualificationUpdater::class); + $this->fileManager = self::getContainer()->get(FileManager::class); $this->entityManager = $kernel->getContainer()->get('doctrine')->getManager(); } @@ -66,6 +69,7 @@ public function testLoadSignalementImport() $this->logger, $this->criticiteCalculator, $this->signalementQualificationUpdater, + $this->fileManager, ); $territory = $this->entityManager->getRepository(Territory::class)->findOneBy(['zip' => '01']);