diff --git a/src/Command/InseeImportCommand.php b/src/Command/InseeImportCommand.php index c23680d..019e6e6 100644 --- a/src/Command/InseeImportCommand.php +++ b/src/Command/InseeImportCommand.php @@ -4,13 +4,13 @@ use App\Service\InseeService; use DateTime; -use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; +use Symfony\Component\HttpKernel\KernelInterface; #[AsCommand( name: 'app:insee:import', @@ -19,8 +19,8 @@ class InseeImportCommand extends Command { public function __construct( - private readonly EntityManagerInterface $em, private readonly InseeService $inseeService, + private readonly KernelInterface $kernel, ) { parent::__construct(); } @@ -105,8 +105,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int $io->writeln("{$startTime->format('d-m-Y G:i:s')} - Start processing"); $output->writeln("Import ID is {$this->inseeService->getImportId()}"); - // Turn off Doctrine's default SQL logger to save memory - $this->em->getConnection()->getConfiguration()->setSQLLogger(); + // In debug mode Doctrine keeps every executed query in memory to feed the profiler. This + // import runs ~95 000 INSERTs, which the collector alone turns into some 290 MB that is + // never released. setSQLLogger(), used here before, is the DBAL 2 API: it is deprecated + // and the debug middleware never reads it, so it saved nothing. + if ($this->kernel->isDebug()) { + $io->warning( + 'Debug mode is on: Doctrine will keep every executed query in memory and the ' + . 'import will need roughly 290 MB more. Run it with --no-debug.' + ); + } // If purge is requested, execute it before importing if ($purge) { diff --git a/src/Service/InseeService.php b/src/Service/InseeService.php index f62bff6..27ee1be 100644 --- a/src/Service/InseeService.php +++ b/src/Service/InseeService.php @@ -14,6 +14,15 @@ class InseeService extends ImporterService { + /** + * Rows kept in the unit of work before it is flushed and detached. + * + * The INSEE files run to tens of thousands of rows (37k communes, 42k historical ones), and + * a run with --target=all chains seven of them. Holding them all until the end of the file + * made the import peak around 670 MB and get killed by the OOM killer. + */ + private const int BATCH_SIZE = 500; + private bool $verbose = false; private int $nbSuccessCommune = 0; private int $nbSuccessCommune1943 = 0; @@ -62,9 +71,13 @@ public function importData(string $filePath, string $type, string $separator = ' $this->processLine($data, $type); ++$lineCounter; - if (0 === $lineCounter % 500 && !$this->verbose) { - // Output a progress message every 500 lines - $this->output->writeln("Processed $lineCounter lines..."); + if (0 === $lineCounter % self::BATCH_SIZE) { + $this->flushBatch(); + + if (!$this->verbose) { + $memory = round(memory_get_usage(true) / 1048576); + $this->output->writeln("Processed $lineCounter lines... ({$memory} MB)"); + } } } @@ -75,11 +88,21 @@ public function importData(string $filePath, string $type, string $separator = ' return false; } - $this->em->flush(); + $this->flushBatch(); return true; } + /** + * Writes the pending rows and detaches them, so that the unit of work does not keep growing + * for the whole file. Nothing here reads back a persisted entity, so detaching them is safe. + */ + private function flushBatch(): void + { + $this->em->flush(); + $this->em->clear(); + } + /** * Process a single CSV row based on the dataset type. */