[Bug] - Corrige l'OOM de l'import INSEE

src/Command/InseeImportCommand.php

@@ -4,13 +4,13 @@
 
 use App\Service\InseeService;
 use DateTime;
-use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\HttpKernel\KernelInterface;
 
 #[AsCommand(
     name: 'app:insee:import',
@@ -19,8 +19,8 @@
 class InseeImportCommand extends Command
 {
     public function __construct(
-        private readonly EntityManagerInterface $em,
         private readonly InseeService $inseeService,
+        private readonly KernelInterface $kernel,
     ) {
         parent::__construct();
     }
@@ -105,8 +105,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $io->writeln("<comment>{$startTime->format('d-m-Y G:i:s')} - Start processing</comment>");
         $output->writeln("Import ID is {$this->inseeService->getImportId()}");
 
-        // Turn off Doctrine's default SQL logger to save memory
-        $this->em->getConnection()->getConfiguration()->setSQLLogger();
+        // In debug mode Doctrine keeps every executed query in memory to feed the profiler. This
+        // import runs ~95 000 INSERTs, which the collector alone turns into some 290 MB that is
+        // never released. setSQLLogger(), used here before, is the DBAL 2 API: it is deprecated
+        // and the debug middleware never reads it, so it saved nothing.
+        if ($this->kernel->isDebug()) {
+            $io->warning(
+                'Debug mode is on: Doctrine will keep every executed query in memory and the '
+                . 'import will need roughly 290 MB more. Run it with --no-debug.'
+            );
+        }
 
         // If purge is requested, execute it before importing
         if ($purge) {

src/Service/InseeService.php

@@ -14,6 +14,15 @@
 
 class InseeService extends ImporterService
 {
+    /**
+     * Rows kept in the unit of work before it is flushed and detached.
+     *
+     * The INSEE files run to tens of thousands of rows (37k communes, 42k historical ones), and
+     * a run with --target=all chains seven of them. Holding them all until the end of the file
+     * made the import peak around 670 MB and get killed by the OOM killer.
+     */
+    private const int BATCH_SIZE = 500;
+
     private bool $verbose = false;
     private int $nbSuccessCommune = 0;
     private int $nbSuccessCommune1943 = 0;
@@ -62,9 +71,13 @@ public function importData(string $filePath, string $type, string $separator = '
                 $this->processLine($data, $type);
 
                 ++$lineCounter;
-                if (0 === $lineCounter % 500 && !$this->verbose) {
-                    // Output a progress message every 500 lines
-                    $this->output->writeln("<comment>Processed $lineCounter lines...</comment>");
+                if (0 === $lineCounter % self::BATCH_SIZE) {
+                    $this->flushBatch();
+
+                    if (!$this->verbose) {
+                        $memory = round(memory_get_usage(true) / 1048576);
+                        $this->output->writeln("<comment>Processed $lineCounter lines... ({$memory} MB)</comment>");
+                    }
                 }
             }
 
@@ -75,11 +88,21 @@ public function importData(string $filePath, string $type, string $separator = '
             return false;
         }
 
-        $this->em->flush();
+        $this->flushBatch();
 
         return true;
     }
 
+    /**
+     * Writes the pending rows and detaches them, so that the unit of work does not keep growing
+     * for the whole file. Nothing here reads back a persisted entity, so detaching them is safe.
+     */
+    private function flushBatch(): void
+    {
+        $this->em->flush();
+        $this->em->clear();
+    }
+
     /**
      * Process a single CSV row based on the dataset type.
      */