diff --git a/src/Command/InseeImportCommand.php b/src/Command/InseeImportCommand.php
index c23680d..019e6e6 100644
--- a/src/Command/InseeImportCommand.php
+++ b/src/Command/InseeImportCommand.php
@@ -4,13 +4,13 @@
use App\Service\InseeService;
use DateTime;
-use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\HttpKernel\KernelInterface;
#[AsCommand(
name: 'app:insee:import',
@@ -19,8 +19,8 @@
class InseeImportCommand extends Command
{
public function __construct(
- private readonly EntityManagerInterface $em,
private readonly InseeService $inseeService,
+ private readonly KernelInterface $kernel,
) {
parent::__construct();
}
@@ -105,8 +105,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$io->writeln("{$startTime->format('d-m-Y G:i:s')} - Start processing");
$output->writeln("Import ID is {$this->inseeService->getImportId()}");
- // Turn off Doctrine's default SQL logger to save memory
- $this->em->getConnection()->getConfiguration()->setSQLLogger();
+ // In debug mode Doctrine keeps every executed query in memory to feed the profiler. This
+ // import runs ~95 000 INSERTs, which the collector alone turns into some 290 MB that is
+ // never released. setSQLLogger(), used here before, is the DBAL 2 API: it is deprecated
+ // and the debug middleware never reads it, so it saved nothing.
+ if ($this->kernel->isDebug()) {
+ $io->warning(
+ 'Debug mode is on: Doctrine will keep every executed query in memory and the '
+ . 'import will need roughly 290 MB more. Run it with --no-debug.'
+ );
+ }
// If purge is requested, execute it before importing
if ($purge) {
diff --git a/src/Service/InseeService.php b/src/Service/InseeService.php
index f62bff6..27ee1be 100644
--- a/src/Service/InseeService.php
+++ b/src/Service/InseeService.php
@@ -14,6 +14,15 @@
class InseeService extends ImporterService
{
+ /**
+ * Rows kept in the unit of work before it is flushed and detached.
+ *
+ * The INSEE files run to tens of thousands of rows (37k communes, 42k historical ones), and
+ * a run with --target=all chains seven of them. Holding them all until the end of the file
+ * made the import peak around 670 MB and get killed by the OOM killer.
+ */
+ private const int BATCH_SIZE = 500;
+
private bool $verbose = false;
private int $nbSuccessCommune = 0;
private int $nbSuccessCommune1943 = 0;
@@ -62,9 +71,13 @@ public function importData(string $filePath, string $type, string $separator = '
$this->processLine($data, $type);
++$lineCounter;
- if (0 === $lineCounter % 500 && !$this->verbose) {
- // Output a progress message every 500 lines
- $this->output->writeln("Processed $lineCounter lines...");
+ if (0 === $lineCounter % self::BATCH_SIZE) {
+ $this->flushBatch();
+
+ if (!$this->verbose) {
+ $memory = round(memory_get_usage(true) / 1048576);
+ $this->output->writeln("Processed $lineCounter lines... ({$memory} MB)");
+ }
}
}
@@ -75,11 +88,21 @@ public function importData(string $filePath, string $type, string $separator = '
return false;
}
- $this->em->flush();
+ $this->flushBatch();
return true;
}
+ /**
+ * Writes the pending rows and detaches them, so that the unit of work does not keep growing
+ * for the whole file. Nothing here reads back a persisted entity, so detaching them is safe.
+ */
+ private function flushBatch(): void
+ {
+ $this->em->flush();
+ $this->em->clear();
+ }
+
/**
* Process a single CSV row based on the dataset type.
*/