diff --git a/src/Command/RppsImport.php b/src/Command/RppsImport.php index 597d290..481b9ba 100755 --- a/src/Command/RppsImport.php +++ b/src/Command/RppsImport.php @@ -7,11 +7,13 @@ use Doctrine\DBAL\Platforms\AbstractMySQLPlatform; use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; +use RuntimeException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\DependencyInjection\Attribute\Autowire; use Throwable; #[AsCommand( @@ -34,6 +36,8 @@ class RppsImport extends Command public function __construct( protected readonly RPPSService $rppsService, protected readonly EntityManagerInterface $em, + #[Autowire('%kernel.debug%')] + private readonly bool $debug = false, private readonly ?LoggerInterface $logger = null, ) { parent::__construct(); @@ -85,12 +89,32 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->rppsService->setOutput($output); $this->rppsService->setMaxPurgeRatio((float) $input->getOption('max-purge-ratio')); + // With the debug toolbar active, doctrine-bundle keeps every executed query -- with a + // backtrace -- for the whole process. At roughly five queries per row that exhausts the + // container a few hundred thousand rows in, and the task is SIGKILLed with no usable + // error. Note the setSQLLogger(null) below does not prevent this: it is the DBAL 2 + // mechanism, and collection now happens through a middleware. APP_ENV is `staging` on + // the scheduled task, so debug is on unless it is explicitly turned off. + if ($this->debug) { + throw new RuntimeException( + 'Refusing to run with debug enabled: query collection would exhaust memory ' + . 'long before the import completes. Re-run with --no-debug.' + ); + } + if (!$this->acquireLock()) { + // Deliberately a failure, not a success. A scheduled run that silently does nothing + // is the exact outcome this command exists to avoid: ECS would record a clean exit + // and nothing would reach Sentry, so the register could go weeks without an import. + // This also covers a lock left behind by a killed task -- MySQL frees it only once + // it notices the dead connection, which can take longer than the gap between runs. $output->writeln( - 'Another RPPS import is already running, skipping this run.' + 'Another RPPS import holds the lock, so this run did nothing. If no import ' + . 'is actually running, a previous one was killed and the lock has not been ' + . "released yet: check SELECT IS_USED_LOCK('rpps_import')." ); - return Command::SUCCESS; + return Command::FAILURE; } try { diff --git a/src/Service/FileParserService.php b/src/Service/FileParserService.php index 400646d..07267f7 100755 --- a/src/Service/FileParserService.php +++ b/src/Service/FileParserService.php @@ -75,9 +75,10 @@ protected function processFile( // of the consecutive rows belonging to the same practitioner. $batchSize = 500; - // Progress every 50k rows rather than every batch: at one line per batch a full run - // emitted ~40k lines, each of which is a CloudWatch PutLogEvents call on ECS. - $progressEvery = 50000; + // Progress every 10k rows: one line per batch meant ~40k lines per run, but 50k was + // too sparse to tell a working import from a hung one -- minutes of silence either way. + // At ~600 rows/s this is a heartbeat every ~17s, and ~227 lines for a full run. + $progressEvery = 10000; $lineCount = $this->fileProcessor->getLinesCount($file);