Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions src/Command/RppsImport.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@
use Doctrine\DBAL\Platforms\AbstractMySQLPlatform;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use RuntimeException;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Throwable;

#[AsCommand(
Expand All @@ -34,6 +36,8 @@ class RppsImport extends Command
public function __construct(
protected readonly RPPSService $rppsService,
protected readonly EntityManagerInterface $em,
#[Autowire('%kernel.debug%')]
private readonly bool $debug = false,
private readonly ?LoggerInterface $logger = null,
) {
parent::__construct();
Expand Down Expand Up @@ -85,12 +89,32 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$this->rppsService->setOutput($output);
$this->rppsService->setMaxPurgeRatio((float) $input->getOption('max-purge-ratio'));

// With the debug toolbar active, doctrine-bundle keeps every executed query -- with a
// backtrace -- for the whole process. At roughly five queries per row that exhausts the
// container a few hundred thousand rows in, and the task is SIGKILLed with no usable
// error. Note the setSQLLogger(null) below does not prevent this: it is the DBAL 2
// mechanism, and collection now happens through a middleware. APP_ENV is `staging` on
// the scheduled task, so debug is on unless it is explicitly turned off.
if ($this->debug) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Pass --no-debug from development import targets

In the default development Docker flow, Makefile.import:22 invokes this command without --no-debug, while .env selects APP_ENV=dev, making %kernel.debug% true. This guard therefore always throws when running make import-rpps, which also makes the existing make import-data and make install workflows fail before importing RPPS data. Update that Make target to pass --no-debug, as the production targets already do, or otherwise avoid rejecting this supported entry point.

Useful? React with 👍 / 👎.

throw new RuntimeException(
'Refusing to run with debug enabled: query collection would exhaust memory '
. 'long before the import completes. Re-run with --no-debug.'
);
}

if (!$this->acquireLock()) {
// Deliberately a failure, not a success. A scheduled run that silently does nothing
// is the exact outcome this command exists to avoid: ECS would record a clean exit
// and nothing would reach Sentry, so the register could go weeks without an import.
// This also covers a lock left behind by a killed task -- MySQL frees it only once
// it notices the dead connection, which can take longer than the gap between runs.
$output->writeln(
'<comment>Another RPPS import is already running, skipping this run.</comment>'
'<error>Another RPPS import holds the lock, so this run did nothing. If no import '
. 'is actually running, a previous one was killed and the lock has not been '
. "released yet: check SELECT IS_USED_LOCK('rpps_import').</error>"
);

return Command::SUCCESS;
return Command::FAILURE;
}

try {
Expand Down
7 changes: 4 additions & 3 deletions src/Service/FileParserService.php
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ protected function processFile(
// of the consecutive rows belonging to the same practitioner.
$batchSize = 500;

// Progress every 50k rows rather than every batch: at one line per batch a full run
// emitted ~40k lines, each of which is a CloudWatch PutLogEvents call on ECS.
$progressEvery = 50000;
// Progress every 10k rows: one line per batch meant ~40k lines per run, but 50k was
// too sparse to tell a working import from a hung one -- minutes of silence either way.
// At ~600 rows/s this is a heartbeat every ~17s, and ~227 lines for a full run.
$progressEvery = 10000;

$lineCount = $this->fileProcessor->getLinesCount($file);

Expand Down