Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions apps/core/src/modules/process/manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,62 @@ describe('ProcessManager', () => {
});
});

describe('Force Crash Handling', () => {
it('should bypass graceful shutdown and terminate immediately when forceCrash is true', async () => {
await processManager.start();

pushToStream(stdoutController, 'Authenticated with cfx.re Nucleus\n');
await Bun.sleep(15);

const stopPromise = processManager.stop({ forceCrash: true });

// Resolve the process exit promise
if (currentTriggerExit) currentTriggerExit(0);
await stopPromise;

// Should skip the graceful 'stopping' phase entirely
expect(processManager.getState().status).not.toBe('stopping');
expect(stoppingServerSpy).not.toHaveBeenCalled();
expect(txEmitSpy).not.toHaveBeenCalledWith(
'serverShuttingDown',
expect.anything(),
);

// Should inject the red stderr termination message instead of the standard stdout one
expect(mockBufferPush).toHaveBeenCalledWith(
expect.objectContaining({
source: 'stderr',
line: expect.stringContaining('fxServer process was forcefully terminated'),
}),
);
});

it('should automatically trigger a force crash if fxManager resource missing error is parsed from stdout', async () => {
await processManager.start();

const stopSpy = spyOn(processManager, 'stop');

// Push the critical error to the standard output stream
pushToStream(stdoutController, "Couldn't find resource fxManager\n");
await Bun.sleep(15); // Allow stream processing to catch up

// Verify the custom internal error was injected
expect(mockBufferPush).toHaveBeenCalledWith(
expect.objectContaining({
line: expect.stringContaining('The server can not run without the fxManager resource.'),
}),
);

// Verify the stream parser invoked stop() with forceCrash
expect(stopSpy).toHaveBeenCalledWith(
expect.objectContaining({
forceCrash: true,
message: expect.stringContaining('can not function properly without the fxManager resource'),
}),
);
});
});

describe('restart()', () => {
it('should cleanly chain sequential termination and initialization procedures smoothly', async () => {
await processManager.start();
Expand Down
69 changes: 47 additions & 22 deletions apps/core/src/modules/process/manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const KILL_GRACE_MS = 5_000;
const SHUTDOWN_EVENT_GRACE_MS = 10_000;
const CONSOLE_FLUSH_MS = 50;

type ShutdownOpts = { author?: string; message?: string };
type ShutdownOpts = { author?: string; message?: string; forceCrash?: boolean };
type RawOutputLine = Omit<ProcessOutputLine, 'seq'>;

export class ProcessManager {
Expand Down Expand Up @@ -148,38 +148,48 @@ export class ProcessManager {

const proc = this.proc;
const wasRunning = this.state.status === 'running';
const isForceCrash = opts?.forceCrash === true;

console.log(`[core] Stopping fxServer`);
this.setState('stopping');
this.clearStartupWatchdog();
if (isForceCrash) {
console.log(`[core] Force crashing fxServer due to unmet requirements`);
this.clearStartupWatchdog();
} else {
console.log(`[core] Stopping fxServer`);
this.setState('stopping');
this.clearStartupWatchdog();

this.injectConsoleLine({
payload: {
line:
'\n' +
'\x1b[1m\x1b[31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n' +
'\x1b[1m\x1b[33m 🛑 fxManager is stopping the server... \x1b[0m\n' +
'\x1b[1m\x1b[31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n\n',
ts: Date.now(),
source: 'stdout',
} satisfies RawOutputLine,
});
this.injectConsoleLine({
payload: {
line:
'\n' +
'\x1b[1m\x1b[31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n' +
'\x1b[1m\x1b[33m 🛑 fxManager is stopping the server... \x1b[0m\n' +
'\x1b[1m\x1b[31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n\n',
ts: Date.now(),
source: 'stdout',
} satisfies RawOutputLine,
});

if (wasRunning) await this.announceShutdown(opts);
if (wasRunning) await this.announceShutdown(opts);
}

await this.terminate(proc);

console.log(`[core] fxServer has stopped`);

this.injectConsoleLine({
payload: {
line:
'\n' +
'\x1b[2m\x1b[37m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n' +
'\x1b[2m\x1b[37m ⚪ fxServer has been stopped. \x1b[0m\n' +
'\x1b[2m\x1b[37m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n\n',
line: isForceCrash
? '\n' +
'\x1b[2m\x1b[31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n' +
'\x1b[2m\x1b[31m ⚫ fxServer process was forcefully terminated. \x1b[0m\n' +
'\x1b[2m\x1b[31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n\n'
: '\n' +
'\x1b[2m\x1b[37m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n' +
'\x1b[2m\x1b[37m ⚪ fxServer has been stopped. \x1b[0m\n' +
'\x1b[2m\x1b[37m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\x1b[0m\n\n',
ts: Date.now(),
source: 'stdout',
source: isForceCrash ? 'stderr' : 'stdout',
} satisfies RawOutputLine,
});

Expand Down Expand Up @@ -461,6 +471,7 @@ export class ProcessManager {
try {
while (true) {
const { done, value } = await reader.read();
let forceCrash = false;
if (done) break;

// skip the fxserver empty prompt
Expand All @@ -479,11 +490,25 @@ export class ProcessManager {
} else {
this.armStartupWatchdog();
}

if (value.includes("Couldn't find resource fxManager")) {
this.injectConsoleLine({
process: 'fxManager',
value: 'The server can not run without the fxManager resource.',
color: '\x1b[31m',
});
forceCrash = true;
}
}

console.log(value);

this.emitLine(event);

if (forceCrash) {
this.stop({ forceCrash: true, message: 'The panel can not function properly without the fxManager resource.' });
return;
}
}
} catch (err) {
console.error(`Stream error:`, err);
Expand Down
Loading