From 82476b5619d49e007ac00198acc039d530845d76 Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Tue, 4 Aug 2026 13:12:54 -0400 Subject: [PATCH 1/9] #4291 move toc detection to first page user input --- .../src/controllers/rules.controllers.ts | 5 +- src/backend/src/routes/rules.routes.ts | 1 + src/backend/src/services/rules.services.ts | 6 +- src/backend/src/utils/parse.utils.ts | 69 ++++++++------ src/frontend/src/apis/rules.api.ts | 3 +- src/frontend/src/hooks/rules.hooks.ts | 1 + .../src/pages/RulesPage/RulesetPage.tsx | 11 ++- .../RulesPage/components/AddNewFileModal.tsx | 90 ++++++++++++++++++- 8 files changed, 146 insertions(+), 40 deletions(-) diff --git a/src/backend/src/controllers/rules.controllers.ts b/src/backend/src/controllers/rules.controllers.ts index d3e75ca083..af182a9048 100644 --- a/src/backend/src/controllers/rules.controllers.ts +++ b/src/backend/src/controllers/rules.controllers.ts @@ -338,7 +338,7 @@ export default class RulesController { static async parseRuleset(req: Request, res: Response, next: NextFunction) { try { - const { fileId, parserType } = req.body; + const { fileId, parserType, firstRulePage } = req.body; const { rulesetId } = req.params as Record; const parseResult = await RulesService.parseRuleset( @@ -346,7 +346,8 @@ export default class RulesController { req.organization.organizationId, fileId, rulesetId, - parserType + parserType, + firstRulePage ); res.status(200).json(parseResult); diff --git a/src/backend/src/routes/rules.routes.ts b/src/backend/src/routes/rules.routes.ts index 7627316e7f..3b9c17c21c 100644 --- a/src/backend/src/routes/rules.routes.ts +++ b/src/backend/src/routes/rules.routes.ts @@ -120,6 +120,7 @@ rulesRouter.post( '/ruleset/:rulesetId/parse', nonEmptyString(body('fileId')), nonEmptyString(body('parserType')), // 'FSAE' or 'FHE' + body('firstRulePage').optional().isInt({ min: 1 }), validateInputs, RulesController.parseRuleset ); diff --git a/src/backend/src/services/rules.services.ts b/src/backend/src/services/rules.services.ts index 5070b11f94..26b63c80ec 100644 --- a/src/backend/src/services/rules.services.ts +++ b/src/backend/src/services/rules.services.ts @@ -1554,6 +1554,7 @@ export default class RulesService { * @param fileId google drive file id of the ruleset pdf * @param rulesetId id of the ruleset to save the parsed rules into * @param parserType type of parser to use (FSAE or FHE) + * @param firstRulePage 1-indexed page rules start on; pages before this are skipped instead of parsed * @returns array of saved rules with parent relationships established * @throws AccessDeniedException if user lacks permissions or ruleset belongs to another organization * @throws NotFoundException if ruleset doesn't exist @@ -1566,7 +1567,8 @@ export default class RulesService { organizationId: string, fileId: string, rulesetId: string, - parserType: 'FSAE' | 'FHE' + parserType: 'FSAE' | 'FHE', + firstRulePage?: number ): Promise { if (!(await userHasPermission(user.userId, organizationId, isLeadership))) { throw new AccessDeniedException('You do not have permissions to upload and parse rulesets'); @@ -1604,7 +1606,7 @@ export default class RulesService { } let parsedRules: ParsedRule[]; try { - parsedRules = await parseRulesFromPdf(buffer, parserType); + parsedRules = await parseRulesFromPdf(buffer, parserType, firstRulePage); if (parsedRules.length === 0) { throw new HttpException(400, 'No rules found in provided file'); } diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index 8da38dc61f..f17521edde 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -6,12 +6,33 @@ export interface ParsedRule { parentRuleCode?: string; } -export const parseRulesFromPdf = async (buffer: Buffer, parserType: 'FSAE' | 'FHE'): Promise => { +const defaultPageRender = pdf.DEFAULT_OPTIONS.pagerender!; + +/** + * Skip text extraction for pages before firstRulePage. + * Useful for skipping TOC and other beginning content. + * + * @param firstRulePage page number to start parsing rules from (1-indexed) + */ +const makePageRenderer = (firstRulePage?: number) => { + if (!firstRulePage || firstRulePage <= 1) return defaultPageRender; + return (pageData: { pageNumber: number }) => { + if (pageData.pageNumber < firstRulePage) return ''; + return defaultPageRender(pageData); + }; +}; + +export const parseRulesFromPdf = async ( + buffer: Buffer, + parserType: 'FSAE' | 'FHE', + firstRulePage?: number +): Promise => { const options = { // max page number to parse, 0 = all pages max: 0, // errors: 0, warnings: 1, infos: 5 - verbosityLevel: 0 as const + verbosityLevel: 0 as const, + pagerender: makePageRenderer(firstRulePage) }; const pdfData = await pdf(buffer, options); @@ -164,11 +185,6 @@ const parseFSAERules = (text: string): ParsedRule[] => { continue; } - // Skip table of contents - if (/\.{4,}\s+\d+\s*$/.test(trimmedLine)) { - continue; - } - // Check if this line starts a new rule const rule = parseRuleNumberFSAE(trimmedLine); if (rule) { @@ -268,7 +284,6 @@ const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { const parseFHERules = (text: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); - let inRulesSection = false; let currentRule: { code: string; text: string } | null = null; const saveCurrentRule = () => { @@ -280,29 +295,25 @@ const parseFHERules = (text: string): ParsedRule[] => { for (const line of lines) { const trimmedLine = line.trim(); if (!trimmedLine) continue; - if (/^Index of Tables/i.test(trimmedLine)) { - inRulesSection = true; + + // Skip repeated running header (e.g. "2025 Formula Hybrid + Electric Rules") + if (/^\d{4}\s+Formula Hybrid.*Rules/i.test(trimmedLine)) { + saveCurrentRule(); + currentRule = null; + continue; } - // Skip table of contents - if (inRulesSection) { - if (/^2025 Formula Hybrid.*Rules/i.test(trimmedLine)) { - saveCurrentRule(); - currentRule = null; - continue; - } - // Check if this line starts a new rule - const rule = parseRuleNumberFHE(trimmedLine); - if (rule) { - saveCurrentRule(); - currentRule = { - code: rule.ruleCode, - text: rule.ruleContent - }; - } else if (currentRule) { - // Append to existing rule - currentRule.text += ' ' + trimmedLine; - } + // Check if this line starts a new rule + const rule = parseRuleNumberFHE(trimmedLine); + if (rule) { + saveCurrentRule(); + currentRule = { + code: rule.ruleCode, + text: rule.ruleContent + }; + } else if (currentRule) { + // Append to existing rule + currentRule.text += ' ' + trimmedLine; } } saveCurrentRule(); diff --git a/src/frontend/src/apis/rules.api.ts b/src/frontend/src/apis/rules.api.ts index e2df230649..bdf0cbd61c 100644 --- a/src/frontend/src/apis/rules.api.ts +++ b/src/frontend/src/apis/rules.api.ts @@ -234,7 +234,8 @@ export const createRuleset = (payload: CreateRulesetPayload) => { export const parseRuleset = (payload: ParseRulesetPayload) => { return axios.post(apiUrls.parseRuleset(payload.rulesetId), { fileId: payload.fileId, - parserType: payload.parserType + parserType: payload.parserType, + firstRulePage: payload.firstRulePage }); }; diff --git a/src/frontend/src/hooks/rules.hooks.ts b/src/frontend/src/hooks/rules.hooks.ts index 8e3cddf799..ac65d749b9 100644 --- a/src/frontend/src/hooks/rules.hooks.ts +++ b/src/frontend/src/hooks/rules.hooks.ts @@ -131,6 +131,7 @@ export interface ParseRulesetPayload { rulesetId: string; fileId: string; parserType: 'FSAE' | 'FHE'; + firstRulePage?: number; } export interface CreateRulesetPayload { diff --git a/src/frontend/src/pages/RulesPage/RulesetPage.tsx b/src/frontend/src/pages/RulesPage/RulesetPage.tsx index 64775f3c7a..5f89869cf8 100644 --- a/src/frontend/src/pages/RulesPage/RulesetPage.tsx +++ b/src/frontend/src/pages/RulesPage/RulesetPage.tsx @@ -31,7 +31,13 @@ const RulesetPage: React.FC = () => { const [AddFileModalShow, setAddFileModalShow] = React.useState(false); const { data: rulesetType, isLoading, isError, error } = useRulesetType(rulesetTypeId); - const handleFileConfirm = async (data: { fileId: string; name: string; carNumber: number; parserType: string }) => { + const handleFileConfirm = async (data: { + fileId: string; + name: string; + carNumber: number; + parserType: string; + firstRulePage?: number; + }) => { setAddFileModalShow(false); toast.info('Creating ruleset and parsing rules...'); @@ -56,7 +62,8 @@ const RulesetPage: React.FC = () => { const parsedRules = await parseRuleset({ rulesetId, fileId: data.fileId, - parserType: data.parserType as 'FSAE' | 'FHE' + parserType: data.parserType as 'FSAE' | 'FHE', + firstRulePage: data.firstRulePage }); toast.success(`Successfully parsed ${parsedRules.length} rules!`); } catch (e) { diff --git a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx index 8df5e0bd7d..6e1f008d82 100644 --- a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx +++ b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx @@ -1,11 +1,25 @@ import NERFormModal from '../../../components/NERFormModal'; import { useForm, Controller } from 'react-hook-form'; -import { Box, FormControl, TextField, Typography, FormLabel, FormHelperText, Button, Select, MenuItem } from '@mui/material'; +import { + Box, + FormControl, + TextField, + Typography, + FormLabel, + FormHelperText, + Button, + Select, + MenuItem, + Accordion, + AccordionSummary, + AccordionDetails +} from '@mui/material'; import { useEffect, useState } from 'react'; import * as yup from 'yup'; import { yupResolver } from '@hookform/resolvers/yup'; import { useToast } from '../../../hooks/toasts.hooks'; import { FileUpload } from '@mui/icons-material'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import { MAX_FILE_SIZE } from 'shared'; import { useUploadRulesetFile } from '../../../hooks/rules.hooks'; import { useGetAllCars } from '../../../hooks/cars.hooks'; @@ -21,6 +35,7 @@ interface NewFileFormData { name: string; carNumber: number; parserType: 'FSAE' | 'FHE'; + firstRulePage?: number; } interface ButtonGroupProps { @@ -47,7 +62,8 @@ const schema = yup.object({ fileId: yup.string().required('File is required'), name: yup.string().required('Name is required'), carNumber: yup.number().min(0).required('Car is required'), - parserType: yup.string().oneOf(['FSAE', 'FHE']).required('Parser type is required') + parserType: yup.string().oneOf(['FSAE', 'FHE']).required('Parser type is required'), + firstRulePage: yup.number().min(1, 'Minimum is page 1').integer('Must be a whole number').optional() }); const ButtonGroup: React.FC = ({ options, value, onChange }) => { @@ -88,6 +104,7 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS const toast = useToast(); const [file, setFile] = useState(null); const [uploading, setUploading] = useState(false); + const [additionalOptionsOpen, setAdditionalOptionsOpen] = useState(false); const { mutateAsync: uploadFile } = useUploadRulesetFile(); const { data: cars, isLoading: carsLoading, isError: carsError } = useGetAllCars(); @@ -103,7 +120,8 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS fileId: '', name: '', carNumber: 100, - parserType: 'FSAE' + parserType: 'FSAE', + firstRulePage: undefined } }); @@ -168,12 +186,14 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS const handleModalClose = () => { setFile(null); reset(); + setAdditionalOptionsOpen(false); onHide(); }; const handleReset = () => { setFile(null); reset(); + setAdditionalOptionsOpen(false); }; return ( @@ -190,7 +210,7 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS > - + {/* File Upload */} Upload Ruleset File: @@ -262,6 +282,68 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS /> {errors.name?.message} + + {/* Additional Options */} + setAdditionalOptionsOpen(isExpanded)} + disableGutters + elevation={0} + square + sx={{ + backgroundColor: 'transparent', + backgroundImage: 'none', + '&:before': { display: 'none' } + }} + > + } + sx={{ + flexDirection: 'row-reverse', + gap: 1, + minHeight: 'unset', + px: 0, + '& .MuiAccordionSummary-content': { my: 0 } + }} + > + + Additional Options + + + + {/* First Rule Page */} + + First Page: + ( + onChange(e.target.value === '' ? undefined : Number(e.target.value))} + slotProps={{ htmlInput: { min: 1 } }} + error={!!errors.firstRulePage} + sx={{ width: 160 }} + /> + )} + /> + + {errors.firstRulePage?.message ?? 'Optional, skips earlier pages'} + + + + {/* Footer Formatting */} + + Footer Format: + + + Footer text to be excluded from parsing + + + + From 431b84be0d6717284b19e6f27e7fc35560260c5d Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Tue, 4 Aug 2026 13:27:50 -0400 Subject: [PATCH 2/9] #4291 footer text matching as user input --- .../src/controllers/rules.controllers.ts | 5 +- src/backend/src/routes/rules.routes.ts | 1 + src/backend/src/services/rules.services.ts | 6 ++- src/backend/src/utils/parse.utils.ts | 54 +++++++------------ src/frontend/src/apis/rules.api.ts | 3 +- src/frontend/src/hooks/rules.hooks.ts | 1 + .../src/pages/RulesPage/RulesetPage.tsx | 4 +- .../RulesPage/components/AddNewFileModal.tsx | 38 ++++++++++--- 8 files changed, 65 insertions(+), 47 deletions(-) diff --git a/src/backend/src/controllers/rules.controllers.ts b/src/backend/src/controllers/rules.controllers.ts index af182a9048..e32893d754 100644 --- a/src/backend/src/controllers/rules.controllers.ts +++ b/src/backend/src/controllers/rules.controllers.ts @@ -338,7 +338,7 @@ export default class RulesController { static async parseRuleset(req: Request, res: Response, next: NextFunction) { try { - const { fileId, parserType, firstRulePage } = req.body; + const { fileId, parserType, firstRulePage, footerText } = req.body; const { rulesetId } = req.params as Record; const parseResult = await RulesService.parseRuleset( @@ -347,7 +347,8 @@ export default class RulesController { fileId, rulesetId, parserType, - firstRulePage + firstRulePage, + footerText ); res.status(200).json(parseResult); diff --git a/src/backend/src/routes/rules.routes.ts b/src/backend/src/routes/rules.routes.ts index 3b9c17c21c..3a3800092b 100644 --- a/src/backend/src/routes/rules.routes.ts +++ b/src/backend/src/routes/rules.routes.ts @@ -121,6 +121,7 @@ rulesRouter.post( nonEmptyString(body('fileId')), nonEmptyString(body('parserType')), // 'FSAE' or 'FHE' body('firstRulePage').optional().isInt({ min: 1 }), + body('footerText').optional().isString(), validateInputs, RulesController.parseRuleset ); diff --git a/src/backend/src/services/rules.services.ts b/src/backend/src/services/rules.services.ts index 26b63c80ec..77b90acfde 100644 --- a/src/backend/src/services/rules.services.ts +++ b/src/backend/src/services/rules.services.ts @@ -1555,6 +1555,7 @@ export default class RulesService { * @param rulesetId id of the ruleset to save the parsed rules into * @param parserType type of parser to use (FSAE or FHE) * @param firstRulePage 1-indexed page rules start on; pages before this are skipped instead of parsed + * @param footerText substring (case-insensitive) identifying repeated page header/footer lines to exclude from rule content * @returns array of saved rules with parent relationships established * @throws AccessDeniedException if user lacks permissions or ruleset belongs to another organization * @throws NotFoundException if ruleset doesn't exist @@ -1568,7 +1569,8 @@ export default class RulesService { fileId: string, rulesetId: string, parserType: 'FSAE' | 'FHE', - firstRulePage?: number + firstRulePage?: number, + footerText?: string ): Promise { if (!(await userHasPermission(user.userId, organizationId, isLeadership))) { throw new AccessDeniedException('You do not have permissions to upload and parse rulesets'); @@ -1606,7 +1608,7 @@ export default class RulesService { } let parsedRules: ParsedRule[]; try { - parsedRules = await parseRulesFromPdf(buffer, parserType, firstRulePage); + parsedRules = await parseRulesFromPdf(buffer, parserType, firstRulePage, footerText); if (parsedRules.length === 0) { throw new HttpException(400, 'No rules found in provided file'); } diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index f17521edde..cfd5618efc 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -25,7 +25,8 @@ const makePageRenderer = (firstRulePage?: number) => { export const parseRulesFromPdf = async ( buffer: Buffer, parserType: 'FSAE' | 'FHE', - firstRulePage?: number + firstRulePage?: number, + footerText?: string ): Promise => { const options = { // max page number to parse, 0 = all pages @@ -37,14 +38,25 @@ export const parseRulesFromPdf = async ( const pdfData = await pdf(buffer, options); if (parserType === 'FSAE') { - return parseFSAERules(pdfData.text); + return parseFSAERules(pdfData.text, footerText); } if (parserType === 'FHE') { - return parseFHERules(pdfData.text); + return parseFHERules(pdfData.text, footerText); } throw new Error(`Invalid parser type: ${parserType}. Must be 'FSAE' or 'FHE'`); }; +/** + * Checks whether a line is a repeated page header/footer that should be excluded from rule content, + * based on user-supplied footer text rather than a hardcoded pattern. + * @param line line to check + * @param footerText substring (case-insensitive) that identifies a header/footer line + */ +const isFooterLine = (line: string, footerText?: string): boolean => { + if (!footerText) return false; + return line.toLowerCase().includes(footerText.toLowerCase()); +}; + /** * Extracts lettered sub-rules from rule content (a, b, c, etc.) * "EV.5.2 Main text a. Sub-rule" becomes: @@ -164,7 +176,7 @@ const handleDuplicateCodes = (rules: ParsedRule[]): ParsedRule[] => { /**************** FSAE ****************/ -const parseFSAERules = (text: string): ParsedRule[] => { +const parseFSAERules = (text: string, footerText?: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); @@ -181,7 +193,7 @@ const parseFSAERules = (text: string): ParsedRule[] => { if (!trimmedLine) continue; // Skip page headers/footers - if (isHeaderFooterFSAE(trimmedLine)) { + if (isFooterLine(trimmedLine, footerText)) { continue; } @@ -226,30 +238,6 @@ const parseRuleNumberFSAE = (line: string): ParsedRule | null => { return null; }; -/** - * Checks if a line is a page header/footer that should be skipped - * @param line line to check - * @returns true if line should be skipped - */ -const isHeaderFooterFSAE = (line: string): boolean => { - const trimmed = line.trim(); - - // Match FSAE headers like "Formula SAE® Rules 2025 © 2024 SAE International Page 7 of 143 Version 1.0 31 Aug 2024" - if (/Formula SAE.*Rules.*\d{4}.*SAE International.*Page \d+ of \d+/i.test(trimmed)) { - return true; - } - // Match standalone page numbers - if (/^Page \d+ of \d+$/i.test(trimmed)) { - return true; - } - // Match version strings - if (/^Version \d+\.\d+.*\d{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{4}$/i.test(trimmed)) { - return true; - } - - return false; -}; - /** * Updates rules to point to nearest existing parent if their assigned parent doesn't exist. * D.8.1.2 -> checks for D.8.1, if missing goes to D.8, then D @@ -281,7 +269,7 @@ const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { /**************** FHE *****************/ -const parseFHERules = (text: string): ParsedRule[] => { +const parseFHERules = (text: string, footerText?: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); let currentRule: { code: string; text: string } | null = null; @@ -296,10 +284,8 @@ const parseFHERules = (text: string): ParsedRule[] => { const trimmedLine = line.trim(); if (!trimmedLine) continue; - // Skip repeated running header (e.g. "2025 Formula Hybrid + Electric Rules") - if (/^\d{4}\s+Formula Hybrid.*Rules/i.test(trimmedLine)) { - saveCurrentRule(); - currentRule = null; + // Skip repeated running footer (e.g. "2025 Formula Hybrid + Electric Rules") + if (isFooterLine(trimmedLine, footerText)) { continue; } diff --git a/src/frontend/src/apis/rules.api.ts b/src/frontend/src/apis/rules.api.ts index bdf0cbd61c..93703c19eb 100644 --- a/src/frontend/src/apis/rules.api.ts +++ b/src/frontend/src/apis/rules.api.ts @@ -235,7 +235,8 @@ export const parseRuleset = (payload: ParseRulesetPayload) => { return axios.post(apiUrls.parseRuleset(payload.rulesetId), { fileId: payload.fileId, parserType: payload.parserType, - firstRulePage: payload.firstRulePage + firstRulePage: payload.firstRulePage, + footerText: payload.footerText }); }; diff --git a/src/frontend/src/hooks/rules.hooks.ts b/src/frontend/src/hooks/rules.hooks.ts index ac65d749b9..a7fd26e012 100644 --- a/src/frontend/src/hooks/rules.hooks.ts +++ b/src/frontend/src/hooks/rules.hooks.ts @@ -132,6 +132,7 @@ export interface ParseRulesetPayload { fileId: string; parserType: 'FSAE' | 'FHE'; firstRulePage?: number; + footerText?: string; } export interface CreateRulesetPayload { diff --git a/src/frontend/src/pages/RulesPage/RulesetPage.tsx b/src/frontend/src/pages/RulesPage/RulesetPage.tsx index 5f89869cf8..30d51d6a9b 100644 --- a/src/frontend/src/pages/RulesPage/RulesetPage.tsx +++ b/src/frontend/src/pages/RulesPage/RulesetPage.tsx @@ -37,6 +37,7 @@ const RulesetPage: React.FC = () => { carNumber: number; parserType: string; firstRulePage?: number; + footerText?: string; }) => { setAddFileModalShow(false); toast.info('Creating ruleset and parsing rules...'); @@ -63,7 +64,8 @@ const RulesetPage: React.FC = () => { rulesetId, fileId: data.fileId, parserType: data.parserType as 'FSAE' | 'FHE', - firstRulePage: data.firstRulePage + firstRulePage: data.firstRulePage, + footerText: data.footerText }); toast.success(`Successfully parsed ${parsedRules.length} rules!`); } catch (e) { diff --git a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx index 6e1f008d82..865f940c02 100644 --- a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx +++ b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx @@ -36,8 +36,16 @@ interface NewFileFormData { carNumber: number; parserType: 'FSAE' | 'FHE'; firstRulePage?: number; + footerText?: string; } +// Default for footer text to exclude from parsing, based on parser type. +// Excluded from parsed rule content +const DEFAULT_FOOTER_TEXT: Record<'FSAE' | 'FHE', string> = { + FSAE: 'SAE International', + FHE: 'Formula Hybrid' +}; + interface ButtonGroupProps { options: string[]; value: string; @@ -63,7 +71,8 @@ const schema = yup.object({ name: yup.string().required('Name is required'), carNumber: yup.number().min(0).required('Car is required'), parserType: yup.string().oneOf(['FSAE', 'FHE']).required('Parser type is required'), - firstRulePage: yup.number().min(1, 'Minimum is page 1').integer('Must be a whole number').optional() + firstRulePage: yup.number().min(1, 'Minimum is page 1').integer('Must be a whole number').optional(), + footerText: yup.string().optional() }); const ButtonGroup: React.FC = ({ options, value, onChange }) => { @@ -121,7 +130,8 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS name: '', carNumber: 100, parserType: 'FSAE', - firstRulePage: undefined + firstRulePage: undefined, + footerText: DEFAULT_FOOTER_TEXT.FSAE } }); @@ -263,7 +273,15 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS name="parserType" control={control} render={({ field: { onChange, value } }) => ( - onChange(val as 'FSAE' | 'FHE')} /> + { + const parserType = val as 'FSAE' | 'FHE'; + onChange(parserType); + setValue('footerText', DEFAULT_FOOTER_TEXT[parserType]); + }} + /> )} /> {errors.parserType?.message} @@ -335,11 +353,17 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS {/* Footer Formatting */} - + Footer Format: - - - Footer text to be excluded from parsing + ( + + )} + /> + + {errors.footerText?.message ?? 'Footer text to be excluded from parsing'} From a1995428262b291bf9ccd92b59658a6b645fa42b Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Tue, 4 Aug 2026 13:40:44 -0400 Subject: [PATCH 3/9] #4291 multiple excluded phrases w/ new line --- src/backend/src/utils/parse.utils.ts | 11 +++++-- .../RulesPage/components/AddNewFileModal.tsx | 33 +++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index cfd5618efc..8f04ae201a 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -49,12 +49,19 @@ export const parseRulesFromPdf = async ( /** * Checks whether a line is a repeated page header/footer that should be excluded from rule content, * based on user-supplied footer text rather than a hardcoded pattern. + * footerText may contain multiple newline-separated phrases (e.g. Formula SAE® Rules 2026 and 2025 SAE International); + * a line is excluded if it contains any one of the phrases * @param line line to check - * @param footerText substring (case-insensitive) that identifies a header/footer line + * @param footerText newline-separated substrings (case-insensitive) that identify header/footers and other excluded phrases */ const isFooterLine = (line: string, footerText?: string): boolean => { if (!footerText) return false; - return line.toLowerCase().includes(footerText.toLowerCase()); + const lowerLine = line.toLowerCase(); + const phrases = footerText + .split('\n') + .map((phrase) => phrase.trim().toLowerCase()) + .filter(Boolean); + return phrases.some((phrase) => lowerLine.includes(phrase)); }; /** diff --git a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx index 865f940c02..7be9ca5e57 100644 --- a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx +++ b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx @@ -39,10 +39,10 @@ interface NewFileFormData { footerText?: string; } -// Default for footer text to exclude from parsing, based on parser type. -// Excluded from parsed rule content +// Default footer text to exclude from parsing, based on parser type. +// Each line is matched independently, so multiple phrases can be excluded at once. const DEFAULT_FOOTER_TEXT: Record<'FSAE' | 'FHE', string> = { - FSAE: 'SAE International', + FSAE: 'Formula SAE\nSAE International', FHE: 'Formula Hybrid' }; @@ -331,7 +331,7 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS {/* First Rule Page */} - First Page: + First Page #: = ({ open, onHide, onFormS /> )} /> - + {errors.firstRulePage?.message ?? 'Optional, skips earlier pages'} - {/* Footer Formatting */} + {/* Excluded Phrases Formatting */} - Footer Format: + Excluded Phrases: ( - + )} /> - - {errors.footerText?.message ?? 'Footer text to be excluded from parsing'} + + {errors.footerText?.message ?? ( + <> + One phrase per line +
+ Skipped when parsing + + )} From 910436ae66ae5d02241f43c264eedc5b5fea5f86 Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Tue, 4 Aug 2026 16:45:15 -0400 Subject: [PATCH 4/9] #4291 cleaned up excluded phrases stripping logic --- src/backend/src/utils/parse.utils.ts | 54 +++++++++++++++------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index 8f04ae201a..a94179dcba 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -47,21 +47,25 @@ export const parseRulesFromPdf = async ( }; /** - * Checks whether a line is a repeated page header/footer that should be excluded from rule content, - * based on user-supplied footer text rather than a hardcoded pattern. - * footerText may contain multiple newline-separated phrases (e.g. Formula SAE® Rules 2026 and 2025 SAE International); - * a line is excluded if it contains any one of the phrases - * @param line line to check - * @param footerText newline-separated substrings (case-insensitive) that identify header/footers and other excluded phrases + * Remove user-supplied excluded phrases from a line, keeping the rest of the content. + * excludedText may contain multiple newline-separated phrases (e.g. Formula SAE® Rules 2026 and 2025 SAE International); + * matching is case-sensitive, so phrases should be entered exactly as they appear in the PDF. + * @param line line to strip + * @param excludedText newline-separated substrings to remove + * @returns the line with excluded phrases removed; unchanged if excludedText is empty or excluded phrases not found */ -const isFooterLine = (line: string, footerText?: string): boolean => { - if (!footerText) return false; - const lowerLine = line.toLowerCase(); - const phrases = footerText +const stripExcluded = (line: string, excludedText?: string): string => { + if (!excludedText) return line; + const phrases = excludedText .split('\n') - .map((phrase) => phrase.trim().toLowerCase()) + .map((phrase) => phrase.trim()) .filter(Boolean); - return phrases.some((phrase) => lowerLine.includes(phrase)); + + let result = line; + for (const phrase of phrases) { + result = result.replaceAll(phrase, ' '); + } + return result.replace(/\s+/g, ' ').trim(); }; /** @@ -199,13 +203,12 @@ const parseFSAERules = (text: string, footerText?: string): ParsedRule[] => { const trimmedLine = line.trim(); if (!trimmedLine) continue; - // Skip page headers/footers - if (isFooterLine(trimmedLine, footerText)) { - continue; - } + // Strip excluded phrases, keeping any real content + const cleanedLine = stripExcluded(trimmedLine, footerText); + if (!cleanedLine) continue; // Check if this line starts a new rule - const rule = parseRuleNumberFSAE(trimmedLine); + const rule = parseRuleNumberFSAE(cleanedLine); if (rule) { saveCurrentRule(); currentRule = { @@ -213,7 +216,7 @@ const parseFSAERules = (text: string, footerText?: string): ParsedRule[] => { text: rule.ruleContent }; } else if (currentRule) { - currentRule.text += ' ' + trimmedLine; // else append to existing rule + currentRule.text += ' ' + cleanedLine; // else append to existing rule } } saveCurrentRule(); @@ -291,13 +294,12 @@ const parseFHERules = (text: string, footerText?: string): ParsedRule[] => { const trimmedLine = line.trim(); if (!trimmedLine) continue; - // Skip repeated running footer (e.g. "2025 Formula Hybrid + Electric Rules") - if (isFooterLine(trimmedLine, footerText)) { - continue; - } + // Strip excluded phrases (e.g. repeated header/footer "2025 Formula Hybrid + Electric Rules"), keeping any real content + const cleanedLine = stripExcluded(trimmedLine, footerText); + if (!cleanedLine) continue; // Check if this line starts a new rule - const rule = parseRuleNumberFHE(trimmedLine); + const rule = parseRuleNumberFHE(cleanedLine); if (rule) { saveCurrentRule(); currentRule = { @@ -306,7 +308,7 @@ const parseFHERules = (text: string, footerText?: string): ParsedRule[] => { }; } else if (currentRule) { // Append to existing rule - currentRule.text += ' ' + trimmedLine; + currentRule.text += ' ' + cleanedLine; } } saveCurrentRule(); @@ -324,6 +326,8 @@ const parseFHERules = (text: string, footerText?: string): ParsedRule[] => { const parseRuleNumberFHE = (line: string): ParsedRule | null => { // Match FHE rule patterns like "1T3.17.1" followed by text const rulePattern = /^(\d+[A-Z]+\d+(?:\.\d+)*)\s+(.+)$/; + // Match FHE rule codes with no leading digit, like "EV5.6" (e.g. Electric Vehicle sections) + const plainLetterPattern = /^([A-Z]{1,4}\d+(?:\.\d+)*)\s+(.+)$/; // "PART A1 - ADMINISTRATIVE REGULATIONS" removes "PART" and captures "A1" as rule code, rest as content const partMatch = line.match(/^PART\s+([A-Z0-9]+)\s+-\s+(.+)$/); @@ -344,7 +348,7 @@ const parseRuleNumberFHE = (line: string): ParsedRule | null => { }; } - const match = line.match(rulePattern); + const match = line.match(rulePattern) || line.match(plainLetterPattern); if (match) { return { ruleCode: match[1], From 8df873ae266cf494259632fdf202f221118ec02f Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Tue, 4 Aug 2026 16:50:18 -0400 Subject: [PATCH 5/9] #4291 codes with content on new line fix --- src/backend/src/utils/parse.utils.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index a94179dcba..83f9ab5aca 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -236,6 +236,8 @@ const parseRuleNumberFSAE = (line: string): ParsedRule | null => { const rulePattern = /^([A-Z]{1,4}(?:\.[\d]+)+)\s+(.+)$/; // Match section patterns like "GR - GENERAL REGULATIONS or PS - PRE-COMPETITION SUBMISSIONS" const sectionPattern = /^([A-Z]{1,4})\s*-\s*(.+)$/; + // Match a rule code alone on its own line, with body text starting on the next line + const bareCodePattern = /^([A-Z]{1,4}(?:\.[\d]+)+)$/; const match = line.match(rulePattern) || line.match(sectionPattern); if (match) { @@ -245,6 +247,15 @@ const parseRuleNumberFSAE = (line: string): ParsedRule | null => { ruleContent: cleanContent }; } + + const bareMatch = line.match(bareCodePattern); + if (bareMatch) { + return { + ruleCode: bareMatch[1], + ruleContent: '' + }; + } + return null; }; @@ -328,6 +339,8 @@ const parseRuleNumberFHE = (line: string): ParsedRule | null => { const rulePattern = /^(\d+[A-Z]+\d+(?:\.\d+)*)\s+(.+)$/; // Match FHE rule codes with no leading digit, like "EV5.6" (e.g. Electric Vehicle sections) const plainLetterPattern = /^([A-Z]{1,4}\d+(?:\.\d+)*)\s+(.+)$/; + // Match a rule code alone on its own line, with body text starting on the next line + const bareCodePattern = /^(\d+[A-Z]+\d+(?:\.\d+)*|[A-Z]{1,4}\d+(?:\.\d+)*)$/; // "PART A1 - ADMINISTRATIVE REGULATIONS" removes "PART" and captures "A1" as rule code, rest as content const partMatch = line.match(/^PART\s+([A-Z0-9]+)\s+-\s+(.+)$/); @@ -356,6 +369,14 @@ const parseRuleNumberFHE = (line: string): ParsedRule | null => { }; } + const bareMatch = line.match(bareCodePattern); + if (bareMatch) { + return { + ruleCode: bareMatch[1], + ruleContent: '' + }; + } + return null; }; From 28983c94012ef64a76ca6546290f5f978184546a Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Tue, 4 Aug 2026 17:52:01 -0400 Subject: [PATCH 6/9] #4291 ensure no real rule content is dropped --- .../src/controllers/rules.controllers.ts | 5 +- src/backend/src/routes/rules.routes.ts | 1 - src/backend/src/services/rules.services.ts | 6 +- src/backend/src/utils/parse.utils.ts | 90 +++++++++++-------- src/frontend/src/apis/rules.api.ts | 3 +- src/frontend/src/hooks/rules.hooks.ts | 1 - .../src/pages/RulesPage/RulesetPage.tsx | 4 +- .../RulesPage/components/AddNewFileModal.tsx | 52 +---------- 8 files changed, 64 insertions(+), 98 deletions(-) diff --git a/src/backend/src/controllers/rules.controllers.ts b/src/backend/src/controllers/rules.controllers.ts index e32893d754..af182a9048 100644 --- a/src/backend/src/controllers/rules.controllers.ts +++ b/src/backend/src/controllers/rules.controllers.ts @@ -338,7 +338,7 @@ export default class RulesController { static async parseRuleset(req: Request, res: Response, next: NextFunction) { try { - const { fileId, parserType, firstRulePage, footerText } = req.body; + const { fileId, parserType, firstRulePage } = req.body; const { rulesetId } = req.params as Record; const parseResult = await RulesService.parseRuleset( @@ -347,8 +347,7 @@ export default class RulesController { fileId, rulesetId, parserType, - firstRulePage, - footerText + firstRulePage ); res.status(200).json(parseResult); diff --git a/src/backend/src/routes/rules.routes.ts b/src/backend/src/routes/rules.routes.ts index 3a3800092b..3b9c17c21c 100644 --- a/src/backend/src/routes/rules.routes.ts +++ b/src/backend/src/routes/rules.routes.ts @@ -121,7 +121,6 @@ rulesRouter.post( nonEmptyString(body('fileId')), nonEmptyString(body('parserType')), // 'FSAE' or 'FHE' body('firstRulePage').optional().isInt({ min: 1 }), - body('footerText').optional().isString(), validateInputs, RulesController.parseRuleset ); diff --git a/src/backend/src/services/rules.services.ts b/src/backend/src/services/rules.services.ts index 77b90acfde..26b63c80ec 100644 --- a/src/backend/src/services/rules.services.ts +++ b/src/backend/src/services/rules.services.ts @@ -1555,7 +1555,6 @@ export default class RulesService { * @param rulesetId id of the ruleset to save the parsed rules into * @param parserType type of parser to use (FSAE or FHE) * @param firstRulePage 1-indexed page rules start on; pages before this are skipped instead of parsed - * @param footerText substring (case-insensitive) identifying repeated page header/footer lines to exclude from rule content * @returns array of saved rules with parent relationships established * @throws AccessDeniedException if user lacks permissions or ruleset belongs to another organization * @throws NotFoundException if ruleset doesn't exist @@ -1569,8 +1568,7 @@ export default class RulesService { fileId: string, rulesetId: string, parserType: 'FSAE' | 'FHE', - firstRulePage?: number, - footerText?: string + firstRulePage?: number ): Promise { if (!(await userHasPermission(user.userId, organizationId, isLeadership))) { throw new AccessDeniedException('You do not have permissions to upload and parse rulesets'); @@ -1608,7 +1606,7 @@ export default class RulesService { } let parsedRules: ParsedRule[]; try { - parsedRules = await parseRulesFromPdf(buffer, parserType, firstRulePage, footerText); + parsedRules = await parseRulesFromPdf(buffer, parserType, firstRulePage); if (parsedRules.length === 0) { throw new HttpException(400, 'No rules found in provided file'); } diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index 83f9ab5aca..5fa4629395 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -25,8 +25,7 @@ const makePageRenderer = (firstRulePage?: number) => { export const parseRulesFromPdf = async ( buffer: Buffer, parserType: 'FSAE' | 'FHE', - firstRulePage?: number, - footerText?: string + firstRulePage?: number ): Promise => { const options = { // max page number to parse, 0 = all pages @@ -38,35 +37,20 @@ export const parseRulesFromPdf = async ( const pdfData = await pdf(buffer, options); if (parserType === 'FSAE') { - return parseFSAERules(pdfData.text, footerText); + return parseFSAERules(pdfData.text); } if (parserType === 'FHE') { - return parseFHERules(pdfData.text, footerText); + return parseFHERules(pdfData.text); } throw new Error(`Invalid parser type: ${parserType}. Must be 'FSAE' or 'FHE'`); }; /** - * Remove user-supplied excluded phrases from a line, keeping the rest of the content. - * excludedText may contain multiple newline-separated phrases (e.g. Formula SAE® Rules 2026 and 2025 SAE International); - * matching is case-sensitive, so phrases should be entered exactly as they appear in the PDF. - * @param line line to strip - * @param excludedText newline-separated substrings to remove - * @returns the line with excluded phrases removed; unchanged if excludedText is empty or excluded phrases not found + * Checks whether a line is nothing but a page number (e.g. "7" or "Page 7 of 143") + * the only content ever automatically excluded; safe to skip without risking dropping anything real + * @param line line to check */ -const stripExcluded = (line: string, excludedText?: string): string => { - if (!excludedText) return line; - const phrases = excludedText - .split('\n') - .map((phrase) => phrase.trim()) - .filter(Boolean); - - let result = line; - for (const phrase of phrases) { - result = result.replaceAll(phrase, ' '); - } - return result.replace(/\s+/g, ' ').trim(); -}; +const isPageNumberLine = (line: string): boolean => /^\d+$/.test(line) || /^Page\s+\d+\s+of\s+\d+$/i.test(line); /** * Extracts lettered sub-rules from rule content (a, b, c, etc.) @@ -187,11 +171,13 @@ const handleDuplicateCodes = (rules: ParsedRule[]): ParsedRule[] => { /**************** FSAE ****************/ -const parseFSAERules = (text: string, footerText?: string): ParsedRule[] => { +const parseFSAERules = (text: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); let currentRule: { code: string; text: string } | null = null; + let unparsedText = ''; + let unparsedCount = 0; const saveCurrentRule = () => { if (!currentRule) return; @@ -199,27 +185,43 @@ const parseFSAERules = (text: string, footerText?: string): ParsedRule[] => { rules.push(...parsedRules); }; + // Text encountered with no rule open yet would otherwise disappear (e.g. before the first + // recognized rule) - keep it as its own top-level rule under an abstract code instead of dropping it. + const saveUnparsed = () => { + if (!unparsedText.trim()) return; + unparsedCount += 1; + rules.push({ + ruleCode: `UNPARSED.${unparsedCount}`, + ruleContent: unparsedText.trim(), + parentRuleCode: undefined + }); + unparsedText = ''; + }; + for (const line of lines) { const trimmedLine = line.trim(); if (!trimmedLine) continue; - // Strip excluded phrases, keeping any real content - const cleanedLine = stripExcluded(trimmedLine, footerText); - if (!cleanedLine) continue; + // Skip bare page numbers - the only content ever automatically excluded + if (isPageNumberLine(trimmedLine)) continue; // Check if this line starts a new rule - const rule = parseRuleNumberFSAE(cleanedLine); + const rule = parseRuleNumberFSAE(trimmedLine); if (rule) { saveCurrentRule(); + saveUnparsed(); currentRule = { code: rule.ruleCode, text: rule.ruleContent }; } else if (currentRule) { - currentRule.text += ' ' + cleanedLine; // else append to existing rule + currentRule.text += ' ' + trimmedLine; // else append to existing rule + } else { + unparsedText += (unparsedText ? ' ' : '') + trimmedLine; } } saveCurrentRule(); + saveUnparsed(); const fixedRules = fixOrphanedRulesFSAE(rules); return handleDuplicateCodes(fixedRules); @@ -290,10 +292,12 @@ const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { /**************** FHE *****************/ -const parseFHERules = (text: string, footerText?: string): ParsedRule[] => { +const parseFHERules = (text: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); let currentRule: { code: string; text: string } | null = null; + let unparsedText = ''; + let unparsedCount = 0; const saveCurrentRule = () => { if (!currentRule) return; @@ -301,28 +305,44 @@ const parseFHERules = (text: string, footerText?: string): ParsedRule[] => { rules.push(...parsedRules); }; + // Text encountered with no rule open yet would otherwise disappear (e.g. before the first + // recognized rule) - keep it as its own top-level rule under an abstract code instead of dropping it. + const saveUnparsed = () => { + if (!unparsedText.trim()) return; + unparsedCount += 1; + rules.push({ + ruleCode: `UNPARSED.${unparsedCount}`, + ruleContent: unparsedText.trim(), + parentRuleCode: undefined + }); + unparsedText = ''; + }; + for (const line of lines) { const trimmedLine = line.trim(); if (!trimmedLine) continue; - // Strip excluded phrases (e.g. repeated header/footer "2025 Formula Hybrid + Electric Rules"), keeping any real content - const cleanedLine = stripExcluded(trimmedLine, footerText); - if (!cleanedLine) continue; + // Skip bare page numbers - the only content ever automatically excluded + if (isPageNumberLine(trimmedLine)) continue; // Check if this line starts a new rule - const rule = parseRuleNumberFHE(cleanedLine); + const rule = parseRuleNumberFHE(trimmedLine); if (rule) { saveCurrentRule(); + saveUnparsed(); currentRule = { code: rule.ruleCode, text: rule.ruleContent }; } else if (currentRule) { // Append to existing rule - currentRule.text += ' ' + cleanedLine; + currentRule.text += ' ' + trimmedLine; + } else { + unparsedText += (unparsedText ? ' ' : '') + trimmedLine; } } saveCurrentRule(); + saveUnparsed(); const fixedRules = fixOrphanedRulesFHE(rules); return handleDuplicateCodes(fixedRules); diff --git a/src/frontend/src/apis/rules.api.ts b/src/frontend/src/apis/rules.api.ts index 93703c19eb..bdf0cbd61c 100644 --- a/src/frontend/src/apis/rules.api.ts +++ b/src/frontend/src/apis/rules.api.ts @@ -235,8 +235,7 @@ export const parseRuleset = (payload: ParseRulesetPayload) => { return axios.post(apiUrls.parseRuleset(payload.rulesetId), { fileId: payload.fileId, parserType: payload.parserType, - firstRulePage: payload.firstRulePage, - footerText: payload.footerText + firstRulePage: payload.firstRulePage }); }; diff --git a/src/frontend/src/hooks/rules.hooks.ts b/src/frontend/src/hooks/rules.hooks.ts index a7fd26e012..ac65d749b9 100644 --- a/src/frontend/src/hooks/rules.hooks.ts +++ b/src/frontend/src/hooks/rules.hooks.ts @@ -132,7 +132,6 @@ export interface ParseRulesetPayload { fileId: string; parserType: 'FSAE' | 'FHE'; firstRulePage?: number; - footerText?: string; } export interface CreateRulesetPayload { diff --git a/src/frontend/src/pages/RulesPage/RulesetPage.tsx b/src/frontend/src/pages/RulesPage/RulesetPage.tsx index 30d51d6a9b..5f89869cf8 100644 --- a/src/frontend/src/pages/RulesPage/RulesetPage.tsx +++ b/src/frontend/src/pages/RulesPage/RulesetPage.tsx @@ -37,7 +37,6 @@ const RulesetPage: React.FC = () => { carNumber: number; parserType: string; firstRulePage?: number; - footerText?: string; }) => { setAddFileModalShow(false); toast.info('Creating ruleset and parsing rules...'); @@ -64,8 +63,7 @@ const RulesetPage: React.FC = () => { rulesetId, fileId: data.fileId, parserType: data.parserType as 'FSAE' | 'FHE', - firstRulePage: data.firstRulePage, - footerText: data.footerText + firstRulePage: data.firstRulePage }); toast.success(`Successfully parsed ${parsedRules.length} rules!`); } catch (e) { diff --git a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx index 7be9ca5e57..aaf20a6464 100644 --- a/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx +++ b/src/frontend/src/pages/RulesPage/components/AddNewFileModal.tsx @@ -36,16 +36,8 @@ interface NewFileFormData { carNumber: number; parserType: 'FSAE' | 'FHE'; firstRulePage?: number; - footerText?: string; } -// Default footer text to exclude from parsing, based on parser type. -// Each line is matched independently, so multiple phrases can be excluded at once. -const DEFAULT_FOOTER_TEXT: Record<'FSAE' | 'FHE', string> = { - FSAE: 'Formula SAE\nSAE International', - FHE: 'Formula Hybrid' -}; - interface ButtonGroupProps { options: string[]; value: string; @@ -71,8 +63,7 @@ const schema = yup.object({ name: yup.string().required('Name is required'), carNumber: yup.number().min(0).required('Car is required'), parserType: yup.string().oneOf(['FSAE', 'FHE']).required('Parser type is required'), - firstRulePage: yup.number().min(1, 'Minimum is page 1').integer('Must be a whole number').optional(), - footerText: yup.string().optional() + firstRulePage: yup.number().min(1, 'Minimum is page 1').integer('Must be a whole number').optional() }); const ButtonGroup: React.FC = ({ options, value, onChange }) => { @@ -130,8 +121,7 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS name: '', carNumber: 100, parserType: 'FSAE', - firstRulePage: undefined, - footerText: DEFAULT_FOOTER_TEXT.FSAE + firstRulePage: undefined } }); @@ -273,15 +263,7 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS name="parserType" control={control} render={({ field: { onChange, value } }) => ( - { - const parserType = val as 'FSAE' | 'FHE'; - onChange(parserType); - setValue('footerText', DEFAULT_FOOTER_TEXT[parserType]); - }} - /> + onChange(val as 'FSAE' | 'FHE')} /> )} /> {errors.parserType?.message} @@ -351,34 +333,6 @@ const AddNewFileModal: React.FC = ({ open, onHide, onFormS {errors.firstRulePage?.message ?? 'Optional, skips earlier pages'} - - {/* Excluded Phrases Formatting */} - - Excluded Phrases: - ( - - )} - /> - - {errors.footerText?.message ?? ( - <> - One phrase per line -
- Skipped when parsing - - )} - - From 07fdddf292c000ce0ddaa2929354c064c106682f Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Wed, 5 Aug 2026 11:15:15 -0400 Subject: [PATCH 7/9] #4291 subrule parsing fix --- src/backend/src/utils/parse.utils.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index 5fa4629395..173c743c96 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -52,18 +52,22 @@ export const parseRulesFromPdf = async ( */ const isPageNumberLine = (line: string): boolean => /^\d+$/.test(line) || /^Page\s+\d+\s+of\s+\d+$/i.test(line); +// Collapses embedded newlines back into normal text for final rule content. +// Kept so extractSubRules can tell real line starts from mid-sentence text. +const normalizeContent = (text: string): string => text.replace(/\s+/g, ' ').trim(); + /** - * Extracts lettered sub-rules from rule content (a, b, c, etc.) - * "EV.5.2 Main text a. Sub-rule" becomes: + * Extracts lettered sub-rules from rule content (a, b, c, etc.) when sub-rule starts on its own line. + * "EV.5.2 Main text\na. Sub-rule" becomes: * - EV.5.2 Main text * - EV.5.2.a Sub-rule - * If no subrules exist, returns the original rule + * If no subrules exist, returns the original rule. * @param ruleCode parent rule code * @param content rule content to extract from * @returns array of parsed rules including main rule and any subrules */ const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { - const letterPattern = /\s+([a-z])\.\s+/g; + const letterPattern = /(?:^|:\s)\s*([a-z])\.\s+/gm; const matches = [...content.matchAll(letterPattern)]; if (matches.length === 0) { @@ -71,7 +75,7 @@ const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { return [ { ruleCode, - ruleContent: content.trim(), + ruleContent: normalizeContent(content), parentRuleCode: findParentRuleCode(ruleCode) } ]; @@ -80,7 +84,7 @@ const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { // Extract the main rule content (everything before the first lettered item) const firstMatchIndex = matches[0].index!; - const mainContent = content.substring(0, firstMatchIndex).trim(); + const mainContent = normalizeContent(content.substring(0, firstMatchIndex)); // add main rule subRules.push({ @@ -96,7 +100,7 @@ const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { // Find where this sub-rule ends (either at next letter or end of rule content) const endIndex = i < matches.length - 1 ? matches[i + 1].index! : content.length; - const subRuleContent = content.substring(startIndex, endIndex).trim(); + const subRuleContent = normalizeContent(content.substring(startIndex, endIndex)); const subRuleCode = `${ruleCode}.${letter}`; subRules.push({ @@ -215,7 +219,7 @@ const parseFSAERules = (text: string): ParsedRule[] => { text: rule.ruleContent }; } else if (currentRule) { - currentRule.text += ' ' + trimmedLine; // else append to existing rule + currentRule.text += '\n' + trimmedLine; // else append to existing rule } else { unparsedText += (unparsedText ? ' ' : '') + trimmedLine; } @@ -336,7 +340,7 @@ const parseFHERules = (text: string): ParsedRule[] => { }; } else if (currentRule) { // Append to existing rule - currentRule.text += ' ' + trimmedLine; + currentRule.text += '\n' + trimmedLine; } else { unparsedText += (unparsedText ? ' ' : '') + trimmedLine; } From eb57b353577576a7e66a54e9bd4ff4190af56105 Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Wed, 5 Aug 2026 15:24:02 -0400 Subject: [PATCH 8/9] #4291 parsing test coverage and FHE subrule fix --- src/backend/src/utils/parse.utils.ts | 33 +- src/backend/tests/unit/parse.utils.test.ts | 523 +++++++++++++++++++++ 2 files changed, 540 insertions(+), 16 deletions(-) create mode 100644 src/backend/tests/unit/parse.utils.test.ts diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index 173c743c96..91fc4c9556 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -14,7 +14,7 @@ const defaultPageRender = pdf.DEFAULT_OPTIONS.pagerender!; * * @param firstRulePage page number to start parsing rules from (1-indexed) */ -const makePageRenderer = (firstRulePage?: number) => { +export const makePageRenderer = (firstRulePage?: number) => { if (!firstRulePage || firstRulePage <= 1) return defaultPageRender; return (pageData: { pageNumber: number }) => { if (pageData.pageNumber < firstRulePage) return ''; @@ -50,11 +50,10 @@ export const parseRulesFromPdf = async ( * the only content ever automatically excluded; safe to skip without risking dropping anything real * @param line line to check */ -const isPageNumberLine = (line: string): boolean => /^\d+$/.test(line) || /^Page\s+\d+\s+of\s+\d+$/i.test(line); +export const isPageNumberLine = (line: string): boolean => /^\d+$/.test(line) || /^Page\s+\d+\s+of\s+\d+$/i.test(line); -// Collapses embedded newlines back into normal text for final rule content. -// Kept so extractSubRules can tell real line starts from mid-sentence text. -const normalizeContent = (text: string): string => text.replace(/\s+/g, ' ').trim(); +// clean up whitespace and newlines in rule content +export const normalizeContent = (text: string): string => text.replace(/\s+/g, ' ').trim(); /** * Extracts lettered sub-rules from rule content (a, b, c, etc.) when sub-rule starts on its own line. @@ -66,8 +65,9 @@ const normalizeContent = (text: string): string => text.replace(/\s+/g, ' ').tri * @param content rule content to extract from * @returns array of parsed rules including main rule and any subrules */ -const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { - const letterPattern = /(?:^|:\s)\s*([a-z])\.\s+/gm; +export const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { + // "a." and "(a)" subrule styles + const letterPattern = /(?<=^|:\s)\s*(?:([a-z])\.|\(([a-z])\))\s+/gm; const matches = [...content.matchAll(letterPattern)]; if (matches.length === 0) { @@ -95,7 +95,8 @@ const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { // Extract lettered sub-rules for (let i = 0; i < matches.length; i++) { - const [, letter] = matches[i]; + const [, dotLetter, parenLetter] = matches[i]; + const letter = dotLetter ?? parenLetter; const startIndex = matches[i].index! + matches[i][0].length; // Find where this sub-rule ends (either at next letter or end of rule content) @@ -120,7 +121,7 @@ const extractSubRules = (ruleCode: string, content: string): ParsedRule[] => { * @param ruleCode rule code to find a parent for * @returns Parent rule code, or undefined if top level */ -const findParentRuleCode = (ruleCode: string): string | undefined => { +export const findParentRuleCode = (ruleCode: string): string | undefined => { const parts = ruleCode.split('.'); if (parts.length <= 1) { return undefined; @@ -134,7 +135,7 @@ const findParentRuleCode = (ruleCode: string): string | undefined => { * @param rules array of parsed rules * @returns array of rules without duplicate rule codes and updated parent references */ -const handleDuplicateCodes = (rules: ParsedRule[]): ParsedRule[] => { +export const handleDuplicateCodes = (rules: ParsedRule[]): ParsedRule[] => { const seenRuleCodes = new Map(); const codeMapping = new Map(); // Maps original code to new code for duplicates @@ -175,7 +176,7 @@ const handleDuplicateCodes = (rules: ParsedRule[]): ParsedRule[] => { /**************** FSAE ****************/ -const parseFSAERules = (text: string): ParsedRule[] => { +export const parseFSAERules = (text: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); @@ -237,7 +238,7 @@ const parseFSAERules = (text: string): ParsedRule[] => { * @param line single line in the extracted text from the ruleset pdf * @returns rule code and content, or null if this line does not start a new rule */ -const parseRuleNumberFSAE = (line: string): ParsedRule | null => { +export const parseRuleNumberFSAE = (line: string): ParsedRule | null => { // Match rule patterns like "GR.1.1" followed by text const rulePattern = /^([A-Z]{1,4}(?:\.[\d]+)+)\s+(.+)$/; // Match section patterns like "GR - GENERAL REGULATIONS or PS - PRE-COMPETITION SUBMISSIONS" @@ -271,7 +272,7 @@ const parseRuleNumberFSAE = (line: string): ParsedRule | null => { * @param rules array of parsed rules * @returns rules with corrected parent references */ -const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { +export const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { const existingCodes = new Set(rules.map((r) => r.ruleCode)); return rules.map((rule) => { @@ -296,7 +297,7 @@ const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { /**************** FHE *****************/ -const parseFHERules = (text: string): ParsedRule[] => { +export const parseFHERules = (text: string): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); let currentRule: { code: string; text: string } | null = null; @@ -358,7 +359,7 @@ const parseFHERules = (text: string): ParsedRule[] => { * @param line single line in the extracted text from the ruleset pdf * @returns rule code and content, or null if this line does not start a new rule */ -const parseRuleNumberFHE = (line: string): ParsedRule | null => { +export const parseRuleNumberFHE = (line: string): ParsedRule | null => { // Match FHE rule patterns like "1T3.17.1" followed by text const rulePattern = /^(\d+[A-Z]+\d+(?:\.\d+)*)\s+(.+)$/; // Match FHE rule codes with no leading digit, like "EV5.6" (e.g. Electric Vehicle sections) @@ -411,7 +412,7 @@ const parseRuleNumberFHE = (line: string): ParsedRule | null => { * @param rules array of parsed rules * @returns rules with corrected parent references */ -const fixOrphanedRulesFHE = (rules: ParsedRule[]): ParsedRule[] => { +export const fixOrphanedRulesFHE = (rules: ParsedRule[]): ParsedRule[] => { const existingCodes = new Set(rules.map((r) => r.ruleCode)); return rules.map((rule) => { diff --git a/src/backend/tests/unit/parse.utils.test.ts b/src/backend/tests/unit/parse.utils.test.ts new file mode 100644 index 0000000000..d3830cb818 --- /dev/null +++ b/src/backend/tests/unit/parse.utils.test.ts @@ -0,0 +1,523 @@ +import { vi } from 'vitest'; +import { + ParsedRule, + isPageNumberLine, + normalizeContent, + findParentRuleCode, + handleDuplicateCodes, + extractSubRules, + parseRuleNumberFSAE, + fixOrphanedRulesFSAE, + parseFSAERules, + parseRuleNumberFHE, + fixOrphanedRulesFHE, + parseFHERules, + makePageRenderer, + parseRulesFromPdf +} from '../../src/utils/parse.utils.js'; + +describe('Parse Utils Tests', () => { + describe('isPageNumberLine', () => { + it('matches a bare page number', () => { + expect(isPageNumberLine('7')).toBe(true); + expect(isPageNumberLine('143')).toBe(true); + }); + + it('matches "Page X of Y", case-insensitively', () => { + expect(isPageNumberLine('Page 7 of 143')).toBe(true); + expect(isPageNumberLine('page 7 of 143')).toBe(true); + }); + + it('does not match a real rule line', () => { + expect(isPageNumberLine('T.1.1 Some requirement text')).toBe(false); + }); + + it('does not match a line that merely contains a number', () => { + expect(isPageNumberLine('See page 7 for details')).toBe(false); + }); + }); + + describe('normalizeContent', () => { + it('collapses embedded newlines and repeated whitespace into single spaces', () => { + expect(normalizeContent('Main text\na. Sub-rule\n\nb. Another')).toBe('Main text a. Sub-rule b. Another'); + }); + + it('trims leading and trailing whitespace', () => { + expect(normalizeContent(' \n padded content \n ')).toBe('padded content'); + }); + }); + + describe('findParentRuleCode', () => { + it('strips the last segment for a nested code', () => { + expect(findParentRuleCode('EV.5.2.2')).toBe('EV.5.2'); + }); + + it('returns undefined for a top-level code', () => { + expect(findParentRuleCode('GR')).toBeUndefined(); + }); + }); + + describe('handleDuplicateCodes', () => { + it('leaves rules with unique codes unchanged', () => { + const input: ParsedRule[] = [ + { ruleCode: 'GR.1', ruleContent: 'First' }, + { ruleCode: 'GR.2', ruleContent: 'Second' } + ]; + expect(handleDuplicateCodes(input)).toEqual(input); + }); + + it('suffixes a duplicate code and remaps children pointing at the original code', () => { + const input: ParsedRule[] = [ + { ruleCode: 'GR.1', ruleContent: 'First occurrence' }, + { ruleCode: 'GR.1', ruleContent: 'Second occurrence (duplicate)' }, + { ruleCode: 'GR.1.a', ruleContent: 'Child', parentRuleCode: 'GR.1' } + ]; + const result = handleDuplicateCodes(input); + expect(result[0].ruleCode).toBe('GR.1'); + expect(result[1].ruleCode).toBe('GR.1.duplicate'); + expect(result[2].parentRuleCode).toBe('GR.1.duplicate'); + }); + + it('suffixes a third occurrence with an incrementing number', () => { + const input: ParsedRule[] = [ + { ruleCode: 'GR.1', ruleContent: 'First' }, + { ruleCode: 'GR.1', ruleContent: 'Second' }, + { ruleCode: 'GR.1', ruleContent: 'Third' } + ]; + const result = handleDuplicateCodes(input); + expect(result.map((r) => r.ruleCode)).toEqual(['GR.1', 'GR.1.duplicate', 'GR.1.duplicate2']); + }); + }); + + describe('extractSubRules', () => { + it('returns the original rule unsplit when there are no lettered items', () => { + const result = extractSubRules('T.1', 'Just some plain rule text.'); + expect(result).toEqual([{ ruleCode: 'T.1', ruleContent: 'Just some plain rule text.', parentRuleCode: 'T' }]); + }); + + it('splits lettered sub-rules that each start their own line', () => { + const content = 'Main text\na. First sub-rule\nb. Second sub-rule'; + const result = extractSubRules('EV.5.2', content); + expect(result).toEqual([ + { ruleCode: 'EV.5.2', ruleContent: 'Main text', parentRuleCode: 'EV.5' }, + { ruleCode: 'EV.5.2.a', ruleContent: 'First sub-rule', parentRuleCode: 'EV.5.2' }, + { ruleCode: 'EV.5.2.b', ruleContent: 'Second sub-rule', parentRuleCode: 'EV.5.2' } + ]); + }); + + it('splits a lettered item that immediately follows a list-introducing colon on the same line', () => { + const content = 'Requirements: a. First item\nb. Second item'; + const result = extractSubRules('T.1', content); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + ['T.1', 'Requirements:'], + ['T.1.a', 'First item'], + ['T.1.b', 'Second item'] + ]); + }); + + it('splits a lettered item starting at the very beginning of the content, with no intro text', () => { + const content = 'a. First item\nb. Second item'; + const result = extractSubRules('T.1', content); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + ['T.1', ''], + ['T.1.a', 'First item'], + ['T.1.b', 'Second item'] + ]); + }); + + it('does not split on a cross-reference like "p. 10" mid-sentence', () => { + const content = 'See p. 10 for the full diagram.'; + const result = extractSubRules('T.1', content); + expect(result).toEqual([{ ruleCode: 'T.1', ruleContent: content, parentRuleCode: 'T' }]); + }); + + it('does not split on an abbreviation like "fig. b." mid-sentence', () => { + const content = 'Refer to fig. b. above for clarification.'; + const result = extractSubRules('T.1', content); + expect(result).toEqual([{ ruleCode: 'T.1', ruleContent: content, parentRuleCode: 'T' }]); + }); + }); + + // FSAE Testing + describe('parseRuleNumberFSAE', () => { + it('matches a rule code followed by inline content', () => { + expect(parseRuleNumberFSAE('GR.1.1 Cars must have a roll bar.')).toEqual({ + ruleCode: 'GR.1.1', + ruleContent: 'Cars must have a roll bar.' + }); + }); + + it('matches a section header', () => { + expect(parseRuleNumberFSAE('GR - GENERAL REGULATIONS')).toEqual({ + ruleCode: 'GR', + ruleContent: 'GENERAL REGULATIONS' + }); + }); + + it('matches a bare rule code with no inline content, returning empty content', () => { + expect(parseRuleNumberFSAE('GR.1.1')).toEqual({ ruleCode: 'GR.1.1', ruleContent: '' }); + }); + + it('collapses runs of 5+ dots down to exactly 5', () => { + const result = parseRuleNumberFSAE('GR.1.1 See table.......... 12'); + expect(result?.ruleContent).toBe('See table..... 12'); + }); + + it('returns null for a line that does not look like a rule', () => { + expect(parseRuleNumberFSAE('This is just ordinary prose.')).toBeNull(); + }); + }); + + describe('fixOrphanedRulesFSAE', () => { + it('leaves a rule unchanged when its parent exists', () => { + const input: ParsedRule[] = [ + { ruleCode: 'GR.1', ruleContent: 'Parent' }, + { ruleCode: 'GR.1.1', ruleContent: 'Child', parentRuleCode: 'GR.1' } + ]; + expect(fixOrphanedRulesFSAE(input)).toEqual(input); + }); + + it('walks up to the nearest existing ancestor when the direct parent is missing', () => { + const input: ParsedRule[] = [ + { ruleCode: 'D', ruleContent: 'Top level' }, + { ruleCode: 'D.8.1.2', ruleContent: 'Deep child', parentRuleCode: 'D.8.1' } + ]; + const result = fixOrphanedRulesFSAE(input); + expect(result[1].parentRuleCode).toBe('D'); + }); + + it('becomes top-level when no ancestor exists at all', () => { + const input: ParsedRule[] = [{ ruleCode: 'D.8.1.2', ruleContent: 'Orphan', parentRuleCode: 'D.8.1' }]; + const result = fixOrphanedRulesFSAE(input); + expect(result[0].parentRuleCode).toBeUndefined(); + }); + }); + + describe('parseRuleNumberFHE', () => { + it('matches a digit-prefixed rule code', () => { + expect(parseRuleNumberFHE('1T3.17.1 Battery enclosures must be sealed.')).toEqual({ + ruleCode: '1T3.17.1', + ruleContent: 'Battery enclosures must be sealed.' + }); + }); + + it('matches a plain letter+digit rule code with no leading digit', () => { + expect(parseRuleNumberFHE('EV5.6 Accumulator systems must address stack arrangement.')).toEqual({ + ruleCode: 'EV5.6', + ruleContent: 'Accumulator systems must address stack arrangement.' + }); + }); + + it('matches a PART header and strips the "PART" keyword from the code', () => { + expect(parseRuleNumberFHE('PART A1 - ADMINISTRATIVE REGULATIONS')).toEqual({ + ruleCode: 'A1', + ruleContent: 'ADMINISTRATIVE REGULATIONS' + }); + }); + + it('matches an ARTICLE header and strips the "ARTICLE" keyword from the code', () => { + expect(parseRuleNumberFHE('ARTICLE A11 FORMULA HYBRID + ELECTRIC OVERVIEW')).toEqual({ + ruleCode: 'A11', + ruleContent: 'FORMULA HYBRID + ELECTRIC OVERVIEW' + }); + }); + + it('matches a bare rule code with no inline content, returning empty content', () => { + expect(parseRuleNumberFHE('EV5.6')).toEqual({ ruleCode: 'EV5.6', ruleContent: '' }); + expect(parseRuleNumberFHE('1T3.17.1')).toEqual({ ruleCode: '1T3.17.1', ruleContent: '' }); + }); + + it('returns null for a line that does not look like a rule', () => { + expect(parseRuleNumberFHE('This is just ordinary prose.')).toBeNull(); + }); + }); + + describe('fixOrphanedRulesFHE', () => { + it('leaves a rule unchanged when its parent exists', () => { + const input: ParsedRule[] = [ + { ruleCode: '1T3', ruleContent: 'Parent' }, + { ruleCode: '1T3.17', ruleContent: 'Child', parentRuleCode: '1T3' } + ]; + expect(fixOrphanedRulesFHE(input)).toEqual(input); + }); + + it('falls back to the article-format parent (1A11 -> A11) when the digit-prefixed parent is missing', () => { + const input: ParsedRule[] = [ + { ruleCode: 'A11', ruleContent: 'Article A11 overview' }, + { ruleCode: '1A11.1', ruleContent: 'Sub-rule', parentRuleCode: '1A11' } + ]; + const result = fixOrphanedRulesFHE(input); + expect(result[1].parentRuleCode).toBe('A11'); + }); + + it('strips a leading digit from an intermediate ancestor found while walking up the hierarchy', () => { + const input: ParsedRule[] = [ + { ruleCode: 'A5', ruleContent: 'Article A5 overview' }, + { ruleCode: '1A5.1.2', ruleContent: 'Deep sub-rule', parentRuleCode: '1A5.1' } + ]; + const result = fixOrphanedRulesFHE(input); + expect(result[1].parentRuleCode).toBe('A5'); + }); + + it('becomes top-level when no ancestor exists at all', () => { + const input: ParsedRule[] = [{ ruleCode: '1A5.1', ruleContent: 'Orphan', parentRuleCode: '1A5' }]; + const result = fixOrphanedRulesFHE(input); + expect(result[0].parentRuleCode).toBeUndefined(); + }); + }); + + describe('parseFSAERules', () => { + it('parses multiple top-level and nested rules from multi-line text', () => { + const text = ['T - TECHNICAL ASPECTS', 'T.1 COCKPIT', 'T.1.1 Cockpit Opening'].join('\n'); + const result = parseFSAERules(text); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + ['T', 'TECHNICAL ASPECTS'], + ['T.1', 'COCKPIT'], + ['T.1.1', 'Cockpit Opening'] + ]); + }); + + it('opens a new rule from a bare code line instead of merging it into the previous rule', () => { + const text = ['GR.1.1 First rule content.', 'GR.1.2', 'Body text starting on the next line.'].join('\n'); + const result = parseFSAERules(text); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + ['GR.1.1', 'First rule content.'], + ['GR.1.2', 'Body text starting on the next line.'] + ]); + }); + + it('captures text before the first recognized rule as an UNPARSED rule instead of dropping it', () => { + const text = ['Cover page filler text.', 'More preamble.', 'GR.1.1 The real first rule.'].join('\n'); + const result = parseFSAERules(text); + expect(result[0]).toEqual({ + ruleCode: 'UNPARSED.1', + ruleContent: 'Cover page filler text. More preamble.', + parentRuleCode: undefined + }); + expect(result[1].ruleCode).toBe('GR.1.1'); + }); + + it('skips bare page number lines without dropping surrounding content', () => { + const text = ['GR.1.1 First rule.', '7', 'Page 7 of 143', 'GR.1.2 Second rule.'].join('\n'); + const result = parseFSAERules(text); + expect(result.map((r) => r.ruleCode)).toEqual(['GR.1.1', 'GR.1.2']); + }); + }); + + describe('parseFHERules', () => { + it('recognizes a plain letter+digit code (EV5.6) as its own rule instead of merging into the enclosing ARTICLE', () => { + const text = [ + 'ARTICLE EV1 POUCH TYPE LITHIUM-ION CELLS', + 'Important Note: Designing an accumulator system utilizing pouch cells is a substantial undertaking.', + 'EV5.6 Accumulator systems using pouch cells must address stack arrangement.', + 'EV5.7 Teams must provide details of the design in their ESF1 and ESF2 submissions.' + ].join('\n'); + const result = parseFHERules(text); + expect(result.map((r) => r.ruleCode)).toEqual(['EV1', 'EV5.6', 'EV5.7']); + expect(result[1].ruleContent).toBe('Accumulator systems using pouch cells must address stack arrangement.'); + expect(result[2].ruleContent).toBe('Teams must provide details of the design in their ESF1 and ESF2 submissions.'); + }); + + it('opens a new rule from a bare code line instead of merging it into the previous rule', () => { + const text = ['1T3.17.1 First rule content.', '1T3.17.2', 'Body text starting on the next line.'].join('\n'); + const result = parseFHERules(text); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + ['1T3.17.1', 'First rule content.'], + ['1T3.17.2', 'Body text starting on the next line.'] + ]); + }); + + it('captures text before the first recognized rule as an UNPARSED rule instead of dropping it', () => { + const text = ['2026 Formula Hybrid + Electric Rules', 'Table of contents filler.', '1T3.1 The real first rule.'].join( + '\n' + ); + const result = parseFHERules(text); + expect(result[0]).toEqual({ + ruleCode: 'UNPARSED.1', + ruleContent: '2026 Formula Hybrid + Electric Rules Table of contents filler.', + parentRuleCode: undefined + }); + expect(result[1].ruleCode).toBe('1T3.1'); + }); + }); +}); + +// mocks pdf-parse-new, used to test parsing logic without real PDF files +vi.mock('pdf-parse-new', () => { + // loop over pages 1..N, call options.pagerender for each one and join with '\n\n'. + const fn: any = async (_buffer: Buffer, options: any) => { + const pages = (globalThis as any).__testPages as string[]; + let text = ''; + for (let i = 1; i <= pages.length; i++) { + text += `\n\n${await options.pagerender({ pageNumber: i })}`; + } + return { text, numpages: pages.length, numrender: pages.length, info: null, metadata: null }; + }; + // mimics the library's default per-page text extractor + fn.DEFAULT_OPTIONS = { + pagerender: async (pageData: any) => { + const pages = (globalThis as any).__testPages as string[]; + return pages[pageData.pageNumber - 1] ?? ''; + } + }; + return { default: fn }; +}); + +describe('parseRulesFromPdf (mocked pdf-parse-new)', () => { + describe('makePageRenderer', () => { + beforeEach(() => { + (globalThis as any).__testPages = ['page one text', 'page two text', 'page three text']; + }); + + it('delegates to the default renderer for every page when firstRulePage is not given', async () => { + expect(await makePageRenderer(undefined)({ pageNumber: 1 })).toBe('page one text'); + expect(await makePageRenderer(undefined)({ pageNumber: 3 })).toBe('page three text'); + }); + + it('returns empty text for pages before firstRulePage and delegates from firstRulePage onward', async () => { + expect(await makePageRenderer(3)({ pageNumber: 1 })).toBe(''); + expect(await makePageRenderer(3)({ pageNumber: 2 })).toBe(''); + expect(await makePageRenderer(3)({ pageNumber: 3 })).toBe('page three text'); + }); + }); + + describe('parseRulesFromPdf', () => { + it('dispatches to the FSAE parser for parserType "FSAE"', async () => { + (globalThis as any).__testPages = ['GR.1.1 Cars must have a roll bar.']; + const result = await parseRulesFromPdf(Buffer.from(''), 'FSAE'); + expect(result).toEqual([{ ruleCode: 'GR.1.1', ruleContent: 'Cars must have a roll bar.', parentRuleCode: undefined }]); + }); + + it('dispatches to the FHE parser for parserType "FHE"', async () => { + (globalThis as any).__testPages = ['EV5.6 Accumulator systems must address stack arrangement.']; + const result = await parseRulesFromPdf(Buffer.from(''), 'FHE'); + expect(result).toEqual([ + { ruleCode: 'EV5.6', ruleContent: 'Accumulator systems must address stack arrangement.', parentRuleCode: undefined } + ]); + }); + + it('throws for an unrecognized parser type', async () => { + (globalThis as any).__testPages = ['GR.1.1 Cars must have a roll bar.']; + await expect(parseRulesFromPdf(Buffer.from(''), 'INVALID' as unknown as 'FSAE')).rejects.toThrow( + "Invalid parser type: INVALID. Must be 'FSAE' or 'FHE'" + ); + }); + + it('skips pages before firstRulePage entirely', async () => { + (globalThis as any).__testPages = ['TABLE OF CONTENTS\nGR.1.1 ..... 5', 'GR.1.1 Cars must have a roll bar.']; + const result = await parseRulesFromPdf(Buffer.from(''), 'FSAE', 2); + expect(result).toEqual([{ ruleCode: 'GR.1.1', ruleContent: 'Cars must have a roll bar.', parentRuleCode: undefined }]); + }); + + it('real 2026 FSAE rules with subrules', async () => { + (globalThis as any).__testPages = [ + '2026 FSAE Rules\nTABLE OF CONTENTS', + [ + 'PS.3.2 Penalty Detail', + 'PS.3.2.1 Late Submissions get a point penalty as shown in Table PS-2, subject to official discretion', + 'PS.3.2.2 Additional penalties will apply if Not Submitted, subject to official discretion', + 'PS.3.2.3 Penalties up to and including Removal of Team Entry may apply based on document reviews,', + 'subject to official discretion', + 'PS.3.3 Removal of Team Entry', + 'PS.3.3.1 The organizer may remove the team entry when a:' + ].join('\n'), + [ + 'a. Grounds for Removal document is Not Submitted in 24 hours or less after the deadline.', + 'Removals will occur after each Document Submission deadline', + 'b. Team does not respond to Reviewer requests or organizer communications', + 'PS.3.3.2 When a team entry will be removed:', + 'a. The team will be notified prior to cancelling registration', + 'b. No refund of entry fees will be given' + ].join('\n') + ]; + const result = await parseRulesFromPdf(Buffer.from(''), 'FSAE', 2); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + ['PS.3.2', 'Penalty Detail'], + ['PS.3.2.1', 'Late Submissions get a point penalty as shown in Table PS-2, subject to official discretion'], + ['PS.3.2.2', 'Additional penalties will apply if Not Submitted, subject to official discretion'], + [ + 'PS.3.2.3', + 'Penalties up to and including Removal of Team Entry may apply based on document reviews, subject to official discretion' + ], + ['PS.3.3', 'Removal of Team Entry'], + ['PS.3.3.1', 'The organizer may remove the team entry when a:'], + [ + 'PS.3.3.1.a', + 'Grounds for Removal document is Not Submitted in 24 hours or less after the deadline. Removals will occur after each Document Submission deadline' + ], + ['PS.3.3.1.b', 'Team does not respond to Reviewer requests or organizer communications'], + ['PS.3.3.2', 'When a team entry will be removed:'], + ['PS.3.3.2.a', 'The team will be notified prior to cancelling registration'], + ['PS.3.3.2.b', 'No refund of entry fees will be given'] + ]); + }); + + it('real 2026 FHE rules with a page boundary', async () => { + (globalThis as any).__testPages = [ + '2026 Formula Hybrid + Electric Rules\nTABLE OF CONTENTS', + [ + 'A2.3.2 Teams planning to enter a vehicle in the HIP category will initially register as a Hybrid. To', + 'change to the HIP category, the team must submit a request to the organizers in writing before', + 'the start of the design event.', + 'Note: The advantages of entering as an HIP are:', + '(a) Receive a full technical inspection of the vehicle and electrical drive systems.', + '(b) Participate in all the competition events. (Provided tech inspection is passed).', + '(c) Receive feedback from the design judges.', + 'Note: Teams can maximize the benefits of an HIP entry by including the full-hybrid', + 'designs in their document submissions and design event presentations, as well as', + 'including the full multi-year program in their Project Management materials.', + '(d) When the vehicle is completed and entered as a hybrid, in a subsequent competition, it is', + 'considered an all-new vehicle, and not a second-year entry.', + '2026 Formula Hybrid + Electric Rules – Rev. 1 3 September 11, 2025' + ].join('\n'), + [ + 'A2.4 Static Events Only (SEO)', + 'A2.4.1 SEO is a category that may only be declared after arrival at the competition. All teams must', + 'initially register as either Hybrid/HIP or Electric.', + 'A2.4.2 A team may declare themselves as SEO and participate in the design2 and other static events', + 'even if the vehicle is in an unfinished state.', + '(a) An SEO vehicle may not participate in any of the dynamic events.', + '(b) An SEO vehicle may continue the technical inspection process, but will be given a lower', + 'priority than the non-SEO teams.', + 'A2.4.3 An SEO declaration must be submitted in writing to the organizers before the scheduled start of', + 'the design events.' + ].join('\n') + ]; + const result = await parseRulesFromPdf(Buffer.from(''), 'FHE', 2); + expect(result.map((r) => [r.ruleCode, r.ruleContent])).toEqual([ + [ + 'A2.3.2', + 'Teams planning to enter a vehicle in the HIP category will initially register as a Hybrid. To change to the HIP category, the team must submit a request to the organizers in writing before the start of the design event. Note: The advantages of entering as an HIP are:' + ], + ['A2.3.2.a', 'Receive a full technical inspection of the vehicle and electrical drive systems.'], + ['A2.3.2.b', 'Participate in all the competition events. (Provided tech inspection is passed).'], + [ + 'A2.3.2.c', + 'Receive feedback from the design judges. Note: Teams can maximize the benefits of an HIP entry by including the full-hybrid designs in their document submissions and design event presentations, as well as including the full multi-year program in their Project Management materials.' + ], + [ + 'A2.3.2.d', + 'When the vehicle is completed and entered as a hybrid, in a subsequent competition, it is considered an all-new vehicle, and not a second-year entry. 2026 Formula Hybrid + Electric Rules – Rev. 1 3 September 11, 2025' + ], + ['A2.4', 'Static Events Only (SEO)'], + [ + 'A2.4.1', + 'SEO is a category that may only be declared after arrival at the competition. All teams must initially register as either Hybrid/HIP or Electric.' + ], + [ + 'A2.4.2', + 'A team may declare themselves as SEO and participate in the design2 and other static events even if the vehicle is in an unfinished state.' + ], + ['A2.4.2.a', 'An SEO vehicle may not participate in any of the dynamic events.'], + [ + 'A2.4.2.b', + 'An SEO vehicle may continue the technical inspection process, but will be given a lower priority than the non-SEO teams.' + ], + [ + 'A2.4.3', + 'An SEO declaration must be submitted in writing to the organizers before the scheduled start of the design events.' + ] + ]); + }); + }); +}); From 7fbcb4de597b4473e4c81939869ff7e575b610b6 Mon Sep 17 00:00:00 2001 From: Ciel Bellerose Date: Wed, 5 Aug 2026 16:26:24 -0400 Subject: [PATCH 9/9] #4291 combine duplicate logic and exclude page number fix --- src/backend/src/utils/parse.utils.ts | 106 ++++++++------------- src/backend/tests/unit/parse.utils.test.ts | 30 ++++-- 2 files changed, 63 insertions(+), 73 deletions(-) diff --git a/src/backend/src/utils/parse.utils.ts b/src/backend/src/utils/parse.utils.ts index 91fc4c9556..90aeb7cc69 100644 --- a/src/backend/src/utils/parse.utils.ts +++ b/src/backend/src/utils/parse.utils.ts @@ -46,11 +46,22 @@ export const parseRulesFromPdf = async ( }; /** - * Checks whether a line is nothing but a page number (e.g. "7" or "Page 7 of 143") - * the only content ever automatically excluded; safe to skip without risking dropping anything real + * Checks whether a line is only a page number (e.g. "7" or "Page 7 of 143"). + * Safe to skip without risking dropping rule content. * @param line line to check */ -export const isPageNumberLine = (line: string): boolean => /^\d+$/.test(line) || /^Page\s+\d+\s+of\s+\d+$/i.test(line); +export const isPageNumberLine = (line: string): boolean => /^\d+$/.test(line); + +/** + * Removes "Page X of Y" wherever it occurs in a line + * @param line line to strip + * @returns the line with any "Page X of Y" occurrences removed and whitespace collapsed + */ +export const stripPageNumberPhrase = (line: string): string => + line + .replace(/Page\s+\d+\s+of\s+\d+/gi, '') + .replace(/\s+/g, ' ') + .trim(); // clean up whitespace and newlines in rule content export const normalizeContent = (text: string): string => text.replace(/\s+/g, ' ').trim(); @@ -174,9 +185,19 @@ export const handleDuplicateCodes = (rules: ParsedRule[]): ParsedRule[] => { }); }; -/**************** FSAE ****************/ - -export const parseFSAERules = (text: string): ParsedRule[] => { +/** + * Shared line-by-line parsing loop used by FSAE and FHE parsers. + * Parsers differ in how rule code is recognized and logic for orphaned parent ref fixes. + * @param text full extracted PDF text to parse + * @param parseRuleNumber recognizes whether a line starts a new rule (FSAE- or FHE-specific) + * @param fixOrphanedRules re-parents rules whose inferred parent code doesn't exist (FSAE- or FHE-specific) + * @returns array of parsed rules + */ +const parseRules = ( + text: string, + parseRuleNumber: (line: string) => ParsedRule | null, + fixOrphanedRules: (rules: ParsedRule[]) => ParsedRule[] +): ParsedRule[] => { const rules: ParsedRule[] = []; const lines = text.split('\n'); @@ -207,11 +228,12 @@ export const parseFSAERules = (text: string): ParsedRule[] => { const trimmedLine = line.trim(); if (!trimmedLine) continue; - // Skip bare page numbers - the only content ever automatically excluded - if (isPageNumberLine(trimmedLine)) continue; + // Remove "Page X of Y", then if nothing real remains or just a page number left, skip entire line + const cleanedLine = stripPageNumberPhrase(trimmedLine); + if (!cleanedLine || isPageNumberLine(cleanedLine)) continue; // Check if this line starts a new rule - const rule = parseRuleNumberFSAE(trimmedLine); + const rule = parseRuleNumber(cleanedLine); if (rule) { saveCurrentRule(); saveUnparsed(); @@ -220,18 +242,22 @@ export const parseFSAERules = (text: string): ParsedRule[] => { text: rule.ruleContent }; } else if (currentRule) { - currentRule.text += '\n' + trimmedLine; // else append to existing rule + currentRule.text += '\n' + cleanedLine; // else append to existing rule } else { - unparsedText += (unparsedText ? ' ' : '') + trimmedLine; + unparsedText += (unparsedText ? ' ' : '') + cleanedLine; } } saveCurrentRule(); saveUnparsed(); - const fixedRules = fixOrphanedRulesFSAE(rules); + const fixedRules = fixOrphanedRules(rules); return handleDuplicateCodes(fixedRules); }; +/**************** FSAE ****************/ + +export const parseFSAERules = (text: string): ParsedRule[] => parseRules(text, parseRuleNumberFSAE, fixOrphanedRulesFSAE); + /** * Determines if this line starts a new rule, if so extracts code and content of the rule * Matches rule pattern (e.g. GR.1.1 some text) or section pattern (e.g. GR - TEXT) @@ -297,61 +323,7 @@ export const fixOrphanedRulesFSAE = (rules: ParsedRule[]): ParsedRule[] => { /**************** FHE *****************/ -export const parseFHERules = (text: string): ParsedRule[] => { - const rules: ParsedRule[] = []; - const lines = text.split('\n'); - let currentRule: { code: string; text: string } | null = null; - let unparsedText = ''; - let unparsedCount = 0; - - const saveCurrentRule = () => { - if (!currentRule) return; - const parsedRules = extractSubRules(currentRule.code, currentRule.text); - rules.push(...parsedRules); - }; - - // Text encountered with no rule open yet would otherwise disappear (e.g. before the first - // recognized rule) - keep it as its own top-level rule under an abstract code instead of dropping it. - const saveUnparsed = () => { - if (!unparsedText.trim()) return; - unparsedCount += 1; - rules.push({ - ruleCode: `UNPARSED.${unparsedCount}`, - ruleContent: unparsedText.trim(), - parentRuleCode: undefined - }); - unparsedText = ''; - }; - - for (const line of lines) { - const trimmedLine = line.trim(); - if (!trimmedLine) continue; - - // Skip bare page numbers - the only content ever automatically excluded - if (isPageNumberLine(trimmedLine)) continue; - - // Check if this line starts a new rule - const rule = parseRuleNumberFHE(trimmedLine); - if (rule) { - saveCurrentRule(); - saveUnparsed(); - currentRule = { - code: rule.ruleCode, - text: rule.ruleContent - }; - } else if (currentRule) { - // Append to existing rule - currentRule.text += '\n' + trimmedLine; - } else { - unparsedText += (unparsedText ? ' ' : '') + trimmedLine; - } - } - saveCurrentRule(); - saveUnparsed(); - - const fixedRules = fixOrphanedRulesFHE(rules); - return handleDuplicateCodes(fixedRules); -}; +export const parseFHERules = (text: string): ParsedRule[] => parseRules(text, parseRuleNumberFHE, fixOrphanedRulesFHE); /** * Determines if this line starts a new rule, if so extracts code and content of the rule diff --git a/src/backend/tests/unit/parse.utils.test.ts b/src/backend/tests/unit/parse.utils.test.ts index d3830cb818..9d650ef4f7 100644 --- a/src/backend/tests/unit/parse.utils.test.ts +++ b/src/backend/tests/unit/parse.utils.test.ts @@ -2,6 +2,7 @@ import { vi } from 'vitest'; import { ParsedRule, isPageNumberLine, + stripPageNumberPhrase, normalizeContent, findParentRuleCode, handleDuplicateCodes, @@ -23,11 +24,6 @@ describe('Parse Utils Tests', () => { expect(isPageNumberLine('143')).toBe(true); }); - it('matches "Page X of Y", case-insensitively', () => { - expect(isPageNumberLine('Page 7 of 143')).toBe(true); - expect(isPageNumberLine('page 7 of 143')).toBe(true); - }); - it('does not match a real rule line', () => { expect(isPageNumberLine('T.1.1 Some requirement text')).toBe(false); }); @@ -37,6 +33,28 @@ describe('Parse Utils Tests', () => { }); }); + describe('stripPageNumberPhrase', () => { + it('removes "Page X of Y" when it is the whole line', () => { + expect(stripPageNumberPhrase('Page 7 of 143')).toBe(''); + }); + + it('matches case-insensitively', () => { + expect(stripPageNumberPhrase('page 7 of 143')).toBe(''); + }); + + it('removes "Page X of Y" embedded in a longer line, keeping the real content on both sides', () => { + const line = + 'Violations on Intent The violation of the intent of a rule will be considered a violation of the rule itself Formula SAE® Rules 2026 © 2025 SAE International Page 8 of 145 Version 1.0 10 Sept 2025'; + expect(stripPageNumberPhrase(line)).toBe( + 'Violations on Intent The violation of the intent of a rule will be considered a violation of the rule itself Formula SAE® Rules 2026 © 2025 SAE International Version 1.0 10 Sept 2025' + ); + }); + + it('leaves a line unchanged when it does not contain the phrase', () => { + expect(stripPageNumberPhrase('GR.1.1 Cars must have a roll bar.')).toBe('GR.1.1 Cars must have a roll bar.'); + }); + }); + describe('normalizeContent', () => { it('collapses embedded newlines and repeated whitespace into single spaces', () => { expect(normalizeContent('Main text\na. Sub-rule\n\nb. Another')).toBe('Main text a. Sub-rule b. Another'); @@ -363,7 +381,7 @@ vi.mock('pdf-parse-new', () => { return { default: fn }; }); -describe('parseRulesFromPdf (mocked pdf-parse-new)', () => { +describe('Parsing pdf Tests', () => { describe('makePageRenderer', () => { beforeEach(() => { (globalThis as any).__testPages = ['page one text', 'page two text', 'page three text'];