From 5cfc52c7653365e7aae2f316db3ef16b0c166dd2 Mon Sep 17 00:00:00 2001 From: Ankit Rathod Date: Thu, 30 Jul 2026 17:57:19 +0530 Subject: [PATCH 01/11] Enhance source encoding handling in Aviator - Introduced SourceEncodingOptions to manage source encoding candidates for decoding and encoding source files. - Updated AuditFprOptions to include default source encoding options. - Modified AuditProcessor and RemediationProcessor to accept and utilize SourceEncodingOptions. - Enhanced FileUtils to read files using specified source encoding options. - Updated SourceCodeEnricher to decode source files based on encoding options. - Adjusted AviatorGrpcClient and AviatorStreamProcessor to pass source encoding options through gRPC calls. - Updated AviatorSSCApplyRemediationsCommand and AviatorSSCAuditCommand to accept source encoding parameters from the command line. - Added new properties for source encoding options in internationalization files for both Aviator and FoD. --- .../ApplyAutoRemediationOnSource.java | 9 +- .../fortify/cli/aviator/audit/AuditFPR.java | 28 ++-- .../cli/aviator/audit/IssueAuditor.java | 18 ++- .../aviator/audit/model/AuditFprOptions.java | 2 + .../aviator/fpr/processor/AuditProcessor.java | 26 +++- .../fpr/processor/RemediationProcessor.java | 107 +++++-------- .../cli/aviator/fpr/utils/FileUtils.java | 27 +++- .../aviator/fpr/utils/SourceCodeEnricher.java | 13 +- .../fpr/utils/SourceEncodingOptions.java | 143 ++++++++++++++++++ .../cli/aviator/grpc/AviatorGrpcClient.java | 9 +- .../aviator/grpc/AviatorStreamProcessor.java | 13 +- .../AviatorSSCApplyRemediationsCommand.java | 9 +- .../ssc/cli/cmd/AviatorSSCAuditCommand.java | 4 + .../aviator/i18n/AviatorMessages.properties | 2 + .../FoDAviatorApplyRemediationsCommand.java | 6 +- .../cli/fod/i18n/FoDMessages.properties | 1 + 16 files changed, 321 insertions(+), 96 deletions(-) create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java index e2db9d777cb..3788d5ce0a9 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java @@ -20,6 +20,7 @@ import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor.RemediationMetric; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.FprHandle; @@ -28,6 +29,12 @@ public class ApplyAutoRemediationOnSource { public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, IAviatorLogger logger) throws AviatorSimpleException, AviatorTechnicalException { + return applyRemediations(fprHandle, sourceCodeDirectory, SourceEncodingOptions.defaults(), logger); + } + + public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, + SourceEncodingOptions sourceEncodingOptions, IAviatorLogger logger) + throws AviatorSimpleException, AviatorTechnicalException { LOG.info("Starting apply auto-remediation process for file: {}", fprHandle.getFprPath()); @@ -37,7 +44,7 @@ public static RemediationMetric applyRemediations(FprHandle fprHandle, String so } LOG.info("FPR validation successful"); - RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory); + RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory, sourceEncodingOptions); return remediationProcessor.processRemediationXML(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 4571af0bc7f..e090ab863bb 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -39,6 +39,7 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.ResourceUtil; @@ -52,8 +53,12 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) options.getFprHandle().validate(); AviatorConfigManager.getInstance(); + SourceEncodingOptions sourceEncodingOptions = options.getSourceEncodingOptions() == null + ? SourceEncodingOptions.defaults() + : options.getSourceEncodingOptions(); + // --- STAGE 1: PARSING --- - ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle()); + ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle(), sourceEncodingOptions); TagMappingConfig tagMappingConfig = loadTagMappingConfig(options.getTagMappingPath()); Map issueCategoryLookup = tagMappingConfig.requiresCategoryForSuppressionEvaluation() ? buildIssueCategoryLookup(parsedData.vulnerabilities) @@ -69,20 +74,20 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) Map auditResponses = new ConcurrentHashMap<>(); AuditOutcome auditOutcome = performAviatorAudit( parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(), - auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder() + auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder(), sourceEncodingOptions ); // --- STAGE 4: FINALIZATION --- return finalizeFprAudit( auditOutcome, auditResponses, parsedData.auditProcessor, - tagMappingConfig, issueCategoryLookup, parsedData.fprInfo + tagMappingConfig, issueCategoryLookup, parsedData.fprInfo, parsedData.streamingFVDLProcessor, sourceEncodingOptions ); } - private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle) { + private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions) { try { // Processors now take the FprHandle directly, no more extracted path - AuditProcessor auditProcessor = new AuditProcessor(fprHandle); + AuditProcessor auditProcessor = new AuditProcessor(fprHandle, sourceEncodingOptions); //FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle); StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle); @@ -126,7 +131,8 @@ private static Map buildIssueCategoryLookup(List private static AuditOutcome performAviatorAudit( ParsedFprData parsedData, IAviatorLogger logger, String token, String appVersion, String url, String sscAppName, String sscAppVersion, - Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, List folderPriorityOrder) { + Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, + List folderPriorityOrder, SourceEncodingOptions sourceEncodingOptions) { SourceLanguageResolver sourceLanguageResolver = new SourceLanguageResolver(parsedData.streamingFVDLProcessor.getFvdlMetadata()); parsedData.streamingFVDLProcessor.getFvdlMetadata().clearSourceFileTypeIndexes(); @@ -141,7 +147,9 @@ private static AuditOutcome performAviatorAudit( filterSelection, logger, folderPriorityOrder, - sourceLanguageResolver + sourceLanguageResolver, + sourceEncodingOptions, + parsedData.streamingFVDLProcessor.getFvdlMetadata() ); return issueAuditor.performAudit( auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url, fprHandle @@ -151,7 +159,8 @@ private static AuditOutcome performAviatorAudit( private static FPRAuditResult finalizeFprAudit( AuditOutcome auditOutcome, Map auditResponses, AuditProcessor auditProcessor, TagMappingConfig tagMappingConfig, - Map issueCategoryLookup, FPRInfo fprInfo) { + Map issueCategoryLookup, FPRInfo fprInfo, StreamingFVDLProcessor streamingFVDLProcessor, + SourceEncodingOptions sourceEncodingOptions) { int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit(); if (auditResponses.isEmpty()) { @@ -192,7 +201,8 @@ private static FPRAuditResult finalizeFprAudit( File updatedFile = null; if (issuesSuccessfullyAudited > 0) { updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml( - auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo); + auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, + streamingFVDLProcessor.getFvdlMetadata(), sourceEncodingOptions); } LOG.info("FPR audit process completed with status: {}", status); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index d25c1286183..f53d6eeb78e 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -48,7 +48,9 @@ import com.fortify.cli.aviator.fpr.filter.VulnerabilityFilterer; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.grpc.AviatorGrpcClient; import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper; import com.fortify.cli.aviator.util.Constants; @@ -80,6 +82,8 @@ public class IssueAuditor { private TagDefinition humanAuditTag; private TagDefinition aviatorStatusTag; private final SourceLanguageResolver sourceLanguageResolver; + private final SourceEncodingOptions sourceEncodingOptions; + private final FVDLMetadata fvdlMetadata; private final IAviatorLogger logger; private final List customPriorityOrder; @@ -88,6 +92,15 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, SourceLanguageResolver sourceLanguageResolver) { + this(vulnerabilities, auditProcessor, auditIssueMap, fprInfo, SSCApplicationName, SSCApplicationVersion, + filterSelection, logger, customPriorityOrder, sourceLanguageResolver, SourceEncodingOptions.defaults(), null); + } + + public IssueAuditor(List vulnerabilities, AuditProcessor auditProcessor, Map auditIssueMap, + FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, + FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, + SourceLanguageResolver sourceLanguageResolver, SourceEncodingOptions sourceEncodingOptions, + FVDLMetadata fvdlMetadata) { this.logger = logger; this.customPriorityOrder = customPriorityOrder; this.MAX_PER_CATEGORY = Constants.MAX_PER_CATEGORY; @@ -103,6 +116,8 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro this.SSCApplicationName = SSCApplicationName; this.SSCApplicationVersion = SSCApplicationVersion; this.sourceLanguageResolver = sourceLanguageResolver; + this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.fvdlMetadata = fvdlMetadata; this.analysisTag = fprInfo.getFilterTemplate().getTagDefinitions().stream().filter(t -> "Analysis".equalsIgnoreCase(t.getName())).findFirst().orElse(null); this.resultsTag = resolveResultTag("", "", analysisTag); } @@ -162,7 +177,8 @@ public AuditOutcome performAudit(Map auditResponses, Stri } else { try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) { CompletableFuture> future = - client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token, fprHandle, customPriorityOrder); + client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, + SSCApplicationVersion, token, fprHandle, customPriorityOrder, sourceEncodingOptions, fvdlMetadata); Map responses = future.get(500, TimeUnit.MINUTES); responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response)); logger.progress("Audit completed"); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java index 54b8c803038..b97adb88bb7 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java @@ -14,6 +14,7 @@ import java.util.List; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.FprHandle; import lombok.Builder; @@ -34,4 +35,5 @@ public class AuditFprOptions { private final boolean noFilterSet; private final List folderNames; private final List folderPriorityOrder; + @Builder.Default private final SourceEncodingOptions sourceEncodingOptions = SourceEncodingOptions.defaults(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index 602367fbf03..dcbf0706702 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -59,7 +59,9 @@ import com.fortify.cli.aviator.config.TagMappingConfig; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -83,9 +85,15 @@ public class AuditProcessor { private final Map auditIssueMap = new HashMap<>(); private final FprHandle fprHandle; + private final SourceEncodingOptions sourceEncodingOptions; public AuditProcessor(FprHandle fprHandle) { + this(fprHandle, SourceEncodingOptions.defaults()); + } + + public AuditProcessor(FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions) { this.fprHandle = fprHandle; + this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; } /** @@ -709,6 +717,16 @@ private String addCommentToIssueElement(Element issueElement, String commentText public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, TagMappingConfig tagMappingConfig, Map issueCategoryLookup, FPRInfo fprInfo) throws AviatorTechnicalException { + return updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, null, + sourceEncodingOptions); + } + + public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, + TagMappingConfig tagMappingConfig, Map issueCategoryLookup, + FPRInfo fprInfo, FVDLMetadata fvdlMetadata, SourceEncodingOptions sourceEncodingOptions) throws AviatorTechnicalException { + SourceEncodingOptions effectiveSourceEncodingOptions = sourceEncodingOptions == null + ? this.sourceEncodingOptions + : sourceEncodingOptions; // Step 1: Apply this save's audit responses. writtenInstanceIds is the local retain set. Map effectiveIssueCategoryLookup = issueCategoryLookup == null ? Map.of() : issueCategoryLookup; AuditXmlUpdateResult updateResult = updateAuditXml( @@ -728,7 +746,8 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi // Step 4: Generate the in-memory remediations.xml document if needed. if (hasRemediations && !remediationCommentTimestamps.isEmpty()) { - this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo); + this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo, + fvdlMetadata, effectiveSourceEncodingOptions); } else { this.remediationsDoc = null; if (hasRemediations) { @@ -765,7 +784,8 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi private Document generateRemediationsXml(Map auditResponses, Map remediationCommentTimestamps, - FPRInfo fprInfo) throws AviatorTechnicalException { + FPRInfo fprInfo, FVDLMetadata fvdlMetadata, + SourceEncodingOptions sourceEncodingOptions) throws AviatorTechnicalException { try { DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -829,7 +849,7 @@ private Document generateRemediationsXml(Map auditRespons fileChangesElement.appendChild(filenameElement); //Optional originalFileContentOptional = fvdlProcessor.getSourceFileContent(filename); - FileUtils fileUtils = new FileUtils(); + FileUtils fileUtils = new FileUtils(sourceEncodingOptions, fvdlMetadata); Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); if (originalFileContentOptional.isEmpty()) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java index 97c1ffce863..ce12b574f9d 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java @@ -14,11 +14,7 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; -import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -49,6 +45,10 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions.SourceEncodeException; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.FuzzyContextSearcher; @@ -58,6 +58,7 @@ public class RemediationProcessor { private final FprHandle fprHandle; private final String sourceCodeDirectory; + private final SourceEncodingOptions sourceEncodingOptions; public record RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles, Map skippedByReason) { @@ -68,7 +69,10 @@ public RemediationMetric(int totalRemediations, int appliedRemediations, int ski private record FvdlMetadataResult(FVDLMetadata metadata, SkipReason skipReason) {} - private record PendingFileWrite(String filename, Path filePath, String content, byte[] updatedBytes) {} + private record SourceFileContent(String content, Charset charset, String encodingSource) {} + + private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, + byte[] updatedBytes) {} private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} @@ -136,8 +140,13 @@ private static class RollbackRemediationException extends AviatorTechnicalExcept } public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { + this(fprHandle, sourceCodeDirectory, SourceEncodingOptions.defaults()); + } + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, SourceEncodingOptions sourceEncodingOptions) { this.fprHandle = fprHandle; this.sourceCodeDirectory = sourceCodeDirectory; + this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; } public RemediationMetric processRemediationXML() { @@ -260,23 +269,25 @@ private boolean processFileChanges(Element remediation, Element fileChanges, Pat } String fileHash = getRequiredElementText(fileChanges, "Hash"); - Charset sourceEncoding = getRequiredSourceEncoding(filename, fvdlMetadataResult); NodeList changesNodes = fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); if (changesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); } - LOG.debug("Remediation {} has {} change(s) for '{}' using FVDL encoding {}", instanceId, changesNodes.getLength(), filename, - sourceEncoding.name()); + SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadataResult, pendingWrites); + Charset sourceEncoding = sourceFileContent.charset(); + LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, changesNodes.getLength(), filename, + sourceFileContent.encodingSource()); - String updatedContent = getPendingOrSourceContent(filePath, filename, sourceEncoding, pendingWrites); + String updatedContent = sourceFileContent.content(); for (int k = 0; k < changesNodes.getLength(); k++) { updatedContent = applyChange(instanceId, filename, fileHash, sourceEncoding, updatedContent, (Element) changesNodes.item(k), k + 1); } - byte[] updatedBytes = encodeStrict(updatedContent, sourceEncoding, filename); - pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, updatedBytes)); - LOG.debug("Staged remediation {} for '{}' using FVDL encoding {}; changes={}, encodedBytes={}", instanceId, filename, - sourceEncoding.name(), changesNodes.getLength(), updatedBytes.length); + byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); + pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, + sourceFileContent.encodingSource(), updatedBytes)); + LOG.debug("Staged remediation {} for '{}' using source encoding {}; changes={}, encodedBytes={}", instanceId, filename, + sourceFileContent.encodingSource(), changesNodes.getLength(), updatedBytes.length); return true; } @@ -334,10 +345,12 @@ private String applyChange(String instanceId, String filename, String fileHash, return String.join(lineSeparator, updatedLines); } - private String getPendingOrSourceContent(Path filePath, String filename, Charset sourceEncoding, + private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FvdlMetadataResult fvdlMetadataResult, Map pendingWrites) { PendingFileWrite pendingWrite = pendingWrites.get(filePath); - return pendingWrite == null ? readSourceFile(filePath, filename, sourceEncoding) : pendingWrite.content(); + return pendingWrite == null + ? readSourceFile(filePath, filename, fvdlMetadataResult) + : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); } private void commitRemediationWrites(String instanceId, Map pendingWrites, Set modifiedFiles) @@ -410,66 +423,26 @@ private FvdlMetadataResult loadFvdlMetadata() { } } - private Charset getRequiredSourceEncoding(String filename, FvdlMetadataResult fvdlMetadataResult) { - if (fvdlMetadataResult.skipReason() != null || fvdlMetadataResult.metadata() == null) { - throw new SkipRemediationException(SkipReason.FVDL_METADATA_UNAVAILABLE, - "FVDL metadata is unavailable; cannot determine source encoding for file '" + filename + "'"); - } - - String encoding = fvdlMetadataResult.metadata().findSourceFileEncodingForFileName(filename); - if (encoding == null || encoding.isBlank()) { - LOG.debug("FVDL source encoding lookup failed for '{}'", filename); - throw new SkipRemediationException(SkipReason.FVDL_ENCODING_MISSING, - "FVDL does not declare a source encoding for file '" + filename + "'"); - } - - try { - Charset charset = Charset.forName(encoding); - LOG.debug("FVDL source encoding for '{}' resolved to '{}'", filename, charset.name()); - return charset; - } catch (Exception e) { - throw new SkipRemediationException(SkipReason.FVDL_ENCODING_UNSUPPORTED, - "FVDL declares unsupported source encoding '" + encoding + "' for file '" + filename + "'", e); - } - } - - private String readSourceFile(Path filePath, String filename, Charset sourceEncoding) { + private SourceFileContent readSourceFile(Path filePath, String filename, FvdlMetadataResult fvdlMetadataResult) { try { byte[] sourceBytes = Files.readAllBytes(filePath); - String decodedContent = decodeStrict(sourceBytes, sourceEncoding); - LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, sourceEncoding.name(), sourceBytes.length, - decodedContent.length()); - return decodedContent; - } catch (CharacterCodingException e) { - throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, - "FVDL declares source encoding '" + sourceEncoding.name() + "' for file '" + filename + - "', but the source file cannot be decoded using that encoding", e); + FVDLMetadata fvdlMetadata = fvdlMetadataResult.skipReason() == null ? fvdlMetadataResult.metadata() : null; + DecodeResult decodeResult = sourceEncodingOptions.decode(sourceBytes, filename, fvdlMetadata); + LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, + decodeResult.content().length()); + return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); + } catch (SourceDecodeException e) { + throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, e.getMessage(), e); } catch (IOException e) { throw new SkipRemediationException(SkipReason.SOURCE_READ_FAILED, "Error reading source code file '" + filePath + "'", e); } } - private String decodeStrict(byte[] bytes, Charset charset) throws CharacterCodingException { - return charset.newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString(); - } - - private byte[] encodeStrict(String content, Charset charset, String filename) { + private byte[] encodeSourceFile(String content, Charset charset, String filename) { try { - ByteBuffer buffer = charset.newEncoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .encode(CharBuffer.wrap(content)); - byte[] result = new byte[buffer.remaining()]; - buffer.get(result); - return result; - } catch (CharacterCodingException e) { - throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, - "Remediation content for file '" + filename + "' cannot be encoded using FVDL source encoding '" + - charset.name() + "'", e); + return sourceEncodingOptions.encode(content, charset, filename); + } catch (SourceEncodeException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index 6ecd49aed6c..528f9d1680f 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -14,7 +14,6 @@ import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -28,6 +27,7 @@ import org.slf4j.LoggerFactory; import com.fortify.cli.aviator.audit.model.Fragment; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; import com.fortify.cli.aviator.util.FileUtil; import com.fortify.cli.aviator.util.FprHandle; @@ -37,6 +37,17 @@ public class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private final Map> fileContentCache = new ConcurrentHashMap<>(); + private final SourceEncodingOptions sourceEncodingOptions; + private final FVDLMetadata fvdlMetadata; + + public FileUtils() { + this(SourceEncodingOptions.defaults(), null); + } + + public FileUtils(SourceEncodingOptions sourceEncodingOptions, FVDLMetadata fvdlMetadata) { + this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.fvdlMetadata = fvdlMetadata; + } /** * Reads all lines from a file, caching the result to avoid repeated reads. @@ -46,12 +57,16 @@ public class FileUtils { * @return List of lines, or empty list if file not found or error occurs */ public List readFileWithFallback(Path filePath) { + return readFileWithFallback(filePath, filePath.getFileName().toString()); + } + + private List readFileWithFallback(Path filePath, String filename) { return fileContentCache.computeIfAbsent(filePath, path -> { try { byte[] fileBytes = Files.readAllBytes(path); - String content = new String(fileBytes, StandardCharsets.UTF_8); + String content = sourceEncodingOptions.decode(fileBytes, filename, fvdlMetadata).content(); return Arrays.asList(content.split("\\r?\\n")); - } catch (IOException e) { + } catch (IOException | SourceEncodingOptions.SourceDecodeException e) { logger.error("Failed to read file: {}", path, e); return Collections.emptyList(); } @@ -80,7 +95,7 @@ public String getLineFromFile(FprHandle fprHandle, String relativePath, int line Path fullSourcePath = resolveFullPath(fprHandle, relativePath); if (fullSourcePath == null) return ""; - List lines = readFileWithFallback(fullSourcePath); + List lines = readFileWithFallback(fullSourcePath, relativePath); if (lineNumber > 0 && lines.size() >= lineNumber) { return appendLineNumbers(lines.get(lineNumber - 1), relativePath, lineNumber - 1); } @@ -97,7 +112,7 @@ public Fragment getFragmentFromFile(FprHandle fprHandle, String relativePath, in return new Fragment("", 0, 0); } - List lines = readFileWithFallback(fullSourcePath); + List lines = readFileWithFallback(fullSourcePath, relativePath); if (lines.isEmpty() || lineNumber <= 0) { return new Fragment("", 0, 0); } @@ -136,7 +151,7 @@ public Optional getSourceFileContent(FprHandle fprHandle, String relativ } try { - return Optional.of(String.join(System.lineSeparator(), readFileWithFallback(actualSourcePath))); + return Optional.of(String.join(System.lineSeparator(), readFileWithFallback(actualSourcePath, relativePath))); } catch (Exception e) { logger.warn("Could not read source file content for path: {}", relativePath, e); return Optional.empty(); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index 57e28043765..d7c7801a5b0 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -24,6 +24,7 @@ import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -43,6 +44,8 @@ public class SourceCodeEnricher { private final Map sourceFileMap;*/ private final FprHandle fprHandle; private final FileUtils fileUtils; + private final SourceEncodingOptions sourceEncodingOptions; + private final FVDLMetadata fvdlMetadata; /** * Creates a new SourceCodeEnricher with the required dependencies. @@ -55,8 +58,14 @@ public class SourceCodeEnricher { }*/ public SourceCodeEnricher(FprHandle fprHandle){ + this(fprHandle, SourceEncodingOptions.defaults(), null); + } + + public SourceCodeEnricher(FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions, FVDLMetadata fvdlMetadata){ this.fprHandle = fprHandle; this.fileUtils = new FileUtils(); + this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.fvdlMetadata = fvdlMetadata; } /** @@ -135,10 +144,10 @@ private void processFileForElement(StackTraceElement element, Map try { if (Files.exists(actualSourcePath)) { byte[] encodedBytes = Files.readAllBytes(actualSourcePath); - String content = new String(encodedBytes); + String content = sourceEncodingOptions.decode(encodedBytes, filename, fvdlMetadata).content(); // Keep line markers in prompt file content; downstream gRPC/template rendering is pass-through. file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); - file.setEndLine(fileUtils.countLines(actualSourcePath)); + file.setEndLine(content.split("\\R", -1).length); } else { // This warning is now more accurate. logger.warn("Source file not found at internal path: {}. This may indicate a corrupt FPR.", actualSourcePath); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java new file mode 100644 index 00000000000..8418c48a489 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java @@ -0,0 +1,143 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +public final class SourceEncodingOptions { + public static final String FPR_ENCODING = "FPR"; + public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,CP850"; + + private final List candidates; + + private SourceEncodingOptions(List candidates) { + this.candidates = List.copyOf(candidates); + } + + public static SourceEncodingOptions defaults() { + return parse(DEFAULT_SOURCE_ENCODINGS); + } + + public static SourceEncodingOptions parse(String value) { + String effectiveValue = value == null || value.isBlank() ? DEFAULT_SOURCE_ENCODINGS : value; + List candidates = new ArrayList<>(); + for (String candidate : effectiveValue.split(",")) { + String trimmed = candidate.trim(); + if (!trimmed.isEmpty()) { + candidates.add(trimmed); + } + } + if (candidates.isEmpty()) { + candidates.add(FPR_ENCODING); + candidates.add("UTF-8"); + candidates.add("CP850"); + } + return new SourceEncodingOptions(candidates); + } + + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + List failures = new ArrayList<>(); + for (String candidate : candidates) { + Optional charset = resolveCharset(candidate, filename, fvdlMetadata, failures); + if (charset.isEmpty()) { + continue; + } + try { + String content = charset.get().newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + return new DecodeResult(content, charset.get(), displayName(candidate, charset.get())); + } catch (CharacterCodingException e) { + failures.add(displayName(candidate, charset.get()) + " failed to decode source bytes"); + } + } + throw new SourceDecodeException("Could not decode source file '" + filename + "' using source encodings " + describe() + + "; attempted: " + String.join("; ", failures)); + } + + public byte[] encode(String content, Charset charset, String filename) { + try { + ByteBuffer buffer = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(content)); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; + } catch (CharacterCodingException e) { + throw new SourceEncodeException("Source file '" + filename + "' cannot be encoded using " + charset.name(), e); + } + } + + public String describe() { + return String.join(",", candidates); + } + + private Optional resolveCharset(String candidate, String filename, FVDLMetadata fvdlMetadata, List failures) { + String encodingName = candidate; + if (FPR_ENCODING.equalsIgnoreCase(candidate)) { + if (fvdlMetadata == null) { + failures.add("FPR metadata unavailable"); + return Optional.empty(); + } + encodingName = fvdlMetadata.findSourceFileEncodingForFileName(filename); + if (encodingName == null || encodingName.isBlank()) { + failures.add("FPR encoding missing for '" + filename + "'"); + return Optional.empty(); + } + } + + try { + return Optional.of(Charset.forName(encodingName)); + } catch (Exception e) { + failures.add(candidate + " resolved to unsupported encoding '" + encodingName + "'"); + return Optional.empty(); + } + } + + private String displayName(String candidate, Charset charset) { + return FPR_ENCODING.equalsIgnoreCase(candidate) ? FPR_ENCODING + "(" + charset.name() + ")" : charset.name(); + } + + public record DecodeResult(String content, Charset charset, String source) {} + + public static class SourceDecodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceDecodeException(String message) { + super(message); + } + } + + public static class SourceEncodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceEncodeException(String message, Throwable cause) { + super(message, cause); + } + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java index 874e192e1f2..764d63e6f73 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java @@ -52,6 +52,8 @@ import com.fortify.cli.aviator.audit.model.AuditResponse; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.grpc.token.DeleteTokenRequest; @@ -123,8 +125,11 @@ public AviatorGrpcClient(ManagedChannel channel, long defaultTimeoutSeconds, IAv this(channel, defaultTimeoutSeconds, logger, 30); } - public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, List customPriorityOrder) { - AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, pingIntervalSeconds, defaultTimeoutSeconds, fprHandle); + public CompletableFuture> processBatchRequests(Queue requests, String projectName, + String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, + List customPriorityOrder, SourceEncodingOptions sourceEncodingOptions, FVDLMetadata fvdlMetadata) { + AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, + pingIntervalSeconds, defaultTimeoutSeconds, fprHandle, sourceEncodingOptions, fvdlMetadata); CompletableFuture> future = processor.processBatchRequests(requests, projectName, FPRBuildId, SSCApplicationName, SSCApplicationVersion, token, customPriorityOrder); future.whenComplete((res, th) -> processor.close()); return future.exceptionally(ex -> { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 89d376801f5..ec11a822cdd 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -48,7 +48,9 @@ import com.fortify.cli.aviator.audit.model.AuditResponse; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.utils.SourceCodeEnricher; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -92,8 +94,13 @@ class AviatorStreamProcessor implements AutoCloseable { private volatile Future processingTask; private final Object retryLock = new Object(); private final FprHandle fprHandle; + private final SourceEncodingOptions sourceEncodingOptions; + private final FVDLMetadata fvdlMetadata; - public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, long defaultTimeoutSeconds, FprHandle fprHandle) { + public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, + ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, + long defaultTimeoutSeconds, FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions, + FVDLMetadata fvdlMetadata) { this.client = client; this.logger = logger; this.asyncStub = asyncStub; @@ -102,6 +109,8 @@ public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, A this.pingIntervalSeconds = pingIntervalSeconds; this.defaultTimeoutSeconds = defaultTimeoutSeconds; this.fprHandle = fprHandle; + this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.fvdlMetadata = fvdlMetadata; } public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, List customPriorityOrder) { @@ -708,7 +717,7 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); // Lazy Loading of source code files for individual issue - SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle); + SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle, sourceEncodingOptions, fvdlMetadata); Map enrichedFiles = sourceCodeEnricher.enrichWithSourceCode(wrapper.userPrompt.getStackTrace()); diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java index 30fe3af0ae2..76bc4a70ffe 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java @@ -29,6 +29,7 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.ssc.cli.mixin.AviatorSSCApplyRemediationsArtifactSelectorMixin; import com.fortify.cli.aviator.ssc.helper.AviatorSSCApplyRemediationsHelper; import com.fortify.cli.aviator.ssc.helper.SinceOptionHelper; @@ -63,6 +64,8 @@ public class AviatorSSCApplyRemediationsCommand extends AbstractSSCJsonNodeOutpu private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCApplyRemediationsCommand.class); @Option(names = {"--source-dir"}, descriptionKey = "fcli.aviator.ssc.apply-remediations.source-dir") private String sourceCodeDirectory = System.getProperty("user.dir"); + @Option(names = {"--source-encodings"}, descriptionKey = "fcli.aviator.ssc.apply-remediations.source-encodings") + private String sourceEncodings = SourceEncodingOptions.DEFAULT_SOURCE_ENCODINGS; @Override @SneakyThrows @@ -128,7 +131,8 @@ JsonNode processAllAviatorArtifacts(OffsetDateTime sinceDate) { try { fprPath = downloadArtifactFpr(ad); try (FprHandle fprHandle = new FprHandle(fprPath)) { - var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + SourceEncodingOptions.parse(sourceEncodings), logger); totalRemediations += metric.totalRemediations(); appliedRemediations += metric.appliedRemediations(); skippedRemediations += metric.skippedRemediations(); @@ -181,7 +185,8 @@ JsonNode processFprRemediations(SSCArtifactDescriptor ad) { try { logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(fprPath)) { - var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + SourceEncodingOptions.parse(sourceEncodings), logger); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java index 2a4e73281df..c6d13356146 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java @@ -33,6 +33,7 @@ import com.fortify.cli.aviator.audit.model.FPRAuditResult; import com.fortify.cli.aviator.config.AviatorLoggerImpl; import com.fortify.cli.aviator.config.TagMappingConfig; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.ssc.helper.AviatorSSCAuditHelper; import com.fortify.cli.aviator.ssc.helper.AviatorSSCTagValidator; import com.fortify.cli.aviator.util.FprHandle; @@ -79,6 +80,8 @@ public class AviatorSSCAuditCommand extends AbstractSSCJsonNodeOutputCommand imp @ArgGroup(exclusive = true, multiplicity = "0..1") private QuotaHandlingArgGroup quotaHandlingArgGroup = new QuotaHandlingArgGroup(); @Option(names = {"--test-exceeding-quota"}) private boolean testExceedingQuota; @Option(names = {"--default-quota-fallback"}) private boolean defaultQuotaFallback; + @Option(names = {"--source-encodings"}, descriptionKey = "fcli.aviator.ssc.audit.source-encodings") + private String sourceEncodings = SourceEncodingOptions.DEFAULT_SOURCE_ENCODINGS; private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCAuditCommand.class); private Long checkedQuotaBefore; @@ -275,6 +278,7 @@ private JsonNode processFpr(UnirestInstance unirest, SSCAppVersionDescriptor av, .noFilterSet(isNoFilterSet()) .folderNames(folderNames) .folderPriorityOrder(getFolderPriorityOrder()) + .sourceEncodingOptions(SourceEncodingOptions.parse(sourceEncodings)) .build()); } catch (Exception e) { LOG.error("FPR audit failed for {}:{}: {}", av.getApplicationName(), av.getVersionName(), e.getMessage(), e); diff --git a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties index da3da97cfb2..532888c0fdc 100644 --- a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties +++ b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties @@ -129,6 +129,7 @@ fcli.aviator.ssc.audit.skip-if-exceeding-quota = Skip audit if the number of ope fcli.aviator.ssc.audit.test-exceeding-quota = Check whether the number of open issues exceeds the available Fortify Remediation Aviator quota and report the result without performing an audit. fcli.aviator.ssc.audit.default-quota-fallback = (Internal) When the Fortify Aviator application does not exist, use the tenant default quota instead of reporting app not found. Used by bulk audit. fcli.aviator.ssc.audit.folder-priority-order = Custom priority order for folder-based filtering when quota is exceeded (comma-separated, highest priority first). Example: Critical,High,Medium,Low. If not specified, uses default priority order. +fcli.aviator.ssc.audit.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files from the FPR. Use FPR to try the source encoding recorded in audit.fvdl. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.audit.refresh = By default, this command will refresh the source application version's metrics when copying from it. \ Note that for large applications this can lead to an error if the timeout expires. fcli.aviator.ssc.audit.refresh-timeout = Time-out, for example 30s (30 seconds), 5m (5 minutes), 1h (1 hour). Default value: ${DEFAULT-VALUE} @@ -156,6 +157,7 @@ fcli.aviator.ssc.apply-remediations.all = Apply remediations from all Fortify Re in chronological order. Aggregates remediation statistics across all artifacts. \ Requires --av/--appversion. Mutually exclusive with --artifact-id and --latest. fcli.aviator.ssc.apply-remediations.source-dir = Source code directory where remediations will be applied. Defaults to current directory. +fcli.aviator.ssc.apply-remediations.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. Remediated files are written using the accepted encoding. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.apply-remediations.since = Filter artifacts by upload date. Supports relative durations (e.g. 7d, 2w, 1M, 90d) \ or absolute dates (e.g. 2025-01-01, 2025-01-01T10:30:00, 2025-01-01T10:30:00Z). \ Can only be used with --latest or --all; not compatible with --artifact-id. diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java index f792edbef1e..c39bbba8f80 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; +import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; @@ -56,6 +57,8 @@ public class FoDAviatorApplyRemediationsCommand extends AbstractFoDJsonNodeOutpu @Mixin private FoDReleaseByQualifiedNameOrIdResolverMixin.RequiredOption releaseResolver; private static final Logger LOG = LoggerFactory.getLogger(FoDAviatorApplyRemediationsCommand.class); @Option(names = {"--source-dir"}) private String sourceCodeDirectory = System.getProperty("user.dir"); + @Option(names = {"--source-encodings"}, descriptionKey = "fcli.fod.aviator.apply-remediations.source-encodings") + private String sourceEncodings = SourceEncodingOptions.DEFAULT_SOURCE_ENCODINGS; @Override @SneakyThrows public JsonNode getJsonNode(UnirestInstance unirest) { @@ -82,7 +85,8 @@ private JsonNode processFprRemediations(UnirestInstance unirest, FoDReleaseDescr logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(downloadedFprPath)) { - var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + SourceEncodingOptions.parse(sourceEncodings), logger); LOG.info("Applied remediation {}", remediationMetric.appliedRemediations()); LOG.info("Total remediation {}", remediationMetric.totalRemediations()); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 62dcf5b6c6d..e9a97bb5749 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -1038,6 +1038,7 @@ fcli.fod.attribute.update.values = List of picklist values (only for Picklist da fcli.fod.aviator.usage.header = Use Fortify Remediation Aviator with FoD. fcli.fod.aviator.apply-remediations.usage.header = Apply Fortify Remediation Aviator auto-remediations to source code. fcli.fod.aviator.apply-remediations.source-dir = Directory containing source code to apply remediations to. Default value: ${DEFAULT-VALUE}. +fcli.fod.aviator.apply-remediations.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. Remediated files are written using the accepted encoding. Default value: ${DEFAULT-VALUE}. # various messages displayed during execution From 9a5395e2be270080d81b5ee5ed85bf753dcc1c15 Mon Sep 17 00:00:00 2001 From: Ankit Rathod Date: Fri, 31 Jul 2026 12:49:53 +0530 Subject: [PATCH 02/11] feat: Enhance audit processing with skipped issues tracking and remediation metrics --- .../fortify/cli/aviator/audit/AuditFPR.java | 61 ++++++++++++++++++- .../aviator/audit/model/FPRAuditResult.java | 17 ++++++ .../aviator/fpr/processor/AuditProcessor.java | 46 +++++++++++++- .../cli/aviator/fpr/utils/FileUtils.java | 5 +- .../aviator/fpr/utils/SourceCodeEnricher.java | 10 +-- .../ssc/helper/AviatorSSCAuditHelper.java | 24 +++++++- .../aviator/i18n/AviatorMessages.properties | 2 +- 7 files changed, 151 insertions(+), 14 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index e090ab863bb..9a6384038ee 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -14,6 +14,7 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -176,6 +177,8 @@ private static FPRAuditResult finalizeFprAudit( long issuesSuccessfullyAudited = auditResponses.values().stream() .filter(response -> "SUCCESS".equalsIgnoreCase(response.getStatus())) .count(); + Map skippedByReason = getSkippedAuditReasons(auditResponses, totalIssuesToAudit); + int issuesSkipped = skippedByReason.values().stream().mapToInt(Integer::intValue).sum(); String status; String message = null; @@ -204,8 +207,64 @@ private static FPRAuditResult finalizeFprAudit( auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, streamingFVDLProcessor.getFvdlMetadata(), sourceEncodingOptions); } + AuditProcessor.RemediationGenerationMetric remediationGenerationMetric = auditProcessor.getLastRemediationGenerationMetric(); + + if (!skippedByReason.isEmpty()) { + LOG.info("Skipped audit issues by reason: {}", skippedByReason); + } + if (!remediationGenerationMetric.skippedByReason().isEmpty()) { + LOG.info("Skipped audit remediation generation by reason: {}", remediationGenerationMetric.skippedByReason()); + } LOG.info("FPR audit process completed with status: {}", status); - return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit); + return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit, + issuesSkipped, skippedByReason, remediationGenerationMetric.skippedRemediations(), + remediationGenerationMetric.skippedByReason()); + } + + private static Map getSkippedAuditReasons(Map auditResponses, int totalIssuesToAudit) { + Map skippedByReason = new LinkedHashMap<>(); + auditResponses.values().stream() + .filter(response -> !"SUCCESS".equalsIgnoreCase(response.getStatus())) + .map(AuditFPR::getSkippedAuditReason) + .forEach(reason -> recordSkipped(skippedByReason, reason)); + int missingResponses = Math.max(0, totalIssuesToAudit - auditResponses.size()); + if (missingResponses > 0) { + skippedByReason.merge("No audit response received", missingResponses, Integer::sum); + } + return skippedByReason; + } + + private static String getSkippedAuditReason(AuditResponse response) { + String statusMessage = response == null ? null : response.getStatusMessage(); + String message = statusMessage == null || statusMessage.isBlank() + ? response == null ? null : response.getStatus() + : statusMessage; + if (message == null || message.isBlank()) { + return "Unknown audit failure"; + } + if (message.startsWith("Client-side pre-processing error: ")) { + message = message.substring("Client-side pre-processing error: ".length()); + } + if (message.startsWith("Could not decode source file")) { + return "Source file decode failed"; + } + if (message.contains("was not found in the FPR")) { + return "Source file not found in FPR"; + } + if (message.contains("could not be read from the FPR")) { + return "Source file read failed"; + } + if ("FAILED".equalsIgnoreCase(message)) { + return "Audit failed"; + } + if ("SKIPPED".equalsIgnoreCase(message)) { + return "Skipped by Aviator"; + } + return message; + } + + private static void recordSkipped(Map skippedByReason, String reason) { + skippedByReason.merge(reason, 1, Integer::sum); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java index 94d8b9a4e18..e2f55d51b62 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java @@ -13,6 +13,7 @@ package com.fortify.cli.aviator.audit.model; import java.io.File; +import java.util.Map; import lombok.Data; @@ -23,13 +24,29 @@ public class FPRAuditResult { private String message; private int issuesSuccessfullyAudited; private int totalIssuesToAudit; + private int issuesSkipped; + private Map skippedByReason; + private int remediationGenerationSkipped; + private Map remediationGenerationSkippedByReason; public FPRAuditResult(File updatedFile, String status, String message, int issuesSuccessfullyAudited, int totalIssuesToAudit) { + this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, + Math.max(0, totalIssuesToAudit - issuesSuccessfullyAudited), Map.of(), 0, Map.of()); + } + + public FPRAuditResult(File updatedFile, String status, String message, + int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSkipped, + Map skippedByReason, int remediationGenerationSkipped, + Map remediationGenerationSkippedByReason) { this.updatedFile = updatedFile; this.status = status; this.message = message; this.issuesSuccessfullyAudited = issuesSuccessfullyAudited; this.totalIssuesToAudit = totalIssuesToAudit; + this.issuesSkipped = issuesSkipped; + this.skippedByReason = skippedByReason == null ? Map.of() : Map.copyOf(skippedByReason); + this.remediationGenerationSkipped = remediationGenerationSkipped; + this.remediationGenerationSkippedByReason = remediationGenerationSkippedByReason == null ? Map.of() : Map.copyOf(remediationGenerationSkippedByReason); } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index dcbf0706702..16d70d252ed 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -27,6 +27,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -86,6 +87,26 @@ public class AuditProcessor { private final Map auditIssueMap = new HashMap<>(); private final FprHandle fprHandle; private final SourceEncodingOptions sourceEncodingOptions; + private RemediationGenerationMetric lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); + + public record RemediationGenerationMetric(int skippedRemediations, Map skippedByReason) { + public static RemediationGenerationMetric empty() { + return new RemediationGenerationMetric(0, Map.of()); + } + } + + private enum RemediationSkipReason { + SOURCE_READ_OR_DECODE_FAILED("Source file read/decode failed"), + INVALID_LINE_NUMBER("Invalid line number"), + STRUCTURALLY_INVALID("Structurally invalid remediation"), + NO_VALID_CHANGES("No valid changes"); + + private final String label; + + RemediationSkipReason(String label) { + this.label = label; + } + } public AuditProcessor(FprHandle fprHandle) { this(fprHandle, SourceEncodingOptions.defaults()); @@ -96,6 +117,10 @@ public AuditProcessor(FprHandle fprHandle, SourceEncodingOptions sourceEncodingO this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; } + public RemediationGenerationMetric getLastRemediationGenerationMetric() { + return lastRemediationGenerationMetric; + } + /** * Per-save result of applying Aviator responses to the in-memory {@code audit.xml} DOM. * {@code writtenInstanceIds} is the local retain set for upload isolation (no processor state). @@ -727,6 +752,7 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi SourceEncodingOptions effectiveSourceEncodingOptions = sourceEncodingOptions == null ? this.sourceEncodingOptions : sourceEncodingOptions; + lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); // Step 1: Apply this save's audit responses. writtenInstanceIds is the local retain set. Map effectiveIssueCategoryLookup = issueCategoryLookup == null ? Map.of() : issueCategoryLookup; AuditXmlUpdateResult updateResult = updateAuditXml( @@ -746,8 +772,10 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi // Step 4: Generate the in-memory remediations.xml document if needed. if (hasRemediations && !remediationCommentTimestamps.isEmpty()) { + Map skippedByReason = new LinkedHashMap<>(); this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo, - fvdlMetadata, effectiveSourceEncodingOptions); + fvdlMetadata, effectiveSourceEncodingOptions, skippedByReason); + lastRemediationGenerationMetric = toRemediationGenerationMetric(skippedByReason); } else { this.remediationsDoc = null; if (hasRemediations) { @@ -785,7 +813,8 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi private Document generateRemediationsXml(Map auditResponses, Map remediationCommentTimestamps, FPRInfo fprInfo, FVDLMetadata fvdlMetadata, - SourceEncodingOptions sourceEncodingOptions) throws AviatorTechnicalException { + SourceEncodingOptions sourceEncodingOptions, + Map skippedByReason) throws AviatorTechnicalException { try { DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -854,6 +883,7 @@ private Document generateRemediationsXml(Map auditRespons if (originalFileContentOptional.isEmpty()) { logger.warn("WARN: Could not retrieve source code for file '{}'. Skipping remediation generation for this file for instanceId '{}'.", filename, instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.SOURCE_READ_OR_DECODE_FAILED); continue; } @@ -920,6 +950,7 @@ private Document generateRemediationsXml(Map auditRespons fileChangesElement.appendChild(changeElement); } catch (NumberFormatException e) { logger.error("Skipping change for issue {} due to invalid line number format. Details: {}", instanceId, e.getMessage()); + recordSkipped(skippedByReason, RemediationSkipReason.INVALID_LINE_NUMBER); } } if (fileChangesElement.getElementsByTagNameNS(REMEDIATIONS_NAMESPACE_URI, "Change").getLength() > 0) { @@ -935,9 +966,11 @@ private Document generateRemediationsXml(Map auditRespons validRemediationCount++; } else { logger.warn("WARN: Skipping structurally invalid remediation for issue instanceId: {}", instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.STRUCTURALLY_INVALID); } } else { logger.warn("WARN: Skipping remediation for instanceId '{}' because all of its proposed changes were invalid and could not be processed.", instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.NO_VALID_CHANGES); } } @@ -949,6 +982,15 @@ private Document generateRemediationsXml(Map auditRespons } } + private RemediationGenerationMetric toRemediationGenerationMetric(Map skippedByReason) { + int skippedRemediations = skippedByReason.values().stream().mapToInt(Integer::intValue).sum(); + return new RemediationGenerationMetric(skippedRemediations, Map.copyOf(skippedByReason)); + } + + private void recordSkipped(Map skippedByReason, RemediationSkipReason reason) { + skippedByReason.merge(reason.label, 1, Integer::sum); + } + private String calculateHashBase64(String content, String algorithm) { if (content == null) return ""; try { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index 528f9d1680f..ec13c6ccba4 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -151,8 +151,9 @@ public Optional getSourceFileContent(FprHandle fprHandle, String relativ } try { - return Optional.of(String.join(System.lineSeparator(), readFileWithFallback(actualSourcePath, relativePath))); - } catch (Exception e) { + byte[] fileBytes = Files.readAllBytes(actualSourcePath); + return Optional.of(sourceEncodingOptions.decode(fileBytes, relativePath, fvdlMetadata).content()); + } catch (IOException | SourceEncodingOptions.SourceDecodeException e) { logger.warn("Could not read source file content for path: {}", relativePath, e); return Optional.empty(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index d7c7801a5b0..aa6631dc6a5 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.StackTraceElement; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; @@ -149,15 +150,10 @@ private void processFileForElement(StackTraceElement element, Map file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); file.setEndLine(content.split("\\R", -1).length); } else { - // This warning is now more accurate. - logger.warn("Source file not found at internal path: {}. This may indicate a corrupt FPR.", actualSourcePath); - file.setContent(""); - file.setEndLine(0); + throw new AviatorSimpleException("Source file '" + filename + "' was not found in the FPR"); } } catch (IOException e) { - logger.warn("Error processing file: {}", filename, e); - file.setContent(""); - file.setEndLine(0); + throw new AviatorSimpleException("Source file '" + filename + "' could not be read from the FPR", e); } uniqueFiles.put(filename, file); } diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java index 542ee85bb86..2596476ec1d 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java @@ -137,10 +137,32 @@ public static void setAuditStats(ObjectNode result, FPRAuditResult auditResult) audit.put("message", message); audit.put("submitted", auditResult.getTotalIssuesToAudit()); audit.put("succeeded", auditResult.getIssuesSuccessfullyAudited()); - audit.put("skipped", Math.max(0, auditResult.getTotalIssuesToAudit() - auditResult.getIssuesSuccessfullyAudited())); + audit.put("skipped", auditResult.getIssuesSkipped()); + audit.put("skippedReasons", formatSkippedReasons(auditResult.getSkippedByReason())); + audit.set("skippedByReason", toObjectNode(auditResult.getSkippedByReason())); + audit.put("remediationGenerationSkipped", auditResult.getRemediationGenerationSkipped()); + audit.put("remediationGenerationSkippedReasons", formatSkippedReasons(auditResult.getRemediationGenerationSkippedByReason())); + audit.set("remediationGenerationSkippedByReason", toObjectNode(auditResult.getRemediationGenerationSkippedByReason())); ((ObjectNode) result.get("operation")).set("audit", audit); } + private static ObjectNode toObjectNode(Map skippedByReason) { + ObjectNode object = JsonHelper.getObjectMapper().createObjectNode(); + if (skippedByReason != null) { + skippedByReason.forEach(object::put); + } + return object; + } + + private static String formatSkippedReasons(Map skippedByReason) { + if (skippedByReason == null || skippedByReason.isEmpty()) { + return ""; + } + List parts = new ArrayList<>(); + skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); + return String.join(", ", parts); + } + /** * Sets only the {@code operation.audit.message} field without audit stats. * Used for code paths that don't perform an actual audit (SKIPPED, FAILED, QUOTA_EXCEEDED, etc.). diff --git a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties index 532888c0fdc..781ed10f230 100644 --- a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties +++ b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties @@ -206,7 +206,7 @@ fcli.env.default.prefix=FCLI_DEFAULT # Table output columns configuration fcli.aviator.session.output.table.args = name,type,url,created,expires,expired fcli.aviator.admin-config.output.table.args = name,type,url,created -fcli.aviator.ssc.audit.output.table.args = id,application.name,name,artifactId,action +fcli.aviator.ssc.audit.output.table.args = id,application.name,name,artifactId,action,operation.audit.skippedReasons fcli.aviator.app.create.output.table.args = id,name,entitlement_id,disclaimer,quota_last_updated,quota fcli.aviator.app.add-entitlement.output.table.args = id,name,entitlement_id,entitlement_multiplier,entitlements_consumed,quota_last_updated,quota fcli.aviator.app.delete.output.table.args = message From 524228c5d3277c220d9d83966c0b9cc41ca7973a Mon Sep 17 00:00:00 2001 From: kireetivar Date: Sun, 2 Aug 2026 22:03:52 +0530 Subject: [PATCH 03/11] refactor: `fcli aviator`: Replace SourceEncodingOptions with ISourceDecoder hierarchy --- .../cli/converter/SourceDecoderConverter.java | 36 +++++ .../cli/mixin/SourceEncodingsMixin.java | 44 ++++++ .../ApplyAutoRemediationOnSource.java | 17 ++- .../fortify/cli/aviator/audit/AuditFPR.java | 29 ++-- .../cli/aviator/audit/IssueAuditor.java | 14 +- .../aviator/audit/model/AuditFprOptions.java | 5 +- .../aviator/fpr/processor/AuditProcessor.java | 28 ++-- .../fpr/processor/RemediationProcessor.java | 68 ++++----- .../fpr/processor/StreamingFVDLProcessor.java | 19 ++- .../fpr/utils/CharsetSourceDecoder.java | 45 ++++++ .../fpr/utils/CompositeSourceDecoder.java | 53 +++++++ .../cli/aviator/fpr/utils/FileUtils.java | 21 ++- .../aviator/fpr/utils/FprSourceDecoder.java | 57 +++++++ .../cli/aviator/fpr/utils/ISourceDecoder.java | 55 +++++++ .../aviator/fpr/utils/SourceCodeEnricher.java | 59 ++------ .../cli/aviator/fpr/utils/SourceDecoders.java | 82 ++++++++++ .../cli/aviator/fpr/utils/SourceEncoder.java | 64 ++++++++ .../fpr/utils/SourceEncodingOptions.java | 143 ------------------ .../cli/aviator/grpc/AviatorGrpcClient.java | 6 +- .../aviator/grpc/AviatorStreamProcessor.java | 15 +- .../converter/SourceDecoderConverterTest.java | 41 +++++ .../aviator/fpr/utils/SourceDecodersTest.java | 102 +++++++++++++ .../AviatorSSCApplyRemediationsCommand.java | 9 +- .../ssc/cli/cmd/AviatorSSCAuditCommand.java | 7 +- .../aviator/i18n/AviatorMessages.properties | 3 +- .../FoDAviatorApplyRemediationsCommand.java | 7 +- .../cli/fod/i18n/FoDMessages.properties | 2 +- 27 files changed, 725 insertions(+), 306 deletions(-) create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java create mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java delete mode 100644 fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java new file mode 100644 index 00000000000..083d74bbfe1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.converter; + +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; + +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +/** + * Picocli adapter: maps a single {@code --source-encodings} token to an + * {@link ISourceDecoder} via the domain factory {@link SourceDecoders}. + */ +public final class SourceDecoderConverter implements ITypeConverter { + @Override + public ISourceDecoder convert(String value) { + try { + return SourceDecoders.fromToken(value); + } catch (IllegalArgumentException e) { + // Covers blank tokens, IllegalCharsetNameException, UnsupportedCharsetException + throw new TypeConversionException( + e.getMessage() != null ? e.getMessage() : "Invalid source encoding '" + value + "'"); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java new file mode 100644 index 00000000000..e58f0e86d0d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.mixin; + +import java.util.List; + +import com.fortify.cli.aviator._common.cli.converter.SourceDecoderConverter; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; + +import lombok.Getter; +import picocli.CommandLine.Option; + +/** + * Shared {@code --source-encodings} option for Aviator commands that decode + * (and optionally re-encode) source files from an FPR. + */ +public class SourceEncodingsMixin { + @Getter + @Option(names = {"--source-encodings"}, + split = ",", + converter = SourceDecoderConverter.class, + defaultValue = SourceDecoders.DEFAULT_SOURCE_ENCODINGS, + paramLabel = "encoding", + descriptionKey = "fcli.aviator.source-encodings") + private List sourceDecoders; + + /** + * Returns a single decoder that tries the configured candidates in order. + */ + public ISourceDecoder getSourceDecoder() { + return SourceDecoders.of(sourceDecoders); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java index 3788d5ce0a9..23b77f0119a 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java @@ -12,6 +12,8 @@ */ package com.fortify.cli.aviator.applyRemediation; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +22,8 @@ import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor.RemediationMetric; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; @@ -29,11 +32,11 @@ public class ApplyAutoRemediationOnSource { public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, IAviatorLogger logger) throws AviatorSimpleException, AviatorTechnicalException { - return applyRemediations(fprHandle, sourceCodeDirectory, SourceEncodingOptions.defaults(), logger); - } + return applyRemediations(fprHandle, sourceCodeDirectory, SourceDecoders.defaults(), logger); + } - public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, - SourceEncodingOptions sourceEncodingOptions, IAviatorLogger logger) + public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, + ISourceDecoder sourceDecoder, IAviatorLogger logger) throws AviatorSimpleException, AviatorTechnicalException { LOG.info("Starting apply auto-remediation process for file: {}", fprHandle.getFprPath()); @@ -44,8 +47,8 @@ public static RemediationMetric applyRemediations(FprHandle fprHandle, String so } LOG.info("FPR validation successful"); - RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory, sourceEncodingOptions); + RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory, + Objects.requireNonNull(sourceDecoder, "sourceDecoder")); return remediationProcessor.processRemediationXML(); - } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 9a6384038ee..a89dc206597 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -40,7 +40,7 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.ResourceUtil; @@ -54,12 +54,11 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) options.getFprHandle().validate(); AviatorConfigManager.getInstance(); - SourceEncodingOptions sourceEncodingOptions = options.getSourceEncodingOptions() == null - ? SourceEncodingOptions.defaults() - : options.getSourceEncodingOptions(); + // Non-null: AuditFprOptions defaults via @Builder.Default; CLI mixin always supplies a decoder. + ISourceDecoder sourceDecoder = options.getSourceDecoder(); // --- STAGE 1: PARSING --- - ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle(), sourceEncodingOptions); + ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle(), sourceDecoder); TagMappingConfig tagMappingConfig = loadTagMappingConfig(options.getTagMappingPath()); Map issueCategoryLookup = tagMappingConfig.requiresCategoryForSuppressionEvaluation() ? buildIssueCategoryLookup(parsedData.vulnerabilities) @@ -75,22 +74,21 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) Map auditResponses = new ConcurrentHashMap<>(); AuditOutcome auditOutcome = performAviatorAudit( parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(), - auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder(), sourceEncodingOptions + auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder(), sourceDecoder ); // --- STAGE 4: FINALIZATION --- return finalizeFprAudit( auditOutcome, auditResponses, parsedData.auditProcessor, - tagMappingConfig, issueCategoryLookup, parsedData.fprInfo, parsedData.streamingFVDLProcessor, sourceEncodingOptions + tagMappingConfig, issueCategoryLookup, parsedData.fprInfo, parsedData.streamingFVDLProcessor ); } - private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions) { + private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle, ISourceDecoder sourceDecoder) { try { // Processors now take the FprHandle directly, no more extracted path - AuditProcessor auditProcessor = new AuditProcessor(fprHandle, sourceEncodingOptions); - //FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle); - StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle); + AuditProcessor auditProcessor = new AuditProcessor(fprHandle, sourceDecoder); + StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); Map auditIssueMap = auditProcessor.processAuditXML(); FPRProcessor fprProcessor = new FPRProcessor(fprHandle, auditIssueMap, auditProcessor); @@ -133,7 +131,7 @@ private static AuditOutcome performAviatorAudit( ParsedFprData parsedData, IAviatorLogger logger, String token, String appVersion, String url, String sscAppName, String sscAppVersion, Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, - List folderPriorityOrder, SourceEncodingOptions sourceEncodingOptions) { + List folderPriorityOrder, ISourceDecoder sourceDecoder) { SourceLanguageResolver sourceLanguageResolver = new SourceLanguageResolver(parsedData.streamingFVDLProcessor.getFvdlMetadata()); parsedData.streamingFVDLProcessor.getFvdlMetadata().clearSourceFileTypeIndexes(); @@ -149,7 +147,7 @@ private static AuditOutcome performAviatorAudit( logger, folderPriorityOrder, sourceLanguageResolver, - sourceEncodingOptions, + sourceDecoder, parsedData.streamingFVDLProcessor.getFvdlMetadata() ); return issueAuditor.performAudit( @@ -160,8 +158,7 @@ private static AuditOutcome performAviatorAudit( private static FPRAuditResult finalizeFprAudit( AuditOutcome auditOutcome, Map auditResponses, AuditProcessor auditProcessor, TagMappingConfig tagMappingConfig, - Map issueCategoryLookup, FPRInfo fprInfo, StreamingFVDLProcessor streamingFVDLProcessor, - SourceEncodingOptions sourceEncodingOptions) { + Map issueCategoryLookup, FPRInfo fprInfo, StreamingFVDLProcessor streamingFVDLProcessor) { int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit(); if (auditResponses.isEmpty()) { @@ -205,7 +202,7 @@ private static FPRAuditResult finalizeFprAudit( if (issuesSuccessfullyAudited > 0) { updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml( auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, - streamingFVDLProcessor.getFvdlMetadata(), sourceEncodingOptions); + streamingFVDLProcessor.getFvdlMetadata()); } AuditProcessor.RemediationGenerationMetric remediationGenerationMetric = auditProcessor.getLastRemediationGenerationMetric(); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index f53d6eeb78e..dc5b6897892 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -50,7 +51,8 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.grpc.AviatorGrpcClient; import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper; import com.fortify.cli.aviator.util.Constants; @@ -82,7 +84,7 @@ public class IssueAuditor { private TagDefinition humanAuditTag; private TagDefinition aviatorStatusTag; private final SourceLanguageResolver sourceLanguageResolver; - private final SourceEncodingOptions sourceEncodingOptions; + private final ISourceDecoder sourceDecoder; private final FVDLMetadata fvdlMetadata; private final IAviatorLogger logger; @@ -93,13 +95,13 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, SourceLanguageResolver sourceLanguageResolver) { this(vulnerabilities, auditProcessor, auditIssueMap, fprInfo, SSCApplicationName, SSCApplicationVersion, - filterSelection, logger, customPriorityOrder, sourceLanguageResolver, SourceEncodingOptions.defaults(), null); + filterSelection, logger, customPriorityOrder, sourceLanguageResolver, SourceDecoders.defaults(), null); } public IssueAuditor(List vulnerabilities, AuditProcessor auditProcessor, Map auditIssueMap, FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, - SourceLanguageResolver sourceLanguageResolver, SourceEncodingOptions sourceEncodingOptions, + SourceLanguageResolver sourceLanguageResolver, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { this.logger = logger; this.customPriorityOrder = customPriorityOrder; @@ -116,7 +118,7 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro this.SSCApplicationName = SSCApplicationName; this.SSCApplicationVersion = SSCApplicationVersion; this.sourceLanguageResolver = sourceLanguageResolver; - this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); this.fvdlMetadata = fvdlMetadata; this.analysisTag = fprInfo.getFilterTemplate().getTagDefinitions().stream().filter(t -> "Analysis".equalsIgnoreCase(t.getName())).findFirst().orElse(null); this.resultsTag = resolveResultTag("", "", analysisTag); @@ -178,7 +180,7 @@ public AuditOutcome performAudit(Map auditResponses, Stri try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) { CompletableFuture> future = client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, - SSCApplicationVersion, token, fprHandle, customPriorityOrder, sourceEncodingOptions, fvdlMetadata); + SSCApplicationVersion, token, fprHandle, customPriorityOrder, sourceDecoder, fvdlMetadata); Map responses = future.get(500, TimeUnit.MINUTES); responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response)); logger.progress("Audit completed"); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java index b97adb88bb7..8fa13198222 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java @@ -14,7 +14,8 @@ import java.util.List; import com.fortify.cli.aviator.config.IAviatorLogger; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; import lombok.Builder; @@ -35,5 +36,5 @@ public class AuditFprOptions { private final boolean noFilterSet; private final List folderNames; private final List folderPriorityOrder; - @Builder.Default private final SourceEncodingOptions sourceEncodingOptions = SourceEncodingOptions.defaults(); + @Builder.Default private final ISourceDecoder sourceDecoder = SourceDecoders.defaults(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index 16d70d252ed..d919ff321b6 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -30,6 +30,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -62,7 +63,8 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.utils.FileUtils; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -86,7 +88,7 @@ public class AuditProcessor { private final Map auditIssueMap = new HashMap<>(); private final FprHandle fprHandle; - private final SourceEncodingOptions sourceEncodingOptions; + private final ISourceDecoder sourceDecoder; private RemediationGenerationMetric lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); public record RemediationGenerationMetric(int skippedRemediations, Map skippedByReason) { @@ -109,12 +111,12 @@ private enum RemediationSkipReason { } public AuditProcessor(FprHandle fprHandle) { - this(fprHandle, SourceEncodingOptions.defaults()); + this(fprHandle, SourceDecoders.defaults()); } - public AuditProcessor(FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions) { + public AuditProcessor(FprHandle fprHandle, ISourceDecoder sourceDecoder) { this.fprHandle = fprHandle; - this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); } public RemediationGenerationMetric getLastRemediationGenerationMetric() { @@ -742,16 +744,12 @@ private String addCommentToIssueElement(Element issueElement, String commentText public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, TagMappingConfig tagMappingConfig, Map issueCategoryLookup, FPRInfo fprInfo) throws AviatorTechnicalException { - return updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, null, - sourceEncodingOptions); + return updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, null); } public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, TagMappingConfig tagMappingConfig, Map issueCategoryLookup, - FPRInfo fprInfo, FVDLMetadata fvdlMetadata, SourceEncodingOptions sourceEncodingOptions) throws AviatorTechnicalException { - SourceEncodingOptions effectiveSourceEncodingOptions = sourceEncodingOptions == null - ? this.sourceEncodingOptions - : sourceEncodingOptions; + FPRInfo fprInfo, FVDLMetadata fvdlMetadata) throws AviatorTechnicalException { lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); // Step 1: Apply this save's audit responses. writtenInstanceIds is the local retain set. Map effectiveIssueCategoryLookup = issueCategoryLookup == null ? Map.of() : issueCategoryLookup; @@ -774,7 +772,7 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi if (hasRemediations && !remediationCommentTimestamps.isEmpty()) { Map skippedByReason = new LinkedHashMap<>(); this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo, - fvdlMetadata, effectiveSourceEncodingOptions, skippedByReason); + fvdlMetadata, skippedByReason); lastRemediationGenerationMetric = toRemediationGenerationMetric(skippedByReason); } else { this.remediationsDoc = null; @@ -813,7 +811,6 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi private Document generateRemediationsXml(Map auditResponses, Map remediationCommentTimestamps, FPRInfo fprInfo, FVDLMetadata fvdlMetadata, - SourceEncodingOptions sourceEncodingOptions, Map skippedByReason) throws AviatorTechnicalException { try { DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); @@ -843,6 +840,7 @@ private Document generateRemediationsXml(Map auditRespons Element remediationListElement = finalDoc.createElementNS(REMEDIATIONS_NAMESPACE_URI, "RemediationList"); rootElement.appendChild(remediationListElement); + FileUtils fileUtils = new FileUtils(this.sourceDecoder, fvdlMetadata); int validRemediationCount = 0; for (Map.Entry entry : auditResponses.entrySet()) { @@ -877,9 +875,7 @@ private Document generateRemediationsXml(Map auditRespons filenameElement.setTextContent(filename); fileChangesElement.appendChild(filenameElement); - //Optional originalFileContentOptional = fvdlProcessor.getSourceFileContent(filename); - FileUtils fileUtils = new FileUtils(sourceEncodingOptions, fvdlMetadata); - Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); + Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); if (originalFileContentOptional.isEmpty()) { logger.warn("WARN: Could not retrieve source code for file '{}'. Skipping remediation generation for this file for instanceId '{}'.", filename, instanceId); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java index ce12b574f9d..2a059b8380b 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java @@ -28,6 +28,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.zip.ZipFile; @@ -45,10 +46,12 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions.DecodeResult; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions.SourceDecodeException; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions.SourceEncodeException; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.FuzzyContextSearcher; @@ -58,7 +61,7 @@ public class RemediationProcessor { private final FprHandle fprHandle; private final String sourceCodeDirectory; - private final SourceEncodingOptions sourceEncodingOptions; + private final ISourceDecoder sourceDecoder; public record RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles, Map skippedByReason) { @@ -67,8 +70,6 @@ public RemediationMetric(int totalRemediations, int appliedRemediations, int ski } } - private record FvdlMetadataResult(FVDLMetadata metadata, SkipReason skipReason) {} - private record SourceFileContent(String content, Charset charset, String encodingSource) {} private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, @@ -77,9 +78,6 @@ private record PendingFileWrite(String filename, Path filePath, String content, private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} private enum SkipReason { - FVDL_METADATA_UNAVAILABLE("FVDL metadata unavailable"), - FVDL_ENCODING_MISSING("FVDL source encoding missing"), - FVDL_ENCODING_UNSUPPORTED("FVDL source encoding unsupported"), SOURCE_FILE_MISSING("Source file missing"), SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), SOURCE_READ_FAILED("Source file read failed"), @@ -140,13 +138,13 @@ private static class RollbackRemediationException extends AviatorTechnicalExcept } public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { - this(fprHandle, sourceCodeDirectory, SourceEncodingOptions.defaults()); + this(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); } - public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, SourceEncodingOptions sourceEncodingOptions) { + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, ISourceDecoder sourceDecoder) { this.fprHandle = fprHandle; this.sourceCodeDirectory = sourceCodeDirectory; - this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); } public RemediationMetric processRemediationXML() { @@ -166,7 +164,7 @@ public RemediationMetric processRemediationXML() { } final Path sourceBasePath = Paths.get(trimmedSourceDir).toAbsolutePath().normalize(); LOG.debug("Applying remediations from {} to source directory {}", remediationPath, sourceBasePath); - final FvdlMetadataResult fvdlMetadataResult = loadFvdlMetadata(); + final FVDLMetadata fvdlMetadata = loadFvdlMetadata(); try (InputStream remediationStream = Files.newInputStream(remediationPath)) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -185,7 +183,7 @@ public RemediationMetric processRemediationXML() { appliedRemediations = 0; for (int i = 0; i < remediationNodes.getLength(); i++) { Element remediation = (Element) remediationNodes.item(i); - if (processRemediation(remediation, sourceBasePath, fvdlMetadataResult, modifiedFiles, skippedByReason)) { + if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason)) { appliedRemediations++; } } @@ -207,11 +205,11 @@ public RemediationMetric processRemediationXML() { return new RemediationMetric(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, skippedByReason); } - private boolean processRemediation(Element remediation, Path sourceBasePath, FvdlMetadataResult fvdlMetadataResult, + private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Set modifiedFiles, Map skippedByReason) { String instanceId = remediation.getAttribute("instanceId"); try { - Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadataResult); + Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata); if (pendingWrites.isEmpty()) { recordSkipped(skippedByReason, SkipReason.NO_CHANGES); return false; @@ -239,7 +237,7 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, Fvd } private Map prepareFileChanges(Element remediation, Path sourceBasePath, - FvdlMetadataResult fvdlMetadataResult) { + FVDLMetadata fvdlMetadata) { NodeList fileChangesNodes = remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); if (fileChangesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); @@ -247,12 +245,12 @@ private Map prepareFileChanges(Element remediation, Path Map pendingWrites = new LinkedHashMap<>(); for (int j = 0; j < fileChangesNodes.getLength(); j++) { - processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadataResult, pendingWrites); + processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites); } return pendingWrites; } - private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FvdlMetadataResult fvdlMetadataResult, + private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map pendingWrites) { String instanceId = remediation.getAttribute("instanceId"); String filename = getRequiredElementText(fileChanges, "Filename"); @@ -273,7 +271,7 @@ private boolean processFileChanges(Element remediation, Element fileChanges, Pat if (changesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); } - SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadataResult, pendingWrites); + SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadata, pendingWrites); Charset sourceEncoding = sourceFileContent.charset(); LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, changesNodes.getLength(), filename, sourceFileContent.encodingSource()); @@ -345,11 +343,11 @@ private String applyChange(String instanceId, String filename, String fileHash, return String.join(lineSeparator, updatedLines); } - private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FvdlMetadataResult fvdlMetadataResult, + private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, Map pendingWrites) { PendingFileWrite pendingWrite = pendingWrites.get(filePath); return pendingWrite == null - ? readSourceFile(filePath, filename, fvdlMetadataResult) + ? readSourceFile(filePath, filename, fvdlMetadata) : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); } @@ -405,29 +403,31 @@ private boolean isFilePresent(Path path) { return Files.exists(path) && Files.isRegularFile(path); } - private FvdlMetadataResult loadFvdlMetadata() { + /** Nullable: missing/unreadable FVDL means FPR encoding candidate is skipped. */ + private FVDLMetadata loadFvdlMetadata() { if (!Files.exists(fprHandle.getPath("/audit.fvdl"))) { - LOG.warn("FVDL file '/audit.fvdl' is missing; source remediations will be skipped"); - return new FvdlMetadataResult(null, SkipReason.FVDL_METADATA_UNAVAILABLE); + LOG.warn("FVDL file '/audit.fvdl' is missing; FPR encoding candidate will be skipped"); + return null; } try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { LOG.debug("Loading FVDL build metadata from '{}' to resolve source encodings", fprHandle.getFprPath()); - StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle); + // Decoder unused for metadata-only parse; ctor requires one for FileUtils wiring. + StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); processor.parseBuildMetadata(zipFile, "audit.fvdl"); LOG.debug("Loaded FVDL build metadata from '{}'", fprHandle.getFprPath()); - return new FvdlMetadataResult(processor.getFvdlMetadata(), null); + return processor.getFvdlMetadata(); } catch (Exception e) { - LOG.warn("Error reading source file encodings from audit.fvdl; source remediations will be skipped", e); - return new FvdlMetadataResult(null, SkipReason.FVDL_METADATA_UNAVAILABLE); + LOG.warn("Error reading source file encodings from audit.fvdl; FPR encoding candidate will be skipped", e); + return null; } } - private SourceFileContent readSourceFile(Path filePath, String filename, FvdlMetadataResult fvdlMetadataResult) { + private SourceFileContent readSourceFile(Path filePath, String filename, FVDLMetadata fvdlMetadata) { try { byte[] sourceBytes = Files.readAllBytes(filePath); - FVDLMetadata fvdlMetadata = fvdlMetadataResult.skipReason() == null ? fvdlMetadataResult.metadata() : null; - DecodeResult decodeResult = sourceEncodingOptions.decode(sourceBytes, filename, fvdlMetadata); + // Metadata may be null (FVDL missing); FPR candidate fails and other encodings are tried. + DecodeResult decodeResult = sourceDecoder.decode(sourceBytes, filename, fvdlMetadata); LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, decodeResult.content().length()); return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); @@ -440,7 +440,7 @@ private SourceFileContent readSourceFile(Path filePath, String filename, FvdlMet private byte[] encodeSourceFile(String content, Charset charset, String filename) { try { - return sourceEncodingOptions.encode(content, charset, filename); + return SourceEncoder.encode(content, charset, filename); } catch (SourceEncodeException e) { throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java index 9a3f8ffa9e8..3dace7f6e75 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java @@ -32,6 +32,8 @@ import com.fortify.cli.aviator.fpr.filter.AnalyzerType; import com.fortify.cli.aviator.fpr.model.*; import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.fpr.utils.XmlUtils; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -70,17 +72,26 @@ public class StreamingFVDLProcessor { private long peakMemoryPass2 = 0; private long peakMemoryPostProcessing = 0; - public StreamingFVDLProcessor(FprHandle fprHandle){ + public StreamingFVDLProcessor(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults()); + } + + /** + * @param sourceDecoder used for stack-trace line/fragment source reads; shares {@link #fvdlMetadata} + * so FPR encoding candidates resolve after build metadata is parsed. + */ + public StreamingFVDLProcessor(FprHandle fprHandle, ISourceDecoder sourceDecoder) { this.vulnFinalizer = new VulnFinalizer(); - this.fileUtils = new FileUtils(); this.fprHandle = fprHandle; this.sourceFileMap = fprHandle.getSourceFileMap(); this.xmlInputFactory = XMLInputFactory.newInstance(); // Security: Disable external entity processing xmlInputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - //this.parsingMetadata = new ParsingMetadata(); this.fvdlMetadata = new FVDLMetadata(); + // Same metadata instance FileUtils will see once encodings are registered during parse. + this.fileUtils = new FileUtils( + Objects.requireNonNull(sourceDecoder, "sourceDecoder"), this.fvdlMetadata); this.rawVulnerabilities = new ArrayList<>(); this.vulnerabilities = new ArrayList<>(); this.descriptionProcessor = new DescriptionProcessor(); @@ -92,8 +103,6 @@ public StreamingFVDLProcessor(FprHandle fprHandle){ this.traceParser.setNodeParser(nodeParser); // Circular dependency for Reason parsing this.descriptionParser = new DescriptionParser(); this.metadataParser = new MetadataParser(); - /*this.extractedPath = extractedPath; - this.indexXMLProcessor = new IndexXMLProcessor(extractedPath, sourceFileMap);*/ } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java new file mode 100644 index 00000000000..b0cf5d06621 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.util.Objects; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Fixed-{@link Charset} decoder. Created via {@link SourceDecoders}. */ +final class CharsetSourceDecoder implements ISourceDecoder { + private final Charset charset; + private final String label; + + CharsetSourceDecoder(String charsetName) { + this.charset = Charset.forName(charsetName); + this.label = charsetName; + } + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + try { + return new DecodeResult(SourceEncoder.decodeStrict(bytes, charset), charset, label); + } catch (CharacterCodingException e) { + throw new SourceDecodeException(label + " failed to decode source bytes", e); + } + } + + @Override + public String describe() { + return label; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java new file mode 100644 index 00000000000..9c335cb550f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Tries candidates in order. Created via {@link SourceDecoders#of}. */ +final class CompositeSourceDecoder implements ISourceDecoder { + private final List decoders; + + CompositeSourceDecoder(List decoders) { + Objects.requireNonNull(decoders, "decoders must not be null"); + if (decoders.isEmpty()) { + throw new IllegalArgumentException("decoders must not be empty"); + } + this.decoders = List.copyOf(decoders); + } + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + List failures = new ArrayList<>(); + for (ISourceDecoder decoder : decoders) { + try { + return decoder.decode(bytes, filename, fvdlMetadata); + } catch (SourceDecodeException e) { + failures.add(e.getMessage() != null ? e.getMessage() : decoder.describe() + " failed"); + } + } + throw new SourceDecodeException("Could not decode source file '" + filename + "' using source encodings " + + describe() + "; attempted: " + String.join("; ", failures)); + } + + @Override + public String describe() { + return decoders.stream().map(ISourceDecoder::describe).collect(Collectors.joining(",")); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index ec13c6ccba4..b45ea13f4f2 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -34,18 +35,22 @@ import com.fortify.cli.aviator.util.LanguageCommentMapperUtil; import com.fortify.cli.aviator.util.StringUtil; +/** + * Source file helpers. Decode failures are soft (empty result + log): snippets/lines are best-effort. + * Callers that must fail or skip with metrics (e.g. remediation apply) should decode themselves. + */ public class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private final Map> fileContentCache = new ConcurrentHashMap<>(); - private final SourceEncodingOptions sourceEncodingOptions; + private final ISourceDecoder sourceDecoder; private final FVDLMetadata fvdlMetadata; public FileUtils() { - this(SourceEncodingOptions.defaults(), null); + this(SourceDecoders.defaults(), null); } - public FileUtils(SourceEncodingOptions sourceEncodingOptions, FVDLMetadata fvdlMetadata) { - this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + public FileUtils(ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); this.fvdlMetadata = fvdlMetadata; } @@ -64,9 +69,9 @@ private List readFileWithFallback(Path filePath, String filename) { return fileContentCache.computeIfAbsent(filePath, path -> { try { byte[] fileBytes = Files.readAllBytes(path); - String content = sourceEncodingOptions.decode(fileBytes, filename, fvdlMetadata).content(); + String content = sourceDecoder.decode(fileBytes, filename, fvdlMetadata).content(); return Arrays.asList(content.split("\\r?\\n")); - } catch (IOException | SourceEncodingOptions.SourceDecodeException e) { + } catch (IOException | ISourceDecoder.SourceDecodeException e) { logger.error("Failed to read file: {}", path, e); return Collections.emptyList(); } @@ -152,8 +157,8 @@ public Optional getSourceFileContent(FprHandle fprHandle, String relativ try { byte[] fileBytes = Files.readAllBytes(actualSourcePath); - return Optional.of(sourceEncodingOptions.decode(fileBytes, relativePath, fvdlMetadata).content()); - } catch (IOException | SourceEncodingOptions.SourceDecodeException e) { + return Optional.of(sourceDecoder.decode(fileBytes, relativePath, fvdlMetadata).content()); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { logger.warn("Could not read source file content for path: {}", relativePath, e); return Optional.empty(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java new file mode 100644 index 00000000000..f7241d3ed73 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.util.Objects; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Resolves encoding from FVDL metadata, then strict-decodes. Created via {@link SourceDecoders}. */ +final class FprSourceDecoder implements ISourceDecoder { + static final String TOKEN = "FPR"; + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + Charset charset = resolveCharset(filename, fvdlMetadata); + try { + String content = SourceEncoder.decodeStrict(bytes, charset); + return new DecodeResult(content, charset, TOKEN + "(" + charset.name() + ")"); + } catch (CharacterCodingException e) { + throw new SourceDecodeException(TOKEN + "(" + charset.name() + ") failed to decode source bytes", e); + } + } + + private static Charset resolveCharset(String filename, FVDLMetadata fvdlMetadata) { + if (fvdlMetadata == null) { + throw new SourceDecodeException("FPR metadata unavailable"); + } + String encodingName = fvdlMetadata.findSourceFileEncodingForFileName(filename); + if (encodingName == null || encodingName.isBlank()) { + throw new SourceDecodeException("FPR encoding missing for '" + filename + "'"); + } + try { + return Charset.forName(encodingName); + } catch (Exception e) { + throw new SourceDecodeException( + TOKEN + " resolved to unsupported encoding '" + encodingName + "'", e); + } + } + + @Override + public String describe() { + return TOKEN; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java new file mode 100644 index 00000000000..96e76154a8b --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.Charset; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** + * Decodes source file bytes using a specific encoding strategy. + * Implementations may resolve the encoding from FVDL metadata, use a fixed + * {@link Charset}, or try multiple candidate decoders in order. + */ +public interface ISourceDecoder { + /** + * Decode source file bytes into text. + * + * @param bytes raw source file bytes + * @param filename source file name (used for FPR encoding lookup and error messages) + * @param fvdlMetadata optional FVDL metadata; required only by FPR-based decoders + * @return decoded content together with the charset that produced it + * @throws SourceDecodeException if this decoder cannot decode the bytes + */ + DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata); + + /** + * Human-readable description of this decoder (used in CLI help and error messages). + */ + String describe(); + + record DecodeResult(String content, Charset charset, String source) {} + + class SourceDecodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceDecodeException(String message) { + super(message); + } + + public SourceDecodeException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index aa6631dc6a5..235c3d60512 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -12,17 +12,15 @@ */ package com.fortify.cli.aviator.fpr.utils; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.StackTraceElement; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; @@ -41,32 +39,17 @@ public class SourceCodeEnricher { private static final Logger logger = LoggerFactory.getLogger(SourceCodeEnricher.class); - /*private final Path extractedPath; - private final Map sourceFileMap;*/ private final FprHandle fprHandle; private final FileUtils fileUtils; - private final SourceEncodingOptions sourceEncodingOptions; - private final FVDLMetadata fvdlMetadata; - /** - * Creates a new SourceCodeEnricher with the required dependencies. - * @param fprHandle Utility for file operations (line numbering, line counting) - */ - /*public SourceCodeEnricher(Path extractedPath, Map sourceFileMap, FileUtils fileUtils) { - this.extractedPath = extractedPath; - this.sourceFileMap = sourceFileMap; - this.fileUtils = fileUtils; - }*/ - - public SourceCodeEnricher(FprHandle fprHandle){ - this(fprHandle, SourceEncodingOptions.defaults(), null); + public SourceCodeEnricher(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults(), null); } - public SourceCodeEnricher(FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions, FVDLMetadata fvdlMetadata){ + public SourceCodeEnricher(FprHandle fprHandle, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { this.fprHandle = fprHandle; - this.fileUtils = new FileUtils(); - this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; - this.fvdlMetadata = fvdlMetadata; + // Single soft-fail decode path via FileUtils (same policy as snippets/lines). + this.fileUtils = new FileUtils(Objects.requireNonNull(sourceDecoder, "sourceDecoder"), fvdlMetadata); } /** @@ -132,29 +115,19 @@ private void processFileForElement(StackTraceElement element, Map String filename = element.getFilename(); if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) && !uniqueFiles.containsKey(filename)) { - String internalPath = fprHandle.getSourceFileMap().get(filename); - if (internalPath == null) { return; } // Should not happen due to containsKey check, but safe. - - Path actualSourcePath = fprHandle.getPath("/" + internalPath); - + // Soft-fail decode via FileUtils: omit file rather than fail the whole issue. + Optional contentOpt = fileUtils.getSourceFileContent(fprHandle, filename); + if (contentOpt.isEmpty()) { + return; + } + String content = contentOpt.get(); File file = new File(); file.setName(filename); file.setSegment(false); file.setStartLine(1); - - try { - if (Files.exists(actualSourcePath)) { - byte[] encodedBytes = Files.readAllBytes(actualSourcePath); - String content = sourceEncodingOptions.decode(encodedBytes, filename, fvdlMetadata).content(); - // Keep line markers in prompt file content; downstream gRPC/template rendering is pass-through. - file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); - file.setEndLine(content.split("\\R", -1).length); - } else { - throw new AviatorSimpleException("Source file '" + filename + "' was not found in the FPR"); - } - } catch (IOException e) { - throw new AviatorSimpleException("Source file '" + filename + "' could not be read from the FPR", e); - } + // Keep line markers in prompt file content; downstream gRPC/template rendering is pass-through. + file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); + file.setEndLine(content.split("\\R", -1).length); uniqueFiles.put(filename, file); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java new file mode 100644 index 00000000000..7c047085fe0 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java @@ -0,0 +1,82 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Domain factory for {@link ISourceDecoder} instances. Single source of truth for + * token/CSV parsing and default candidate order. CLI converters should delegate here. + */ +public final class SourceDecoders { + public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,CP850"; + + private SourceDecoders() {} + + /** + * Map a single token to a decoder: {@code FPR} or a charset name. + * + * @throws IllegalArgumentException if token is blank + * @throws java.nio.charset.IllegalCharsetNameException if charset name is illegal + * @throws java.nio.charset.UnsupportedCharsetException if charset is unsupported + */ + public static ISourceDecoder fromToken(String token) { + if (token == null || token.isBlank()) { + throw new IllegalArgumentException("Source encoding must not be blank"); + } + String trimmed = token.trim(); + if (FprSourceDecoder.TOKEN.equalsIgnoreCase(trimmed)) { + return new FprSourceDecoder(); + } + return new CharsetSourceDecoder(trimmed); + } + + /** + * Parse a comma-separated list of encoding tokens into a single decoder + * (composite when more than one candidate). + */ + public static ISourceDecoder fromCsv(String csv) { + String effective = csv == null || csv.isBlank() ? DEFAULT_SOURCE_ENCODINGS : csv; + List decoders = new ArrayList<>(); + for (String part : effective.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + decoders.add(fromToken(trimmed)); + } + } + if (decoders.isEmpty()) { + return defaults(); + } + return of(decoders); + } + + /** + * Compose an ordered candidate list into one decoder. + */ + public static ISourceDecoder of(List decoders) { + Objects.requireNonNull(decoders, "decoders must not be null"); + if (decoders.isEmpty()) { + return defaults(); + } + if (decoders.size() == 1) { + return decoders.get(0); + } + return new CompositeSourceDecoder(decoders); + } + + public static ISourceDecoder defaults() { + return fromCsv(DEFAULT_SOURCE_ENCODINGS); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java new file mode 100644 index 00000000000..93668218dd1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; + +/** + * Strict source encode/decode helpers ({@link CodingErrorAction#REPORT}). + * Public {@link #encode} is for write-back; package-private decode is shared by decoders. + */ +public final class SourceEncoder { + private SourceEncoder() {} + + public static byte[] encode(String content, Charset charset, String filename) { + try { + return encodeStrict(content, charset); + } catch (CharacterCodingException e) { + throw new SourceEncodeException( + "Source file '" + filename + "' cannot be encoded using " + charset.name(), e); + } + } + + static String decodeStrict(byte[] bytes, Charset charset) throws CharacterCodingException { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + } + + private static byte[] encodeStrict(String content, Charset charset) throws CharacterCodingException { + ByteBuffer buffer = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(content)); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; + } + + public static class SourceEncodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceEncodeException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java deleted file mode 100644 index 8418c48a489..00000000000 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncodingOptions.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2021-2026 Open Text. - * - * The only warranties for products and services of Open Text - * and its affiliates and licensors ("Open Text") are as may - * be set forth in the express warranty statements accompanying - * such products and services. Nothing herein should be construed - * as constituting an additional warranty. Open Text shall not be - * liable for technical or editorial errors or omissions contained - * herein. The information contained herein is subject to change - * without notice. - */ -package com.fortify.cli.aviator.fpr.utils; - -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.Charset; -import java.nio.charset.CodingErrorAction; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import com.fortify.cli.aviator._common.exception.AviatorSimpleException; -import com.fortify.cli.aviator.fpr.model.FVDLMetadata; - -public final class SourceEncodingOptions { - public static final String FPR_ENCODING = "FPR"; - public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,CP850"; - - private final List candidates; - - private SourceEncodingOptions(List candidates) { - this.candidates = List.copyOf(candidates); - } - - public static SourceEncodingOptions defaults() { - return parse(DEFAULT_SOURCE_ENCODINGS); - } - - public static SourceEncodingOptions parse(String value) { - String effectiveValue = value == null || value.isBlank() ? DEFAULT_SOURCE_ENCODINGS : value; - List candidates = new ArrayList<>(); - for (String candidate : effectiveValue.split(",")) { - String trimmed = candidate.trim(); - if (!trimmed.isEmpty()) { - candidates.add(trimmed); - } - } - if (candidates.isEmpty()) { - candidates.add(FPR_ENCODING); - candidates.add("UTF-8"); - candidates.add("CP850"); - } - return new SourceEncodingOptions(candidates); - } - - public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { - Objects.requireNonNull(bytes, "bytes must not be null"); - List failures = new ArrayList<>(); - for (String candidate : candidates) { - Optional charset = resolveCharset(candidate, filename, fvdlMetadata, failures); - if (charset.isEmpty()) { - continue; - } - try { - String content = charset.get().newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString(); - return new DecodeResult(content, charset.get(), displayName(candidate, charset.get())); - } catch (CharacterCodingException e) { - failures.add(displayName(candidate, charset.get()) + " failed to decode source bytes"); - } - } - throw new SourceDecodeException("Could not decode source file '" + filename + "' using source encodings " + describe() + - "; attempted: " + String.join("; ", failures)); - } - - public byte[] encode(String content, Charset charset, String filename) { - try { - ByteBuffer buffer = charset.newEncoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .encode(CharBuffer.wrap(content)); - byte[] result = new byte[buffer.remaining()]; - buffer.get(result); - return result; - } catch (CharacterCodingException e) { - throw new SourceEncodeException("Source file '" + filename + "' cannot be encoded using " + charset.name(), e); - } - } - - public String describe() { - return String.join(",", candidates); - } - - private Optional resolveCharset(String candidate, String filename, FVDLMetadata fvdlMetadata, List failures) { - String encodingName = candidate; - if (FPR_ENCODING.equalsIgnoreCase(candidate)) { - if (fvdlMetadata == null) { - failures.add("FPR metadata unavailable"); - return Optional.empty(); - } - encodingName = fvdlMetadata.findSourceFileEncodingForFileName(filename); - if (encodingName == null || encodingName.isBlank()) { - failures.add("FPR encoding missing for '" + filename + "'"); - return Optional.empty(); - } - } - - try { - return Optional.of(Charset.forName(encodingName)); - } catch (Exception e) { - failures.add(candidate + " resolved to unsupported encoding '" + encodingName + "'"); - return Optional.empty(); - } - } - - private String displayName(String candidate, Charset charset) { - return FPR_ENCODING.equalsIgnoreCase(candidate) ? FPR_ENCODING + "(" + charset.name() + ")" : charset.name(); - } - - public record DecodeResult(String content, Charset charset, String source) {} - - public static class SourceDecodeException extends AviatorSimpleException { - private static final long serialVersionUID = 1L; - - public SourceDecodeException(String message) { - super(message); - } - } - - public static class SourceEncodeException extends AviatorSimpleException { - private static final long serialVersionUID = 1L; - - public SourceEncodeException(String message, Throwable cause) { - super(message, cause); - } - } -} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java index 764d63e6f73..e3766515ecd 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java @@ -53,7 +53,7 @@ import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.grpc.token.DeleteTokenRequest; @@ -127,9 +127,9 @@ public AviatorGrpcClient(ManagedChannel channel, long defaultTimeoutSeconds, IAv public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, - List customPriorityOrder, SourceEncodingOptions sourceEncodingOptions, FVDLMetadata fvdlMetadata) { + List customPriorityOrder, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, - pingIntervalSeconds, defaultTimeoutSeconds, fprHandle, sourceEncodingOptions, fvdlMetadata); + pingIntervalSeconds, defaultTimeoutSeconds, fprHandle, sourceDecoder, fvdlMetadata); CompletableFuture> future = processor.processBatchRequests(requests, projectName, FPRBuildId, SSCApplicationName, SSCApplicationVersion, token, customPriorityOrder); future.whenComplete((res, th) -> processor.close()); return future.exceptionally(ex -> { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index ec11a822cdd..31dcef6ad58 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -17,6 +17,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.UUID; @@ -49,8 +50,8 @@ import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.fpr.utils.SourceCodeEnricher; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -94,12 +95,12 @@ class AviatorStreamProcessor implements AutoCloseable { private volatile Future processingTask; private final Object retryLock = new Object(); private final FprHandle fprHandle; - private final SourceEncodingOptions sourceEncodingOptions; - private final FVDLMetadata fvdlMetadata; + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; - public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, + public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, - long defaultTimeoutSeconds, FprHandle fprHandle, SourceEncodingOptions sourceEncodingOptions, + long defaultTimeoutSeconds, FprHandle fprHandle, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { this.client = client; this.logger = logger; @@ -109,7 +110,7 @@ public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, A this.pingIntervalSeconds = pingIntervalSeconds; this.defaultTimeoutSeconds = defaultTimeoutSeconds; this.fprHandle = fprHandle; - this.sourceEncodingOptions = sourceEncodingOptions == null ? SourceEncodingOptions.defaults() : sourceEncodingOptions; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); this.fvdlMetadata = fvdlMetadata; } @@ -717,7 +718,7 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); // Lazy Loading of source code files for individual issue - SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle, sourceEncodingOptions, fvdlMetadata); + SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle, sourceDecoder, fvdlMetadata); Map enrichedFiles = sourceCodeEnricher.enrichWithSourceCode(wrapper.userPrompt.getStackTrace()); diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java new file mode 100644 index 00000000000..0ad64fab89d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.converter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import picocli.CommandLine.TypeConversionException; + +/** + * Only the CLI adapter layer: domain factory exceptions become {@link TypeConversionException}. + * Happy-path token semantics are covered by {@code SourceDecodersTest}. + */ +class SourceDecoderConverterTest { + + private final SourceDecoderConverter converter = new SourceDecoderConverter(); + + @Test + void convert_delegatesHappyPathToFactory() { + assertEquals("FPR", converter.convert("FPR").describe()); + assertEquals("UTF-8", converter.convert("UTF-8").describe()); + } + + @Test + void convert_mapsFactoryFailuresToTypeConversionException() { + assertThrows(TypeConversionException.class, () -> converter.convert(" ")); + assertThrows(TypeConversionException.class, () -> converter.convert("NOT-A-REAL-CHARSET")); + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java new file mode 100644 index 00000000000..fea7f98fed1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; + +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; + +/** + * Behavior tests for source encoding factory + decode/encode. + * Prefer real byte decode outcomes over describe()-only checks. + */ +class SourceDecodersTest { + + @Test + void fromToken_rejectsBlankAndUnknownCharset() { + assertThrows(IllegalArgumentException.class, () -> SourceDecoders.fromToken(" ")); + assertThrows(UnsupportedCharsetException.class, () -> SourceDecoders.fromToken("NOT-A-CHARSET")); + } + + @Test + void defaults_matchDocumentedCandidateOrder() { + assertEquals("FPR,UTF-8,CP850", SourceDecoders.DEFAULT_SOURCE_ENCODINGS); + assertEquals(SourceDecoders.DEFAULT_SOURCE_ENCODINGS, SourceDecoders.defaults().describe()); + } + + @Test + void decode_fallsThroughFprWhenMetadataMissing() { + byte[] utf8 = "hello".getBytes(StandardCharsets.UTF_8); + DecodeResult result = SourceDecoders.defaults().decode(utf8, "Main.java", null); + assertEquals("hello", result.content()); + assertEquals(StandardCharsets.UTF_8, result.charset()); + assertEquals("UTF-8", result.source()); + } + + @Test + void decode_usesFprEncodingWhenMetadataPresent() { + FVDLMetadata metadata = new FVDLMetadata(); + metadata.registerSourceFileEncoding("src/Main.java", "ISO-8859-1"); + byte[] latin1 = "caf\u00e9".getBytes(StandardCharsets.ISO_8859_1); + + DecodeResult result = SourceDecoders.defaults().decode(latin1, "src/Main.java", metadata); + + assertEquals("caf\u00e9", result.content()); + assertEquals(StandardCharsets.ISO_8859_1, result.charset()); + assertTrue(result.source().startsWith("FPR("), result.source()); + } + + @Test + void decode_allCandidatesFail_messageListsAttempts() { + ISourceDecoder decoder = SourceDecoders.fromCsv("UTF-8,US-ASCII"); + byte[] invalid = new byte[] {(byte) 0xFF, (byte) 0xFE, (byte) 0x00}; + + SourceDecodeException ex = assertThrows(SourceDecodeException.class, + () -> decoder.decode(invalid, "bad.bin", null)); + + assertTrue(ex.getMessage().contains("bad.bin"), ex.getMessage()); + assertTrue(ex.getMessage().contains("UTF-8"), ex.getMessage()); + assertTrue(ex.getMessage().contains("US-ASCII"), ex.getMessage()); + } + + @Test + void encode_roundTripAndRejectsUnmappable() { + String content = "secure code"; + byte[] encoded = SourceEncoder.encode(content, StandardCharsets.UTF_8, "a.java"); + assertEquals(content, SourceDecoders.fromToken("UTF-8").decode(encoded, "a.java", null).content()); + + assertThrows(SourceEncodeException.class, + () -> SourceEncoder.encode("caf\u00e9", StandardCharsets.US_ASCII, "a.java")); + } + + @Test + void fromToken_fprIsCaseInsensitive() { + assertEquals("FPR", SourceDecoders.fromToken("fpr").describe()); + } + + @Test + void fromCsv_compositesMultipleCandidates() { + assertEquals("UTF-8,CP850", SourceDecoders.fromCsv("UTF-8,CP850").describe()); + assertEquals("UTF-8", SourceDecoders.fromCsv("UTF-8").describe()); + } +} diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java index 76bc4a70ffe..c6bff9c596e 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java @@ -26,10 +26,10 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.ssc.cli.mixin.AviatorSSCApplyRemediationsArtifactSelectorMixin; import com.fortify.cli.aviator.ssc.helper.AviatorSSCApplyRemediationsHelper; import com.fortify.cli.aviator.ssc.helper.SinceOptionHelper; @@ -64,8 +64,7 @@ public class AviatorSSCApplyRemediationsCommand extends AbstractSSCJsonNodeOutpu private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCApplyRemediationsCommand.class); @Option(names = {"--source-dir"}, descriptionKey = "fcli.aviator.ssc.apply-remediations.source-dir") private String sourceCodeDirectory = System.getProperty("user.dir"); - @Option(names = {"--source-encodings"}, descriptionKey = "fcli.aviator.ssc.apply-remediations.source-encodings") - private String sourceEncodings = SourceEncodingOptions.DEFAULT_SOURCE_ENCODINGS; + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; @Override @SneakyThrows @@ -132,7 +131,7 @@ JsonNode processAllAviatorArtifacts(OffsetDateTime sinceDate) { fprPath = downloadArtifactFpr(ad); try (FprHandle fprHandle = new FprHandle(fprPath)) { var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, - SourceEncodingOptions.parse(sourceEncodings), logger); + sourceEncodingsMixin.getSourceDecoder(), logger); totalRemediations += metric.totalRemediations(); appliedRemediations += metric.appliedRemediations(); skippedRemediations += metric.skippedRemediations(); @@ -186,7 +185,7 @@ JsonNode processFprRemediations(SSCArtifactDescriptor ad) { logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(fprPath)) { var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, - SourceEncodingOptions.parse(sourceEncodings), logger); + sourceEncodingsMixin.getSourceDecoder(), logger); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java index c6d13356146..5a0c876ef9e 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator._common.config.AviatorConfigManager; import com.fortify.cli.aviator._common.session.user.cli.mixin.AviatorUserSessionDescriptorSupplier; import com.fortify.cli.aviator._common.session.user.helper.AviatorUserSessionDescriptor; @@ -33,7 +34,6 @@ import com.fortify.cli.aviator.audit.model.FPRAuditResult; import com.fortify.cli.aviator.config.AviatorLoggerImpl; import com.fortify.cli.aviator.config.TagMappingConfig; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.ssc.helper.AviatorSSCAuditHelper; import com.fortify.cli.aviator.ssc.helper.AviatorSSCTagValidator; import com.fortify.cli.aviator.util.FprHandle; @@ -80,8 +80,7 @@ public class AviatorSSCAuditCommand extends AbstractSSCJsonNodeOutputCommand imp @ArgGroup(exclusive = true, multiplicity = "0..1") private QuotaHandlingArgGroup quotaHandlingArgGroup = new QuotaHandlingArgGroup(); @Option(names = {"--test-exceeding-quota"}) private boolean testExceedingQuota; @Option(names = {"--default-quota-fallback"}) private boolean defaultQuotaFallback; - @Option(names = {"--source-encodings"}, descriptionKey = "fcli.aviator.ssc.audit.source-encodings") - private String sourceEncodings = SourceEncodingOptions.DEFAULT_SOURCE_ENCODINGS; + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCAuditCommand.class); private Long checkedQuotaBefore; @@ -278,7 +277,7 @@ private JsonNode processFpr(UnirestInstance unirest, SSCAppVersionDescriptor av, .noFilterSet(isNoFilterSet()) .folderNames(folderNames) .folderPriorityOrder(getFolderPriorityOrder()) - .sourceEncodingOptions(SourceEncodingOptions.parse(sourceEncodings)) + .sourceDecoder(sourceEncodingsMixin.getSourceDecoder()) .build()); } catch (Exception e) { LOG.error("FPR audit failed for {}:{}: {}", av.getApplicationName(), av.getVersionName(), e.getMessage(), e); diff --git a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties index 781ed10f230..7a2c64f68e3 100644 --- a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties +++ b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties @@ -129,7 +129,7 @@ fcli.aviator.ssc.audit.skip-if-exceeding-quota = Skip audit if the number of ope fcli.aviator.ssc.audit.test-exceeding-quota = Check whether the number of open issues exceeds the available Fortify Remediation Aviator quota and report the result without performing an audit. fcli.aviator.ssc.audit.default-quota-fallback = (Internal) When the Fortify Aviator application does not exist, use the tenant default quota instead of reporting app not found. Used by bulk audit. fcli.aviator.ssc.audit.folder-priority-order = Custom priority order for folder-based filtering when quota is exceeded (comma-separated, highest priority first). Example: Critical,High,Medium,Low. If not specified, uses default priority order. -fcli.aviator.ssc.audit.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files from the FPR. Use FPR to try the source encoding recorded in audit.fvdl. Default value: ${DEFAULT-VALUE}. +fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.audit.refresh = By default, this command will refresh the source application version's metrics when copying from it. \ Note that for large applications this can lead to an error if the timeout expires. fcli.aviator.ssc.audit.refresh-timeout = Time-out, for example 30s (30 seconds), 5m (5 minutes), 1h (1 hour). Default value: ${DEFAULT-VALUE} @@ -157,7 +157,6 @@ fcli.aviator.ssc.apply-remediations.all = Apply remediations from all Fortify Re in chronological order. Aggregates remediation statistics across all artifacts. \ Requires --av/--appversion. Mutually exclusive with --artifact-id and --latest. fcli.aviator.ssc.apply-remediations.source-dir = Source code directory where remediations will be applied. Defaults to current directory. -fcli.aviator.ssc.apply-remediations.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. Remediated files are written using the accepted encoding. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.apply-remediations.since = Filter artifacts by upload date. Supports relative durations (e.g. 7d, 2w, 1M, 90d) \ or absolute dates (e.g. 2025-01-01, 2025-01-01T10:30:00, 2025-01-01T10:30:00Z). \ Can only be used with --latest or --all; not compatible with --artifact-id. diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java index c39bbba8f80..471232a1e03 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java @@ -20,9 +20,9 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; -import com.fortify.cli.aviator.fpr.utils.SourceEncodingOptions; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; @@ -57,8 +57,7 @@ public class FoDAviatorApplyRemediationsCommand extends AbstractFoDJsonNodeOutpu @Mixin private FoDReleaseByQualifiedNameOrIdResolverMixin.RequiredOption releaseResolver; private static final Logger LOG = LoggerFactory.getLogger(FoDAviatorApplyRemediationsCommand.class); @Option(names = {"--source-dir"}) private String sourceCodeDirectory = System.getProperty("user.dir"); - @Option(names = {"--source-encodings"}, descriptionKey = "fcli.fod.aviator.apply-remediations.source-encodings") - private String sourceEncodings = SourceEncodingOptions.DEFAULT_SOURCE_ENCODINGS; + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; @Override @SneakyThrows public JsonNode getJsonNode(UnirestInstance unirest) { @@ -86,7 +85,7 @@ private JsonNode processFprRemediations(UnirestInstance unirest, FoDReleaseDescr logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(downloadedFprPath)) { var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, - SourceEncodingOptions.parse(sourceEncodings), logger); + sourceEncodingsMixin.getSourceDecoder(), logger); LOG.info("Applied remediation {}", remediationMetric.appliedRemediations()); LOG.info("Total remediation {}", remediationMetric.totalRemediations()); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index e9a97bb5749..0fcff19749e 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -1038,7 +1038,7 @@ fcli.fod.attribute.update.values = List of picklist values (only for Picklist da fcli.fod.aviator.usage.header = Use Fortify Remediation Aviator with FoD. fcli.fod.aviator.apply-remediations.usage.header = Apply Fortify Remediation Aviator auto-remediations to source code. fcli.fod.aviator.apply-remediations.source-dir = Directory containing source code to apply remediations to. Default value: ${DEFAULT-VALUE}. -fcli.fod.aviator.apply-remediations.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. Remediated files are written using the accepted encoding. Default value: ${DEFAULT-VALUE}. +fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. # various messages displayed during execution From 2b03224a3b32cebabb1fd11cd9a864374803a080 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Mon, 3 Aug 2026 15:07:24 +0530 Subject: [PATCH 04/11] chore: add aviator-common dependency to fcli-app --- fcli-core/fcli-app/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fcli-core/fcli-app/build.gradle.kts b/fcli-core/fcli-app/build.gradle.kts index 12c38c2a221..3cbddfca67c 100644 --- a/fcli-core/fcli-app/build.gradle.kts +++ b/fcli-core/fcli-app/build.gradle.kts @@ -8,7 +8,7 @@ plugins { // Inter-project dependencies val refs = listOf( "fcliCommonRef","fcliCommonThirdpartyRef","fcliCommonCiRef","fcliCommonActionRef","fcliCommonToolRef", - "fcliActionRef","fcliAiAssistRef","fcliAviatorRef","fcliConfigRef", + "fcliActionRef","fcliAiAssistRef","fcliAviatorCommonRef","fcliAviatorRef","fcliConfigRef", "fcliFoDRef","fcliSSCRef","fcliSCSastRef","fcliSCDastRef", "fcliToolRef","fcliLicenseRef","fcliUtilRef" ) From d7ac45dd04b78f3cab68a3dfaea5a2229c86b4f6 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Fri, 7 Aug 2026 12:14:26 +0530 Subject: [PATCH 05/11] chore: skip issues with undecodable source files --- .../fortify/cli/aviator/audit/AuditFPR.java | 32 ++++-- .../cli/aviator/fpr/utils/FileUtils.java | 25 +++-- .../aviator/fpr/utils/SourceCodeEnricher.java | 65 +++++++----- .../aviator/grpc/AviatorStreamProcessor.java | 100 +++++++++++++++--- .../cli/aviator/grpc/RequestWrapper.java | 1 + .../cli/aviator/audit/AuditFprStatusTest.java | 60 +++++++++++ .../fpr/utils/SourceCodeEnricherTest.java | 94 ++++++++++++++++ .../aviator/fpr/utils/SourceDecodersTest.java | 13 +++ .../ssc/helper/AviatorSSCAuditHelper.java | 3 +- 9 files changed, 334 insertions(+), 59 deletions(-) create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java create mode 100644 fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index a89dc206597..377a75a5dfb 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -180,13 +180,12 @@ private static FPRAuditResult finalizeFprAudit( String status; String message = null; - if (issuesSuccessfullyAudited == totalIssuesToAudit) { - status = "AUDITED"; - } else if (issuesSuccessfullyAudited > 0) { - status = "PARTIALLY_AUDITED"; - } else { - status = "FAILED"; + status = determineAuditStatus(issuesSuccessfullyAudited, issuesSkipped, totalIssuesToAudit, auditResponses.size()); + if ("SKIPPED".equals(status)) { + message = String.format("All %d issues were skipped", totalIssuesToAudit); + } else if ("FAILED".equals(status)) { String commonFailureReason = auditResponses.values().stream() + .filter(response -> !"SKIPPED".equalsIgnoreCase(response.getStatus())) .map(AuditResponse::getStatusMessage) .filter(msg -> msg != null && !msg.isBlank()) .findFirst() @@ -195,7 +194,8 @@ private static FPRAuditResult finalizeFprAudit( if (commonFailureReason.startsWith("Client-side pre-processing error: ")) { commonFailureReason = commonFailureReason.substring("Client-side pre-processing error: ".length()); } - message = String.format("All %d issues failed (%s)", totalIssuesToAudit, commonFailureReason); + message = String.format("No issues were audited (%d skipped; failure details: %s)", + issuesSkipped, commonFailureReason); } File updatedFile = null; @@ -219,10 +219,24 @@ private static FPRAuditResult finalizeFprAudit( remediationGenerationMetric.skippedByReason()); } - private static Map getSkippedAuditReasons(Map auditResponses, int totalIssuesToAudit) { + static String determineAuditStatus(long issuesSuccessfullyAudited, int issuesSkipped, + int totalIssuesToAudit, int responseCount) { + if (issuesSuccessfullyAudited == totalIssuesToAudit) { + return "AUDITED"; + } + if (issuesSuccessfullyAudited > 0) { + return "PARTIALLY_AUDITED"; + } + if (issuesSkipped == totalIssuesToAudit && responseCount == totalIssuesToAudit) { + return "SKIPPED"; + } + return "FAILED"; + } + + static Map getSkippedAuditReasons(Map auditResponses, int totalIssuesToAudit) { Map skippedByReason = new LinkedHashMap<>(); auditResponses.values().stream() - .filter(response -> !"SUCCESS".equalsIgnoreCase(response.getStatus())) + .filter(response -> "SKIPPED".equalsIgnoreCase(response.getStatus())) .map(AuditFPR::getSkippedAuditReason) .forEach(reason -> recordSkipped(skippedByReason, reason)); int missingResponses = Math.max(0, totalIssuesToAudit - auditResponses.size()); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index b45ea13f4f2..f5878cc62d8 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -72,7 +72,7 @@ private List readFileWithFallback(Path filePath, String filename) { String content = sourceDecoder.decode(fileBytes, filename, fvdlMetadata).content(); return Arrays.asList(content.split("\\r?\\n")); } catch (IOException | ISourceDecoder.SourceDecodeException e) { - logger.error("Failed to read file: {}", path, e); + logger.warn("Could not read or decode source file {}: {}", path, e.getMessage()); return Collections.emptyList(); } }); @@ -147,23 +147,26 @@ private Path resolveFullPath(FprHandle fprHandle, String relativePath) { return fprHandle.getPath(internalPath); } - - // Assumes you have already updated the signature to accept extractedPath public Optional getSourceFileContent(FprHandle fprHandle, String relativePath) { - Path actualSourcePath = resolveFullPath(fprHandle, relativePath); - if (actualSourcePath == null) { - return Optional.empty(); - } - try { - byte[] fileBytes = Files.readAllBytes(actualSourcePath); - return Optional.of(sourceDecoder.decode(fileBytes, relativePath, fvdlMetadata).content()); + return Optional.of(readSourceFileContentStrict(fprHandle, relativePath)); } catch (IOException | ISourceDecoder.SourceDecodeException e) { - logger.warn("Could not read source file content for path: {}", relativePath, e); + logger.warn("Could not read source file content for path {}: {}", relativePath, e.getMessage()); return Optional.empty(); } } + /** Reads and decodes source content while preserving read or decode failures for the caller. */ + String readSourceFileContentStrict(FprHandle fprHandle, String relativePath) throws IOException { + Path actualSourcePath = resolveFullPath(fprHandle, relativePath); + if (actualSourcePath == null) { + throw new IOException("Source file key not found in sourceFileMap: " + relativePath); + } + + byte[] fileBytes = Files.readAllBytes(actualSourcePath); + return sourceDecoder.decode(fileBytes, relativePath, fvdlMetadata).content(); + } + public String appendLineNumbers(String content, String fileName, int startLineNo) { return appendLineNumbers(content, fileName, startLineNo, null); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index 235c3d60512..b5cb97c5ab4 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -12,11 +12,13 @@ */ package com.fortify.cli.aviator.fpr.utils; +import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +50,6 @@ public SourceCodeEnricher(FprHandle fprHandle) { public SourceCodeEnricher(FprHandle fprHandle, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { this.fprHandle = fprHandle; - // Single soft-fail decode path via FileUtils (same policy as snippets/lines). this.fileUtils = new FileUtils(Objects.requireNonNull(sourceDecoder, "sourceDecoder"), fvdlMetadata); } @@ -67,36 +68,41 @@ public SourceCodeEnricher(FprHandle fprHandle, ISourceDecoder sourceDecoder, FVD * @return Map of filename → File objects with content loaded */ public Map enrichWithSourceCode(List> stackTraces) { - Map uniqueFiles = new HashMap<>(); + return new HashMap<>(enrichWithSourceCodeDetailed(stackTraces).files()); + } + + public EnrichmentResult enrichWithSourceCodeDetailed(List> stackTraces) { + Map uniqueFiles = new LinkedHashMap<>(); + Map failuresByFilename = new LinkedHashMap<>(); if (stackTraces == null || stackTraces.isEmpty()) { logger.debug("No stack traces to enrich"); - return uniqueFiles; + return new EnrichmentResult(uniqueFiles, List.of()); } - processStackTraces(stackTraces, uniqueFiles); + processStackTraces(stackTraces, uniqueFiles, failuresByFilename); logger.debug("Enriched {} unique source files from {} stack traces", uniqueFiles.size(), stackTraces.size()); - return uniqueFiles; + return new EnrichmentResult(uniqueFiles, new ArrayList<>(failuresByFilename.values())); } /** * Processes all stack traces to extract and load unique source files. * Replicates FVDLProcessor.processStackTraceElements() logic. */ - private void processStackTraces(List> stackTraces, Map uniqueFiles) { + private void processStackTraces(List> stackTraces, Map uniqueFiles, + Map failuresByFilename) { for (List stackTrace : stackTraces) { if (stackTrace == null) continue; for (StackTraceElement element : stackTrace) { - processFileForElement(element, uniqueFiles); + processFileForElement(element, uniqueFiles, failuresByFilename); - // Process inner stack traces recursively if (element.getInnerStackTrace() != null) { for (StackTraceElement innerElement : element.getInnerStackTrace()) { - processFileForElement(innerElement, uniqueFiles); + processFileForElement(innerElement, uniqueFiles, failuresByFilename); } } } @@ -110,25 +116,34 @@ private void processStackTraces(List> stackTraces, Map uniqueFiles) { + private void processFileForElement(StackTraceElement element, Map uniqueFiles, + Map failuresByFilename) { if (element == null) return; String filename = element.getFilename(); - if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) && !uniqueFiles.containsKey(filename)) { - // Soft-fail decode via FileUtils: omit file rather than fail the whole issue. - Optional contentOpt = fileUtils.getSourceFileContent(fprHandle, filename); - if (contentOpt.isEmpty()) { - return; + if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) + && !uniqueFiles.containsKey(filename) && !failuresByFilename.containsKey(filename)) { + try { + String content = fileUtils.readSourceFileContentStrict(fprHandle, filename); + File file = new File(); + file.setName(filename); + file.setSegment(false); + file.setStartLine(1); + file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); + file.setEndLine(content.split("\\R", -1).length); + uniqueFiles.put(filename, file); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { + logger.warn("Could not read source file content for path {}: {}", filename, e.getMessage()); + failuresByFilename.put(filename, new SourceFileFailure(filename, e.getMessage())); } - String content = contentOpt.get(); - File file = new File(); - file.setName(filename); - file.setSegment(false); - file.setStartLine(1); - // Keep line markers in prompt file content; downstream gRPC/template rendering is pass-through. - file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); - file.setEndLine(content.split("\\R", -1).length); - uniqueFiles.put(filename, file); } } + + public record EnrichmentResult(Map files, List failures) { + public boolean hasFailures() { + return !failures.isEmpty(); + } + } + + public record SourceFileFailure(String filename, String message) {} } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 31dcef6ad58..2d86749c719 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -399,7 +400,7 @@ public void onNext(AuditorResponse response) { response.getRequestId(), instanceId, response.getStatus(), metrics.getDuration()); } - outstandingRequests.decrementAndGet(); + decrementOutstanding(completedWrapper); requestSemaphore.release(); AuditResponse auditResponse = GrpcUtil.convertToAuditResponse(response); @@ -446,11 +447,14 @@ public void onError(Throwable t) { if (infinite || currentStreamState.streamRetryCount < Constants.MAX_STREAM_RETRIES) { LOG.debug("Stream encountered retryable error: {}. Will retry...", t.getMessage()); - int reAdded = inflightRequests.size(); - inflightRequests.values().forEach(processingQueue::addFirst); + int requeuedRequestCount = inflightRequests.size(); + inflightRequests.values().forEach(wrapper -> { + wrapper.outstandingTracked = false; + processingQueue.addFirst(wrapper); + }); inflightRequests.clear(); - outstandingRequests.addAndGet(-reAdded); - requestSemaphore.release(reAdded); + outstandingRequests.addAndGet(-requeuedRequestCount); + requestSemaphore.release(requeuedRequestCount); if (outstandingRequests.get() < 0) { outstandingRequests.set(0); } @@ -720,10 +724,15 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque // Lazy Loading of source code files for individual issue SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle, sourceDecoder, fvdlMetadata); - Map enrichedFiles = - sourceCodeEnricher.enrichWithSourceCode(wrapper.userPrompt.getStackTrace()); - List sourceCodeFiles = new ArrayList<>(enrichedFiles.values()); + SourceCodeEnricher.EnrichmentResult enrichmentResult = + sourceCodeEnricher.enrichWithSourceCodeDetailed(wrapper.userPrompt.getStackTrace()); + if (enrichmentResult.hasFailures()) { + completeSkippedRequest(wrapper, enrichmentResult, responses, processedRequests, totalRequests, + resultFuture, streamLatch); + continue; + } + List sourceCodeFiles = new ArrayList<>(enrichmentResult.files().values()); wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); logger.info("Size of files {}", wrapper.userPrompt.getFiles().size()); @@ -734,8 +743,8 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque continue; } - if (wrapper.attemptCount == 0) { - outstandingRequests.incrementAndGet(); + if (!wrapper.outstandingTracked) { + incrementOutstanding(wrapper); } if (wrapper.attemptCount > 0) { @@ -771,7 +780,7 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque currentStreamState.pendingIssueIds.remove(instanceId); int completed = processedRequests.incrementAndGet(); - outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapper); requestSemaphore.release(); logger.progress("Processed " + completed + " out of " + totalRequests + " issues (1 failed)."); @@ -802,6 +811,70 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque processingQueue.size(), processedRequests.get(), totalRequests, outstandingRequests.get()); } + private void incrementOutstanding(RequestWrapper wrapper) { + if (!wrapper.outstandingTracked) { + outstandingRequests.incrementAndGet(); + wrapper.outstandingTracked = true; + } + } + + private void decrementOutstanding(RequestWrapper wrapper) { + if (wrapper != null && wrapper.outstandingTracked) { + wrapper.outstandingTracked = false; + if (outstandingRequests.decrementAndGet() < 0) { + outstandingRequests.set(0); + } + } + } + + private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.EnrichmentResult enrichmentResult, + Map responses, AtomicInteger processedRequests, + int totalRequests, CompletableFuture> resultFuture, + CountDownLatch streamLatch) { + String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); + AuditResponse skippedResponse = new AuditResponse(); + skippedResponse.setIssueId(instanceId); + skippedResponse.setStatus("SKIPPED"); + skippedResponse.setStatusMessage(formatSourceFailureMessage(enrichmentResult)); + responses.put(instanceId, skippedResponse); + + currentStreamState.processedIssueIds.add(instanceId); + currentStreamState.pendingIssueIds.remove(instanceId); + decrementOutstanding(wrapper); + requestSemaphore.release(); + + int completed = processedRequests.incrementAndGet(); + logger.warn("Skipping issue %s because required source files could not be loaded", instanceId); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (1 skipped)."); + + if (completed >= totalRequests) { + logger.info("All requests accounted for, completing stream."); + if (requestHandler != null && !requestHandler.isCompleted()) { + requestHandler.complete(); + } + streamLatch.countDown(); + if (!resultFuture.isDone()) { + resultFuture.complete(responses); + } + } + } + + private String formatSourceFailureMessage(SourceCodeEnricher.EnrichmentResult enrichmentResult) { + List filenames = enrichmentResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename) + .distinct() + .collect(Collectors.toList()); + String prefix = filenames.size() == 1 + ? "Could not decode source file: " + : "Could not decode source files: "; + String details = enrichmentResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::message) + .filter(message -> message != null && !message.isBlank()) + .distinct() + .collect(Collectors.joining("; ")); + return prefix + String.join(", ", filenames) + (details.isBlank() ? "" : " (" + details + ")"); + } + private void handleServerBusy(String requestId, int totalRequests, AtomicInteger processedRequests, Map responses, CompletableFuture> resultFuture, CountDownLatch streamLatch) { RequestWrapper wrapperToRetry = inflightRequests.remove(requestId); if (wrapperToRetry == null) { @@ -825,7 +898,8 @@ private void handleServerBusy(String requestId, int totalRequests, AtomicInteger int completed = processedRequests.incrementAndGet(); logger.progress("Request permanently failed due to server busy - Processed " + completed + " out of " + totalRequests + " issues"); - int stillOutstanding = outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapperToRetry); + int stillOutstanding = outstandingRequests.get(); LOG.warn("WARN: Request for instance {} permanently failed. Remaining outstanding requests: {}", wrapperToRetry.userPrompt.getIssueData().getInstanceID(), stillOutstanding); if (completed >= totalRequests) { logger.info("All requests accounted for after permanent failure, completing stream."); @@ -858,7 +932,7 @@ private void submitUserPrompt(RequestWrapper wrapper) { LOG.error("Failed to send request for instance {} after all retries. Re-queueing for later attempt.", wrapper.userPrompt.getIssueData().getInstanceID()); inflightRequests.remove(requestId); requestMetricsMap.remove(requestId); - outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapper); requestSemaphore.release(); wrapper.attemptCount++; processingQueue.addLast(wrapper); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java index 954f048f4b2..9b1d3277f11 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java @@ -17,6 +17,7 @@ class RequestWrapper { final UserPrompt userPrompt; int attemptCount = 0; + volatile boolean outstandingTracked; RequestWrapper(UserPrompt userPrompt) { this.userPrompt = userPrompt; diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java new file mode 100644 index 00000000000..6e821417652 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -0,0 +1,60 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.audit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.audit.model.AuditResponse; + +class AuditFprStatusTest { + + @Test + void determinesAuditStatusForSuccessSkipAndFailureCombinations() { + assertEquals("AUDITED", AuditFPR.determineAuditStatus(3, 0, 3, 3)); + assertEquals("PARTIALLY_AUDITED", AuditFPR.determineAuditStatus(2, 1, 3, 3)); + assertEquals("SKIPPED", AuditFPR.determineAuditStatus(0, 3, 3, 3)); + assertEquals("FAILED", AuditFPR.determineAuditStatus(0, 2, 3, 3)); + assertEquals("FAILED", AuditFPR.determineAuditStatus(0, 3, 3, 2)); + } + + @Test + void countsOnlyExplicitSkippedResponsesAsSkipped() { + AuditResponse skipped = AuditResponse.builder() + .status("SKIPPED") + .statusMessage("Could not decode source file: payments.c") + .build(); + AuditResponse failed = AuditResponse.builder() + .status("FAILED") + .statusMessage("backend error") + .build(); + AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); + + assertEquals(Map.of("Source file decode failed", 1), + AuditFPR.getSkippedAuditReasons(Map.of( + "skipped", skipped, + "failed", failed, + "success", success), 3)); + } + + @Test + void preservesMissingResponseAccountingForFilteredIssues() { + AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); + + assertEquals(Map.of("No audit response received", 1), + AuditFPR.getSkippedAuditReasons(Map.of("success", success), 2)); + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java new file mode 100644 index 00000000000..d0596751d93 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.util.FprHandle; + +class SourceCodeEnricherTest { + + @TempDir + Path tempDir; + + @Test + void reportsEveryUniqueDecodeFailureAndDoesNotHideSuccessfulFiles() throws Exception { + Map sourceFiles = new LinkedHashMap<>(); + sourceFiles.put("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + sourceFiles.put("bad-one.java", new byte[] {(byte) 0xFF}); + sourceFiles.put("bad-two.java", new byte[] {(byte) 0xFE}); + Path fprPath = createFpr(sourceFiles); + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher.EnrichmentResult result = new SourceCodeEnricher( + fprHandle, SourceDecoders.fromToken("UTF-8"), null) + .enrichWithSourceCodeDetailed(List.of(List.of( + element("good.java"), + element("bad-one.java"), + element("bad-two.java"), + element("bad-one.java")))); + + assertTrue(result.hasFailures()); + assertEquals(List.of("bad-one.java", "bad-two.java"), + result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of("good.java"), result.files().keySet().stream().toList()); + } + } + + private static StackTraceElement element(String filename) { + return new StackTraceElement(filename, 1, "", "", null, null, null); + } + + private Path createFpr(Map sourceFiles) throws IOException { + Path fprPath = tempDir.resolve("source-enricher-test.fpr"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(fprPath))) { + writeEntry(zip, "src-archive/index.xml", createIndex(sourceFiles.keySet()).getBytes(StandardCharsets.UTF_8)); + int index = 1; + for (Map.Entry sourceFile : sourceFiles.entrySet()) { + writeEntry(zip, "src-archive/" + index++, sourceFile.getValue()); + } + } + return fprPath; + } + + private static String createIndex(Iterable filenames) { + StringBuilder index = new StringBuilder(""); + int entry = 1; + for (String filename : filenames) { + index.append("src-archive/") + .append(entry++).append(""); + } + return index.append("").toString(); + } + + private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) throws IOException { + zip.putNextEntry(new ZipEntry(name)); + zip.write(bytes); + zip.closeEntry(); + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java index fea7f98fed1..86cf907b83e 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java @@ -66,6 +66,19 @@ void decode_usesFprEncodingWhenMetadataPresent() { assertTrue(result.source().startsWith("FPR("), result.source()); } + @Test + void decode_usesWindows1252FromFprMetadata() { + FVDLMetadata metadata = new FVDLMetadata(); + metadata.registerSourceFileEncoding("payments.c", "windows-1252"); + byte[] windows1252 = new byte[] {(byte) 0x93, 'p', 'r', 'e', 'm', 'i', 'u', 'm', (byte) 0x94}; + + DecodeResult result = SourceDecoders.defaults().decode(windows1252, "payments.c", metadata); + + assertEquals("\u201cpremium\u201d", result.content()); + assertEquals("windows-1252", result.charset().name()); + assertEquals("FPR(windows-1252)", result.source()); + } + @Test void decode_allCandidatesFail_messageListsAttempts() { ISourceDecoder decoder = SourceDecoders.fromCsv("UTF-8,US-ASCII"); diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java index 2596476ec1d..533a607e2d3 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java @@ -197,7 +197,8 @@ public static void setAvailableQuotaBefore(ObjectNode result, long quota) { public static String getProgressMessage(FPRAuditResult auditResult) { switch (auditResult.getStatus()) { case "SKIPPED": - return "No issues to audit, skipping upload"; + return (auditResult.getMessage() != null ? auditResult.getMessage() : "No issues to audit") + + ", skipping upload"; case "FAILED": String message = auditResult.getMessage() != null ? auditResult.getMessage() : "Unknown error"; return "Audit failed: " + message; From c57efdbe935827d633c3a7a582748d64147e6018 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Mon, 10 Aug 2026 14:37:18 +0530 Subject: [PATCH 06/11] chore: make source enrichment concurrency-safe, retry-safe, and logging-consistent --- .../aviator/fpr/utils/SourceCodeEnricher.java | 45 ++++-- .../aviator/grpc/AviatorStreamProcessor.java | 44 ++++-- .../cli/aviator/grpc/RequestWrapper.java | 1 + .../fpr/utils/SourceCodeEnricherTest.java | 149 ++++++++++++++++++ 4 files changed, 214 insertions(+), 25 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index b5cb97c5ab4..9336ce6f139 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +44,7 @@ public class SourceCodeEnricher { private final FprHandle fprHandle; private final FileUtils fileUtils; + private final Map sourceFileCache = new ConcurrentHashMap<>(); public SourceCodeEnricher(FprHandle fprHandle) { this(fprHandle, SourceDecoders.defaults(), null); @@ -123,22 +125,41 @@ private void processFileForElement(StackTraceElement element, Map String filename = element.getFilename(); if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) && !uniqueFiles.containsKey(filename) && !failuresByFilename.containsKey(filename)) { - try { - String content = fileUtils.readSourceFileContentStrict(fprHandle, filename); - File file = new File(); - file.setName(filename); - file.setSegment(false); - file.setStartLine(1); - file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); - file.setEndLine(content.split("\\R", -1).length); - uniqueFiles.put(filename, file); - } catch (IOException | ISourceDecoder.SourceDecodeException e) { - logger.warn("Could not read source file content for path {}: {}", filename, e.getMessage()); - failuresByFilename.put(filename, new SourceFileFailure(filename, e.getMessage())); + CachedSourceResult result = sourceFileCache.computeIfAbsent(filename, this::loadSourceFile); + if (result.failure() != null) { + failuresByFilename.put(filename, result.failure()); + } else { + uniqueFiles.put(filename, result.sourceFile().toFile(filename)); } } } + private CachedSourceResult loadSourceFile(String filename) { + try { + String content = fileUtils.readSourceFileContentStrict(fprHandle, filename); + CachedSourceFile sourceFile = new CachedSourceFile( + fileUtils.appendLineNumbers(content, filename, 0), content.split("\\R", -1).length); + return new CachedSourceResult(sourceFile, null); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { + logger.warn("Could not read source file content for path {}: {}", filename, e.getMessage()); + return new CachedSourceResult(null, new SourceFileFailure(filename, e.getMessage())); + } + } + + private record CachedSourceFile(String content, int endLine) { + private File toFile(String filename) { + File file = new File(); + file.setName(filename); + file.setSegment(false); + file.setStartLine(1); + file.setContent(content); + file.setEndLine(endLine); + return file; + } + } + + private record CachedSourceResult(CachedSourceFile sourceFile, SourceFileFailure failure) {} + public record EnrichmentResult(Map files, List failures) { public boolean hasFailures() { return !failures.isEmpty(); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 2d86749c719..1d1efd948e6 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -98,6 +98,9 @@ class AviatorStreamProcessor implements AutoCloseable { private final FprHandle fprHandle; private final ISourceDecoder sourceDecoder; private final FVDLMetadata fvdlMetadata; + private final SourceCodeEnricher sourceCodeEnricher; + private final AtomicInteger skippedRequests = new AtomicInteger(); + private final AtomicInteger failedRequests = new AtomicInteger(); public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, @@ -113,6 +116,7 @@ public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, A this.fprHandle = fprHandle; this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); this.fvdlMetadata = fvdlMetadata; + this.sourceCodeEnricher = new SourceCodeEnricher(fprHandle, this.sourceDecoder, fvdlMetadata); } public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, List customPriorityOrder) { @@ -721,20 +725,13 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); - // Lazy Loading of source code files for individual issue - SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle, sourceDecoder, fvdlMetadata); - - SourceCodeEnricher.EnrichmentResult enrichmentResult = - sourceCodeEnricher.enrichWithSourceCodeDetailed(wrapper.userPrompt.getStackTrace()); - if (enrichmentResult.hasFailures()) { + SourceCodeEnricher.EnrichmentResult enrichmentResult = enrichSourceCode(wrapper); + if (enrichmentResult != null && enrichmentResult.hasFailures()) { completeSkippedRequest(wrapper, enrichmentResult, responses, processedRequests, totalRequests, resultFuture, streamLatch); continue; } - List sourceCodeFiles = new ArrayList<>(enrichmentResult.files().values()); - wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); - logger.info("Size of files {}", wrapper.userPrompt.getFiles().size()); logger.info("Size of programming language {}", wrapper.userPrompt.getProgrammingLanguages().size()); @@ -780,10 +777,11 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque currentStreamState.pendingIssueIds.remove(instanceId); int completed = processedRequests.incrementAndGet(); + int failed = failedRequests.incrementAndGet(); decrementOutstanding(wrapper); requestSemaphore.release(); - logger.progress("Processed " + completed + " out of " + totalRequests + " issues (1 failed)."); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (" + failed + " failed)."); } else { LOG.error("Caught AviatorSimpleException but the request wrapper was null.", e); } @@ -811,6 +809,24 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque processingQueue.size(), processedRequests.get(), totalRequests, outstandingRequests.get()); } + private SourceCodeEnricher.EnrichmentResult enrichSourceCode(RequestWrapper wrapper) { + synchronized (wrapper) { + if (wrapper.sourceCodeEnriched) { + return null; + } + + SourceCodeEnricher.EnrichmentResult enrichmentResult = + sourceCodeEnricher.enrichWithSourceCodeDetailed(wrapper.userPrompt.getStackTrace()); + if (!enrichmentResult.hasFailures()) { + List sourceCodeFiles = + new ArrayList<>(enrichmentResult.files().values()); + wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); + wrapper.sourceCodeEnriched = true; + } + return enrichmentResult; + } + } + private void incrementOutstanding(RequestWrapper wrapper) { if (!wrapper.outstandingTracked) { outstandingRequests.incrementAndGet(); @@ -832,10 +848,11 @@ private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.E int totalRequests, CompletableFuture> resultFuture, CountDownLatch streamLatch) { String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); + String failureMessage = formatSourceFailureMessage(enrichmentResult); AuditResponse skippedResponse = new AuditResponse(); skippedResponse.setIssueId(instanceId); skippedResponse.setStatus("SKIPPED"); - skippedResponse.setStatusMessage(formatSourceFailureMessage(enrichmentResult)); + skippedResponse.setStatusMessage(failureMessage); responses.put(instanceId, skippedResponse); currentStreamState.processedIssueIds.add(instanceId); @@ -844,8 +861,9 @@ private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.E requestSemaphore.release(); int completed = processedRequests.incrementAndGet(); - logger.warn("Skipping issue %s because required source files could not be loaded", instanceId); - logger.progress("Processed " + completed + " out of " + totalRequests + " issues (1 skipped)."); + int skipped = skippedRequests.incrementAndGet(); + LOG.warn("Skipping issue {}: {}", instanceId, failureMessage); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (" + skipped + " skipped)."); if (completed >= totalRequests) { logger.info("All requests accounted for, completing stream."); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java index 9b1d3277f11..65ddee600c5 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java @@ -18,6 +18,7 @@ class RequestWrapper { final UserPrompt userPrompt; int attemptCount = 0; volatile boolean outstandingTracked; + volatile boolean sourceCodeEnriched; RequestWrapper(UserPrompt userPrompt) { this.userPrompt = userPrompt; diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java index d0596751d93..3856ddca2eb 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java @@ -22,6 +22,12 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -29,6 +35,7 @@ import org.junit.jupiter.api.io.TempDir; import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FprHandle; class SourceCodeEnricherTest { @@ -60,6 +67,148 @@ void reportsEveryUniqueDecodeFailureAndDoesNotHideSuccessfulFiles() throws Excep } } + @Test + void cachesSuccessfulAndFailedLoadsAcrossEnrichmentCalls() throws Exception { + Map sourceFiles = new LinkedHashMap<>(); + sourceFiles.put("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + sourceFiles.put("bad.java", new byte[] {(byte) 0xFF}); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + if (bytes.length > 0 && bytes[0] == (byte) 0xFF) { + throw new SourceDecodeException("decode failed"); + } + return new DecodeResult(new String(bytes, StandardCharsets.UTF_8), StandardCharsets.UTF_8, "test"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("good.java"), element("bad.java"))); + + SourceCodeEnricher.EnrichmentResult firstResult = enricher.enrichWithSourceCodeDetailed(stackTraces); + SourceCodeEnricher.EnrichmentResult secondResult = enricher.enrichWithSourceCodeDetailed(stackTraces); + + assertEquals(2, decodeCalls.get()); + assertEquals(List.of("good.java"), secondResult.files().keySet().stream().toList()); + assertEquals(List.of("bad.java"), secondResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(firstResult.files().get("good.java").getContent(), secondResult.files().get("good.java").getContent()); + } + } + + @Test + void loadsAFileOnceWhenEnrichmentCallsAreConcurrent() throws Exception { + Map sourceFiles = Map.of("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + CountDownLatch decodeStarted = new CountDownLatch(1); + CountDownLatch allowDecode = new CountDownLatch(1); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + decodeStarted.countDown(); + try { + if (!allowDecode.await(5, TimeUnit.SECONDS)) { + throw new SourceDecodeException("Timed out waiting to decode"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SourceDecodeException("Interrupted while decoding", e); + } + return new DecodeResult(new String(bytes, StandardCharsets.UTF_8), StandardCharsets.UTF_8, "test"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("good.java"))); + Future first = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + Future second = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + + assertTrue(decodeStarted.await(5, TimeUnit.SECONDS)); + allowDecode.countDown(); + + assertEquals(List.of("good.java"), first.get(5, TimeUnit.SECONDS).files().keySet().stream().toList()); + assertEquals(List.of("good.java"), second.get(5, TimeUnit.SECONDS).files().keySet().stream().toList()); + assertEquals(1, decodeCalls.get()); + } finally { + executor.shutdownNow(); + } + } + } + + @Test + void cachesConcurrentDecodeFailureOnce() throws Exception { + Map sourceFiles = Map.of("bad.java", new byte[] {(byte) 0xFF}); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + CountDownLatch decodeStarted = new CountDownLatch(1); + CountDownLatch allowDecode = new CountDownLatch(1); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + decodeStarted.countDown(); + try { + if (!allowDecode.await(5, TimeUnit.SECONDS)) { + throw new SourceDecodeException("Timed out waiting to decode"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SourceDecodeException("Interrupted while decoding", e); + } + throw new SourceDecodeException("decode failed"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("bad.java"))); + Future first = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + Future second = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + + assertTrue(decodeStarted.await(5, TimeUnit.SECONDS)); + allowDecode.countDown(); + + assertEquals(List.of("bad.java"), first.get(5, TimeUnit.SECONDS).failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of("bad.java"), second.get(5, TimeUnit.SECONDS).failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(1, decodeCalls.get()); + } finally { + executor.shutdownNow(); + } + } + } + private static StackTraceElement element(String filename) { return new StackTraceElement(filename, 1, "", "", null, null, null); } From 8efbc8b9dd46db8b64631329245f6fce43cb9f73 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Thu, 13 Aug 2026 15:07:42 +0530 Subject: [PATCH 07/11] chore: exclude locally skipped issues from submitted count --- .../fortify/cli/aviator/audit/AuditFPR.java | 15 ++++-- .../aviator/audit/model/AuditResponse.java | 19 ++++++++ .../aviator/audit/model/FPRAuditResult.java | 12 ++++- .../aviator/grpc/AviatorStreamProcessor.java | 1 + .../fortify/cli/aviator/grpc/GrpcUtil.java | 1 + .../cli/aviator/audit/AuditFprStatusTest.java | 24 ++++++++++ .../ssc/helper/AviatorSSCAuditHelper.java | 2 +- .../cli/cmd/AviatorSSCAuditCommandTest.java | 48 +++++++++++++++++++ 8 files changed, 117 insertions(+), 5 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 377a75a5dfb..63fd4a43e28 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -161,13 +161,16 @@ private static FPRAuditResult finalizeFprAudit( Map issueCategoryLookup, FPRInfo fprInfo, StreamingFVDLProcessor streamingFVDLProcessor) { int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit(); + int issuesSubmitted = getSubmittedAuditCount(auditResponses); if (auditResponses.isEmpty()) { if (totalIssuesToAudit == 0) { LOG.info("No issues were audited, skipping update and upload"); - return new FPRAuditResult(null, "SKIPPED", "No issues to audit", 0, totalIssuesToAudit); + return new FPRAuditResult(null, "SKIPPED", "No issues to audit", 0, totalIssuesToAudit, + issuesSubmitted, 0, Map.of(), 0, Map.of()); } else { LOG.error("No audit responses received for {} issues", totalIssuesToAudit); - return new FPRAuditResult(null, "FAILED", "No audit responses received from server", 0, totalIssuesToAudit); + return new FPRAuditResult(null, "FAILED", "No audit responses received from server", 0, totalIssuesToAudit, + issuesSubmitted, 0, Map.of(), 0, Map.of()); } } @@ -215,10 +218,16 @@ private static FPRAuditResult finalizeFprAudit( LOG.info("FPR audit process completed with status: {}", status); return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit, - issuesSkipped, skippedByReason, remediationGenerationMetric.skippedRemediations(), + issuesSubmitted, issuesSkipped, skippedByReason, remediationGenerationMetric.skippedRemediations(), remediationGenerationMetric.skippedByReason()); } + static int getSubmittedAuditCount(Map auditResponses) { + return (int) auditResponses.values().stream() + .filter(AuditResponse::isSubmittedToAviator) + .count(); + } + static String determineAuditStatus(long issuesSuccessfullyAudited, int issuesSkipped, int totalIssuesToAudit, int responseCount) { if (issuesSuccessfullyAudited == totalIssuesToAudit) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java index 220379cf776..1f2d912b41d 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java @@ -12,6 +12,7 @@ */ package com.fortify.cli.aviator.audit.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; import lombok.AllArgsConstructor; @@ -33,6 +34,8 @@ public class AuditResponse { private String status; private String statusMessage; private String issueId; + @JsonIgnore + private boolean submittedToAviator; private String tier; private String aviatorPredictionTag; @@ -40,4 +43,20 @@ public class AuditResponse { private String userPrompt; private String systemPrompt; + public AuditResponse(AuditResult auditResult, int inputToken, int outputToken, String status, + String statusMessage, String issueId, String tier, String aviatorPredictionTag, + Boolean isAviatorProcessed, String userPrompt, String systemPrompt) { + this.auditResult = auditResult; + this.inputToken = inputToken; + this.outputToken = outputToken; + this.status = status; + this.statusMessage = statusMessage; + this.issueId = issueId; + this.tier = tier; + this.aviatorPredictionTag = aviatorPredictionTag; + this.isAviatorProcessed = isAviatorProcessed; + this.userPrompt = userPrompt; + this.systemPrompt = systemPrompt; + } + } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java index e2f55d51b62..db7704cd69e 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java @@ -24,6 +24,7 @@ public class FPRAuditResult { private String message; private int issuesSuccessfullyAudited; private int totalIssuesToAudit; + private int issuesSubmitted; private int issuesSkipped; private Map skippedByReason; private int remediationGenerationSkipped; @@ -32,18 +33,27 @@ public class FPRAuditResult { public FPRAuditResult(File updatedFile, String status, String message, int issuesSuccessfullyAudited, int totalIssuesToAudit) { this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, - Math.max(0, totalIssuesToAudit - issuesSuccessfullyAudited), Map.of(), 0, Map.of()); + totalIssuesToAudit, Math.max(0, totalIssuesToAudit - issuesSuccessfullyAudited), Map.of(), 0, Map.of()); } public FPRAuditResult(File updatedFile, String status, String message, int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSkipped, Map skippedByReason, int remediationGenerationSkipped, Map remediationGenerationSkippedByReason) { + this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, totalIssuesToAudit, + issuesSkipped, skippedByReason, remediationGenerationSkipped, remediationGenerationSkippedByReason); + } + + public FPRAuditResult(File updatedFile, String status, String message, + int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSubmitted, + int issuesSkipped, Map skippedByReason, int remediationGenerationSkipped, + Map remediationGenerationSkippedByReason) { this.updatedFile = updatedFile; this.status = status; this.message = message; this.issuesSuccessfullyAudited = issuesSuccessfullyAudited; this.totalIssuesToAudit = totalIssuesToAudit; + this.issuesSubmitted = issuesSubmitted; this.issuesSkipped = issuesSkipped; this.skippedByReason = skippedByReason == null ? Map.of() : Map.copyOf(skippedByReason); this.remediationGenerationSkipped = remediationGenerationSkipped; diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 1d1efd948e6..8b2bb1a3fa2 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -912,6 +912,7 @@ private void handleServerBusy(String requestId, int totalRequests, AtomicInteger failedResponse.setIssueId(wrapperToRetry.userPrompt.getIssueData().getInstanceID()); failedResponse.setStatus("RETRY_LIMIT_EXCEEDED"); failedResponse.setStatusMessage("Request failed after " + Constants.MAX_RETRIES + " retries due to server overload."); + failedResponse.setSubmittedToAviator(true); responses.put(wrapperToRetry.userPrompt.getIssueData().getInstanceID(), failedResponse); int completed = processedRequests.incrementAndGet(); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java index 2d44b56f599..8b33c95dd7b 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java @@ -196,6 +196,7 @@ static AuditResponse convertToAuditResponse(AuditorResponse response) { auditResponse.setStatus(response.getStatus()); auditResponse.setStatusMessage(response.getStatusMessage()); auditResponse.setIssueId(response.getIssueId()); + auditResponse.setSubmittedToAviator(true); auditResponse.setTier(response.getTier()); auditResponse.setAviatorPredictionTag(response.getAviatorPredictionTag()); auditResponse.setIsAviatorProcessed(response.getIsAviatorProcessed()); diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java index 6e821417652..342edb28383 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -57,4 +57,28 @@ void preservesMissingResponseAccountingForFilteredIssues() { assertEquals(Map.of("No audit response received", 1), AuditFPR.getSkippedAuditReasons(Map.of("success", success), 2)); } + + @Test + void countsOnlyResponsesOriginatingFromAviatorAsSubmitted() { + AuditResponse localSkip = AuditResponse.builder().status("SKIPPED").build(); + AuditResponse localFailure = AuditResponse.builder() + .status("FAILED") + .statusMessage("Request validation failed") + .build(); + AuditResponse serverSkip = AuditResponse.builder() + .status("SKIPPED") + .submittedToAviator(true) + .build(); + AuditResponse serverFailure = AuditResponse.builder() + .status("FAILED") + .statusMessage("Aviator processing failed") + .submittedToAviator(true) + .build(); + + assertEquals(2, AuditFPR.getSubmittedAuditCount(Map.of( + "local", localSkip, + "localFailure", localFailure, + "server", serverSkip, + "serverFailure", serverFailure))); + } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java index 533a607e2d3..e9ce3228879 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java @@ -135,7 +135,7 @@ public static void setAuditStats(ObjectNode result, FPRAuditResult auditResult) break; } audit.put("message", message); - audit.put("submitted", auditResult.getTotalIssuesToAudit()); + audit.put("submitted", auditResult.getIssuesSubmitted()); audit.put("succeeded", auditResult.getIssuesSuccessfullyAudited()); audit.put("skipped", auditResult.getIssuesSkipped()); audit.put("skippedReasons", formatSkippedReasons(auditResult.getSkippedByReason())); diff --git a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java index e98fa9a5587..d8988e8a860 100644 --- a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java +++ b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java @@ -19,9 +19,15 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Map; import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator.audit.model.FPRAuditResult; +import com.fortify.cli.aviator.ssc.helper.AviatorSSCAuditHelper; +import com.fortify.cli.common.json.JsonHelper; + import picocli.CommandLine; class AviatorSSCAuditCommandTest { @@ -46,6 +52,48 @@ void testAllowsNoFilterSetOption() { assertNull(cmd.getFilterSetTitleOrId()); } + @Test + void reportsDecodeSkippedIssuesAsNotSubmitted() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "SKIPPED", "All 6 issues were skipped", 0, 6, 0, 6, + Map.of("Source file decode failed", 6), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + var audit = result.path("operation").path("audit"); + assertEquals(0, audit.get("submitted").asInt()); + assertEquals(0, audit.get("succeeded").asInt()); + assertEquals(6, audit.get("skipped").asInt()); + } + + @Test + void reportsServerSkippedIssuesAsSubmitted() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "SKIPPED", "Skipped by Aviator", 0, 1, 1, 1, + Map.of("Skipped by Aviator", 1), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + assertEquals(1, result.path("operation").path("audit").get("submitted").asInt()); + } + + @Test + void excludesOnlyDecodeSkippedIssuesFromSubmittedCount() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "PARTIALLY_AUDITED", null, 2, 4, 3, 2, + Map.of("Source file decode failed", 1, "Skipped by Aviator", 1), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + var audit = result.path("operation").path("audit"); + assertEquals(3, audit.get("submitted").asInt()); + assertEquals(2, audit.get("succeeded").asInt()); + assertEquals(2, audit.get("skipped").asInt()); + } + private static AviatorSSCAuditCommand parse(String... args) { var cmd = new AviatorSSCAuditCommand(); var fullArgs = new ArrayList(); From f05ccd6eef641986a7d62cfaae4792a5be3a4045 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Wed, 19 Aug 2026 15:11:11 +0530 Subject: [PATCH 08/11] chore: replace CP850 with ISO-8859-1 default source encoding --- .../cli/aviator/fpr/utils/SourceDecoders.java | 2 +- .../cli/aviator/fpr/utils/SourceDecodersTest.java | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java index 7c047085fe0..7b231926127 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java @@ -21,7 +21,7 @@ * token/CSV parsing and default candidate order. CLI converters should delegate here. */ public final class SourceDecoders { - public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,CP850"; + public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,ISO-8859-1"; private SourceDecoders() {} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java index 86cf907b83e..bdc8f254957 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java @@ -40,7 +40,7 @@ void fromToken_rejectsBlankAndUnknownCharset() { @Test void defaults_matchDocumentedCandidateOrder() { - assertEquals("FPR,UTF-8,CP850", SourceDecoders.DEFAULT_SOURCE_ENCODINGS); + assertEquals("FPR,UTF-8,ISO-8859-1", SourceDecoders.DEFAULT_SOURCE_ENCODINGS); assertEquals(SourceDecoders.DEFAULT_SOURCE_ENCODINGS, SourceDecoders.defaults().describe()); } @@ -53,6 +53,17 @@ void decode_fallsThroughFprWhenMetadataMissing() { assertEquals("UTF-8", result.source()); } + @Test + void decode_fallsThroughUtf8ToIso88591ByDefault() { + byte[] iso88591 = {(byte) 0xE9}; + + DecodeResult result = SourceDecoders.defaults().decode(iso88591, "Main.java", null); + + assertEquals("\u00e9", result.content()); + assertEquals(StandardCharsets.ISO_8859_1, result.charset()); + assertEquals("ISO-8859-1", result.source()); + } + @Test void decode_usesFprEncodingWhenMetadataPresent() { FVDLMetadata metadata = new FVDLMetadata(); From 7eb018951e8d7eef6273a14ae86331c32d6a6448 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Wed, 19 Aug 2026 17:52:07 +0530 Subject: [PATCH 09/11] chore: enhance audit response handling with skip reasons and improve source file failure reporting --- .../fortify/cli/aviator/audit/AuditFPR.java | 29 ++------ .../aviator/audit/model/AuditResponse.java | 72 ++++++++++++++++++- .../aviator/fpr/utils/SourceCodeEnricher.java | 19 +++-- .../aviator/grpc/AviatorStreamProcessor.java | 19 +++-- .../fortify/cli/aviator/grpc/GrpcUtil.java | 4 ++ .../cli/aviator/audit/AuditFprStatusTest.java | 12 +++- .../fpr/utils/SourceCodeEnricherTest.java | 24 +++++++ 7 files changed, 142 insertions(+), 37 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 63fd4a43e28..2a228e6daaa 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -28,6 +28,7 @@ import com.fortify.cli.aviator.audit.model.AuditFprOptions; import com.fortify.cli.aviator.audit.model.AuditOutcome; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.FPRAuditResult; import com.fortify.cli.aviator.audit.model.FilterSelection; import com.fortify.cli.aviator.audit.model.ParsedFprData; @@ -256,32 +257,10 @@ static Map getSkippedAuditReasons(Map au } private static String getSkippedAuditReason(AuditResponse response) { - String statusMessage = response == null ? null : response.getStatusMessage(); - String message = statusMessage == null || statusMessage.isBlank() - ? response == null ? null : response.getStatus() - : statusMessage; - if (message == null || message.isBlank()) { - return "Unknown audit failure"; + if (response == null) { + return AuditSkipReason.UNKNOWN.displayMessage(null, null); } - if (message.startsWith("Client-side pre-processing error: ")) { - message = message.substring("Client-side pre-processing error: ".length()); - } - if (message.startsWith("Could not decode source file")) { - return "Source file decode failed"; - } - if (message.contains("was not found in the FPR")) { - return "Source file not found in FPR"; - } - if (message.contains("could not be read from the FPR")) { - return "Source file read failed"; - } - if ("FAILED".equalsIgnoreCase(message)) { - return "Audit failed"; - } - if ("SKIPPED".equalsIgnoreCase(message)) { - return "Skipped by Aviator"; - } - return message; + return response.getAuditSkipReason().displayMessage(response.getStatus(), response.getStatusMessage()); } private static void recordSkipped(Map skippedByReason, String reason) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java index 1f2d912b41d..cae9d0a2d45 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java @@ -12,6 +12,8 @@ */ package com.fortify.cli.aviator.audit.model; +import java.util.regex.Pattern; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; @@ -26,13 +28,76 @@ @Builder @Reflectable public class AuditResponse { + public enum AuditSkipReason { + SOURCE_FILE_DECODE_FAILED( + "Could not decode source file%s: %s%s", + "Source file decode failed", + Pattern.compile("^Could not decode source file.*")), + SOURCE_FILE_NOT_FOUND( + "%s was not found in the FPR", + "Source file not found in FPR", + Pattern.compile(".* was not found in the FPR.*")), + SOURCE_FILE_READ_FAILED( + "%s could not be read from the FPR%s", + "Source file read failed", + Pattern.compile(".* could not be read from the FPR.*")), + AUDIT_FAILED("FAILED", "Audit failed", Pattern.compile("^FAILED$", Pattern.CASE_INSENSITIVE)), + SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator", Pattern.compile("^SKIPPED$", Pattern.CASE_INSENSITIVE)), + UNKNOWN(null, "Unknown audit failure", null), + OTHER(null, null, null); + + private static final String CLIENT_SIDE_ERROR_PREFIX = "Client-side pre-processing error: "; + + private final String messageFormat; + private final String displayMessage; + private final Pattern messagePattern; + + AuditSkipReason(String messageFormat, String displayMessage, Pattern messagePattern) { + this.messageFormat = messageFormat; + this.displayMessage = displayMessage; + this.messagePattern = messagePattern; + } + + public String format(Object... args) { + return String.format(messageFormat, args); + } + + public static AuditSkipReason from(String status, String statusMessage) { + String message = effectiveMessage(status, statusMessage); + if (message.isBlank()) { + return UNKNOWN; + } + for (AuditSkipReason reason : values()) { + if (reason.messagePattern != null && reason.messagePattern.matcher(message).matches()) { + return reason; + } + } + return OTHER; + } + + public String displayMessage(String status, String statusMessage) { + return displayMessage == null ? effectiveMessage(status, statusMessage) : displayMessage; + } + + private static String effectiveMessage(String status, String statusMessage) { + String message = statusMessage == null || statusMessage.isBlank() ? status : statusMessage; + if (message == null || message.isBlank()) { + return ""; + } + return message.startsWith(CLIENT_SIDE_ERROR_PREFIX) + ? message.substring(CLIENT_SIDE_ERROR_PREFIX.length()) + : message; + } + } + private AuditResult auditResult; private int inputToken; private int outputToken; - private String status; private String statusMessage; + @JsonIgnore + private AuditSkipReason auditSkipReason; private String issueId; @JsonIgnore private boolean submittedToAviator; @@ -43,6 +108,11 @@ public class AuditResponse { private String userPrompt; private String systemPrompt; + @JsonIgnore + public AuditSkipReason getAuditSkipReason() { + return auditSkipReason == null ? AuditSkipReason.from(status, statusMessage) : auditSkipReason; + } + public AuditResponse(AuditResult auditResult, int inputToken, int outputToken, String status, String statusMessage, String issueId, String tier, String aviatorPredictionTag, Boolean isAviatorProcessed, String userPrompt, String systemPrompt) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index 9336ce6f139..873a5a82fec 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.StackTraceElement; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; @@ -140,12 +141,18 @@ private CachedSourceResult loadSourceFile(String filename) { CachedSourceFile sourceFile = new CachedSourceFile( fileUtils.appendLineNumbers(content, filename, 0), content.split("\\R", -1).length); return new CachedSourceResult(sourceFile, null); - } catch (IOException | ISourceDecoder.SourceDecodeException e) { - logger.warn("Could not read source file content for path {}: {}", filename, e.getMessage()); - return new CachedSourceResult(null, new SourceFileFailure(filename, e.getMessage())); + } catch (ISourceDecoder.SourceDecodeException e) { + return failedSourceFile(filename, e, AuditSkipReason.SOURCE_FILE_DECODE_FAILED); + } catch (IOException e) { + return failedSourceFile(filename, e, AuditSkipReason.SOURCE_FILE_READ_FAILED); } } + private CachedSourceResult failedSourceFile(String filename, Exception exception, AuditSkipReason reason) { + logger.warn("Could not read source file content for path {}: {}", filename, exception.getMessage()); + return new CachedSourceResult(null, new SourceFileFailure(filename, exception.getMessage(), reason)); + } + private record CachedSourceFile(String content, int endLine) { private File toFile(String filename) { File file = new File(); @@ -166,5 +173,9 @@ public boolean hasFailures() { } } - public record SourceFileFailure(String filename, String message) {} + public record SourceFileFailure(String filename, String message, AuditSkipReason reason) { + public SourceFileFailure { + Objects.requireNonNull(reason, "reason"); + } + } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 8b2bb1a3fa2..533018004fd 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -48,6 +48,7 @@ import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.QuotaBasedFilter; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; @@ -848,10 +849,12 @@ private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.E int totalRequests, CompletableFuture> resultFuture, CountDownLatch streamLatch) { String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); - String failureMessage = formatSourceFailureMessage(enrichmentResult); + AuditSkipReason skipReason = enrichmentResult.failures().get(0).reason(); + String failureMessage = formatSourceFailureMessage(enrichmentResult, skipReason); AuditResponse skippedResponse = new AuditResponse(); skippedResponse.setIssueId(instanceId); skippedResponse.setStatus("SKIPPED"); + skippedResponse.setAuditSkipReason(skipReason); skippedResponse.setStatusMessage(failureMessage); responses.put(instanceId, skippedResponse); @@ -877,20 +880,24 @@ private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.E } } - private String formatSourceFailureMessage(SourceCodeEnricher.EnrichmentResult enrichmentResult) { + private String formatSourceFailureMessage(SourceCodeEnricher.EnrichmentResult enrichmentResult, + AuditSkipReason skipReason) { List filenames = enrichmentResult.failures().stream() .map(SourceCodeEnricher.SourceFileFailure::filename) .distinct() .collect(Collectors.toList()); - String prefix = filenames.size() == 1 - ? "Could not decode source file: " - : "Could not decode source files: "; String details = enrichmentResult.failures().stream() .map(SourceCodeEnricher.SourceFileFailure::message) .filter(message -> message != null && !message.isBlank()) .distinct() .collect(Collectors.joining("; ")); - return prefix + String.join(", ", filenames) + (details.isBlank() ? "" : " (" + details + ")"); + String detailSuffix = details.isBlank() ? "" : " (" + details + ")"; + return switch (skipReason) { + case SOURCE_FILE_DECODE_FAILED -> skipReason.format( + filenames.size() == 1 ? "" : "s", String.join(", ", filenames), detailSuffix); + case SOURCE_FILE_READ_FAILED -> skipReason.format(String.join(", ", filenames), detailSuffix); + default -> details.isBlank() ? skipReason.displayMessage("SKIPPED", null) : details; + }; } private void handleServerBusy(String requestId, int totalRequests, AtomicInteger processedRequests, Map responses, CompletableFuture> resultFuture, CountDownLatch streamLatch) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java index 8b33c95dd7b..db16a416a73 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java @@ -29,6 +29,7 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.Autoremediation; import com.fortify.cli.aviator.audit.model.Change; import com.fortify.cli.aviator.audit.model.StackTraceElement; @@ -195,6 +196,9 @@ static AuditResponse convertToAuditResponse(AuditorResponse response) { auditResponse.setOutputToken(response.getOutputToken()); auditResponse.setStatus(response.getStatus()); auditResponse.setStatusMessage(response.getStatusMessage()); + if ("SKIPPED".equalsIgnoreCase(response.getStatus())) { + auditResponse.setAuditSkipReason(AuditSkipReason.from(response.getStatus(), response.getStatusMessage())); + } auditResponse.setIssueId(response.getIssueId()); auditResponse.setSubmittedToAviator(true); auditResponse.setTier(response.getTier()); diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java index 342edb28383..2a0b91316bc 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Test; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; class AuditFprStatusTest { @@ -35,7 +36,8 @@ void determinesAuditStatusForSuccessSkipAndFailureCombinations() { void countsOnlyExplicitSkippedResponsesAsSkipped() { AuditResponse skipped = AuditResponse.builder() .status("SKIPPED") - .statusMessage("Could not decode source file: payments.c") + .auditSkipReason(AuditSkipReason.SOURCE_FILE_DECODE_FAILED) + .statusMessage("The source decoder wording can change") .build(); AuditResponse failed = AuditResponse.builder() .status("FAILED") @@ -50,6 +52,14 @@ void countsOnlyExplicitSkippedResponsesAsSkipped() { "success", success), 3)); } + @Test + void classifiesLegacyServerMessagesAtTheResponseBoundary() { + assertEquals(AuditSkipReason.SOURCE_FILE_READ_FAILED, + AuditSkipReason.from("SKIPPED", "example could not be read from the FPR")); + assertEquals(AuditSkipReason.SOURCE_FILE_NOT_FOUND, + AuditSkipReason.from("SKIPPED", "example was not found in the FPR")); + } + @Test void preservesMissingResponseAccountingForFilteredIssues() { AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java index 3856ddca2eb..3e42be508d8 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.StackTraceElement; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FprHandle; @@ -63,10 +64,25 @@ void reportsEveryUniqueDecodeFailureAndDoesNotHideSuccessfulFiles() throws Excep assertTrue(result.hasFailures()); assertEquals(List.of("bad-one.java", "bad-two.java"), result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of(AuditSkipReason.SOURCE_FILE_DECODE_FAILED, AuditSkipReason.SOURCE_FILE_DECODE_FAILED), + result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::reason).toList()); assertEquals(List.of("good.java"), result.files().keySet().stream().toList()); } } + @Test + void classifiesReadFailuresSeparatelyFromDecodeFailures() throws Exception { + Path fprPath = createFprWithMissingSource("missing.java"); + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher.EnrichmentResult result = new SourceCodeEnricher( + fprHandle, SourceDecoders.fromToken("UTF-8"), null) + .enrichWithSourceCodeDetailed(List.of(List.of(element("missing.java")))); + + assertEquals(AuditSkipReason.SOURCE_FILE_READ_FAILED, result.failures().get(0).reason()); + } + } + @Test void cachesSuccessfulAndFailedLoadsAcrossEnrichmentCalls() throws Exception { Map sourceFiles = new LinkedHashMap<>(); @@ -225,6 +241,14 @@ private Path createFpr(Map sourceFiles) throws IOException { return fprPath; } + private Path createFprWithMissingSource(String filename) throws IOException { + Path fprPath = tempDir.resolve("source-enricher-missing-source.fpr"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(fprPath))) { + writeEntry(zip, "src-archive/index.xml", createIndex(List.of(filename)).getBytes(StandardCharsets.UTF_8)); + } + return fprPath; + } + private static String createIndex(Iterable filenames) { StringBuilder index = new StringBuilder(""); int entry = 1; From 227627dcb8b7e8a610e93ee2e7e19b646f2fc988 Mon Sep 17 00:00:00 2001 From: kireetivar Date: Wed, 19 Aug 2026 18:08:54 +0530 Subject: [PATCH 10/11] chore: update audit skip reason handling to use predefined constants and improve test coverage --- .../aviator/audit/model/AuditResponse.java | 54 +++++-------------- .../fortify/cli/aviator/grpc/GrpcUtil.java | 2 +- .../cli/aviator/audit/AuditFprStatusTest.java | 12 +++-- .../cli/aviator/grpc/GrpcUtilTest.java | 15 ++++++ 4 files changed, 35 insertions(+), 48 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java index cae9d0a2d45..bbe5094f79f 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java @@ -12,8 +12,6 @@ */ package com.fortify.cli.aviator.audit.model; -import java.util.regex.Pattern; - import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; @@ -31,62 +29,34 @@ public class AuditResponse { public enum AuditSkipReason { SOURCE_FILE_DECODE_FAILED( "Could not decode source file%s: %s%s", - "Source file decode failed", - Pattern.compile("^Could not decode source file.*")), + "Source file decode failed"), SOURCE_FILE_NOT_FOUND( "%s was not found in the FPR", - "Source file not found in FPR", - Pattern.compile(".* was not found in the FPR.*")), + "Source file not found in FPR"), SOURCE_FILE_READ_FAILED( "%s could not be read from the FPR%s", - "Source file read failed", - Pattern.compile(".* could not be read from the FPR.*")), - AUDIT_FAILED("FAILED", "Audit failed", Pattern.compile("^FAILED$", Pattern.CASE_INSENSITIVE)), - SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator", Pattern.compile("^SKIPPED$", Pattern.CASE_INSENSITIVE)), - UNKNOWN(null, "Unknown audit failure", null), - OTHER(null, null, null); - - private static final String CLIENT_SIDE_ERROR_PREFIX = "Client-side pre-processing error: "; + "Source file read failed"), + AUDIT_FAILED("FAILED", "Audit failed"), + SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator"), + UNKNOWN(null, "Unknown audit failure"), + OTHER(null, null); private final String messageFormat; private final String displayMessage; - private final Pattern messagePattern; - AuditSkipReason(String messageFormat, String displayMessage, Pattern messagePattern) { + AuditSkipReason(String messageFormat, String displayMessage) { this.messageFormat = messageFormat; this.displayMessage = displayMessage; - this.messagePattern = messagePattern; } public String format(Object... args) { return String.format(messageFormat, args); } - public static AuditSkipReason from(String status, String statusMessage) { - String message = effectiveMessage(status, statusMessage); - if (message.isBlank()) { - return UNKNOWN; - } - for (AuditSkipReason reason : values()) { - if (reason.messagePattern != null && reason.messagePattern.matcher(message).matches()) { - return reason; - } - } - return OTHER; - } - public String displayMessage(String status, String statusMessage) { - return displayMessage == null ? effectiveMessage(status, statusMessage) : displayMessage; - } - - private static String effectiveMessage(String status, String statusMessage) { - String message = statusMessage == null || statusMessage.isBlank() ? status : statusMessage; - if (message == null || message.isBlank()) { - return ""; - } - return message.startsWith(CLIENT_SIDE_ERROR_PREFIX) - ? message.substring(CLIENT_SIDE_ERROR_PREFIX.length()) - : message; + return displayMessage == null + ? statusMessage == null || statusMessage.isBlank() ? status : statusMessage + : displayMessage; } } @@ -110,7 +80,7 @@ private static String effectiveMessage(String status, String statusMessage) { @JsonIgnore public AuditSkipReason getAuditSkipReason() { - return auditSkipReason == null ? AuditSkipReason.from(status, statusMessage) : auditSkipReason; + return auditSkipReason == null ? AuditSkipReason.UNKNOWN : auditSkipReason; } public AuditResponse(AuditResult auditResult, int inputToken, int outputToken, String status, diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java index db16a416a73..1fc5785ca33 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java @@ -197,7 +197,7 @@ static AuditResponse convertToAuditResponse(AuditorResponse response) { auditResponse.setStatus(response.getStatus()); auditResponse.setStatusMessage(response.getStatusMessage()); if ("SKIPPED".equalsIgnoreCase(response.getStatus())) { - auditResponse.setAuditSkipReason(AuditSkipReason.from(response.getStatus(), response.getStatusMessage())); + auditResponse.setAuditSkipReason(AuditSkipReason.SKIPPED_BY_AVIATOR); } auditResponse.setIssueId(response.getIssueId()); auditResponse.setSubmittedToAviator(true); diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java index 2a0b91316bc..1aee8e87934 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -53,11 +53,13 @@ void countsOnlyExplicitSkippedResponsesAsSkipped() { } @Test - void classifiesLegacyServerMessagesAtTheResponseBoundary() { - assertEquals(AuditSkipReason.SOURCE_FILE_READ_FAILED, - AuditSkipReason.from("SKIPPED", "example could not be read from the FPR")); - assertEquals(AuditSkipReason.SOURCE_FILE_NOT_FOUND, - AuditSkipReason.from("SKIPPED", "example was not found in the FPR")); + void doesNotInferSkipReasonFromStatusMessage() { + AuditResponse response = AuditResponse.builder() + .status("SKIPPED") + .statusMessage("example could not be read from the FPR") + .build(); + + assertEquals(AuditSkipReason.UNKNOWN, response.getAuditSkipReason()); } @Test diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java index 2f503d3d0a5..09285bae9e9 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java @@ -19,6 +19,9 @@ import org.junit.jupiter.api.Test; +import com.fortify.aviator.grpc.AuditorResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.UserPrompt; @@ -43,4 +46,16 @@ void shouldPreserveResolvedProgrammingLanguages() { assertEquals(List.of("PLSQL"), auditRequest.getProgrammingLanguagesList()); assertEquals("PLSQL", auditRequest.getLanguage()); } + + @Test + void shouldClassifySkippedServerResponseWithoutInspectingMessage() { + AuditorResponse response = AuditorResponse.newBuilder() + .setStatus("SKIPPED") + .setStatusMessage("example could not be read from the FPR") + .build(); + + AuditResponse auditResponse = GrpcUtil.convertToAuditResponse(response); + + assertEquals(AuditSkipReason.SKIPPED_BY_AVIATOR, auditResponse.getAuditSkipReason()); + } } \ No newline at end of file From f493388ef1faf7ad0fc19428aec6032417fc8d3d Mon Sep 17 00:00:00 2001 From: kireetivar Date: Wed, 19 Aug 2026 19:05:59 +0530 Subject: [PATCH 11/11] chore: simplify audit skip reason handling --- .../fortify/cli/aviator/audit/AuditFPR.java | 6 +-- .../aviator/audit/model/AuditResponse.java | 52 ++++--------------- .../aviator/grpc/AviatorStreamProcessor.java | 2 +- .../cli/aviator/audit/AuditFprStatusTest.java | 10 ---- 4 files changed, 11 insertions(+), 59 deletions(-) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 2a228e6daaa..5888d18affc 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -28,7 +28,6 @@ import com.fortify.cli.aviator.audit.model.AuditFprOptions; import com.fortify.cli.aviator.audit.model.AuditOutcome; import com.fortify.cli.aviator.audit.model.AuditResponse; -import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.FPRAuditResult; import com.fortify.cli.aviator.audit.model.FilterSelection; import com.fortify.cli.aviator.audit.model.ParsedFprData; @@ -257,10 +256,7 @@ static Map getSkippedAuditReasons(Map au } private static String getSkippedAuditReason(AuditResponse response) { - if (response == null) { - return AuditSkipReason.UNKNOWN.displayMessage(null, null); - } - return response.getAuditSkipReason().displayMessage(response.getStatus(), response.getStatusMessage()); + return response.getAuditSkipReason().getDisplayMessage(); } private static void recordSkipped(Map skippedByReason, String reason) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java index bbe5094f79f..8406c07cdde 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java @@ -15,49 +15,37 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; @Data -@AllArgsConstructor +@AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor @Builder @Reflectable public class AuditResponse { + @Getter + @RequiredArgsConstructor public enum AuditSkipReason { SOURCE_FILE_DECODE_FAILED( "Could not decode source file%s: %s%s", - "Source file decode failed"), - SOURCE_FILE_NOT_FOUND( - "%s was not found in the FPR", - "Source file not found in FPR"), + "Source file decode failed"), SOURCE_FILE_READ_FAILED( "%s could not be read from the FPR%s", - "Source file read failed"), - AUDIT_FAILED("FAILED", "Audit failed"), - SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator"), - UNKNOWN(null, "Unknown audit failure"), - OTHER(null, null); + "Source file read failed"), + SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator"); private final String messageFormat; private final String displayMessage; - AuditSkipReason(String messageFormat, String displayMessage) { - this.messageFormat = messageFormat; - this.displayMessage = displayMessage; - } - public String format(Object... args) { return String.format(messageFormat, args); } - - public String displayMessage(String status, String statusMessage) { - return displayMessage == null - ? statusMessage == null || statusMessage.isBlank() ? status : statusMessage - : displayMessage; - } } private AuditResult auditResult; @@ -77,26 +65,4 @@ public String displayMessage(String status, String statusMessage) { private Boolean isAviatorProcessed; private String userPrompt; private String systemPrompt; - - @JsonIgnore - public AuditSkipReason getAuditSkipReason() { - return auditSkipReason == null ? AuditSkipReason.UNKNOWN : auditSkipReason; - } - - public AuditResponse(AuditResult auditResult, int inputToken, int outputToken, String status, - String statusMessage, String issueId, String tier, String aviatorPredictionTag, - Boolean isAviatorProcessed, String userPrompt, String systemPrompt) { - this.auditResult = auditResult; - this.inputToken = inputToken; - this.outputToken = outputToken; - this.status = status; - this.statusMessage = statusMessage; - this.issueId = issueId; - this.tier = tier; - this.aviatorPredictionTag = aviatorPredictionTag; - this.isAviatorProcessed = isAviatorProcessed; - this.userPrompt = userPrompt; - this.systemPrompt = systemPrompt; - } - } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 533018004fd..959c68a53f2 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -896,7 +896,7 @@ private String formatSourceFailureMessage(SourceCodeEnricher.EnrichmentResult en case SOURCE_FILE_DECODE_FAILED -> skipReason.format( filenames.size() == 1 ? "" : "s", String.join(", ", filenames), detailSuffix); case SOURCE_FILE_READ_FAILED -> skipReason.format(String.join(", ", filenames), detailSuffix); - default -> details.isBlank() ? skipReason.displayMessage("SKIPPED", null) : details; + case SKIPPED_BY_AVIATOR -> details.isBlank() ? skipReason.getDisplayMessage() : details; }; } diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java index 1aee8e87934..0dc06bc834f 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -52,16 +52,6 @@ void countsOnlyExplicitSkippedResponsesAsSkipped() { "success", success), 3)); } - @Test - void doesNotInferSkipReasonFromStatusMessage() { - AuditResponse response = AuditResponse.builder() - .status("SKIPPED") - .statusMessage("example could not be read from the FPR") - .build(); - - assertEquals(AuditSkipReason.UNKNOWN, response.getAuditSkipReason()); - } - @Test void preservesMissingResponseAccountingForFilteredIssues() { AuditResponse success = AuditResponse.builder().status("SUCCESS").build();