diff --git a/fcli-core/fcli-app/build.gradle.kts b/fcli-core/fcli-app/build.gradle.kts index 12c38c2a221..3cbddfca67c 100644 --- a/fcli-core/fcli-app/build.gradle.kts +++ b/fcli-core/fcli-app/build.gradle.kts @@ -8,7 +8,7 @@ plugins { // Inter-project dependencies val refs = listOf( "fcliCommonRef","fcliCommonThirdpartyRef","fcliCommonCiRef","fcliCommonActionRef","fcliCommonToolRef", - "fcliActionRef","fcliAiAssistRef","fcliAviatorRef","fcliConfigRef", + "fcliActionRef","fcliAiAssistRef","fcliAviatorCommonRef","fcliAviatorRef","fcliConfigRef", "fcliFoDRef","fcliSSCRef","fcliSCSastRef","fcliSCDastRef", "fcliToolRef","fcliLicenseRef","fcliUtilRef" ) diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java new file mode 100644 index 00000000000..083d74bbfe1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.converter; + +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; + +import picocli.CommandLine.ITypeConverter; +import picocli.CommandLine.TypeConversionException; + +/** + * Picocli adapter: maps a single {@code --source-encodings} token to an + * {@link ISourceDecoder} via the domain factory {@link SourceDecoders}. + */ +public final class SourceDecoderConverter implements ITypeConverter { + @Override + public ISourceDecoder convert(String value) { + try { + return SourceDecoders.fromToken(value); + } catch (IllegalArgumentException e) { + // Covers blank tokens, IllegalCharsetNameException, UnsupportedCharsetException + throw new TypeConversionException( + e.getMessage() != null ? e.getMessage() : "Invalid source encoding '" + value + "'"); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java new file mode 100644 index 00000000000..e58f0e86d0d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/_common/cli/mixin/SourceEncodingsMixin.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.mixin; + +import java.util.List; + +import com.fortify.cli.aviator._common.cli.converter.SourceDecoderConverter; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; + +import lombok.Getter; +import picocli.CommandLine.Option; + +/** + * Shared {@code --source-encodings} option for Aviator commands that decode + * (and optionally re-encode) source files from an FPR. + */ +public class SourceEncodingsMixin { + @Getter + @Option(names = {"--source-encodings"}, + split = ",", + converter = SourceDecoderConverter.class, + defaultValue = SourceDecoders.DEFAULT_SOURCE_ENCODINGS, + paramLabel = "encoding", + descriptionKey = "fcli.aviator.source-encodings") + private List sourceDecoders; + + /** + * Returns a single decoder that tries the configured candidates in order. + */ + public ISourceDecoder getSourceDecoder() { + return SourceDecoders.of(sourceDecoders); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java index e2db9d777cb..23b77f0119a 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/applyRemediation/ApplyAutoRemediationOnSource.java @@ -12,6 +12,8 @@ */ package com.fortify.cli.aviator.applyRemediation; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +22,8 @@ import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor; import com.fortify.cli.aviator.fpr.processor.RemediationProcessor.RemediationMetric; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; @@ -28,6 +32,12 @@ public class ApplyAutoRemediationOnSource { public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, IAviatorLogger logger) throws AviatorSimpleException, AviatorTechnicalException { + return applyRemediations(fprHandle, sourceCodeDirectory, SourceDecoders.defaults(), logger); + } + + public static RemediationMetric applyRemediations(FprHandle fprHandle, String sourceCodeDirectory, + ISourceDecoder sourceDecoder, IAviatorLogger logger) + throws AviatorSimpleException, AviatorTechnicalException { LOG.info("Starting apply auto-remediation process for file: {}", fprHandle.getFprPath()); @@ -37,8 +47,8 @@ public static RemediationMetric applyRemediations(FprHandle fprHandle, String so } LOG.info("FPR validation successful"); - RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory); + RemediationProcessor remediationProcessor = new RemediationProcessor(fprHandle, sourceCodeDirectory, + Objects.requireNonNull(sourceDecoder, "sourceDecoder")); return remediationProcessor.processRemediationXML(); - } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 4571af0bc7f..5888d18affc 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -14,6 +14,7 @@ import java.io.File; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -39,6 +40,7 @@ import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.ResourceUtil; @@ -52,8 +54,11 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) options.getFprHandle().validate(); AviatorConfigManager.getInstance(); + // Non-null: AuditFprOptions defaults via @Builder.Default; CLI mixin always supplies a decoder. + ISourceDecoder sourceDecoder = options.getSourceDecoder(); + // --- STAGE 1: PARSING --- - ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle()); + ParsedFprData parsedData = prepareAndParseFpr(options.getFprHandle(), sourceDecoder); TagMappingConfig tagMappingConfig = loadTagMappingConfig(options.getTagMappingPath()); Map issueCategoryLookup = tagMappingConfig.requiresCategoryForSuppressionEvaluation() ? buildIssueCategoryLookup(parsedData.vulnerabilities) @@ -69,22 +74,21 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) Map auditResponses = new ConcurrentHashMap<>(); AuditOutcome auditOutcome = performAviatorAudit( parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(), - auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder() + auditResponses, filterSelection, options.getFprHandle(), options.getFolderPriorityOrder(), sourceDecoder ); // --- STAGE 4: FINALIZATION --- return finalizeFprAudit( auditOutcome, auditResponses, parsedData.auditProcessor, - tagMappingConfig, issueCategoryLookup, parsedData.fprInfo + tagMappingConfig, issueCategoryLookup, parsedData.fprInfo, parsedData.streamingFVDLProcessor ); } - private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle) { + private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle, ISourceDecoder sourceDecoder) { try { // Processors now take the FprHandle directly, no more extracted path - AuditProcessor auditProcessor = new AuditProcessor(fprHandle); - //FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle); - StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle); + AuditProcessor auditProcessor = new AuditProcessor(fprHandle, sourceDecoder); + StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); Map auditIssueMap = auditProcessor.processAuditXML(); FPRProcessor fprProcessor = new FPRProcessor(fprHandle, auditIssueMap, auditProcessor); @@ -126,7 +130,8 @@ private static Map buildIssueCategoryLookup(List private static AuditOutcome performAviatorAudit( ParsedFprData parsedData, IAviatorLogger logger, String token, String appVersion, String url, String sscAppName, String sscAppVersion, - Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, List folderPriorityOrder) { + Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle, + List folderPriorityOrder, ISourceDecoder sourceDecoder) { SourceLanguageResolver sourceLanguageResolver = new SourceLanguageResolver(parsedData.streamingFVDLProcessor.getFvdlMetadata()); parsedData.streamingFVDLProcessor.getFvdlMetadata().clearSourceFileTypeIndexes(); @@ -141,7 +146,9 @@ private static AuditOutcome performAviatorAudit( filterSelection, logger, folderPriorityOrder, - sourceLanguageResolver + sourceLanguageResolver, + sourceDecoder, + parsedData.streamingFVDLProcessor.getFvdlMetadata() ); return issueAuditor.performAudit( auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url, fprHandle @@ -151,33 +158,37 @@ private static AuditOutcome performAviatorAudit( private static FPRAuditResult finalizeFprAudit( AuditOutcome auditOutcome, Map auditResponses, AuditProcessor auditProcessor, TagMappingConfig tagMappingConfig, - Map issueCategoryLookup, FPRInfo fprInfo) { + Map issueCategoryLookup, FPRInfo fprInfo, StreamingFVDLProcessor streamingFVDLProcessor) { int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit(); + int issuesSubmitted = getSubmittedAuditCount(auditResponses); if (auditResponses.isEmpty()) { if (totalIssuesToAudit == 0) { LOG.info("No issues were audited, skipping update and upload"); - return new FPRAuditResult(null, "SKIPPED", "No issues to audit", 0, totalIssuesToAudit); + return new FPRAuditResult(null, "SKIPPED", "No issues to audit", 0, totalIssuesToAudit, + issuesSubmitted, 0, Map.of(), 0, Map.of()); } else { LOG.error("No audit responses received for {} issues", totalIssuesToAudit); - return new FPRAuditResult(null, "FAILED", "No audit responses received from server", 0, totalIssuesToAudit); + return new FPRAuditResult(null, "FAILED", "No audit responses received from server", 0, totalIssuesToAudit, + issuesSubmitted, 0, Map.of(), 0, Map.of()); } } long issuesSuccessfullyAudited = auditResponses.values().stream() .filter(response -> "SUCCESS".equalsIgnoreCase(response.getStatus())) .count(); + Map skippedByReason = getSkippedAuditReasons(auditResponses, totalIssuesToAudit); + int issuesSkipped = skippedByReason.values().stream().mapToInt(Integer::intValue).sum(); String status; String message = null; - if (issuesSuccessfullyAudited == totalIssuesToAudit) { - status = "AUDITED"; - } else if (issuesSuccessfullyAudited > 0) { - status = "PARTIALLY_AUDITED"; - } else { - status = "FAILED"; + status = determineAuditStatus(issuesSuccessfullyAudited, issuesSkipped, totalIssuesToAudit, auditResponses.size()); + if ("SKIPPED".equals(status)) { + message = String.format("All %d issues were skipped", totalIssuesToAudit); + } else if ("FAILED".equals(status)) { String commonFailureReason = auditResponses.values().stream() + .filter(response -> !"SKIPPED".equalsIgnoreCase(response.getStatus())) .map(AuditResponse::getStatusMessage) .filter(msg -> msg != null && !msg.isBlank()) .findFirst() @@ -186,16 +197,69 @@ private static FPRAuditResult finalizeFprAudit( if (commonFailureReason.startsWith("Client-side pre-processing error: ")) { commonFailureReason = commonFailureReason.substring("Client-side pre-processing error: ".length()); } - message = String.format("All %d issues failed (%s)", totalIssuesToAudit, commonFailureReason); + message = String.format("No issues were audited (%d skipped; failure details: %s)", + issuesSkipped, commonFailureReason); } File updatedFile = null; if (issuesSuccessfullyAudited > 0) { updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml( - auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo); + auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, + streamingFVDLProcessor.getFvdlMetadata()); + } + AuditProcessor.RemediationGenerationMetric remediationGenerationMetric = auditProcessor.getLastRemediationGenerationMetric(); + + if (!skippedByReason.isEmpty()) { + LOG.info("Skipped audit issues by reason: {}", skippedByReason); + } + if (!remediationGenerationMetric.skippedByReason().isEmpty()) { + LOG.info("Skipped audit remediation generation by reason: {}", remediationGenerationMetric.skippedByReason()); } LOG.info("FPR audit process completed with status: {}", status); - return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit); + return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit, + issuesSubmitted, issuesSkipped, skippedByReason, remediationGenerationMetric.skippedRemediations(), + remediationGenerationMetric.skippedByReason()); + } + + static int getSubmittedAuditCount(Map auditResponses) { + return (int) auditResponses.values().stream() + .filter(AuditResponse::isSubmittedToAviator) + .count(); + } + + static String determineAuditStatus(long issuesSuccessfullyAudited, int issuesSkipped, + int totalIssuesToAudit, int responseCount) { + if (issuesSuccessfullyAudited == totalIssuesToAudit) { + return "AUDITED"; + } + if (issuesSuccessfullyAudited > 0) { + return "PARTIALLY_AUDITED"; + } + if (issuesSkipped == totalIssuesToAudit && responseCount == totalIssuesToAudit) { + return "SKIPPED"; + } + return "FAILED"; + } + + static Map getSkippedAuditReasons(Map auditResponses, int totalIssuesToAudit) { + Map skippedByReason = new LinkedHashMap<>(); + auditResponses.values().stream() + .filter(response -> "SKIPPED".equalsIgnoreCase(response.getStatus())) + .map(AuditFPR::getSkippedAuditReason) + .forEach(reason -> recordSkipped(skippedByReason, reason)); + int missingResponses = Math.max(0, totalIssuesToAudit - auditResponses.size()); + if (missingResponses > 0) { + skippedByReason.merge("No audit response received", missingResponses, Integer::sum); + } + return skippedByReason; + } + + private static String getSkippedAuditReason(AuditResponse response) { + return response.getAuditSkipReason().getDisplayMessage(); + } + + private static void recordSkipped(Map skippedByReason, String reason) { + skippedByReason.merge(reason, 1, Integer::sum); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index d25c1286183..dc5b6897892 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -48,7 +49,10 @@ import com.fortify.cli.aviator.fpr.filter.VulnerabilityFilterer; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.grpc.AviatorGrpcClient; import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper; import com.fortify.cli.aviator.util.Constants; @@ -80,6 +84,8 @@ public class IssueAuditor { private TagDefinition humanAuditTag; private TagDefinition aviatorStatusTag; private final SourceLanguageResolver sourceLanguageResolver; + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; private final IAviatorLogger logger; private final List customPriorityOrder; @@ -88,6 +94,15 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, SourceLanguageResolver sourceLanguageResolver) { + this(vulnerabilities, auditProcessor, auditIssueMap, fprInfo, SSCApplicationName, SSCApplicationVersion, + filterSelection, logger, customPriorityOrder, sourceLanguageResolver, SourceDecoders.defaults(), null); + } + + public IssueAuditor(List vulnerabilities, AuditProcessor auditProcessor, Map auditIssueMap, + FPRInfo fprInfo, String SSCApplicationName, String SSCApplicationVersion, + FilterSelection filterSelection, IAviatorLogger logger, List customPriorityOrder, + SourceLanguageResolver sourceLanguageResolver, ISourceDecoder sourceDecoder, + FVDLMetadata fvdlMetadata) { this.logger = logger; this.customPriorityOrder = customPriorityOrder; this.MAX_PER_CATEGORY = Constants.MAX_PER_CATEGORY; @@ -103,6 +118,8 @@ public IssueAuditor(List vulnerabilities, AuditProcessor auditPro this.SSCApplicationName = SSCApplicationName; this.SSCApplicationVersion = SSCApplicationVersion; this.sourceLanguageResolver = sourceLanguageResolver; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fvdlMetadata = fvdlMetadata; this.analysisTag = fprInfo.getFilterTemplate().getTagDefinitions().stream().filter(t -> "Analysis".equalsIgnoreCase(t.getName())).findFirst().orElse(null); this.resultsTag = resolveResultTag("", "", analysisTag); } @@ -162,7 +179,8 @@ public AuditOutcome performAudit(Map auditResponses, Stri } else { try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) { CompletableFuture> future = - client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token, fprHandle, customPriorityOrder); + client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, + SSCApplicationVersion, token, fprHandle, customPriorityOrder, sourceDecoder, fvdlMetadata); Map responses = future.get(500, TimeUnit.MINUTES); responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response)); logger.progress("Audit completed"); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java index 54b8c803038..8fa13198222 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditFprOptions.java @@ -14,6 +14,8 @@ import java.util.List; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.FprHandle; import lombok.Builder; @@ -34,4 +36,5 @@ public class AuditFprOptions { private final boolean noFilterSet; private final List folderNames; private final List folderPriorityOrder; + @Builder.Default private final ISourceDecoder sourceDecoder = SourceDecoders.defaults(); } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java index 220379cf776..8406c07cdde 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/AuditResponse.java @@ -12,32 +12,57 @@ */ package com.fortify.cli.aviator.audit.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; @Data -@AllArgsConstructor +@AllArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor @Builder @Reflectable public class AuditResponse { + @Getter + @RequiredArgsConstructor + public enum AuditSkipReason { + SOURCE_FILE_DECODE_FAILED( + "Could not decode source file%s: %s%s", + "Source file decode failed"), + SOURCE_FILE_READ_FAILED( + "%s could not be read from the FPR%s", + "Source file read failed"), + SKIPPED_BY_AVIATOR("SKIPPED", "Skipped by Aviator"); + + private final String messageFormat; + private final String displayMessage; + + public String format(Object... args) { + return String.format(messageFormat, args); + } + } + private AuditResult auditResult; private int inputToken; private int outputToken; - private String status; private String statusMessage; + @JsonIgnore + private AuditSkipReason auditSkipReason; private String issueId; + @JsonIgnore + private boolean submittedToAviator; private String tier; private String aviatorPredictionTag; private Boolean isAviatorProcessed; private String userPrompt; private String systemPrompt; - } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java index 94d8b9a4e18..db7704cd69e 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/FPRAuditResult.java @@ -13,6 +13,7 @@ package com.fortify.cli.aviator.audit.model; import java.io.File; +import java.util.Map; import lombok.Data; @@ -23,13 +24,39 @@ public class FPRAuditResult { private String message; private int issuesSuccessfullyAudited; private int totalIssuesToAudit; + private int issuesSubmitted; + private int issuesSkipped; + private Map skippedByReason; + private int remediationGenerationSkipped; + private Map remediationGenerationSkippedByReason; public FPRAuditResult(File updatedFile, String status, String message, int issuesSuccessfullyAudited, int totalIssuesToAudit) { + this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, + totalIssuesToAudit, Math.max(0, totalIssuesToAudit - issuesSuccessfullyAudited), Map.of(), 0, Map.of()); + } + + public FPRAuditResult(File updatedFile, String status, String message, + int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSkipped, + Map skippedByReason, int remediationGenerationSkipped, + Map remediationGenerationSkippedByReason) { + this(updatedFile, status, message, issuesSuccessfullyAudited, totalIssuesToAudit, totalIssuesToAudit, + issuesSkipped, skippedByReason, remediationGenerationSkipped, remediationGenerationSkippedByReason); + } + + public FPRAuditResult(File updatedFile, String status, String message, + int issuesSuccessfullyAudited, int totalIssuesToAudit, int issuesSubmitted, + int issuesSkipped, Map skippedByReason, int remediationGenerationSkipped, + Map remediationGenerationSkippedByReason) { this.updatedFile = updatedFile; this.status = status; this.message = message; this.issuesSuccessfullyAudited = issuesSuccessfullyAudited; this.totalIssuesToAudit = totalIssuesToAudit; + this.issuesSubmitted = issuesSubmitted; + this.issuesSkipped = issuesSkipped; + this.skippedByReason = skippedByReason == null ? Map.of() : Map.copyOf(skippedByReason); + this.remediationGenerationSkipped = remediationGenerationSkipped; + this.remediationGenerationSkippedByReason = remediationGenerationSkippedByReason == null ? Map.of() : Map.copyOf(remediationGenerationSkippedByReason); } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index 602367fbf03..d919ff321b6 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -27,8 +27,10 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,7 +61,10 @@ import com.fortify.cli.aviator.config.TagMappingConfig; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -83,9 +88,39 @@ public class AuditProcessor { private final Map auditIssueMap = new HashMap<>(); private final FprHandle fprHandle; + private final ISourceDecoder sourceDecoder; + private RemediationGenerationMetric lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); + + public record RemediationGenerationMetric(int skippedRemediations, Map skippedByReason) { + public static RemediationGenerationMetric empty() { + return new RemediationGenerationMetric(0, Map.of()); + } + } + + private enum RemediationSkipReason { + SOURCE_READ_OR_DECODE_FAILED("Source file read/decode failed"), + INVALID_LINE_NUMBER("Invalid line number"), + STRUCTURALLY_INVALID("Structurally invalid remediation"), + NO_VALID_CHANGES("No valid changes"); + + private final String label; + + RemediationSkipReason(String label) { + this.label = label; + } + } public AuditProcessor(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults()); + } + + public AuditProcessor(FprHandle fprHandle, ISourceDecoder sourceDecoder) { this.fprHandle = fprHandle; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + } + + public RemediationGenerationMetric getLastRemediationGenerationMetric() { + return lastRemediationGenerationMetric; } /** @@ -709,6 +744,13 @@ private String addCommentToIssueElement(Element issueElement, String commentText public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, TagMappingConfig tagMappingConfig, Map issueCategoryLookup, FPRInfo fprInfo) throws AviatorTechnicalException { + return updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, issueCategoryLookup, fprInfo, null); + } + + public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, + TagMappingConfig tagMappingConfig, Map issueCategoryLookup, + FPRInfo fprInfo, FVDLMetadata fvdlMetadata) throws AviatorTechnicalException { + lastRemediationGenerationMetric = RemediationGenerationMetric.empty(); // Step 1: Apply this save's audit responses. writtenInstanceIds is the local retain set. Map effectiveIssueCategoryLookup = issueCategoryLookup == null ? Map.of() : issueCategoryLookup; AuditXmlUpdateResult updateResult = updateAuditXml( @@ -728,7 +770,10 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi // Step 4: Generate the in-memory remediations.xml document if needed. if (hasRemediations && !remediationCommentTimestamps.isEmpty()) { - this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo); + Map skippedByReason = new LinkedHashMap<>(); + this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo, + fvdlMetadata, skippedByReason); + lastRemediationGenerationMetric = toRemediationGenerationMetric(skippedByReason); } else { this.remediationsDoc = null; if (hasRemediations) { @@ -765,7 +810,8 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi private Document generateRemediationsXml(Map auditResponses, Map remediationCommentTimestamps, - FPRInfo fprInfo) throws AviatorTechnicalException { + FPRInfo fprInfo, FVDLMetadata fvdlMetadata, + Map skippedByReason) throws AviatorTechnicalException { try { DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -794,6 +840,7 @@ private Document generateRemediationsXml(Map auditRespons Element remediationListElement = finalDoc.createElementNS(REMEDIATIONS_NAMESPACE_URI, "RemediationList"); rootElement.appendChild(remediationListElement); + FileUtils fileUtils = new FileUtils(this.sourceDecoder, fvdlMetadata); int validRemediationCount = 0; for (Map.Entry entry : auditResponses.entrySet()) { @@ -828,12 +875,11 @@ private Document generateRemediationsXml(Map auditRespons filenameElement.setTextContent(filename); fileChangesElement.appendChild(filenameElement); - //Optional originalFileContentOptional = fvdlProcessor.getSourceFileContent(filename); - FileUtils fileUtils = new FileUtils(); - Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); + Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); if (originalFileContentOptional.isEmpty()) { logger.warn("WARN: Could not retrieve source code for file '{}'. Skipping remediation generation for this file for instanceId '{}'.", filename, instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.SOURCE_READ_OR_DECODE_FAILED); continue; } @@ -900,6 +946,7 @@ private Document generateRemediationsXml(Map auditRespons fileChangesElement.appendChild(changeElement); } catch (NumberFormatException e) { logger.error("Skipping change for issue {} due to invalid line number format. Details: {}", instanceId, e.getMessage()); + recordSkipped(skippedByReason, RemediationSkipReason.INVALID_LINE_NUMBER); } } if (fileChangesElement.getElementsByTagNameNS(REMEDIATIONS_NAMESPACE_URI, "Change").getLength() > 0) { @@ -915,9 +962,11 @@ private Document generateRemediationsXml(Map auditRespons validRemediationCount++; } else { logger.warn("WARN: Skipping structurally invalid remediation for issue instanceId: {}", instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.STRUCTURALLY_INVALID); } } else { logger.warn("WARN: Skipping remediation for instanceId '{}' because all of its proposed changes were invalid and could not be processed.", instanceId); + recordSkipped(skippedByReason, RemediationSkipReason.NO_VALID_CHANGES); } } @@ -929,6 +978,15 @@ private Document generateRemediationsXml(Map auditRespons } } + private RemediationGenerationMetric toRemediationGenerationMetric(Map skippedByReason) { + int skippedRemediations = skippedByReason.values().stream().mapToInt(Integer::intValue).sum(); + return new RemediationGenerationMetric(skippedRemediations, Map.copyOf(skippedByReason)); + } + + private void recordSkipped(Map skippedByReason, RemediationSkipReason reason) { + skippedByReason.merge(reason.label, 1, Integer::sum); + } + private String calculateHashBase64(String content, String algorithm) { if (content == null) return ""; try { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java index 97c1ffce863..2a059b8380b 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/RemediationProcessor.java @@ -14,11 +14,7 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; -import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -32,6 +28,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.zip.ZipFile; @@ -49,6 +46,12 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.FuzzyContextSearcher; @@ -58,6 +61,7 @@ public class RemediationProcessor { private final FprHandle fprHandle; private final String sourceCodeDirectory; + private final ISourceDecoder sourceDecoder; public record RemediationMetric(int totalRemediations, int appliedRemediations, int skippedRemediations, Set modifiedFiles, Map skippedByReason) { @@ -66,16 +70,14 @@ public RemediationMetric(int totalRemediations, int appliedRemediations, int ski } } - private record FvdlMetadataResult(FVDLMetadata metadata, SkipReason skipReason) {} + private record SourceFileContent(String content, Charset charset, String encodingSource) {} - private record PendingFileWrite(String filename, Path filePath, String content, byte[] updatedBytes) {} + private record PendingFileWrite(String filename, Path filePath, String content, Charset charset, String encodingSource, + byte[] updatedBytes) {} private record RollbackFileWrite(String filename, Path filePath, byte[] originalBytes) {} private enum SkipReason { - FVDL_METADATA_UNAVAILABLE("FVDL metadata unavailable"), - FVDL_ENCODING_MISSING("FVDL source encoding missing"), - FVDL_ENCODING_UNSUPPORTED("FVDL source encoding unsupported"), SOURCE_FILE_MISSING("Source file missing"), SOURCE_FILE_OUTSIDE_SOURCE_DIR("Source file outside source directory"), SOURCE_READ_FAILED("Source file read failed"), @@ -136,8 +138,13 @@ private static class RollbackRemediationException extends AviatorTechnicalExcept } public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory) { + this(fprHandle, sourceCodeDirectory, SourceDecoders.defaults()); + } + + public RemediationProcessor(FprHandle fprHandle, String sourceCodeDirectory, ISourceDecoder sourceDecoder) { this.fprHandle = fprHandle; this.sourceCodeDirectory = sourceCodeDirectory; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); } public RemediationMetric processRemediationXML() { @@ -157,7 +164,7 @@ public RemediationMetric processRemediationXML() { } final Path sourceBasePath = Paths.get(trimmedSourceDir).toAbsolutePath().normalize(); LOG.debug("Applying remediations from {} to source directory {}", remediationPath, sourceBasePath); - final FvdlMetadataResult fvdlMetadataResult = loadFvdlMetadata(); + final FVDLMetadata fvdlMetadata = loadFvdlMetadata(); try (InputStream remediationStream = Files.newInputStream(remediationPath)) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -176,7 +183,7 @@ public RemediationMetric processRemediationXML() { appliedRemediations = 0; for (int i = 0; i < remediationNodes.getLength(); i++) { Element remediation = (Element) remediationNodes.item(i); - if (processRemediation(remediation, sourceBasePath, fvdlMetadataResult, modifiedFiles, skippedByReason)) { + if (processRemediation(remediation, sourceBasePath, fvdlMetadata, modifiedFiles, skippedByReason)) { appliedRemediations++; } } @@ -198,11 +205,11 @@ public RemediationMetric processRemediationXML() { return new RemediationMetric(totalRemediations, appliedRemediations, skippedRemediations, modifiedFiles, skippedByReason); } - private boolean processRemediation(Element remediation, Path sourceBasePath, FvdlMetadataResult fvdlMetadataResult, + private boolean processRemediation(Element remediation, Path sourceBasePath, FVDLMetadata fvdlMetadata, Set modifiedFiles, Map skippedByReason) { String instanceId = remediation.getAttribute("instanceId"); try { - Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadataResult); + Map pendingWrites = prepareFileChanges(remediation, sourceBasePath, fvdlMetadata); if (pendingWrites.isEmpty()) { recordSkipped(skippedByReason, SkipReason.NO_CHANGES); return false; @@ -230,7 +237,7 @@ private boolean processRemediation(Element remediation, Path sourceBasePath, Fvd } private Map prepareFileChanges(Element remediation, Path sourceBasePath, - FvdlMetadataResult fvdlMetadataResult) { + FVDLMetadata fvdlMetadata) { NodeList fileChangesNodes = remediation.getElementsByTagNameNS(NAMESPACE_URI, "FileChanges"); if (fileChangesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No file changes found"); @@ -238,12 +245,12 @@ private Map prepareFileChanges(Element remediation, Path Map pendingWrites = new LinkedHashMap<>(); for (int j = 0; j < fileChangesNodes.getLength(); j++) { - processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadataResult, pendingWrites); + processFileChanges(remediation, (Element) fileChangesNodes.item(j), sourceBasePath, fvdlMetadata, pendingWrites); } return pendingWrites; } - private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FvdlMetadataResult fvdlMetadataResult, + private boolean processFileChanges(Element remediation, Element fileChanges, Path sourceBasePath, FVDLMetadata fvdlMetadata, Map pendingWrites) { String instanceId = remediation.getAttribute("instanceId"); String filename = getRequiredElementText(fileChanges, "Filename"); @@ -260,23 +267,25 @@ private boolean processFileChanges(Element remediation, Element fileChanges, Pat } String fileHash = getRequiredElementText(fileChanges, "Hash"); - Charset sourceEncoding = getRequiredSourceEncoding(filename, fvdlMetadataResult); NodeList changesNodes = fileChanges.getElementsByTagNameNS(NAMESPACE_URI, "Change"); if (changesNodes.getLength() == 0) { throw new SkipRemediationException(SkipReason.NO_CHANGES, "No changes found for file: " + filename); } - LOG.debug("Remediation {} has {} change(s) for '{}' using FVDL encoding {}", instanceId, changesNodes.getLength(), filename, - sourceEncoding.name()); + SourceFileContent sourceFileContent = getPendingOrSourceContent(filePath, filename, fvdlMetadata, pendingWrites); + Charset sourceEncoding = sourceFileContent.charset(); + LOG.debug("Remediation {} has {} change(s) for '{}' using source encoding {}", instanceId, changesNodes.getLength(), filename, + sourceFileContent.encodingSource()); - String updatedContent = getPendingOrSourceContent(filePath, filename, sourceEncoding, pendingWrites); + String updatedContent = sourceFileContent.content(); for (int k = 0; k < changesNodes.getLength(); k++) { updatedContent = applyChange(instanceId, filename, fileHash, sourceEncoding, updatedContent, (Element) changesNodes.item(k), k + 1); } - byte[] updatedBytes = encodeStrict(updatedContent, sourceEncoding, filename); - pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, updatedBytes)); - LOG.debug("Staged remediation {} for '{}' using FVDL encoding {}; changes={}, encodedBytes={}", instanceId, filename, - sourceEncoding.name(), changesNodes.getLength(), updatedBytes.length); + byte[] updatedBytes = encodeSourceFile(updatedContent, sourceEncoding, filename); + pendingWrites.put(filePath, new PendingFileWrite(filename, filePath, updatedContent, sourceEncoding, + sourceFileContent.encodingSource(), updatedBytes)); + LOG.debug("Staged remediation {} for '{}' using source encoding {}; changes={}, encodedBytes={}", instanceId, filename, + sourceFileContent.encodingSource(), changesNodes.getLength(), updatedBytes.length); return true; } @@ -334,10 +343,12 @@ private String applyChange(String instanceId, String filename, String fileHash, return String.join(lineSeparator, updatedLines); } - private String getPendingOrSourceContent(Path filePath, String filename, Charset sourceEncoding, + private SourceFileContent getPendingOrSourceContent(Path filePath, String filename, FVDLMetadata fvdlMetadata, Map pendingWrites) { PendingFileWrite pendingWrite = pendingWrites.get(filePath); - return pendingWrite == null ? readSourceFile(filePath, filename, sourceEncoding) : pendingWrite.content(); + return pendingWrite == null + ? readSourceFile(filePath, filename, fvdlMetadata) + : new SourceFileContent(pendingWrite.content(), pendingWrite.charset(), pendingWrite.encodingSource()); } private void commitRemediationWrites(String instanceId, Map pendingWrites, Set modifiedFiles) @@ -392,84 +403,46 @@ private boolean isFilePresent(Path path) { return Files.exists(path) && Files.isRegularFile(path); } - private FvdlMetadataResult loadFvdlMetadata() { + /** Nullable: missing/unreadable FVDL means FPR encoding candidate is skipped. */ + private FVDLMetadata loadFvdlMetadata() { if (!Files.exists(fprHandle.getPath("/audit.fvdl"))) { - LOG.warn("FVDL file '/audit.fvdl' is missing; source remediations will be skipped"); - return new FvdlMetadataResult(null, SkipReason.FVDL_METADATA_UNAVAILABLE); + LOG.warn("FVDL file '/audit.fvdl' is missing; FPR encoding candidate will be skipped"); + return null; } try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { LOG.debug("Loading FVDL build metadata from '{}' to resolve source encodings", fprHandle.getFprPath()); - StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle); + // Decoder unused for metadata-only parse; ctor requires one for FileUtils wiring. + StreamingFVDLProcessor processor = new StreamingFVDLProcessor(fprHandle, sourceDecoder); processor.parseBuildMetadata(zipFile, "audit.fvdl"); LOG.debug("Loaded FVDL build metadata from '{}'", fprHandle.getFprPath()); - return new FvdlMetadataResult(processor.getFvdlMetadata(), null); + return processor.getFvdlMetadata(); } catch (Exception e) { - LOG.warn("Error reading source file encodings from audit.fvdl; source remediations will be skipped", e); - return new FvdlMetadataResult(null, SkipReason.FVDL_METADATA_UNAVAILABLE); + LOG.warn("Error reading source file encodings from audit.fvdl; FPR encoding candidate will be skipped", e); + return null; } } - private Charset getRequiredSourceEncoding(String filename, FvdlMetadataResult fvdlMetadataResult) { - if (fvdlMetadataResult.skipReason() != null || fvdlMetadataResult.metadata() == null) { - throw new SkipRemediationException(SkipReason.FVDL_METADATA_UNAVAILABLE, - "FVDL metadata is unavailable; cannot determine source encoding for file '" + filename + "'"); - } - - String encoding = fvdlMetadataResult.metadata().findSourceFileEncodingForFileName(filename); - if (encoding == null || encoding.isBlank()) { - LOG.debug("FVDL source encoding lookup failed for '{}'", filename); - throw new SkipRemediationException(SkipReason.FVDL_ENCODING_MISSING, - "FVDL does not declare a source encoding for file '" + filename + "'"); - } - - try { - Charset charset = Charset.forName(encoding); - LOG.debug("FVDL source encoding for '{}' resolved to '{}'", filename, charset.name()); - return charset; - } catch (Exception e) { - throw new SkipRemediationException(SkipReason.FVDL_ENCODING_UNSUPPORTED, - "FVDL declares unsupported source encoding '" + encoding + "' for file '" + filename + "'", e); - } - } - - private String readSourceFile(Path filePath, String filename, Charset sourceEncoding) { + private SourceFileContent readSourceFile(Path filePath, String filename, FVDLMetadata fvdlMetadata) { try { byte[] sourceBytes = Files.readAllBytes(filePath); - String decodedContent = decodeStrict(sourceBytes, sourceEncoding); - LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, sourceEncoding.name(), sourceBytes.length, - decodedContent.length()); - return decodedContent; - } catch (CharacterCodingException e) { - throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, - "FVDL declares source encoding '" + sourceEncoding.name() + "' for file '" + filename + - "', but the source file cannot be decoded using that encoding", e); + // Metadata may be null (FVDL missing); FPR candidate fails and other encodings are tried. + DecodeResult decodeResult = sourceDecoder.decode(sourceBytes, filename, fvdlMetadata); + LOG.debug("Strict decoded '{}' using {}; sourceBytes={}, decodedChars={}", filename, decodeResult.source(), sourceBytes.length, + decodeResult.content().length()); + return new SourceFileContent(decodeResult.content(), decodeResult.charset(), decodeResult.source()); + } catch (SourceDecodeException e) { + throw new SkipRemediationException(SkipReason.SOURCE_DECODE_FAILED, e.getMessage(), e); } catch (IOException e) { throw new SkipRemediationException(SkipReason.SOURCE_READ_FAILED, "Error reading source code file '" + filePath + "'", e); } } - private String decodeStrict(byte[] bytes, Charset charset) throws CharacterCodingException { - return charset.newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(bytes)) - .toString(); - } - - private byte[] encodeStrict(String content, Charset charset, String filename) { + private byte[] encodeSourceFile(String content, Charset charset, String filename) { try { - ByteBuffer buffer = charset.newEncoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - .encode(CharBuffer.wrap(content)); - byte[] result = new byte[buffer.remaining()]; - buffer.get(result); - return result; - } catch (CharacterCodingException e) { - throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, - "Remediation content for file '" + filename + "' cannot be encoded using FVDL source encoding '" + - charset.name() + "'", e); + return SourceEncoder.encode(content, charset, filename); + } catch (SourceEncodeException e) { + throw new SkipRemediationException(SkipReason.REMEDIATION_ENCODE_FAILED, e.getMessage(), e); } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java index 9a3f8ffa9e8..3dace7f6e75 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java @@ -32,6 +32,8 @@ import com.fortify.cli.aviator.fpr.filter.AnalyzerType; import com.fortify.cli.aviator.fpr.model.*; import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; +import com.fortify.cli.aviator.fpr.utils.SourceDecoders; import com.fortify.cli.aviator.fpr.utils.XmlUtils; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -70,17 +72,26 @@ public class StreamingFVDLProcessor { private long peakMemoryPass2 = 0; private long peakMemoryPostProcessing = 0; - public StreamingFVDLProcessor(FprHandle fprHandle){ + public StreamingFVDLProcessor(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults()); + } + + /** + * @param sourceDecoder used for stack-trace line/fragment source reads; shares {@link #fvdlMetadata} + * so FPR encoding candidates resolve after build metadata is parsed. + */ + public StreamingFVDLProcessor(FprHandle fprHandle, ISourceDecoder sourceDecoder) { this.vulnFinalizer = new VulnFinalizer(); - this.fileUtils = new FileUtils(); this.fprHandle = fprHandle; this.sourceFileMap = fprHandle.getSourceFileMap(); this.xmlInputFactory = XMLInputFactory.newInstance(); // Security: Disable external entity processing xmlInputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - //this.parsingMetadata = new ParsingMetadata(); this.fvdlMetadata = new FVDLMetadata(); + // Same metadata instance FileUtils will see once encodings are registered during parse. + this.fileUtils = new FileUtils( + Objects.requireNonNull(sourceDecoder, "sourceDecoder"), this.fvdlMetadata); this.rawVulnerabilities = new ArrayList<>(); this.vulnerabilities = new ArrayList<>(); this.descriptionProcessor = new DescriptionProcessor(); @@ -92,8 +103,6 @@ public StreamingFVDLProcessor(FprHandle fprHandle){ this.traceParser.setNodeParser(nodeParser); // Circular dependency for Reason parsing this.descriptionParser = new DescriptionParser(); this.metadataParser = new MetadataParser(); - /*this.extractedPath = extractedPath; - this.indexXMLProcessor = new IndexXMLProcessor(extractedPath, sourceFileMap);*/ } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java new file mode 100644 index 00000000000..b0cf5d06621 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CharsetSourceDecoder.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.util.Objects; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Fixed-{@link Charset} decoder. Created via {@link SourceDecoders}. */ +final class CharsetSourceDecoder implements ISourceDecoder { + private final Charset charset; + private final String label; + + CharsetSourceDecoder(String charsetName) { + this.charset = Charset.forName(charsetName); + this.label = charsetName; + } + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + try { + return new DecodeResult(SourceEncoder.decodeStrict(bytes, charset), charset, label); + } catch (CharacterCodingException e) { + throw new SourceDecodeException(label + " failed to decode source bytes", e); + } + } + + @Override + public String describe() { + return label; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java new file mode 100644 index 00000000000..9c335cb550f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/CompositeSourceDecoder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Tries candidates in order. Created via {@link SourceDecoders#of}. */ +final class CompositeSourceDecoder implements ISourceDecoder { + private final List decoders; + + CompositeSourceDecoder(List decoders) { + Objects.requireNonNull(decoders, "decoders must not be null"); + if (decoders.isEmpty()) { + throw new IllegalArgumentException("decoders must not be empty"); + } + this.decoders = List.copyOf(decoders); + } + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + List failures = new ArrayList<>(); + for (ISourceDecoder decoder : decoders) { + try { + return decoder.decode(bytes, filename, fvdlMetadata); + } catch (SourceDecodeException e) { + failures.add(e.getMessage() != null ? e.getMessage() : decoder.describe() + " failed"); + } + } + throw new SourceDecodeException("Could not decode source file '" + filename + "' using source encodings " + + describe() + "; attempted: " + String.join("; ", failures)); + } + + @Override + public String describe() { + return decoders.stream().map(ISourceDecoder::describe).collect(Collectors.joining(",")); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index 6ecd49aed6c..f5878cc62d8 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -14,13 +14,13 @@ import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -28,15 +28,31 @@ import org.slf4j.LoggerFactory; import com.fortify.cli.aviator.audit.model.Fragment; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; import com.fortify.cli.aviator.util.FileUtil; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.LanguageCommentMapperUtil; import com.fortify.cli.aviator.util.StringUtil; +/** + * Source file helpers. Decode failures are soft (empty result + log): snippets/lines are best-effort. + * Callers that must fail or skip with metrics (e.g. remediation apply) should decode themselves. + */ public class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private final Map> fileContentCache = new ConcurrentHashMap<>(); + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; + + public FileUtils() { + this(SourceDecoders.defaults(), null); + } + + public FileUtils(ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fvdlMetadata = fvdlMetadata; + } /** * Reads all lines from a file, caching the result to avoid repeated reads. @@ -46,13 +62,17 @@ public class FileUtils { * @return List of lines, or empty list if file not found or error occurs */ public List readFileWithFallback(Path filePath) { + return readFileWithFallback(filePath, filePath.getFileName().toString()); + } + + private List readFileWithFallback(Path filePath, String filename) { return fileContentCache.computeIfAbsent(filePath, path -> { try { byte[] fileBytes = Files.readAllBytes(path); - String content = new String(fileBytes, StandardCharsets.UTF_8); + String content = sourceDecoder.decode(fileBytes, filename, fvdlMetadata).content(); return Arrays.asList(content.split("\\r?\\n")); - } catch (IOException e) { - logger.error("Failed to read file: {}", path, e); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { + logger.warn("Could not read or decode source file {}: {}", path, e.getMessage()); return Collections.emptyList(); } }); @@ -80,7 +100,7 @@ public String getLineFromFile(FprHandle fprHandle, String relativePath, int line Path fullSourcePath = resolveFullPath(fprHandle, relativePath); if (fullSourcePath == null) return ""; - List lines = readFileWithFallback(fullSourcePath); + List lines = readFileWithFallback(fullSourcePath, relativePath); if (lineNumber > 0 && lines.size() >= lineNumber) { return appendLineNumbers(lines.get(lineNumber - 1), relativePath, lineNumber - 1); } @@ -97,7 +117,7 @@ public Fragment getFragmentFromFile(FprHandle fprHandle, String relativePath, in return new Fragment("", 0, 0); } - List lines = readFileWithFallback(fullSourcePath); + List lines = readFileWithFallback(fullSourcePath, relativePath); if (lines.isEmpty() || lineNumber <= 0) { return new Fragment("", 0, 0); } @@ -127,20 +147,24 @@ private Path resolveFullPath(FprHandle fprHandle, String relativePath) { return fprHandle.getPath(internalPath); } - - // Assumes you have already updated the signature to accept extractedPath public Optional getSourceFileContent(FprHandle fprHandle, String relativePath) { - Path actualSourcePath = resolveFullPath(fprHandle, relativePath); - if (actualSourcePath == null) { + try { + return Optional.of(readSourceFileContentStrict(fprHandle, relativePath)); + } catch (IOException | ISourceDecoder.SourceDecodeException e) { + logger.warn("Could not read source file content for path {}: {}", relativePath, e.getMessage()); return Optional.empty(); } + } - try { - return Optional.of(String.join(System.lineSeparator(), readFileWithFallback(actualSourcePath))); - } catch (Exception e) { - logger.warn("Could not read source file content for path: {}", relativePath, e); - return Optional.empty(); + /** Reads and decodes source content while preserving read or decode failures for the caller. */ + String readSourceFileContentStrict(FprHandle fprHandle, String relativePath) throws IOException { + Path actualSourcePath = resolveFullPath(fprHandle, relativePath); + if (actualSourcePath == null) { + throw new IOException("Source file key not found in sourceFileMap: " + relativePath); } + + byte[] fileBytes = Files.readAllBytes(actualSourcePath); + return sourceDecoder.decode(fileBytes, relativePath, fvdlMetadata).content(); } public String appendLineNumbers(String content, String fileName, int startLineNo) { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java new file mode 100644 index 00000000000..f7241d3ed73 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FprSourceDecoder.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.util.Objects; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** Resolves encoding from FVDL metadata, then strict-decodes. Created via {@link SourceDecoders}. */ +final class FprSourceDecoder implements ISourceDecoder { + static final String TOKEN = "FPR"; + + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata) { + Objects.requireNonNull(bytes, "bytes must not be null"); + Charset charset = resolveCharset(filename, fvdlMetadata); + try { + String content = SourceEncoder.decodeStrict(bytes, charset); + return new DecodeResult(content, charset, TOKEN + "(" + charset.name() + ")"); + } catch (CharacterCodingException e) { + throw new SourceDecodeException(TOKEN + "(" + charset.name() + ") failed to decode source bytes", e); + } + } + + private static Charset resolveCharset(String filename, FVDLMetadata fvdlMetadata) { + if (fvdlMetadata == null) { + throw new SourceDecodeException("FPR metadata unavailable"); + } + String encodingName = fvdlMetadata.findSourceFileEncodingForFileName(filename); + if (encodingName == null || encodingName.isBlank()) { + throw new SourceDecodeException("FPR encoding missing for '" + filename + "'"); + } + try { + return Charset.forName(encodingName); + } catch (Exception e) { + throw new SourceDecodeException( + TOKEN + " resolved to unsupported encoding '" + encodingName + "'", e); + } + } + + @Override + public String describe() { + return TOKEN; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java new file mode 100644 index 00000000000..96e76154a8b --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/ISourceDecoder.java @@ -0,0 +1,55 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.charset.Charset; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; + +/** + * Decodes source file bytes using a specific encoding strategy. + * Implementations may resolve the encoding from FVDL metadata, use a fixed + * {@link Charset}, or try multiple candidate decoders in order. + */ +public interface ISourceDecoder { + /** + * Decode source file bytes into text. + * + * @param bytes raw source file bytes + * @param filename source file name (used for FPR encoding lookup and error messages) + * @param fvdlMetadata optional FVDL metadata; required only by FPR-based decoders + * @return decoded content together with the charset that produced it + * @throws SourceDecodeException if this decoder cannot decode the bytes + */ + DecodeResult decode(byte[] bytes, String filename, FVDLMetadata fvdlMetadata); + + /** + * Human-readable description of this decoder (used in CLI help and error messages). + */ + String describe(); + + record DecodeResult(String content, Charset charset, String source) {} + + class SourceDecodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceDecodeException(String message) { + super(message); + } + + public SourceDecodeException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java index 57e28043765..873a5a82fec 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -13,17 +13,21 @@ package com.fortify.cli.aviator.fpr.utils; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -39,24 +43,17 @@ public class SourceCodeEnricher { private static final Logger logger = LoggerFactory.getLogger(SourceCodeEnricher.class); - /*private final Path extractedPath; - private final Map sourceFileMap;*/ private final FprHandle fprHandle; private final FileUtils fileUtils; + private final Map sourceFileCache = new ConcurrentHashMap<>(); - /** - * Creates a new SourceCodeEnricher with the required dependencies. - * @param fprHandle Utility for file operations (line numbering, line counting) - */ - /*public SourceCodeEnricher(Path extractedPath, Map sourceFileMap, FileUtils fileUtils) { - this.extractedPath = extractedPath; - this.sourceFileMap = sourceFileMap; - this.fileUtils = fileUtils; - }*/ + public SourceCodeEnricher(FprHandle fprHandle) { + this(fprHandle, SourceDecoders.defaults(), null); + } - public SourceCodeEnricher(FprHandle fprHandle){ + public SourceCodeEnricher(FprHandle fprHandle, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { this.fprHandle = fprHandle; - this.fileUtils = new FileUtils(); + this.fileUtils = new FileUtils(Objects.requireNonNull(sourceDecoder, "sourceDecoder"), fvdlMetadata); } /** @@ -74,36 +71,41 @@ public SourceCodeEnricher(FprHandle fprHandle){ * @return Map of filename → File objects with content loaded */ public Map enrichWithSourceCode(List> stackTraces) { - Map uniqueFiles = new HashMap<>(); + return new HashMap<>(enrichWithSourceCodeDetailed(stackTraces).files()); + } + + public EnrichmentResult enrichWithSourceCodeDetailed(List> stackTraces) { + Map uniqueFiles = new LinkedHashMap<>(); + Map failuresByFilename = new LinkedHashMap<>(); if (stackTraces == null || stackTraces.isEmpty()) { logger.debug("No stack traces to enrich"); - return uniqueFiles; + return new EnrichmentResult(uniqueFiles, List.of()); } - processStackTraces(stackTraces, uniqueFiles); + processStackTraces(stackTraces, uniqueFiles, failuresByFilename); logger.debug("Enriched {} unique source files from {} stack traces", uniqueFiles.size(), stackTraces.size()); - return uniqueFiles; + return new EnrichmentResult(uniqueFiles, new ArrayList<>(failuresByFilename.values())); } /** * Processes all stack traces to extract and load unique source files. * Replicates FVDLProcessor.processStackTraceElements() logic. */ - private void processStackTraces(List> stackTraces, Map uniqueFiles) { + private void processStackTraces(List> stackTraces, Map uniqueFiles, + Map failuresByFilename) { for (List stackTrace : stackTraces) { if (stackTrace == null) continue; for (StackTraceElement element : stackTrace) { - processFileForElement(element, uniqueFiles); + processFileForElement(element, uniqueFiles, failuresByFilename); - // Process inner stack traces recursively if (element.getInnerStackTrace() != null) { for (StackTraceElement innerElement : element.getInnerStackTrace()) { - processFileForElement(innerElement, uniqueFiles); + processFileForElement(innerElement, uniqueFiles, failuresByFilename); } } } @@ -117,40 +119,63 @@ private void processStackTraces(List> stackTraces, Map uniqueFiles) { + private void processFileForElement(StackTraceElement element, Map uniqueFiles, + Map failuresByFilename) { if (element == null) return; String filename = element.getFilename(); - if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) && !uniqueFiles.containsKey(filename)) { - String internalPath = fprHandle.getSourceFileMap().get(filename); - if (internalPath == null) { return; } // Should not happen due to containsKey check, but safe. + if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) + && !uniqueFiles.containsKey(filename) && !failuresByFilename.containsKey(filename)) { + CachedSourceResult result = sourceFileCache.computeIfAbsent(filename, this::loadSourceFile); + if (result.failure() != null) { + failuresByFilename.put(filename, result.failure()); + } else { + uniqueFiles.put(filename, result.sourceFile().toFile(filename)); + } + } + } + + private CachedSourceResult loadSourceFile(String filename) { + try { + String content = fileUtils.readSourceFileContentStrict(fprHandle, filename); + CachedSourceFile sourceFile = new CachedSourceFile( + fileUtils.appendLineNumbers(content, filename, 0), content.split("\\R", -1).length); + return new CachedSourceResult(sourceFile, null); + } catch (ISourceDecoder.SourceDecodeException e) { + return failedSourceFile(filename, e, AuditSkipReason.SOURCE_FILE_DECODE_FAILED); + } catch (IOException e) { + return failedSourceFile(filename, e, AuditSkipReason.SOURCE_FILE_READ_FAILED); + } + } - Path actualSourcePath = fprHandle.getPath("/" + internalPath); + private CachedSourceResult failedSourceFile(String filename, Exception exception, AuditSkipReason reason) { + logger.warn("Could not read source file content for path {}: {}", filename, exception.getMessage()); + return new CachedSourceResult(null, new SourceFileFailure(filename, exception.getMessage(), reason)); + } + private record CachedSourceFile(String content, int endLine) { + private File toFile(String filename) { File file = new File(); file.setName(filename); file.setSegment(false); file.setStartLine(1); + file.setContent(content); + file.setEndLine(endLine); + return file; + } + } - try { - if (Files.exists(actualSourcePath)) { - byte[] encodedBytes = Files.readAllBytes(actualSourcePath); - String content = new String(encodedBytes); - // Keep line markers in prompt file content; downstream gRPC/template rendering is pass-through. - file.setContent(fileUtils.appendLineNumbers(content, filename, 0)); - file.setEndLine(fileUtils.countLines(actualSourcePath)); - } else { - // This warning is now more accurate. - logger.warn("Source file not found at internal path: {}. This may indicate a corrupt FPR.", actualSourcePath); - file.setContent(""); - file.setEndLine(0); - } - } catch (IOException e) { - logger.warn("Error processing file: {}", filename, e); - file.setContent(""); - file.setEndLine(0); - } - uniqueFiles.put(filename, file); + private record CachedSourceResult(CachedSourceFile sourceFile, SourceFileFailure failure) {} + + public record EnrichmentResult(Map files, List failures) { + public boolean hasFailures() { + return !failures.isEmpty(); + } + } + + public record SourceFileFailure(String filename, String message, AuditSkipReason reason) { + public SourceFileFailure { + Objects.requireNonNull(reason, "reason"); } } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java new file mode 100644 index 00000000000..7b231926127 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceDecoders.java @@ -0,0 +1,82 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Domain factory for {@link ISourceDecoder} instances. Single source of truth for + * token/CSV parsing and default candidate order. CLI converters should delegate here. + */ +public final class SourceDecoders { + public static final String DEFAULT_SOURCE_ENCODINGS = "FPR,UTF-8,ISO-8859-1"; + + private SourceDecoders() {} + + /** + * Map a single token to a decoder: {@code FPR} or a charset name. + * + * @throws IllegalArgumentException if token is blank + * @throws java.nio.charset.IllegalCharsetNameException if charset name is illegal + * @throws java.nio.charset.UnsupportedCharsetException if charset is unsupported + */ + public static ISourceDecoder fromToken(String token) { + if (token == null || token.isBlank()) { + throw new IllegalArgumentException("Source encoding must not be blank"); + } + String trimmed = token.trim(); + if (FprSourceDecoder.TOKEN.equalsIgnoreCase(trimmed)) { + return new FprSourceDecoder(); + } + return new CharsetSourceDecoder(trimmed); + } + + /** + * Parse a comma-separated list of encoding tokens into a single decoder + * (composite when more than one candidate). + */ + public static ISourceDecoder fromCsv(String csv) { + String effective = csv == null || csv.isBlank() ? DEFAULT_SOURCE_ENCODINGS : csv; + List decoders = new ArrayList<>(); + for (String part : effective.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + decoders.add(fromToken(trimmed)); + } + } + if (decoders.isEmpty()) { + return defaults(); + } + return of(decoders); + } + + /** + * Compose an ordered candidate list into one decoder. + */ + public static ISourceDecoder of(List decoders) { + Objects.requireNonNull(decoders, "decoders must not be null"); + if (decoders.isEmpty()) { + return defaults(); + } + if (decoders.size() == 1) { + return decoders.get(0); + } + return new CompositeSourceDecoder(decoders); + } + + public static ISourceDecoder defaults() { + return fromCsv(DEFAULT_SOURCE_ENCODINGS); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java new file mode 100644 index 00000000000..93668218dd1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceEncoder.java @@ -0,0 +1,64 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; + +import com.fortify.cli.aviator._common.exception.AviatorSimpleException; + +/** + * Strict source encode/decode helpers ({@link CodingErrorAction#REPORT}). + * Public {@link #encode} is for write-back; package-private decode is shared by decoders. + */ +public final class SourceEncoder { + private SourceEncoder() {} + + public static byte[] encode(String content, Charset charset, String filename) { + try { + return encodeStrict(content, charset); + } catch (CharacterCodingException e) { + throw new SourceEncodeException( + "Source file '" + filename + "' cannot be encoded using " + charset.name(), e); + } + } + + static String decodeStrict(byte[] bytes, Charset charset) throws CharacterCodingException { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + } + + private static byte[] encodeStrict(String content, Charset charset) throws CharacterCodingException { + ByteBuffer buffer = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(content)); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; + } + + public static class SourceEncodeException extends AviatorSimpleException { + private static final long serialVersionUID = 1L; + + public SourceEncodeException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java index 874e192e1f2..e3766515ecd 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java @@ -52,6 +52,8 @@ import com.fortify.cli.aviator.audit.model.AuditResponse; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.grpc.token.DeleteTokenRequest; @@ -123,8 +125,11 @@ public AviatorGrpcClient(ManagedChannel channel, long defaultTimeoutSeconds, IAv this(channel, defaultTimeoutSeconds, logger, 30); } - public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, List customPriorityOrder) { - AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, pingIntervalSeconds, defaultTimeoutSeconds, fprHandle); + public CompletableFuture> processBatchRequests(Queue requests, String projectName, + String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle, + List customPriorityOrder, ISourceDecoder sourceDecoder, FVDLMetadata fvdlMetadata) { + AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, + pingIntervalSeconds, defaultTimeoutSeconds, fprHandle, sourceDecoder, fvdlMetadata); CompletableFuture> future = processor.processBatchRequests(requests, projectName, FPRBuildId, SSCApplicationName, SSCApplicationVersion, token, customPriorityOrder); future.whenComplete((res, th) -> processor.close()); return future.exceptionally(ex -> { diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index 89d376801f5..959c68a53f2 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -17,6 +17,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.UUID; @@ -36,6 +37,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,8 +48,11 @@ import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.QuotaBasedFilter; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder; import com.fortify.cli.aviator.fpr.utils.SourceCodeEnricher; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -92,8 +97,16 @@ class AviatorStreamProcessor implements AutoCloseable { private volatile Future processingTask; private final Object retryLock = new Object(); private final FprHandle fprHandle; - - public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, long defaultTimeoutSeconds, FprHandle fprHandle) { + private final ISourceDecoder sourceDecoder; + private final FVDLMetadata fvdlMetadata; + private final SourceCodeEnricher sourceCodeEnricher; + private final AtomicInteger skippedRequests = new AtomicInteger(); + private final AtomicInteger failedRequests = new AtomicInteger(); + + public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, + ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, + long defaultTimeoutSeconds, FprHandle fprHandle, ISourceDecoder sourceDecoder, + FVDLMetadata fvdlMetadata) { this.client = client; this.logger = logger; this.asyncStub = asyncStub; @@ -102,6 +115,9 @@ public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, A this.pingIntervalSeconds = pingIntervalSeconds; this.defaultTimeoutSeconds = defaultTimeoutSeconds; this.fprHandle = fprHandle; + this.sourceDecoder = Objects.requireNonNull(sourceDecoder, "sourceDecoder"); + this.fvdlMetadata = fvdlMetadata; + this.sourceCodeEnricher = new SourceCodeEnricher(fprHandle, this.sourceDecoder, fvdlMetadata); } public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, List customPriorityOrder) { @@ -389,7 +405,7 @@ public void onNext(AuditorResponse response) { response.getRequestId(), instanceId, response.getStatus(), metrics.getDuration()); } - outstandingRequests.decrementAndGet(); + decrementOutstanding(completedWrapper); requestSemaphore.release(); AuditResponse auditResponse = GrpcUtil.convertToAuditResponse(response); @@ -436,11 +452,14 @@ public void onError(Throwable t) { if (infinite || currentStreamState.streamRetryCount < Constants.MAX_STREAM_RETRIES) { LOG.debug("Stream encountered retryable error: {}. Will retry...", t.getMessage()); - int reAdded = inflightRequests.size(); - inflightRequests.values().forEach(processingQueue::addFirst); + int requeuedRequestCount = inflightRequests.size(); + inflightRequests.values().forEach(wrapper -> { + wrapper.outstandingTracked = false; + processingQueue.addFirst(wrapper); + }); inflightRequests.clear(); - outstandingRequests.addAndGet(-reAdded); - requestSemaphore.release(reAdded); + outstandingRequests.addAndGet(-requeuedRequestCount); + requestSemaphore.release(requeuedRequestCount); if (outstandingRequests.get() < 0) { outstandingRequests.set(0); } @@ -707,14 +726,12 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); - // Lazy Loading of source code files for individual issue - SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle); - - Map enrichedFiles = - sourceCodeEnricher.enrichWithSourceCode(wrapper.userPrompt.getStackTrace()); - List sourceCodeFiles = new ArrayList<>(enrichedFiles.values()); - - wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); + SourceCodeEnricher.EnrichmentResult enrichmentResult = enrichSourceCode(wrapper); + if (enrichmentResult != null && enrichmentResult.hasFailures()) { + completeSkippedRequest(wrapper, enrichmentResult, responses, processedRequests, totalRequests, + resultFuture, streamLatch); + continue; + } logger.info("Size of files {}", wrapper.userPrompt.getFiles().size()); logger.info("Size of programming language {}", wrapper.userPrompt.getProgrammingLanguages().size()); @@ -724,8 +741,8 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque continue; } - if (wrapper.attemptCount == 0) { - outstandingRequests.incrementAndGet(); + if (!wrapper.outstandingTracked) { + incrementOutstanding(wrapper); } if (wrapper.attemptCount > 0) { @@ -761,10 +778,11 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque currentStreamState.pendingIssueIds.remove(instanceId); int completed = processedRequests.incrementAndGet(); - outstandingRequests.decrementAndGet(); + int failed = failedRequests.incrementAndGet(); + decrementOutstanding(wrapper); requestSemaphore.release(); - logger.progress("Processed " + completed + " out of " + totalRequests + " issues (1 failed)."); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (" + failed + " failed)."); } else { LOG.error("Caught AviatorSimpleException but the request wrapper was null.", e); } @@ -792,6 +810,96 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque processingQueue.size(), processedRequests.get(), totalRequests, outstandingRequests.get()); } + private SourceCodeEnricher.EnrichmentResult enrichSourceCode(RequestWrapper wrapper) { + synchronized (wrapper) { + if (wrapper.sourceCodeEnriched) { + return null; + } + + SourceCodeEnricher.EnrichmentResult enrichmentResult = + sourceCodeEnricher.enrichWithSourceCodeDetailed(wrapper.userPrompt.getStackTrace()); + if (!enrichmentResult.hasFailures()) { + List sourceCodeFiles = + new ArrayList<>(enrichmentResult.files().values()); + wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); + wrapper.sourceCodeEnriched = true; + } + return enrichmentResult; + } + } + + private void incrementOutstanding(RequestWrapper wrapper) { + if (!wrapper.outstandingTracked) { + outstandingRequests.incrementAndGet(); + wrapper.outstandingTracked = true; + } + } + + private void decrementOutstanding(RequestWrapper wrapper) { + if (wrapper != null && wrapper.outstandingTracked) { + wrapper.outstandingTracked = false; + if (outstandingRequests.decrementAndGet() < 0) { + outstandingRequests.set(0); + } + } + } + + private void completeSkippedRequest(RequestWrapper wrapper, SourceCodeEnricher.EnrichmentResult enrichmentResult, + Map responses, AtomicInteger processedRequests, + int totalRequests, CompletableFuture> resultFuture, + CountDownLatch streamLatch) { + String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); + AuditSkipReason skipReason = enrichmentResult.failures().get(0).reason(); + String failureMessage = formatSourceFailureMessage(enrichmentResult, skipReason); + AuditResponse skippedResponse = new AuditResponse(); + skippedResponse.setIssueId(instanceId); + skippedResponse.setStatus("SKIPPED"); + skippedResponse.setAuditSkipReason(skipReason); + skippedResponse.setStatusMessage(failureMessage); + responses.put(instanceId, skippedResponse); + + currentStreamState.processedIssueIds.add(instanceId); + currentStreamState.pendingIssueIds.remove(instanceId); + decrementOutstanding(wrapper); + requestSemaphore.release(); + + int completed = processedRequests.incrementAndGet(); + int skipped = skippedRequests.incrementAndGet(); + LOG.warn("Skipping issue {}: {}", instanceId, failureMessage); + logger.progress("Processed " + completed + " out of " + totalRequests + " issues (" + skipped + " skipped)."); + + if (completed >= totalRequests) { + logger.info("All requests accounted for, completing stream."); + if (requestHandler != null && !requestHandler.isCompleted()) { + requestHandler.complete(); + } + streamLatch.countDown(); + if (!resultFuture.isDone()) { + resultFuture.complete(responses); + } + } + } + + private String formatSourceFailureMessage(SourceCodeEnricher.EnrichmentResult enrichmentResult, + AuditSkipReason skipReason) { + List filenames = enrichmentResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename) + .distinct() + .collect(Collectors.toList()); + String details = enrichmentResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::message) + .filter(message -> message != null && !message.isBlank()) + .distinct() + .collect(Collectors.joining("; ")); + String detailSuffix = details.isBlank() ? "" : " (" + details + ")"; + return switch (skipReason) { + case SOURCE_FILE_DECODE_FAILED -> skipReason.format( + filenames.size() == 1 ? "" : "s", String.join(", ", filenames), detailSuffix); + case SOURCE_FILE_READ_FAILED -> skipReason.format(String.join(", ", filenames), detailSuffix); + case SKIPPED_BY_AVIATOR -> details.isBlank() ? skipReason.getDisplayMessage() : details; + }; + } + private void handleServerBusy(String requestId, int totalRequests, AtomicInteger processedRequests, Map responses, CompletableFuture> resultFuture, CountDownLatch streamLatch) { RequestWrapper wrapperToRetry = inflightRequests.remove(requestId); if (wrapperToRetry == null) { @@ -811,11 +919,13 @@ private void handleServerBusy(String requestId, int totalRequests, AtomicInteger failedResponse.setIssueId(wrapperToRetry.userPrompt.getIssueData().getInstanceID()); failedResponse.setStatus("RETRY_LIMIT_EXCEEDED"); failedResponse.setStatusMessage("Request failed after " + Constants.MAX_RETRIES + " retries due to server overload."); + failedResponse.setSubmittedToAviator(true); responses.put(wrapperToRetry.userPrompt.getIssueData().getInstanceID(), failedResponse); int completed = processedRequests.incrementAndGet(); logger.progress("Request permanently failed due to server busy - Processed " + completed + " out of " + totalRequests + " issues"); - int stillOutstanding = outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapperToRetry); + int stillOutstanding = outstandingRequests.get(); LOG.warn("WARN: Request for instance {} permanently failed. Remaining outstanding requests: {}", wrapperToRetry.userPrompt.getIssueData().getInstanceID(), stillOutstanding); if (completed >= totalRequests) { logger.info("All requests accounted for after permanent failure, completing stream."); @@ -848,7 +958,7 @@ private void submitUserPrompt(RequestWrapper wrapper) { LOG.error("Failed to send request for instance {} after all retries. Re-queueing for later attempt.", wrapper.userPrompt.getIssueData().getInstanceID()); inflightRequests.remove(requestId); requestMetricsMap.remove(requestId); - outstandingRequests.decrementAndGet(); + decrementOutstanding(wrapper); requestSemaphore.release(); wrapper.attemptCount++; processingQueue.addLast(wrapper); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java index 2d44b56f599..1fc5785ca33 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java @@ -29,6 +29,7 @@ import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.Autoremediation; import com.fortify.cli.aviator.audit.model.Change; import com.fortify.cli.aviator.audit.model.StackTraceElement; @@ -195,7 +196,11 @@ static AuditResponse convertToAuditResponse(AuditorResponse response) { auditResponse.setOutputToken(response.getOutputToken()); auditResponse.setStatus(response.getStatus()); auditResponse.setStatusMessage(response.getStatusMessage()); + if ("SKIPPED".equalsIgnoreCase(response.getStatus())) { + auditResponse.setAuditSkipReason(AuditSkipReason.SKIPPED_BY_AVIATOR); + } auditResponse.setIssueId(response.getIssueId()); + auditResponse.setSubmittedToAviator(true); auditResponse.setTier(response.getTier()); auditResponse.setAviatorPredictionTag(response.getAviatorPredictionTag()); auditResponse.setIsAviatorProcessed(response.getIsAviatorProcessed()); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java index 954f048f4b2..65ddee600c5 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/RequestWrapper.java @@ -17,6 +17,8 @@ class RequestWrapper { final UserPrompt userPrompt; int attemptCount = 0; + volatile boolean outstandingTracked; + volatile boolean sourceCodeEnriched; RequestWrapper(UserPrompt userPrompt) { this.userPrompt = userPrompt; diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java new file mode 100644 index 00000000000..0ad64fab89d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/_common/cli/converter/SourceDecoderConverterTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator._common.cli.converter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import picocli.CommandLine.TypeConversionException; + +/** + * Only the CLI adapter layer: domain factory exceptions become {@link TypeConversionException}. + * Happy-path token semantics are covered by {@code SourceDecodersTest}. + */ +class SourceDecoderConverterTest { + + private final SourceDecoderConverter converter = new SourceDecoderConverter(); + + @Test + void convert_delegatesHappyPathToFactory() { + assertEquals("FPR", converter.convert("FPR").describe()); + assertEquals("UTF-8", converter.convert("UTF-8").describe()); + } + + @Test + void convert_mapsFactoryFailuresToTypeConversionException() { + assertThrows(TypeConversionException.class, () -> converter.convert(" ")); + assertThrows(TypeConversionException.class, () -> converter.convert("NOT-A-REAL-CHARSET")); + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java new file mode 100644 index 00000000000..0dc06bc834f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/audit/AuditFprStatusTest.java @@ -0,0 +1,86 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.audit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; + +class AuditFprStatusTest { + + @Test + void determinesAuditStatusForSuccessSkipAndFailureCombinations() { + assertEquals("AUDITED", AuditFPR.determineAuditStatus(3, 0, 3, 3)); + assertEquals("PARTIALLY_AUDITED", AuditFPR.determineAuditStatus(2, 1, 3, 3)); + assertEquals("SKIPPED", AuditFPR.determineAuditStatus(0, 3, 3, 3)); + assertEquals("FAILED", AuditFPR.determineAuditStatus(0, 2, 3, 3)); + assertEquals("FAILED", AuditFPR.determineAuditStatus(0, 3, 3, 2)); + } + + @Test + void countsOnlyExplicitSkippedResponsesAsSkipped() { + AuditResponse skipped = AuditResponse.builder() + .status("SKIPPED") + .auditSkipReason(AuditSkipReason.SOURCE_FILE_DECODE_FAILED) + .statusMessage("The source decoder wording can change") + .build(); + AuditResponse failed = AuditResponse.builder() + .status("FAILED") + .statusMessage("backend error") + .build(); + AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); + + assertEquals(Map.of("Source file decode failed", 1), + AuditFPR.getSkippedAuditReasons(Map.of( + "skipped", skipped, + "failed", failed, + "success", success), 3)); + } + + @Test + void preservesMissingResponseAccountingForFilteredIssues() { + AuditResponse success = AuditResponse.builder().status("SUCCESS").build(); + + assertEquals(Map.of("No audit response received", 1), + AuditFPR.getSkippedAuditReasons(Map.of("success", success), 2)); + } + + @Test + void countsOnlyResponsesOriginatingFromAviatorAsSubmitted() { + AuditResponse localSkip = AuditResponse.builder().status("SKIPPED").build(); + AuditResponse localFailure = AuditResponse.builder() + .status("FAILED") + .statusMessage("Request validation failed") + .build(); + AuditResponse serverSkip = AuditResponse.builder() + .status("SKIPPED") + .submittedToAviator(true) + .build(); + AuditResponse serverFailure = AuditResponse.builder() + .status("FAILED") + .statusMessage("Aviator processing failed") + .submittedToAviator(true) + .build(); + + assertEquals(2, AuditFPR.getSubmittedAuditCount(Map.of( + "local", localSkip, + "localFailure", localFailure, + "server", serverSkip, + "serverFailure", serverFailure))); + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java new file mode 100644 index 00000000000..3e42be508d8 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricherTest.java @@ -0,0 +1,267 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; +import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.util.FprHandle; + +class SourceCodeEnricherTest { + + @TempDir + Path tempDir; + + @Test + void reportsEveryUniqueDecodeFailureAndDoesNotHideSuccessfulFiles() throws Exception { + Map sourceFiles = new LinkedHashMap<>(); + sourceFiles.put("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + sourceFiles.put("bad-one.java", new byte[] {(byte) 0xFF}); + sourceFiles.put("bad-two.java", new byte[] {(byte) 0xFE}); + Path fprPath = createFpr(sourceFiles); + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher.EnrichmentResult result = new SourceCodeEnricher( + fprHandle, SourceDecoders.fromToken("UTF-8"), null) + .enrichWithSourceCodeDetailed(List.of(List.of( + element("good.java"), + element("bad-one.java"), + element("bad-two.java"), + element("bad-one.java")))); + + assertTrue(result.hasFailures()); + assertEquals(List.of("bad-one.java", "bad-two.java"), + result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of(AuditSkipReason.SOURCE_FILE_DECODE_FAILED, AuditSkipReason.SOURCE_FILE_DECODE_FAILED), + result.failures().stream().map(SourceCodeEnricher.SourceFileFailure::reason).toList()); + assertEquals(List.of("good.java"), result.files().keySet().stream().toList()); + } + } + + @Test + void classifiesReadFailuresSeparatelyFromDecodeFailures() throws Exception { + Path fprPath = createFprWithMissingSource("missing.java"); + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher.EnrichmentResult result = new SourceCodeEnricher( + fprHandle, SourceDecoders.fromToken("UTF-8"), null) + .enrichWithSourceCodeDetailed(List.of(List.of(element("missing.java")))); + + assertEquals(AuditSkipReason.SOURCE_FILE_READ_FAILED, result.failures().get(0).reason()); + } + } + + @Test + void cachesSuccessfulAndFailedLoadsAcrossEnrichmentCalls() throws Exception { + Map sourceFiles = new LinkedHashMap<>(); + sourceFiles.put("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + sourceFiles.put("bad.java", new byte[] {(byte) 0xFF}); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + if (bytes.length > 0 && bytes[0] == (byte) 0xFF) { + throw new SourceDecodeException("decode failed"); + } + return new DecodeResult(new String(bytes, StandardCharsets.UTF_8), StandardCharsets.UTF_8, "test"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("good.java"), element("bad.java"))); + + SourceCodeEnricher.EnrichmentResult firstResult = enricher.enrichWithSourceCodeDetailed(stackTraces); + SourceCodeEnricher.EnrichmentResult secondResult = enricher.enrichWithSourceCodeDetailed(stackTraces); + + assertEquals(2, decodeCalls.get()); + assertEquals(List.of("good.java"), secondResult.files().keySet().stream().toList()); + assertEquals(List.of("bad.java"), secondResult.failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(firstResult.files().get("good.java").getContent(), secondResult.files().get("good.java").getContent()); + } + } + + @Test + void loadsAFileOnceWhenEnrichmentCallsAreConcurrent() throws Exception { + Map sourceFiles = Map.of("good.java", "class Good {}".getBytes(StandardCharsets.UTF_8)); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + CountDownLatch decodeStarted = new CountDownLatch(1); + CountDownLatch allowDecode = new CountDownLatch(1); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + decodeStarted.countDown(); + try { + if (!allowDecode.await(5, TimeUnit.SECONDS)) { + throw new SourceDecodeException("Timed out waiting to decode"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SourceDecodeException("Interrupted while decoding", e); + } + return new DecodeResult(new String(bytes, StandardCharsets.UTF_8), StandardCharsets.UTF_8, "test"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("good.java"))); + Future first = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + Future second = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + + assertTrue(decodeStarted.await(5, TimeUnit.SECONDS)); + allowDecode.countDown(); + + assertEquals(List.of("good.java"), first.get(5, TimeUnit.SECONDS).files().keySet().stream().toList()); + assertEquals(List.of("good.java"), second.get(5, TimeUnit.SECONDS).files().keySet().stream().toList()); + assertEquals(1, decodeCalls.get()); + } finally { + executor.shutdownNow(); + } + } + } + + @Test + void cachesConcurrentDecodeFailureOnce() throws Exception { + Map sourceFiles = Map.of("bad.java", new byte[] {(byte) 0xFF}); + Path fprPath = createFpr(sourceFiles); + AtomicInteger decodeCalls = new AtomicInteger(); + CountDownLatch decodeStarted = new CountDownLatch(1); + CountDownLatch allowDecode = new CountDownLatch(1); + ISourceDecoder decoder = new ISourceDecoder() { + @Override + public DecodeResult decode(byte[] bytes, String filename, FVDLMetadata metadata) { + decodeCalls.incrementAndGet(); + decodeStarted.countDown(); + try { + if (!allowDecode.await(5, TimeUnit.SECONDS)) { + throw new SourceDecodeException("Timed out waiting to decode"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SourceDecodeException("Interrupted while decoding", e); + } + throw new SourceDecodeException("decode failed"); + } + + @Override + public String describe() { + return "test"; + } + }; + + try (FprHandle fprHandle = new FprHandle(fprPath)) { + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + SourceCodeEnricher enricher = new SourceCodeEnricher(fprHandle, decoder, null); + List> stackTraces = List.of(List.of(element("bad.java"))); + Future first = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + Future second = executor.submit( + () -> enricher.enrichWithSourceCodeDetailed(stackTraces)); + + assertTrue(decodeStarted.await(5, TimeUnit.SECONDS)); + allowDecode.countDown(); + + assertEquals(List.of("bad.java"), first.get(5, TimeUnit.SECONDS).failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(List.of("bad.java"), second.get(5, TimeUnit.SECONDS).failures().stream() + .map(SourceCodeEnricher.SourceFileFailure::filename).toList()); + assertEquals(1, decodeCalls.get()); + } finally { + executor.shutdownNow(); + } + } + } + + private static StackTraceElement element(String filename) { + return new StackTraceElement(filename, 1, "", "", null, null, null); + } + + private Path createFpr(Map sourceFiles) throws IOException { + Path fprPath = tempDir.resolve("source-enricher-test.fpr"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(fprPath))) { + writeEntry(zip, "src-archive/index.xml", createIndex(sourceFiles.keySet()).getBytes(StandardCharsets.UTF_8)); + int index = 1; + for (Map.Entry sourceFile : sourceFiles.entrySet()) { + writeEntry(zip, "src-archive/" + index++, sourceFile.getValue()); + } + } + return fprPath; + } + + private Path createFprWithMissingSource(String filename) throws IOException { + Path fprPath = tempDir.resolve("source-enricher-missing-source.fpr"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(fprPath))) { + writeEntry(zip, "src-archive/index.xml", createIndex(List.of(filename)).getBytes(StandardCharsets.UTF_8)); + } + return fprPath; + } + + private static String createIndex(Iterable filenames) { + StringBuilder index = new StringBuilder(""); + int entry = 1; + for (String filename : filenames) { + index.append("src-archive/") + .append(entry++).append(""); + } + return index.append("").toString(); + } + + private static void writeEntry(ZipOutputStream zip, String name, byte[] bytes) throws IOException { + zip.putNextEntry(new ZipEntry(name)); + zip.write(bytes); + zip.closeEntry(); + } +} \ No newline at end of file diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java new file mode 100644 index 00000000000..bdc8f254957 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/fpr/utils/SourceDecodersTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; + +import org.junit.jupiter.api.Test; + +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.DecodeResult; +import com.fortify.cli.aviator.fpr.utils.ISourceDecoder.SourceDecodeException; +import com.fortify.cli.aviator.fpr.utils.SourceEncoder.SourceEncodeException; + +/** + * Behavior tests for source encoding factory + decode/encode. + * Prefer real byte decode outcomes over describe()-only checks. + */ +class SourceDecodersTest { + + @Test + void fromToken_rejectsBlankAndUnknownCharset() { + assertThrows(IllegalArgumentException.class, () -> SourceDecoders.fromToken(" ")); + assertThrows(UnsupportedCharsetException.class, () -> SourceDecoders.fromToken("NOT-A-CHARSET")); + } + + @Test + void defaults_matchDocumentedCandidateOrder() { + assertEquals("FPR,UTF-8,ISO-8859-1", SourceDecoders.DEFAULT_SOURCE_ENCODINGS); + assertEquals(SourceDecoders.DEFAULT_SOURCE_ENCODINGS, SourceDecoders.defaults().describe()); + } + + @Test + void decode_fallsThroughFprWhenMetadataMissing() { + byte[] utf8 = "hello".getBytes(StandardCharsets.UTF_8); + DecodeResult result = SourceDecoders.defaults().decode(utf8, "Main.java", null); + assertEquals("hello", result.content()); + assertEquals(StandardCharsets.UTF_8, result.charset()); + assertEquals("UTF-8", result.source()); + } + + @Test + void decode_fallsThroughUtf8ToIso88591ByDefault() { + byte[] iso88591 = {(byte) 0xE9}; + + DecodeResult result = SourceDecoders.defaults().decode(iso88591, "Main.java", null); + + assertEquals("\u00e9", result.content()); + assertEquals(StandardCharsets.ISO_8859_1, result.charset()); + assertEquals("ISO-8859-1", result.source()); + } + + @Test + void decode_usesFprEncodingWhenMetadataPresent() { + FVDLMetadata metadata = new FVDLMetadata(); + metadata.registerSourceFileEncoding("src/Main.java", "ISO-8859-1"); + byte[] latin1 = "caf\u00e9".getBytes(StandardCharsets.ISO_8859_1); + + DecodeResult result = SourceDecoders.defaults().decode(latin1, "src/Main.java", metadata); + + assertEquals("caf\u00e9", result.content()); + assertEquals(StandardCharsets.ISO_8859_1, result.charset()); + assertTrue(result.source().startsWith("FPR("), result.source()); + } + + @Test + void decode_usesWindows1252FromFprMetadata() { + FVDLMetadata metadata = new FVDLMetadata(); + metadata.registerSourceFileEncoding("payments.c", "windows-1252"); + byte[] windows1252 = new byte[] {(byte) 0x93, 'p', 'r', 'e', 'm', 'i', 'u', 'm', (byte) 0x94}; + + DecodeResult result = SourceDecoders.defaults().decode(windows1252, "payments.c", metadata); + + assertEquals("\u201cpremium\u201d", result.content()); + assertEquals("windows-1252", result.charset().name()); + assertEquals("FPR(windows-1252)", result.source()); + } + + @Test + void decode_allCandidatesFail_messageListsAttempts() { + ISourceDecoder decoder = SourceDecoders.fromCsv("UTF-8,US-ASCII"); + byte[] invalid = new byte[] {(byte) 0xFF, (byte) 0xFE, (byte) 0x00}; + + SourceDecodeException ex = assertThrows(SourceDecodeException.class, + () -> decoder.decode(invalid, "bad.bin", null)); + + assertTrue(ex.getMessage().contains("bad.bin"), ex.getMessage()); + assertTrue(ex.getMessage().contains("UTF-8"), ex.getMessage()); + assertTrue(ex.getMessage().contains("US-ASCII"), ex.getMessage()); + } + + @Test + void encode_roundTripAndRejectsUnmappable() { + String content = "secure code"; + byte[] encoded = SourceEncoder.encode(content, StandardCharsets.UTF_8, "a.java"); + assertEquals(content, SourceDecoders.fromToken("UTF-8").decode(encoded, "a.java", null).content()); + + assertThrows(SourceEncodeException.class, + () -> SourceEncoder.encode("caf\u00e9", StandardCharsets.US_ASCII, "a.java")); + } + + @Test + void fromToken_fprIsCaseInsensitive() { + assertEquals("FPR", SourceDecoders.fromToken("fpr").describe()); + } + + @Test + void fromCsv_compositesMultipleCandidates() { + assertEquals("UTF-8,CP850", SourceDecoders.fromCsv("UTF-8,CP850").describe()); + assertEquals("UTF-8", SourceDecoders.fromCsv("UTF-8").describe()); + } +} diff --git a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java index 2f503d3d0a5..09285bae9e9 100644 --- a/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java +++ b/fcli-core/fcli-aviator-common/src/test/java/com/fortify/cli/aviator/grpc/GrpcUtilTest.java @@ -19,6 +19,9 @@ import org.junit.jupiter.api.Test; +import com.fortify.aviator.grpc.AuditorResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse; +import com.fortify.cli.aviator.audit.model.AuditResponse.AuditSkipReason; import com.fortify.cli.aviator.audit.model.File; import com.fortify.cli.aviator.audit.model.UserPrompt; @@ -43,4 +46,16 @@ void shouldPreserveResolvedProgrammingLanguages() { assertEquals(List.of("PLSQL"), auditRequest.getProgrammingLanguagesList()); assertEquals("PLSQL", auditRequest.getLanguage()); } + + @Test + void shouldClassifySkippedServerResponseWithoutInspectingMessage() { + AuditorResponse response = AuditorResponse.newBuilder() + .setStatus("SKIPPED") + .setStatusMessage("example could not be read from the FPR") + .build(); + + AuditResponse auditResponse = GrpcUtil.convertToAuditResponse(response); + + assertEquals(AuditSkipReason.SKIPPED_BY_AVIATOR, auditResponse.getAuditSkipReason()); + } } \ No newline at end of file diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java index 30fe3af0ae2..c6bff9c596e 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCApplyRemediationsCommand.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; @@ -63,6 +64,7 @@ public class AviatorSSCApplyRemediationsCommand extends AbstractSSCJsonNodeOutpu private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCApplyRemediationsCommand.class); @Option(names = {"--source-dir"}, descriptionKey = "fcli.aviator.ssc.apply-remediations.source-dir") private String sourceCodeDirectory = System.getProperty("user.dir"); + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; @Override @SneakyThrows @@ -128,7 +130,8 @@ JsonNode processAllAviatorArtifacts(OffsetDateTime sinceDate) { try { fprPath = downloadArtifactFpr(ad); try (FprHandle fprHandle = new FprHandle(fprPath)) { - var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var metric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + sourceEncodingsMixin.getSourceDecoder(), logger); totalRemediations += metric.totalRemediations(); appliedRemediations += metric.appliedRemediations(); skippedRemediations += metric.skippedRemediations(); @@ -181,7 +184,8 @@ JsonNode processFprRemediations(SSCArtifactDescriptor ad) { try { logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(fprPath)) { - var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + sourceEncodingsMixin.getSourceDecoder(), logger); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java index 2a4e73281df..5a0c876ef9e 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommand.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator._common.config.AviatorConfigManager; import com.fortify.cli.aviator._common.session.user.cli.mixin.AviatorUserSessionDescriptorSupplier; import com.fortify.cli.aviator._common.session.user.helper.AviatorUserSessionDescriptor; @@ -79,6 +80,7 @@ public class AviatorSSCAuditCommand extends AbstractSSCJsonNodeOutputCommand imp @ArgGroup(exclusive = true, multiplicity = "0..1") private QuotaHandlingArgGroup quotaHandlingArgGroup = new QuotaHandlingArgGroup(); @Option(names = {"--test-exceeding-quota"}) private boolean testExceedingQuota; @Option(names = {"--default-quota-fallback"}) private boolean defaultQuotaFallback; + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; private static final Logger LOG = LoggerFactory.getLogger(AviatorSSCAuditCommand.class); private Long checkedQuotaBefore; @@ -275,6 +277,7 @@ private JsonNode processFpr(UnirestInstance unirest, SSCAppVersionDescriptor av, .noFilterSet(isNoFilterSet()) .folderNames(folderNames) .folderPriorityOrder(getFolderPriorityOrder()) + .sourceDecoder(sourceEncodingsMixin.getSourceDecoder()) .build()); } catch (Exception e) { LOG.error("FPR audit failed for {}:{}: {}", av.getApplicationName(), av.getVersionName(), e.getMessage(), e); diff --git a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java index 542ee85bb86..e9ce3228879 100644 --- a/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java +++ b/fcli-core/fcli-aviator/src/main/java/com/fortify/cli/aviator/ssc/helper/AviatorSSCAuditHelper.java @@ -135,12 +135,34 @@ public static void setAuditStats(ObjectNode result, FPRAuditResult auditResult) break; } audit.put("message", message); - audit.put("submitted", auditResult.getTotalIssuesToAudit()); + audit.put("submitted", auditResult.getIssuesSubmitted()); audit.put("succeeded", auditResult.getIssuesSuccessfullyAudited()); - audit.put("skipped", Math.max(0, auditResult.getTotalIssuesToAudit() - auditResult.getIssuesSuccessfullyAudited())); + audit.put("skipped", auditResult.getIssuesSkipped()); + audit.put("skippedReasons", formatSkippedReasons(auditResult.getSkippedByReason())); + audit.set("skippedByReason", toObjectNode(auditResult.getSkippedByReason())); + audit.put("remediationGenerationSkipped", auditResult.getRemediationGenerationSkipped()); + audit.put("remediationGenerationSkippedReasons", formatSkippedReasons(auditResult.getRemediationGenerationSkippedByReason())); + audit.set("remediationGenerationSkippedByReason", toObjectNode(auditResult.getRemediationGenerationSkippedByReason())); ((ObjectNode) result.get("operation")).set("audit", audit); } + private static ObjectNode toObjectNode(Map skippedByReason) { + ObjectNode object = JsonHelper.getObjectMapper().createObjectNode(); + if (skippedByReason != null) { + skippedByReason.forEach(object::put); + } + return object; + } + + private static String formatSkippedReasons(Map skippedByReason) { + if (skippedByReason == null || skippedByReason.isEmpty()) { + return ""; + } + List parts = new ArrayList<>(); + skippedByReason.forEach((reason, count) -> parts.add(reason + "=" + count)); + return String.join(", ", parts); + } + /** * Sets only the {@code operation.audit.message} field without audit stats. * Used for code paths that don't perform an actual audit (SKIPPED, FAILED, QUOTA_EXCEEDED, etc.). @@ -175,7 +197,8 @@ public static void setAvailableQuotaBefore(ObjectNode result, long quota) { public static String getProgressMessage(FPRAuditResult auditResult) { switch (auditResult.getStatus()) { case "SKIPPED": - return "No issues to audit, skipping upload"; + return (auditResult.getMessage() != null ? auditResult.getMessage() : "No issues to audit") + + ", skipping upload"; case "FAILED": String message = auditResult.getMessage() != null ? auditResult.getMessage() : "Unknown error"; return "Audit failed: " + message; diff --git a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties index da3da97cfb2..7a2c64f68e3 100644 --- a/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties +++ b/fcli-core/fcli-aviator/src/main/resources/com/fortify/cli/aviator/i18n/AviatorMessages.properties @@ -129,6 +129,7 @@ fcli.aviator.ssc.audit.skip-if-exceeding-quota = Skip audit if the number of ope fcli.aviator.ssc.audit.test-exceeding-quota = Check whether the number of open issues exceeds the available Fortify Remediation Aviator quota and report the result without performing an audit. fcli.aviator.ssc.audit.default-quota-fallback = (Internal) When the Fortify Aviator application does not exist, use the tenant default quota instead of reporting app not found. Used by bulk audit. fcli.aviator.ssc.audit.folder-priority-order = Custom priority order for folder-based filtering when quota is exceeded (comma-separated, highest priority first). Example: Critical,High,Medium,Low. If not specified, uses default priority order. +fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. fcli.aviator.ssc.audit.refresh = By default, this command will refresh the source application version's metrics when copying from it. \ Note that for large applications this can lead to an error if the timeout expires. fcli.aviator.ssc.audit.refresh-timeout = Time-out, for example 30s (30 seconds), 5m (5 minutes), 1h (1 hour). Default value: ${DEFAULT-VALUE} @@ -204,7 +205,7 @@ fcli.env.default.prefix=FCLI_DEFAULT # Table output columns configuration fcli.aviator.session.output.table.args = name,type,url,created,expires,expired fcli.aviator.admin-config.output.table.args = name,type,url,created -fcli.aviator.ssc.audit.output.table.args = id,application.name,name,artifactId,action +fcli.aviator.ssc.audit.output.table.args = id,application.name,name,artifactId,action,operation.audit.skippedReasons fcli.aviator.app.create.output.table.args = id,name,entitlement_id,disclaimer,quota_last_updated,quota fcli.aviator.app.add-entitlement.output.table.args = id,name,entitlement_id,entitlement_multiplier,entitlements_consumed,quota_last_updated,quota fcli.aviator.app.delete.output.table.args = message diff --git a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java index e98fa9a5587..d8988e8a860 100644 --- a/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java +++ b/fcli-core/fcli-aviator/src/test/java/com/fortify/cli/aviator/ssc/cli/cmd/AviatorSSCAuditCommandTest.java @@ -19,9 +19,15 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Map; import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.aviator.audit.model.FPRAuditResult; +import com.fortify.cli.aviator.ssc.helper.AviatorSSCAuditHelper; +import com.fortify.cli.common.json.JsonHelper; + import picocli.CommandLine; class AviatorSSCAuditCommandTest { @@ -46,6 +52,48 @@ void testAllowsNoFilterSetOption() { assertNull(cmd.getFilterSetTitleOrId()); } + @Test + void reportsDecodeSkippedIssuesAsNotSubmitted() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "SKIPPED", "All 6 issues were skipped", 0, 6, 0, 6, + Map.of("Source file decode failed", 6), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + var audit = result.path("operation").path("audit"); + assertEquals(0, audit.get("submitted").asInt()); + assertEquals(0, audit.get("succeeded").asInt()); + assertEquals(6, audit.get("skipped").asInt()); + } + + @Test + void reportsServerSkippedIssuesAsSubmitted() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "SKIPPED", "Skipped by Aviator", 0, 1, 1, 1, + Map.of("Skipped by Aviator", 1), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + assertEquals(1, result.path("operation").path("audit").get("submitted").asInt()); + } + + @Test + void excludesOnlyDecodeSkippedIssuesFromSubmittedCount() { + ObjectNode result = JsonHelper.getObjectMapper().createObjectNode(); + result.set("operation", JsonHelper.getObjectMapper().createObjectNode()); + var auditResult = new FPRAuditResult(null, "PARTIALLY_AUDITED", null, 2, 4, 3, 2, + Map.of("Source file decode failed", 1, "Skipped by Aviator", 1), 0, Map.of()); + + AviatorSSCAuditHelper.setAuditStats(result, auditResult); + + var audit = result.path("operation").path("audit"); + assertEquals(3, audit.get("submitted").asInt()); + assertEquals(2, audit.get("succeeded").asInt()); + assertEquals(2, audit.get("skipped").asInt()); + } + private static AviatorSSCAuditCommand parse(String... args) { var cmd = new AviatorSSCAuditCommand(); var fullArgs = new ArrayList(); diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java index f792edbef1e..471232a1e03 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/aviator/cmd/FoDAviatorApplyRemediationsCommand.java @@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.aviator._common.cli.mixin.SourceEncodingsMixin; import com.fortify.cli.aviator.applyRemediation.ApplyAutoRemediationOnSource; import com.fortify.cli.aviator.config.AviatorLoggerImpl; import com.fortify.cli.aviator.util.FprHandle; @@ -56,6 +57,7 @@ public class FoDAviatorApplyRemediationsCommand extends AbstractFoDJsonNodeOutpu @Mixin private FoDReleaseByQualifiedNameOrIdResolverMixin.RequiredOption releaseResolver; private static final Logger LOG = LoggerFactory.getLogger(FoDAviatorApplyRemediationsCommand.class); @Option(names = {"--source-dir"}) private String sourceCodeDirectory = System.getProperty("user.dir"); + @Mixin private SourceEncodingsMixin sourceEncodingsMixin; @Override @SneakyThrows public JsonNode getJsonNode(UnirestInstance unirest) { @@ -82,7 +84,8 @@ private JsonNode processFprRemediations(UnirestInstance unirest, FoDReleaseDescr logger.progress("Status: Processing FPR with Aviator for Applying Auto Remediations"); try (FprHandle fprHandle = new FprHandle(downloadedFprPath)) { - var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, logger); + var remediationMetric = ApplyAutoRemediationOnSource.applyRemediations(fprHandle, sourceCodeDirectory, + sourceEncodingsMixin.getSourceDecoder(), logger); LOG.info("Applied remediation {}", remediationMetric.appliedRemediations()); LOG.info("Total remediation {}", remediationMetric.totalRemediations()); String status = remediationMetric.appliedRemediations() > 0 ? "Remediation-Applied" : "No-Remediation-Applied"; diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 62dcf5b6c6d..0fcff19749e 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -1038,6 +1038,7 @@ fcli.fod.attribute.update.values = List of picklist values (only for Picklist da fcli.fod.aviator.usage.header = Use Fortify Remediation Aviator with FoD. fcli.fod.aviator.apply-remediations.usage.header = Apply Fortify Remediation Aviator auto-remediations to source code. fcli.fod.aviator.apply-remediations.source-dir = Directory containing source code to apply remediations to. Default value: ${DEFAULT-VALUE}. +fcli.aviator.source-encodings = Comma-separated source encoding candidates to try in order when decoding source files. Use FPR to try the source encoding recorded in audit.fvdl. When writing remediated files, the accepted encoding is used. Default value: ${DEFAULT-VALUE}. # various messages displayed during execution