XRAY-156301 Implement http HEAD GET security for pip and poetry

_typos.toml

@@ -2,6 +2,7 @@
 testng = "testng"
 afe = "afe"
 Bouned = "Bouned"
+certifi = "certifi"
 
 [files]
 extend-exclude = ["go.mod", "go.sum", "tests/testdata"]

cli/docs/scan/curation/help.go

@@ -8,14 +8,14 @@ func GetAIDescription() string {
 	return `Inspect the project's package-manager dependencies against the JFrog Curation service and report which packages were blocked by curation policy along with the matching policies. Use when an agent suspects a package install failure was due to curation, or wants a preemptive curation report.
 
 When to use:
-- Diagnose 403/forbidden errors during npm/pip/maven/gradle/nuget/go install steps in a curation-enabled remote.
+- Diagnose 403/forbidden errors during npm/pip/pipenv/poetry/maven/gradle/nuget/go install steps in a curation-enabled remote.
 - Produce a curation-status report (blocked packages and policies) for the current project.
 - Run automatically after a failed install via the JFROG_CLI_SKIP_CURATION_AFTER_FAILURE workflow.
 
 Prerequisites:
 - A configured JFrog Platform server (jf c add) with JFrog Curation entitlement.
-- Project must use a supported package manager (npm, yarn, pip, maven, gradle, nuget, go) resolved through a curation-configured remote. Docker images and Hugging Face models are audited via dedicated flags.
-- The package manager and its lockfile must be present in the working directory.
+- Project must use a supported package manager (npm, yarn, pnpm, pip, pipenv, poetry, maven, gradle, nuget, go) resolved through a curation-configured remote. Docker images and Hugging Face models are audited via dedicated flags.
+- The package manager must be present in the working directory. Lockfiles are required where the package manager's flow depends on one (e.g. npm, yarn, pnpm, poetry); pipenv generates/refreshes Pipfile.lock as part of the audit, so a pre-existing lockfile is not required.
 
 Common patterns:
   $ jf curation-audit

commands/curation/curationaudit.go

@@ -137,6 +137,9 @@ var supportedTech = map[techutils.Technology]func(ca *CurationAuditCommand) (boo
 	techutils.Poetry: func(ca *CurationAuditCommand) (bool, error) {
 		return ca.checkSupportByVersionOrEnv(techutils.Poetry, MinArtiPassThroughSupport)
 	},
+	techutils.Pipenv: func(ca *CurationAuditCommand) (bool, error) {
+		return ca.checkSupportByVersionOrEnv(techutils.Pipenv, MinArtiPassThroughSupport)
+	},
 }
 
 func (ca *CurationAuditCommand) checkSupportByVersionOrEnv(tech techutils.Technology, minArtiVersion string) (bool, error) {
@@ -844,6 +847,12 @@ func (ca *CurationAuditCommand) getRtManagerAndAuth(tech techutils.Technology) (
 	return
 }
 
+// boundedRedirectManager must use zero retries: HttpClient.Send retries
+// on CheckRedirect errors, desyncing SendWithBoundedRedirects's hop counter.
+func boundedRedirectManager(serverDetails *config.ServerDetails) (artifactory.ArtifactoryServicesManager, error) {
+	return rtUtils.CreateServiceManager(serverDetails, 0, 0, false)
+}
+
 func (ca *CurationAuditCommand) GetAuth(tech techutils.Technology) (serverDetails *config.ServerDetails, err error) {
 	if ca.PackageManagerConfig == nil {
 		if err = ca.SetRepo(tech); err != nil {
@@ -857,8 +866,18 @@ func (ca *CurationAuditCommand) GetAuth(tech techutils.Technology) (serverDetail
 	return
 }
 
+// getBuildInfoParamsByTech resolves install-time server details, preferring an already-set
+// ca.PackageManagerConfig (native detection, e.g. Pipenv) over the generic server so install
+// and the later GetAuth-based probes hit the same endpoint. Does not trigger SetRepo eagerly,
+// so techs resolving their repo later via SetResolutionRepoInParamsIfExists are unaffected.
 func (ca *CurationAuditCommand) getBuildInfoParamsByTech() (technologies.BuildInfoBomGeneratorParams, error) {
-	serverDetails, err := ca.ServerDetails()
+	var serverDetails *config.ServerDetails
+	var err error
+	if ca.PackageManagerConfig != nil {
+		serverDetails, err = ca.PackageManagerConfig.ServerDetails()
+	} else {
+		serverDetails, err = ca.ServerDetails()
+	}
 	return technologies.BuildInfoBomGeneratorParams{
 		XrayVersion:      ca.GetXrayVersion(),
 		ExclusionPattern: technologies.GetScaExcludePattern(ca.GetConfigProfile(), ca.IsRecursiveScan(), ca.Exclusions()...),
@@ -923,6 +942,14 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 	if err := validateRunNativeForTech(tech, ca.RunNative()); err != nil {
 		return err
 	}
+	// Resolve Pipenv's native repo/server before getBuildInfoParamsByTech so install and the
+	// later probes share an endpoint. Other techs resolve later via SetResolutionRepoInParamsIfExists
+	// and must not be forced through SetRepo this early (they tolerate having no config file yet).
+	if tech == techutils.Pipenv && ca.PackageManagerConfig == nil {
+		if err := ca.SetRepo(tech); err != nil {
+			return err
+		}
+	}
 	params, err := ca.getBuildInfoParamsByTech()
 	if err != nil {
 		return errorutils.CheckErrorf("failed to get build info params for %s: %v", tech.String(), err)
@@ -956,7 +983,7 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 		// Instead of aborting with no output, run the metadata-API fallback to
 		// recover the curation policy and render a partial table.
 		var cvsErr *python.CvsBlockedError
-		if (tech == techutils.Pip || tech == techutils.Poetry) && errors.As(err, &cvsErr) {
+		if (tech == techutils.Pip || tech == techutils.Poetry || tech == techutils.Pipenv) && errors.As(err, &cvsErr) {
 			return ca.runCvsFallback(cvsErr, tech, results)
 		}
 		return err
@@ -980,6 +1007,12 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 	if err != nil {
 		return err
 	}
+	if tech == techutils.Pip || tech == techutils.Poetry || tech == techutils.Pipenv {
+		rtManager, err = boundedRedirectManager(serverDetails)
+		if err != nil {
+			return err
+		}
+	}
 	rtAuth, err := serverDetails.CreateArtAuthConfig()
 	if err != nil {
 		return err
@@ -1309,6 +1342,10 @@ func (ca *CurationAuditCommand) SetRepo(tech techutils.Technology) error {
 		return ca.setRepoFromNpmrcForPnpm()
 	}
 
+	if tech == techutils.Pipenv {
+		return ca.setRepoFromPipfile()
+	}
+
 	// Yarn V4 uses native mode: no jf yarn-config / yarn.yaml required.
 	// Detect the running yarn version and route to the appropriate path.
 	// Version detection failures are fatal — silently falling through to the
@@ -1359,6 +1396,78 @@ func (ca *CurationAuditCommand) SetRepo(tech techutils.Technology) error {
 	return nil
 }
 
+// setRepoFromPipfile detects the Artifactory PyPI source for pipenv curation.
+//
+// Detection priority:
+//  1. pipenv.yaml — explicit 'jf pipenv-config'.
+//  2. User pip.conf index-url — Artifactory "Set me up" for pip.
+//  3. Pipfile [[source]] — Artifactory URL declared directly in the Pipfile.
+func (ca *CurationAuditCommand) setRepoFromPipfile() error {
+	projectType := techutils.Pipenv.GetProjectType()
+	_, configExists, err := project.GetProjectConfFilePath(projectType)
+	if err != nil {
+		return err
+	}
+
+	if configExists {
+		resolverParams, err := ca.getRepoParams(projectType)
+		if err != nil {
+			return err
+		}
+		configuredServer, err := resolverParams.ServerDetails()
+		if err != nil {
+			return err
+		}
+		fallbackServer, _ := ca.ServerDetails()
+		server, repo, err := python.ResolvePipfileArtifactorySource("Pipfile", configuredServer, resolverParams.TargetRepo(), fallbackServer)
+		if err != nil {
+			return err
+		}
+		repoConfig := (&project.RepositoryConfig{}).SetTargetRepo(repo).SetServerDetails(server)
+		ca.setPackageManagerConfig(repoConfig)
+		ca.SetDepsRepo(repo)
+		return nil
+	}
+
+	configuredServer, configuredRepo, pipConfPath, err := python.ParsePipConfigIndexUrl(python.DefaultPipConfPaths()...)
+	if err != nil {
+		return err
+	}
+	fallbackServer, _ := ca.ServerDetails()
+	if configuredRepo != "" && configuredServer != nil {
+		server, repo, err := python.ResolvePipfileArtifactorySource("Pipfile", configuredServer, configuredRepo, fallbackServer)
+		if err != nil {
+			return err
+		}
+		repoConfig := (&project.RepositoryConfig{}).
+			SetTargetRepo(repo).
+			SetServerDetails(server)
+		ca.setPackageManagerConfig(repoConfig)
+		ca.SetDepsRepo(repo)
+		log.Info(fmt.Sprintf("pipenv: using Artifactory repository %q from %s", repo, pipConfPath))
+		return nil
+	}
+
+	server, repo, err := python.ResolvePipfileArtifactorySource("Pipfile", nil, "", fallbackServer)
+	if err != nil {
+		return err
+	}
+	if repo != "" && server != nil {
+		repoConfig := (&project.RepositoryConfig{}).
+			SetTargetRepo(repo).
+			SetServerDetails(server)
+		ca.setPackageManagerConfig(repoConfig)
+		ca.SetDepsRepo(repo)
+		log.Info(fmt.Sprintf("pipenv: using Artifactory repository %q from Pipfile [[source]]", repo))
+		return nil
+	}
+
+	return errorutils.CheckErrorf(
+		"curation-audit for pipenv requires an Artifactory PyPI resolver. " +
+			"Either run 'jf pipenv-config', configure index-url in your user pip.conf via " +
+			"Artifactory 'Set me up', or add an Artifactory [[source]] entry to your Pipfile.")
+}
+
 // validateRunNativeForTech rejects --run-native for techs that don't implement
 // native-config semantics. npm uses it to read Artifactory details from .npmrc;
 // pnpm accepts it as a no-op (it always resolves from .npmrc). Extend the
@@ -1380,6 +1489,12 @@ func validateRunNativeForTech(tech techutils.Technology, runNative bool) error {
 	if _, ok := supported[tech]; ok {
 		return nil
 	}
+	if tech == techutils.Pipenv {
+		return errorutils.CheckErrorf(
+			"--run-native is not supported for 'pipenv' projects. " +
+				"Run 'jf ca' without --run-native; the repository is resolved automatically from " +
+				"'jf pipenv-config', ~/.pip/pip.conf, or the Artifactory [[source]] entry in your Pipfile.")
+	}
 	return errorutils.CheckErrorf(
 		"--run-native is not supported for '%s' projects. "+
 			"Run 'jf ca' without --run-native; configure the resolution repository using 'jf %s-config'.",
@@ -1626,7 +1741,13 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e
 			return nil
 		}
 		requestDetails := nc.httpClientDetails.Clone()
-		resp, _, err := nc.rtManager.Client().SendHead(packageUrl, requestDetails)
+		var resp *http.Response
+		var err error
+		if nc.tech == techutils.Pip || nc.tech == techutils.Poetry || nc.tech == techutils.Pipenv {
+			resp, _, err = nc.sendBoundedRequest(http.MethodHead, packageUrl, requestDetails)
+		} else {
+			resp, _, err = nc.rtManager.Client().SendHead(packageUrl, requestDetails)
+		}
 		if resp != nil && resp.StatusCode == http.StatusUnauthorized {
 			// Store the error silently (not returned) so errorsQueue.AddError is never
 			// called and the message is not logged here. fetchNodesStatus picks it up
@@ -1683,6 +1804,16 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e
 	return nil
 }
 
+func (nc *treeAnalyzer) sendBoundedRequest(method, requestURL string, details *httputils.HttpClientDetails) (*http.Response, []byte, error) {
+	repositoryURL := fmt.Sprintf("%s/api/pypi/%s/", strings.TrimSuffix(nc.url, "/"), nc.repo)
+	boundary, err := utils.NewEndpointBoundary(repositoryURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	return utils.SendWithBoundedRedirects(nc.rtManager.Client(), method, requestURL, details,
+		boundary, utils.MaxAuthenticatedRedirects)
+}
+
 // runCvsFallback is called when pip or poetry resolution failed because CVS
 // stripped a pinned version from the simple index (CvsBlockedError). It uses
 // the PyPI metadata API to recover each blocker's real download URL, probes
@@ -1693,6 +1824,12 @@ func (ca *CurationAuditCommand) runCvsFallback(cvsErr *python.CvsBlockedError, t
 	if err != nil {
 		return fmt.Errorf("curation-blocked resolution fallback: failed to get Artifactory manager (%w); %s error: %w", err, tech, cvsErr)
 	}
+	if tech == techutils.Pip || tech == techutils.Poetry || tech == techutils.Pipenv {
+		rtManager, err = boundedRedirectManager(serverDetails)
+		if err != nil {
+			return fmt.Errorf("curation-blocked resolution fallback: failed to create bounded HTTP manager: %w", err)
+		}
+	}
 	rtAuth, err := serverDetails.CreateArtAuthConfig()
 	if err != nil {
 		return fmt.Errorf("curation-blocked resolution fallback: failed to create auth config (%w); %s error: %w", err, tech, cvsErr)
@@ -1733,7 +1870,14 @@ func (nc *treeAnalyzer) lookupPypiAllVersions(name string) ([]string, error) {
 		strings.TrimSuffix(nc.url, "/"), nc.repo, name)
 
 	requestDetails := nc.httpClientDetails.Clone()
-	resp, body, _, err := nc.rtManager.Client().SendGet(metadataURL, true, requestDetails)
+	var resp *http.Response
+	var body []byte
+	var err error
+	if nc.tech == techutils.Pip || nc.tech == techutils.Poetry || nc.tech == techutils.Pipenv {
+		resp, body, err = nc.sendBoundedRequest(http.MethodGet, metadataURL, requestDetails)
+	} else {
+		resp, body, _, err = nc.rtManager.Client().SendGet(metadataURL, true, requestDetails)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("all-versions metadata API request failed for %s: %w", name, err)
 	}
@@ -1766,7 +1910,14 @@ func (nc *treeAnalyzer) lookupPypiNormalDownloadURL(name, ver string) (string, e
 		strings.TrimSuffix(nc.url, "/"), nc.repo, name, ver)
 
 	requestDetails := nc.httpClientDetails.Clone()
-	resp, body, _, err := nc.rtManager.Client().SendGet(metadataURL, true, requestDetails)
+	var resp *http.Response
+	var body []byte
+	var err error
+	if nc.tech == techutils.Pip || nc.tech == techutils.Poetry || nc.tech == techutils.Pipenv {
+		resp, body, err = nc.sendBoundedRequest(http.MethodGet, metadataURL, requestDetails)
+	} else {
+		resp, body, _, err = nc.rtManager.Client().SendGet(metadataURL, true, requestDetails)
+	}
 	if err != nil {
 		return "", fmt.Errorf("metadata API request failed for %s==%s: %w", name, ver, err)
 	}
@@ -1861,7 +2012,13 @@ func (nc *treeAnalyzer) fetchCvsBlockedStatus(pins []python.PinnedRequirement) [
 
 		// ── Step 3a: HEAD probe — detect whether the version is download-blocked
 		headDetails := nc.httpClientDetails.Clone()
-		headResp, _, headErr := nc.rtManager.Client().SendHead(dlURL, headDetails)
+		var headResp *http.Response
+		var headErr error
+		if nc.tech == techutils.Pip || nc.tech == techutils.Poetry || nc.tech == techutils.Pipenv {
+			headResp, _, headErr = nc.sendBoundedRequest(http.MethodHead, dlURL, headDetails)
+		} else {
+			headResp, _, headErr = nc.rtManager.Client().SendHead(dlURL, headDetails)
+		}
 		if headErr != nil && (headResp == nil || headResp.StatusCode != http.StatusForbidden) {
 			log.Debug(fmt.Sprintf("curation-blocked resolution fallback: HEAD probe failed for %s==%s: %v",
 				pin.Name, resolvedVersion, headErr))
@@ -1954,7 +2111,14 @@ func effectiveParentVersion(pin python.PinnedRequirement) string {
 func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, version string) (*PackageStatus, error) {
 	requestDetails := nc.httpClientDetails.Clone()
 	requestDetails.Headers["X-Artifactory-Curation-Request-Waiver"] = "syn"
-	getResp, respBody, _, err := nc.rtManager.Client().SendGet(packageUrl, true, requestDetails)
+	var getResp *http.Response
+	var respBody []byte
+	var err error
+	if nc.tech == techutils.Pip || nc.tech == techutils.Poetry || nc.tech == techutils.Pipenv {
+		getResp, respBody, err = nc.sendBoundedRequest(http.MethodGet, packageUrl, requestDetails)
+	} else {
+		getResp, respBody, _, err = nc.rtManager.Client().SendGet(packageUrl, true, requestDetails)
+	}
 	if err != nil {
 		if getResp == nil {
 			return nil, err
@@ -2072,7 +2236,7 @@ func getUrlNameAndVersionByTech(tech techutils.Technology, node *xrayUtils.Graph
 		return getGradleNameScopeAndVersion(node.Id, artiUrl, repo, node)
 	case techutils.Gem:
 		return getGemNameScopeAndVersion(node.Id, artiUrl, repo)
-	case techutils.Pip, techutils.Poetry:
+	case techutils.Pip, techutils.Poetry, techutils.Pipenv:
 		downloadUrls, name, version = getPythonNameVersion(node.Id, downloadUrlsMap)
 		return
 	case techutils.Go: