parser.go

// Package httpparser parses .http files, as used by JetBrains IDEs and the
// VS Code REST Client extension, into structured Go types.
//
// It is a pure parsing library: it performs no request execution and no
// variable substitution. {{placeholder}} tokens are preserved verbatim in
// URLs, header values, and bodies. The package depends only on the standard
// library.
package httpparser

import (
	"io"
	"net/http"
	"os"
	"regexp"
	"strings"
)

// File is the parsed representation of a .http file.
type File struct {
	// Vars holds file-level "@key = value" declarations that appear before the
	// first request (evaluated eagerly, once).
	Vars map[string]string
	// LazyVars holds file-level "@key := value" declarations, whose values are
	// re-evaluated on each use.
	LazyVars map[string]string
	// Requests holds the parsed requests in the order they appear.
	Requests []Request
}

// Script is a JetBrains HTTP Client pre-request or response-handler script
// attached to a request. It is either inline content (from a "{% ... %}" block)
// or a reference to an external JavaScript file.
type Script struct {
	// Source is the inline script content (External false) or the external file
	// path (External true), with any {{placeholders}} left intact.
	Source string
	// External is true when the script was declared as a "> /path/to/file.js"
	// (or "< /path/to/file.js") external-file reference rather than an inline
	// "{% ... %}" block.
	External bool
	// Line is the 1-indexed file line number of the script directive.
	Line int
}

// Request is a single request defined in a .http file.
type Request struct {
	// Name is the value of a "# @name <value>" annotation, or "" if absent.
	Name string
	// Method is the upper-cased HTTP method (GET, POST, ...).
	Method string
	// URL is the request target, with any {{placeholders}} left intact.
	URL string
	// Headers holds the request headers, with values left intact.
	Headers http.Header
	// Body is the request body, trimmed of leading/trailing whitespace, or
	// nil if the request has no body. It is nil when BodyFile is set.
	Body []byte
	// BodyFile is the path of a file whose contents are the request body,
	// declared with "< /path/to/body" after the headers. The path may contain
	// {{placeholders}} and is resolved relative to the .http file. Empty when the
	// request has an inline Body or no body.
	BodyFile string
	// Line is the 1-indexed line number of the "METHOD URL" line.
	Line int
	// Assertions holds the "# @expect" checks declared for the request, in the
	// order they appear. Nil when the request declares none.
	Assertions []Assertion
	// SpreadHeaders holds the names of object variables spread into the request
	// headers with a "...variableName" line, in the order they appear. Each
	// named object's key/value pairs are injected as headers at execution time.
	SpreadHeaders []string
	// NoCookieJar is set by a "# @no-cookie-jar" tag: the request neither sends
	// stored cookies nor stores the response's cookies.
	NoCookieJar bool
	// NoRedirect is set by a "# @no-redirect" tag: the request does not follow
	// HTTP redirects.
	NoRedirect bool
	// PreScript is the pre-request script declared with "< {% ... %}" or
	// "< /path/to/file.js", or nil when the request declares none.
	PreScript *Script
	// ResponseScript is the response-handler script declared with "> {% ... %}"
	// or "> /path/to/file.js", or nil when the request declares none.
	ResponseScript *Script
}

// supportedMethods is the set of HTTP methods a request line may use.
var supportedMethods = map[string]bool{
	"GET":     true,
	"POST":    true,
	"PUT":     true,
	"DELETE":  true,
	"PATCH":   true,
	"HEAD":    true,
	"OPTIONS": true,
}

// Parse reads a .http file from r and returns its structured representation.
func Parse(r io.Reader) (*File, error) {
	data, err := io.ReadAll(r)
	if err != nil {
		return nil, err
	}

	f := &File{Vars: map[string]string{}, LazyVars: map[string]string{}}

	lines := strings.Split(string(data), "\n")
	for i := range lines {
		lines[i] = strings.TrimRight(lines[i], "\r")
	}

	// Split into blocks separated by "###" lines, remembering the 1-indexed
	// file line number at which each block starts.
	start := 0
	title := "" // text after the preceding "###" separator, applied to the next request
	for idx := 0; idx <= len(lines); idx++ {
		if idx == len(lines) || strings.HasPrefix(strings.TrimSpace(lines[idx]), "###") {
			req, err := parseBlock(lines[start:idx], start+1, f, title)
			if err != nil {
				return nil, err
			}
			if req != nil {
				f.Requests = append(f.Requests, *req)
			}
			if idx < len(lines) {
				title = separatorTitle(lines[idx])
			}
			start = idx + 1
		}
	}

	return f, nil
}

// ParseFile reads and parses the .http file at path.
func ParseFile(path string) (*File, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return Parse(file)
}

// ParseString parses a .http file from the string s.
func ParseString(s string) (*File, error) {
	return Parse(strings.NewReader(s))
}

// parseBlock parses a single request block. lines are the block's lines and
// startLine is the 1-indexed file line number of lines[0]. It returns nil if
// the block contains no request. File-level @vars are recorded on f, but only
// while no request has been parsed yet.
func parseBlock(lines []string, startLine int, f *File, title string) (*Request, error) {
	seenRequest := len(f.Requests) > 0

	// Pre-request scan: skip blanks and comments, collect @name and, before
	// the first request, file-level @vars. Stop at the request line.
	name := ""
	noCookieJar, noRedirect := false, false
	var assertions []Assertion
	var preScript *Script
	i := 0
	for ; i < len(lines); i++ {
		t := strings.TrimSpace(lines[i])
		switch {
		case t == "":
			continue
		case strings.HasPrefix(t, "/*"):
			i = blockCommentEnd(lines, i)
			continue
		case strings.HasPrefix(t, "//"), strings.HasPrefix(t, "#"):
			if n, ok := parseName(t); ok {
				name = n
			} else if isMetaFlag(t, "no-cookie-jar") {
				noCookieJar = true
			} else if isMetaFlag(t, "no-redirect") {
				noRedirect = true
			} else if a, ok, err := parseExpect(t); err != nil {
				return nil, &ParseError{Line: startLine + i, Kind: ErrMalformedAssertion, Detail: lines[i]}
			} else if ok {
				a.Line = startLine + i
				assertions = append(assertions, a)
			}
			continue
		case strings.HasPrefix(t, "<"):
			sc, consumed, err := parseScript(lines, i, startLine, '<')
			if err != nil {
				return nil, err
			}
			preScript = sc
			i += consumed - 1 // the loop's i++ advances past the last consumed line
			continue
		case strings.HasPrefix(t, "@") && !seenRequest:
			if k, v, lazy, ok := parseVar(t); ok {
				if lazy {
					f.LazyVars[k] = v
				} else {
					f.Vars[k] = v
				}
				continue
			}
		}
		// Anything else is the request line.
		break
	}

	if i >= len(lines) {
		return nil, nil // no request in this block
	}

	// A "### <title>" separator names the request when no "# @name" is given.
	if name == "" {
		name = title
	}

	methodLine := startLine + i
	method, url, err := parseRequestLine(strings.TrimSpace(lines[i]), methodLine)
	if err != nil {
		return nil, err
	}
	i++

	// URL continuation: indented lines immediately following the request line
	// extend the URL, as used to split long query strings across lines. The
	// continuation ends at the first blank line, comment, or non-indented line.
	for i < len(lines) {
		line := lines[i]
		if line == "" || (line[0] != ' ' && line[0] != '\t') {
			break
		}
		t := strings.TrimSpace(line)
		if t == "" || strings.HasPrefix(t, "#") || strings.HasPrefix(t, "//") {
			break
		}
		url += t
		i++
	}

	req := &Request{
		Name:        name,
		Method:      method,
		URL:         url,
		Headers:     http.Header{},
		Line:        methodLine,
		PreScript:   preScript,
		NoCookieJar: noCookieJar,
		NoRedirect:  noRedirect,
	}

	// Headers: Key: Value pairs up to the first blank line.
	lastKey := ""
	for ; i < len(lines); i++ {
		t := strings.TrimSpace(lines[i])
		if t == "" {
			i++ // consume the blank separator line
			break
		}
		if strings.HasPrefix(t, "/*") {
			i = blockCommentEnd(lines, i)
			continue
		}
		if strings.HasPrefix(t, "//") || strings.HasPrefix(t, "#") {
			if a, ok, err := parseExpect(t); err != nil {
				return nil, &ParseError{Line: startLine + i, Kind: ErrMalformedAssertion, Detail: lines[i]}
			} else if ok {
				a.Line = startLine + i
				assertions = append(assertions, a)
			}
			continue // comment line among the headers
		}
		if strings.HasPrefix(t, "...") {
			if n := strings.TrimSpace(t[3:]); n != "" {
				req.SpreadHeaders = append(req.SpreadHeaders, n)
			}
			lastKey = ""
			continue // "...var" spreads an object variable into the headers
		}
		// Folded header value: an indented continuation line extends the
		// previous header's value, joined with a single space (RFC 7230
		// obs-fold unfolding).
		if lastKey != "" && (lines[i][0] == ' ' || lines[i][0] == '\t') {
			ck := http.CanonicalHeaderKey(lastKey)
			if vals := req.Headers[ck]; len(vals) > 0 {
				vals[len(vals)-1] += " " + t
			}
			continue
		}
		key, value, ok := parseHeader(lines[i])
		if !ok {
			return nil, &ParseError{Line: startLine + i, Kind: ErrMalformedHeader, Detail: lines[i]}
		}
		req.Headers.Add(key, value)
		lastKey = key
	}

	req.Assertions = assertions

	// Body region: everything after the blank line, up to an optional trailing
	// "> {% ... %}" / "> /path.js" response-handler script. The script, when
	// present, is the last thing in the block.
	bodyEnd := len(lines)
	for j := i; j < len(lines); j++ {
		if strings.HasPrefix(strings.TrimSpace(lines[j]), ">") {
			sc, _, err := parseScript(lines, j, startLine, '>')
			if err != nil {
				return nil, err
			}
			req.ResponseScript = sc
			bodyEnd = j
			break
		}
	}

	if i < bodyEnd {
		body := strings.TrimSpace(strings.Join(lines[i:bodyEnd], "\n"))
		if body != "" {
			if path, ok := parseBodyFile(body); ok {
				req.BodyFile = path
			} else {
				if isFormURLEncoded(req.Headers) {
					body = collapseFormNewlines(body)
				}
				req.Body = []byte(body)
			}
		}
	}

	return req, nil
}

// formContinuationRe matches a newline that begins a form-body continuation
// line ("&key=value"), so it can be collapsed into the field separator.
var formContinuationRe = regexp.MustCompile(`\n[ \t]*&`)

// isFormURLEncoded reports whether the request declares an
// application/x-www-form-urlencoded body.
func isFormURLEncoded(h http.Header) bool {
	ct := h.Get("Content-Type")
	if i := strings.IndexByte(ct, ';'); i >= 0 {
		ct = ct[:i]
	}
	return strings.EqualFold(strings.TrimSpace(ct), "application/x-www-form-urlencoded")
}

// collapseFormNewlines joins "&"-prefixed continuation lines of a form body onto
// the preceding line, matching the JetBrains HTTP Client: "a=1\n&b=2\n&c=3"
// becomes "a=1&b=2&c=3".
func collapseFormNewlines(body string) string {
	return formContinuationRe.ReplaceAllString(body, "&")
}

// blockCommentEnd returns the index of the last line occupied by a "/* ... */"
// block comment that begins at lines[i] (its trimmed text starts with "/*").
// Recognised only in the pre-request and header regions, never inside a body,
// matching how line comments are treated. An unterminated block runs to the end.
func blockCommentEnd(lines []string, i int) int {
	if strings.Contains(strings.TrimSpace(lines[i])[2:], "*/") {
		return i // opens and closes on the same line
	}
	for j := i + 1; j < len(lines); j++ {
		if strings.Contains(lines[j], "*/") {
			return j
		}
	}
	return len(lines) - 1
}

// parseBodyFile recognises a "< /path/to/file" body-from-file directive. It
// returns the path when body is a single such line. A body that merely starts
// with '<' (for example inline XML like "<note>") is not a directive: the marker
// must be followed by whitespace, matching the .http format. Multi-line bodies
// are treated as literal content.
func parseBodyFile(body string) (string, bool) {
	if strings.ContainsRune(body, '\n') {
		return "", false
	}
	if len(body) < 2 || body[0] != '<' {
		return "", false
	}
	if body[1] != ' ' && body[1] != '\t' {
		return "", false
	}
	path := strings.TrimSpace(body[1:])
	if path == "" {
		return "", false
	}
	return path, true
}

// parseScript parses a script directive beginning at lines[i], introduced by
// marker ('<' for a pre-request script, '>' for a response handler). An inline
// "{% ... %}" block may span multiple lines; an external reference is a single
// "marker /path/to/file.js" line. It returns the parsed Script, the number of
// block lines consumed, and any error. Script.Line is the 1-indexed file line
// of the directive (startLine + i).
func parseScript(lines []string, i, startLine int, marker byte) (*Script, int, error) {
	line := strings.TrimSpace(lines[i])
	rest := strings.TrimSpace(line[1:]) // line[0] is the marker

	if strings.HasPrefix(rest, "{%") {
		var b strings.Builder
		for j := i; j < len(lines); j++ {
			seg := lines[j]
			if j == i {
				// Skip the marker and the opening "{%" on the first line.
				seg = strings.TrimSpace(lines[j])[1:]
				seg = strings.TrimSpace(seg)[len("{%"):]
			}
			if idx := strings.Index(seg, "%}"); idx >= 0 {
				b.WriteString(seg[:idx])
				return &Script{
					Source: strings.TrimSpace(b.String()),
					Line:   startLine + i,
				}, j - i + 1, nil
			}
			b.WriteString(seg)
			b.WriteByte('\n')
		}
		return nil, 0, &ParseError{Line: startLine + i, Kind: ErrMalformedScript, Detail: lines[i]}
	}

	if rest == "" {
		return nil, 0, &ParseError{Line: startLine + i, Kind: ErrMalformedScript, Detail: lines[i]}
	}
	return &Script{Source: rest, External: true, Line: startLine + i}, 1, nil
}

// parseRequestLine parses a "[METHOD] URL [HTTP-version]" line. The method may
// be omitted, in which case it defaults to GET. The HTTP version, if present, is
// accepted and ignored.
func parseRequestLine(line string, lineNo int) (method, url string, err error) {
	fields := strings.Fields(line)
	if len(fields) == 0 {
		return "", "", &ParseError{Line: lineNo, Kind: ErrMalformedRequestLine, Detail: line}
	}
	method = strings.ToUpper(fields[0])
	if supportedMethods[method] {
		if len(fields) < 2 {
			return "", "", &ParseError{Line: lineNo, Kind: ErrMalformedRequestLine, Detail: line}
		}
		return method, fields[1], nil
	}
	// The first token is not a supported method. A URL-shaped token means the
	// method was omitted and defaults to GET; anything else is an error.
	if looksLikeURL(fields[0]) {
		return "GET", fields[0], nil
	}
	if len(fields) < 2 {
		return "", "", &ParseError{Line: lineNo, Kind: ErrMalformedRequestLine, Detail: line}
	}
	return "", "", &ParseError{Line: lineNo, Kind: ErrUnsupportedMethod, Detail: line}
}

// looksLikeURL reports whether s has the shape of a request URL (as opposed to
// an HTTP method token), used to detect a request line whose method is omitted.
func looksLikeURL(s string) bool {
	return strings.Contains(s, "://") ||
		strings.HasPrefix(s, "{{") ||
		strings.HasPrefix(s, "/") ||
		strings.ContainsAny(s, ".:")
}

// separatorTitle extracts the title text following a "###" request separator,
// e.g. "### Create the user" yields "Create the user".
func separatorTitle(line string) string {
	s := strings.TrimSpace(line)
	s = strings.TrimLeft(s, "#")
	return strings.TrimSpace(s)
}

// parseVar parses an "@key = value" (eager) or "@key := value" (lazy)
// declaration. lazy is true for the ":=" form.
func parseVar(line string) (key, value string, lazy, ok bool) {
	line = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "@"))
	eq := strings.Index(line, "=")
	if eq <= 0 {
		return "", "", false, false
	}
	// A ":=" whose "=" is the first "=" in the line is a lazy declaration.
	if colonEq := strings.Index(line, ":="); colonEq >= 0 && colonEq+1 == eq {
		key = strings.TrimSpace(line[:colonEq])
		value = strings.TrimSpace(line[colonEq+2:])
		return key, value, true, key != ""
	}
	key = strings.TrimSpace(line[:eq])
	value = strings.TrimSpace(line[eq+1:])
	return key, value, false, key != ""
}

// parseName extracts the value of a "# @name <value>" (or "// @name <value>")
// annotation. comment is the trimmed comment line including its marker. The
// value may be separated from @name by whitespace and/or an '=' sign.
func parseName(comment string) (string, bool) {
	s := strings.TrimSpace(comment)
	s = strings.TrimPrefix(s, "//")
	s = strings.TrimPrefix(s, "#")
	s = strings.TrimSpace(s)

	if !strings.HasPrefix(s, "@name") {
		return "", false
	}
	rest := s[len("@name"):]
	// The character after @name must be a separator, else it's a different
	// annotation (e.g. @named).
	if rest != "" && rest[0] != ' ' && rest[0] != '\t' && rest[0] != '=' {
		return "", false
	}
	rest = strings.TrimSpace(rest)
	rest = strings.TrimPrefix(rest, "=")
	rest = strings.TrimSpace(rest)
	if rest == "" {
		return "", false
	}
	return rest, true
}

// isMetaFlag reports whether a comment line is exactly the metadata flag
// "@<flag>" (for example "# @no-redirect").
func isMetaFlag(comment, flag string) bool {
	s := strings.TrimSpace(comment)
	s = strings.TrimPrefix(s, "//")
	s = strings.TrimPrefix(s, "#")
	return strings.TrimSpace(s) == "@"+flag
}

// parseHeader parses a "Key: Value" header line, leaving the value intact.
func parseHeader(line string) (key, value string, ok bool) {
	idx := strings.Index(line, ":")
	if idx <= 0 {
		return "", "", false
	}
	key = strings.TrimSpace(line[:idx])
	value = strings.TrimSpace(line[idx+1:])
	if key == "" {
		return "", "", false
	}
	return key, value, true
}