diff --git a/README.md b/README.md
index 2106257..e7ac360 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,27 @@ echo $highlighter->highlight($code, 'php');
`…
`. Emit a theme stylesheet
once per page, then reuse the highlighter for every code block.
+## Parse without rendering
+
+Use `CodeParser` when another component needs semantic tokens instead of HTML:
+
+```php
+use Alto\Code\Highlight\CodeParser;
+
+$stream = (new CodeParser())->parse(
+ '$total = array_sum($prices);',
+ 'php',
+);
+
+foreach ($stream as $token) {
+ echo $token->text.' '.$token->scope->value.PHP_EOL;
+}
+```
+
+`parse()` returns a `ParsedStream` and preserves the caller's source exactly:
+`$stream->toString()` is the original code. It also resolves configured
+embedded languages, without requiring a theme or choosing an output format.
+
## What it covers
- **27 languages:** the PHP web stack plus common programming, markup, data,
@@ -76,7 +97,7 @@ once per page, then reuse the highlighter for every code block.
| [Create a theme](docs/theming/creating.md) | Implement `ThemeInterface` |
| [Embedded languages](docs/languages/embedded.md) | HTML, SVG, Markdown, and Twig |
| [Theme adapters](docs/theming/adapters.md) | Highlight.js, Prism, and TextMate |
-| [Public API](docs/api/index.md) | Supported entry points and extension contracts |
+| [Public API](docs/api/index.md) | Parsing, rendering, and extension contracts |
| [Examples](docs/examples.md) | Compact examples and generated previews |
The complete source examples are available in [`examples/languages/`](examples/languages/).
diff --git a/docs/api/index.md b/docs/api/index.md
index 50f7d33..244b72f 100644
--- a/docs/api/index.md
+++ b/docs/api/index.md
@@ -4,9 +4,26 @@ Alto Code Highlight follows semantic versioning for the supported entry points
and extension contracts described here. Patch and minor releases preserve
their documented signatures and behavior throughout the 1.x series.
-## Main entry point
+## Parsing
-`Highlighter` is the primary facade. Its supported operations are:
+`CodeParser` parses source into a `ParsedStream` without rendering it. Its
+supported operations are:
+
+- construction with an optional embedding registry and optional language list;
+- `parse()` for semantically scoped tokens;
+- `registerLanguage()` for adding or replacing a parser;
+- `getEmbeddedRegistry()` for inspecting embedding plans;
+- `setEmbeddingEnabled()` for toggling a configured host and target pair.
+
+`parse()` preserves the source supplied by the caller. Concatenating the token
+text, or calling `ParsedStream::toString()`, returns that source exactly.
+Selecting `php` parses PHP from the first byte even when the opening tag is
+omitted.
+
+## HTML rendering
+
+`Highlighter` renders the same parsed representation as escaped HTML. Its
+supported operations are:
- construction with a `ThemeInterface`, optional embedding registry, and
optional language list;
@@ -17,8 +34,7 @@ their documented signatures and behavior throughout the 1.x series.
- `setEmbeddingEnabled()` for toggling a configured host and target pair.
`HighlighterInterface` defines the portable highlighting operation for code
-that depends on an abstraction rather than the concrete facade. Selecting
-`php` parses PHP from the first byte even when the opening tag is omitted.
+that depends on an abstraction rather than the concrete facade.
## Theme extension contract
diff --git a/docs/index.md b/docs/index.md
index 0e986ab..796b7ad 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -34,12 +34,15 @@ require a browser-side highlighter.
## Public API at a glance
-`Alto\Code\Highlight\Highlighter` is the main entry point:
+Use `CodeParser` when you need tokens, and `Highlighter` when you need HTML:
```php
+use Alto\Code\Highlight\CodeParser;
use Alto\Code\Highlight\Highlighter;
use Alto\Code\Highlight\Theme\AltoTheme;
+$tokens = (new CodeParser())->parse('$answer = 42;', 'php');
+
$highlighter = new Highlighter(new AltoTheme());
$html = $highlighter->highlight('getTheme()->getStylesheet();
diff --git a/src/CodeParser.php b/src/CodeParser.php
new file mode 100644
index 0000000..576b800
--- /dev/null
+++ b/src/CodeParser.php
@@ -0,0 +1,139 @@
+
+ */
+final class CodeParser
+{
+ /**
+ * @var array
+ */
+ private array $languages = [];
+
+ private EmbeddedLanguageRegistry $embeddedRegistry;
+
+ /**
+ * @var array
+ */
+ private array $embeddedLanguageToggles = [];
+
+ /**
+ * @param list|null $languages
+ */
+ public function __construct(
+ ?EmbeddedLanguageRegistry $embeddedRegistry = null,
+ ?array $languages = null,
+ ) {
+ $this->embeddedRegistry = $embeddedRegistry ?? new EmbeddedLanguageRegistry();
+
+ $languages ??= Languages::getDefaultLanguages();
+ foreach ($languages as $language) {
+ if (!$language instanceof LanguageInterface) {
+ throw new \InvalidArgumentException('All languages must implement LanguageInterface.');
+ }
+
+ $this->registerLanguage($language);
+ }
+ }
+
+ /**
+ * Parse source code without rendering it.
+ */
+ public function parse(string $code, string $language): ParsedStream
+ {
+ $language = strtolower(trim($language));
+
+ if ('php-snippet' === $language) {
+ $language = 'php';
+ }
+
+ return $this->parseWithLanguage($this->getLanguage($language), $code);
+ }
+
+ /**
+ * Register a language parser.
+ */
+ public function registerLanguage(LanguageInterface $language): void
+ {
+ $this->languages[$language->getIdentifier()] = $language;
+ }
+
+ public function getEmbeddedRegistry(): EmbeddedLanguageRegistry
+ {
+ return $this->embeddedRegistry;
+ }
+
+ /**
+ * Enable or disable a specific embedded language.
+ *
+ * @param string $host The host language (e.g., 'html')
+ * @param string $target The embedded language to toggle (e.g., 'javascript')
+ */
+ public function setEmbeddingEnabled(string $host, string $target, bool $enabled): void
+ {
+ $this->embeddedLanguageToggles[strtolower($host) . ':' . strtolower($target)] = $enabled;
+ }
+
+ /**
+ * @throws LanguageNotFoundException
+ */
+ private function getLanguage(string $identifier): LanguageInterface
+ {
+ if (!isset($this->languages[$identifier])) {
+ throw new LanguageNotFoundException($identifier);
+ }
+
+ return $this->languages[$identifier];
+ }
+
+ private function parseWithLanguage(LanguageInterface $language, string $code): ParsedStream
+ {
+ if ($language instanceof EmbeddedLanguageCapable) {
+ $plan = $this->embeddedRegistry->getPlan($language->getIdentifier());
+
+ if (null !== $plan) {
+ $host = $language->getIdentifier();
+ $triggers = array_filter($plan->getTriggers(), function ($trigger) use ($host) {
+ $key = $host . ':' . $trigger->targetLanguage;
+
+ return $this->embeddedLanguageToggles[$key] ?? true;
+ });
+ $plan = EmbeddedLanguagePlan::forHost($host, array_values($triggers));
+ }
+
+ $context = EmbeddedLanguageContext::fromResolver(function (string $identifier, string $embeddedCode): ParsedStream {
+ $embeddedLanguage = $this->getLanguage($identifier);
+
+ return $this->parseWithLanguage($embeddedLanguage, $embeddedCode);
+ }, $plan);
+
+ return $language->parseWithEmbedding($code, $context);
+ }
+
+ return $language->parse($code);
+ }
+}
diff --git a/src/Highlighter.php b/src/Highlighter.php
index 6fd38c2..bec98ca 100644
--- a/src/Highlighter.php
+++ b/src/Highlighter.php
@@ -13,13 +13,8 @@
namespace Alto\Code\Highlight;
-use Alto\Code\Highlight\Embedded\EmbeddedLanguagePlan;
use Alto\Code\Highlight\Embedded\EmbeddedLanguageRegistry;
-use Alto\Code\Highlight\Exception\LanguageNotFoundException;
-use Alto\Code\Highlight\Language\EmbeddedLanguageCapable;
-use Alto\Code\Highlight\Language\EmbeddedLanguageContext;
use Alto\Code\Highlight\Language\LanguageInterface;
-use Alto\Code\Highlight\Language\Languages;
use Alto\Code\Highlight\Parser\ParsedStream;
/**
@@ -31,17 +26,7 @@
*/
final class Highlighter implements HighlighterInterface
{
- /**
- * @var array
- */
- private array $languages = [];
-
- private EmbeddedLanguageRegistry $embeddedRegistry;
-
- /**
- * @var array
- */
- private array $embeddedLanguageToggles = [];
+ private CodeParser $parser;
/**
* @param list|null $languages
@@ -51,17 +36,7 @@ public function __construct(
?EmbeddedLanguageRegistry $embeddedRegistry = null,
?array $languages = null,
) {
- $this->embeddedRegistry = $embeddedRegistry ?? new EmbeddedLanguageRegistry();
-
- // Register built-in languages
- $languages ??= Languages::getDefaultLanguages();
- foreach ($languages as $language) {
- if (!$language instanceof LanguageInterface) {
- throw new \InvalidArgumentException('All languages must implement LanguageInterface.');
- }
-
- $this->registerLanguage($language);
- }
+ $this->parser = new CodeParser($embeddedRegistry, $languages);
}
/**
@@ -88,11 +63,7 @@ public function highlight(
$language = 'php';
}
- // Get the language parser
- $languageParser = $this->getLanguage($language);
-
- // Parse the code (recursively handling embedded languages when supported)
- $parsedStream = $this->parseWithLanguage($languageParser, $code);
+ $parsedStream = $this->parser->parse($code, $language);
// Format the output
return $this->format($parsedStream, $language, $lineNumbers, $highlightLines);
@@ -103,7 +74,7 @@ public function highlight(
*/
public function registerLanguage(LanguageInterface $language): void
{
- $this->languages[$language->getIdentifier()] = $language;
+ $this->parser->registerLanguage($language);
}
/**
@@ -114,54 +85,9 @@ public function getTheme(): ThemeInterface
return $this->theme;
}
- /**
- * Get a registered language parser.
- *
- * @throws LanguageNotFoundException
- */
- private function getLanguage(string $identifier): LanguageInterface
- {
- if (!isset($this->languages[$identifier])) {
- throw new LanguageNotFoundException($identifier);
- }
-
- return $this->languages[$identifier];
- }
-
- /**
- * Parse source code with the given language, handling embedded delegation when available.
- */
- private function parseWithLanguage(LanguageInterface $language, string $code): ParsedStream
- {
- if ($language instanceof EmbeddedLanguageCapable) {
- $plan = $this->embeddedRegistry->getPlan($language->getIdentifier());
-
- if (null !== $plan) {
- $host = $language->getIdentifier();
- $triggers = array_filter($plan->getTriggers(), function ($trigger) use ($host) {
- $key = $host . ':' . $trigger->targetLanguage;
-
- return $this->embeddedLanguageToggles[$key] ?? true;
- });
- // Create a new plan with filtered triggers
- $plan = EmbeddedLanguagePlan::forHost($host, array_values($triggers));
- }
-
- $context = EmbeddedLanguageContext::fromResolver(function (string $identifier, string $embeddedCode): ParsedStream {
- $embeddedLanguage = $this->getLanguage($identifier);
-
- return $this->parseWithLanguage($embeddedLanguage, $embeddedCode);
- }, $plan);
-
- return $language->parseWithEmbedding($code, $context);
- }
-
- return $language->parse($code);
- }
-
public function getEmbeddedRegistry(): EmbeddedLanguageRegistry
{
- return $this->embeddedRegistry;
+ return $this->parser->getEmbeddedRegistry();
}
/**
@@ -172,7 +98,7 @@ public function getEmbeddedRegistry(): EmbeddedLanguageRegistry
*/
public function setEmbeddingEnabled(string $host, string $target, bool $enabled): void
{
- $this->embeddedLanguageToggles[strtolower($host) . ':' . strtolower($target)] = $enabled;
+ $this->parser->setEmbeddingEnabled($host, $target, $enabled);
}
/**
diff --git a/tests/Unit/CodeParserTest.php b/tests/Unit/CodeParserTest.php
new file mode 100644
index 0000000..778a234
--- /dev/null
+++ b/tests/Unit/CodeParserTest.php
@@ -0,0 +1,143 @@
+getByRole("button")->click();';
+
+ $stream = (new CodeParser())->parse($code, 'php');
+
+ self::assertSame($code, $stream->toString());
+ self::assertContains(Scope::Variable, array_map(
+ static fn(ParsedToken $token): Scope => $token->scope,
+ $stream->tokens,
+ ));
+ self::assertContains(Scope::FunctionCall, array_map(
+ static fn(ParsedToken $token): Scope => $token->scope,
+ $stream->tokens,
+ ));
+ }
+
+ public function testNormalizesLanguageIdentifierAndPhpSnippetAlias(): void
+ {
+ $parser = new CodeParser();
+ $code = '$answer = 42;';
+
+ self::assertSame($code, $parser->parse($code, ' PHP ')->toString());
+ self::assertSame($code, $parser->parse($code, 'php-snippet')->toString());
+ }
+
+ public function testThrowsForUnknownLanguage(): void
+ {
+ $this->expectException(LanguageNotFoundException::class);
+
+ (new CodeParser())->parse('code', 'unknown-language');
+ }
+
+ public function testCanRegisterAndParseACustomLanguage(): void
+ {
+ $language = self::customLanguage('custom');
+ $parser = new CodeParser(languages: []);
+ $parser->registerLanguage($language);
+
+ $stream = $parser->parse('source', 'custom');
+
+ self::assertSame('source', $stream->toString());
+ self::assertSame(Scope::String, $stream->tokens[0]->scope);
+ }
+
+ public function testAcceptsLanguagesInConstructor(): void
+ {
+ $parser = new CodeParser(languages: [self::customLanguage('custom')]);
+
+ self::assertSame('source', $parser->parse('source', 'custom')->toString());
+ }
+
+ public function testRejectsInvalidConstructorLanguage(): void
+ {
+ $this->expectException(\InvalidArgumentException::class);
+ $this->expectExceptionMessage('All languages must implement LanguageInterface.');
+
+ new CodeParser(languages: [new \stdClass()]);
+ }
+
+ public function testExposesInjectedEmbeddedLanguageRegistry(): void
+ {
+ $registry = new EmbeddedLanguageRegistry([]);
+
+ $parser = new CodeParser($registry);
+
+ self::assertSame($registry, $parser->getEmbeddedRegistry());
+ self::assertSame(
+ '',
+ $parser->parse('', 'html')->toString(),
+ );
+ }
+
+ public function testCanDisableAndReEnableEmbeddedLanguageParsing(): void
+ {
+ $parser = new CodeParser();
+ $code = '';
+
+ self::assertTrue($this->hasToken($parser->parse($code, 'html'), 'var', Scope::KeywordDeclaration));
+
+ $parser->setEmbeddingEnabled('HTML', 'JAVASCRIPT', false);
+ self::assertFalse($this->hasToken($parser->parse($code, 'html'), 'var', Scope::KeywordDeclaration));
+
+ $parser->setEmbeddingEnabled('html', 'javascript', true);
+ self::assertTrue($this->hasToken($parser->parse($code, 'html'), 'var', Scope::KeywordDeclaration));
+ }
+
+ private static function customLanguage(string $identifier): LanguageInterface
+ {
+ return new class ($identifier) implements LanguageInterface {
+ public function __construct(private readonly string $identifier) {}
+
+ public function parse(string $code): ParsedStream
+ {
+ return new ParsedStream([new ParsedToken($code, Scope::String)]);
+ }
+
+ public function getIdentifier(): string
+ {
+ return $this->identifier;
+ }
+ };
+ }
+
+ private function hasToken(ParsedStream $stream, string $text, Scope $scope): bool
+ {
+ foreach ($stream as $token) {
+ if ($text === $token->text && $scope === $token->scope) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+}