From d89a156ea76185295e380f4ecd30b6f0c25846bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 21 Jul 2026 11:55:29 +0200 Subject: [PATCH 1/6] Support building against a system uchardet library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `system-uchardet` option to build against the system installed `uchardet` library instead of building a bundled copy. This requires a git version of `uchardet` right now, and upstream did not increment the version yet, so the code is explicitly checking whether `uchardet_get_n_candidates` is available. Signed-off-by: Michał Górny --- meson.options | 2 ++ src/cchardet/meson.build | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 meson.options diff --git a/meson.options b/meson.options new file mode 100644 index 0000000..c46ecab --- /dev/null +++ b/meson.options @@ -0,0 +1,2 @@ +option('system-uchardet', type: 'boolean', value: false, + description: 'Build against the system uchardet library (requires chardet from git)') diff --git a/src/cchardet/meson.build b/src/cchardet/meson.build index 58bc3aa..4562211 100644 --- a/src/cchardet/meson.build +++ b/src/cchardet/meson.build @@ -67,15 +67,28 @@ uchardet_sources = files( uchardet_inc = include_directories('../ext/uchardet/src') +if get_option('system-uchardet') + uchardet = dependency('uchardet') + # TODO: replace with a version constraint once new uchardet is released + cpp = meson.get_compiler('cpp') + if not cpp.has_function('uchardet_get_n_candidates', dependencies: [uchardet]) + error('system-uchardet requires git version of uchardet from https://gitlab.freedesktop.org/uchardet/uchardet.git') + endif +else + uchardet = declare_dependency( + sources: uchardet_sources, + include_directories: uchardet_inc, + ) +endif + py.extension_module( '_cchardet', '_cchardet.pyx', - uchardet_sources, + dependencies: [uchardet], # Build the Cython output as C++ (the module wraps a C++ library). Meson then # links the C++ runtime that matches the active compiler (libstdc++ or # libc++), so no manual `-lstdc++` is needed. override_options: ['cython_language=cpp'], - include_directories: uchardet_inc, install: true, subdir: 'cchardet', ) From c8bc48bf5bd18be5e9de461a478b13b530b68a86 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 24 Jul 2026 13:12:07 +0000 Subject: [PATCH 2/6] meson: make system-uchardet a feature option (auto/enabled/disabled) Refine the system-uchardet switch from a hard boolean into a Meson `feature` option so one switch serves more environments: - enabled -> require the system library; hard fail if missing or too old (what distro packaging wants). - disabled -> always build the bundled copy (the default; unchanged behaviour for the published wheels). - auto -> use the system library when it is new enough, else fall back to the bundled copy (handy for source installs on a box that happens to have a recent-enough uchardet). Passing the feature to dependency(..., required: system_uchardet) gives the enabled/auto hard-fail-vs-fallback distinction for free. The existing has_function('uchardet_get_n_candidates') probe still rejects a found-but-too-old system library, and now only errors out under `enabled` -- `auto` falls through to the bundled copy. Also document the option for packagers in the README (the three states, the --config-settings invocation, and the git-uchardet requirement), and fix the option description ("chardet" -> "uchardet"). Validated with meson setup for all three states: disabled and auto (no system lib) configure the bundled copy and the full test suite passes (125 passed, 1 skipped); enabled with no system lib fails configuration as intended. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BnTZQEXqpedyrFAgwRnbsA --- README.md | 26 ++++++++++++++++++++++++ meson.options | 4 ++-- src/cchardet/meson.build | 43 +++++++++++++++++++++++++++++++++------- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 75134bc..e76f2c6 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,32 @@ import cchardet # same import name as upstream - On 32-bit (`i686`) builds a few near-equivalent labels differ — e.g. Thai `TIS-620` is detected as `ISO-8859-11`. +## Building against a system `uchardet` + +By default cChardet builds a **bundled** copy of `uchardet` (vendored as a git +submodule), which is what the published wheels ship. Distributions and source +builds can instead link the **system** `uchardet` through the `system-uchardet` +Meson [feature option](https://mesonbuild.com/Build-options.html#features): + +| `-Dsystem-uchardet=` | Behaviour | +| --- | --- | +| `disabled` (default) | Always build the bundled copy (used by the wheels). | +| `enabled` | Require the system library; the build **fails** if it is missing or too old. Recommended for distro packaging. | +| `auto` | Use the system library if it is new enough, otherwise fall back to the bundled copy. | + +Pass the option through your build front-end. With `pip` / `build` the +meson-python backend reads it from `setup-args`: + +```bash +pip install . --config-settings=setup-args=-Dsystem-uchardet=enabled +``` + +> **Requirement:** the system `uchardet` must be recent enough to expose +> `uchardet_get_n_candidates`. That symbol currently ships only in the git +> version () — no tagged +> release provides it yet — so `enabled`/`auto` probe for it and reject an +> older library. + ## Supported Languages/Encodings - International (Unicode) diff --git a/meson.options b/meson.options index c46ecab..e89386c 100644 --- a/meson.options +++ b/meson.options @@ -1,2 +1,2 @@ -option('system-uchardet', type: 'boolean', value: false, - description: 'Build against the system uchardet library (requires chardet from git)') +option('system-uchardet', type: 'feature', value: 'disabled', + description: 'Build against the system uchardet library instead of the bundled copy. enabled: require the system library (hard fail if missing or too old); disabled: always use the bundled copy (the wheel default); auto: use the system library if it is new enough, else fall back to the bundled copy. Requires uchardet from git until upstream tags a release exposing uchardet_get_n_candidates.') diff --git a/src/cchardet/meson.build b/src/cchardet/meson.build index 4562211..cfb6b97 100644 --- a/src/cchardet/meson.build +++ b/src/cchardet/meson.build @@ -67,14 +67,43 @@ uchardet_sources = files( uchardet_inc = include_directories('../ext/uchardet/src') -if get_option('system-uchardet') - uchardet = dependency('uchardet') - # TODO: replace with a version constraint once new uchardet is released - cpp = meson.get_compiler('cpp') - if not cpp.has_function('uchardet_get_n_candidates', dependencies: [uchardet]) - error('system-uchardet requires git version of uchardet from https://gitlab.freedesktop.org/uchardet/uchardet.git') +# Select the uchardet implementation to build against. `system-uchardet` is a +# feature option: +# enabled -> require the system library; hard fail if it is missing or too +# old. This is what distro packaging wants. +# disabled -> always build the bundled copy (the default, used by the wheels). +# auto -> use the system library if it is new enough, otherwise fall back +# to the bundled copy (convenient for source installs). +system_uchardet = get_option('system-uchardet') + +use_bundled = true +if not system_uchardet.disabled() + # Passing the feature as `required:` makes meson hard fail for `enabled` when + # no uchardet is installed, while `auto` continues quietly if it is absent. + system_uchardet_dep = dependency('uchardet', required: system_uchardet) + if system_uchardet_dep.found() + # The candidate API (uchardet_get_n_candidates) only exists in uchardet + # newer than the last tagged release, so a found-but-too-old system library + # must still be rejected. + # TODO: replace this has_function() probe with a `version: '>=X'` constraint + # on the dependency() call above once upstream tags a release exposing it. + cpp = meson.get_compiler('cpp') + if cpp.has_function('uchardet_get_n_candidates', dependencies: [system_uchardet_dep]) + uchardet = system_uchardet_dep + use_bundled = false + elif system_uchardet.enabled() + error( + 'system-uchardet is enabled but the installed uchardet is too old: ' + + 'it does not provide uchardet_get_n_candidates. Install the git ' + + 'version from https://gitlab.freedesktop.org/uchardet/uchardet.git, ' + + 'or set -Dsystem-uchardet=disabled to build the bundled copy.' + ) + endif + # auto + a too-old system library: fall through to the bundled copy below. endif -else +endif + +if use_bundled uchardet = declare_dependency( sources: uchardet_sources, include_directories: uchardet_inc, From 10f3eddc6a62456ae96f3ed3b6a5f783e7eef68a Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Fri, 24 Jul 2026 13:38:59 +0000 Subject: [PATCH 3/6] ci: test building against a system uchardet The existing Tests workflow only builds the bundled uchardet default, so the system-uchardet path is never exercised in CI. Add a Linux job that builds uchardet from git, installs it, then builds cChardet with -Dsystem-uchardet=enabled and runs the test suite. The job also asserts (via ldd) that the built extension links the shared system libuchardet rather than silently falling back to the bundled sources, and checks that the system header exposes uchardet_get_n_candidates (the symbol the enabled/auto probe requires). Stacked on the system-uchardet feature-option change; the option it exercises does not exist on master yet. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BnTZQEXqpedyrFAgwRnbsA --- .github/workflows/test-system-uchardet.yml | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/test-system-uchardet.yml diff --git a/.github/workflows/test-system-uchardet.yml b/.github/workflows/test-system-uchardet.yml new file mode 100644 index 0000000..d7602a4 --- /dev/null +++ b/.github/workflows/test-system-uchardet.yml @@ -0,0 +1,70 @@ +name: Test (system uchardet) + +# Exercises the `system-uchardet` feature option (see meson.options): build +# cChardet against a system uchardet built from git and confirm the resulting +# extension links the shared library instead of the bundled sources. The +# regular Tests workflow only covers the bundled default. +on: + pull_request: + push: + branches: + - master + +jobs: + system-uchardet: + name: Build against system uchardet (from git) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + # meson's files() validates the vendored uchardet sources exist at + # configure time even for a system build, so the submodule still has + # to be checked out. + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build and install uchardet from git + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends cmake ninja-build pkg-config + git clone --depth 1 \ + https://gitlab.freedesktop.org/uchardet/uchardet.git /tmp/uchardet-src + cmake -S /tmp/uchardet-src -B /tmp/uchardet-build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DBUILD_BINARY=OFF + sudo cmake --build /tmp/uchardet-build --target install + sudo ldconfig + + - name: Confirm the system uchardet exposes the candidate API + run: | + pkg-config --exists uchardet + echo "system uchardet version: $(pkg-config --modversion uchardet)" + # `enabled`/`auto` reject a uchardet without this symbol; the last + # tagged release still reports 0.0.8 but only the git build ships it. + grep -q uchardet_get_n_candidates /usr/include/uchardet/uchardet.h + + - name: Build cChardet against the system uchardet + run: | + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install . \ + --config-settings=setup-args=-Dsystem-uchardet=enabled + + - name: Verify the extension links the system libuchardet + run: | + so=$(python -c "import cchardet._cchardet as m; print(m.__file__)") + echo "built extension: $so" + ldd "$so" + # Fail if it did not link the shared system library -- i.e. it + # silently fell back to compiling the bundled sources. + ldd "$so" | grep -q libuchardet + + - name: Run tests + run: python -m pytest src/tests From f45a26c3aa70f85c007bc79b45c91606c00497f6 Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Thu, 30 Jul 2026 13:24:14 +0000 Subject: [PATCH 4/6] ci: set least-privilege permissions on system-uchardet workflow CodeQL (actions/missing-workflow-permissions) flagged the workflow for not constraining the GITHUB_TOKEN. The job only checks out and builds the repository, so declare the minimal `contents: read` at the workflow level. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01BnTZQEXqpedyrFAgwRnbsA --- .github/workflows/test-system-uchardet.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-system-uchardet.yml b/.github/workflows/test-system-uchardet.yml index d7602a4..35fdc9d 100644 --- a/.github/workflows/test-system-uchardet.yml +++ b/.github/workflows/test-system-uchardet.yml @@ -10,6 +10,11 @@ on: branches: - master +# Least-privilege GITHUB_TOKEN: this workflow only checks out and builds the +# repository, so read access to its contents is all it needs. +permissions: + contents: read + jobs: system-uchardet: name: Build against system uchardet (from git) From 7daf400f1e94eea02256081817f3289727bf33ba Mon Sep 17 00:00:00 2001 From: William Barnhart Date: Tue, 4 Aug 2026 07:40:18 -0400 Subject: [PATCH 5/6] docs: warn packagers that a system uchardet detects non-UTF-8 less accurately (#67) --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e76f2c6..b276a14 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Meson [feature option](https://mesonbuild.com/Build-options.html#features): | `-Dsystem-uchardet=` | Behaviour | | --- | --- | | `disabled` (default) | Always build the bundled copy (used by the wheels). | -| `enabled` | Require the system library; the build **fails** if it is missing or too old. Recommended for distro packaging. | +| `enabled` | Require the system library; the build **fails** if it is missing or too old. Intended for distro packaging — read the detection-quality caveat below first. | | `auto` | Use the system library if it is new enough, otherwise fall back to the bundled copy. | Pass the option through your build front-end. With `pip` / `build` the @@ -72,6 +72,35 @@ pip install . --config-settings=setup-args=-Dsystem-uchardet=enabled > release provides it yet — so `enabled`/`auto` probe for it and reject an > older library. +> [!WARNING] +> **A system build currently detects non-UTF-8 input less accurately than the +> bundled build.** The bundled copy compiles a cChardet-specific replacement +> for uchardet's multi-byte group prober, which both restores detection +> throughput and rejects non-UTF-8 byte sequences that upstream reports as +> UTF-8. A system build links upstream's prober, so neither applies. +> +> Measured over 1650 non-UTF-8 documents spanning 7 encodings +> (`benchmarks/make_nonutf8_corpus.py`, median of 3 runs): +> +> | Build | Throughput | Non-UTF-8 reported as UTF-8 | +> | --- | ---: | ---: | +> | bundled (wheel default) | 3.10 MB/s | 0.0% | +> | system `uchardet` | 1.44 MB/s | **16.2%** | +> +> Reporting non-UTF-8 bytes as UTF-8 is the defect that caused v3.0.0 to be +> yanked from PyPI, so this is not a cosmetic difference. It is an upstream +> issue rather than a packaging mistake: `nsUTF8Prober` does not reject +> invalid sequences on its own, and its confidence never falls low enough for +> the candidate to be discarded. Patches have been sent upstream; once they +> land, the build can require a `uchardet` version that includes them and this +> caveat goes away. +> +> If you are packaging cChardet for a distribution that forbids bundled +> libraries, consider carrying the overlay +> (`src/cchardet/uchardet-overlay/nsMBCSGroupProber.cpp`) as a patch against +> your system `uchardet` until then. The corpus generator and benchmark above +> are in-tree, so you can verify the result yourself. + ## Supported Languages/Encodings - International (Unicode) From b85755f7d066f0ad6cc4110846555187f72da5f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 11:44:52 +0000 Subject: [PATCH 6/6] chore: bump version to 3.1.0 system-uchardet is a new build-time feature rather than a fix, so the next release is a minor bump instead of 3.0.2. Both version sources move together: meson.build holds the value and src/cchardet/version.py mirrors it for cchardet.__version__. Also adds the 3.1.0 changelog entry describing the feature option, credits the original work, and records the detection-quality caveat for a system build so it is not only in the README. The 3.0.1 heading was still marked unreleased; it shipped on 2026-08-04. This branch was several weeks behind master, so master is merged in first to bring the 3.0.1 release the bump builds on. Verified: builds from source with both version sources reporting 3.1.0, 130 passed 1 skipped, CHANGES.rst parses as reStructuredText. --- CHANGES.rst | 22 +++++++++++++++++++++- meson.build | 2 +- src/cchardet/version.py | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 22eda3a..b58d59e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,27 @@ CHANGES ======= -3.0.1 (unreleased) +3.1.0 (unreleased) +------------------ + +- add a ``system-uchardet`` Meson feature option, so distributions and source + builds can link the system ``uchardet`` instead of the vendored copy + (`#56`_ by `@mgorny`_, `#64`_). ``disabled`` (the default, and what the + published wheels use) always builds the bundled copy, ``enabled`` requires + the system library and fails if it is missing or too old, and ``auto`` + falls back to the bundled copy. The system library must be recent enough to + expose ``uchardet_get_n_candidates``. + + Note that a system build currently detects non-UTF-8 input less accurately + than the bundled one, because the encoding-only multibyte prober added in + 3.0.1 is part of the vendored copy. See the README for measurements and + guidance for packagers. + +.. _#56: https://github.com/faust-streaming/cChardet/pull/56 +.. _#64: https://github.com/faust-streaming/cChardet/pull/64 +.. _@mgorny: https://github.com/mgorny + +3.0.1 (2026-08-04) ------------------ - fix the severe detection slowdown introduced in 3.0.0 (`#57`_). freedesktop diff --git a/meson.build b/meson.build index a65835f..f2b968f 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'faust-cchardet', 'cpp', 'cython', - version: '3.0.1', + version: '3.1.0', meson_version: '>=1.2.0', default_options: [ 'cpp_std=c++11', diff --git a/src/cchardet/version.py b/src/cchardet/version.py index b7a5531..7f5601d 100644 --- a/src/cchardet/version.py +++ b/src/cchardet/version.py @@ -1 +1 @@ -__version__ = '3.0.1' +__version__ = '3.1.0'