diff --git a/.github/workflows/test-system-uchardet.yml b/.github/workflows/test-system-uchardet.yml new file mode 100644 index 0000000..35fdc9d --- /dev/null +++ b/.github/workflows/test-system-uchardet.yml @@ -0,0 +1,75 @@ +name: Test (system uchardet) + +# Exercises the `system-uchardet` feature option (see meson.options): build +# cChardet against a system uchardet built from git and confirm the resulting +# extension links the shared library instead of the bundled sources. The +# regular Tests workflow only covers the bundled default. +on: + pull_request: + push: + branches: + - master + +# Least-privilege GITHUB_TOKEN: this workflow only checks out and builds the +# repository, so read access to its contents is all it needs. +permissions: + contents: read + +jobs: + system-uchardet: + name: Build against system uchardet (from git) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + # meson's files() validates the vendored uchardet sources exist at + # configure time even for a system build, so the submodule still has + # to be checked out. + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Build and install uchardet from git + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends cmake ninja-build pkg-config + git clone --depth 1 \ + https://gitlab.freedesktop.org/uchardet/uchardet.git /tmp/uchardet-src + cmake -S /tmp/uchardet-src -B /tmp/uchardet-build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DBUILD_BINARY=OFF + sudo cmake --build /tmp/uchardet-build --target install + sudo ldconfig + + - name: Confirm the system uchardet exposes the candidate API + run: | + pkg-config --exists uchardet + echo "system uchardet version: $(pkg-config --modversion uchardet)" + # `enabled`/`auto` reject a uchardet without this symbol; the last + # tagged release still reports 0.0.8 but only the git build ships it. + grep -q uchardet_get_n_candidates /usr/include/uchardet/uchardet.h + + - name: Build cChardet against the system uchardet + run: | + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install . \ + --config-settings=setup-args=-Dsystem-uchardet=enabled + + - name: Verify the extension links the system libuchardet + run: | + so=$(python -c "import cchardet._cchardet as m; print(m.__file__)") + echo "built extension: $so" + ldd "$so" + # Fail if it did not link the shared system library -- i.e. it + # silently fell back to compiling the bundled sources. + ldd "$so" | grep -q libuchardet + + - name: Run tests + run: python -m pytest src/tests diff --git a/CHANGES.rst b/CHANGES.rst index 22eda3a..b58d59e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,27 @@ CHANGES ======= -3.0.1 (unreleased) +3.1.0 (unreleased) +------------------ + +- add a ``system-uchardet`` Meson feature option, so distributions and source + builds can link the system ``uchardet`` instead of the vendored copy + (`#56`_ by `@mgorny`_, `#64`_). ``disabled`` (the default, and what the + published wheels use) always builds the bundled copy, ``enabled`` requires + the system library and fails if it is missing or too old, and ``auto`` + falls back to the bundled copy. The system library must be recent enough to + expose ``uchardet_get_n_candidates``. + + Note that a system build currently detects non-UTF-8 input less accurately + than the bundled one, because the encoding-only multibyte prober added in + 3.0.1 is part of the vendored copy. See the README for measurements and + guidance for packagers. + +.. _#56: https://github.com/faust-streaming/cChardet/pull/56 +.. _#64: https://github.com/faust-streaming/cChardet/pull/64 +.. _@mgorny: https://github.com/mgorny + +3.0.1 (2026-08-04) ------------------ - fix the severe detection slowdown introduced in 3.0.0 (`#57`_). freedesktop diff --git a/README.md b/README.md index 75134bc..b276a14 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,61 @@ import cchardet # same import name as upstream - On 32-bit (`i686`) builds a few near-equivalent labels differ — e.g. Thai `TIS-620` is detected as `ISO-8859-11`. +## Building against a system `uchardet` + +By default cChardet builds a **bundled** copy of `uchardet` (vendored as a git +submodule), which is what the published wheels ship. Distributions and source +builds can instead link the **system** `uchardet` through the `system-uchardet` +Meson [feature option](https://mesonbuild.com/Build-options.html#features): + +| `-Dsystem-uchardet=` | Behaviour | +| --- | --- | +| `disabled` (default) | Always build the bundled copy (used by the wheels). | +| `enabled` | Require the system library; the build **fails** if it is missing or too old. Intended for distro packaging — read the detection-quality caveat below first. | +| `auto` | Use the system library if it is new enough, otherwise fall back to the bundled copy. | + +Pass the option through your build front-end. With `pip` / `build` the +meson-python backend reads it from `setup-args`: + +```bash +pip install . --config-settings=setup-args=-Dsystem-uchardet=enabled +``` + +> **Requirement:** the system `uchardet` must be recent enough to expose +> `uchardet_get_n_candidates`. That symbol currently ships only in the git +> version () — no tagged +> release provides it yet — so `enabled`/`auto` probe for it and reject an +> older library. + +> [!WARNING] +> **A system build currently detects non-UTF-8 input less accurately than the +> bundled build.** The bundled copy compiles a cChardet-specific replacement +> for uchardet's multi-byte group prober, which both restores detection +> throughput and rejects non-UTF-8 byte sequences that upstream reports as +> UTF-8. A system build links upstream's prober, so neither applies. +> +> Measured over 1650 non-UTF-8 documents spanning 7 encodings +> (`benchmarks/make_nonutf8_corpus.py`, median of 3 runs): +> +> | Build | Throughput | Non-UTF-8 reported as UTF-8 | +> | --- | ---: | ---: | +> | bundled (wheel default) | 3.10 MB/s | 0.0% | +> | system `uchardet` | 1.44 MB/s | **16.2%** | +> +> Reporting non-UTF-8 bytes as UTF-8 is the defect that caused v3.0.0 to be +> yanked from PyPI, so this is not a cosmetic difference. It is an upstream +> issue rather than a packaging mistake: `nsUTF8Prober` does not reject +> invalid sequences on its own, and its confidence never falls low enough for +> the candidate to be discarded. Patches have been sent upstream; once they +> land, the build can require a `uchardet` version that includes them and this +> caveat goes away. +> +> If you are packaging cChardet for a distribution that forbids bundled +> libraries, consider carrying the overlay +> (`src/cchardet/uchardet-overlay/nsMBCSGroupProber.cpp`) as a patch against +> your system `uchardet` until then. The corpus generator and benchmark above +> are in-tree, so you can verify the result yourself. + ## Supported Languages/Encodings - International (Unicode) diff --git a/meson.build b/meson.build index a65835f..f2b968f 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'faust-cchardet', 'cpp', 'cython', - version: '3.0.1', + version: '3.1.0', meson_version: '>=1.2.0', default_options: [ 'cpp_std=c++11', diff --git a/meson.options b/meson.options new file mode 100644 index 0000000..e89386c --- /dev/null +++ b/meson.options @@ -0,0 +1,2 @@ +option('system-uchardet', type: 'feature', value: 'disabled', + description: 'Build against the system uchardet library instead of the bundled copy. enabled: require the system library (hard fail if missing or too old); disabled: always use the bundled copy (the wheel default); auto: use the system library if it is new enough, else fall back to the bundled copy. Requires uchardet from git until upstream tags a release exposing uchardet_get_n_candidates.') diff --git a/src/cchardet/meson.build b/src/cchardet/meson.build index d3006cc..c5bb494 100644 --- a/src/cchardet/meson.build +++ b/src/cchardet/meson.build @@ -66,15 +66,57 @@ uchardet_sources = files( uchardet_inc = include_directories('../ext/uchardet/src') +# Select the uchardet implementation to build against. `system-uchardet` is a +# feature option: +# enabled -> require the system library; hard fail if it is missing or too +# old. This is what distro packaging wants. +# disabled -> always build the bundled copy (the default, used by the wheels). +# auto -> use the system library if it is new enough, otherwise fall back +# to the bundled copy (convenient for source installs). +system_uchardet = get_option('system-uchardet') + +use_bundled = true +if not system_uchardet.disabled() + # Passing the feature as `required:` makes meson hard fail for `enabled` when + # no uchardet is installed, while `auto` continues quietly if it is absent. + system_uchardet_dep = dependency('uchardet', required: system_uchardet) + if system_uchardet_dep.found() + # The candidate API (uchardet_get_n_candidates) only exists in uchardet + # newer than the last tagged release, so a found-but-too-old system library + # must still be rejected. + # TODO: replace this has_function() probe with a `version: '>=X'` constraint + # on the dependency() call above once upstream tags a release exposing it. + cpp = meson.get_compiler('cpp') + if cpp.has_function('uchardet_get_n_candidates', dependencies: [system_uchardet_dep]) + uchardet = system_uchardet_dep + use_bundled = false + elif system_uchardet.enabled() + error( + 'system-uchardet is enabled but the installed uchardet is too old: ' + + 'it does not provide uchardet_get_n_candidates. Install the git ' + + 'version from https://gitlab.freedesktop.org/uchardet/uchardet.git, ' + + 'or set -Dsystem-uchardet=disabled to build the bundled copy.' + ) + endif + # auto + a too-old system library: fall through to the bundled copy below. + endif +endif + +if use_bundled + uchardet = declare_dependency( + sources: uchardet_sources, + include_directories: uchardet_inc, + ) +endif + py.extension_module( '_cchardet', '_cchardet.pyx', - uchardet_sources, + dependencies: [uchardet], # Build the Cython output as C++ (the module wraps a C++ library). Meson then # links the C++ runtime that matches the active compiler (libstdc++ or # libc++), so no manual `-lstdc++` is needed. override_options: ['cython_language=cpp'], - include_directories: uchardet_inc, install: true, subdir: 'cchardet', ) diff --git a/src/cchardet/version.py b/src/cchardet/version.py index b7a5531..7f5601d 100644 --- a/src/cchardet/version.py +++ b/src/cchardet/version.py @@ -1 +1 @@ -__version__ = '3.0.1' +__version__ = '3.1.0'