fix utf-8 file paths on windows

bindings/python/tests/test_utf8_paths.py

@@ -0,0 +1,87 @@
+"""Non-ASCII (UTF-8) path handling through the C ABI.
+
+Regression coverage for Handy issue #1585: on Windows, model loading
+and backend artifact-dir validation failed for non-ASCII paths because
+the library's narrow ``::stat()`` / ``ifstream`` calls read the UTF-8
+path bytes in the process ANSI code page. src/transcribe-path.h now
+converts UTF-8 to wide paths on Windows.
+
+The C++ counterpart (tests/utf8_path_unit.cpp) pins the same contract,
+but ctest only runs on Linux/macOS in CI — this file is what executes
+on a real Windows runner (the wheel lanes run the pytest suite via
+scripts/ci/wheel_smoke.py).
+
+Model-free tests always run; the full-load test skips without a model
+(``TRANSCRIBE_SMOKE_MODEL``, set by the wheel lanes).
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+import pytest
+
+import transcribe_cpp as t
+from transcribe_cpp import errors
+
+# Same characters as the C++ counterpart's directory name.
+NON_ASCII_DIR = "transcribe-utf8-café-日本語"
+
+
+def _nonascii_dir(tmp_path: Path) -> Path:
+    d = tmp_path / NON_ASCII_DIR
+    try:
+        d.mkdir()
+    except OSError as e:  # filesystem that cannot represent the name
+        pytest.skip(f"cannot create non-ASCII directory: {e}")
+    return d
+
+
+def test_missing_file_in_nonascii_dir_raises_file_not_found(tmp_path):
+    # The not-found classification must survive the wide-path port: a
+    # genuinely absent file is FILE_NOT_FOUND, not some generic error.
+    d = _nonascii_dir(tmp_path)
+    with pytest.raises(t.ModelFileNotFound) as ei:
+        t.Model(d / "definitely-missing.gguf")
+    assert ei.value.status == errors.ERR_FILE_NOT_FOUND
+
+
+def test_junk_file_in_nonascii_dir_is_found_then_rejected(tmp_path):
+    # The #1585 failure shape: the file EXISTS at the non-ASCII path,
+    # but pre-fix Windows raised ModelFileNotFound (the ANSI-code-page
+    # stat could not see it). ModelLoadError is a sibling class, so
+    # this cannot pass via FILE_NOT_FOUND.
+    d = _nonascii_dir(tmp_path)
+    junk = d / "junk.gguf"
+    junk.write_bytes(b"this is not a gguf file" * 64)
+    with pytest.raises(t.ModelLoadError):
+        t.Model(junk)
+
+
+def test_smoke_model_loads_from_nonascii_dir(model_path, tmp_path):
+    # Full successful load through the non-ASCII path: existence
+    # pre-check, magic sniff, gguf parse, and the tensor-data streaming
+    # reopen all consume the path. Constructing the Model is the
+    # assertion.
+    d = _nonascii_dir(tmp_path)
+    local = d / model_path.name
+    shutil.copyfile(model_path, local)
+    with t.Model(local):
+        pass
+
+
+def test_init_backends_nonascii_dirs(tmp_path):
+    # The existing non-ASCII dir must not be misread as absent;
+    # FILE_NOT_FOUND specifically is the #1585 encoding bug. We do not
+    # assert OK: a module-less dir may legitimately report ERR_BACKEND
+    # depending on build posture.
+    lib = t._lib
+    d = _nonascii_dir(tmp_path)
+    st = lib.transcribe_init_backends(os.fspath(d).encode("utf-8"))
+    assert st != errors.ERR_FILE_NOT_FOUND
+
+    missing = d / "missing-subdir"
+    st = lib.transcribe_init_backends(os.fspath(missing).encode("utf-8"))
+    assert st == errors.ERR_FILE_NOT_FOUND

include/transcribe.h

@@ -719,6 +719,9 @@ typedef enum {
  * is automatic under the supported usage (call once, before the first
  * model load).
  *
+ * artifact_dir is UTF-8, like every path crossing this API; on Windows
+ * it is converted to a wide path internally.
+ *
  * Returns:
  *   TRANSCRIBE_ERR_INVALID_ARG     artifact_dir is NULL or empty.
  *   TRANSCRIBE_ERR_FILE_NOT_FOUND  artifact_dir is not an existing directory.
@@ -1335,6 +1338,10 @@ TRANSCRIBE_API const char * transcribe_model_meta_val_str(const struct transcrib
 /*
  * Load a GGUF model from disk.
  *
+ * path is UTF-8, like every path crossing this API; on Windows it is
+ * converted to a wide path internally, so non-ASCII paths load
+ * correctly regardless of the process ANSI code page.
+ *
  * params may be NULL for all library defaults. To customize, initialize
  * a struct with transcribe_model_load_params_init() and set fields. A
  * struct with struct_size == 0 (including `{0}`) is rejected with

src/transcribe-bin-loader.cpp

@@ -5,10 +5,8 @@
 #include "ggml-backend.h"
 #include "ggml.h"
 #include "transcribe-log.h"
+#include "transcribe-path.h"
 
-#include <sys/stat.h>
-
-#include <cerrno>
 #include <cstdio>
 #include <cstring>
 #include <fstream>
@@ -20,21 +18,6 @@ namespace {
 
 constexpr const char * kTag = "whisper-bin";
 
-// Same path-existence check transcribe-loader uses for GGUFs: ENOENT /
-// ENOTDIR are conclusive "file is not at this path"; anything else
-// means we couldn't reach the path and the open below should surface
-// the underlying error.
-bool path_is_present(const char * path) {
-    struct stat st{};
-    if (::stat(path, &st) == 0) {
-        return true;
-    }
-    if (errno == ENOENT || errno == ENOTDIR) {
-        return false;
-    }
-    return true;
-}
-
 // Read sizeof(T) bytes into `out`. Returns false on short read or
 // stream failure.
 template <typename T> bool read_pod(std::ifstream & fin, T & out) {
@@ -158,7 +141,7 @@ transcribe_status parse_whisper_bin(const char * path, WhisperBinModel & out) {
     }
     out.path = path;
 
-    std::ifstream fin(path, std::ios::binary);
+    std::ifstream fin(path_from_utf8(path), std::ios::binary);
     if (!fin) {
         log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to open %s", kTag, path);
         return TRANSCRIBE_ERR_GGUF;
@@ -395,7 +378,7 @@ transcribe_status parse_whisper_bin(const char * path, WhisperBinModel & out) {
 transcribe_status stream_tensor_data_from_bin(const std::string &                path,
                                               const std::vector<BinStreamSlot> & slots,
                                               const char *                       error_tag) {
-    std::ifstream fin(path, std::ios::binary);
+    std::ifstream fin(path_from_utf8(path), std::ios::binary);
     if (!fin) {
         log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to reopen %s for tensor data", error_tag, path.c_str());
         return TRANSCRIBE_ERR_GGUF;

src/transcribe-load-common.cpp

@@ -11,6 +11,7 @@
 #include "gguf.h"
 #include "transcribe-backend.h"
 #include "transcribe-log.h"
+#include "transcribe-path.h"
 
 #include <cstdint>
 #include <cstdio>
@@ -382,7 +383,7 @@ transcribe_status stream_tensor_data(const std::string &  path,
     // time: ifstream::seekg takes a streamoff (signed 64-bit on every
     // platform we target), so multi-GB tensor offsets work without
     // #ifdef'ing fseeko vs _fseeki64.
-    std::ifstream fin(path, std::ios::binary);
+    std::ifstream fin(path_from_utf8(path), std::ios::binary);
     if (!fin) {
         log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to reopen %s for tensor data", error_tag, path.c_str());
         return TRANSCRIBE_ERR_GGUF;
@@ -574,7 +575,7 @@ ReadF32Result read_f32_tensor_checked(gguf_context *       gguf_ctx,
     const size_t data_off = gguf_get_data_offset(gguf_ctx);
     const size_t t_off    = gguf_get_tensor_offset(gguf_ctx, idx);
 
-    std::ifstream fin(gguf_path, std::ios::binary);
+    std::ifstream fin(path_from_utf8(gguf_path), std::ios::binary);
     if (!fin) {
         log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to open %s for tensor \"%s\"", error_tag, gguf_path.c_str(),
                 tensor_name);

src/transcribe-loader.cpp

@@ -9,10 +9,7 @@
 
 #include "gguf.h"
 #include "transcribe-meta.h"
-
-#include <sys/stat.h>
-
-#include <cerrno>
+#include "transcribe-path.h"
 
 namespace transcribe {
 
@@ -29,51 +26,16 @@ gguf_context * Loader::release_gguf() {
     return out;
 }
 
-namespace {
-
-// Narrow stat-based pre-check whose ONLY job is to distinguish
-// "the file is not at this path" from every other reason
-// gguf_init_from_file might return nullptr.
-//
-// We deliberately do not check S_ISREG here. If stat() succeeds and the
-// path points at a directory (or a fifo, or a socket), the file IS at
-// that path — it's just not a usable GGUF. fopen() inside ggml will
-// return EISDIR / etc. and we will surface that as TRANSCRIBE_ERR_GGUF,
-// which is strictly more accurate than collapsing it into FILE_NOT_FOUND.
-//
-// We also deliberately only short-circuit on ENOENT / ENOTDIR. Other
-// stat() failures (EACCES, ENAMETOOLONG, ELOOP, EIO, ...) do NOT
-// conclusively prove the file is missing — they prove we can't reach
-// it. Returning FILE_NOT_FOUND for those would be wrong. We let
-// gguf_init_from_file try and surface whatever it returns as ERR_GGUF.
-//
-// Returns true if the path either exists OR if we cannot prove it
-// doesn't (in which case the caller should hand off to
-// gguf_init_from_file unchanged).
-bool path_is_present(const char * path) {
-    struct stat st{};
-    if (::stat(path, &st) == 0) {
-        return true;
-    }
-    // ENOENT: the named file does not exist.
-    // ENOTDIR: a non-final component of the path is not a directory,
-    //          which is the same observable failure mode as "the file
-    //          is not at this path."
-    if (errno == ENOENT || errno == ENOTDIR) {
-        return false;
-    }
-    return true;
-}
-
-}  // namespace
-
 transcribe_status Loader::open(const char * path) {
     if (path == nullptr) {
         return TRANSCRIBE_ERR_INVALID_ARG;
     }
 
     path_ = path;
 
+    // Distinguishes "the file is not at this path" (-> FILE_NOT_FOUND)
+    // from every other reason gguf_init_from_file might return nullptr
+    // (-> ERR_GGUF). Exact semantics in transcribe-path.h.
     if (!path_is_present(path)) {
         return TRANSCRIBE_ERR_FILE_NOT_FOUND;
     }

src/transcribe-path.h

@@ -0,0 +1,57 @@
+// transcribe-path.h - UTF-8 path handling for user-facing file access.
+//
+// Contract: every path crossing the public C ABI is UTF-8 (see
+// include/transcribe.h). On POSIX the narrow file APIs consume UTF-8
+// bytes directly. On Windows they interpret narrow strings in the
+// process ANSI code page, so a non-ASCII path fails with ENOENT even
+// though the file exists (Handy issue #1585). Every stat/open a
+// user-supplied path can reach must therefore go through these
+// helpers, which convert UTF-8 to the native wide encoding on Windows.
+//
+// Deliberately not converted: dev-only I/O (the transcribe-debug.cpp
+// dump writers, the TRANSCRIBE_ENABLE_VALIDATION_HOOKS sidecar
+// readers) — those paths never cross the public ABI.
+
+#pragma once
+
+#include <filesystem>
+#include <string>
+#include <system_error>
+
+namespace transcribe {
+
+// Build a std::filesystem::path from UTF-8 bytes. On Windows this
+// converts to the native wide encoding; elsewhere the narrow encoding
+// IS UTF-8 and this is a passthrough. (fs::u8path is the C++17
+// spelling; it is deprecated-but-present under C++20.)
+inline std::filesystem::path path_from_utf8(const char * utf8) {
+#if defined(_WIN32)
+    return std::filesystem::u8path(utf8);
+#else
+    return std::filesystem::path(utf8);
+#endif
+}
+
+inline std::filesystem::path path_from_utf8(const std::string & utf8) {
+    return path_from_utf8(utf8.c_str());
+}
+
+// Existence pre-check whose ONLY job is to distinguish "the file is
+// not at this path" from every other reason a subsequent open might
+// fail.
+//
+// Deliberately no regular-file check: if the path names a directory
+// (or a fifo, or a socket), the file IS at that path — the caller's
+// open will surface a more accurate error than FILE_NOT_FOUND.
+//
+// Deliberately only the not-found outcome (ENOENT / ENOTDIR and their
+// Windows equivalents, which fs::status maps to file_type::not_found)
+// reports absence. Other failures (EACCES, ELOOP, EIO, ...) don't
+// prove the file is missing — only that we can't reach it — so they
+// return true and the caller's open reports the real error.
+inline bool path_is_present(const char * utf8_path) {
+    std::error_code ec;
+    return std::filesystem::status(path_from_utf8(utf8_path), ec).type() != std::filesystem::file_type::not_found;
+}
+
+}  // namespace transcribe

src/transcribe.cpp

@@ -24,6 +24,7 @@
 #include "transcribe-loader.h"
 #include "transcribe-log.h"
 #include "transcribe-model.h"
+#include "transcribe-path.h"
 #include "transcribe-session.h"
 #include "transcribe-tokenizer.h"
 #include "transcribe/whisper.h"
@@ -40,17 +41,8 @@
 #    include <dlfcn.h>
 #endif
 
-#include <sys/stat.h>
-
-// MSVC's <sys/stat.h> has the _S_IFDIR mode bits but not the POSIX
-// S_ISDIR() macro.
-#if defined(_WIN32) && !defined(S_ISDIR)
-#    define S_ISDIR(m) (((m) & _S_IFMT) == _S_IFDIR)
-#endif
-
 #include <algorithm>
 #include <atomic>
-#include <cerrno>
 #include <climits>
 #include <cmath>
 #include <cstdarg>
@@ -864,8 +856,8 @@ static transcribe_status transcribe_init_backends_impl(const char * artifact_dir
         return TRANSCRIBE_ERR_INVALID_ARG;
     }
     {
-        struct stat st{};
-        if (::stat(artifact_dir, &st) != 0 || !S_ISDIR(st.st_mode)) {
+        std::error_code ec;
+        if (!std::filesystem::is_directory(transcribe::path_from_utf8(artifact_dir), ec)) {
             transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_init_backends: %s is not an existing directory",
                                 artifact_dir);
             return TRANSCRIBE_ERR_FILE_NOT_FOUND;
@@ -880,8 +872,8 @@ static transcribe_status transcribe_init_backends_impl(const char * artifact_dir
     std::lock_guard<std::mutex>  lock(s_mutex);
 
     std::error_code       ec;
-    std::filesystem::path canonical = std::filesystem::weakly_canonical(artifact_dir, ec);
-    const std::string     key       = ec ? std::string(artifact_dir) : canonical.string();
+    std::filesystem::path canonical = std::filesystem::weakly_canonical(transcribe::path_from_utf8(artifact_dir), ec);
+    const std::string     key       = ec ? std::string(artifact_dir) : canonical.u8string();
     if (s_loaded_dirs.find(key) == s_loaded_dirs.end()) {
         ggml_backend_load_all_from_path(artifact_dir);
         s_loaded_dirs.insert(key);
@@ -1419,22 +1411,15 @@ static transcribe_status transcribe_model_load_file_impl(const char *
     // which HF/users mislabel): GGUF magic 0x46554747 is the canonical path,
     // ggml magic 0x67676d6c is a legacy whisper.cpp .bin. Public contract:
     // missing path -> ERR_FILE_NOT_FOUND; every other failure -> ERR_GGUF.
-    {
-        struct stat st{};
-        if (::stat(path, &st) != 0) {
-            if (errno == ENOENT || errno == ENOTDIR) {
-                return TRANSCRIBE_ERR_FILE_NOT_FOUND;
-            }
-            // Other stat() failures (EACCES, etc.) are not "not found";
-            // the open below will surface them as ERR_GGUF.
-        }
+    if (!transcribe::path_is_present(path)) {
+        return TRANSCRIBE_ERR_FILE_NOT_FOUND;
     }
     uint32_t magic = 0;
     {
-        std::ifstream fin(path, std::ios::binary);
+        std::ifstream fin(transcribe::path_from_utf8(path), std::ios::binary);
         if (!fin) {
-            // stat() said the path is reachable but ifstream couldn't
-            // open it — permissions / type issue. Treat as a generic
+            // The existence pre-check said the path is reachable but
+            // the open failed — permissions / type issue. Treat as a generic
             // load failure rather than FILE_NOT_FOUND.
             return TRANSCRIBE_ERR_GGUF;
         }