diff --git a/docs/_build/html/.buildinfo b/docs/_build/html/.buildinfo index d7142fb..943f73f 100644 --- a/docs/_build/html/.buildinfo +++ b/docs/_build/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file records the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 3caef0746bc07fabd8f91030ce7b6533 +config: 949fad3acb2ae8760cf522b5d9f55c0f tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_build/html/.doctrees/docs/evaluation.doctree b/docs/_build/html/.doctrees/docs/evaluation.doctree index 3dff8be..9f1f19c 100644 Binary files a/docs/_build/html/.doctrees/docs/evaluation.doctree and b/docs/_build/html/.doctrees/docs/evaluation.doctree differ diff --git a/docs/_build/html/.doctrees/docs/inference.doctree b/docs/_build/html/.doctrees/docs/inference.doctree index efab5a5..3125083 100644 Binary files a/docs/_build/html/.doctrees/docs/inference.doctree and b/docs/_build/html/.doctrees/docs/inference.doctree differ diff --git a/docs/_build/html/.doctrees/docs/usage.doctree b/docs/_build/html/.doctrees/docs/usage.doctree index 1e56e2c..e09d97d 100644 Binary files a/docs/_build/html/.doctrees/docs/usage.doctree and b/docs/_build/html/.doctrees/docs/usage.doctree differ diff --git a/docs/_build/html/_modules/index.html b/docs/_build/html/_modules/index.html index 50938ce..42e6c66 100644 --- a/docs/_build/html/_modules/index.html +++ b/docs/_build/html/_modules/index.html @@ -4,20 +4,20 @@ - Overview: module code — LLMSQL 0.1.13 documentation + Overview: module code — LLMSQL 0.1.16 documentation - + - - + + - + - + +
- +

All modules for which code is available

@@ -73,11 +72,11 @@

Navigation

  • modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Overview: module code
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Overview: module code
    • - + \ No newline at end of file diff --git a/docs/_build/html/_modules/llmsql/evaluation/evaluate.html b/docs/_build/html/_modules/llmsql/evaluation/evaluate.html index 277b177..c63fc3f 100644 --- a/docs/_build/html/_modules/llmsql/evaluation/evaluate.html +++ b/docs/_build/html/_modules/llmsql/evaluation/evaluate.html @@ -4,20 +4,20 @@ - llmsql.evaluation.evaluate — LLMSQL 0.1.13 documentation + llmsql.evaluation.evaluate — LLMSQL 0.1.16 documentation - + - - + + - + - + +
    - +

    Source code for llmsql.evaluation.evaluate

     """
 LLMSQL Evaluation Module
@@ -51,17 +51,19 @@ 
    Source code for llmsql.evaluation.evaluate
     """
 
 from datetime import datetime, timezone
-from pathlib import Path
 import uuid
 
 from rich.progress import track
 
-from llmsql.config.config import DEFAULT_WORKDIR_PATH
+from llmsql.config.config import (
+    DEFAULT_LLMSQL_VERSION,
+    get_repo_id,
+)
 from llmsql.utils.evaluation_utils import (
     connect_sqlite,
-    download_benchmark_file,
     evaluate_sample,
 )
+from llmsql.utils.inference_utils import _maybe_download, resolve_workdir_path
 from llmsql.utils.rich_utils import log_mismatch, print_summary
 from llmsql.utils.utils import load_jsonl, load_jsonl_dict_by_key, save_json_report
 
@@ -69,11 +71,10 @@ 
    Source code for llmsql.evaluation.evaluate
     
    
 [docs]
 def evaluate(
-    outputs,
+    outputs: str | list[dict[int, str | int]],
     *,
-    workdir_path: str | None = DEFAULT_WORKDIR_PATH,
-    questions_path: str | None = None,
-    db_path: str | None = None,
+    version: str = DEFAULT_LLMSQL_VERSION,
+    workdir_path: str | None = None,
     save_report: str | None = None,
     show_mismatches: bool = True,
     max_mismatches: int = 5,
@@ -82,10 +83,10 @@ 
    Source code for llmsql.evaluation.evaluate
         Evaluate predicted SQL queries against the LLMSQL benchmark.
 
     Args:
+        version: LLMSQL version
         outputs: Either a JSONL file path or a list of dicts.
-        workdir_path: Directory for auto-downloads (ignored if all paths provided).
-        questions_path: Manual path to benchmark questions JSONL.
-        db_path: Manual path to SQLite benchmark DB.
+        workdir_path: Directory to store downloaded benchmark files. If omitted, a
+            temporary directory is created automatically.
         save_report: Optional manual save path. If None → auto-generated.
         show_mismatches: Print mismatches while evaluating.
         max_mismatches: Max mismatches to print.
@@ -96,37 +97,12 @@ 
    Source code for llmsql.evaluation.evaluate
     
     # Determine input type
     input_mode = "jsonl_path" if isinstance(outputs, str) else "dict_list"
+    workdir = resolve_workdir_path(workdir_path)
 
-    # --- Resolve inputs if needed ---
-    workdir = Path(workdir_path) if workdir_path else None
-    if workdir_path and (questions_path is None or db_path is None):
-        workdir.mkdir(parents=True, exist_ok=True)
-
-    if questions_path is None:
-        if workdir is None:
-            raise ValueError(
-                "questions_path not provided, and workdir_path disabled. "
-                "Enable workdir or provide questions_path explicitly."
-            )
-        local_q = workdir / "questions.jsonl"
-        questions_path = (
-            str(local_q)
-            if local_q.is_file()
-            else download_benchmark_file("questions.jsonl", workdir)
-        )
+    repo_id = get_repo_id(version)
 
-    if db_path is None:
-        if workdir is None:
-            raise ValueError(
-                "db_path not provided, and workdir_path disabled. "
-                "Enable workdir or provide db_path explicitly."
-            )
-        local_db = workdir / "sqlite_tables.db"
-        db_path = (
-            str(local_db)
-            if local_db.is_file()
-            else download_benchmark_file("sqlite_tables.db", workdir)
-        )
+    questions_path = _maybe_download(repo_id, "questions.jsonl", workdir)
+    db_path = _maybe_download(repo_id, "sqlite_tables.db", workdir)
 
     # --- Load benchmark questions ---
     questions = load_jsonl_dict_by_key(questions_path, key="question_id")
@@ -226,12 +202,12 @@ 
    Navigation
    
         
  • 
           modules |
    • 
-        
  • LLMSQL 0.1.13 documentation »
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
           
  • Module code »
    • 
-        
  • llmsql.evaluation.evaluate
    • 
+        
  • llmsql.evaluation.evaluate
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/_modules/llmsql/inference/inference_transformers.html b/docs/_build/html/_modules/llmsql/inference/inference_transformers.html
index ec1fdd8..47db67d 100644
--- a/docs/_build/html/_modules/llmsql/inference/inference_transformers.html
+++ b/docs/_build/html/_modules/llmsql/inference/inference_transformers.html
@@ -4,20 +4,20 @@
   
     
     
-    llmsql.inference.inference_transformers — LLMSQL 0.1.13 documentation
+    llmsql.inference.inference_transformers — LLMSQL 0.1.16 documentation
     
-    
+    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
-    
+     
   
     
+    
      
 
     
    
       
    
         
    
           
    
-
+            
   
    Source code for llmsql.inference.inference_transformers
     """
 LLMSQL Transformers Inference Function
@@ -56,17 +56,16 @@ 
    Source code for llmsql.inference.inference_transformers
        results = inference_transformers(
         model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
+        repo_id="llmsql-bench/llmsql-2.0",
         output_file="outputs/preds_transformers.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
         temperature=0.7,
-        model_args={
+        model_kwargs={
             "torch_dtype": "bfloat16",
         },
-        generate_kwargs={
+        generation_kwargs={
             "do_sample": False,
         },
     )
@@ -80,17 +79,23 @@ 
    Source code for llmsql.inference.inference_transformers
    """
 
-from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 
 from dotenv import load_dotenv
 import torch
 from tqdm import tqdm
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
-from llmsql.config.config import DEFAULT_WORKDIR_PATH
+from llmsql.config.config import (
+    DEFAULT_LLMSQL_VERSION,
+    get_repo_id,
+)
 from llmsql.loggers.logging_config import log
-from llmsql.utils.inference_utils import _maybe_download, _setup_seed
+from llmsql.utils.inference_utils import (
+    _maybe_download,
+    _setup_seed,
+    resolve_workdir_path,
+)
 from llmsql.utils.utils import (
     choose_prompt_builder,
     load_jsonl,
@@ -129,12 +134,12 @@ 
    Source code for llmsql.inference.inference_transformers
    top_k: int = 50,
     generation_kwargs: dict[str, Any] | None = None,
     # --- Benchmark Parameters ---
+    version: Literal["1.0", "2.0"] = DEFAULT_LLMSQL_VERSION,
     output_file: str = "llm_sql_predictions.jsonl",
-    questions_path: str | None = None,
-    tables_path: str | None = None,
-    workdir_path: str = DEFAULT_WORKDIR_PATH,
+    workdir_path: str | None = None,
     num_fewshots: int = 5,
     batch_size: int = 8,
+    limit: int | float | None = None,
     seed: int = 42,
 ) -> list[dict[str, str]]:
     """
@@ -172,13 +177,16 @@ 
    Source code for llmsql.inference.inference_transformers
                              'top_p', 'top_k' are handled separately.
 
         # Benchmark:
+        version: LLMSQL version
         output_file: Output JSONL file path for completions.
-        questions_path: Path to benchmark questions JSONL.
-        tables_path: Path to benchmark tables JSONL.
-        workdir_path: Working directory path.
+        workdir_path: Directory to store downloaded benchmark files. If omitted, a
+            temporary directory is created automatically.
         num_fewshots: Number of few-shot examples (0, 1, or 5).
         batch_size: Batch size for inference.
         seed: Random seed for reproducibility.
+        limit: Limit the number of questions to evaluate. If an integer, evaluates
+               the first N samples. If a float between 0.0 and 1.0, evaluates the
+               first X*100% of samples. If None, evaluates all samples (default).
 
     Returns:
         List of generated SQL results with metadata.
@@ -186,9 +194,6 @@ 
    Source code for llmsql.inference.inference_transformers
    # --- Setup ---
     _setup_seed(seed=seed)
 
-    workdir = Path(workdir_path)
-    workdir.mkdir(parents=True, exist_ok=True)
-
     model_kwargs = model_kwargs or {}
     tokenizer_kwargs = tokenizer_kwargs or {}
     generation_kwargs = generation_kwargs or {}
@@ -252,13 +257,33 @@ 
    Source code for llmsql.inference.inference_transformers
    model.eval()
 
     # --- Load necessary files ---
-    questions_path = _maybe_download("questions.jsonl", questions_path)
-    tables_path = _maybe_download("tables.jsonl", tables_path)
+    workdir = resolve_workdir_path(workdir_path)
+    repo_id = get_repo_id(version)
+
+    questions_path = _maybe_download(repo_id, "questions.jsonl", workdir)
+    tables_path = _maybe_download(repo_id, "tables.jsonl", workdir)
 
     questions = load_jsonl(questions_path)
     tables_list = load_jsonl(tables_path)
     tables = {t["table_id"]: t for t in tables_list}
 
+    # --- Apply limit ---
+    if limit is not None:
+        if isinstance(limit, float):
+            if not (0.0 < limit <= 1.0):
+                raise ValueError(
+                    f"When a float, `limit` must be between 0.0 and 1.0, got {limit}."
+                )
+            limit = max(1, int(len(questions) * limit))
+        if not isinstance(limit, int) or limit < 1:
+            raise ValueError(
+                f"`limit` must be a positive integer or a float in (0.0, 1.0], got {limit!r}."
+            )
+        log.info(
+            f"Limiting evaluation to first {limit} questions out of {len(questions)}"
+        )
+        questions = questions[:limit]
+
     # --- Chat template setup ---
     use_chat_template = chat_template or getattr(tokenizer, "chat_template", None)
     if use_chat_template:
@@ -362,12 +387,12 @@ 
    Navigation
    
         
  • 
           modules |
    • 
-        
  • LLMSQL 0.1.13 documentation »
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
           
  • Module code »
    • 
-        
  • llmsql.inference.inference_transformers
    • 
+        
  • llmsql.inference.inference_transformers
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/_sources/docs/evaluation.rst.txt b/docs/_build/html/_sources/docs/evaluation.rst.txt
index 8b98b15..fa60e38 100644
--- a/docs/_build/html/_sources/docs/evaluation.rst.txt
+++ b/docs/_build/html/_sources/docs/evaluation.rst.txt
@@ -36,15 +36,13 @@ Evaluate from a list of Python dicts:
     report = evaluate(predictions)
     print(report)
 
-Providing your own DB and questions (skip workdir):
+Using a persistent cache directory for benchmark downloads:
 
 .. code-block:: python
 
     report = evaluate(
         "path_to_outputs.jsonl",
-        questions_path="bench/questions.jsonl",
-        db_path="bench/sqlite_tables.db",
-        workdir_path=None
+        workdir_path="./benchmark-cache",
     )
 
 Function Arguments
@@ -59,11 +57,7 @@ Function Arguments
    * - outputs
      - Path to JSONL file or a list of prediction dicts (required).
    * - workdir_path
-     - Directory for automatic benchmark downloads. Ignored if both questions_path and db_path are provided. Default: "llmsql_workdir".
-   * - questions_path
-     - Optional path to benchmark questions JSONL file.
-   * - db_path
-     - Optional path to SQLite DB with evaluation tables.
+     - Directory used to cache downloaded benchmark files. If omitted, a temporary directory is created automatically.
    * - save_report
      - Path to save detailed JSON report. Defaults to "evaluation_results_{uuid}.json".
    * - show_mismatches
diff --git a/docs/_build/html/_sources/docs/index.rst.txt b/docs/_build/html/_sources/docs/index.rst.txt
index b2760cd..fe3654c 100644
--- a/docs/_build/html/_sources/docs/index.rst.txt
+++ b/docs/_build/html/_sources/docs/index.rst.txt
@@ -36,8 +36,6 @@ Example: Running your first evaluation (with transformers backend)
     results = inference_transformers(
         model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
         output_file="outputs/preds_transformers.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
diff --git a/docs/_build/html/_sources/docs/inference.rst.txt b/docs/_build/html/_sources/docs/inference.rst.txt
index 5bcf0c6..1a33533 100644
--- a/docs/_build/html/_sources/docs/inference.rst.txt
+++ b/docs/_build/html/_sources/docs/inference.rst.txt
@@ -14,6 +14,12 @@ Inference API Reference
 
 ---
 
+.. automodule:: llmsql.inference.inference_api
+   :members:
+   :undoc-members:
+
+---
+
 .. raw:: html
 
    
    
diff --git a/docs/_build/html/_sources/docs/usage.rst.txt b/docs/_build/html/_sources/docs/usage.rst.txt
index 806a4e8..767fad6 100644
--- a/docs/_build/html/_sources/docs/usage.rst.txt
+++ b/docs/_build/html/_sources/docs/usage.rst.txt
@@ -27,8 +27,7 @@ Using transformers backend.
     results = inference_transformers(
         model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
         output_file="outputs/preds_transformers.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
+        workdir_path="./benchmark-cache",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
@@ -58,8 +57,7 @@ Using vllm backend.
     results = inference_vllm(
         model_name="Qwen/Qwen2.5-1.5B-Instruct",
         output_file="outputs/preds_vllm.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
+        workdir_path="./benchmark-cache",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
@@ -77,6 +75,41 @@ Using vllm backend.
     print(report)
 
 
+Using OpenAI-compateble API.
+
+.. code-block:: python
+
+    from llmsql import inference_api
+    from dotenv import load_dotenv
+    import os
+    load_dotenv()
+
+    # Run inference (will take some time)
+    results = inference_api(
+        model_name="gpt-5-mini",
+        base_url="https://api.openai.com/v1/",
+        api_key=os.environ["OPENAI_API_KEY"],
+        api_kwargs={
+            "response_format": {
+                    "type": "text"
+                },
+                "verbosity": "medium",
+                "reasoning_effort": "medium",
+                "store": False
+        },
+        requests_per_minute=100,
+        output_file="test_output_api.jsonl",
+        limit=50,
+        num_fewshots = 5,
+        seed=42,
+        version="2.0"
+    )
+
+    # Evaluate the results
+    evaluator = LLMSQLEvaluator()
+    report = evaluator.evaluate(outputs_path="outputs/preds_transformers.jsonl")
+    print(report)
+
 ---
 
 .. raw:: html
diff --git a/docs/_build/html/_static/documentation_options.js b/docs/_build/html/_static/documentation_options.js
index eede5b1..5525413 100644
--- a/docs/_build/html/_static/documentation_options.js
+++ b/docs/_build/html/_static/documentation_options.js
@@ -1,5 +1,5 @@
 const DOCUMENTATION_OPTIONS = {
-    VERSION: '0.1.15',
+    VERSION: '0.1.16',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',
@@ -10,4 +10,4 @@ const DOCUMENTATION_OPTIONS = {
     NAVIGATION_WITH_KEYS: false,
     SHOW_SEARCH_SUMMARY: true,
     ENABLE_SEARCH_SHORTCUTS: true,
-};
+};
\ No newline at end of file
diff --git a/docs/_build/html/_static/scripts/front_page.js b/docs/_build/html/_static/scripts/front_page.js
index 1e0c0dc..4f2cdc5 100644
--- a/docs/_build/html/_static/scripts/front_page.js
+++ b/docs/_build/html/_static/scripts/front_page.js
@@ -87,10 +87,8 @@ function renderLeaderboard(rows) {
   rows.forEach((row, i) => {
     const tr = document.createElement('tr');
 
-    // Берём только вторую часть после слеша
     const modelName = row.model.includes('/') ? row.model.split('/')[1] : row.model;
 
-    // Модель с ссылкой
     const modelCell = document.createElement('td');
     if (row.url) {
       const a = document.createElement('a');
@@ -116,7 +114,7 @@ function renderLeaderboard(rows) {
     barContainer.appendChild(text);
     accuracyCell.appendChild(barContainer);
 
-    // Вставка остальных ячеек
+
     tr.innerHTML += `${i+1}`;
     tr.appendChild(modelCell);
     tr.innerHTML += `
diff --git a/docs/_build/html/_static/styles/front_page.css b/docs/_build/html/_static/styles/front_page.css
index 1d3bcfb..45e3f4b 100644
--- a/docs/_build/html/_static/styles/front_page.css
+++ b/docs/_build/html/_static/styles/front_page.css
@@ -1,26 +1,29 @@
 /* === LLMSQL Front Page CSS === */
 
+/* Three-column page layout: nav | content | TOC */
+.page-layout {
+  display: grid;
+  grid-template-columns: 180px minmax(0, 1fr) 200px;
+  gap: 32px;
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 28px 24px;
+  align-items: start;
+}
+
 .sidebar {
-  position: fixed;
-  top: 16px;
-  left: 16px;
-  height: auto;
-  width: 160px;
-  background-color: #f4f4f4;
-  border: 1px solid #e0e0e0;
-  border-radius: 8px;
-  padding: 12px;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  z-index: 110;
-  box-shadow: 0 2px 6px rgba(0,0,0,0.04);
+  position: sticky;
+  top: 24px;
+  background-color: #f8f9fa;
+  border: 1px solid #e8e8e8;
+  border-radius: 10px;
+  padding: 14px;
+  box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04);
 }
 
 .sidebar-content {
   display: flex;
   flex-direction: column;
-  align-items: center;
   gap: 10px;
   width: 100%;
 }
@@ -30,10 +33,11 @@
   text-align: center;
   background-color: #eef6ff;
   color: #0056b3;
-  padding: 10px;
+  padding: 10px 12px;
   border-radius: 6px;
   text-decoration: none;
   font-weight: 600;
+  font-size: 0.9rem;
   transition: background-color 0.2s, color 0.2s;
 }
 .sidebar-button:hover {
@@ -41,12 +45,19 @@
   color: white;
 }
 
+/* Standalone back-link on docs pages (no .page-layout wrapper) */
+body > .sidebar-button,
+.document .sidebar-button {
+  display: inline-block;
+  margin-bottom: 1.5rem;
+}
+
 .sidebar-search {
   width: 100%;
   padding: 8px 10px;
   border-radius: 6px;
   border: 1px solid #ccc;
-  font-size: 0.9rem;
+  font-size: 0.85rem;
   box-sizing: border-box;
   transition: border-color 0.2s ease, box-shadow 0.2s ease;
 }
@@ -64,41 +75,60 @@
 }
 
 .on-this-page {
-  position: fixed;
-  top: 16px;
-  right: 16px;
-  width: 220px;
-  background: #fafafa;
-  border: 1px solid #eee;
-  border-radius: 8px;
-  padding: 12px;
-  font-size: 0.95rem;
-  z-index: 105;
-  box-shadow: 0 2px 6px rgba(0,0,0,0.04);
+  position: sticky;
+  top: 24px;
+  background: #f8f9fa;
+  border: 1px solid #e8e8e8;
+  border-radius: 10px;
+  padding: 14px 16px;
+  font-size: 0.88rem;
+  box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04);
 }
 .on-this-page h4 {
-  margin: 0 0 8px 0;
-  padding-bottom: 6px;
-  border-bottom: 1px solid #e9e9e9;
-  font-size: 0.95rem;
+  margin: 0 0 10px 0;
+  padding-bottom: 8px;
+  border-bottom: 1px solid #e0e0e0;
+  font-size: 0.8rem;
+  font-weight: 700;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  color: #555;
 }
 .on-this-page ul {
   list-style: none;
   padding: 0;
-  margin: 8px 0 0 0;
+  margin: 0;
 }
 .on-this-page ul li {
   margin-bottom: 6px;
 }
+.on-this-page ul li a {
+  color: #444;
+  text-decoration: none;
+  line-height: 1.4;
+}
+.on-this-page ul li a:hover {
+  color: #007bff;
+}
 
 body {
   font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
   line-height: 1.6;
-  margin: 0 auto;
-  padding: 28px 36px;
+  margin: 0;
+  padding: 0;
   color: #333;
   background-color: #fff;
-  max-width: 1100px;
+}
+
+.page-layout > main {
+  min-width: 0;
+}
+
+/* Sphinx docs pages (no .page-layout wrapper) */
+body > .document {
+  max-width: 900px;
+  margin: 0 auto;
+  padding: 28px 24px;
 }
 
 .center-content {
@@ -227,12 +257,24 @@ div.highlight span {
 }
 
 /* Responsive */
-@media (max-width: 1000px) {
-  .on-this-page { position: static; width: auto; margin: 12px 0 18px; }
-}
-
-@media (max-width: 720px) {
-  .sidebar, .on-this-page { display: none; }
+@media (max-width: 1100px) {
+  .page-layout {
+    grid-template-columns: 160px minmax(0, 1fr);
+  }
+  .on-this-page {
+    display: none;
+  }
+}
+
+@media (max-width: 768px) {
+  .page-layout {
+    grid-template-columns: 1fr;
+    gap: 16px;
+    padding: 16px;
+  }
+  .sidebar {
+    position: static;
+  }
   h1 { font-size: 1.5rem; }
   .badges img { height: 18px; }
 }
diff --git a/docs/_build/html/docs/evaluation.html b/docs/_build/html/docs/evaluation.html
index 22d13e4..47ebb12 100644
--- a/docs/_build/html/docs/evaluation.html
+++ b/docs/_build/html/docs/evaluation.html
@@ -5,23 +5,21 @@
     
     
 
-    Evaluation API Reference — LLMSQL 0.1.15 documentation
+    Evaluation API Reference — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
-
-
+            
   
    
 
    Evaluation API Reference
    
 
    The evaluate() function allows you to benchmark Text-to-SQL model outputs
@@ -79,12 +77,10 @@ 
    Usage Examplesprint(report)
 
    
 
    
-
    Providing your own DB and questions (skip workdir):
    
+
    Using a persistent cache directory for benchmark downloads:
    
 
    report = evaluate(
     "path_to_outputs.jsonl",
-    questions_path="bench/questions.jsonl",
-    db_path="bench/sqlite_tables.db",
-    workdir_path=None
+    workdir_path="./benchmark-cache",
 )
 
    
 
    
@@ -106,13 +102,7 @@ 
    Function Arguments
    workdir_path
    
-
    Directory for automatic benchmark downloads. Ignored if both questions_path and db_path are provided. Default: “llmsql_workdir”.
    
-
-
    questions_path
    
-
    Optional path to benchmark questions JSONL file.
    
-
-
    db_path
    
-
    Optional path to SQLite DB with evaluation tables.
    
+
    Directory used to cache downloaded benchmark files. If omitted, a temporary directory is created automatically.
    
 
 
    save_report
    
 
    Path to save detailed JSON report. Defaults to “evaluation_results_{uuid}.json”.
    
@@ -155,6 +145,37 @@ 
    Report Saving
+
    LLMSQL Evaluation Module
    
+
    Provides the evaluate() function to benchmark Text-to-SQL model outputs
+on the LLMSQL benchmark.
    
+
    See the documentation for full usage details.
    
+
+
    
+
    
+llmsql.evaluation.evaluate.evaluate(outputs: str | list[dict[int, str | int]], *, version: str = '2.0', workdir_path: str | None = None, save_report: str | None = None, show_mismatches: bool = True, max_mismatches: int = 5) → dict[source]
    
+
    Evaluate predicted SQL queries against the LLMSQL benchmark.
    
+
    
+
    Parameters:
    
+
      
+
    • version – LLMSQL version
      • 
+
    • outputs – Either a JSONL file path or a list of dicts.
      • 
+
    • workdir_path – Directory to store downloaded benchmark files. If omitted, a
+temporary directory is created automatically.
      • 
+
    • save_report – Optional manual save path. If None → auto-generated.
      • 
+
    • show_mismatches – Print mismatches while evaluating.
      • 
+
    • max_mismatches – Max mismatches to print.
      • 
+
    
+
    
+
    Returns:
    
+
    Metrics and mismatches.
    
+
    
+
    Return type:
    
+
    dict
    
+
    
+
    
+
    
+
 
    —
    
 
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/docs/index.html b/docs/_build/html/docs/index.html
index 9c26bf9..f4406e8 100644
--- a/docs/_build/html/docs/index.html
+++ b/docs/_build/html/docs/index.html
@@ -5,24 +5,22 @@
     
     
 
-    LLMSQL package Documentation — LLMSQL 0.1.15 documentation
+    LLMSQL package Documentation — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
-
-
+            
   
    
 
    LLMSQL package Documentation
    
 
@@ -73,8 +72,6 @@ 
    Example: Running your first evaluation (with transformers backend)results = inference_transformers(
     model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
     output_file="outputs/preds_transformers.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
     num_fewshots=5,
     batch_size=8,
     max_new_tokens=256,
@@ -163,17 +160,20 @@ 
    Navigation
    
         
  • 
           index
    • 
+        
  • 
+          modules |
    • 
         
  • 
           next |
    • 
         
  • 
           previous |
    • 
-        
  • LLMSQL 0.1.15 documentation »
    • 
-        
  • LLMSQL package Documentation
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
+        
  • LLMSQL package Documentation
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/docs/inference.html b/docs/_build/html/docs/inference.html
index 2aa340f..ee04295 100644
--- a/docs/_build/html/docs/inference.html
+++ b/docs/_build/html/docs/inference.html
@@ -5,24 +5,22 @@
     
     
 
-    Inference API Reference — LLMSQL 0.1.15 documentation
+    Inference API Reference — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
+            
+  
    
+
    Inference API Reference
    
+
    
+
    LLMSQL Transformers Inference Function
    
+
    This module provides a single function inference_transformers() that performs
+text-to-SQL generation using large language models via the Transformers backend.
    
+
    Example
    
+
    from llmsql.inference import inference_transformers
 
-  
    
-
    Inference API Reference
    
+results = inference_transformers(
+    model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
+    repo_id="llmsql-bench/llmsql-2.0",
+    output_file="outputs/preds_transformers.jsonl",
+    num_fewshots=5,
+    batch_size=8,
+    max_new_tokens=256,
+    temperature=0.7,
+    model_kwargs={
+        "torch_dtype": "bfloat16",
+    },
+    generation_kwargs={
+        "do_sample": False,
+    },
+)
+
    
+
    
+
    Notes
    
+
    This function uses the HuggingFace Transformers backend and may produce
+slightly different outputs than the vLLM backend even with the same inputs
+due to differences in implementation and numerical precision.
    
+
    
+
    
+
    
+llmsql.inference.inference_transformers.inference_transformers(model_or_model_name_or_path: str | AutoModelForCausalLM, tokenizer_or_name: str | Any | None = None, *, trust_remote_code: bool = True, dtype: dtype = torch.float16, device_map: str | dict[str, int] | None = 'auto', hf_token: str | None = None, model_kwargs: dict[str, Any] | None = None, tokenizer_kwargs: dict[str, Any] | None = None, chat_template: str | None = None, max_new_tokens: int = 256, temperature: float = 0.0, do_sample: bool = False, top_p: float = 1.0, top_k: int = 50, generation_kwargs: dict[str, Any] | None = None, version: Literal['1.0', '2.0'] = '2.0', output_file: str = 'llm_sql_predictions.jsonl', workdir_path: str | None = None, num_fewshots: int = 5, batch_size: int = 8, limit: int | float | None = None, seed: int = 42) → list[dict[str, str]][source]
    
+
    Inference a causal model (Transformers) on the LLMSQL benchmark.
    
+
    
+
    Parameters:
    
+
      
+
    • model_or_model_name_or_path – Model object or HF model name/path.
      • 
+
    • tokenizer_or_name – Tokenizer object or HF tokenizer name/path.
      • 
+
    • Loading (# Tokenizer)
      • 
+
    • trust_remote_code – Whether to trust remote code (default: True).
      • 
+
    • dtype – Torch dtype for model (default: float16).
      • 
+
    • device_map – Device placement strategy (default: “auto”).
      • 
+
    • hf_token – Hugging Face authentication token.
      • 
+
    • model_kwargs – Additional arguments for AutoModelForCausalLM.from_pretrained().
+Note: ‘dtype’, ‘device_map’, ‘trust_remote_code’, ‘token’
+are handled separately and will override values here.
      • 
+
    • Loading
      • 
+
    • tokenizer_kwargs – Additional arguments for AutoTokenizer.from_pretrained(). ‘padding_side’ defaults to “left”.
+Note: ‘trust_remote_code’, ‘token’ are handled separately and will override values here.
      • 
+
    • Chat (# Prompt &)
      • 
+
    • chat_template – Optional chat template to apply before tokenization.
      • 
+
    • Generation (#)
      • 
+
    • max_new_tokens – Maximum tokens to generate per sequence.
      • 
+
    • temperature – Sampling temperature (0.0 = greedy).
      • 
+
    • do_sample – Whether to use sampling vs greedy decoding.
      • 
+
    • top_p – Nucleus sampling parameter.
      • 
+
    • top_k – Top-k sampling parameter.
      • 
+
    • generation_kwargs – Additional arguments for model.generate().
+Note: ‘max_new_tokens’, ‘temperature’, ‘do_sample’,
+‘top_p’, ‘top_k’ are handled separately.
      • 
+
    • Benchmark (#)
      • 
+
    • version – LLMSQL version
      • 
+
    • output_file – Output JSONL file path for completions.
      • 
+
    • workdir_path – Directory to store downloaded benchmark files. If omitted, a
+temporary directory is created automatically.
      • 
+
    • num_fewshots – Number of few-shot examples (0, 1, or 5).
      • 
+
    • batch_size – Batch size for inference.
      • 
+
    • seed – Random seed for reproducibility.
      • 
+
    • limit – Limit the number of questions to evaluate. If an integer, evaluates
+the first N samples. If a float between 0.0 and 1.0, evaluates the
+first X*100% of samples. If None, evaluates all samples (default).
      • 
+
    
+
    
+
    Returns:
    
+
    List of generated SQL results with metadata.
    
+
    
+
    
+
    
 
-  
    
-
    Inference API Reference
    
+
    —
    
 
    —
    
 
    —
    
 
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/docs/usage.html b/docs/_build/html/docs/usage.html
index 2e73e7a..9c2d572 100644
--- a/docs/_build/html/docs/usage.html
+++ b/docs/_build/html/docs/usage.html
@@ -5,24 +5,22 @@
     
     
 
-    Usage Overview — LLMSQL 0.1.15 documentation
+    Usage Overview — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
-
-
+            
   
    
 
    Usage Overview
    
 
    LLMSQL package provides two primary components:
    
@@ -75,18 +73,15 @@ 
    Basic Exampleresults = inference_transformers(
     model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
     output_file="outputs/preds_transformers.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
+    workdir_path="./benchmark-cache",
     num_fewshots=5,
     batch_size=8,
     max_new_tokens=256,
     temperature=0.7,
-    model_kwargs={
     model_kwargs={
         "attn_implementation": "flash_attention_2",
         "torch_dtype": "bfloat16",
     },
-    generation_kwargs={
     generation_kwargs={
         "do_sample": False,
     },
@@ -106,8 +101,7 @@ 
    Basic Exampleresults = inference_vllm(
     model_name="Qwen/Qwen2.5-1.5B-Instruct",
     output_file="outputs/preds_vllm.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
+    workdir_path="./benchmark-cache",
     num_fewshots=5,
     batch_size=8,
     max_new_tokens=256,
@@ -119,6 +113,39 @@ 
    Basic Example},
 )
 
+# Evaluate the results
+evaluator = LLMSQLEvaluator()
+report = evaluator.evaluate(outputs_path="outputs/preds_transformers.jsonl")
+print(report)
+
    
+
    
+
    Using OpenAI-compateble API.
    
+
    from llmsql import inference_api
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+# Run inference (will take some time)
+results = inference_api(
+    model_name="gpt-5-mini",
+    base_url="https://api.openai.com/v1/",
+    api_key=os.environ["OPENAI_API_KEY"],
+    api_kwargs={
+        "response_format": {
+                "type": "text"
+            },
+            "verbosity": "medium",
+            "reasoning_effort": "medium",
+            "store": False
+    },
+    requests_per_minute=100,
+    output_file="test_output_api.jsonl",
+    limit=50,
+    num_fewshots = 5,
+    seed=42,
+    version="2.0"
+)
+
 # Evaluate the results
 evaluator = LLMSQLEvaluator()
 report = evaluator.evaluate(outputs_path="outputs/preds_transformers.jsonl")
@@ -186,19 +213,21 @@ 
    Navigation
    
         
  • 
           index
    • 
+        
  • 
+          modules |
    • 
         
  • 
           next |
    • 
         
  • 
           previous |
    • 
-        
  • LLMSQL 0.1.15 documentation »
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
           
  • LLMSQL package Documentation »
    • 
-        
  • Usage Overview
    • 
-        
  • Usage Overview
    • 
+        
  • Usage Overview
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/genindex.html b/docs/_build/html/genindex.html
index 2caaec9..e566bac 100644
--- a/docs/_build/html/genindex.html
+++ b/docs/_build/html/genindex.html
@@ -4,22 +4,20 @@
   
     
     
-    Index — LLMSQL 0.1.15 documentation
+    Index — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
-    
-    
+     
   
     
-    
    
+
    - - +

    Index

    + E + | I + | L + | M + +
    +

    E

    + + +
    +

    I

    + + +
    + +

    L

    + + + +
      +
    • + llmsql.evaluation.evaluate + +
      • +
      +
    • + llmsql.inference.inference_transformers + +
      • +
    + +

    M

    + + +
    -
    @@ -74,11 +129,14 @@

    Navigation

  • index
    • -
  • LLMSQL 0.1.15 documentation »
    • -
  • Index
    • +
  • + modules |
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Index
    • - + \ No newline at end of file diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html index bd4199d..f66124c 100644 --- a/docs/_build/html/index.html +++ b/docs/_build/html/index.html @@ -17,31 +17,15 @@ - -
    -
    - 📚 Documentation - -
    -
    - - - +
    + - -
    +

    Welcome to LLMSQL Project

    @@ -68,21 +52,20 @@

    Welcome to LLMSQL Project

    LLMSQL is a Python package for evaluation Hugging Face models on LLMSQL benchmark with transformers and vLLM.

    -

    💡 Description

    +

    Description

    LLMSQL Benchmark is an open-source framework providing a modernized, cleaned, and extended version of the original WikiSQL dataset, specifically designed for evaluating Hugging Face style Large Language Models (LLMs) on Text-to-SQL tasks.

    -

    Key improvements

    -
      -
    • Data Cleaning: Resolved duplicates, datatype mismatches, and inconsistent casing, reducing the widespread occurrence of empty query results.
      • -
    • LLM-Ready Format: Reformatted SQL queries stored in WikiSQL’s custom encoding into standard SQL syntax.
      • -
    +

    📣 Latest News

    +
    +

    Loading latest news...

    +
    -

    📚 Documentation

    +

    Documentation

    Note: Documentation pages (installation guide, API reference) are under construction.
    See Quick Start below or the README files inside the repo.

    -

    ⚡ Quick Start

    +

    Quick Start

    ⚠️ WARNING — Reproducibility

    @@ -113,7 +96,7 @@

    1️⃣ Installation

    2️⃣ Inference from CLI

    vLLM Backend (Recommended)

    -
    llmsql inference --method vllm \
+
    llmsql inference vllm \
 --model-name Qwen/Qwen2.5-1.5B-Instruct \
 --output-file outputs/preds.jsonl \
 --batch-size 8 \
@@ -121,7 +104,7 @@ 
    2️⃣ Inference from CLI
    
 --temperature 0.0
    
 
 
    Transformers Backend
    
-
    llmsql inference --method transformers \
+
    llmsql inference transformers \
 --model-or-model-name-or-path Qwen/Qwen2.5-1.5B-Instruct \
 --output-file outputs/preds.jsonl \
 --batch-size 8 \
@@ -150,9 +133,26 @@ 
         📦 PyPI Projectllmsql on PyPI
         💾 Dataset on Hugging Facellmsql-bench dataset
         💻 Source CodeGitHub repo
+        💻 PlaygroundHF Space
       
     
 
+    
    🤝 Contributing
    
+    
    We welcome contributions — bug reports, documentation improvements, new features, and benchmark submissions. Check the contributing guide for full details.
    
+    
    
+      
    Quick start for developers:
    
+      
    git clone https://github.com/<YOUR_USERNAME>/llmsql-benchmark.git
+cd llmsql-benchmark
+pip install pdm
+pdm install --without default --with dev
+pre-commit install
    
+    
    
+    
+
     
    📊 Leaderboard — Execution Accuracy (EX)
    
     
    
       
    Loading leaderboard...
    
@@ -163,7 +163,7 @@ 
    📄 Citation
    
     
    @inproceedings{llmsql_bench,
   title={LLMSQL: Upgrading WikiSQL for the LLM Era of Text-to-SQL},
   author={Pihulski, Dzmitry and Charchut, Karol and Novogrodskaia, Viktoria and Koco{'n}, Jan},
-  booktitle={2025 IEEE ICувцDMW},
+  booktitle={2025 IEEE International Conference on Data Mining Workshops (ICDMW)},
   year={2025},
   organization={IEEE}
 }
@@ -172,7 +172,22 @@ 
    📄 Citation
    
     
    
       💬 Made with ❤️ by the LLMSQL Team
    
     
    
-
    +
    + + +
    @@ -200,6 +215,75 @@

    📄 Citation

    }); }); + diff --git a/docs/_build/html/objects.inv b/docs/_build/html/objects.inv index ef93dc2..8e77ee8 100644 Binary files a/docs/_build/html/objects.inv and b/docs/_build/html/objects.inv differ diff --git a/docs/_build/html/py-modindex.html b/docs/_build/html/py-modindex.html index a77a212..2488387 100644 --- a/docs/_build/html/py-modindex.html +++ b/docs/_build/html/py-modindex.html @@ -4,21 +4,21 @@ - Python Module Index — LLMSQL 0.1.13 documentation + Python Module Index — LLMSQL 0.1.16 documentation - + - - + + - + - + @@ -31,16 +31,16 @@

    Navigation

  • modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Python Module Index
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Python Module Index
    • -
    +
    - +

    Python Module Index

    @@ -68,11 +68,6 @@

    Python Module Index

        llmsql.inference.inference_transformers - - -     - llmsql.inference.inference_vllm - @@ -105,11 +100,11 @@

    Navigation

  • modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Python Module Index
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Python Module Index
    • - + \ No newline at end of file diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html index 1a05821..7910195 100644 --- a/docs/_build/html/search.html +++ b/docs/_build/html/search.html @@ -4,19 +4,18 @@ - Search — LLMSQL 0.1.15 documentation + Search — LLMSQL 0.1.16 documentation - - - - + + + - + @@ -24,8 +23,7 @@ - - + -
    +
    - - +

    Search

    - - + - - - - + +

    Searching for multiple words only shows matches that contain all words.

    - - - - + +
    - - - - + +
    - - +
    @@ -98,11 +89,14 @@

    Navigation

  • index
    • -
  • LLMSQL 0.1.15 documentation »
    • -
  • Search
    • +
  • + modules |
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Search
    • - + \ No newline at end of file diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index f5962dd..0d3d7b3 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"Basic Example":[[3,"basic-example"]],"Contents":[[1,null]],"Evaluation API Reference":[[0,null]],"Example: Running your first evaluation (with transformers backend)":[[1,"example-running-your-first-evaluation-with-transformers-backend"]],"Features":[[0,"features"]],"Full Documentation":[[1,"full-documentation"]],"Function Arguments":[[0,"function-arguments"]],"Getting Started":[[1,"getting-started"]],"Inference API Reference":[[2,null]],"Input Format":[[0,"input-format"]],"Installation":[[1,"installation"]],"LLMSQL package Documentation":[[1,null]],"Output Metrics":[[0,"output-metrics"]],"Report Saving":[[0,"report-saving"]],"Typical workflow":[[3,"typical-workflow"]],"Usage Examples":[[0,"usage-examples"]],"Usage Overview":[[3,null]]},"docnames":["docs/evaluation","docs/index","docs/inference","docs/usage","index"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["docs\\evaluation.rst","docs\\index.rst","docs\\inference.rst","docs\\usage.rst","index.rst"],"indexentries":{},"objects":{},"objnames":{},"objtypes":{},"terms":{"0":[1,3],"1":[0,1,3],"2":0,"256":[1,3],"3":0,"30":0,"4096":3,"5":[0,1,3],"5b":[1,3],"7":[1,3],"8":[1,3],"9":3,"By":0,"It":0,"The":0,"accuraci":[0,3],"activ":0,"ag":0,"against":0,"allow":0,"api":1,"ar":0,"attn_implement":3,"automat":0,"back":1,"backend":3,"batch_siz":[1,3],"bench":0,"benchmark":0,"bfloat16":[1,3],"both":0,"can":0,"compon":3,"comput":3,"configur":0,"contain":0,"count":0,"cover":1,"current":0,"data":[1,3],"databas":0,"dataset":3,"db":0,"db_path":0,"default":0,"descript":0,"detail":0,"dict":0,"dict_list":0,"dictionari":0,"directori":0,"displai":0,"do_sampl":[1,3],"download":0,"error":0,"evalu":3,"evaluation_results_":0,"everyth":1,"exact":0,"execut":0,"fals":[1,3],"file":0,"flash_attention_2":3,"follow":0,"from":[0,1,3],"gener":3,"generate_kwarg":1,"generation_kwarg":3,"gold":0,"gold_non":0,"gpu_memory_util":3,"guid":1,"how":0,"i":0,"ignor":0,"import":[0,1,3],"infer":[1,3],"inference_transform":[1,3],"inference_vllm":3,"input_mod":0,"inspect":3,"instruct":[1,3],"invalid":0,"json":0,"jsonl":[0,1,3],"jsonl_path":0,"kei":0,"level":3,"list":0,"llm":3,"llm_kwarg":3,"llmsql":[0,2,3],"llmsql_workdir":0,"llmsqlevalu":3,"log":0,"made":[0,1,2,3],"main":1,"match":0,"max_mismatch":0,"max_model_len":3,"max_new_token":[1,3],"maximum":0,"metric":3,"mismatch":0,"miss":0,"mode":0,"model":[0,1,3],"model_arg":1,"model_kwarg":3,"model_nam":3,"model_or_model_name_or_path":[1,3],"name":0,"need":1,"none":0,"null":0,"num_fewshot":[1,3],"number":0,"option":0,"output":[1,3],"output_fil":[1,3],"outputs_path":3,"overal":0,"overrid":0,"overview":1,"own":0,"packag":3,"page":1,"pass":3,"path":0,"path_to_output":0,"perform":3,"pip":1,"pred_non":0,"predict":[0,3],"predicted_sql":0,"preds_transform":[1,3],"preds_vllm":3,"primari":3,"print":[0,1,3],"project":1,"provid":[0,3],"python":0,"queri":[0,3],"question":[0,1,3],"question_id":0,"questions_path":[0,1,3],"qwen":[1,3],"qwen2":[1,3],"refer":1,"report":3,"requir":0,"result":[0,1,3],"return":0,"run":3,"save_report":0,"select":0,"should":0,"show_mismatch":0,"skip":0,"some":3,"sourc":1,"sql":[0,1,3],"sql_error":0,"sqlite":0,"sqlite_t":0,"summari":0,"support":0,"tabl":[0,1,3],"tables_path":[1,3],"take":3,"task":3,"team":[0,1,2,3],"temperatur":[1,3],"tensor_parallel_s":3,"text":[0,1],"thi":[0,1],"time":3,"timestamp":0,"torch_dtyp":[1,3],"total":0,"transform":3,"true":0,"two":3,"us":[0,1,3],"usag":1,"uuid":0,"vllm":3,"wa":0,"welcom":1,"were":0,"where":0,"while":0,"workdir":0,"workdir_path":0,"you":[0,1],"your":0},"titles":["Evaluation API Reference","LLMSQL package Documentation","Inference API Reference","Usage Overview","<no title>"],"titleterms":{"api":[0,2],"argument":0,"backend":1,"basic":3,"content":1,"document":1,"evalu":[0,1],"exampl":[0,1,3],"featur":0,"first":1,"format":0,"full":1,"function":0,"get":1,"infer":2,"input":0,"instal":1,"llmsql":1,"metric":0,"output":0,"overview":3,"packag":1,"refer":[0,2],"report":0,"run":1,"save":0,"start":1,"transform":1,"typic":3,"usag":[0,3],"workflow":3,"your":1}}) \ No newline at end of file +Search.setIndex({"alltitles":{"Basic Example":[[3,"basic-example"]],"Contents":[[1,null]],"Evaluation API Reference":[[0,null]],"Example: Running your first evaluation (with transformers backend)":[[1,"example-running-your-first-evaluation-with-transformers-backend"]],"Features":[[0,"features"]],"Full Documentation":[[1,"full-documentation"]],"Function Arguments":[[0,"function-arguments"]],"Getting Started":[[1,"getting-started"]],"Inference API Reference":[[2,null]],"Input Format":[[0,"input-format"]],"Installation":[[1,"installation"]],"LLMSQL Evaluation Module":[[0,"llmsql-evaluation-module"]],"LLMSQL Transformers Inference Function":[[2,"llmsql-transformers-inference-function"]],"LLMSQL package Documentation":[[1,null]],"Output Metrics":[[0,"output-metrics"]],"Report Saving":[[0,"report-saving"]],"Typical workflow":[[3,"typical-workflow"]],"Usage Examples":[[0,"usage-examples"]],"Usage Overview":[[3,null]]},"docnames":["docs/evaluation","docs/index","docs/inference","docs/usage","index"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["docs\\evaluation.rst","docs\\index.rst","docs\\inference.rst","docs\\usage.rst","index.rst"],"indexentries":{"evaluate() (in module llmsql.evaluation.evaluate)":[[0,"llmsql.evaluation.evaluate.evaluate",false]],"inference_transformers() (in module llmsql.inference.inference_transformers)":[[2,"llmsql.inference.inference_transformers.inference_transformers",false]],"llmsql.evaluation.evaluate":[[0,"module-llmsql.evaluation.evaluate",false]],"llmsql.inference.inference_transformers":[[2,"module-llmsql.inference.inference_transformers",false]],"module":[[0,"module-llmsql.evaluation.evaluate",false],[2,"module-llmsql.inference.inference_transformers",false]]},"objects":{"llmsql.evaluation":[[0,0,0,"-","evaluate"]],"llmsql.evaluation.evaluate":[[0,1,1,"","evaluate"]],"llmsql.inference":[[2,0,0,"-","inference_transformers"]],"llmsql.inference.inference_transformers":[[2,1,1,"","inference_transformers"]]},"objnames":{"0":["py","module","Python module"],"1":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:function"},"terms":{"0":[0,1,2,3],"1":[0,1,2,3],"100":[2,3],"2":[0,2,3],"256":[1,2,3],"3":0,"30":0,"4096":3,"42":[2,3],"5":[0,1,2,3],"50":[2,3],"5b":[1,2,3],"7":[1,2,3],"8":[1,2,3],"9":3,"By":0,"If":[0,2],"It":0,"The":0,"accuraci":[0,3],"activ":0,"addit":2,"ag":0,"against":0,"all":2,"allow":0,"an":2,"ani":2,"api":[1,3],"api_kei":3,"api_kwarg":3,"appli":2,"ar":2,"argument":2,"attn_implement":3,"authent":2,"auto":[0,2],"automat":[0,2],"automodelforcausallm":2,"autotoken":2,"back":1,"backend":[2,3],"base_url":3,"batch":2,"batch_siz":[1,2,3],"befor":2,"bench":2,"benchmark":[0,2,3],"between":2,"bfloat16":[1,2,3],"bool":[0,2],"both":[],"cach":[0,3],"can":0,"causal":2,"chat":2,"chat_templ":2,"code":2,"com":3,"compatebl":3,"complet":2,"compon":3,"comput":3,"configur":0,"contain":0,"count":0,"cover":1,"creat":[0,2],"current":0,"data":[],"databas":0,"dataset":3,"db":0,"db_path":[],"decod":2,"default":[0,2],"descript":0,"detail":0,"devic":2,"device_map":2,"dict":[0,2],"dict_list":0,"dictionari":0,"differ":2,"directori":[0,2],"displai":0,"do_sampl":[1,2,3],"document":0,"dotenv":3,"download":[0,2],"dtype":2,"due":2,"either":0,"environ":3,"error":0,"evalu":[2,3],"evaluation_results_":0,"even":2,"everyth":1,"exact":0,"exampl":2,"execut":0,"face":2,"fals":[1,2,3],"few":2,"file":[0,2],"first":2,"flash_attention_2":3,"float":2,"float16":2,"follow":0,"from":[0,1,2,3],"from_pretrain":2,"full":0,"gener":[0,2,3],"generate_kwarg":1,"generation_kwarg":[2,3],"gold":0,"gold_non":0,"gpt":3,"gpu_memory_util":3,"greedi":2,"guid":1,"handl":2,"here":2,"hf":2,"hf_token":2,"how":0,"http":3,"hug":2,"huggingfac":2,"i":[0,2],"ignor":[],"implement":2,"import":[0,1,2,3],"infer":[1,3],"inference_api":3,"inference_transform":[1,2,3],"inference_vllm":3,"input":2,"input_mod":0,"inspect":3,"instruct":[1,2,3],"int":[0,2],"integ":2,"invalid":0,"json":0,"jsonl":[0,1,2,3],"jsonl_path":0,"k":2,"kei":0,"languag":2,"larg":2,"left":2,"level":3,"limit":[2,3],"list":[0,2],"liter":2,"llm":3,"llm_kwarg":3,"llm_sql_predict":2,"llmsql":3,"llmsql_workdir":[],"llmsqlevalu":3,"load":2,"load_dotenv":3,"log":0,"made":[0,1,2,3],"mai":2,"main":1,"manual":0,"match":0,"max":0,"max_mismatch":0,"max_model_len":3,"max_new_token":[1,2,3],"maximum":[0,2],"medium":3,"metadata":2,"metric":3,"mini":3,"mismatch":0,"miss":0,"mode":0,"model":[0,1,2,3],"model_arg":1,"model_kwarg":[2,3],"model_nam":3,"model_or_model_name_or_path":[1,2,3],"modul":2,"n":2,"name":[0,2],"need":1,"none":[0,2],"note":2,"nucleu":2,"null":0,"num_fewshot":[1,2,3],"number":[0,2],"numer":2,"o":3,"object":2,"omit":[0,2],"openai":3,"openai_api_kei":3,"option":[0,2],"output":[1,2,3],"output_fil":[1,2,3],"outputs_path":3,"overal":0,"overrid":[0,2],"overview":1,"own":[],"packag":3,"padding_sid":2,"page":1,"paramet":[0,2],"pass":3,"path":[0,2],"path_to_output":0,"per":2,"perform":[2,3],"persist":0,"pip":1,"placement":2,"precis":2,"pred_non":0,"predict":[0,3],"predicted_sql":0,"preds_transform":[1,2,3],"preds_vllm":3,"primari":3,"print":[0,1,3],"produc":2,"project":1,"prompt":2,"provid":[0,2,3],"python":0,"queri":[0,3],"question":[0,2],"question_id":0,"questions_path":[],"qwen":[1,2,3],"qwen2":[1,2,3],"random":2,"reasoning_effort":3,"refer":1,"remot":2,"repo_id":2,"report":3,"reproduc":2,"requests_per_minut":3,"requir":0,"response_format":3,"result":[0,1,2,3],"return":[0,2],"run":3,"same":2,"sampl":2,"save_report":0,"see":0,"seed":[2,3],"select":0,"separ":2,"sequenc":2,"shot":2,"should":0,"show_mismatch":0,"singl":2,"size":2,"skip":[],"slightli":2,"some":3,"sourc":[0,1,2],"sql":[0,1,2,3],"sql_error":0,"sqlite":0,"sqlite_t":[],"store":[0,2,3],"str":[0,2],"strategi":2,"summari":0,"support":0,"tabl":0,"tables_path":[],"take":3,"task":3,"team":[0,1,2,3],"temperatur":[1,2,3],"templat":2,"temporari":[0,2],"tensor_parallel_s":3,"test_output_api":3,"text":[0,1,2,3],"than":2,"thi":[0,1,2],"time":3,"timestamp":0,"token":2,"tokenizer_kwarg":2,"tokenizer_or_nam":2,"top":2,"top_k":2,"top_p":2,"torch":2,"torch_dtyp":[1,2,3],"total":0,"transform":3,"true":[0,2],"trust":2,"trust_remote_cod":2,"two":3,"type":[0,3],"us":[0,1,2,3],"usag":1,"uuid":0,"v":2,"v1":3,"valu":2,"verbos":3,"version":[0,2,3],"via":2,"vllm":[2,3],"wa":0,"welcom":1,"were":0,"where":0,"whether":2,"while":0,"workdir":[],"workdir_path":[0,2,3],"x":2,"you":[0,1],"your":[]},"titles":["Evaluation API Reference","LLMSQL package Documentation","Inference API Reference","Usage Overview","<no title>"],"titleterms":{"api":[0,2],"argument":0,"backend":1,"basic":3,"content":1,"document":1,"evalu":[0,1],"exampl":[0,1,3],"featur":0,"first":1,"format":0,"full":1,"function":[0,2],"get":1,"infer":2,"input":0,"instal":1,"llmsql":[0,1,2],"metric":0,"modul":0,"output":0,"overview":3,"packag":1,"refer":[0,2],"report":0,"run":1,"save":0,"start":1,"transform":[1,2],"typic":3,"usag":[0,3],"workflow":3,"your":1}}) \ No newline at end of file diff --git a/docs/_static/styles/front_page.css b/docs/_static/styles/front_page.css index 1d3bcfb..45e3f4b 100644 --- a/docs/_static/styles/front_page.css +++ b/docs/_static/styles/front_page.css @@ -1,26 +1,29 @@ /* === LLMSQL Front Page CSS === */ +/* Three-column page layout: nav | content | TOC */ +.page-layout { + display: grid; + grid-template-columns: 180px minmax(0, 1fr) 200px; + gap: 32px; + max-width: 1280px; + margin: 0 auto; + padding: 28px 24px; + align-items: start; +} + .sidebar { - position: fixed; - top: 16px; - left: 16px; - height: auto; - width: 160px; - background-color: #f4f4f4; - border: 1px solid #e0e0e0; - border-radius: 8px; - padding: 12px; - display: flex; - align-items: center; - justify-content: center; - z-index: 110; - box-shadow: 0 2px 6px rgba(0,0,0,0.04); + position: sticky; + top: 24px; + background-color: #f8f9fa; + border: 1px solid #e8e8e8; + border-radius: 10px; + padding: 14px; + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); } .sidebar-content { display: flex; flex-direction: column; - align-items: center; gap: 10px; width: 100%; } @@ -30,10 +33,11 @@ text-align: center; background-color: #eef6ff; color: #0056b3; - padding: 10px; + padding: 10px 12px; border-radius: 6px; text-decoration: none; font-weight: 600; + font-size: 0.9rem; transition: background-color 0.2s, color 0.2s; } .sidebar-button:hover { @@ -41,12 +45,19 @@ color: white; } +/* Standalone back-link on docs pages (no .page-layout wrapper) */ +body > .sidebar-button, +.document .sidebar-button { + display: inline-block; + margin-bottom: 1.5rem; +} + .sidebar-search { width: 100%; padding: 8px 10px; border-radius: 6px; border: 1px solid #ccc; - font-size: 0.9rem; + font-size: 0.85rem; box-sizing: border-box; transition: border-color 0.2s ease, box-shadow 0.2s ease; } @@ -64,41 +75,60 @@ } .on-this-page { - position: fixed; - top: 16px; - right: 16px; - width: 220px; - background: #fafafa; - border: 1px solid #eee; - border-radius: 8px; - padding: 12px; - font-size: 0.95rem; - z-index: 105; - box-shadow: 0 2px 6px rgba(0,0,0,0.04); + position: sticky; + top: 24px; + background: #f8f9fa; + border: 1px solid #e8e8e8; + border-radius: 10px; + padding: 14px 16px; + font-size: 0.88rem; + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); } .on-this-page h4 { - margin: 0 0 8px 0; - padding-bottom: 6px; - border-bottom: 1px solid #e9e9e9; - font-size: 0.95rem; + margin: 0 0 10px 0; + padding-bottom: 8px; + border-bottom: 1px solid #e0e0e0; + font-size: 0.8rem; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; + color: #555; } .on-this-page ul { list-style: none; padding: 0; - margin: 8px 0 0 0; + margin: 0; } .on-this-page ul li { margin-bottom: 6px; } +.on-this-page ul li a { + color: #444; + text-decoration: none; + line-height: 1.4; +} +.on-this-page ul li a:hover { + color: #007bff; +} body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; - margin: 0 auto; - padding: 28px 36px; + margin: 0; + padding: 0; color: #333; background-color: #fff; - max-width: 1100px; +} + +.page-layout > main { + min-width: 0; +} + +/* Sphinx docs pages (no .page-layout wrapper) */ +body > .document { + max-width: 900px; + margin: 0 auto; + padding: 28px 24px; } .center-content { @@ -227,12 +257,24 @@ div.highlight span { } /* Responsive */ -@media (max-width: 1000px) { - .on-this-page { position: static; width: auto; margin: 12px 0 18px; } -} - -@media (max-width: 720px) { - .sidebar, .on-this-page { display: none; } +@media (max-width: 1100px) { + .page-layout { + grid-template-columns: 160px minmax(0, 1fr); + } + .on-this-page { + display: none; + } +} + +@media (max-width: 768px) { + .page-layout { + grid-template-columns: 1fr; + gap: 16px; + padding: 16px; + } + .sidebar { + position: static; + } h1 { font-size: 1.5rem; } .badges img { height: 18px; } } diff --git a/docs/_templates/index.html b/docs/_templates/index.html index d81ed4f..aa843dc 100644 --- a/docs/_templates/index.html +++ b/docs/_templates/index.html @@ -17,31 +17,15 @@ - -
    -
    - 📚 Documentation - -
    -
    - - - +
    + - -
    +

    Welcome to LLMSQL Project

    @@ -153,6 +137,22 @@ +

    🤝 Contributing

    +

    We welcome contributions — bug reports, documentation improvements, new features, and benchmark submissions. Check the contributing guide for full details.

    +
    +

    Quick start for developers:

    + 
    git clone https://github.com/<YOUR_USERNAME>/llmsql-benchmark.git
+cd llmsql-benchmark
+pip install pdm
+pdm install --without default --with dev
+pre-commit install
    +
    + +

    📊 Leaderboard — Execution Accuracy (EX)

    Loading leaderboard...

    @@ -172,7 +172,22 @@

    📄 Citation

    💬 Made with ❤️ by the LLMSQL Team
    -
    +
    + + +