From 88ecb51eae3be6a13cb588aad4114648b2708cfe Mon Sep 17 00:00:00 2001 From: Viknov Date: Mon, 22 Jun 2026 22:30:11 +0200 Subject: [PATCH] Change layout style of sidebars ("Documentation", "On this page") in Docs --- docs/_build/html/.buildinfo | 2 +- .../html/.doctrees/docs/evaluation.doctree | Bin 17486 -> 34645 bytes .../html/.doctrees/docs/inference.doctree | Bin 3054 -> 60561 bytes docs/_build/html/.doctrees/docs/usage.doctree | Bin 8936 -> 10566 bytes docs/_build/html/_modules/index.html | 27 ++- .../_modules/llmsql/evaluation/evaluate.html | 80 ++++----- .../inference/inference_transformers.html | 89 ++++++---- .../html/_sources/docs/evaluation.rst.txt | 12 +- docs/_build/html/_sources/docs/index.rst.txt | 2 - .../html/_sources/docs/inference.rst.txt | 6 + docs/_build/html/_sources/docs/usage.rst.txt | 41 ++++- .../html/_static/documentation_options.js | 4 +- .../_build/html/_static/scripts/front_page.js | 4 +- .../_build/html/_static/styles/front_page.css | 126 +++++++++----- docs/_build/html/docs/evaluation.html | 88 ++++++---- docs/_build/html/docs/index.html | 36 ++-- docs/_build/html/docs/inference.html | 129 ++++++++++++--- docs/_build/html/docs/usage.html | 77 ++++++--- docs/_build/html/genindex.html | 92 +++++++++-- docs/_build/html/index.html | 156 ++++++++++++++---- docs/_build/html/objects.inv | Bin 323 -> 393 bytes docs/_build/html/py-modindex.html | 31 ++-- docs/_build/html/search.html | 60 +++---- docs/_build/html/searchindex.js | 2 +- docs/_static/styles/front_page.css | 126 +++++++++----- docs/_templates/index.html | 65 +++++--- 26 files changed, 829 insertions(+), 426 deletions(-) diff --git a/docs/_build/html/.buildinfo b/docs/_build/html/.buildinfo index d7142fb..943f73f 100644 --- a/docs/_build/html/.buildinfo +++ b/docs/_build/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file records the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 3caef0746bc07fabd8f91030ce7b6533 +config: 949fad3acb2ae8760cf522b5d9f55c0f tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/_build/html/.doctrees/docs/evaluation.doctree b/docs/_build/html/.doctrees/docs/evaluation.doctree index 3dff8bedc85d28d10ee6abedc9791edf7cbdc371..9f1f19ced0de84b44b67c610eb36395b38b377e4 100644 GIT binary patch literal 34645 zcmdsA3ydU3dG`ADal3oBXCFR%KA)Xt4ffu#v*!nXu=X8(o$ce!d&l`~0(aK9ccyo{ z`(~!c(>?nDENn35mbCJ~10ewvf(#0eyoe%MiG+|MOh|YHVoNANC~+b}B1J?ZI0}I% z-(Qce>Ymx!?zy!QIq7z0s;ldN)L;Mq*I$2CJwEiIH^2V^_Aj2O29#t=e)LorgQ_1@TDeR3a~EB=y}P zci0_yxHE-^Ly;HN9PLrO`|y%oYuogOb;pS#)=6i-oYz_r<@f6Zm9Xb4o7Ub zY2($X<6abxmJ__)E%C5GcP<{Rc@3vC*PgfIF*}M{-h4X(nMBqWuNs1cc%)LZ!w`k| zKGd-5{M*)QyQzN6+m*8|XMyUpch!rkMJyO+CL+`aMbcg>!8FmzhsnMSY_ zES|aB3C~7B^GvN)4IyPqm72DPg7XxnLdPH5SUs)a%?a;&Ny+4FYj%veVvtJ(4zQE1g2 zu(A?PTeV;@9IJa_-Hs}*g93Unw3pCC)roAc=2Weg(+pa8$!UkBQyE?=Ad&=E~ z4hIe$(WhYO3}@(^dj+};+}Ocns0{WCL6L#nGaesBre6^|(%Y0ciEH*33BLIhk3 z5;uweuf_kX@qY^4>+W}NA}j!VSNz3aa2~J2RmVvFw7b`RJ%n*kN;gX$1=W1Npg6gX#pY&d?ZTQZPQ>uGO5VT=OtelmsYMGfGd&5ZJo* zIKEA&8%M~0v*lDhHfTa?p%v7vH$QNE?wGXzK@F{-Wt~`!Txce$6LJxCOOjI87X+SP zqlziD{7jL3G6v?j&}POCTo4^dl`8}T99)!C#CPAJk3*{(EH`R_UDebElt4%dI;95g zzSHs+EDZI~Yb(Jd&F^26X z*JzcXvEVJXTlRd-5o)d%b3P)NbH2?gw)}Q+wrm5XAI{4bs@%<%6Az>nJB=Mdwzcd< zt}+X{MTk6@)X1(kxpD9sFhFcH&Wv%5kqof-_$C988fnwpRXy>?#d~5qnEU&Adx9#9 zo>*UN$HQDZcH)5&vtYKS0hlgr&#SlNtqW3P+tU1{$~03r3h=_iNtCg{7EC-C+KY~L zcm>@JjVk${EmrLNPJZ^0U3h_4XQ*5Rt)@ih@M;=3tyM!~sO@X47UmeSx0W4_rGe_@}b`4xo7IS5X?6dDXvJi1b^&4x{zrW0FXkKr>f;p@;s zKr@WBcx%mWEVdy=op`v(%yB24aJ@yhhW}C7ZY`4kPJY^d3Ms!tW>qX{9WBGIcS|8v+(bolhO{Cz3SwwHF@3Sv^6Pe zI@K~}6;%4b;bVvII%Uy>#JcC?@%yb)%o@G*zQZRETQnFCSvOoe`Pg*tHm^6edDrm= z=T1%S->0=p4L#gTDm4n$k;MPsL`vcJFN0i|ok@@p#Lyd%eK)Lux zxJDyOjfA&$>pLd6>nJ@g)yqNaET}G%VmvrGb0C=jml#JTr^ga?__DR1^#9G^#-{fF zKE)`_hEKG~(v}=)X_Y1CPb}?;!Jd;5HQ4i=_s9^#9j!$c7D{Y-uSZHIP8O00n0Eho z@&>c+*D*^vgwR7Oe#GZ{hzblxi>;vD)Qqc|3XW8QTG(_dT@TltH6f@K)Rw(!7#UNZUzH>g)zWb-(REF_xMk-s{RM?k>Qon zTH}a{Ra#z?1q{WkC{?C#{uH%Z80TAn$R`;h1#niwO1i^ZG{}FF5ZU8Dk)zdj^Qv5G zSdY&K)$Z<_^52JwX(mmX88;%FJ=x32Z^f{`A>f4>_C}EQ^&F&$PW~%a)r{+HC!mKZ z5UFA-6k4$@>3}I;fd^VFIIWMiVYaIKL||c8e1456uKY`1T$u$|{<$bua*5GbI0Fj{ zz&72;NhwGdDxya&>_-%C1<`|ebAzd=9Pb0Hs#z3*yAx-m9U>GAp%6}rh&rJxvDQ2a zhcNSk71X^ba;k_JSP1B0N@T;GQI3rlT9p>&uz0&iP#JOCR~)Y5{gr+3{x5cRLGqyN2Zm z$l|1$f`ihKip%e8x4r6PY*Ia$XrogF8-DTekC~f(@$rw(sKp8Y!M-?n2pl|Fl!Gs; z(Ci4^V7aWKe^PL|%9Zf6`y%~KAib5J^ng`03(vJI=t~i9v6y8oW1-WLffowwO5w3i zwc1W19K~N$)W>~Me>`{CQ}Z7XTFC+74MJ}uduY}Eg`AE&aB4zkMOtE{E*LK6zq7XTnH~A zI4EYRYCfDiwe+WkIWbK=4RhjcN7xF+JwXe4LMpg)q3YtF7V2V}2mg2S2Ac;T1~O{? zw4c_>mHQl8(6?rh@Lo5mM6nRDAJ+`GE5At3Of8edcI^WyeVCGm?bH(GB z-EG`piiPgif}oZNu3{Z|$vJr4U-0-U_uFq&xq8AkKOah4Hd-On>xWKU*`)yTM^(8!<>eQojv8__3H zBYKubwA5c%OhqOIt-G^{F;<3X-outw5jz68l;=ZX-?PrD3gezn6N;`o5-f|vh)!zs zfv)*XP(w>Qcuf>UcFlV6@u#g*?3+wAl51U6&ih2!`KJgYN($Q*=XhCB@=#xtoB|~$ z@#Y4zUsonqQ2pZq5-3;s6?aPakw|@u$fsI%k-Q`|H>RzWa3y9O7OT(GuS8Ln6K54C zHluaLs(N3ndJv?01z1J(d$Q_wGG*mP&~TVsq?tt$&y&{pr~udI9z1r8LP?E4FfYrf z_b5i4f7^;pPxi&8hd}n@1=vLOd$Q>sO3gE|Danl!Y_XQ@aARiumSWasK~OCFXkRRQ zD_HQG1z1M)d$Mc>tD|M7)e2f66Rjf+3KWWheWQbB4`WtB1D_!4C#)M#0#g7h0o*R*7XEcIVe)E8((5&qS_ z2!9(0`EmimseVtwZ_?&(q=S!%b^oMTrMWWG$`e(^!j>{W-&0)Zw?)O0f9Q)PZwHCr zEx;10-;*ViNfH(l29X&-@}E4&3c-&R`FY+}r2bc5q@D)qKPo^f)$d8_z074SljUHv z-WMz#H z@D;9-NG;c#eepzQ7vhP*-6!wb;O=if$pYFI5)I);%g)G@Dp|gv5HFdtpS;1G{c_6L zf8kE7V(|DgEp3Qg2drjLygIX%VZnsU>~;*)Uh(NiXwpw^67)Q6gitFaGh@irM|LVQ z@G+Z6vh@=Uh(!Hl;CH2ET5rQXf{NX+0@#ceh6-Q+N6N0S+AQnj5p8YY5grYrO#TA} z$$q74SfT%wx_yA za%P=%o74TXVL)HK$l|lUHXuzP(+h0U1OkK9DA`gfl_cg{UVOZxQ63h;SVF3%AjLbv zrt39UX6$NJYRZ=i7k z@-zY28`tx0U*edbQDaShb!M8Yv$VMhoI!{JadZ=rR~#MCNHZ(#-C~E5=C<6=crB%5 z{R&EzKT}9_NR|I?^2X1BJG2ul@kQhL6D{VnVAT@pUYdpUCNL~jsHODvcYcx2hc*OD zlQ)zW-=2b{4Zc4~_|c`K^Y6p*|2)&m#=4i_GTnPb1wmm8l zkyK}u9&pHHVgXugrR2LEDI0SEo0wojDJ@sbI+C@df^R7T{SV-q$R3e!M`xR%vnL2UZnQ+Rm=0MWwy8Su}*wx z&!^B`k8w2o4}tO!>3Q$Pr*s!KNY6`a?qTv~kY5;ICn&{;vZmlvaUu)~Y+on7YJK&V zDB*P zYd#2Sc$=2XrxA3}mxyQKNJV&d<36i&h=seEHWxH+V>FY6rj530zh8X0f$P#780)Bk zGdg6NC{4xVV0j4vaJyDfqzQIW2zYZUIhFy`@|^|dZ- z;#!^Mjn0w!3pjhu;sZi)T4&EXeQ?f5n7@d}U0ETb69kkur=Ywm>FIZA?EA18%0s=N z2oWnl%QI#uIcRBWv?QW{b1a|&mDqm42xdK5@ZXKslq~p9&`**TDF_^lK@$9@=oMp+ zr~Be`(if*S^q(`L@8{@?-drM&M+5V`5lsJFVqji=F8#oWe&1Z$0>?{Z(o04teKUzc z*-RcyCz3R+VLvu(a^mvvD36BPZHChCsLm+(tBp|l?+G#^=hqW?cr* zG)Ep<9x_AeyT8*TuWE$Se}5B{^Xu=l4o305kuvgc{&Rosc^FK-S z@sqE%+4J))XUSvXqMmhj#b{YM0iQGSM#nrJM=S;V>Uq=g^igLOll^K52M4TTVMk1} zuny(W51?pGoj82Ly5YKOuir;o?fuUXs6GCt@sV92oQhwKbhM%X)?gtV91N$=4VhBRHM*zk5E8;s)m5MG5H%AGZd2o5>WCCiX}4?lOPEwDFrmf z1KSjc7HrO=h|U?2__-*eCwWKZVd2M&*c)AH0VORCy#phkHA69pLjhWzHA69pgMiYr zzXzE>s1fv>5lqewkGBN#emX?P;q+;HI00z|lxHyKk&%Bi5^R(aG0gJVo*$Z_m~2l0 zmGLt(6qAeyC~rawi5=yACRA=#P{y|!ZT3+#l(ME9n;DKmx|el<%k9!$X zCZ@nB`>c^(qlgG7c}&c+W+)~TQ-GG|%uq}sBB10qG0&S}^9xXXKYV;ONVk4lW49y@V z#xOSb4yW+vij|ie!Rr!G$i#3keQKObfm1dXXw*0b*nOjsUZchlQ1WP;yUb8b8m9m) zZ!ts3Av$S|L&;Q3bdZfc$bKj}DVt&Sp>a5&o1k$nT_^pkm$Jzo8zOW~wph`BDpyj(}C^_UNrDq0ilu8EQ)JXd`Gn76w3@5QK4U^}k z__8|A_42T-ui;>;pIGYR_eiXz9!4$PS3>z%l>n#xcg|4+d73Zssa(#kR7IYtDh_VY zR_&0}OUWqfk39Zdad!Kd$;Ie<=y5Q8+W1l6l(q3XCTV|jgecGxUo~bSe8P9#;7XdA zCC^!|V1s5S-XXu1ZQPHOS{mIblUvVQx{CIDV)IRSfG4UQP?_uUE2X|6(+TP0{*~y| zco;vBpbr#i&x)*tq-eI@Ih7@aXTB6L?uyI zu(Oqw-jG{o^gYbeVD)1RaN5hsit_^My@Ml|95>U&+M7k@II;Gyhiyo7@~(CgFWbh8 z1P^r_FWD(^*%3}AM*B$sJ;@T6To!F>Z27|sTZ|O=g3F)dMC^*YxJ*wnT%ImOh3bQB zQycfe?=iHD6yW%u9DVQ(eVv#(LLGyPjcXzDYiM&Azjq1RG*Op5Prw~3egbZ$8`G=J zqA$~hif!M?^nj5avO}L=w?pYkrbGQ#kQ<7wEFv3=!?!j|91d&iIV26o+4<)<46AN8 z+W4TyY&UO)c?=j!l&*`Ul3-9g(YFf3|MF z(vwWT-qa^u3dd}t8~4#qHmi?r%mhTKpEf=aRs=6nO0z2>D*C0XT+zM1pT4;oFZDK6 z^d!?alR0Qn_+%T{xF4=!$QWfRpX%J74iix~;DFZ5k*w^dqn-mI8zK_C&lW3EAIR{^ zNP<*%$Jb>&J;|_s_qzBiEV8d{%8XxU`8k8`$LY-gA>A9|T0?b#xv$@#;d(n~qn zUo>3u_1p;QOCo$X9PMw-(A*#n#pL6=wI5s6;XG7FiS&tb7l?lOf z8`qOUKpZV31jdo?pT>ZI6v|1eJYmHA(!Km88Byr{(v<)AK$W5y{Ep&mNJQ*T-fN}SfMa41`gUISM?H)pSJIrUp{22?F*&0clWO*h>- zc?dUY{m1Wo@bfUJIMaz8efQ$WpZVdlzh=#^>SuzSa_supZ3h79kkrSoI9#rHz>Y~U z(sa^rJS+|YjtAYSUhAYIk`{FX=WQd!TP5M~c76<330^q$y*gxAJRtX+KQJ6ShI78O zCU>jwW65rMB~my1l5^F%R>wje90sQ9WB_g9lEe-a!?`f)%TjXjx%>L~5`?oGVU6wh zC^uWdBF>iO7oQKYlXyEv5mb_OoGN~{DP%SfPl(gR%E=|_+hhq#>Yl@Xe?CrdUHrv# z#tFMW9yeQ3EU}7P8y})1?J$wsc$kL`I&<#CTs$gfG>G-$2QC-qm&4-6cq<4Zpo$>v z3igg1z#b-!0>$pnqQaQ?*>2VFYdgh7>0)+zir6UKP=ur4(MAIq;M$XO@pihS9_Ra) zS4dcJ2+{;1N?wx%`Z01sw01c!acWMTO3E}m%JOD-0242WtK#S4ZCq-~HD}4G;hg(; z%(ZJcJdpn!iZ6Cnyr_(OZ=-goN{6b>e0ve#CYJ41gVq|*E`}RQM$s1)hv*^v`00SS zGd`dl;t2M(qlMBfB^GGjn|{cTIW4=mdA;Ufqu(&0iWqdfgR{M?S{Y)kbmA*APclvT zXGsfBu%l6XyuCO5AoFs35zzp_3th0N#;*C?F0b?i$0_Q6Ah`@40 zjuVWzZ6F3WXNzrCkQz~~%w86PRsi!Aj639S0i;7q7-;3-);AE>Zq8 znE$@cNsMSL9YQxiH0and4lq%MdNJJ1iEtG+$<}3QV!_>^>}*;5-WA__AN*z<(9<|; zadEZ8If#ZM4ks-fCki92d>%wM6OZqC$dk$A;|qIx{BNP2(HR;N#&{qeWRr<_@W|Xf z$H_njZMtgO{{$ohS_XO+{HA5oZ}KMN@q|HaM2>rnO08Wb4X{X>W~@qi(GchlMoJm`F0wnD_%RL}L8F@L}RV{2)=|NA!K)+YhJl2gZ1R zoSb{k^PF?u^PY3>wde4MPa`2NobBnoAS^7KbnZ}J6JWh@(F_kEX!=$&Hh+!`| z7?Yx!UkOJ^Q*^=~A%_)p!s}P9T}iokEGd^y|10k4GEZR24@mAu67BW^{L^Eb|0#)lx& zX(Fl;iSSX8ymrzn`NUXI>mqf*V2F0#m1%XIJ=@?@|B#vRYYmC&1auioeE&rax8g4^ zuHsgF)}GAInFWJrZx=Sv&Q$Tvyt~rVSehFwe%5T+e=)<>%FV*Vl(>Nt->9sgbPPw> zQiwsq#*}DE{zUoyk%5un?L)-p4@$X?+&eLUggv;RK(3MtWi!lHwFr;UigxjeeyZ{` z6ld-zdxqS5o*&a&u$l5y&bw=_53T*%Nk(=K(LG}Of7nmYmRSvYT9R3IoQLcF?L z=%)D-JpXsq9%!s=hSQdHs0ZG*?1IlK--1NVI@AjbH7>B4C45^idJ84pU?hAU!=~kR zP*EMjKkL4&)c0f7_kzy?L>0hMUIcemJ_x%lZKx7%Hg**CQCLMGykoIqQz4I0T5}Aw zabG|6*>k?dF^bYP1kojg+S)-Bpzu~xDTR+0W^Y4oma7K`;e72Obc9EHl)KXh>TV-E zRS0uamdrjk6b*A1%+!yfL1;5r;Qod-G?xpSS{!r~;temNc@|W7j51TOvvDuF1g|%C zqW8G<32I${S8dG#(=NhCHWRp3kCc4M0`U(t>|ecQX?_Sk=d^tk{0XNqO{YE6v=e;` zUo?%P?>Xj4>bb))Ow%zJtb1{>0ZMH9@Dc-CsG>3I6AUnCvk6RPun}xAKtpp6COTw4 zwVQPaQ}sw+H+SI<1FUM8MoS+aGJv&p5O3FE2dHmAhcQ)$-EDPZ zj{%zOomkRgBK3(njHw)!{?d*RKCT1Y)Oth*F!dq~IGntRDLCoaherAB{OVYP=NP_) z-+9Wg=+32dsBJZ(_j7h_2*1f;;BXqKp~E>%TW9<9;DsAOE;;a=JJyh9ti?%6_{6vrteFSV@I^RlX7!qCcR(HYU7uY62n?FEKI0 zH5xrI!-R>k%BYAVgc^(eHy@A7nJU;M{MgRpY$h!kTiWGGuD1nd=K7iv3wpk*W)$H!<&Ft;& zEN5m9cQok0DoP^CQ>7HSvMlNmt76A=9#USWtum9SigX+~Dn}1X7NgjzFiFX>NSm&d z5=)Nd{C{`PbkFSUzdgIBhh1V7cso7a{XhHr|L*CYxitEr_rJP{{TI$u+;Tf`>i$C0 ztyq31oWO#H)eJh%bdLOd=ZQ`sob=2KzT5W7Rwvv9HOjVAuXt9o^JvH36ps5=IdI%& zT;CqG$L#S(I|aBo8aP4SQf`G;-`%WQ(70?FC-1-8c+e789c%(Eww|-rkaq8i@9S{3 z##$!y6LjpI;bbx9+TIe531*kVt#zkqb(Y#CGn_Jmz;jCN0E9%Lwm20Z1PI5=b<_8u z5Ppv~%?AH9QE9j2Mae9$cvcnbw8QbhT*g}+d&b^lPuqL#E%v_f?WY#c|CDce{`sc6 z>Moyur{%8%ZtHx#-tZr*&zC@8yJ31O=Ydh5|IdjOvEcb2#r}IYw@cv;zhyhkwFR?M z5e6A`niXqJQ6bl^{RWW2Hb7?w;NNxd?-2YephTb)&tY;D!mTaS@jBtrhFfXZt=kOF z&w`{K|0)Ka+4QTf*8nLx;R0_luiL;LTM9a6+rHkOmugwE_k;57Deg$pF?K`+;h{5U z&OY?O8RMKn+>kDEy4~c?#k^xyySnfZ;N&DX5a-0~sWoIA?Rkf==LdA{$@OzSR-^ot zQ0PKyeGyF$mT4{b^4wJiMzLWUK9H$f zMpfv~cu&k=f~~@l_ZWd~21d*B(DeRP0K;!SaOcq|%U09!OdQ-s8#^@WrnhXt|C`Hg z6Mg_$tL_`CjtQMuNgLvrv6__yHm(cyY%gLMpj559W^nu# z!N?Q5u^j9fJyhJH&wK>hpEm2h#oIvBBf@KbV#mMjcI2ZSM(_LmQg7r7C)XPFMZaZY zHaG!34oS}{{2Ybw0Y#U?i6}gD!m-wRU_)jY&e+bfU59@`5q%R9FpRfP3WS)95Cf2+ z9cKokOD^VW0+Ym*M5K=nj!16+IuFEjzQE#A94On63$kc-S9^IGV%}*GG&8apX0u{6 z%yo>qm3G;h@|l43b)(``S%3-*0R0z zV?49#glA$d6EUW6#3+ETh>Va+*Eiii(h2X-tSb9TknK?U|6I5gvQjnYh-Qd$Fad=- z5b|J7g~3eC4E!ydHuVsai0X?5L~!Gx@pGzya5e*}>Vnx0T>cw|c9C73grc)}_q6Ms zGTXjcKXY~tMjymmh{GNx3z#&>Zr5ATcHv}m9f&NsO$!nb{4-}<56!%`A3%cI0PHH3 zq1Q#DVKVA_B?#}OQZ;GU&{+!AmHhE)+(0E(Cmg6g3=pQnM) z{(|eRR2-g+W|~gT5o3wwAb><^VpMqRoKbfgPQb4(3WcQn)3>Zj)Zi^Q9x>+cfJT1s zh;#{Xu}5JQfi2$5h1=NB#FYqK?ueFy?9s5GixpAy)^8vo{_DLVz7S3WV9bL`8E4^k zWR$Jruxm?Q4zWA-hY*2Z1_b^J?=t+vkICe|CzkHNcLV~&TsX-rkXHXy*`OFHyVq^0 z+CQUIJAta@dYwS>y;b_ERH^-Km}qjIcQc-Gv7%wYln*^j@tucH0jU23y;;=w33~qo zy;<}d0KH?7=J``R+?1gfzExwKWo18c-wanVH)9!)+Vx(d>j;O)Ix!g4w!!*OZ#)V| znvh8=%iDnEwJB_}r#1$E7))NtAP4zu;d*DA&5u-eM;0Td;Fl!5zYpnFH|f18(sBA) zX2UJe ziNfJ2l0(%P@My@YR@{c^z$AZjYaJdwKmo2Z!GZt_$Q@QCz;)xG!wH4cP{(h$LQIz+TatlM{z^_aVCi9fkyBVgmpI8)bOQdgOSO4bMLq2BZc@tH z+u(MW;|W6uD*F;tF2>frsOZwBsT8uP2p`KK%PLJpLzX)gSz^DqGSjW(_<52f+sjdE z4?r{ND7A<15*;b>lG}!uxpfHdsy&R9D94ct132BUT@CfSd>{cn$p)&#BJZqig+2hQ zHo+fo%u)CgkKOC#EGsq1?3!FM-zp@B{;sa(kq2hnkw$wC8DbADv!02OGXDw}BI)

i@>nUkyIC3HiT<}WzO@j7oo=}S*iV7?4EDY+GdtY9;{KPr&+SjYzTcwOa3+~ zu*zLHZLPuDF+2-^h0;zqcdxY$i=>r#cx(<2+M+cj-|#KGfOT`DaR0mRH*Pw9?8YPb zOtJPEu7&5}5A6C%?2jmfSFI@B&BuLoYEMD+#Ht~7C7kJBXLO~qtfA01(2oT~mXSzB zc6Bw0#z@8;sp+qy_=6>Pf5_ylCF6K@a`!Eo3Weu_iVpDnUz!Sq=YmQ?qg(Mo40r6z z_|ET3RZhY}Wp7og9*`@c<75WJ`BC60?bamD=TP@SQeF}X1r-%9dV%_JnhFlYt{ImY zh&g0=J55Eybv_D=MA!#=QsLoUH2(O7^pmqRtzSz^)E~mRphH3UKJlP2^9f@X-i)%E zk&NO>3R$Ivq$bKGXbLl59b5@$Dip2=Dmu9GUYZK!9D+&$R}{_FGJ&N_nH`dWfstie zd8j<-)Zrg8V*z=a&pt_08Ps350FEiLe43;(xFwLvMtUpX zPX+sNM;=wUf05*Vqb%`7nu>-cl3taKg5RL2XpB4^uX>H9q9IG>_)ag*{S`?@*R3{o zd6%PLGCL!)@*K&>?C7w`hfY?0kfuUqT#$w}MVl%Kv{}SoqL{vYLQx-@`AEqUx((QB5R5RVK(o`rQ5ma=5B%rBK zULvR@^f$%RZT6Ii?VQbC1xV!)?fz1&bbDtwmYp3JTV6`i>998HCC zha9pzPg9}XK~T|(D=(2`bPc|-afS1rL>n@$4S6wX4@%BsI<#FXp#cF*p!Ah0JQb#^2x#B5WWM`0#eC_~Mg zXa!KAWXRfDT}yNY%?;RNm1&Udss3+}LTr>lUZkmL7(^!}`yx$+f=!_@9q0N6O+`bN%we9M z>c2*k8Q8lxnE{uOxR#Q^noP-ZM5FO>GPR)8DyZnBWV19CDkaMy%R!oohAjFi*>RfI z04W(4G-+kk-!xLP1-bis(6nr4x{D|^>%Jk=vwLV|Q0dueXsVf>Jxo)fa7Iwk!I>&e zg>oW6WrNeRHIgCKLwEk z9UbX-_6sx>%CkvJB-69sBdO?G;s=qQWf~+qJ^N!)h>bGHt27l2gXpAZ-=nEeuqiaA z<6ILH1LqNxbEVU>t4J~fd)H9u8RtJ4xXJV^M>IM{%1f@82`W12*&Q?$Dm}{~%Nd#q zcp+>`hrsOW^47ilV# zJLHh%i!>F=9R!ukOfnr}zCn`FHF#G7GgOG-{3pSO3^6%I*>_2KNf!}RbVAJdB$*#j zAtr|`vosaTMFbW75Oa{GH9&~rEF`V0`kO|Gxi9R-(kv5YRa z(=t=h?jWr&Sx zRc4?Z+x&Cr`bkn;5@Q4v9gKOJrb1y%4q1MgrlR3D35?kijZN4M$bQSw_9>du00_gy z913Bs>GejwEP$C-Do5y&D3ikeE-fqtEiXb-O=$TFO@)FMK}82z{s&EkauGo#0WG8K zRfbG|+2A57NM(M9WME);5p)K8jPV!DOgR81hsJkIk?Dno#&rN@KTU-Km>ja)KvU6> zB?&O_w#;bkZNcBtP2NUR8USFpm_q@~)jeM9nT0QtaotElr9{g4AT1|_Elbc;6IEqPBsUn0p5j4VT)fWF9Vlml3DsQXz`UNUnLRCIvlIhqOu zEIDL(o~EMVItgGIW$)XK2A;T*gYG4o+5q6fMI8!Uc6YxVI15)sS+!U+ekDcwBU&^H zPhN(mnt1YUnhJ#{f{G5Fe3zy|IfbB-z?1yB%Jei9x(7y-;c}I6U@F&36>@0&&7{C2 zqzEcHka81Eg@TkEvYe!;Xn0QoQntW5w4+hSe#riJ+o`B~Q{+C~pu{5?GRNN#z$w;sfIdV`9J=&(q96 zITDi`y8f4>xFm`QDmp0g?`bL&isX>x^E4F=ze%9T=9^;>!JiUC{wqyq01V+m4uv5{ z$xjg|d%EG90?L7+#7NoI)klOk8F!>U{x_`;3VZ$>nrdRt$Tlj#lbb076&>u^K~tf8 zNKi>=bog!cZz35Q7>|b9R=*#3>atq)zO9={_DQP=Dw$8)Qv0?}kyHj&(lAd~Gj)`` zZ|i4B?sase^+WMDpU7`LWljFZ zMbju9i?3x8wvQH>O2V#zrd{DY`}LgH(^M!l5ma>0G!-f!=8)wNX(}4BBm!a|xA|W*wE;pR7d2^T6$dp!;)3#ZYmLtu3dl}?`?S(U zG-eWI5^}yztAGlT--D)_A#&RcnR`$pM^MoTk$Y$=l=BEG2`#djF0t(l;9Y^BIg+^Q z15_;R`Z&)p)9nJVlRiUD2FUJ&{Q;JNkk)Z zP3UbjtpVZ-7c^;Q)!#G{;MvU2aA!spwPqYu5`{9VJV?t*MU^FJsu@+@LsOw(L{QOz z#3oIJ@)1EL5moZ-2Dn5LSB;%7(;eb~#YJYI9J>K>==xbwTr&I!DmvKm98HD7mK?G? zPgBwGn*_Fu+#C%$_$5c$OEi@MP=$*)6slZNGJ~=WUsrzIijHpGnuRG-vR)*dS|p|X zBU(xdPhN(mnt1YUnhJ#{f{G5Fe3zy|`Gue|1fERqq=NU41W(3+rCc-T9J+opDK3d8 zf{G5F+(c8M@Fa&UCuu4gev`x#J|gku?YAm;avx1)06gI$4uvO1-Dx-hhmwm1J4R$& zWV?un%|xTDQG_evNw6A8{RzqcM%n!T&{Q_a=394B0bWOzOq-_{J$931bWOgoiyqwK$v{Ic zdgMUZIZ|Fyt%8b9BC$wQp%RH4vfM>ephJ_jg~AI#MJL|<0!@W-20kzwGH^Fxl_bXr5EE*dBde2bS@`u2A=6at>t3bZ8XfzQ?RYb z{=U7({tEvbv;A#*|Ggva5^QnW#L#`MP{~H2x^xLsQ6pt-@Bu zhg9j--BgM&@TlWt20UW433%$l#q9lK*O2U!d#VJL>Al$5`wEU`vX`SGzt_HMtBgLPr7IBKmy{S6y%p-$ifPPIGO+S_>9d9qU5btyrCKNA$C3Hp{l_DJXA` zv7O~_c%oCV4Upyl{JReR9fE&_M>{MX3QTX=f)6?#E5bKu+I7o6TCX?!$LdF&X4Uen zX4#UziV%33e%19FmgjfEQO~M6kl(%@j=`^rTkeE2@PA)6=%6&8Muo-FEZO^)+9gE? zpEr@OLb%0umOJ4T{D0mp)vR*Ru@cOVasE0XZq@+?7h(qGuOx#t+?!>Og%j0wz1}n% z@P+!MB6mNC2rey8@IkhRy}51L@5tWJ1M+;O72!V#3l{<6J~$5#;P3m9*eI9qm` zfdya3e5Mm^gXI<|*bhVn;XCe*^lF$!*=+k}-C$q;F$(7t?H)1QrV-edapug~haNa% zl)w^p!}L}bqWQ$|9m!z+WzwfrGoT9YxA%pgIJJ2Gr+jdO^G$cvT|WO#%U^->-{-k) z=3{n6!TDIH@>fE;3$67<^c}VV+io!E##t;i_!u=@ z&$#QfaA2%>6l-xGEkOP`qwK7AlRNA85@aG}l69nZcU=LBf}v>dGromYt&PK< z2vr2oOB=pJez^iHhlWARopV>LCOl5{lGcms8fgpGb}B@|U2#Qp2do>}DdUb*#v-zv z*E+W_A+xN5chv5n&JB44lZ>T%>Tji+7*`u+rP(vCS%G<+Q8*}VbA&0ikW~G+s2M3d zzLwIIQ+lDf6oa4`sd=+eP(}b2t2)k_63&G?qrq)8TDIvsi3#SkB2vV4qD*rBc#0a$ z`5M;VQ)@HMdskK0dcw`Eb$%wvT5HvvvV#-qamZQV*pk&@X@Z&y)hfK>v6i~Af=Z^^dI zC~Fs4LtIHQznW56?{S2VCcxrLJuQHDQWiLJ&edgKvX$~Ty-mRBx)45fMH(#Js_$z8g;{$}~hjWx?c^a*Y>%3bZ{Wel39!6*i>YlC5&vKcry zW5Nt46RPA#yG5cd3#6chE0cQDGQ)MyC_9nFJ1H|9(BNM3lkT9u;@`xWq4~}cH$Kag zbp8P~RQ}0I#X(KOtQ8gx+BhhD1JUUu_zg>vPrKeJHmyB#cA@IIjbaOypJBneX;lhG z7N(XU04*Aa89W`HgCFsL!>g`%!V1x-ho?->GHkP1saqApw{U?xuT&aED}}G{FNY8`7#3NO`uWOW>W9OMG^Nbo$DrUt(H) z<mv-YDsA0GmP{&eRY4Q3y>^&RnuyJCHW=$q5iCd5(#MhGj$5YnP zGWKeJ31G=&PtEUNFHcJMIXN5C9G>wkDPvq3hg&8LZFr{S!w@=bh%Cd$-0(mhGUWsD z(CzXfIYt2!HfW4?e_>IK)9xG6d>>|J$;vD7PHL?F4mnmy1;dC`RScsop=1cX zsAdSgk)okNm#WhPc0h$om8E6uKVUl>>&ClI*=B$f;QJv~v;yN$=KOan){0Unej#Oe zDPeFVulpM6*2LtvPg z+}9Fm3EnaAQww#-4-mf0RtwUEk_A}`Ub7`ImMvx`2dR&V3DilIN~=!HCjF(isg#M- zR%n!+G2$K7{Q#EUGLU4F!*rMZZ8gH!qb13aL(cp?t7$>9hWUBnVCL|il2zhCq!dYE zj;HiPGbysM`AFT#p^n9>uKN!4k%JL)@JsS^I@J%&lNi{i7*D;c4l}Utg1-9>>{uy@ zft}V#e`mSAk03b*_G0oYOgOU7nrlv@-QYu)B_!enpO(>r#Vg->tc{N;Ge>j!>rG-f zr;DXA+qsM>9PBZsgRB$+@2EZobqd<@NUqY|^fya$W6g4t^6eV5$wMaKD^9}I@&bHN z1CCNGd$8+qJ$peU@x2>ij6|7~`4cJC^&Tq5_l(NiR-kakQ}AE=x>D&Z>$w= zjlb}LHc*y0+rDM^kx5p4VFJi6VI>!=lsPn%kAnz9C?!(p-%aUF%M35}Gy~pAnc;>! zf=I^FJ@vQ7XE)Xw*Rd}lpe2^t<+|1Or3vEQdzqT=^Ff8zQ!3KZ_^UlNj(1WT-%Bwp zHJonQU&G(pSPdUwZY zL94iOxW@!GbKGRSzFQzy#vQ5XJ5vUt6(~ab9X;)hcT)ELXwOxFUP^R6jINVjA!S>_s56t|Q+jKnpr^|5PDEN`sI2iG_J(mIKEN#ELi-i;L}mJ z0V^+MvtD1%v}f{-Zo;6+LaDrwtCJ?svglv;v?$(5S+w>;)P|HR%}9Tz_}<1k#oN?3 zAfZz%wHqZ^+kkzrRqMjMZ@YnEt>FX@o@*A4%^An%aOdUC*^}&TYHv{-O%_VouOGyw zhoDh*HjH;tHa(&*jASd_Pk)mfWIWL>nIk(sEZ-`F2Dua8MPuN*Xc$b!+nF;5mHPS_ zF{IQYDdj^cZPDC|sB~XX<>Q@{@~`d9xMU;UuD`O+Y^<^icu*GA{2)BdcN?bUO* zJgeowrn|CJf&*_sHiVIf+yuYH`8>{FqvDcZPrqAHZ zP}m6rKnsC2u#oDr(2t?W!fg}~MOer&n+Ckj!&(f3jX{e*%A^hr8nRPVF_NEF6DOoTe4X9$!mE{kN1u(EcS( zd%TQ|;?o|rzgB2=ZSS8u0|z;9xJ1h_6$1|md6o|wc750m1=&0tOED{EVDbnB6O??7 zq%^PPk%sMb`BM$E3j?9lxeP>m$Sa^V@Q~=(Pjn!JZuI-C5kJk9-J-yst;0dGW*Gr8 zeKZYX7Hn|q@LJ<=G=^_1zv>!zAU}aeu+{6PJMT0r)*9|XZ@SUXnW%Ukw$yp+aC4iu z8C~y!8sdbRA6MGZbfdDU9`=Ovd@BfG@xOB!rdRgwGl1G-Pe%eON7A;;&(B9`;+9t~ zb(A^Wbs&KXZKNmT!7z4OX#&p1O=C025#^Z_g3OHrk=c=#%sU1mGpmr<6mId%3(D!K z(7nCltirr)y>6X25WvyQd9&^;Hy6ub4$HgEfHQlrTXi0m=@xH7YFQcg`{!Q@0-lN_Rwt`k~Y1 zfB=K*e$b*{>g@^P7-ViR>+FPE?VwTbMDJ2rLhW-0xmv1J4$Xpndw6x6SkId+XC97d z@H40|-~@6jRbXd_1DF?8F{5*>=E7=)W*gJyD z;*fm8(@Z$x0Vilfcx)4x_0GL;8ZfE@Z+C&Cc}Q<0oGG_GI4>I%MPEzdwx|T(=+IdT zC&Y+^qqDoOOA8!$YH$PB`|{Y~O0^T;UjBSLrO-GfUy5fO^-1o+bgj%oaRTR)YiQKYQ{fCA(QL6dPe4OB{1WcvC03nnk@t(}#zl5A#=_g%MFTu4 z!wR-RQtx_jOwn4kK%JYzDch{am!sh>Ys~>I2JrmAk1CCZqZO;vUWWc=E|^}EZBz_k zm7@VAlVFarg z4tIzZE!oQ`>!lO!Pu)p%QF|JkJe=YEcDLS_ypd`g?nD`UP?QZS!ZQb*aC_ZsF1O8P zerTr?Znm18@CsNla=l8iVzt0?fQ-WwI|h-U3WumcBTgl92sr;dgUdigSrl%DQ`_P8 zO1sf2#x4npWd)e#d$7G5O}B%UFlR^W%c7QVwc)YE8jS9xa3^dxbA79*wq#yX4vMRe z@4%84_EoGnzFDfHAcD*BXsE!j+XlkG+@s0H9aP^gsugj0((k}w^{em~5P?I-Y;0Oh z?5zSOp9V+ivJM;qvs&|SKK2&S=Zse^r@F@y+5MrY zd(6tvWmhb3!LfoWqXB?$6yXwxO$KHOMo}fOJNHh6`@tt0#dvVW0u=ogddTH)r@!6| z%(bHJEZY!zYy{pq9<~j)`4D}A;$v+X)jlZd3U`sXlk8NA0W16G3%2wd2WK;gGbscKFGeHRxqF%<#E5R$-UV9Hs6 zF%D7Fw=H=8z~ykZ47FB@7GzTOBJ4#*++|$v!k!V0eXCyOn07hbVl`Lc;9V)4;Amo@ zkL`qHiY^52D; z1!KHc!f5t_{h4++iT7B-g<*o-y==b#S(kyAtlqT!8T$_Vb~vLwCf>Io50Q)HK7L8V z?d*LB>U<^~b$ob`$lmi)`x=X);Rk+9&fj?cDP;jR;Iu0aORSSRM06e zK~Re7aYLykIFeW?cz2`Z*5d)d3a?BR#m7D&k4rJnJAu)i<#3-JI!*v WUBN-Jj5r4e!~om{4)P>=;r|1BWN^L! delta 213 zcmbPullh%EYXj@l$J`rPr!h|6%q%%M?4cA>hTddFOEt;T46O{E44Dk=3|ViE45!Do9&<7V^+vW&PXWk j;mk|SO)aS`NG+aHJ0&Bzvp7QmNGD`)7jL$I9mEI#&;d!S diff --git a/docs/_build/html/.doctrees/docs/usage.doctree b/docs/_build/html/.doctrees/docs/usage.doctree index 1e56e2cde0f2db7355a610ef5ad4ff60c6b79c89..e09d97d85ee3c795f976d3f72647b1a2530f99d2 100644 GIT binary patch delta 1552 zcmeHH&rcIU6z)PBLyJf#kjjs+-NXbU{Rv#E8`DEg2oV|+5|xn6ww*H5?(QrzvqfSE zdf-3;rzt1YD~T7Q{R8yG(LX`r&7&T@IkQ_44{G$Pb9l*q?|bvTZ)Wy=ES=tsJdPaQ zn0Xa>7Yo1X35B@QALnNSU#7M&Y1R-iT$4dvOJ^#;sY2T%%~aK_LeNPL9w{++X}t|c z?)_om^ByI~-wut3o8JYa{MpFq??icWu+%9J&nkagc}?M?iS@NPIM#?Op%5I0y

T zW}dWn+S4#{FtJLJv!SlIz$q@Ks@N9pLB#@UvAmdW2X22&;!hJrW!ZQ4>4Lk0>7rsy^pN^_x*|-KQ!!d23Ya1?0Eu?JZ zAZ^x9R!j;Ek65|}8FOhNlM!`kEZmqVE!<>UxlETT5=Ikj>zJlRJBVOsO1K>IZMoi_r&nwhkbcv zv-3VNq$o!sulI8~k|yAFIJrbSg|V^T*qJ~M1^^+L@Z@B?LncpqS6>UT@lu9C87|B4 sLWUC&yn{YD9OQ3ED_gVWMR3$DqfHA)s9yd-phsibNE24+foB#j- delta 515 zcmX>W^um?3fpzL#rH!oTxEWtgZje@(yr1vAXJKh-aY<%=UU7UuVo8RrQc7Y;qCQ+u zFRM5|FGop-O92Q<5|eULQPe?1Ac{7h<)6yPk-;82B||iVHs;S`VlLBS(uv8~lxU)1v Xp>~P}SVsmoSb4_mtj^7^)uR~!=FG3@ diff --git a/docs/_build/html/_modules/index.html b/docs/_build/html/_modules/index.html index 50938ce..42e6c66 100644 --- a/docs/_build/html/_modules/index.html +++ b/docs/_build/html/_modules/index.html @@ -4,20 +4,20 @@ - Overview: module code — LLMSQL 0.1.13 documentation + Overview: module code — LLMSQL 0.1.16 documentation - + - - + + - + - +

+
- +

All modules for which code is available

@@ -73,11 +72,11 @@

Navigation

  • modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Overview: module code
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Overview: module code
    • - + \ No newline at end of file diff --git a/docs/_build/html/_modules/llmsql/evaluation/evaluate.html b/docs/_build/html/_modules/llmsql/evaluation/evaluate.html index 277b177..c63fc3f 100644 --- a/docs/_build/html/_modules/llmsql/evaluation/evaluate.html +++ b/docs/_build/html/_modules/llmsql/evaluation/evaluate.html @@ -4,20 +4,20 @@ - llmsql.evaluation.evaluate — LLMSQL 0.1.13 documentation + llmsql.evaluation.evaluate — LLMSQL 0.1.16 documentation - + - - + + - + - + +
    - +

    Source code for llmsql.evaluation.evaluate

     """
 LLMSQL Evaluation Module
@@ -51,17 +51,19 @@ 
    Source code for llmsql.evaluation.evaluate
     """
 
 from datetime import datetime, timezone
-from pathlib import Path
 import uuid
 
 from rich.progress import track
 
-from llmsql.config.config import DEFAULT_WORKDIR_PATH
+from llmsql.config.config import (
+    DEFAULT_LLMSQL_VERSION,
+    get_repo_id,
+)
 from llmsql.utils.evaluation_utils import (
     connect_sqlite,
-    download_benchmark_file,
     evaluate_sample,
 )
+from llmsql.utils.inference_utils import _maybe_download, resolve_workdir_path
 from llmsql.utils.rich_utils import log_mismatch, print_summary
 from llmsql.utils.utils import load_jsonl, load_jsonl_dict_by_key, save_json_report
 
@@ -69,11 +71,10 @@ 
    Source code for llmsql.evaluation.evaluate
     
    
 [docs]
 def evaluate(
-    outputs,
+    outputs: str | list[dict[int, str | int]],
     *,
-    workdir_path: str | None = DEFAULT_WORKDIR_PATH,
-    questions_path: str | None = None,
-    db_path: str | None = None,
+    version: str = DEFAULT_LLMSQL_VERSION,
+    workdir_path: str | None = None,
     save_report: str | None = None,
     show_mismatches: bool = True,
     max_mismatches: int = 5,
@@ -82,10 +83,10 @@ 
    Source code for llmsql.evaluation.evaluate
         Evaluate predicted SQL queries against the LLMSQL benchmark.
 
     Args:
+        version: LLMSQL version
         outputs: Either a JSONL file path or a list of dicts.
-        workdir_path: Directory for auto-downloads (ignored if all paths provided).
-        questions_path: Manual path to benchmark questions JSONL.
-        db_path: Manual path to SQLite benchmark DB.
+        workdir_path: Directory to store downloaded benchmark files. If omitted, a
+            temporary directory is created automatically.
         save_report: Optional manual save path. If None → auto-generated.
         show_mismatches: Print mismatches while evaluating.
         max_mismatches: Max mismatches to print.
@@ -96,37 +97,12 @@ 
    Source code for llmsql.evaluation.evaluate
     
     # Determine input type
     input_mode = "jsonl_path" if isinstance(outputs, str) else "dict_list"
+    workdir = resolve_workdir_path(workdir_path)
 
-    # --- Resolve inputs if needed ---
-    workdir = Path(workdir_path) if workdir_path else None
-    if workdir_path and (questions_path is None or db_path is None):
-        workdir.mkdir(parents=True, exist_ok=True)
-
-    if questions_path is None:
-        if workdir is None:
-            raise ValueError(
-                "questions_path not provided, and workdir_path disabled. "
-                "Enable workdir or provide questions_path explicitly."
-            )
-        local_q = workdir / "questions.jsonl"
-        questions_path = (
-            str(local_q)
-            if local_q.is_file()
-            else download_benchmark_file("questions.jsonl", workdir)
-        )
+    repo_id = get_repo_id(version)
 
-    if db_path is None:
-        if workdir is None:
-            raise ValueError(
-                "db_path not provided, and workdir_path disabled. "
-                "Enable workdir or provide db_path explicitly."
-            )
-        local_db = workdir / "sqlite_tables.db"
-        db_path = (
-            str(local_db)
-            if local_db.is_file()
-            else download_benchmark_file("sqlite_tables.db", workdir)
-        )
+    questions_path = _maybe_download(repo_id, "questions.jsonl", workdir)
+    db_path = _maybe_download(repo_id, "sqlite_tables.db", workdir)
 
     # --- Load benchmark questions ---
     questions = load_jsonl_dict_by_key(questions_path, key="question_id")
@@ -226,12 +202,12 @@ 
    Navigation
    
         
  • 
           modules |
    • 
-        
  • LLMSQL 0.1.13 documentation »
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
           
  • Module code »
    • 
-        
  • llmsql.evaluation.evaluate
    • 
+        
  • llmsql.evaluation.evaluate
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/_modules/llmsql/inference/inference_transformers.html b/docs/_build/html/_modules/llmsql/inference/inference_transformers.html
index ec1fdd8..47db67d 100644
--- a/docs/_build/html/_modules/llmsql/inference/inference_transformers.html
+++ b/docs/_build/html/_modules/llmsql/inference/inference_transformers.html
@@ -4,20 +4,20 @@
   
     
     
-    llmsql.inference.inference_transformers — LLMSQL 0.1.13 documentation
+    llmsql.inference.inference_transformers — LLMSQL 0.1.16 documentation
     
-    
+    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
-    
+     
   
     
+    
      
 
     
    
       
    
         
    
           
    
-
+            
   
    Source code for llmsql.inference.inference_transformers
     """
 LLMSQL Transformers Inference Function
@@ -56,17 +56,16 @@ 
    Source code for llmsql.inference.inference_transformers
        results = inference_transformers(
         model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
+        repo_id="llmsql-bench/llmsql-2.0",
         output_file="outputs/preds_transformers.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
         temperature=0.7,
-        model_args={
+        model_kwargs={
             "torch_dtype": "bfloat16",
         },
-        generate_kwargs={
+        generation_kwargs={
             "do_sample": False,
         },
     )
@@ -80,17 +79,23 @@ 
    Source code for llmsql.inference.inference_transformers
    """
 
-from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 
 from dotenv import load_dotenv
 import torch
 from tqdm import tqdm
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
-from llmsql.config.config import DEFAULT_WORKDIR_PATH
+from llmsql.config.config import (
+    DEFAULT_LLMSQL_VERSION,
+    get_repo_id,
+)
 from llmsql.loggers.logging_config import log
-from llmsql.utils.inference_utils import _maybe_download, _setup_seed
+from llmsql.utils.inference_utils import (
+    _maybe_download,
+    _setup_seed,
+    resolve_workdir_path,
+)
 from llmsql.utils.utils import (
     choose_prompt_builder,
     load_jsonl,
@@ -129,12 +134,12 @@ 
    Source code for llmsql.inference.inference_transformers
    top_k: int = 50,
     generation_kwargs: dict[str, Any] | None = None,
     # --- Benchmark Parameters ---
+    version: Literal["1.0", "2.0"] = DEFAULT_LLMSQL_VERSION,
     output_file: str = "llm_sql_predictions.jsonl",
-    questions_path: str | None = None,
-    tables_path: str | None = None,
-    workdir_path: str = DEFAULT_WORKDIR_PATH,
+    workdir_path: str | None = None,
     num_fewshots: int = 5,
     batch_size: int = 8,
+    limit: int | float | None = None,
     seed: int = 42,
 ) -> list[dict[str, str]]:
     """
@@ -172,13 +177,16 @@ 
    Source code for llmsql.inference.inference_transformers
                              'top_p', 'top_k' are handled separately.
 
         # Benchmark:
+        version: LLMSQL version
         output_file: Output JSONL file path for completions.
-        questions_path: Path to benchmark questions JSONL.
-        tables_path: Path to benchmark tables JSONL.
-        workdir_path: Working directory path.
+        workdir_path: Directory to store downloaded benchmark files. If omitted, a
+            temporary directory is created automatically.
         num_fewshots: Number of few-shot examples (0, 1, or 5).
         batch_size: Batch size for inference.
         seed: Random seed for reproducibility.
+        limit: Limit the number of questions to evaluate. If an integer, evaluates
+               the first N samples. If a float between 0.0 and 1.0, evaluates the
+               first X*100% of samples. If None, evaluates all samples (default).
 
     Returns:
         List of generated SQL results with metadata.
@@ -186,9 +194,6 @@ 
    Source code for llmsql.inference.inference_transformers
    # --- Setup ---
     _setup_seed(seed=seed)
 
-    workdir = Path(workdir_path)
-    workdir.mkdir(parents=True, exist_ok=True)
-
     model_kwargs = model_kwargs or {}
     tokenizer_kwargs = tokenizer_kwargs or {}
     generation_kwargs = generation_kwargs or {}
@@ -252,13 +257,33 @@ 
    Source code for llmsql.inference.inference_transformers
    model.eval()
 
     # --- Load necessary files ---
-    questions_path = _maybe_download("questions.jsonl", questions_path)
-    tables_path = _maybe_download("tables.jsonl", tables_path)
+    workdir = resolve_workdir_path(workdir_path)
+    repo_id = get_repo_id(version)
+
+    questions_path = _maybe_download(repo_id, "questions.jsonl", workdir)
+    tables_path = _maybe_download(repo_id, "tables.jsonl", workdir)
 
     questions = load_jsonl(questions_path)
     tables_list = load_jsonl(tables_path)
     tables = {t["table_id"]: t for t in tables_list}
 
+    # --- Apply limit ---
+    if limit is not None:
+        if isinstance(limit, float):
+            if not (0.0 < limit <= 1.0):
+                raise ValueError(
+                    f"When a float, `limit` must be between 0.0 and 1.0, got {limit}."
+                )
+            limit = max(1, int(len(questions) * limit))
+        if not isinstance(limit, int) or limit < 1:
+            raise ValueError(
+                f"`limit` must be a positive integer or a float in (0.0, 1.0], got {limit!r}."
+            )
+        log.info(
+            f"Limiting evaluation to first {limit} questions out of {len(questions)}"
+        )
+        questions = questions[:limit]
+
     # --- Chat template setup ---
     use_chat_template = chat_template or getattr(tokenizer, "chat_template", None)
     if use_chat_template:
@@ -362,12 +387,12 @@ 
    Navigation
    
         
  • 
           modules |
    • 
-        
  • LLMSQL 0.1.13 documentation »
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
           
  • Module code »
    • 
-        
  • llmsql.inference.inference_transformers
    • 
+        
  • llmsql.inference.inference_transformers
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/_sources/docs/evaluation.rst.txt b/docs/_build/html/_sources/docs/evaluation.rst.txt
index 8b98b15..fa60e38 100644
--- a/docs/_build/html/_sources/docs/evaluation.rst.txt
+++ b/docs/_build/html/_sources/docs/evaluation.rst.txt
@@ -36,15 +36,13 @@ Evaluate from a list of Python dicts:
     report = evaluate(predictions)
     print(report)
 
-Providing your own DB and questions (skip workdir):
+Using a persistent cache directory for benchmark downloads:
 
 .. code-block:: python
 
     report = evaluate(
         "path_to_outputs.jsonl",
-        questions_path="bench/questions.jsonl",
-        db_path="bench/sqlite_tables.db",
-        workdir_path=None
+        workdir_path="./benchmark-cache",
     )
 
 Function Arguments
@@ -59,11 +57,7 @@ Function Arguments
    * - outputs
      - Path to JSONL file or a list of prediction dicts (required).
    * - workdir_path
-     - Directory for automatic benchmark downloads. Ignored if both questions_path and db_path are provided. Default: "llmsql_workdir".
-   * - questions_path
-     - Optional path to benchmark questions JSONL file.
-   * - db_path
-     - Optional path to SQLite DB with evaluation tables.
+     - Directory used to cache downloaded benchmark files. If omitted, a temporary directory is created automatically.
    * - save_report
      - Path to save detailed JSON report. Defaults to "evaluation_results_{uuid}.json".
    * - show_mismatches
diff --git a/docs/_build/html/_sources/docs/index.rst.txt b/docs/_build/html/_sources/docs/index.rst.txt
index b2760cd..fe3654c 100644
--- a/docs/_build/html/_sources/docs/index.rst.txt
+++ b/docs/_build/html/_sources/docs/index.rst.txt
@@ -36,8 +36,6 @@ Example: Running your first evaluation (with transformers backend)
     results = inference_transformers(
         model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
         output_file="outputs/preds_transformers.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
diff --git a/docs/_build/html/_sources/docs/inference.rst.txt b/docs/_build/html/_sources/docs/inference.rst.txt
index 5bcf0c6..1a33533 100644
--- a/docs/_build/html/_sources/docs/inference.rst.txt
+++ b/docs/_build/html/_sources/docs/inference.rst.txt
@@ -14,6 +14,12 @@ Inference API Reference
 
 ---
 
+.. automodule:: llmsql.inference.inference_api
+   :members:
+   :undoc-members:
+
+---
+
 .. raw:: html
 
    
    
diff --git a/docs/_build/html/_sources/docs/usage.rst.txt b/docs/_build/html/_sources/docs/usage.rst.txt
index 806a4e8..767fad6 100644
--- a/docs/_build/html/_sources/docs/usage.rst.txt
+++ b/docs/_build/html/_sources/docs/usage.rst.txt
@@ -27,8 +27,7 @@ Using transformers backend.
     results = inference_transformers(
         model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
         output_file="outputs/preds_transformers.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
+        workdir_path="./benchmark-cache",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
@@ -58,8 +57,7 @@ Using vllm backend.
     results = inference_vllm(
         model_name="Qwen/Qwen2.5-1.5B-Instruct",
         output_file="outputs/preds_vllm.jsonl",
-        questions_path="data/questions.jsonl",
-        tables_path="data/tables.jsonl",
+        workdir_path="./benchmark-cache",
         num_fewshots=5,
         batch_size=8,
         max_new_tokens=256,
@@ -77,6 +75,41 @@ Using vllm backend.
     print(report)
 
 
+Using OpenAI-compateble API.
+
+.. code-block:: python
+
+    from llmsql import inference_api
+    from dotenv import load_dotenv
+    import os
+    load_dotenv()
+
+    # Run inference (will take some time)
+    results = inference_api(
+        model_name="gpt-5-mini",
+        base_url="https://api.openai.com/v1/",
+        api_key=os.environ["OPENAI_API_KEY"],
+        api_kwargs={
+            "response_format": {
+                    "type": "text"
+                },
+                "verbosity": "medium",
+                "reasoning_effort": "medium",
+                "store": False
+        },
+        requests_per_minute=100,
+        output_file="test_output_api.jsonl",
+        limit=50,
+        num_fewshots = 5,
+        seed=42,
+        version="2.0"
+    )
+
+    # Evaluate the results
+    evaluator = LLMSQLEvaluator()
+    report = evaluator.evaluate(outputs_path="outputs/preds_transformers.jsonl")
+    print(report)
+
 ---
 
 .. raw:: html
diff --git a/docs/_build/html/_static/documentation_options.js b/docs/_build/html/_static/documentation_options.js
index eede5b1..5525413 100644
--- a/docs/_build/html/_static/documentation_options.js
+++ b/docs/_build/html/_static/documentation_options.js
@@ -1,5 +1,5 @@
 const DOCUMENTATION_OPTIONS = {
-    VERSION: '0.1.15',
+    VERSION: '0.1.16',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',
@@ -10,4 +10,4 @@ const DOCUMENTATION_OPTIONS = {
     NAVIGATION_WITH_KEYS: false,
     SHOW_SEARCH_SUMMARY: true,
     ENABLE_SEARCH_SHORTCUTS: true,
-};
+};
\ No newline at end of file
diff --git a/docs/_build/html/_static/scripts/front_page.js b/docs/_build/html/_static/scripts/front_page.js
index 1e0c0dc..4f2cdc5 100644
--- a/docs/_build/html/_static/scripts/front_page.js
+++ b/docs/_build/html/_static/scripts/front_page.js
@@ -87,10 +87,8 @@ function renderLeaderboard(rows) {
   rows.forEach((row, i) => {
     const tr = document.createElement('tr');
 
-    // Берём только вторую часть после слеша
     const modelName = row.model.includes('/') ? row.model.split('/')[1] : row.model;
 
-    // Модель с ссылкой
     const modelCell = document.createElement('td');
     if (row.url) {
       const a = document.createElement('a');
@@ -116,7 +114,7 @@ function renderLeaderboard(rows) {
     barContainer.appendChild(text);
     accuracyCell.appendChild(barContainer);
 
-    // Вставка остальных ячеек
+
     tr.innerHTML += `${i+1}`;
     tr.appendChild(modelCell);
     tr.innerHTML += `
diff --git a/docs/_build/html/_static/styles/front_page.css b/docs/_build/html/_static/styles/front_page.css
index 1d3bcfb..45e3f4b 100644
--- a/docs/_build/html/_static/styles/front_page.css
+++ b/docs/_build/html/_static/styles/front_page.css
@@ -1,26 +1,29 @@
 /* === LLMSQL Front Page CSS === */
 
+/* Three-column page layout: nav | content | TOC */
+.page-layout {
+  display: grid;
+  grid-template-columns: 180px minmax(0, 1fr) 200px;
+  gap: 32px;
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 28px 24px;
+  align-items: start;
+}
+
 .sidebar {
-  position: fixed;
-  top: 16px;
-  left: 16px;
-  height: auto;
-  width: 160px;
-  background-color: #f4f4f4;
-  border: 1px solid #e0e0e0;
-  border-radius: 8px;
-  padding: 12px;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  z-index: 110;
-  box-shadow: 0 2px 6px rgba(0,0,0,0.04);
+  position: sticky;
+  top: 24px;
+  background-color: #f8f9fa;
+  border: 1px solid #e8e8e8;
+  border-radius: 10px;
+  padding: 14px;
+  box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04);
 }
 
 .sidebar-content {
   display: flex;
   flex-direction: column;
-  align-items: center;
   gap: 10px;
   width: 100%;
 }
@@ -30,10 +33,11 @@
   text-align: center;
   background-color: #eef6ff;
   color: #0056b3;
-  padding: 10px;
+  padding: 10px 12px;
   border-radius: 6px;
   text-decoration: none;
   font-weight: 600;
+  font-size: 0.9rem;
   transition: background-color 0.2s, color 0.2s;
 }
 .sidebar-button:hover {
@@ -41,12 +45,19 @@
   color: white;
 }
 
+/* Standalone back-link on docs pages (no .page-layout wrapper) */
+body > .sidebar-button,
+.document .sidebar-button {
+  display: inline-block;
+  margin-bottom: 1.5rem;
+}
+
 .sidebar-search {
   width: 100%;
   padding: 8px 10px;
   border-radius: 6px;
   border: 1px solid #ccc;
-  font-size: 0.9rem;
+  font-size: 0.85rem;
   box-sizing: border-box;
   transition: border-color 0.2s ease, box-shadow 0.2s ease;
 }
@@ -64,41 +75,60 @@
 }
 
 .on-this-page {
-  position: fixed;
-  top: 16px;
-  right: 16px;
-  width: 220px;
-  background: #fafafa;
-  border: 1px solid #eee;
-  border-radius: 8px;
-  padding: 12px;
-  font-size: 0.95rem;
-  z-index: 105;
-  box-shadow: 0 2px 6px rgba(0,0,0,0.04);
+  position: sticky;
+  top: 24px;
+  background: #f8f9fa;
+  border: 1px solid #e8e8e8;
+  border-radius: 10px;
+  padding: 14px 16px;
+  font-size: 0.88rem;
+  box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04);
 }
 .on-this-page h4 {
-  margin: 0 0 8px 0;
-  padding-bottom: 6px;
-  border-bottom: 1px solid #e9e9e9;
-  font-size: 0.95rem;
+  margin: 0 0 10px 0;
+  padding-bottom: 8px;
+  border-bottom: 1px solid #e0e0e0;
+  font-size: 0.8rem;
+  font-weight: 700;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  color: #555;
 }
 .on-this-page ul {
   list-style: none;
   padding: 0;
-  margin: 8px 0 0 0;
+  margin: 0;
 }
 .on-this-page ul li {
   margin-bottom: 6px;
 }
+.on-this-page ul li a {
+  color: #444;
+  text-decoration: none;
+  line-height: 1.4;
+}
+.on-this-page ul li a:hover {
+  color: #007bff;
+}
 
 body {
   font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
   line-height: 1.6;
-  margin: 0 auto;
-  padding: 28px 36px;
+  margin: 0;
+  padding: 0;
   color: #333;
   background-color: #fff;
-  max-width: 1100px;
+}
+
+.page-layout > main {
+  min-width: 0;
+}
+
+/* Sphinx docs pages (no .page-layout wrapper) */
+body > .document {
+  max-width: 900px;
+  margin: 0 auto;
+  padding: 28px 24px;
 }
 
 .center-content {
@@ -227,12 +257,24 @@ div.highlight span {
 }
 
 /* Responsive */
-@media (max-width: 1000px) {
-  .on-this-page { position: static; width: auto; margin: 12px 0 18px; }
-}
-
-@media (max-width: 720px) {
-  .sidebar, .on-this-page { display: none; }
+@media (max-width: 1100px) {
+  .page-layout {
+    grid-template-columns: 160px minmax(0, 1fr);
+  }
+  .on-this-page {
+    display: none;
+  }
+}
+
+@media (max-width: 768px) {
+  .page-layout {
+    grid-template-columns: 1fr;
+    gap: 16px;
+    padding: 16px;
+  }
+  .sidebar {
+    position: static;
+  }
   h1 { font-size: 1.5rem; }
   .badges img { height: 18px; }
 }
diff --git a/docs/_build/html/docs/evaluation.html b/docs/_build/html/docs/evaluation.html
index 22d13e4..47ebb12 100644
--- a/docs/_build/html/docs/evaluation.html
+++ b/docs/_build/html/docs/evaluation.html
@@ -5,23 +5,21 @@
     
     
 
-    Evaluation API Reference — LLMSQL 0.1.15 documentation
+    Evaluation API Reference — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
-
-
+            
   
    
 
    Evaluation API Reference
    
 
    The evaluate() function allows you to benchmark Text-to-SQL model outputs
@@ -79,12 +77,10 @@ 
    Usage Examplesprint(report)
 
    
 
    
-
    Providing your own DB and questions (skip workdir):
    
+
    Using a persistent cache directory for benchmark downloads:
    
 
    report = evaluate(
     "path_to_outputs.jsonl",
-    questions_path="bench/questions.jsonl",
-    db_path="bench/sqlite_tables.db",
-    workdir_path=None
+    workdir_path="./benchmark-cache",
 )
 
    
 
    
@@ -106,13 +102,7 @@ 
    Function Arguments
    workdir_path
    
-
    Directory for automatic benchmark downloads. Ignored if both questions_path and db_path are provided. Default: “llmsql_workdir”.
    
-
-
    questions_path
    
-
    Optional path to benchmark questions JSONL file.
    
-
-
    db_path
    
-
    Optional path to SQLite DB with evaluation tables.
    
+
    Directory used to cache downloaded benchmark files. If omitted, a temporary directory is created automatically.
    
 
 
    save_report
    
 
    Path to save detailed JSON report. Defaults to “evaluation_results_{uuid}.json”.
    
@@ -155,6 +145,37 @@ 
    Report Saving
+
    LLMSQL Evaluation Module
    
+
    Provides the evaluate() function to benchmark Text-to-SQL model outputs
+on the LLMSQL benchmark.
    
+
    See the documentation for full usage details.
    
+
+
    
+
    
+llmsql.evaluation.evaluate.evaluate(outputs: str | list[dict[int, str | int]], *, version: str = '2.0', workdir_path: str | None = None, save_report: str | None = None, show_mismatches: bool = True, max_mismatches: int = 5) → dict[source]
    
+
    Evaluate predicted SQL queries against the LLMSQL benchmark.
    
+
    
+
    Parameters:
    
+
      
+
    • version – LLMSQL version
      • 
+
    • outputs – Either a JSONL file path or a list of dicts.
      • 
+
    • workdir_path – Directory to store downloaded benchmark files. If omitted, a
+temporary directory is created automatically.
      • 
+
    • save_report – Optional manual save path. If None → auto-generated.
      • 
+
    • show_mismatches – Print mismatches while evaluating.
      • 
+
    • max_mismatches – Max mismatches to print.
      • 
+
    
+
    
+
    Returns:
    
+
    Metrics and mismatches.
    
+
    
+
    Return type:
    
+
    dict
    
+
    
+
    
+
    
+
 
    —
    
 
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/docs/index.html b/docs/_build/html/docs/index.html
index 9c26bf9..f4406e8 100644
--- a/docs/_build/html/docs/index.html
+++ b/docs/_build/html/docs/index.html
@@ -5,24 +5,22 @@
     
     
 
-    LLMSQL package Documentation — LLMSQL 0.1.15 documentation
+    LLMSQL package Documentation — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
-
-
+            
   
    
 
    LLMSQL package Documentation
    
 
@@ -73,8 +72,6 @@ 
    Example: Running your first evaluation (with transformers backend)results = inference_transformers(
     model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
     output_file="outputs/preds_transformers.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
     num_fewshots=5,
     batch_size=8,
     max_new_tokens=256,
@@ -163,17 +160,20 @@ 
    Navigation
    
         
  • 
           index
    • 
+        
  • 
+          modules |
    • 
         
  • 
           next |
    • 
         
  • 
           previous |
    • 
-        
  • LLMSQL 0.1.15 documentation »
    • 
-        
  • LLMSQL package Documentation
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
+        
  • LLMSQL package Documentation
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/docs/inference.html b/docs/_build/html/docs/inference.html
index 2aa340f..ee04295 100644
--- a/docs/_build/html/docs/inference.html
+++ b/docs/_build/html/docs/inference.html
@@ -5,24 +5,22 @@
     
     
 
-    Inference API Reference — LLMSQL 0.1.15 documentation
+    Inference API Reference — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
+            
+  
    
+
    Inference API Reference
    
+
    
+
    LLMSQL Transformers Inference Function
    
+
    This module provides a single function inference_transformers() that performs
+text-to-SQL generation using large language models via the Transformers backend.
    
+
    Example
    
+
    from llmsql.inference import inference_transformers
 
-  
    
-
    Inference API Reference
    
+results = inference_transformers(
+    model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
+    repo_id="llmsql-bench/llmsql-2.0",
+    output_file="outputs/preds_transformers.jsonl",
+    num_fewshots=5,
+    batch_size=8,
+    max_new_tokens=256,
+    temperature=0.7,
+    model_kwargs={
+        "torch_dtype": "bfloat16",
+    },
+    generation_kwargs={
+        "do_sample": False,
+    },
+)
+
    
+
    
+
    Notes
    
+
    This function uses the HuggingFace Transformers backend and may produce
+slightly different outputs than the vLLM backend even with the same inputs
+due to differences in implementation and numerical precision.
    
+
    
+
    
+
    
+llmsql.inference.inference_transformers.inference_transformers(model_or_model_name_or_path: str | AutoModelForCausalLM, tokenizer_or_name: str | Any | None = None, *, trust_remote_code: bool = True, dtype: dtype = torch.float16, device_map: str | dict[str, int] | None = 'auto', hf_token: str | None = None, model_kwargs: dict[str, Any] | None = None, tokenizer_kwargs: dict[str, Any] | None = None, chat_template: str | None = None, max_new_tokens: int = 256, temperature: float = 0.0, do_sample: bool = False, top_p: float = 1.0, top_k: int = 50, generation_kwargs: dict[str, Any] | None = None, version: Literal['1.0', '2.0'] = '2.0', output_file: str = 'llm_sql_predictions.jsonl', workdir_path: str | None = None, num_fewshots: int = 5, batch_size: int = 8, limit: int | float | None = None, seed: int = 42) → list[dict[str, str]][source]
    
+
    Inference a causal model (Transformers) on the LLMSQL benchmark.
    
+
    
+
    Parameters:
    
+
      
+
    • model_or_model_name_or_path – Model object or HF model name/path.
      • 
+
    • tokenizer_or_name – Tokenizer object or HF tokenizer name/path.
      • 
+
    • Loading (# Tokenizer)
      • 
+
    • trust_remote_code – Whether to trust remote code (default: True).
      • 
+
    • dtype – Torch dtype for model (default: float16).
      • 
+
    • device_map – Device placement strategy (default: “auto”).
      • 
+
    • hf_token – Hugging Face authentication token.
      • 
+
    • model_kwargs – Additional arguments for AutoModelForCausalLM.from_pretrained().
+Note: ‘dtype’, ‘device_map’, ‘trust_remote_code’, ‘token’
+are handled separately and will override values here.
      • 
+
    • Loading
      • 
+
    • tokenizer_kwargs – Additional arguments for AutoTokenizer.from_pretrained(). ‘padding_side’ defaults to “left”.
+Note: ‘trust_remote_code’, ‘token’ are handled separately and will override values here.
      • 
+
    • Chat (# Prompt &)
      • 
+
    • chat_template – Optional chat template to apply before tokenization.
      • 
+
    • Generation (#)
      • 
+
    • max_new_tokens – Maximum tokens to generate per sequence.
      • 
+
    • temperature – Sampling temperature (0.0 = greedy).
      • 
+
    • do_sample – Whether to use sampling vs greedy decoding.
      • 
+
    • top_p – Nucleus sampling parameter.
      • 
+
    • top_k – Top-k sampling parameter.
      • 
+
    • generation_kwargs – Additional arguments for model.generate().
+Note: ‘max_new_tokens’, ‘temperature’, ‘do_sample’,
+‘top_p’, ‘top_k’ are handled separately.
      • 
+
    • Benchmark (#)
      • 
+
    • version – LLMSQL version
      • 
+
    • output_file – Output JSONL file path for completions.
      • 
+
    • workdir_path – Directory to store downloaded benchmark files. If omitted, a
+temporary directory is created automatically.
      • 
+
    • num_fewshots – Number of few-shot examples (0, 1, or 5).
      • 
+
    • batch_size – Batch size for inference.
      • 
+
    • seed – Random seed for reproducibility.
      • 
+
    • limit – Limit the number of questions to evaluate. If an integer, evaluates
+the first N samples. If a float between 0.0 and 1.0, evaluates the
+first X*100% of samples. If None, evaluates all samples (default).
      • 
+
    
+
    
+
    Returns:
    
+
    List of generated SQL results with metadata.
    
+
    
+
    
+
    
 
-  
    
-
    Inference API Reference
    
+
    —
    
 
    —
    
 
    —
    
 
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/docs/usage.html b/docs/_build/html/docs/usage.html
index 2e73e7a..9c2d572 100644
--- a/docs/_build/html/docs/usage.html
+++ b/docs/_build/html/docs/usage.html
@@ -5,24 +5,22 @@
     
     
 
-    Usage Overview — LLMSQL 0.1.15 documentation
+    Usage Overview — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
     
     
-    
-    
+     
   
     
-    
    
+    
      
 
     
    
       
    
         
    
           
    
-
-
+            
   
    
 
    Usage Overview
    
 
    LLMSQL package provides two primary components:
    
@@ -75,18 +73,15 @@ 
    Basic Exampleresults = inference_transformers(
     model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
     output_file="outputs/preds_transformers.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
+    workdir_path="./benchmark-cache",
     num_fewshots=5,
     batch_size=8,
     max_new_tokens=256,
     temperature=0.7,
-    model_kwargs={
     model_kwargs={
         "attn_implementation": "flash_attention_2",
         "torch_dtype": "bfloat16",
     },
-    generation_kwargs={
     generation_kwargs={
         "do_sample": False,
     },
@@ -106,8 +101,7 @@ 
    Basic Exampleresults = inference_vllm(
     model_name="Qwen/Qwen2.5-1.5B-Instruct",
     output_file="outputs/preds_vllm.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
+    workdir_path="./benchmark-cache",
     num_fewshots=5,
     batch_size=8,
     max_new_tokens=256,
@@ -119,6 +113,39 @@ 
    Basic Example},
 )
 
+# Evaluate the results
+evaluator = LLMSQLEvaluator()
+report = evaluator.evaluate(outputs_path="outputs/preds_transformers.jsonl")
+print(report)
+
    
+
    
+
    Using OpenAI-compateble API.
    
+
    from llmsql import inference_api
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+# Run inference (will take some time)
+results = inference_api(
+    model_name="gpt-5-mini",
+    base_url="https://api.openai.com/v1/",
+    api_key=os.environ["OPENAI_API_KEY"],
+    api_kwargs={
+        "response_format": {
+                "type": "text"
+            },
+            "verbosity": "medium",
+            "reasoning_effort": "medium",
+            "store": False
+    },
+    requests_per_minute=100,
+    output_file="test_output_api.jsonl",
+    limit=50,
+    num_fewshots = 5,
+    seed=42,
+    version="2.0"
+)
+
 # Evaluate the results
 evaluator = LLMSQLEvaluator()
 report = evaluator.evaluate(outputs_path="outputs/preds_transformers.jsonl")
@@ -186,19 +213,21 @@ 
    Navigation
    
         
  • 
           index
    • 
+        
  • 
+          modules |
    • 
         
  • 
           next |
    • 
         
  • 
           previous |
    • 
-        
  • LLMSQL 0.1.15 documentation »
    • 
+        
  • LLMSQL 0.1.16 documentation »
    • 
           
  • LLMSQL package Documentation »
    • 
-        
  • Usage Overview
    • 
-        
  • Usage Overview
    • 
+        
  • Usage Overview
    •  
       
     
    
     
   
-
+
\ No newline at end of file
diff --git a/docs/_build/html/genindex.html b/docs/_build/html/genindex.html
index 2caaec9..e566bac 100644
--- a/docs/_build/html/genindex.html
+++ b/docs/_build/html/genindex.html
@@ -4,22 +4,20 @@
   
     
     
-    Index — LLMSQL 0.1.15 documentation
+    Index — LLMSQL 0.1.16 documentation
     
     
-    
     
-    
-    
+    
+    
     
     
     
     
-    
+    
     
     
-    
-    
+     
   
     
-    
    
+
    - - +

    Index

    + E + | I + | L + | M + +
    +

    E

    + + +
    +

    I

    + + +
    + +

    L

    + + + +
      +
    • + llmsql.evaluation.evaluate + +
      • +
      +
    • + llmsql.inference.inference_transformers + +
      • +
    + +

    M

    + + +
    -
    @@ -74,11 +129,14 @@

    Navigation

  • index
    • -
  • LLMSQL 0.1.15 documentation »
    • -
  • Index
    • +
  • + modules |
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Index
    • - + \ No newline at end of file diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html index bd4199d..f66124c 100644 --- a/docs/_build/html/index.html +++ b/docs/_build/html/index.html @@ -17,31 +17,15 @@ - -
    -
    - 📚 Documentation - -
    -
    - - - +
    + - -
    +

    Welcome to LLMSQL Project

    @@ -68,21 +52,20 @@

    Welcome to LLMSQL Project

    LLMSQL is a Python package for evaluation Hugging Face models on LLMSQL benchmark with transformers and vLLM.

    -

    💡 Description

    +

    Description

    LLMSQL Benchmark is an open-source framework providing a modernized, cleaned, and extended version of the original WikiSQL dataset, specifically designed for evaluating Hugging Face style Large Language Models (LLMs) on Text-to-SQL tasks.

    -

    Key improvements

    -
      -
    • Data Cleaning: Resolved duplicates, datatype mismatches, and inconsistent casing, reducing the widespread occurrence of empty query results.
      • -
    • LLM-Ready Format: Reformatted SQL queries stored in WikiSQL’s custom encoding into standard SQL syntax.
      • -
    +

    📣 Latest News

    +
    +

    Loading latest news...

    +
    -

    📚 Documentation

    +

    Documentation

    Note: Documentation pages (installation guide, API reference) are under construction.
    See Quick Start below or the README files inside the repo.

    -

    ⚡ Quick Start

    +

    Quick Start

    ⚠️ WARNING — Reproducibility

    @@ -113,7 +96,7 @@

    1️⃣ Installation

    2️⃣ Inference from CLI

    vLLM Backend (Recommended)

    -
    llmsql inference --method vllm \
+
    llmsql inference vllm \
 --model-name Qwen/Qwen2.5-1.5B-Instruct \
 --output-file outputs/preds.jsonl \
 --batch-size 8 \
@@ -121,7 +104,7 @@ 
    2️⃣ Inference from CLI
    
 --temperature 0.0
    
 
 
    Transformers Backend
    
-
    llmsql inference --method transformers \
+
    llmsql inference transformers \
 --model-or-model-name-or-path Qwen/Qwen2.5-1.5B-Instruct \
 --output-file outputs/preds.jsonl \
 --batch-size 8 \
@@ -150,9 +133,26 @@ 
         📦 PyPI Projectllmsql on PyPI
         💾 Dataset on Hugging Facellmsql-bench dataset
         💻 Source CodeGitHub repo
+        💻 PlaygroundHF Space
       
     
 
+    
    🤝 Contributing
    
+    
    We welcome contributions — bug reports, documentation improvements, new features, and benchmark submissions. Check the contributing guide for full details.
    
+    
    
+      
    Quick start for developers:
    
+      
    git clone https://github.com/<YOUR_USERNAME>/llmsql-benchmark.git
+cd llmsql-benchmark
+pip install pdm
+pdm install --without default --with dev
+pre-commit install
    
+    
    
+    
+
     
    📊 Leaderboard — Execution Accuracy (EX)
    
     
    
       
    Loading leaderboard...
    
@@ -163,7 +163,7 @@ 
    📄 Citation
    
     
    @inproceedings{llmsql_bench,
   title={LLMSQL: Upgrading WikiSQL for the LLM Era of Text-to-SQL},
   author={Pihulski, Dzmitry and Charchut, Karol and Novogrodskaia, Viktoria and Koco{'n}, Jan},
-  booktitle={2025 IEEE ICувцDMW},
+  booktitle={2025 IEEE International Conference on Data Mining Workshops (ICDMW)},
   year={2025},
   organization={IEEE}
 }
@@ -172,7 +172,22 @@ 
    📄 Citation
    
     
    
       💬 Made with ❤️ by the LLMSQL Team
    
     
    
-
    +
    + + +
    @@ -200,6 +215,75 @@

    📄 Citation

    }); }); + diff --git a/docs/_build/html/objects.inv b/docs/_build/html/objects.inv index ef93dc279cb0908985244f5ac23dd6dec053de78..8e77ee8ea44cd8cf7db8a6c048af507226f33537 100644 GIT binary patch delta 285 zcmV+&0pk9{0*M2Vb$^mcPs1<}h420qBf&K+*9sva!J$$r0u^UuoJkYmC3p$s-{Zs$ zahB5CXy(0d-pJ-$m@j^X4dss7O39%_sN0QDx#I{okSk`cBT;DuzX;Rh5)#5sVkW%8 zN!Cr_cAXfZDp{aL$#AURt)|ixN(b=2B~J{)EIifvk&vk-%Wl2aWyQ*+*OQ) delta 214 zcmV;{04e{81H%H4b$^l34uT*QhVOX_Ucg$nt+ln)g_|zR^#BAXbs;zb%ePNKQ`4LK#+6&_H6pnz;U6Aa!nhn+e+ z2*UUh;DXR6NdO-uH8}`vg}tIPE@-;Msr+woVG*NR{jb4J9Z!9;g>waEmB9-oAcoyJ zxdU&Ze%fEy9MSBFqsC51tTWAFk8;TtghlQ3fg?i642MoVO8;O{t<&ZQCbR-l(sT;C QnPXy?N6ov@2fSmk>&iD{aR2}S diff --git a/docs/_build/html/py-modindex.html b/docs/_build/html/py-modindex.html index a77a212..2488387 100644 --- a/docs/_build/html/py-modindex.html +++ b/docs/_build/html/py-modindex.html @@ -4,21 +4,21 @@ - Python Module Index — LLMSQL 0.1.13 documentation + Python Module Index — LLMSQL 0.1.16 documentation - + - - + + - + - + @@ -31,16 +31,16 @@

    Navigation

  • modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Python Module Index
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Python Module Index
    • -
    +
    - +

    Python Module Index

    @@ -68,11 +68,6 @@

    Python Module Index

        llmsql.inference.inference_transformers - - -     - llmsql.inference.inference_vllm - @@ -105,11 +100,11 @@

    Navigation

  • modules |
    • -
  • LLMSQL 0.1.13 documentation »
    • -
  • Python Module Index
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Python Module Index
    • - + \ No newline at end of file diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html index 1a05821..7910195 100644 --- a/docs/_build/html/search.html +++ b/docs/_build/html/search.html @@ -4,19 +4,18 @@ - Search — LLMSQL 0.1.15 documentation + Search — LLMSQL 0.1.16 documentation - - - - + + + - + @@ -24,8 +23,7 @@ - - + -
    +
    - - +

    Search

    - - + - - - - + +

    Searching for multiple words only shows matches that contain all words.

    - - - - + +
    - - - - + +
    - - +
    @@ -98,11 +89,14 @@

    Navigation

  • index
    • -
  • LLMSQL 0.1.15 documentation »
    • -
  • Search
    • +
  • + modules |
    • +
  • LLMSQL 0.1.16 documentation »
    • +
  • Search
    • - + \ No newline at end of file diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index f5962dd..0d3d7b3 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"Basic Example":[[3,"basic-example"]],"Contents":[[1,null]],"Evaluation API Reference":[[0,null]],"Example: Running your first evaluation (with transformers backend)":[[1,"example-running-your-first-evaluation-with-transformers-backend"]],"Features":[[0,"features"]],"Full Documentation":[[1,"full-documentation"]],"Function Arguments":[[0,"function-arguments"]],"Getting Started":[[1,"getting-started"]],"Inference API Reference":[[2,null]],"Input Format":[[0,"input-format"]],"Installation":[[1,"installation"]],"LLMSQL package Documentation":[[1,null]],"Output Metrics":[[0,"output-metrics"]],"Report Saving":[[0,"report-saving"]],"Typical workflow":[[3,"typical-workflow"]],"Usage Examples":[[0,"usage-examples"]],"Usage Overview":[[3,null]]},"docnames":["docs/evaluation","docs/index","docs/inference","docs/usage","index"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["docs\\evaluation.rst","docs\\index.rst","docs\\inference.rst","docs\\usage.rst","index.rst"],"indexentries":{},"objects":{},"objnames":{},"objtypes":{},"terms":{"0":[1,3],"1":[0,1,3],"2":0,"256":[1,3],"3":0,"30":0,"4096":3,"5":[0,1,3],"5b":[1,3],"7":[1,3],"8":[1,3],"9":3,"By":0,"It":0,"The":0,"accuraci":[0,3],"activ":0,"ag":0,"against":0,"allow":0,"api":1,"ar":0,"attn_implement":3,"automat":0,"back":1,"backend":3,"batch_siz":[1,3],"bench":0,"benchmark":0,"bfloat16":[1,3],"both":0,"can":0,"compon":3,"comput":3,"configur":0,"contain":0,"count":0,"cover":1,"current":0,"data":[1,3],"databas":0,"dataset":3,"db":0,"db_path":0,"default":0,"descript":0,"detail":0,"dict":0,"dict_list":0,"dictionari":0,"directori":0,"displai":0,"do_sampl":[1,3],"download":0,"error":0,"evalu":3,"evaluation_results_":0,"everyth":1,"exact":0,"execut":0,"fals":[1,3],"file":0,"flash_attention_2":3,"follow":0,"from":[0,1,3],"gener":3,"generate_kwarg":1,"generation_kwarg":3,"gold":0,"gold_non":0,"gpu_memory_util":3,"guid":1,"how":0,"i":0,"ignor":0,"import":[0,1,3],"infer":[1,3],"inference_transform":[1,3],"inference_vllm":3,"input_mod":0,"inspect":3,"instruct":[1,3],"invalid":0,"json":0,"jsonl":[0,1,3],"jsonl_path":0,"kei":0,"level":3,"list":0,"llm":3,"llm_kwarg":3,"llmsql":[0,2,3],"llmsql_workdir":0,"llmsqlevalu":3,"log":0,"made":[0,1,2,3],"main":1,"match":0,"max_mismatch":0,"max_model_len":3,"max_new_token":[1,3],"maximum":0,"metric":3,"mismatch":0,"miss":0,"mode":0,"model":[0,1,3],"model_arg":1,"model_kwarg":3,"model_nam":3,"model_or_model_name_or_path":[1,3],"name":0,"need":1,"none":0,"null":0,"num_fewshot":[1,3],"number":0,"option":0,"output":[1,3],"output_fil":[1,3],"outputs_path":3,"overal":0,"overrid":0,"overview":1,"own":0,"packag":3,"page":1,"pass":3,"path":0,"path_to_output":0,"perform":3,"pip":1,"pred_non":0,"predict":[0,3],"predicted_sql":0,"preds_transform":[1,3],"preds_vllm":3,"primari":3,"print":[0,1,3],"project":1,"provid":[0,3],"python":0,"queri":[0,3],"question":[0,1,3],"question_id":0,"questions_path":[0,1,3],"qwen":[1,3],"qwen2":[1,3],"refer":1,"report":3,"requir":0,"result":[0,1,3],"return":0,"run":3,"save_report":0,"select":0,"should":0,"show_mismatch":0,"skip":0,"some":3,"sourc":1,"sql":[0,1,3],"sql_error":0,"sqlite":0,"sqlite_t":0,"summari":0,"support":0,"tabl":[0,1,3],"tables_path":[1,3],"take":3,"task":3,"team":[0,1,2,3],"temperatur":[1,3],"tensor_parallel_s":3,"text":[0,1],"thi":[0,1],"time":3,"timestamp":0,"torch_dtyp":[1,3],"total":0,"transform":3,"true":0,"two":3,"us":[0,1,3],"usag":1,"uuid":0,"vllm":3,"wa":0,"welcom":1,"were":0,"where":0,"while":0,"workdir":0,"workdir_path":0,"you":[0,1],"your":0},"titles":["Evaluation API Reference","LLMSQL package Documentation","Inference API Reference","Usage Overview","<no title>"],"titleterms":{"api":[0,2],"argument":0,"backend":1,"basic":3,"content":1,"document":1,"evalu":[0,1],"exampl":[0,1,3],"featur":0,"first":1,"format":0,"full":1,"function":0,"get":1,"infer":2,"input":0,"instal":1,"llmsql":1,"metric":0,"output":0,"overview":3,"packag":1,"refer":[0,2],"report":0,"run":1,"save":0,"start":1,"transform":1,"typic":3,"usag":[0,3],"workflow":3,"your":1}}) \ No newline at end of file +Search.setIndex({"alltitles":{"Basic Example":[[3,"basic-example"]],"Contents":[[1,null]],"Evaluation API Reference":[[0,null]],"Example: Running your first evaluation (with transformers backend)":[[1,"example-running-your-first-evaluation-with-transformers-backend"]],"Features":[[0,"features"]],"Full Documentation":[[1,"full-documentation"]],"Function Arguments":[[0,"function-arguments"]],"Getting Started":[[1,"getting-started"]],"Inference API Reference":[[2,null]],"Input Format":[[0,"input-format"]],"Installation":[[1,"installation"]],"LLMSQL Evaluation Module":[[0,"llmsql-evaluation-module"]],"LLMSQL Transformers Inference Function":[[2,"llmsql-transformers-inference-function"]],"LLMSQL package Documentation":[[1,null]],"Output Metrics":[[0,"output-metrics"]],"Report Saving":[[0,"report-saving"]],"Typical workflow":[[3,"typical-workflow"]],"Usage Examples":[[0,"usage-examples"]],"Usage Overview":[[3,null]]},"docnames":["docs/evaluation","docs/index","docs/inference","docs/usage","index"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["docs\\evaluation.rst","docs\\index.rst","docs\\inference.rst","docs\\usage.rst","index.rst"],"indexentries":{"evaluate() (in module llmsql.evaluation.evaluate)":[[0,"llmsql.evaluation.evaluate.evaluate",false]],"inference_transformers() (in module llmsql.inference.inference_transformers)":[[2,"llmsql.inference.inference_transformers.inference_transformers",false]],"llmsql.evaluation.evaluate":[[0,"module-llmsql.evaluation.evaluate",false]],"llmsql.inference.inference_transformers":[[2,"module-llmsql.inference.inference_transformers",false]],"module":[[0,"module-llmsql.evaluation.evaluate",false],[2,"module-llmsql.inference.inference_transformers",false]]},"objects":{"llmsql.evaluation":[[0,0,0,"-","evaluate"]],"llmsql.evaluation.evaluate":[[0,1,1,"","evaluate"]],"llmsql.inference":[[2,0,0,"-","inference_transformers"]],"llmsql.inference.inference_transformers":[[2,1,1,"","inference_transformers"]]},"objnames":{"0":["py","module","Python module"],"1":["py","function","Python function"]},"objtypes":{"0":"py:module","1":"py:function"},"terms":{"0":[0,1,2,3],"1":[0,1,2,3],"100":[2,3],"2":[0,2,3],"256":[1,2,3],"3":0,"30":0,"4096":3,"42":[2,3],"5":[0,1,2,3],"50":[2,3],"5b":[1,2,3],"7":[1,2,3],"8":[1,2,3],"9":3,"By":0,"If":[0,2],"It":0,"The":0,"accuraci":[0,3],"activ":0,"addit":2,"ag":0,"against":0,"all":2,"allow":0,"an":2,"ani":2,"api":[1,3],"api_kei":3,"api_kwarg":3,"appli":2,"ar":2,"argument":2,"attn_implement":3,"authent":2,"auto":[0,2],"automat":[0,2],"automodelforcausallm":2,"autotoken":2,"back":1,"backend":[2,3],"base_url":3,"batch":2,"batch_siz":[1,2,3],"befor":2,"bench":2,"benchmark":[0,2,3],"between":2,"bfloat16":[1,2,3],"bool":[0,2],"both":[],"cach":[0,3],"can":0,"causal":2,"chat":2,"chat_templ":2,"code":2,"com":3,"compatebl":3,"complet":2,"compon":3,"comput":3,"configur":0,"contain":0,"count":0,"cover":1,"creat":[0,2],"current":0,"data":[],"databas":0,"dataset":3,"db":0,"db_path":[],"decod":2,"default":[0,2],"descript":0,"detail":0,"devic":2,"device_map":2,"dict":[0,2],"dict_list":0,"dictionari":0,"differ":2,"directori":[0,2],"displai":0,"do_sampl":[1,2,3],"document":0,"dotenv":3,"download":[0,2],"dtype":2,"due":2,"either":0,"environ":3,"error":0,"evalu":[2,3],"evaluation_results_":0,"even":2,"everyth":1,"exact":0,"exampl":2,"execut":0,"face":2,"fals":[1,2,3],"few":2,"file":[0,2],"first":2,"flash_attention_2":3,"float":2,"float16":2,"follow":0,"from":[0,1,2,3],"from_pretrain":2,"full":0,"gener":[0,2,3],"generate_kwarg":1,"generation_kwarg":[2,3],"gold":0,"gold_non":0,"gpt":3,"gpu_memory_util":3,"greedi":2,"guid":1,"handl":2,"here":2,"hf":2,"hf_token":2,"how":0,"http":3,"hug":2,"huggingfac":2,"i":[0,2],"ignor":[],"implement":2,"import":[0,1,2,3],"infer":[1,3],"inference_api":3,"inference_transform":[1,2,3],"inference_vllm":3,"input":2,"input_mod":0,"inspect":3,"instruct":[1,2,3],"int":[0,2],"integ":2,"invalid":0,"json":0,"jsonl":[0,1,2,3],"jsonl_path":0,"k":2,"kei":0,"languag":2,"larg":2,"left":2,"level":3,"limit":[2,3],"list":[0,2],"liter":2,"llm":3,"llm_kwarg":3,"llm_sql_predict":2,"llmsql":3,"llmsql_workdir":[],"llmsqlevalu":3,"load":2,"load_dotenv":3,"log":0,"made":[0,1,2,3],"mai":2,"main":1,"manual":0,"match":0,"max":0,"max_mismatch":0,"max_model_len":3,"max_new_token":[1,2,3],"maximum":[0,2],"medium":3,"metadata":2,"metric":3,"mini":3,"mismatch":0,"miss":0,"mode":0,"model":[0,1,2,3],"model_arg":1,"model_kwarg":[2,3],"model_nam":3,"model_or_model_name_or_path":[1,2,3],"modul":2,"n":2,"name":[0,2],"need":1,"none":[0,2],"note":2,"nucleu":2,"null":0,"num_fewshot":[1,2,3],"number":[0,2],"numer":2,"o":3,"object":2,"omit":[0,2],"openai":3,"openai_api_kei":3,"option":[0,2],"output":[1,2,3],"output_fil":[1,2,3],"outputs_path":3,"overal":0,"overrid":[0,2],"overview":1,"own":[],"packag":3,"padding_sid":2,"page":1,"paramet":[0,2],"pass":3,"path":[0,2],"path_to_output":0,"per":2,"perform":[2,3],"persist":0,"pip":1,"placement":2,"precis":2,"pred_non":0,"predict":[0,3],"predicted_sql":0,"preds_transform":[1,2,3],"preds_vllm":3,"primari":3,"print":[0,1,3],"produc":2,"project":1,"prompt":2,"provid":[0,2,3],"python":0,"queri":[0,3],"question":[0,2],"question_id":0,"questions_path":[],"qwen":[1,2,3],"qwen2":[1,2,3],"random":2,"reasoning_effort":3,"refer":1,"remot":2,"repo_id":2,"report":3,"reproduc":2,"requests_per_minut":3,"requir":0,"response_format":3,"result":[0,1,2,3],"return":[0,2],"run":3,"same":2,"sampl":2,"save_report":0,"see":0,"seed":[2,3],"select":0,"separ":2,"sequenc":2,"shot":2,"should":0,"show_mismatch":0,"singl":2,"size":2,"skip":[],"slightli":2,"some":3,"sourc":[0,1,2],"sql":[0,1,2,3],"sql_error":0,"sqlite":0,"sqlite_t":[],"store":[0,2,3],"str":[0,2],"strategi":2,"summari":0,"support":0,"tabl":0,"tables_path":[],"take":3,"task":3,"team":[0,1,2,3],"temperatur":[1,2,3],"templat":2,"temporari":[0,2],"tensor_parallel_s":3,"test_output_api":3,"text":[0,1,2,3],"than":2,"thi":[0,1,2],"time":3,"timestamp":0,"token":2,"tokenizer_kwarg":2,"tokenizer_or_nam":2,"top":2,"top_k":2,"top_p":2,"torch":2,"torch_dtyp":[1,2,3],"total":0,"transform":3,"true":[0,2],"trust":2,"trust_remote_cod":2,"two":3,"type":[0,3],"us":[0,1,2,3],"usag":1,"uuid":0,"v":2,"v1":3,"valu":2,"verbos":3,"version":[0,2,3],"via":2,"vllm":[2,3],"wa":0,"welcom":1,"were":0,"where":0,"whether":2,"while":0,"workdir":[],"workdir_path":[0,2,3],"x":2,"you":[0,1],"your":[]},"titles":["Evaluation API Reference","LLMSQL package Documentation","Inference API Reference","Usage Overview","<no title>"],"titleterms":{"api":[0,2],"argument":0,"backend":1,"basic":3,"content":1,"document":1,"evalu":[0,1],"exampl":[0,1,3],"featur":0,"first":1,"format":0,"full":1,"function":[0,2],"get":1,"infer":2,"input":0,"instal":1,"llmsql":[0,1,2],"metric":0,"modul":0,"output":0,"overview":3,"packag":1,"refer":[0,2],"report":0,"run":1,"save":0,"start":1,"transform":[1,2],"typic":3,"usag":[0,3],"workflow":3,"your":1}}) \ No newline at end of file diff --git a/docs/_static/styles/front_page.css b/docs/_static/styles/front_page.css index 1d3bcfb..45e3f4b 100644 --- a/docs/_static/styles/front_page.css +++ b/docs/_static/styles/front_page.css @@ -1,26 +1,29 @@ /* === LLMSQL Front Page CSS === */ +/* Three-column page layout: nav | content | TOC */ +.page-layout { + display: grid; + grid-template-columns: 180px minmax(0, 1fr) 200px; + gap: 32px; + max-width: 1280px; + margin: 0 auto; + padding: 28px 24px; + align-items: start; +} + .sidebar { - position: fixed; - top: 16px; - left: 16px; - height: auto; - width: 160px; - background-color: #f4f4f4; - border: 1px solid #e0e0e0; - border-radius: 8px; - padding: 12px; - display: flex; - align-items: center; - justify-content: center; - z-index: 110; - box-shadow: 0 2px 6px rgba(0,0,0,0.04); + position: sticky; + top: 24px; + background-color: #f8f9fa; + border: 1px solid #e8e8e8; + border-radius: 10px; + padding: 14px; + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); } .sidebar-content { display: flex; flex-direction: column; - align-items: center; gap: 10px; width: 100%; } @@ -30,10 +33,11 @@ text-align: center; background-color: #eef6ff; color: #0056b3; - padding: 10px; + padding: 10px 12px; border-radius: 6px; text-decoration: none; font-weight: 600; + font-size: 0.9rem; transition: background-color 0.2s, color 0.2s; } .sidebar-button:hover { @@ -41,12 +45,19 @@ color: white; } +/* Standalone back-link on docs pages (no .page-layout wrapper) */ +body > .sidebar-button, +.document .sidebar-button { + display: inline-block; + margin-bottom: 1.5rem; +} + .sidebar-search { width: 100%; padding: 8px 10px; border-radius: 6px; border: 1px solid #ccc; - font-size: 0.9rem; + font-size: 0.85rem; box-sizing: border-box; transition: border-color 0.2s ease, box-shadow 0.2s ease; } @@ -64,41 +75,60 @@ } .on-this-page { - position: fixed; - top: 16px; - right: 16px; - width: 220px; - background: #fafafa; - border: 1px solid #eee; - border-radius: 8px; - padding: 12px; - font-size: 0.95rem; - z-index: 105; - box-shadow: 0 2px 6px rgba(0,0,0,0.04); + position: sticky; + top: 24px; + background: #f8f9fa; + border: 1px solid #e8e8e8; + border-radius: 10px; + padding: 14px 16px; + font-size: 0.88rem; + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); } .on-this-page h4 { - margin: 0 0 8px 0; - padding-bottom: 6px; - border-bottom: 1px solid #e9e9e9; - font-size: 0.95rem; + margin: 0 0 10px 0; + padding-bottom: 8px; + border-bottom: 1px solid #e0e0e0; + font-size: 0.8rem; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; + color: #555; } .on-this-page ul { list-style: none; padding: 0; - margin: 8px 0 0 0; + margin: 0; } .on-this-page ul li { margin-bottom: 6px; } +.on-this-page ul li a { + color: #444; + text-decoration: none; + line-height: 1.4; +} +.on-this-page ul li a:hover { + color: #007bff; +} body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; line-height: 1.6; - margin: 0 auto; - padding: 28px 36px; + margin: 0; + padding: 0; color: #333; background-color: #fff; - max-width: 1100px; +} + +.page-layout > main { + min-width: 0; +} + +/* Sphinx docs pages (no .page-layout wrapper) */ +body > .document { + max-width: 900px; + margin: 0 auto; + padding: 28px 24px; } .center-content { @@ -227,12 +257,24 @@ div.highlight span { } /* Responsive */ -@media (max-width: 1000px) { - .on-this-page { position: static; width: auto; margin: 12px 0 18px; } -} - -@media (max-width: 720px) { - .sidebar, .on-this-page { display: none; } +@media (max-width: 1100px) { + .page-layout { + grid-template-columns: 160px minmax(0, 1fr); + } + .on-this-page { + display: none; + } +} + +@media (max-width: 768px) { + .page-layout { + grid-template-columns: 1fr; + gap: 16px; + padding: 16px; + } + .sidebar { + position: static; + } h1 { font-size: 1.5rem; } .badges img { height: 18px; } } diff --git a/docs/_templates/index.html b/docs/_templates/index.html index d81ed4f..aa843dc 100644 --- a/docs/_templates/index.html +++ b/docs/_templates/index.html @@ -17,31 +17,15 @@ - -
    -
    - 📚 Documentation - -
    -
    - - - +
    + - -
    +

    Welcome to LLMSQL Project

    @@ -153,6 +137,22 @@ +

    🤝 Contributing

    +

    We welcome contributions — bug reports, documentation improvements, new features, and benchmark submissions. Check the contributing guide for full details.

    +
    +

    Quick start for developers:

    + 
    git clone https://github.com/<YOUR_USERNAME>/llmsql-benchmark.git
+cd llmsql-benchmark
+pip install pdm
+pdm install --without default --with dev
+pre-commit install
    +
    + +

    📊 Leaderboard — Execution Accuracy (EX)

    Loading leaderboard...

    @@ -172,7 +172,22 @@

    📄 Citation

    💬 Made with ❤️ by the LLMSQL Team
    -
    +
    + + +