From f3549d125959ae5ba75283cef0685155ddd59831 Mon Sep 17 00:00:00 2001
From: xluox <wdljwan@gmail.com>
Date: Thu, 16 Dec 2021 20:52:55 -0500
Subject: [PATCH 1/2] Support multiple maps in league evaluation

Co-authored-by: Costa Huang <costa.huang@outlook.com>
---
 experiments/new_league.py | 81 +++++++++++++++++++++------------------
 1 file changed, 43 insertions(+), 38 deletions(-)

diff --git a/experiments/new_league.py b/experiments/new_league.py
index d7ca05f5..ef32f3e3 100644
--- a/experiments/new_league.py
+++ b/experiments/new_league.py
@@ -56,6 +56,8 @@ def parse_args():
         help='if toggled, the database will be updated')
     parser.add_argument('--cuda', type=lambda x: bool(strtobool(x)), default=True, nargs='?', const=True,
         help='if toggled, cuda will not be enabled by default')
+    parser.add_argument('--maps', nargs='+', default=["maps/16x16/basesWorkers16x16B.xml","maps/16x16/basesWorkers16x16C.xml","maps/16x16/basesWorkers16x16D.xml", "maps/16x16/basesWorkers16x16E.xml", "maps/16x16/basesWorkers16x16F.xml"], # [],
+        help='the maps')
     # ["randomBiasedAI","workerRushAI","lightRushAI","coacAI"]
     # default=["randomBiasedAI","workerRushAI","lightRushAI","coacAI","randomAI","passiveAI","naiveMCTSAI","mixedBot","rojo","izanagi","tiamat","droplet","guidedRojoA3N"]
     args = parser.parse_args()
@@ -95,7 +97,7 @@ class Outcome(Enum):
     LOSS = -1
 
 class Match:
-    def __init__(self, partial_obs: bool, match_up=None):
+    def __init__(self, partial_obs: bool, match_up=None, map_path="maps/16x16/basesWorkers16x16A.xml"):
         # mode 0: rl-ai vs built-in-ai
         # mode 1: rl-ai vs rl-ai
         # mode 2: built-in-ai vs built-in-ai
@@ -104,6 +106,7 @@ def __init__(self, partial_obs: bool, match_up=None):
         built_in_ais2=None
         rl_ai=None
         rl_ai2=None
+        self.map_path = map_path
         
         # determine mode
         rl_ais = []
@@ -150,7 +153,7 @@ def __init__(self, partial_obs: bool, match_up=None):
                 max_steps=max_steps,
                 render_theme=2,
                 ai2s=built_in_ais,
-                map_paths=["maps/16x16/basesWorkers16x16A.xml"],
+                map_paths=[map_path],
                 reward_weight=np.array([10.0, 1.0, 1.0, 0.2, 1.0, 4.0]),
             )
             self.agent = Agent(self.envs).to(self.device)
@@ -163,7 +166,7 @@ def __init__(self, partial_obs: bool, match_up=None):
                 partial_obs=partial_obs,
                 max_steps=max_steps,
                 render_theme=2,
-                map_paths=["maps/16x16/basesWorkers16x16A.xml"],
+                map_paths=[map_path],
                 reward_weight=np.array([10.0, 1.0, 1.0, 0.2, 1.0, 4.0]),
             )
             self.agent = Agent(self.envs).to(self.device)
@@ -178,7 +181,7 @@ def __init__(self, partial_obs: bool, match_up=None):
                 ai2s=built_in_ais2,
                 max_steps=max_steps,
                 render_theme=2,
-                map_paths=["maps/16x16/basesWorkers16x16.xml"],
+                map_paths=[map_path],
                 reward_weight=np.array([10.0, 1.0, 1.0, 0.2, 1.0, 4.0])
             )
         self.envs = MicroRTSStatsRecorder(self.envs)
@@ -339,41 +342,43 @@ def get_leaderboard_existing_ais(existing_ai_names):
                 if idx == 0:
                     match_up = list(reversed(match_up))
 
-                m = Match(args.partial_obs, match_up)
-                challenger = AI.get_or_none(name=m.p0)
-                defender = AI.get_or_none(name=m.p1)
-                
-                r = m.run(args.num_matches // 2)
-                for item in r:
-                    drawn = False
-                    if item == Outcome.WIN.value:
-                        winner = challenger
-                        loser = defender
-                    elif item == Outcome.DRAW.value:
-                        drawn = True
-                    else:
-                        winner = defender
-                        loser = challenger
-                        
-                    print(f"{winner.name} {'draws' if drawn else 'wins'} {loser.name}")
-                    
-                    winner_rating, loser_rating = rate_1vs1(
-                        Rating(winner.mu, winner.sigma),
-                        Rating(loser.mu, loser.sigma),
-                        drawn=drawn)
-
-                    winner.mu, winner.sigma = winner_rating.mu, winner_rating.sigma
-                    loser.mu, loser.sigma = loser_rating.mu, loser_rating.sigma
-                    winner.save()
-                    loser.save()
+
+                for match in range(args.num_matches // 2):
+                    m = Match(args.partial_obs, match_up, args.maps[match])
+                    challenger = AI.get_or_none(name=m.p0)
+                    defender = AI.get_or_none(name=m.p1)
                     
-                    MatchHistory(
-                        challenger=challenger,
-                        defender=defender,
-                        win=int(item == 1),
-                        draw=int(item == 0),
-                        loss=int(item == -1),
-                    ).save()
+                    r = m.run(1)
+                    for item in r:
+                        drawn = False
+                        if item == Outcome.WIN.value:
+                            winner = challenger
+                            loser = defender
+                        elif item == Outcome.DRAW.value:
+                            drawn = True
+                        else:
+                            winner = defender
+                            loser = challenger
+                            
+                        print(f"{winner.name} {'draws' if drawn else 'wins'} {loser.name}")
+                        
+                        winner_rating, loser_rating = rate_1vs1(
+                            Rating(winner.mu, winner.sigma),
+                            Rating(loser.mu, loser.sigma),
+                            drawn=drawn)
+
+                        winner.mu, winner.sigma = winner_rating.mu, winner_rating.sigma
+                        loser.mu, loser.sigma = loser_rating.mu, loser_rating.sigma
+                        winner.save()
+                        loser.save()
+                        
+                        MatchHistory(
+                            challenger=challenger,
+                            defender=defender,
+                            win=int(item == 1),
+                            draw=int(item == 0),
+                            loss=int(item == -1),
+                        ).save()
         get_leaderboard().to_csv(f"{dbname}.csv", index=False)
 
     # case 2: new AIs

From 4332070bfd125c53fda941223807c9805d557723 Mon Sep 17 00:00:00 2001
From: xluox <wdljwan@gmail.com>
Date: Wed, 5 Jan 2022 00:45:54 -0500
Subject: [PATCH 2/2] Support evaluation in PO training

---
 experiments/league.db        | Bin 73728 -> 73728 bytes
 experiments/league.db.backup | Bin 0 -> 73728 bytes
 experiments/league.temp.csv  |   9 +++++++++
 experiments/new_league.py    |   5 ++++-
 experiments/po_league.csv    |   8 ++++----
 experiments/po_league.db     | Bin 24576 -> 24576 bytes
 experiments/ppo_gridnet.py   |   9 +++++----
 7 files changed, 22 insertions(+), 9 deletions(-)
 create mode 100644 experiments/league.db.backup
 create mode 100644 experiments/league.temp.csv

diff --git a/experiments/league.db b/experiments/league.db
index fd985e1a22177c2a7375442d382434036f93f57b..3d173c101a18235fb2e462e7256bbb94e6c8ac53 100644
GIT binary patch
delta 1272
zcmZoTz|wGlWr8%L?L--8R$B%=|Jscy3;1<JxK=Ro%JBW?-^{m!_ZYu3??>KzzAL=0
ze8qg$ymNVu@MQAvac}01;AY@jv9Ym&t3FJGouQGRkzG7DKP5G%Sl>4@xhOv<B-p(u
zGsPDu6qcIonpYMdUr>-ApAHnuOD&0yHv}Rx6GLMoOG6VAeKR1TS5PA9ps)%ARu|>O
zCuW8T6J>5aE`tpWEwLLMDFifFlpF^eTbdhK7#JGq8(UhKo0yt`++2_1=tx0|%{H>M
zG%_@@zzB~>0jx&X6A{pGcN<z-npqeaV4BU3)ofynu7`z*p@D&+iMbJm*<pM@v&l*y
zaA#vpAL@*343g~Zo0VlBGRcYX`ZMr<<FDX9$KS%gk^ceTZoWc3b>6%Dti02B{Wps!
z=<?RPu>hSxaUvBV*O+=>Vn<4o!k#R^L`br8;9-JDf<nZZRF9th;6@=*oFH*V;WP=6
hz63B#swb*UfIAA2ocPHLakxp))WgWP*++lN2LOBIo`V1Y

delta 105
zcmV-v0G9uNzyyH61dtm6Hjx}d1vUUKOOdf;pbs7m4IBXt91i~vvks&U#t%ac@eOYd
z)D1-ra}GNVowG3@`V6xi9q0lc4G&BJ5B3ju56%yS53vv650DR655W&*5A6?351_Ll
LFe(qTOfR(XiPIq&

diff --git a/experiments/league.db.backup b/experiments/league.db.backup
new file mode 100644
index 0000000000000000000000000000000000000000..79e8a2acfb0e97d4e7eef6f77a958219ad4faab4
GIT binary patch
literal 73728
zcmeHw2bdhiwe@sQ=P-=~0t5mUO@k0&nH&~`5)ufI1VVs_LMXAIgd}pDof#2rz+k{&
zFxh0VNjA}7jLF$xu!V^>m?VRV2K=A8SKAxd?tZra_rCvo|LB8zc4p63cUM<eoqMbL
z*6zFQ(z$$Y+q}g~7tUJVmQSpakfg+zwzfpV+%}O&495T6|5W^-bbkO!y?ze)5oNSb
zJ>JD{Rp+#XH`z1XQ(X-|w*Iwhpj88{8feu(s|H#%(5it}4YX>YRRjND*T6BVzRsvk
zq}oQEi{{Kd3f{=<gAeXpwtVr@qi4>UJ8v$&TspIJPVeuv@zW-Zoj##$(v<Bd%xD|l
z`&YgHWO&=uDSdxCylu1Ly~-RuvX)Tvb;jc0Bffj^*$2;Buwd?@`QJIb8gckOe>Ct3
z^rejKIxeZNv&kk>bDQO}4q7m`_r>)7Sy_G1>0`(3GU2;d)j4N4s$=?u9VSd`n=*BJ
z+mt<a*`;mQX_F?8owj$|&J*_DvTb<Z<9>$?M(?T_==Vo<F8c0K#NREQb>x7*Sg?56
zvH^cFd+FR+%jeFSIR|!l+xEDb(<e=y5S;mzZ4;+Xn=omIDdN;|&-$MF$hK({CQg_(
zVaoUkySELW)hX^Ou770zzv_KCVSh7nysWLW{vfH7?*FXLnTuvEoI9&Cxw^LZm@;X%
zJ%Wa|`qzA0`M83PW|Wcibp{QRx^|{}IIDA3r`-2`1!wu4dhK_YsKB;8$4(o+<Jf7N
z<%-3Ty=rCn!o!E-;>YgN`@3bG^B2zg?$<ck<wq|;?G9MhNH^7%Skq2O?y%H9y<1aP
zx(|Ef-M_huQ!ls^Qp-}?xEDJQIh{_*zSW*#|9{;-wJyF@1Fafp)j+ETS~bwBfmRK)
zYT$oE4eYt5o~U_{?F$#rnY&=w){{GDFI_xs`tCa{?VK|iU+p<}_Jl=8%$&Jo$>N#w
z@!g`i%V*Ba;=fWMo6l7<+3MD%Y@yJ;WckoBZP$r^*DqZ#b5`fxgHi5~f2Z7hwVbJB
zvbnAE)k?WgEPc1=LH}{lxoS0+%~ifz^cw$h(b;OXRLNw%TeSC|7oEvu3+3E*irzDY
zqW`UX7}WKE_i%)yC5GyHdiLU3v&T*vGw$PcjpA|djQ-@oEpPtqwCX{JcP>~yvvUz{
z_GXrrSRceWOBXL$Fn9TwvAH2vjr>7n^an$`_TT^TvAvjE8d@R^=Av1hN6ei(e){ex
zx8sR@cYW=M&quGiarpE1%$d}SJ;LP^Sl&4cF@Q0nPCxWVmu~XY(W?rz*_(EK+>1C;
z)e;+^*rl@;%~`w<WsWRN$(*$B`=dYm{l>>PpPbi=x|OaaHUag>#Y+#JyL8&&%MJ#8
z>;{KjwACZWj{acb=^w3o<4=3hQ}Yw+hdk$-J6d8Rl(b~lvSqj$6tnJwi>6&Mbgwb;
z=C7W7lQFkfG2W&Lr@NS~Z7ne}IN5QXvzE=BgFoJCpN>;gzkX`;r|Z_Xy>^Esz5cje
z(Gnx@2lEf_oHKXMw8e)k9-E&srlVu=me<{X^yrVKopQ<>cg^m_-^}FO(Rs|QMYHC2
zjv4EH`jb^BA2@o|=j(NzdFWBSnBFF`mKY+=1%;03m@)C?$?2J+SG|3~GxL^R+M9P+
zVuS9}{|;`c@jYG8Isf40t1EQW!3X@|l(mMAUiHjl7ryq!?Y)b7H8JixMf^)&j7;*)
zUf6lm+&SYGFCVkrOPB6=*5WfquNw93<@;>*%U%aVxx=H&w(#pU-SZOOXWmiXQ(nXS
znfF@i?$lwa5$;M)b<gw8^7ir`@aB3Sc)NI)Mwiw4ZL0=aHPEVoRt>ajpj88{8feu(
zs|H#%(5it}4g9M$Fi}A~<3El_tvN|S2<l%hCJ2i31fT~0hb8sIn+E-dCG~`t*7#3L
z3ZhBgzgOKsJ*^HNrL5#oi9<E6o7_kW;v&xA<VcA_Gu_b++fJ@8tqvDi;^)<|3e!ub
zrCz}yV~yk{QtwcWE(pCM4VshOV0GDJC#i#yBcwi25=H#FJ5Z1`l0zgB>5$FjP?Mut
zVo|^Dy(AjwVDBH^r`|{2yWX4LO8)9!ZMQ9cs|H#%(5it}4YX>YRRgUWXw^Wg23j@H
zs)1Gwv}&ML1OKx%FsL~S2VA4EH>_Ph9M-B2f;GF^VU5~Ou=+e1R`NcDO}Gr=$JW1A
z4YX>YRRgUWXw^Wg23j@Hs)1Gwv}&ML1Fafp)j+ET{x{Tsqa`I-m)Gd!|9dU}|9``s
zZCyjF23j@Hs)1Gwv}&ML1Fafp)j+ETS~bwBfmRK)YM`eEQsn<jBSmihd5QND-Yed1
z-uJzOy>Z^K)Mu$@QrD##sRL4#)L{30_hI)^_ZWA&+wL098_qq>dCn4NC&#zHwSRB_
z+&;seZ;!J#u)eUKw{Emfwq{w|SVPQ@%ty^D%;U|y%&ch}ZyWa;KQfjXlZ_FEr2j#`
zT|ZOr)W_=^XrF7(YS(K`ZKhV$)>J=Gf2&@m9;@!5ZmpWio63F4kCf%gRAqBTO}>`A
zJ9%F6u;k9kP2_*dFUz;e-;?LdW99XwPo<}&Yo)rhzf_h6;bI2<=g%{+xq)GrV=W_>
z$z`|7X1B^z+On07Vy+`sYA@r3j$Fo{D_EvsgtGFPQpTUtU6!NwEvr!MC}i5J<zgmN
z_GibiN|i#k>>m`z%9L_>e^v~uSjd;M{(&*9Lb*`Q`7^s&wl&b%SMr%mzUm(k!^-6{
znS#H63@ckKRx|#7F|16rP$>KR#<B9bV$R=3u*4lQ``t&>eJ0zUsaC7moWFM*D_1RM
z{TVT=N-2Z$+AD@tu9m8Kf6q8pK3^)Q_W%ngZ;o~QReh!-i$<8OWOMoS^cYgMP^zYP
zk0xabm24$FEs|8KR<fCVdbb!-HdoB1ckL$iYw`JVN3PIbDwQk6TzYB@DVs$}Q=&=5
ze5q1SPmU%PO1L%YU7|_(YBiJdcLs@{Xti&Z^O=k9@>zeUXjZmb#1Z{TF|10pP|Eu|
z#<B9bQqkWbhE*=2A^Q`%S-RHm>`NWRifAx}V#c2k$I6znC4c)ERt807{PEGO61vG;
z&L0=Y67SdfW4l>y|GKBMFP1a8YRMlH!z$&==yONMu!`kEuIO(U!zxq@#i~Clj+M(*
z^8U7hC7xBMUr&=Sb`-1a*>WWh);4i0YF`~OtWvRDt@za#R<T;l=lx0?E1N4<{c^xk
z`n8-wM=^^rDqBQ-m10;WH0X?9jA0egSWA8(j+HHDihe$ZRVd~#D&_=>#^HYLK3DF@
zm)k4VT)t59voWknu~04hnHW~NnyD20t>aj^3hJvphE>W{3mJc_ZkE}v-Q(;t<#yC*
zshsz>jA0eadGr%o#IZ1V=KRfLScO79ll4c&u<}SfSMfI!EYa?b{&ipK$e`U<3*}7S
z9}&Z<RPuOgH;rSVXU+MW#IVZrboeo>61v|?It><$u>Bs|TtQTQxm-qXk{%vIf{G}n
zH;yKi@~DS&TQmv7MlPG)D4JBLR;pEhLy)-I<-QMWE>FTHpA*t^1HlpxhA}WJS14D?
z{;(KUwp`AX{PpRGW@uU2Y^GT7*NZwYh}~*cG_9fCEUVvnL1ClOKu2XDJ=cw4<*S8k
z(O)NymCqJS{@O9DTsenFaY!61o6S}IwYphOvTxmIOC8y2JNmaAy1_MLSVELk{K0Xo
zToI4Ypcq!MRLB?nHR4!EL0RxU!4hr4?bqhB#g2Tjy@X3Dmi<%=D~qbn`)&*i*(uQq
zoj6vunk)NuGz+6y3C@8fSfaa8`?s7dwfhP(VwQX}j+McH<{L3A^r)qRug9=Tkh4`^
zi(wV3<x1681xwVu-mk0AW`sz<6_yHBUx{HA@zCb{WDKiN5$>BD$3pw6_)-iD?qVe`
z{Qo(Lzx+G?f7<@q{+)f3eX@O^U9|^W?^zF77h6YIQ>~F!(p+iYX`XE!VvaY58J`+Y
z8P^zH#(qZ0SVMnTe^9?jU#?HqH`V^B{a*W-cA7R@+g2N*ehBaXQuSzcn!1ImC@Yma
zlrxosm9fft$xo6`B(F-IklZVoNt*H-^4;>e@<Mr{ypi;U^sICpeE<EVqLjjg4E)cp
z^~n1^&v&&WkG{3k4#AWyXZ>1_z7L^gGf*(KK;Y~BY1xikz8#}b5&cswQ1~IVe7T&>
z__ZF1KX7UJOj(rHqwxo(<q9QqE43bxKQIk<9WtgCsC?J%-Ql4^@X(dpQB)OPQ7w@9
z1Jm#bz!mjtJvx8D(hvg3lyC_>LVsXdB`-oSwH~EEFs)ohu*0tfQs3y~l2$u%)sAAO
zJ(H_IjQKUv`lMz;Xt`Xm;@1MPA5t1TSM+GLK<$UnGD2wiwLtC<Kr2)+W|q-L{92&*
z)jk>>mw-XD(hiMVLF}g%2>uXS4)G@3tU&RH(6ZGmlwvKA{9!c2e^6SW`9o+Kxala3
zL_g_~KK&mqff`ad4`&ybK&n52mMdq_eb)lnZwy>oE>jc@DbW2PGz?|%WNU%&520ns
zWw_WiQvRZa=zRvDDn|Z}e5t)q%?lr>M$$imRzVB|6++rSf>y2;3%F7e|53Djz6ctr
z{|H(M9T(1x<iDU<{TjKrgkpO!53dT1Jkb9kv|Obi>Md{p!e}{!9sFA00ff-9#cB>T
zaskAx520l$$aL-30w2KcBVuqD5GyE%OUNMZPz#)Z5L&KOs^FdqFMvcv7!5hLaizi!
zh@xfl#XRawcmh$hOgUEqP51&tQ~LG2&|w+e1q3MIj@E=X5JiL6hXK46_yZxO<umyT
zXyg!xTOUHp6)?E@HS!1|XxRcTB3%<M0qzyVSl?kG1IIPf4reA0pQk2Vf(TZ*m@O64
zHQ^FOu<*EK^XZy!38GlpY#A)!5=5}z=OYwT3tR%V-woi?M|WBg4L9%!!e|JM;64RT
zK?p6M7m?;#;1z_>awWO}fm;wl%OV(CP1l59Ko{D__pU;vbY$}FqMa2A>6-8hqFCrA
z%juf%3nEy^T%X0w6n;Sji#o}4P51>-tSovMzeaw6Xg+qoeuz(Bgv?5)S#k`bXlQ0=
z+JR>fQd+546c2IW8ide_74+bKE$|J(XxU5-G<FW)6g&MN0o;KchVBgV^7}RR4#H`e
z6oBtsBlp0Fpk=dSKqLPkf`%SJjLGC6M9_+2&j7y`cnJ0Ymyksa1XD^-MuCeELc=Bp
zaD!?I+<8eHG=v7<1$W^@!9oKGWnsw3`6tA%3dIUSUdPiB(Td#uk1=jR1`h#diz@zc
zF)YlbU=r@w?jwejmCa`1T^<utRtBmr?;jn_64NR0jgJbB*iTMm@*UYsJNjdIwEmGX
ztUS83l7B=D3j;RB^~2*>m@vxt%VSvBT>+lzGQkp$rqRFdvmM!7JEnOs2>455SnzBx
z3Lh57${_gSFNtB5@(2|8i(^;_mckof)Xj3u-s2Z0c_AXo2zo>7XZ?jStP*ni<^2UQ
zEX>a!dU0qRi=^lwF)XxajOCqzC7z*>DQX1S5feH%j+M`#<;;&^6-wD`f&KrL3GZd^
z=icea^FP`fn))R5Wa{eFiK!W>Y|3)qa_@65aF@EfxSKfNIxji5IHx)XIomjE*&o;s
z+dr|7vUjsLx0BW%t=p~dTl1|k)==|f^KtV^%>M6bZjGyX-MGs**H~oiXbjiC)_<qp
zq@S!Gs8{vD+I!kV+Qr%t+Ei^bO;TS`Z&lAw=c?PO>nN*~$CN9T6O<WBPO+13CGW-D
z|KjANWLo}4eo6kBe40F4-c}wWeJK4_x>Pz^nkH=_DT$SR+<(6-Q0#W^!BjXVa8F9@
zLhM2{bOnk%gqDX}3R<Ap-QHtcP+AG0b4=lM1&V!O8rm`hnco#C_7ECk*_i6-BE>Eq
z$1oaBF6(y%iamr@MxR;oyGXH%2Ie}w9uK|*1Zm;Wm9tg9D^TnqG%+cJD<#DqQ5qyW
z+^a59>=87~<-!5)BE=p>%M=ik?-Gg~EKJJvyMl_ChsL%xnNmL8B?NmEON1xWT|w-A
z;Igts#Ani7LGXTHR;G$un(h*6J>rNNcyqYxfmpYD_pagw;PmtDm{81>ai0RQ9!ARv
zPri%e_o#~y8kS3xanK-uA3}rL6{Q6c{16%@$uhXmK(F^6-NY5ZL#QBdRIOIP3iNsi
zE1!q<?-F{QDs4a(x;tzql<pF8J%Uw1bg7W;5^6nyRmL4h%?4uK9^jA|AP|Mfqb>rm
zJ}^y8G@&Q%3WE3{G|Yivwz(^a;)l^N<B4;lFg~f`0ne?1S#XHJE}_?Hgm!w5ajWkD
z!WCGIBDyOf*du8eCQw?S*h5M~02{%Nu0XMe(GY$|X@O!7p%t*i0M9xpcHFG7Ztu94
zxCG3sV{RJLD57H~#U4Q`ih#J^MT$LwR;VB*;CGQ?kD_6p-J;(`iammcU=TER7b$i@
zOZK^02>Mrq%ERmpMh(A<6ng{>y*C`9u0XMel$N7-VOOBoLui;x#mcQNQtS~2MO32*
z_fSZ7un;ur-Id}BN*#rKI|LlYgLF+u_6SxP(X2|kCM3Hl4mogHC4}w35|TZlEHTSl
zO4o#Bk76O>1(&HuviCnECcP@{2>xb8;IT)u4@@iODG)@WoyOULX_&qgt|qDWC>mx=
z;ir*okDy^-gmkP0x?Snth7d2xv}ZB7hXqVE67IqQ8<>UxsEF>P#>zdMR;gyOOs&S!
zJ%WbCBRDtK?%}jD)-d4Q0`YG3d6F|7xts{Xi%AKrPpSp#J%lD^8{oJG@;!`}E5ebf
z1^PXNmMiA5yrLEe_z)Uu7ow;ZD0sWyB_L*zFJMg0VbajA1rk1lhS;76L6L?Rw?2d>
zlm}=e;-hFtnvOsTsrU$5ri9SHUz>*~Mm#cO&E7&6vEO2`BPUkNpa(?t&W)qxko4WJ
z&55BQ0wpGWX2;OV1q9{%+CeciOtn^U39|%ET&dpsE{IFWiItIJGDl1|92iH-Ad2JH
zX2#IU1+lQRc0ddb=LT1*wtpN=DF1YAKhUV(>OC%t06a#7VtciS^+V`&_l;p8HiJHJ
zpJ*0b2#C(w-qEZ)1S~>|GX#rTN3VXq=e%;23gRTSy}+V2Wc5G&f^htCTJW&_+MY2q
zbo3(Jyhj`j5lXbk=`pkdrqgf(c8{Uuv1}GY)iluP2K1jdq)A>ZKFd|I>Dq3QEcDD+
zu$Qjw8pF!tK~C4EMzbndlYj{8lo%E!t8fD*gT*(Xw{IZs6VFkjD;43mT?9=$DAvF<
zta=p_M?1%qmdC%0@c(BgK8^DK-*oSBf9Ni9cW~RBFP-O{>z#(Pzf*EN`yKlM`$Bt}
zy^Fnx^{w@ib&GYXbr4qnuVsE<K5YKPJj&e7+}unWe>844&NMoW@x})F=lV1HwR)Gn
zuU^ny?a$i%+6CHS+D=+p{aXE<dXsvxdZ1cW2P^L>4=EQbM_}#$W{Q-2C3$P|jO5(p
zcFA?*Rq~_q<?^xeba^XTll~<ALONSIL>ey*!)5%}{`vJlwi~@yVPnL?S|w;Vbog25
z$~wt*@!ZJ0!>EFm?I>j1vHo4m3)V@thtse?2lHTlon(6i4KouM#p@*7qi9&vgoBc7
zkD!$i3NHKgK(?!WVitS}h@|H6d=>n9AlpM|7<RE}piZ(~G`$cSCYbS1*GaZV(XgTc
zmq4;Tf|kKj<Gf$zKsyfV^d8$*--Qg8W|cGWtm_<TkDy^O8xG2W_HY_P8yFbs9B2=x
zAxxdem6B{1mk@GpSYazFgk(F>dUs60CE$h@F=*8}&@M_FkcPE!<k(T9J-jqb3Syz4
z-$kMJC>rDse5@eW9&%9Vd#s}C3WDuUAAL*>DPL{J`Z#j0DcVj49he5M2+`xNK)Z*O
zhPAzTWV!<F9zsL(Tf{d4?H)$MbTuj@h`4LteVnnz4=(^g*!W#R$UTgPMY8ZF1MMC{
zL*xarrYi`#htaU^2;L-Vckwuf(C`|KaIXXHZuAyO+>pdFMGWPlG}7*(v@lw(gjIuG
zq}?NEc$g8M?;`CUMMLvN^^$gvpk)xez|9J@yW8h+&UWOh9feAJQQT5o0%><qS_n<d
zL%`D|?H)zrTTj|Of(G{#6JlL~b`LqISkHjF5NLO&kK2Yzz^a{WJEEBt5%meQdl(I^
z5;W58^vDcc8dNk^Vs(*rkD%eX!YdwKq}`)v*?bAXGNIj3A6PWdOA;df!dD<BNpgsM
z3h5rjLVsUPcM0Vl!9sYTfKZ<h?h!27PAT0bbbAyFORmv4NVbDE*6d$pSgniId?gHi
zs4|l65i~5uz-sp{A=|}4!%D-_U(5@43Edt=gFMUOphCDu(L}%p50y~vOzW-U#2v^B
zZBfQrTr3dl64E`2h7}FsNe;BTIdEw?@gjuZ6=?S`8kWMLv_QLu(6FvYG+ff|q7B)-
z9R^$i*5Bpx?fC+|w$nx0J%WY>C78_Z655?)ZAfWCGobqr;ysF%Lx2(ICe(Wr4XYqU
z?=0jy(ab*b2$xV6amX^3{2|`ZCG<Prg@I`qS|HX%0$yC{z@?$j#iQVN1p+>VhTcOw
z@Fd_PN~`4Xa-!e0Q}?-P{hkH!s!6duTf{^kE@4s(O+-?0&>iDwxg5e0T|30k@a6~J
zM(vsyL(Ad42S|_!0j*zeBrXAKULZikv#@;(4GY8|wz|g0&@e+JbkDdr8iHkb>c_^=
z#2PnL@0f0y+5ZwSYmQfDux?v)HKXHb&}3-6+r`lG#jFVPj*6k>FfoFzX4^O#9DCfX
zZ3IoUH?3bElFfJI5sWOu1we&#be9%F6H_**-fB!~CAgockV*^<%b3KiFUQfKK(gts
z5@<Af^uGdht2vq&DPn4>7{kJ9TCfVyEW{zj*q4uH;kM&7)vjC&3#*-Q1F~RopVdzp
zt)8teiN%P1S0;vr1&gB3+B$|-fVeC2{QvXa{{I|rl(%;3qtqj*%TmXrc28}YQr%bG
zJKeL~PIsKUzVoT`v~#Uf|2O>q?;G=tF~(5+WBqacO8t0!Pkn3M&|cT>($3KqXcM%J
z)GyWN)f?ge&r-KhhbSK@k1Cfd$12m6trRW!r{ph^XD1Iyj!zDgKb4=7e=46O?=9zK
zTY5{nS2|BxEbS<5ocJp7JOTiIJy7P!-U~?~k`W9o7V##MaJT$=pv=vFK>!hOEVN^$
zDi1+b50rTb4Uubf!*x>TBq9bbO)P`PK?7wTQW|2MSmas{lzA8}i>vVKJ<7bFw^gWg
z6iRp%9%34$^(gZYTCRZZq#h`9V_+IwUg)5Dpv=Q)8H~YxJy7N$G`L#`t=0o&uJ@T;
z&vg{=x*oiHc%R~Ckun$68$!c$EG8}Lq|BpeIq^zZos@Y54Ko|i`Sl>G9&%8;_J-%S
z9z@lxer`Kof|v-zC|3`n>LE1I3!}6ksvbgvdPI*{52ETJv<%*9#XY5{y13FX8VW-T
z38LyoA1Q~sknhOhk-__(;@pC$dI+slKuhxLK~z122Jb>dY=WqI7%eB>`>K;X7voY0
zt$@@Q#dIC=+$4?G@A-pU2_;v-f-VHi>rm%WEWF%SEu`xZ=aDQS4bpXJ^9UB=^TldD
zU57M}V&O$*tkkVTnVZ!2_Ys&m;g1zW{b%!7R8fa8k7A)mFXYm7=<+BQR>Nk&f-H|@
z;U%(6x(-zy#lkuu5k3q=x!SMi5T}nBG&t9o8mkAQJcK4vETFVNl!wvqE)m)+iE_~{
zLTH#ez>Bc;K$M5j^5vok;s&Bz`|cgcW!v)^1Yl5FAj$`(VbU&-Kti2FIccc@OUr>J
z0)8aQBWPGBDq@x-%A;tAl%h)@QBD=o{}I62A|yt^k_zz<lPHg(<*?Y*uahW`pkdxo
z^lBu^BWQTRQ;eb{%A;rq<)PjJQEv1b4MeVvQafgrD5@QZ@(>zUUW;BW5al7XVl|6s
zLOl@WVYD22aK9di@(>#Qe=))bqFm{J2_-QVizVjbo!5FG%0p;a$sr~UNR*5IJB%h~
zar}B9%0p=AKtxoVM0rGMSPogJ`1Kx9-tTc1FV<mYMG*@VP;Wh=d|(;^SD3G=3sKH~
z#lSSgtHsR{syvd0Nqt<YkmXS{yljz0vlP0VXllO!4VRFs;FT_VVW%#Hc_a<v9?mUL
z<{_maz=e=-Jy7N$G`##Ev`nDP!)Q4;?S8#ooSV2=T0a@f?*`(1ovL_bwu+(QkZ{}T
zTgK4fD~kD|E#heT9HI&J&0}cLMb&D$K2nSx;u88_0lid@X@9KaPuDk#W{Ei}utr3)
zkWZsrN!K@xVIkEI&TErsRuOB|(1v_*k)qF%`=5T1R(N5eTimB~cU^?hGRSD;*N4Z@
zpkjm{v2n~n@j|HRW!mCs2nK_;QE<@y&mO`OSQmmf?-2E=Zx~0*iZ=u58^qAc7&uUG
z!(wP~BM=LzuOCAbQ;sNYy>42vUw<kxw_&LudMqqntq+Z%;r)7X&~@WzSjB_V)`_8&
z(AVSK){dcJMG#i?*M|t2Xh<Q`%|%2bakJKnp%t*o3$!(3Xo#dEBwQaHM?*lhU?m@r
zzL&fy;eF-(-TTmc$9u#3qxYisjQ5!LkawT=3%nI@gLk!esdu4wF7gAO;&pk)c}I9l
zyhFS>-T~eWZ<@D@H_;p8b$A7@-P_C??hW(S_SWz$Pf2~7`Xcpl>iyK8Q?I37NxhJI
z3UP%8Q}?9qNZyvbN!woAPOGGDPF;go!-c7HQfH)^sS{F1rk12SQwOE?O-)ZtPEAaW
zMh?PkYV%Y&H7qqGl}hQUg!`rYvHKq0OIYc?=sxW};y&Qs<=*Pv;9lik;+~IK#i?$W
zd#t<MUFgnpXSy@o-Q1nr@$R<CyR@vfxt7+3X+zv1vKfwW+uWh<VApmP=NsoU=OgDG
z=XK{5=XvJ|=QqxM&YgJI;acZ%=OX7^=X=fy=S1fy=P>6GXSTDSvxhUq*})m(RGpl&
zh2uLLIBPqeV>puimHmnRzWuiSs{NAvjQyznpnbP}n|-5wHQtrDz&_hP&92+W*@xSU
zkT>x_dvAN1y|cZ&J<2ZGTicu28{6yIYub*jVnxO0)+*~=>kaD<)(h5?*2C8Q)-SA|
zS=U)tSU<LYh<7bcwobB+ww79lT63)Ztv!)>VMlAM)nVnWEv-$g4Xt(5_tdx4mFkP?
z)7Bu%wp8<<=9hRk<2~~&<Y#!%eA;}(e89ZRyw$wHyvn@9Jl{OaJk{(nk2RN@3(a}v
zOmhZSaO{K}4%?bVv)vqFwwXiC!KQ60#y7@i#z)3Gc(>yf<9Xu=<2S~A#+}A3#<j-f
z#zpEQ>I3Rsc$?z}y!Y`vV})^|ag=eGafmS+IUx2hrWiXQ7F0EI#ukQeY+$Tyc!q(z
zkYDMa=<n-q>#yoBVU@_E$P;n5ew%)yezpD+{Q~`L{WQI<AEzI#FVg4Z{gJ)(Y5LCk
z_WCF-rRiD%@uQE`tJF)>^VPG|Q}v3T)3?+&)!X#-^tJSqZs@Z1wf32|O8cw!mbOxR
zQF~f@M0)`7r(3lfw5zmBwDYyIv{SV%?O1KOwosdgjFdC9-L#$5rg|cBQY=#!s`C+>
z+84Pgr>c|G@#-kGtY+2C)wDWH9ipaGT}>!oDj#Fz%3H`?@uKpy@`&<)a+h+ea)WY}
za*1-ja+Y$c(uJ7Ta%G`1Pnn5#TXs`+QpPLWDn+GT8KJZ(LzOiZSJAQJ<sZpUlOHAD
zO}?32nS3$%bn=np1Brx+<QMXv$-XTTc*^oy*uL|T|JtgT7VH1z8W6WrOVYM1TGHlE
z7K6!P2vFs37`|rs2g6qkUkXs<FBm>&_>AFGhQABY<WCqrX80S!Du#~)$nu8_A27Vn
z@E*fosSUnM_8qc+5x>x7k70Fs*}u{!>0O3*7?>&jnLZ_6XLya_PYkaztYmnE;kOJA
zGyI0(A%>e7ZeqBR;Rc55)ubfpyONzsb_&_aWOpGuk?aJr+mjtnb{tuiEK<;-oFrLU
zNlK<J%GcyO8SY@Xo#8fyTNxHG9LjJALnp(*1kw`>k25^R@F)XSfF}KdKk@M-KAyzK
zllXWNRe&Z@1!xjgfF@nfe|H_jkqk#L9L}(uVHv|t43ijkWY~dWBEtlR?HR^1jAOw3
z5$Z+ShM|L@%1~h_6C}=NIE&#-hVL_cPk<`ZpQ|$cxhm72t1|t$DwmkzGB^x2gT-Jn
z7z}jBR9WLsDg&k$a3JYfhG!U_W_XI>Nd{_xszfbNm8b=(616~8`ZZJNhO5%g`P0uB
zZegHCs!FHxr_&ftWjKZ5WP-#m8R&+q3A*8Gf^N8)pc}3xE@H}s3_oJHfZ=?G^B8`}
za4y3+3_lRfShdKSWDT-9S*jpKrpBnqkBL7~WNM6xOpQ^IsV)?m>OzsJE)<#SLXoL1
z6!}5^JG!lkOt)2$>9#8J8T>n{3q_{JsL0e96`2~NB2!~j<fY6!jA04GVunQw3mN7!
z%ww3#Fo$6_!)S)>7)CK{%Ruc_k?y5W(mf1!Gu*{M_eGJoFzF!vau&mZ3^NH5pECTN
z;S&bxZI#5|_|v5fKVi6p;bMj#3rNb;9VTV!4wEu<he`R5{JTFeJizcPhWi=rW1ve(
z%2)8G%NZ_Xpra>cI(kyxoxhyMup7g!3{x4VFid9Hg<)p~>M)Y>ru=CW2A?6#Fq~l+
z!}<*CF$`r`mm$d@Ge`^x0_mR&Uow2b@HxY03{>+;i5`rkL=Q$%q6Z@>(Swnc=)p)z
zCH{jVLxCaBkYi97_>{x|jk_Z8DM@rnvizp_QkLIfc%9)j2I@>?`Dy+{orx?{XCli_
z@RxLgvP>r^%XEUWOeZMIKV%B^6tYY`g)CD~A<NWL$npxN9LsPF!_f>!F&xQ2mnqA1
znX*imDa&-3vYaJ~^ew|T3|}++gW)TND;cg}xSWBSlq^w`k|k<VvP2D9mZ(9?5;bU9
zx`_YaEC%WsWQlqPS)!gnmZ)ctrFEII4#V0ELm1W~#c>bWyUE@~_LpRTLG~K5KP7uL
z*{jH2N%j)57nA)l*^9_tNVZ9~LAFk|i)@YTL1br<J&^27vImfzL3S^)dy?IQ>~yjl
zlWilr5!nsNZa{V&vTKtaLYA*|P4Tnz2ePk_eVOd<$-YGPMY6vm`vTeL$v#K+TC!9p
z64i-Bbs}A5!GAbG{W4)ZfOqt7HEuAjGA=RBH_kFnMZUgcjpfEdW1cb7m|^T@?1c6E
z+Zsiq-Pp_+ZVZFhwuWIDivF$s1!nc%*Z-`)roW=Upg*NQ0?+Lp{SN(R{Tlr;{X+d5
z{S3XSpP(P9FG1G-gRu60I<o&w)JMZh%j%o!X?>VJ1U{OsC$ulMkC7E%U=QsA?FZT!
z+6wI??HKrIi?oBagS36M>Dpv%qBh$5()-wZ&wI;TiTK6S-Xq=v-d*0Uh+bUfUE-bZ
zo#ma1*u}Bla&MtG&zp(J#ct*u=FR3c=4IxE<`T2hJjmSFoR0Su(&jL8h?z3pGFBQd
z8c!RKcsqL&kQbrq<-M)E5#Gk$`rZ)F^Gq+9`X=>x>Tjv{Qg5gJlzKV!T<Y=EL#caH
zcl=kQ8s^L9bI6qNka@2;*_>#OHY;XUt8_;xQhsW~)Vis`DJP}<`%#bRNQZfjd4}0E
zPcV<vhIISU&Xdk>od=w|o!gz85&yWtx!5`1IomniX(9r0w6n}v=*)L!Ir}=(oypEb
zXS7psvd-pC+8O2yaZ--%B<wHkkL~yDx9pYni}us@BY02bF8fyd2Ky@e68n7nEc;Zu
z%Rbg#ZZEXwnYx)UzBE2I-a|BGAA7nz#oiJ7No<RFNCtT&e0xKCU3)MhBAT7BzOw#q
zeQ3Rdn8+Wk7p-Tk$B=XKK14-+Ze4F(iR_c-S!Y_OST*YyYnipcnrj_k?Pcw1O|r(}
zt(1bbm9?q0k+m*jAr>-Jer<kgerW#1eC_{bx3G0?ts3}msR1#P+DV;3V^A4L^w>!f
zJyw!LkCi0RV<k!SSTgA_OD4f!CP<N*i9d)xHxsWgyv%?x3X@p-klmXsMo4^v5fT<7
zB<xmXw<Nm-+0Dt06h#>F77Uv+tp0K{{xpI>`iS8}h7TCtXLyf+B$*-oi9fx{K$6;!
z=mHGsLH_aqhF>w<&u||DNist^mp`4u@B@ak8J02}#y}2>AuZ-liwF{2Sc3Gqk+_Gy
zyqn=JhT9l!W%xP6&lqkIwPP$JyOiu<WS5X#Om;J}Bgk$_b`!Eb*)-YVWT8n=8Z-$k
zGzly;2`n@T>@c$HlU+|7UYCEz@B+j049_t<%RuL=%XGfFd<%bhGs8^`+cS)37{@S{
zVGKitVQYqVhOHR3WEjjaNPsDG)04UB$-n0>Ut)MsfFs|?a0A2j4A(JSE5MZL=9)6y
zTvPr5e|a_oDOXEANqn;86B$lmIG*7+0j9h^!+s3=GVH^!w*Xt7#IPg74h$0+CI~R4
zzcYNo@G-;R7*+|eq_-K~Vjy?KlHTA?uM2Rcg$xTA4rMrmp_75?)0OC1b0vD#T#24F
zSE6UlmFQV>CGzrIiM%{l^7+rx3?_rYK)s7AQSahP)VsKeujqG)FB!gI_?+Q0hF>$#
z<LD;nadZ>(IJ$}ZMN4##Cwm;(W62&v_Gq$4kv)>^5o8Z1yPWJIvJ1&BAbTj;L&!p{
zqkO1!*xkuaBfA^fv1G@P9ZhySvZKg)WK(2avJP2WT#qBIU}!Ql80rjN3^fL7UXDb~
z%aN#gITAH5N22ECNXIhs7>1)6$m4UQgZa~ZhItHg8RjqyXV{pbjbS5(4H;4lE`!6c
z`cf?VlsK7zFD$_qmf#CZz;Ti!XAarfWZ^jB8#qp|aGYSbCA$sT4zg9U6|!ZrC9*}b
z1+savYmr@(>|nBk$gUyI$(E)w?9MQaVK;_d8L|u+2I^&OiFz4ZqF%<9HsIe4V_2VI
zJ%*tSYcfzrWJ`ni)9RD(=u_f7hQBhr%kU1vUl^Wac$VQAhNl^xVz`RoN`@;KE@!xm
z;S7e;8BSw3mEjZyYG`(X8k(IrNmQAABH0tj&Ln#P+5O4xN0vG;EAdZJik0}5;Twjp
z8UDeris2)M4;emSc%Ol~WGg{kvX!9m#7fY3VkMqu%99LFFg(ui7{j9s*D+kna1Fyx
z8Q|VZlGRD}V6yYc&LcaQ?9OC&B0Gufj%0Trn<JYgn<2Y3*><u+$*xOw9kOear3i&7
z9U#ualz0q~cnpxJ3QdW}0Exx`Q<}ldy%_dn*n?pz!xV<e47)Jw%rJ&wG{bfbqZqa&
zNc@@MZHBiP-eh>gvOht`(&l+PdgHzA;NuoB<KB@frZTB5Qk$kW#;p4~slmNG!R5{(
z=V0Xj+YkHweQdpN{n>iWx)Za~*IQRvKQVuc`Q*FJ+u>JlV6I~h!c4J>xBp)>o-rOX
z9>UD;RAUmN2&43k^r8Bix~uE()?f0T^&a<r<JG<6y(7KDyhD+NaHhAnx4SnPbNADb
zcW|ORre{Xoaxjnmt@*jRYPIh!JXrFah3~FEsXwgWum3{-nLbwU(DRrB-lQk0VBBlm
zVcd+p2rlz3^Dgp!=$+}E>K@@PaSw6lxCgi$ZUL(THgkt#hr=_SQ=FP}OwT<3Kr{5l
zx5nqjD&t+_4eV^NHR2*08|&$F^!@R6!&H4oWH~%ZKU!a^A8K4^oMW6}G>sE_RzdWv
z2e2-%&bCgo>eg}AG;3#Tdux<cvbMH1vo^NYv({wqTtq00H4Vp55pVfiUxi&1-oQSJ
zFZ7zf4(7D=>+~!1AL~EVzu!HFt&R8o;eG0T<h|>?*=uda1*sok)y0a`NvZu(d!}|x
z?UdTy9p<j>damJ0&R5PS&il^W&a1qJ!8p=bf*cM98T+n|l!%DMyXG6_AIulbC#}P+
zMacblptZO4vh|$xIQ;T^X`RCn_7eLLdyaj8Jp(HpcCjbgx7#;6d1otUgtM`;p0lRo
zI4X9M{2ZAjpK%^_9(3+b*(t^S#{JCw$bH9s-F?M<-hIOT4YE+)>E7aA>t60&<euw(
z&t2i3m^wPOG<9fdPU`m5O{t%zF2ybqMXg;Mp|$m_J3-85Q*R@0U94ug)4RpHw*NeT
z_u2MbRnL7>K2hFR-d0{!UUEOgxxC^2(S5P!ntRp>V1>X@>S5|3>TKm1ytne8a<_7u
zvazxrX7e3I?OEj^))Q#s#aZ^WMD=p@BK2JLd+G}1M&)YdC&~rN*~)23T{%uUTv??4
zM!iqHQ@usKR++CHsO+swQ+7`NA^Ae`$>hVy`xDv&@}K@&+{0F(tr`$DAchHz*Bodv
zjq{pJ<Gd!5kk@1q@`_BVMv+O?C^D%UMW&%gk!k4B*^SWI`PJF^)!F&g6L*UP>501-
zXk^zDG_otwQt_oC9mcSPVKKuZ0-kwN<W2aK&yZ#ysVlQ6mZkaPOIadOEK76w6I3b!
z>`<w&6yBGhQt>lXD(p-q(U>mtm@e~}F7uc!^O&CGF+Is+dQze>Jt@(co|I@zPs*o>
z!je3hljO;qBv0lfc``?lj};U}K8As$y&{veSL7r4Ty|oZ#IPg74h$0oDAELm?HR^1
zjAIzfK;l7>NIWPKi3dd@@t{Z~9+EtXpOiYpA0(wJLxlnQ9JK>|4hww_3w;g?eGZ!w
zBvm?`VL8Jxg2ZhMw+cwg_b^ZsNXpa%lJYP4%U=kP<x3bYX819~MGP0x;SkHfK@rP<
zZ6mu8*$o9L$wEFU4dYMiGpxq|Er&mXmV<?sgN2rZ9YS_3L6Rks`LaYZUuKyvOC<AU
z$rBWn=do0t$5MG7OO@z`snQXGqDUr#!Jso}45|QGQW%m9GK0jB5NDM5C)sbwena+a
zvi}eyRi>*_<)Qp(T>|MWhBF1oJUcBX{?4C1VfdKgZvvDAsdy#v5q}~TuOvvtD+yBZ
z3cEE*;;&5M4j{oDK!Q2|Ro+PaPL=5<sPYE<iEe@_(@ju#0g#e-jZ66x!>bG{>AL<%
zmZrZFuh7pgll?sxM-5h$x8+aUFl@{~Eg|t7(Vr#z4B4m2J|#%W1i2c?1i2c?1i2c?
z#AE!sM;RVr_$|Z348LJ`$i7iXe~nxPja&tdTm_9>1&tzc8b#tXio|IYiPNZ;Qe}Gj
zRCzm5n8GVRlms<<g;#zkyz)ay+-`qD2O39~pRdHv#owukpE2AbKu+Aua1+Cg3^y=b
zk6xYHn8r@7#twp(cw78lOT5KE4uY0=gN`+k>;$sglcoM7nYdJtlZl@&T*5$4X)^I+
zs^g2u(leU)5&e7t+4Gru9>WhA&Sf}<;Ri%Ko9tO+&m{YOvfmRVP3mOGGGrLGW@s0n
za+E?%tl&>gh6Y2OfxKdsy<(NUVrV+tRh7Kr<niL?<Z)yv-jqCsem+`Q`6#kSl0Aa#
z;bfN!n_NbADcQrwE+M-(b<pbJm0Fj^OKLRv0dY#2{40k08SZ1amth@-wHbymP)DW7
z)KO_1Y|<pUT1}#>)g-!FO`@ySI4Y<~3+T_KLm3WHlBTqWFw=$EU6^UY>?X{v!b}xr
ziZGLfp%rv0InydR(^wrVYL1*~b%zwS3XRu0XuPIK<26N^{3tOgs-}*ipGTA3j_jz^
zD|~6sFg(pbkF6#@DL4uSAy^DTuoNUxC`hDW5W;Fb3P0%-e$p|p;~NS;=?99R)vd_V
zyaRTe6W`FhgF4dtsW_6JIGusUY&~%*e>z26bXu6<!fY%|n=lw~&=@u(y8+o@WY;IV
z9$Abv;85H^##n=&G1kChtU*(um{xL4`gt(fL1fopA*Ls;;Zs<BdROzOGXzJr$eLsg
zvN~B!SXCvfkWG@MC5)<-sz~2YUX}RQ*4X#*F#n<bUHbre{Qjivg17j$)5==LJ=;Cq
zZMrAAN4v|g27kUg%iY)A!=36*!YcexZrRPcTezFJ8@cPcgIvo^I$tBNz=zIXoY$O}
zkzep}=OO1_=MKz0U*lZnT<Dx*ZejX}C#-FHJ)T*QcW>-$Y;TMb{x{a>pA4V<Sm$uW
zDCRo{I(s|QoSmKRol(eExV5ucUw_kHWxs2`VLop@VgAOvPg{)D1hce#wB5Cl+Hh@s
zZ7t1JMk^I1t8A{MyH{W*?^04&ot;R2nfy5U9(E;IsT`#2YyT0m+|SsL*$>(G+IQGD
z+t=8aAur-N_8E55KEXZ`9&o39ki9SFwkN~?9c@?aEM~UT_Aq;hc^LLcoNexB?$Hyu
zz@CWDYL98Z*6zW2htsu&cD#0k`lk9v<VSo8J0xDLo~NFvo}$+5l&xb<{7b}P-m~7a
zR$@l{Y3#aqi+Qbixp|TCpmDcxn{lIYwQ-zrICfu{ZyeYwZg9JHllD{XQtd~Yrv6j?
zLj9ZiSG9mAVpDY^bzOCkSQ@B&t$eC9u?NGE$`Yj$85nO!UZo6EhFFhS4_J3uw^}zK
z=5vX4zI7JzK6Y8hTFb44);w#bHN)D?+Q}Mko@;*3Tw$JQ9%cN*xWG8uIL)Z5d#Ssk
ze;B83qkO3RMR`qmS$PgQ9wsXjMcjlVC_QnGzH2(HZLOl!ZjG?otfAImyw9MR-<Y47
zADQo%ubZ!!Q_LOAF=o}wnFiii{>u2oc;9&2c-43b(WgiC4fS>OHFQ&#)nBQ<RDZ5s
zuU@GhgH`<tu;0W1%Hzm3c`x>xyji(MxlFlGIY&8zV>rg%#<YLg6BLi7$x9<mUK(lg
z(nyn+Mw;@2`i_ZoOt~m!HA9qV^K_Oizr>$j<b(W<;RS~08J=T!mf;PC*BMA>nY^;l
z<duacuPiiqWuYnmK`?EeOtZTu(@67#U5)qyM<!Y5$nWtdl7*&Ba>C?QayGA(w&gqc
z2)8rb#&D|uS0+K@^3<WrQ-`ihg2t6e(6~H@=gK5#T={zbyXzRP72wF!Q93epl#Wat
zr6XU(zq?X^&C4Qf`Ever83PF#o0mn}@{jn-3m8bU*t|H;=EZpqPZl~nS?KU&p(CFy
z{=kvX65z^bGJKzbbc@RiU|pF;7)z#M+LCFQwqzQnEt!UCORn+Xoy2e=!wC$>GaM(t
z;$^fJFQc`18Lh?3Xf0kwYs-fTrY$dFSj@19VW9w1COu%vhw`UG7)WNB^1<ShF3)F}
z$1s;+4#R8#rhE{?EQSLaW-=Tgz~UKFOWuz^?aQza!`=dHc?QE?40|%{!7yEb#Y<N$
zc^ZG(jbT@YsRDF)3d3ZET^M#|*hzrPQ?ahRBY)a~VIso>0fxLi!+3^q3}YF_2yo@U
zF|1<vNPxrdQ8@Ak;*-Ja*$rOLZt!|`gV(bgyq;Z`i-M{1N==<tYGPkX{xU1Tk~0ij
zGqf{o#jvFSn^#TP^5*<$B*SJ5BLp}}l|f-hGRO=PLxO<UdpNw_!{GNz4Eb~MJ461A
z;Zugc3vl`U5?B6Me6r=i41*ZfVDK1H0vy?8a2RX`i@_9N%LaqapfS)j*fL#%E7LW&
zGF^i!(>1s<U4zSOhh1Le>Pp{;KX;|C8UDfWl>kHflHm&mzD$WPQ=-dsd0yU?KH<Nk
z%XB5a{Sw`Ni`S%B(uaa#Np$-y>3#n6o&ZPsE5o}C?=bv@;m-nWp4~^9BmVRz!y62*
z3oxbE82-fYD#J<vu0(ygE0K<NrI-23-!r_#@LPt58Ggg?5W}w-e#mex!#NB;U^rWV
z!)s?8=>h)qD+Y329O*v(M9z!DE9)KUZvOHvhF>yV$nYZu>USMpwe9e#ZHHGOIJ^qM
z;Z+C@uR?I7wD>z)sxh3zKs~0-v*<R@qT30&Fgx)eQFw`g%}WgI1f8`dJu7}^NzX7m
z&F~b%lMKILxRZfKHH+WKvG|P~i&sNi(j`o}nBm6^7cp?}%`^IzbPRuaG{aE@ylBGW
zMH5!y3;vQCrj__i$a!lf*#pS#Pj)}D`wFhfE1XP;##obIPcfyR@$YV7xS4^vRXBe9
zX&S?B47)PWBWp_Z$eI#8vZh3jtSQkWYbK~|nLNX5@(i!ZGrVTvd2wRqBC-p~E+9+G
zFw8^5F${i_$>2Ac41SZz;5V5J={lxd%Ww_DPZ_RexQgLQhAS8@XSj^vQi23C6U<C7
zGr`QnW#YVyPO=A+rCDTS9{o(S$i_bO^WJ1<kll;yo@BQryA9b6vQ@GbvL4wKS(mIs
zmg-WMUKD4jOTS}yf#G?E=NO(~c%0!ehDRA5VK|-PG=@_dPGLBiVJgEEhRF=OFzn2*
zCBqgBn`7Dd`7!#n_3Qun8W4BK;kQT~ev8yeTKuKSU@+(m+P|3p@9$U}el=Es&%lar
z%YM_o$Nr(c*q&skt#A6||NjtgmS1ij*K7a3bBqPX1Y^Vhmi+%ekQPW2rMB+;e+^{s
zLI;DaBW9nZ7szCW?EaN@WYR)nsB|Mp-_<8;8(3xJ_eYW&<Wb0_8$tT6fmz6}S16_%
zl)g(GGK7V_=&R`lrSA%7<+0-uSd_jioRvcckgDH6`Y!ZMhB-Fb|03!1kt;`Rs@*{T
zu1H!A`;(!xAc0p%X~?LI)Dw*$fma9(I}srvUL#206-L8uR%O2tB=Ac1Ns@&-fPHuJ
z?HODG5_mO&1YRLDYzB)2AdMh_R|rjPJ&bb;5_pBtvUwz6Yy=6sLTK1Z1Su06K>{zk
zPs%O21Z??&O}on2O|B6n@Cu`0AJmH9*rEGoh0w5P9&%|lf&^Y6G^9X6;*>^^z$=7?
z3_ZwU(g?EZ>V2}@;1UWV*%K0)WwD8SBS_#CLPL5L<a2EV3A{pR*iT2)8zt}(tv7^*
zB(g|c*$5JNh0&1m7xy$s;HC6Q?2k)80z)J|!*eC-ElA)MM#GLSsF0onUIUkwDI(XT
z-{?u;H2^JxdaIU1_4Xw23ZY@McQkTJ;3X<V?ROWjb376-B70X3DPkIwz$=1=T-w-j
zsSzab3Mmcw=dcTIBS_#CM#G-oxP%~qR|rjPmX7M>1YX!MOz!tMXOYP>*N&V&Vh6Pb
zC-91(;Xy@doWLubhJ1Y3<f6d|yh3T%Qys}6{RSuSilAZl3S22C@B&Tm*Qtt2z-}qn
z$`^^x8l1o@oF;a61C0`RQ6nF?G>i#I)YG5@USfB*kb@%QHnz-YPy(-r(y)ynHZyAk
z3A~JcLs1r6vgO;cja?Zz$QwZduMiqGcN5i%1YSIphS0FXs~8uM!7Gx69bS=kwJ{=&
zh6MU(y~yFkr6v31(#9r8w7mjao7j7^fh1nhH0<z&OQ0-XVmJ*cEsNb8anK-(R|pNc
z{lo(pWbqo9h8=1P`E0s@EMA=WtlvYQ!Nx%a><yXAVH1W1l6Xb2u-8Ez3IsX4qFBf$
zh}57Br0|MjWvkesCEY*<uSgc2QFKN@0<YwNr;j}XME=%BkiaX1hMjao@7$BXYv9uI
zNb!xD?MdJ@FbxSJMf>ha;1xnc{^Ux!fdpQBp?y+mqX}g?3djVW7kdk(8_3@k#X_1_
zWU*@?eODAKQ^I~Q=?1cQMY53VE|YE`c~=ArJN)F3kF7zuyF{DS`#r?s^pW!n+fxYv
zHK_Xu2%+V$eG5jaHDYMkybgO-HoQ1mE{~+hjZ_Q`!y_`NHC#axy_(u5Z5!VK?5vQf
z;)ePSCys`@iKo+!rXl@lIR}~*Lqj$;B!_I6aWu4Wv`B+!VoV<3E@aS%O4xG6Z|E^J
zY?vVQsTLe`;L?!nSPZsmOleujEL^D)L&GklNdMeOcGHsm^eGw=c5T8Xl#t)DAqyJy
zks-7U_SN<qQVcDZCyAYip<yQhZ1_<>G3MN|(49y+ctST#?sIO)=3Yj2WbE~l6PHjw
zK89Arj^)U7cw8I}Ll8<kHim|+O)zNGkBOn>i#d_w{AfX=>g^-$f`s&?BGQcd^`l~F
zc(6p8_9NqH*m(;1|Bi^E6-v2Gnf?Ej3GZcO`a2yt{kHSg#oFm7Qdg%=OwGWm0L%S%
z_W%3W{eNHmR{cHZWzWET>~`up$||g*zCt-(*$Xr9HfGxIO`exroZK<FvHX?%ynKV)
zln;>0@*2{+(u2~Er6Z(WrOhQJv9eFd#cu{uKG`!2RNK(@i)j8?kxm%9?4+BJ^4ux+
zY#Tu=kwstZ7l7pQrF0WYK8l5~LZy&yLdZw5vgoVQP3ZV27E;(^KhY*+d=yIz0!6<`
zDqgfMt*2w*)5kW>MeM+vtKe~Kl8TR@!F9lB-6RzsMZ;rR^_!&PBWTDT4sF{c6(2!E
zI%jcEQt^VO_dMghE&;n^X0c^UlT>^ZEt5yUph+q|j8+gHZ3$gylT>^JO`IC4H%P-f
z(7B<s3U(jFL0QFPuyDS62_oSzQp#dymu8^iLuiEzHXreuq~e9x520c26y)e`l8TR@
ziQObJev?#u6b*MilWqoiczZ@{z6nLKnFRdLLIsTyBAy%Nz$|P4P|2s8K_1?LS;$jg
zz#|0-A5|7a8g@@<LcvF}uuDM6Z}tfI{*M7pAG_~nb6M!eW{-d$m?pe?+)hfvD;h^~
zKpH%^GJ5tVrQwaD332N;gEYJ$r6E*TMz0WP_@14d_y!a@3gvd}VxJW^Akgq3tPDZ`
z=_WKhSK7d=GM<5Qx(Nv%#VR4;ELc$Rkt{?f&}IVxFZZu9oPG(;a}gUk`OQGUhtRN1
z1L7u42zVYd!)PKawBLk+kE9{en8iUM;iGBT6%S7*G(6MH{>>JbfHs7kv`V<05b=>T
z#Fw!VRWnfWA*F#OI=?{0htM#vAmw&5Q1M|j?AV7ENh)4E>q`HADBn@cw^u|$b6f(c
z_;6YoJL8FZV-+7-8a9d-ak?g}_;4Dwp+Q3JCad^x8UjCXKbx%LL6iH9$9crUGT4Z^
znnggQ$tpgAmWPkwHv<(PQW|!c5<PC9;=^dzTMXyMDn9(6h((vt{Xxa^wi!K<A9|9p
zxnI77O>H2%@w`LCN3t@AF{hi*@KG!Tzq9B{A>pG~5WfYe6Dasd7CIPISs>u8er-r>
zL|wtQLS;xyzd-_Cbbev9EH)?b8zkT(XmERke?$U4f`&bGghxXHK8lt{Sj}&cfEP5m
zCx*m#ASZTVz^2F8!_aS#fRCW%;LzYoNx%zdHl#F!t;8%!AmBr2h?)q0JP`09G$<!*
z6VM0*ywa}`Aq*z^(+pDHBg%NV=#1z|9*|bT&MV>)mUs7A1DA&FcCm+{-&hty6Wboc
zZC@I5P!a0Ixg8di)-xnj+H#^N>c~{vv3)2aVSZyt3=Q3f2+=K$p_Q?%5NL~HXs8kF
zPt{l$L&N5vqO=8qCaPEKvylWg>BN7<cHDO%u^NZQ(Q=rD@EeE3&{kh*XABKjDngqF
z$I(Pg!Eem(rYZdb^96*2;UHi?5)n_GCurhX2&2I>#?6`=Ln~tUThQjj92C!=XqK~M
zN`s)shD?ovM874@t=~<^XW`1VW1B-bOzFm~NS5%PML6%kXcnBM972OLV^~>i{hn?d
z5Y58YLXbR-{k!X8;0b9cl!}<2+AoHdK~<p2_Kl|DBC}caGW*2QF#lH&{{PhE*FFA!
z^!|SX&Hpd5cd*;A3jI0c0B9f&eaZ6hj`Xk1i_IgEJ$`dDi3|X@8{d!F|8GO}bM+bZ
zTD1!q)(fhu{8_mlZ$cl2{OD<%Uh7}023j@nf4K(y6(rk5ud4NNcX$`@Qajd}K<fD`
zNVZ4O;LGRz6(rjuXjyEaR`OSnY>%K}-3mO96(rlEXfzwKf@Hg(*#lle35#Xmd8{DW
z9!1NR1dW345j4bep=ei7@I8W-t6(nHUqQk5D4JM&<F5#U@792qAXYCxVXO#(?_sno
zmI(MOdbIn%rDaI?HhZ-D05o{I)j}52GtD0DJ}?b2GKi!mX?G!#)IM=mG^8^5%@ACW
zgH6)z5j4yL3cs1OdlU`xa`2l;yGPJUMMU5ICTsWbbHgT$B4`+Bccmvr4~L=q63W;o
z3G=^A((dA*VYIy1yQ|rw-3KlWk!NuUJ=#5lhHYMP38dX44vKgK1Y9%F?phzW4Ldzy
zB@Z@x!V)e)BkeAlWeBZ+kS4B_w7WQH7!6y`!>13ldk76%7Gu()8EE$q8n%`eEhNzH
za=-SLrIkETs$~%z3bcC&4KiAUM@hSjD-EGxPbd-fA?+ST%fWf}o21<%Xqbn`KBP@(
zcb2Ms98cT?x&ru}n6FAVA>E@`dAMU>LAgh=upTUzZbG<6v2x`s_Oot6w@0$FVlUff
zAlv00rA?1Pmi93hle^-slWZ4_BaDVsil|x0c6OdZXs8TP8gzRk4W6lp$wIhC(sEeu
zf%^pI&a`B|OMu@DXQ2SOE$%v`JG(m}G_1bBgkdw#?jbZReG=iLK)Z+0Fl~#{0_`3`
z!#?rYNU%xTT{Jbb$8V;)kSU6#REUyg;QBZB?!JUD8uqzHNN`3p4UU*tFWB5GhK7K*
zn9|uZj)t%sO53BGruV-Kd9er=+qGvzzc4+9hUGLO&c1sL4M8_-HQAgNM?+{FwB2H8
zSXC&Z_PYuiHS&I=S_U(>kVjRx*XSsx#?UH7^fP{QN(>DlR}s*j97n?<P~3%GVrYmB
z<xtwr-85yuOTY%!Rjhk!?i5YKa>1f_vL?mQFeil#W}7?4(BNQ0MKyPbqhWFo!N`dL
ztxrS39B#Rzm}|$%&Z>|g6Jlr?5xn=C+sDz+iQwGEN7KYQXEB#CE{0aA7KES~D`?_z
j4w;6=YC%C86Gy|YlsM?<7+M7#97@|RhE~P~?AiYVicP=2

literal 0
HcmV?d00001

diff --git a/experiments/league.temp.csv b/experiments/league.temp.csv
new file mode 100644
index 00000000..7ff08ad1
--- /dev/null
+++ b/experiments/league.temp.csv
@@ -0,0 +1,9 @@
+name,mu,sigma,trueskill
+POWorkerRush,38.861939273926204,3.2121311873637186,29.225545711835046
+POLightRush,25.754744223814487,2.2179711777256155,19.100830690637643
+POHeavyRush,22.98870034666808,2.212164591404348,16.352206572455035
+models/MicroRTSGridModeVecEnv__ppo_gridnet__1__1641331302/6144.pt,21.481666441292322,2.812293923609439,13.044784670464004
+models/MicroRTSGridModeVecEnv__ppo_gridnet__1__1641330893/6144.pt,16.10799174066183,5.348608643642864,0.062165809733237154
+models/MicroRTSGridModeVecEnv__ppo_gridnet__1__1641330984/6144.pt,25.0,8.333333333333334,0.0
+models/MicroRTSGridModeVecEnv__ppo_gridnet__1__1641331138/6144.pt,25.0,8.333333333333334,0.0
+PORangedRush,8.881771566353699,3.3669435969510504,-1.2190592244994534
diff --git a/experiments/new_league.py b/experiments/new_league.py
index ef32f3e3..77279dde 100644
--- a/experiments/new_league.py
+++ b/experiments/new_league.py
@@ -68,6 +68,8 @@ def parse_args():
 dbname = "league"
 if(args.partial_obs):
     dbname = 'po_league'
+
+print(dbname)
 db = SqliteDatabase(f"{dbname}.db")
 class BaseModel(Model):
     class Meta:
@@ -146,6 +148,7 @@ def __init__(self, partial_obs: bool, match_up=None, map_path="maps/16x16/basesW
         self.device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
         max_steps = 5000
         if mode == 0:
+            print(partial_obs)
             self.envs = MicroRTSGridModeVecEnv(
                 num_bot_envs=len(built_in_ais),
                 num_selfplay_envs=0,
@@ -452,7 +455,7 @@ def binary_search(leaderboard, low, high, ai, n=5):
             ai = AI.get(name=new_ai_name)
             binary_search(leaderboard, 0, len(leaderboard), ai.name, n=5)
         
-        get_leaderboard().to_csv(f"{dbname}.temp.csv", index=False)
+        get_leaderboard().to_csv(f"league.temp.csv", index=False)
     
     print("=======================")
     print(get_leaderboard())
diff --git a/experiments/po_league.csv b/experiments/po_league.csv
index 83df9bfd..01ab6b0c 100644
--- a/experiments/po_league.csv
+++ b/experiments/po_league.csv
@@ -1,5 +1,5 @@
 name,mu,sigma,trueskill
-POWorkerRush,39.969639589352205,3.468613295987325,29.56379970139023
-POLightRush,26.237614972853905,2.4859644240612635,18.779721700670116
-POHeavyRush,21.253428741884882,2.4920725461958844,13.77721110329723
-PORangedRush,8.351719772683518,3.0734987337830626,-0.8687764286656705
+POWorkerRush,38.861939273926204,3.2121311873637186,29.225545711835046
+POLightRush,25.754744223814487,2.2179711777256155,19.100830690637643
+POHeavyRush,22.98870034666808,2.212164591404348,16.352206572455035
+PORangedRush,8.881771566353699,3.3669435969510504,-1.2190592244994534
diff --git a/experiments/po_league.db b/experiments/po_league.db
index ae6a8a34611ab77478f5ef4d0c7631a5f4c7ad3a..4c331b184e9565d0a0a6d38fccbf249cd6702771 100644
GIT binary patch
literal 24576
zcmeI3du$v>9mnr>_n!9c&(6zvwOQY;leo1nGyC48v22{nqxB=XIH8e(P8a(UA93u|
zagwG`ASp-$As#{?q`)5(QL57N2o*1(mO?;?04k9v6_0{iBxt14@@hdX6*IrY?zz4W
zQX~F=%*y`eZuh&tnf=V{Ugm3i_pX83Ow~9zHGQNqV>q#vSUeu<HH=s+c7c5-+4t<0
zU@x@UH|*bd-Rn1cc_CI9zB|W`CYnDGgF#5=&gPQrrQ9R|Nk9^i1SA1TKoXDyBmqg_
zx+ZW}BGtCJBYt{KZE~!7CtHn-j*r)l%}h<7DvwnUR@u(!a&4@Bf3m++>>DW>JBGFt
zZ#VMwhw4ws8^c5MkLHc_`MR3<&eJh1)z-%vU$?OF(ecW}M0N7eg60!~=I5Q%xC8So
zooC*urrJ6>;_uruQ#mkEt-qN1*V@%aNBZ6}P+WLbwXr<IF;d)CEEz+?BgW94fdOM@
zX~$q+X`gXhabK5_pWp5RH*?gZX>|N#ZE|5N{JiPP$%Y3erj8wJcwlt8TA8VimB*Oo
zjV)|2M|KPr=X%~{Y#lBYcWfKtU1wu9zw4bwskpURDh~A*cN_UijgKn3{?6B*Rew2e
zbWUf#l5AVm8n5B_S8C<S%8_cNre3AJXK2T+J#!Q7>ftNHdZk@%XVZwpQ*EuS@iVvK
z7*=YPnlgX9=6bn+>+41d3mSX-O8wjWO6zUU>#U=aKXN?JF1~M|e&4a$p(B-ryIE&v
zP90@*8`kQ~!GTyR_hJkjxC99}qu-|wvA1%Q1SA1TKoXDyBmqf45|9KW0ZBj-_`e{q
zH<gN=2DVr|GBsA6IJRN1Hab038ri*Vx;8e*cI~Z>7AH@X%SVq+l@GCflhv7WnX)h6
zrH(_X6K?RS>lTjAtmri!;M*6bC(4!D-sH8^B~j=$uKTtGubX<4{Uu@K%+~ECt3G%8
zcPlf!OUM5CfNtB^?JhMLGgB#Z=kWGw<;1Dd@nhq?{=Zh8-1?5M_r}lPv+wY^g9nb+
zCT7aDNp_Ll@noz6-EW<mK3ttfd;jD}^2@vT^k&ZA@kIR0!*zD}O3V;Vnr>~jT&coR
zrZFtn{=JhQX=xeg%|8F0HP7U-b>&#8c2T)lmoj@X1GPirGgvNq%)k7U@sr;8BZnUB
zdHbVv<+7|+O1~U~$KiMID7*l70%dRICJ9Ial7J*22}lBxfFvLZNCJ|8Bp?a=j}q9b
zvBcw=^N!S;FH;|HB(I@NU7C?-Q9I&0hhGa0G*C^pV=l%}ue_g20u)d?GCPM~3oiKY
z=JzDo`u|t(HTWn@z&2>tU)F!Fe@Q>9kLsKCw%ng{7jyUJPUrULg509!=bC@m{MqJr
zHScS7nsZHmXnL&aT+@lBT}@q0sqE9)Z)ZP|J(|5WYh_-|JefJ4`C#T?rZ2NP{Zjf;
z`l0mu)1&F0^orEqQa?@IpE{G;pYl^JtZ}(X0+N6vAPGnU|1|<uPn!L0TRNTTOQ<%n
z?QTN5iEB`QBl{v=Aq=C4de-Lu;Bo^>ed`wP%B0hMidJ9C+vvF)U9VtAjz<D(lh{Qg
z*R}&|qu50P#|uI$61m($<U5quRw#CbP7n~=4AQa8mEJGxyjuuZi*8{0LKFKzkZTju
z6Pi4S_>OJ5LX+#;K5<P)XmUcIc+|9KO%15K1v`ksh?-Prq7iZI&?I7$9eR|Q8-ym}
z`GmnN2uw~P3jN57tZqIoeC{dp=H8Y0;uOM&gpOl%&5d3|R}eTpCDzSim+!F&O01j2
zF3%1_;#r+ym+jeJK&<t%F0}!7r@-2ygj(ywE<)IZ@vXHYmt6>n8@i6wA$Iwc`CQ8q
zyIdOCl$z%305yPW7aTSneAmniO}69sj$^h9P1FuuFE9<EiFi&F1m+rni58-OO)by5
zkzXF4G@7#TKBa}ww`mkvH;7$9;8Pk}tHmzg4Q$`BR*7AnOG!kmm0}k&+0-#tFc*75
zvMJxF^ER8z?5<xfG||Wj!oX}3o2X4(YAzF+2phn_HJ1uZq!4+Y=LhDJSyO|_P8fBM
z)iM_gO@Zrsgqn+lCO@#*{bjZaO<qK8*EL(jCgM<!TEI*=U(^Q3Nx`u_YDbpNUHqe~
zv5STblw;+@F2*w;q17yC(TNzVAYdP<O=1^&s(H+noposo?^9OGV-E)QAkBzf4*LjT
z51F*sMQwIZcvec}3JXyb+77XjViy}~-|@@@bKz`n(D|_7xFI2tsR>QC=SJ*FqzX;c
z^#ejoMQCDev8S3D7nlO}{~gLcPWaFNd-PAmU_10f4@BVVf7Sn_Kdb*<|E>N@mIFMl
zU(|m9_rTq767GP*&;nV|^jGzN=-<)5r9Z5HQNJHvhQGt}@GLyTeuLnf@KtySJ`V+0
z4|!M(%iuTg3-}pagzv*=;2eA$J`87}3T4;_yJ1kjmo+XoNk9^i1SA1TKoXDyBmqf4
z5|9KWf$N$;LXF20J*b<xDz~6+LfwcOp@y28i3ePKE*=+`i^Ij{Lb(tw8@Lqkyl&Jk
z)SFRnLha<Lu18&mx)!wq)#3$AE_p8PTnsL2_+jlv)EiJ&qpm_-$yHf_x*W9)bs6eX
zuG$jR#i)x=TTxrMs(`Aa=1`kao49IO)C_7GHHDhwswPl1R25ax)ObdnTmQcTFTr2n
zIrt-6yWb0+hEKxB;6rSs{$6-D+yxil5%>n2hX-K>CSe>7z<&KH{Ym|j{$rL7JPp5w
zC)i&Q{0R2KPPi4euq0t6ECC(TpzswwgkVD#tOY}VMSoF$LBGuMg}MCTx}N0n;w1q|
zKoXDyBmqf45|9KW0ZBj-kOX45OvUxTitB$3*Z&%>|216yYq<W`Lhe)oR3Fttbx|Fz
znvF_P3F-z^OafF)0#r-_R7?U?OafF)0yInlG)w|COae4a0yK+PP)*c4YCEcdigAE~
zae#txfP!&=f^mSdk|zc$xGd+=#$_3orCgS9S<Gb-msTzq0w@>)C>R1L7y>950w@>)
es2BpM7y_sm0;m`Qs7YQR!A0Yua#2(@lleDRfCvNt

literal 24576
zcmeI4ZEPGz8OQH-*S@{CeeA@ETc>F@w`<4e*q7becgKLQ&LuH%?DXOkEQ+j4e2I@Z
z4%iMM6$ld4NU0)TMTJ!Q0VLE&yhuPmEmfN$1Rp4ZQYsZHwE`MI<pZUHA|yg#=1J_F
z?d$jpstBEx{mW+ezdN(Pon2=h+c|jmNNu62A3ZsDqOzbnv6fgo9vjs4SS<Dg{cNV6
zi%Sz-Q5OF~|BTnIzuD>&vBJL3X6fFh%mXpl4{G*8wwbPqMHmnUgaKhd7!U@80bxKG
z5C-1X2JUN0w7;V>es)W3cB=XjIvY7MJzbk$I5~G_a;kc?N;l3;)~4#WHxHFd#qp9p
zJi4p&UOiX8tNxIjJ~q01Z%*&d)txh!KO0jL?M2%7w$+UvnXb&tRA-N^YQBln{IY`@
zcVKxWf9|7lqP?>-e*eye%Hf%6{l(O;D>oV)FWxm$T76ZusT|cYUfNSC>!V}i`sksN
z5&b}Ucz>}xq3<hA^y<0g?XF7GM7^3u`%l+qSGU5Bo2#5|xMSwz{CvY5N9L-Ph3eGg
z6lG4|MQ`T#@cz<L&wKUVW98EDo>A6yde4@3J+GHbyG!NL=uqjPo~zW@U8UEbUwKsh
z<-FN3`5~#feN$_^hWEcxo1Cqjs8(w74cmuChwnbLG|+DBzJ6P;x69&m7+LW|duwa_
z+&;XAm0G1HEx%t&y<FvW#a+S^>h~1OLwk$mZsK|QIxD#or*ib-izD^h=4;1JR90`M
zon1KdL29?*x$;?fAf{+<#I!fyS?GoLy{%tC;^Kt?VL%uV2801&Ko}4PgaKhd7!U^j
zPYv|On`1^IVH_A6sU4eMD4&|29_$}@?$Bd%7YF5+iwA!9hc6vIRhwCutj$hVYJGBZ
ztP|7a3atlCVO+o5_AqI=e^7n!{8{KJ*T<E`_<K&y9k0$|eCXQdt`9soF_^yc=)vD#
zjn&1e;=TV3&Arm%HSMidK70mS^#*@)`N^SmgYv6Cd_4dDC+k{G(IyqR5`*8uYw!$R
z5sNS&3<v|lfG{8o2m``^Fdz&F1HynXAPoGk8EBC^;|IoW))$oP8*bJoWRMM-SSkfb
znrtNN${p-y^9K@s3|@fC@F|#vyI`~S5AB!QH?<4el=d#IBl~*xne5lH=d+dUK(;;e
zTIT7@S2G{aOk|u)HvPx+57G~%Po>Aw`LvvRG4*Kb^Qqa?aLP>nJNZKLyUEWak0*C0
zx2o6F=hbhk7uBQcpxTl6TjFPlONn!dduhXB5e9?-VL%uV25uPx=736nPcEsdMM)+W
zvHNWL;p?{FZ(IEqDTHniIlj4*p9;Ld@rc>KM#_pp*D)i0%6IHAB4)@>d4U~}zzkSQ
zRg*<YS>1cP5E2?9rq53WJ_!Tg^wvmO^jwzd@>9NR+qQ2y{FE0u#P<!N#**m1fqJ_@
ze9!Y-!{&zU$Z~Dhu=pY3*p_GP;D#*M4ZX-Ha6(ogazdZ@M&DvcT5a7bc(xn(p3%z<
zxi0bCz}U_WIg#rV&*<TYh#v%zk>`f!F!0^b=w=}{URDejE9&>6Lg0tC?HJqmA=`_5
z%jn{U=peRf$2++pI)+2rH%xxW3Q6Rd#^QjI8}{6`qmWo;j-RrKXVFJ%D=!rmB9G89
zZ0h_J9S6jV%q{#>;768C%sZD-D+Ubn9u_>}gpO-&=BHfGVK0U|_$eo%i)U_HBSjww
z-!nV-DH6D$@0z!>6njM~4SFBYQR6z6XWqt7`96I`MrJ!d<ptDNXl`61MQq~R<_3Ps
z4IGDrMjK7h&L@j2UNTI5P#{5M2d=T68?qyhJ<#j;A=mNf6V}QNS%KxyXSIbNvgxh%
z3}7Mr%C!6?!#eI4JdXsnZD`yORp3ztS#HRQNJv~G!w=C1CkTu*H$+@7vLhq47;5mS
zx`oJ(g1|D8+)x;?_oJcmLnN|&VkEesz;_~QyqO=eN$3Y=6AfJ-=nck-TX5(jLu^yw
zr-<tZR$$8f6#W`+`=KeVks_Ao`(~V<vH~Xv9rpeI0qs%@dY}_@*aRD)6*7<jN&6SA
z1N>F{v-XO16|TTz@E!ODJfgj%{YraQ`>FP0?J4d1+GScNxCCE@hu{nFX}ADYn1l&9
zNPp9R4-{c1c(4Qh39rMe@Cv*PFT(Ti3wRoS1fPVD!D;vq9KU7$$tOB03<v|lfG{8o
z2m``^Fdz&F1HynX@Rk^8lH+k{0J#&{kBpEZGC=xB59uNuMNY;EgUw(u>|iJ`^fB}@
zY-i|U$TM^^Y-8wR=wvY231tJBLvBUt$Sue_k(-fsAU7d97?s<Rw;|h+8<86r<u>Gc
z<T_+4vV~Csq=w8QGsrZfl0qhtDl&m=Mm8Z8q>PjlIi6IO=Kn9jui-iPIsAmq&YyzE
z;d}5c_&PiSUxA0=i|{$P2<PDwa33tdEd42g!*DNjKpSYFf^>6+uU*sruKh*(lXkVP
zI&cphfOo?#cqfEF=>IX$1^Ue@!do(UL?gn0Fdz&F1HynXAPfit!hkR!3<v|ai~%-f
zm2v(r<NRO7`M-?we;Mch3eNu(oc}90|5tGSuQ+%tLE1<QxdT~Xl=_gp$nD4;WS&v!
zMs7oPAv=*+4p6WhpkO&b!E%6t<p2fC0TPx2BrFF=SPqb|93WviK*Dl>gyjGU%K;LW
z1EdYC+R(<Zo?#tBD?<wdFlY=}h73cRA;pknP#F>oIRBS%{x9SFU&i^rjPrl_KRBRO
AQvd(}

diff --git a/experiments/ppo_gridnet.py b/experiments/ppo_gridnet.py
index 1f213725..52bba166 100644
--- a/experiments/ppo_gridnet.py
+++ b/experiments/ppo_gridnet.py
@@ -280,7 +280,7 @@ def get_value(self, x):
         + [microrts_ai.randomBiasedAI for _ in range(min(args.num_bot_envs, 2))]
         + [microrts_ai.lightRushAI for _ in range(min(args.num_bot_envs, 2))]
         + [microrts_ai.workerRushAI for _ in range(min(args.num_bot_envs, 2))],
-        map_paths=["maps/16x16/basesWorkers16x16.xml"],
+        map_paths=["maps/16x16/basesWorkers16x16A.xml","maps/16x16/basesWorkers16x16A.xml","maps/16x16/basesWorkers16x16B.xml","maps/16x16/basesWorkers16x16B.xml","maps/16x16/basesWorkers16x16C.xml","maps/16x16/basesWorkers16x16C.xml","maps/16x16/basesWorkers16x16D.xml","maps/16x16/basesWorkers16x16D.xml","maps/16x16/basesWorkers16x16E.xml","maps/16x16/basesWorkers16x16E.xml","maps/16x16/basesWorkers16x16F.xml","maps/16x16/basesWorkers16x16F.xml","maps/16x16/basesWorkers16x16G.xml","maps/16x16/basesWorkers16x16G.xml","maps/16x16/basesWorkers16x16H.xml","maps/16x16/basesWorkers16x16H.xml","maps/16x16/basesWorkers16x16I.xml","maps/16x16/basesWorkers16x16I.xml","maps/16x16/basesWorkers16x16J.xml","maps/16x16/basesWorkers16x16J.xml","maps/16x16/basesWorkers16x16K.xml","maps/16x16/basesWorkers16x16K.xml","maps/16x16/basesWorkers16x16L.xml","maps/16x16/basesWorkers16x16L.xml"],
         reward_weight=np.array([10.0, 1.0, 1.0, 0.2, 1.0, 4.0]),
     )
     envs = MicroRTSStatsRecorder(envs)
@@ -340,8 +340,8 @@ def get_value(self, x):
 
     ## EVALUATION LOGIC:
     eval_queue = []
-    trueskill_df = pd.read_csv("league.csv")
-    trueskill_step_df = pd.read_csv("league.csv")
+    trueskill_df = pd.read_csv("po_league.csv")
+    trueskill_step_df = pd.read_csv("po_league.csv")
     trueskill_step_df["type"] = trueskill_step_df["name"]
     trueskill_step_df["step"] = 0
     preset_trueskill_step_df = trueskill_step_df.copy()
@@ -476,7 +476,7 @@ def get_value(self, x):
                 torch.save(agent.state_dict(), f"models/{experiment_name}/agent.pt")
                 torch.save(agent.state_dict(), f"models/{experiment_name}/{global_step}.pt")
                 wandb.save(f"models/{experiment_name}/agent.pt", base_path=f"models/{experiment_name}", policy="now")
-                subprocess.Popen(["python", "new_league.py", "--evals", f"models/{experiment_name}/{global_step}.pt", "--update-db", "false"])
+                subprocess.Popen(["python", "new_league.py", "--evals", f"models/{experiment_name}/{global_step}.pt", "--update-db", "false", "--partial-obs", str(args.partial_obs)])
                 eval_queue += [f"models/{experiment_name}/{global_step}.pt"]
                 print(f"Evaluating models/{experiment_name}/{global_step}.pt")
 
@@ -500,6 +500,7 @@ def get_value(self, x):
                             "trueskill": league.loc[model_path]["trueskill"]
                         }
                         trueskill_df = trueskill_df.append(trueskill_data, ignore_index=True)
+                        print(trueskill_df)
                         wandb.log({"trueskill": wandb.Table(dataframe=trueskill_df)})
                         trueskill_data["type"] = "training"
                         trueskill_data["step"] = model_global_step