From 4197f74231c96d768e530ddb063bf538065eeeed Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:49:33 -0700 Subject: [PATCH] delete CPU threading support (#17866) * delete CPU threading support * more removals * fixes --- .github/workflows/benchmark.yml | 4 +- examples/mlperf/dataloader.py | 10 ++-- extra/export_model.py | 3 +- extra/onnx_helpers.py | 4 +- spec/tinyspec.pdf | Bin 99412 -> 99291 bytes spec/tinyspec.tex | 2 - test/backend/test_jit.py | 15 +----- test/device/test_hcq2.py | 10 ++++ test/opt/test_kernel_opts.py | 14 ----- tinygrad/codegen/__init__.py | 4 +- tinygrad/codegen/gpudims.py | 7 ++- tinygrad/codegen/opt/__init__.py | 2 +- tinygrad/codegen/opt/heuristic.py | 13 ----- tinygrad/codegen/opt/postrange.py | 10 +--- tinygrad/codegen/opt/search.py | 1 - tinygrad/engine/jit.py | 4 +- tinygrad/helpers.py | 4 +- tinygrad/renderer/__init__.py | 2 - tinygrad/renderer/cstyle.py | 6 +-- tinygrad/renderer/isa/x86.py | 6 +-- tinygrad/renderer/llvmir.py | 6 +-- tinygrad/runtime/graph/hcq.py | 1 - tinygrad/runtime/ops_cpu.py | 85 +++++------------------------- tinygrad/runtime/ops_dsp.py | 1 - tinygrad/runtime/support/hcq2.py | 9 +++- tinygrad/uop/ops.py | 16 +++--- 26 files changed, 64 insertions(+), 175 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6b1d495f97..2a9ff72172 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -648,6 +648,6 @@ jobs: - name: Checkout Code uses: actions/checkout@v6 - name: Speed Test - run: DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py + run: DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py - name: Speed Test (BEAM=2) - run: BEAM=2 DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py + run: BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py diff --git a/examples/mlperf/dataloader.py b/examples/mlperf/dataloader.py index 275bcd4aef..3f5db0c74a 100644 --- a/examples/mlperf/dataloader.py +++ b/examples/mlperf/dataloader.py @@ -5,7 +5,7 @@ from multiprocessing import Queue, Process, shared_memory, connection, Lock import numpy as np from tinygrad import dtypes, Tensor -from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX, NUM_CPU_THREADS +from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX, CPU_COUNT from tinygrad.nn.state import TensorIO ### ResNet @@ -131,7 +131,7 @@ def batch_load_resnet(batch_size=64, val=False, shuffle=True, seed=None, pad_fir else: X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name}") Y = [None] * (batch_size*BATCH_COUNT) - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): p = Process(target=loader_process, args=(q_in, q_out, X, seed)) p.daemon = True p.start() @@ -212,7 +212,7 @@ def batch_load_train_bert(BS:int, seed:int|None=None): rng.shuffle(fs) train_files.append(fs.pop(0)) - cycle_length = min(NUM_CPU_THREADS.value, len(train_files)) + cycle_length = min(CPU_COUNT, len(train_files)) assert cycle_length > 0, "cycle_length must be greater than 0" dataset = InterleavedDataset(train_files, cycle_length) @@ -301,7 +301,7 @@ def batch_load_unet3d(preprocessed_dataset_dir:Path, batch_size:int=6, val:bool= X = Tensor.empty(*sz, dtype=dtypes.float32, device=f"disk:/dev/shm/{shm_name_x}") Y = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name_y}") - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): proc = Process(target=load_unet3d_data, args=(preprocessed_dataset_dir, seed, queue_in, queue_out, X, Y)) proc.daemon = True proc.start() @@ -437,7 +437,7 @@ def batch_load_retinanet(dataset, val:bool, base_dir:Path, batch_size:int=32, sh dataset_iter = iter(image_ids) try: - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): proc = Process( target=load_retinanet_data, args=(base_dir, val, queue_in, queue_out, imgs, boxes, labels), diff --git a/extra/export_model.py b/extra/export_model.py index f5d5a09a84..a6baf20bc7 100644 --- a/extra/export_model.py +++ b/extra/export_model.py @@ -241,8 +241,7 @@ export default {model_name}; def export_model(model, target:str, *inputs, model_name: Optional[str] = "model", stream_weights=False): assert Device.DEFAULT in EXPORT_SUPPORTED_DEVICE, f"only {', '.join(EXPORT_SUPPORTED_DEVICE)} are supported" - # NOTE: NUM_CPU_THREADS=1, since export does not support threading - with Context(JIT=2, NUM_CPU_THREADS=1): linear, output_bufs = jit_model(model, *inputs) + with Context(JIT=2): linear, output_bufs = jit_model(model, *inputs) functions, statements, bufs, bufs_to_save = compile_net(linear, output_bufs) state = get_state_dict(model) weight_names = {(id(b), b.offset, b.size, b.dtype): name for name, x in state.items() if (b:=x.uop.base.realized) is not None} diff --git a/extra/onnx_helpers.py b/extra/onnx_helpers.py index c11a8d41f8..a0a0363d4b 100644 --- a/extra/onnx_helpers.py +++ b/extra/onnx_helpers.py @@ -1,12 +1,12 @@ from tinygrad import Tensor -from tinygrad.helpers import NUM_CPU_THREADS +import os from tinygrad.tensor import _to_np_dtype from tinygrad.nn.onnx import OnnxRunner, OnnxValue import numpy as np import onnxruntime as ort ort_options = ort.SessionOptions() ort_options.log_severity_level = 3 -ort_options.intra_op_num_threads = NUM_CPU_THREADS.value +ort_options.intra_op_num_threads = os.cpu_count() or 1 def get_example_inputs(graph_inputs:dict[str, OnnxValue], config={}): """ diff --git a/spec/tinyspec.pdf b/spec/tinyspec.pdf index 5e8ebe58b3b28885d72f51e27f594eda73dd59ed..7dec94f0b04948bfb83d1914651ec732e868e6c6 100644 GIT binary patch delta 13174 zcmai(LwKMKu%+X4Y}>ZgvD2|_+x}wP?AW$#ckGUBbd33DR&!T(TTh*HwsqcGih$Y- zhpK0%1K4;tIDh_xb#ZkzGqQvA+_=(}cEKCN=snRO=Bii5f9@ToS6{DKQ|sWFpSceN z5fgVJ-%Ofvy1#hn(JxIu8xT=JA&{j_7&a_TZ|l4189fe1*kS%S!K33$d!JbBoB86X z?HRt9di3V`{xV|Ia>%^R2JC8&hMSF?hptSq0S)&D@1o}1YL~yM(cfDib`Q(P4;I-9 zy}Q1_JTc7H50^g$=eW#`Wn|jC++W_pSY~dO+jjWhr$)~reYS!t4lMS6vm7{3jP0z1 zyw?ul(F}r1$mwz`5i*12?UD2U@J_$kHb@1giXixKqy%#$&z=MDziyep(g{f+xJ2ap@B-q-~7gn3`b@q}gD~Vg!Lb6MKpQ6wV*B43Y4W zqMk^YCfIIh;rRs-WsS_h1oD3FA@+CH!-P%j+#P6 z^N(B_8tZWRex)OdHBhT79#t^m<<3Ny5+}3O3y&W=_OW(3IW%cpQVBKpvieR5(Q23( zcYW(Ue#_VPeYsxbm%j8RRc4T3-9r1N`b+jWREumM(jCAy9m^w@gjFMz6b50#)8hB+ z{>|}jZp&{dP~A$q{W?}KV97AOH%4dF_WkvBuAOm}fRx|np7k&Yof`zAH0AhLY+qT> z#)gH+2aU}Je+pZf`YV&`+uY9Cp(p)dg3rc;vMkoY0@zqcxAS6F45u{rKQAT0-ZH(g#yd67 zmU)_`gkBbNxpHpFf?q8}1K*K}x>d*ELS_p+gaXD6lZ+Q4LjJZPoD0~Iq2F*g+Y6wG z&kneLIusp`m0Wc7?Ic2MkOyME;)==mITBtQ*AopL4GVd&N$T=6J_w`C{(xkfi4vPm zwE}B+A3rh3tBq^Yg)OM|iZYV0$j|0JAi=+;-eLVHAl;x^zX)7F(RW<1%kmSfx+A&g zoCCf#cmFv2f{wglUq$=zFbC5bLJy0&|MrfdaZAqH;IdoWwFmD`Dw%CEAefWLc^Qx) z?9`SQt1%4{=QG8GefzK{6eo1}`a6}|fCSvApFeFJ4-^uG?StArp2@z>pa*9*C@H3S z%dF9#H%I)JjApupm*4ZlpU@xVG{%?#A%TDt9~mhv8hvP(n+R4wCk9+eK!TH9Vso5W zX=Q{jdgd5LW>;RG!`MSWy?mu@@HH5zLu(hyp%BS9yh9qS2wP!Jj@klYst7jXi5Nww zO%>W&cqYT4g_WC0kvN64ow+lAAFiaN3`yUyDFxOoMDzmP&B?u{yn|`QjzEFb**74h zoTr?I+N2ki;Gj3yAc>BdtfbzB8;<`=g4NzDbaIvLvWkPVh_0~9o#T7KKDgoE1SZv7 zC$-V{WQV|PjH-&xcqh&+m!0ylSYS#=#zordhCdAUgy8uj1?W_kUwX2l#5ipy#e1*8 z!_QL`r}N;Byt%qsj?GezFDnMH^etdkhv{cIQb_*EC{G)r9p}HNg9NO-KNBLi#f&ub z7gyjXT~1uG(7F~Qd?`?nBeWFspCB+V`e{)bZzdwn2AGY+F^Z_hN8u&G1)+lSS(nWx z%jHNr1Os7>P-rDv)lM`sK5X=SPAn`3FZ8DQw=&*ZNE54*9K^Wi2*%PbBXa=hUlqn> zh(u}D1Yk-+Lao!Kr2hV*^7=kz%H#U7;ORhDz>D;5s8Wq5E+({m$b-R&=x*+rcDdom zX`L{NV~~y3xzlaa#I3{L4?TEA5*PeI1ivL#X=L-}8u7i$LoiNEO1Ln!2F#N9U?sv- zLJA|Ot%GU4%s4ezK5vLcpAMLV7AST(iusuz2Pxn|{|DsYDU&;LCbWJj7L|&+rsxHt zBe2zY3pDY1?s+uv)cW?aYk#6?F8(>(mrSE2buRcqubbzZA1V25APmC_cD4AKu`7?{ zAJt>`q%zLD4I?^lRZbs-<1eEPY-;g77B263?Ng00n}u#d+LjMgYEQree^>|(hx}$< zopK9Oc9SU9u7UkW_TgAs*iU=95G_~f6DiiL^)<^da$KPpimPkz_ z4s^yM54E6AWOlw;4wcmv-QwxY5WveEyv7en33v@9uW3k52h$UrI)G~^tBdp+GfY05 zj&6iBsBgZMNr{XTrQP9)qjs`FZT)>eLZv{D)R?}lmcbxx*!vsUWEFMB{h2L`y60lO zZL7CU%S?ss`%`qvq2Qm^L`m`Ps*Qfz)p1LClJy~bC-WcoGg!?0=*b-0iAybWVpn%B z#3RMGLO9S930!1ltsM`xbf00lQyh@Fyp!1r5Z%rX7>63>x7%%Ul&T3cbymZVE!pDb z!T|<1_pIr|_9_y9g(Re?;h0Z?GRYo)V{sz31AKzuN_-Qm{Wn)`MB?~!;E!kHk$O&|k=HjM5;Ouzpp#$v-uteI)r8npH*HU9`c?+5W3!B^gX(!@AHdy zS+)QT72no7ay`b*F^T0&p`u8!XhjYN}Z-HO5* zTd7)|>DB^JA;lq$>!mxx9dbQdPPQ3mXF(t!ZA|y`Z&YSf-Nf}OuMjA;^eT&+T(4A5 z0?lO?SGyPs$olZ+U@T!XDle)r4mwkW?N{ha5}0yyzbtR`c=II%m9Az?HDv0jc`qqA9A>}>U=r&$@4`4zZ)zHjX6?Dx`@V#gL#F+%G*sd z+jJ#kR!bJyl+VwyK7S+GwCy)pQ2H3=2~ddbXeM_wcGg!r{!C~7V~<)E6YZv*g51tk zK3Zfd+rXZ!5P2R25)SWX(GhE~rnmbb|tDG(j^cLQwB}>xYEHA&FEE9i8(}Ec5(l(M9_nTSDyS+X>*9T}aW1gxm zV$~7_R3a4A)*wEmpW9d?!?av(q(hxKlJR8`5~Ed!!8D)fmZB=vcF34;5r6^V?MuF0 z)SnuiW33HSE?o*NcE6zfxAqJuxgT}Zo zCb)my;~0b~Goc4#$SNVyTso6W^8c)D8N4num-_W^GA^w#0)x_-CD zt4-JJY?Eyn^r+7vS3^Lt7H@yS&}|@dk0TY z&7a63n=)_$#`V^LI6G}^`ul56#`#lNi$dBJR56-s`!R#>pdyaJCxh?8L}wihvo0-0 zR+k{YOHySW$zm2&7eFmwAw-vgu?ER$Rb;`hZ?4v!!lqJtpj1ZV^9NK*hjY|jR)KPS z_kdd#zCIRmITNIVMA9Z$@8K#{;p}NMrjmtEx&1ba_Ryz2!!NQ?V#nmI4 z9mL^YIT7s_eD^uiZ&R$7X8{Vg@`$ENRB0Qp=xS~l%G za)!iwaT2YSs3PJ`E2-`a->(NznHz*q7?kcE*VW)M(0xi(rfNy>zqa_kes==T6iMSw zSJl8BFP~sIWN85XccO8#k~QDW?xOIhWzyppIVz_mU-D9$xvZ7rg_|!J>)0m6^bu_S zdsig?15KXvbzSY{>#olK*#J*CLa6D3*Fph|jXT*aY6mnnSv*<_fKJLW0~L|Xg3%(k zPCS)jbHY58ZYSp=e#{r99o)c#5SW4JLBFQuF5j(Hb~Hv@AXY8iwE*+L@Ktiu7oB4k zRv6HmDFcJ%XvC&`%-=1OKnjC7$yaPK7ihQ9jF0@cTPqBN!C097NfefZBEVTnlK4ux zx67%$U^fR;)OT`m@R)k?#0BLSXB-$lV{LZ+Ei!(8ftiI~c{b6+oaX@?9REKt@Y96R zL?3It1}zQ$rMRJFD)$`m9%i}ZIWP7nLIea&+)0>IILyrv`^gW3+7vG6@}0FDqOg68 zLIP)ryr^(K-4hK1Ew%%$Nnsh z`@Z{72c;C^MO`0XUeIZ|nhC*%x-ox>|nR{-S3!A`M_ZSibxm*^o&B8jl_jw4Xf z6uY!6aZVXeq7hu|a=5452a@SDFqWJ(f2L6tL=ag5q5 zfkal?KN7V+beqE;BX!Yvw`iObpE^S0@V2+GI=O4~PFlTkV_DT{w#hW4KEYFZg$oin zzc*Wo?6^8er@2x{iC~Ft=b>7&s6Ot_~M%}vcy>>nrB zhT5lJsFK|%c_>$Z;ViNpn8#djWSgg1y6^ZFbqs#od7j*TgrG)yVH&U|_Yc5V0D#8B zqA^5G{T+_&(n*u*e(*Z9a@1N-HFXO0z`+Zzk%zake>pinC}gpV=ayj)vvj0$Q~OeN zb+c(zJ*MhRZ?R2QLM0g#q;qT^oDt79w;ncju*guBBEoZ0t-UZUVjtY*@LJLrYAKCKM{W9KFFZi8_rUQ0SnoBn$;tP?)XLJI3Z~UdUne59U{@U5Vc7<=E@1 z+xz-QRB*ZK{mkp1XP$0G=xm2Phmm4leK-I0WsL==EVD~dyxBuH*$rGp=*&B=22}C) zv4-h^7a^t;5~qKB_h;+Sddy7;WRy87!BO}SRE(dVFe`csC#gG8n}^g4z)$Fsk_M8- zfF)v>(qmA_73SyF^uCr{ITxG5r;~A;APTzNbR!bZ4kM5E%O*m{_BdkDzWB2JJ5ScX z0jt3{zrhy9c7XX=yD;j8+*L{)^5lLF;c51HQ zF)sPI&PD&Z_HrJl)xh=WKD4ThsZMJ$t46Ues}?gLD2A{}Z3`t5RkM3{*;P|WgA*-3 zfipb!ba(m@`PzUeLL_=Iu6~!@H6G88Q@RK#?>)-EkJ%RC>~}e_za^=E0OWU899E1K z-+Z{N43d+1h4q&S(6U<&RhtlLVwC#gNG|hBqwrSO-M&<>IB(c}J&zgb_~(a9{_}&L zJBF_Bz!yYnG1~H~Q-5ToqKl@kyS6!Jb{EnSB^~@`kjJuJvLOp+zxo;u0CEVQ+WkA^X+;Vdd0P0;b5{@#x?-fPWD}XJdG@e96&X_5PmXa-11%aw=30%1oSFuXoTYR zU3N`OV=g4cFU*R^X`MwqGWBjbTwfnTfFb3Dyk)l+%6h!;V7CxEjWO8E6E^x&f=qLQ z1D;a+OnX-n7&??B4mBeWSUV-Nl~3TzQ}C7uzD4~VgwSCW=3f&U3E3PFTQrJdOPR$o z;Y(9+uAegOeK+#hw+F#guo?VW=}WH341k(*mx9Ct{5Ny50yS55^aq#`NpX#zu4+wb zxtzCg8E0FUE&DysTz~OB-4EVkZ7jIDkd(+|5dK;NaA0#2S;@Ex2oG3!9&-G|mEs7( ztmp(oy!*TnK@+5cDMHNaic>#vI?{K?_9u3Kv5gUX&=Euz##x%k=JS}I<)h@5A0@3o zRt^+1=Zbp9nLmcHKoD|W?;zjab%(aQAZbzg%Trwmv#uMd&JGy3z8Wn}>_I~14p4`V z0P{4!z@dJ6THM@O6r;0(4UmURiTtC>(%-9mj_(d@q=!yvaPrxm!K`f-6XDl}9OWm* z^UMMc#4lHhiDUvFB~mW9w|-lAwQ2dq=Ux$7cY_MRLeE4EPc{M3p=vKg`%g_EChD`? z_)-`};s_2|)f%@` zdr@t9!DZ{w++TBxS(2H&Rb$~1YWn3rY5bt1eF_jIAbc`o~&Sg%Rpvslj z=0Yxb5zXc0<70a#;1_6Ci{cUb+c7-p*c^7 z4?t!(kjQ{)7~aWN-H#*jQ1L}o>>S^_WCGNV zufbpH2ap#aF|6v@IqjilSOREetQN1qrq!0{<0705h^G`r{u5u$Pf~@g?O`Dl;^NNJ z9+g|z=u|K`PvQb^AwzzJ{eUH_FCEU;eEs4CHhwi0iv@IC?#~rXl$+^&a;0kH;>BDp z-cLb2iPhIbR53J)gozyj{d4wQ$t>t2FLw7mmymE3R5()d3pX(vLOJDAZQwkkYWL`s zy&eAehZ%I$y*7E}eUN<|lRak78BWsKRtsu=3b(kh`M)r+w^h3_4Z!XeHY(N(XNE+z z<9&vI^F&~D-eD1ALoJDl$3at)?TX3XBd<%5WM_OI`+Lu~NRNgd|C~#klF2GR6+#Uv zedG_?c9>p+jVr{@IG=N}ehgh)?SgPOdW^2ZXO*`4Y?WsTro`@=@^AW3S)Oo0fBWU| zUPitVeV#QXe*@<-HekCaiSAkK$zGkz?G6rs-G-!S857O+cQUNCAse(p|MVQ35YnT+ zKNM!QN#xq-^521hf3wL%bq0N27g;`HiY zW!@mh1SLcLp_a7Fm%vAe>Qv3oxQBJCga@k6%k~3xWk&q*1bLm|)XY$)BYMK`PsR)| zD0QaJK=3M}^~%PiL7A%fri-UA6C&G4*@lUVk=)I74yxVRa+Rph_{~N zZsw<)N%An3uGeyIqK>4#Gd4=E&y~g*g4bD|5kmqEu~THDCgmSt@?{+%q13BBOI^(w zf>32I%T3y`@o2^sp5E`UPoPh<-e^Pef;j{K;krMR%e&t%f^cf{2PuXjuRlBDLfKp0 z_MLck1jrR2HNHl)pcWut=^OHMH&Hbi$!M;rOPyi+t?vbOur~g2o?$L6s;74GLydmc zz+)w61*8n7eKzF2z>umJb1m69;U6Ea(!G7GUTlejm(Yn-m5asP7R141Xu*iBs<4*& zwv=BBeq~+a`kfJuZP`wmGZV~8n?Z~35KUR1B;ZUOh0L02rv=1oSs(2^?9sl)4X$ig zJ6!bs?#3ND@9xe*j#R&QTvJX7K(7qjj}k4hT~l%!&?!b)!w`CA=Zngal7uD z=Baukb-Ei9V`v|H+Jf2%q5_r7FUCv7eY(FVER%VJy_~IQA1->u(zczOWi2j^o{bs(8 zgHoOtU;gMpxg#YMmHJ3YiVhKSMoZhud~aCXoo z)ly0i@nW^NVc(55xg4#oDD^3}7*~wV0>IiV38@wdM{4-0e%t=9*->}Xk&|1kJ;IGfe+#6{zewBY5z0C2T(JzzmQ zx>IHXvRC3ox@V>G-14s`gJsawvr*U2yrDU4nmq|#(&l;1fEz>5EtJ++lw1kkl}bzG zJ*vTcXHs(Vme^BAoB;-Yj7kY*+tFv=_KiaLw9=skM9oS5JtRxGa(At&QB$xDJi&3t zsmJl;j_3$Ufko1BTW<7I@6-)t?C1oSRz_PfvbY<@4b!P@ zTwX$}YGuQ32gK{5E+|*QsO+b~@Em#-sSQB>dh+2M2Jia2ZtIj6Ni~nY14iz z`HBIFFx*HQ&uFWfVT5gjGxAag6yS^l#`I59^9(5W28fM+I&$9CO zBZvw6lr_b82APJg?@%P<$f}MFGbi~Ir`8o1%B56k9gVpb$F&75xi+i$NyT3?B0S;K zxX&uu0#f^{Zmzl|P;Tn@8&8*jokp@QB-Dx;T8m_aZvyt76#1Gk1IW&>h=`Iw(q?&@ zapE}Kzl(Tj@9C9v&+~IarJsW3(Hy?7lLCf9x5{eEe9*Ky9Kjr^r>m=n zoGHxP@D{APKhupK?SN-??Pc2|cL4jipMK_H^ONg%f4w@>oyd)!i6vM3NrZ;)s@F0_ z+#08^75*;+9et6O2@AMKM+FKihZ3HRTxV6(LrY#^=b*95;_TaJD8*~bD>NhFPea}~ zhm046yKj)b1&)>EfkCu%BLPr;#02h>?1R9EI(!heP-yf2J8$b103~1t{s&SS(XsJB zlSKXO?KAFW$icUG`&>&N;_)|hj2Re!LPhsOUNLM=q^KjON-*ea>y*qQv>NrawZS5L z=6_RiyKr|?Yopg1WvyT%5u+sds*;)T=_p_40aNZ%(4+Cw#T6q zU_$tV+J8b-%Zq;)4kq=VKrJL+Zh5nl)^nf1=5!Dv@XQPUfOJ_88J zc$^ft$0cu!G{6X~r4W;oag~$h0;5KU6ivC~YbzMgDXkZvQzn?vHE=tr!6uOzA}c3_ zLBpU z)&o?Tvioc6WxJP513YnVfw=VFy9nu?yP`@ znveI3!9yS2FP-D@wf|jQ{`61APD{YkB~he%Dr|*rBw}hrWLzp|WPM~OGOk4Rl%D}i z*AEqhY}n)gP3g_*mjzfIwAMt_iVzs*H11yekMHCy7$7_GSAJc&hAs>p|F22zc9_@U!sXg?pT*jD(vMt0iGc#CQY%9W9jN1ffvJ|ZwIkbEHh=Z5 z9e*XNAN1RKMz;j&WOeaz-o&*bMktee%F61TV^>O$rDvp)K0YKHbK_qA>U%n^fl(@? z?%K@5_>USqnJVehmnP4TsE~f%9v5Z^yz+jyxC(su{^NN6gn3W={(7{k7RZuEgo@su z7y$YpL<26!_eIT8{u*88y?nE~qK7&-qtJh@reH31*%Nib+a4pDshqp- zeM2cQ*Y0~)xD>5^`%x4K2@9c>w!CO2Kua)k;@Rp8O@OQ@3$a)%>4~Px@L7uFeg0h} zIsYjNSh~I?9J9Znf>xc0lCT(S&4)$7xB#vm}a*&oL3Kocg}MG-!`Yy#eKVfrIFX}d9T~6~ zeuLNkGE@hF-1C-s?^P8jQZtDB&5~k>UA%2iD_n0BGe60ql0NvS3gg!aPU&Ykh972!Ee29uDuBE(a+2J?SN z$H6xfAv`5fQ%v?rVk5)ZDYrXrx2LjRdU$Gy+0|` zVtfXkGD|E!J{Yd<6t#txP>eu=M&}#B{Gnk7GE||Bp>d4@BiNJJzvaAcjHR1{NT*)Q z-%#r{WW1?P-nvLq2>rX zDJ2-%RXH$gSEKKVjd$${L((^pJgIY|->MXCs+2#}(_7(l${uR3%#9X(@y1i`>~y~% zSmK2<$12e%Q@%B=fn99SWin!4g?FJ%snh^(1LJ6QYXZ3JG|W*+B4er4_~Mmxbxvn;`=8@#vP#5tqE&{LwS5VTDI2*!9}{!)tdQ zbbApl?FQjk+|%dL`T6wVok%~oJ69ipa_hRAfze?!7za=jeCISk*N{w}TTJjWh&zs6 zZ=>oOIxE|^`ZDLFH!qqvN-iR~Kbha(REcr*6JV^R&NW{fE%-u;dL6NKjaB8s=TlVn zz6JfAbwo6XBeiy0F`?5k?s0;(ly1UXR_59hESm8FuiP`{j;g$Qkv?*IFI(O8esUB( z?|26p-31`%T`67n1eXwR4r~N4c<{RP4{p!JC!oeNx>Kbekea;ic{=jU!M}ZSGKwGy zWX?wCt=;FDw*Yf8+xkegQSKk*qkERRf3(CpQo+QcAqjNY%X?TC39g!&Li|q5zI>fo z2nNx;?&-le@r5F+)}SJFP)`;;*6cD6;ppkxq=9+4w4Qj$8-<^>At;;V(b`ne9b{T^ zGMH*fBd54paw3>oWN;#VxsI0NO623lM9K6uFD0h)ZjEb*rdAhybXu2CTB;t6_@CbG z?|TFxJel&Pn@3SP2Qpi(UyK?F5~{ibcg|=U<+D+A;|eOZ#Qd9yu7on#oA7&{WNMcT zs=(cqe+gty3Qg}Q`fOKw>hX0K>WBlfxChnbE7J+nPOpq+7Z(9in^5w7IY~jJcZ4kH zm)*}1LpvGIAYY=Np|cie5ZP0hd89G@XZCbDTMTg}ID;W{x+@)pe2ygz54CcUm(@xm zu4R)Bb2~jgO?{}LcJmb}Csr^zTfvz5>OiCAkxHXfl=>hSSi9dNWI0A(3T>-A#!euu z-KU3n^p;`t)oj}KLHavUJepPOq2{o+A>D8=KAc8PoOo0UT^xS&*uH4?OUA+dJb^4* z2#k)HKr4ox@#g3(pNbhh<}n0q2SBs3+$&-iL`hF*hfH-T)CTXn_bow=uKt8 zg+Vn>rGK_}GMYaOm}VF(PMoyAbVq2gB~kSe?o2=QMX$5x$w&F#t+Idfzcr1Yj9~2I zDo}ad#vJv?2E(l3H$qotJb0Acw3&P8qLuV zD}0)f1@9Nu)y{va+uo~1_{Vs^!e2)o&#e>UqhI;+;-MdyUMUDru^bs+^?}|zqQf#C z!~l&a?4$45Z)!WGCNbl1T+VIXtM}k%rr90Fvc{eLPWlxz(YyR%WE8*6PFq!mwe&-q zGn>@C=-@R<|6#Wdgx;n3l?Rb|3MSjL-lnSUAJ10;gP)(7ucPui)fjjKF087p&3*_&Rm_U>k=)9X_fL(Trse()5NPCEoD*HX`nn zvC?N98lXKYr(VP7RSuM`?hYwQ1`(}d_%r$c^z!NsA9(V&i;|q#24&AJmy|YV;Hw5% zGLIx~^;<<@cY_cF7Y1NyPGI~w{?q5zfS#^^m}2;54AMR76Xo9IeFhcG`r8=h^+3O` z!Sb%i^B>_wmNgO7oz=PC!Wm3f>>XdU>%~%zH`Q7cRgnras-4vKd!>P$yy7}$kkK9L zWPiV8R!=62D!Ui$Piz?9i0*QubBhYHRFrI#?}0~8rv4QcNo4&#iNFCv~< z&Bak$IP6;Bzgn6Q^xMI%md~=W=8F8XgdTJb@ND++g1T}82(Hw$eUqt1^)50 zXZ6+Kp)7d;_U}%OnSaY+y`J2;^1|Uk?a{Wrhk%%JVk_qkmvZEEjYNcxl?mqc$Umi! zmTyL&ta3#h1#?EUSnUwk=29Bkj}8ZYRXROz^C3g;bPu(`x{YMLljGPamV%o2)oHq~ z)7hW(+Vq!mD#HQsOZ$(Xudh`HwBFmeHcM~%4XEEh^7LSOW@vIHNs}chLZ3$3JecVq zj@k~3x}JMXw$F_mPLpt^PY&Zp_xt|KOPg|Mrf9X&Lio0+a;?&o zg4A*boRR~(oSID`oHGX=-Q6~P#%7li$btW!+P|2V%@(UVo@aHP@N}KFJMQ0RCayQR zO6WIPwg}WNy7bv<}qPoWj14HG3MapH0LtoFy%Bi<>KLC=44@GG5X2>zip(W1c9mn z^L(~5P9P(9F13|~@v;??Rca2543&gEb^;sLO7Vqi%bjAw#CO}j}SbPJl;>=cw3 z#%%1Aq$mEzZH!Y1xMo z9pWpKyd7$#X2DxDZBynQvaOSt>y*#Hq`Q|8A*munxv+k%>&e8B*nJV!VTakALl>MiE! zRs7R0F*dBCvc-8eg(3@wm!B>XF0W1<&f8PE`;}?g4D&iX^8*I+n`l4FLfX-QK8&!j zo@uh{WPMCjB)o(B)9`fV{kzb1ew@fSv!XfF`IA-O zS={}RfUX$q8;?4;&-I=NPk~c?qPOg>9MA6+JyG9`k5!D}bob}uKPJ~B!#BXS-Ef}D zwG6Any_b8)*P*^hnrmmaNS~`T6NQS}^>#t{*3|gh&9!`E!E#TI?c3QkWj5ay-S{=w zbL5|sZMuo+@^{nIzH7C^60U3Sb?WDFKj%JvYQ94hXJUj8=!qEJfOPs0P(oNv79Lm% K3ULJq*#85_bDB&5 delta 13330 zcmaibQ*fXSuxyeIHrB@4Xk**9v$1X4Uu@g9ZQItyw(Z=%?!6D^?YvFZ%*#}F)zox# zuZKV`hCS?f|6LfQ~m*$(C9;unp<`_uf#!gSG4==AhYfSD3f^9Ls?xWCfCO!{N zw~wCStGoKoCmuXlzQ6RD)NRvlG5~AZ!@)*&nWn%a=%yT24Y~OS*n_<6( zbYy!u@SS!5hob+7sI(Tl0zN%h&Mu)(h7J0miS3^vtmml{6hyU3{@s?e;f&NRc zhP9!LMMC_BjiG1eh$yM=d)WM>yJ#GR1(*~}SNNj&qI6jo@e-+oztqG)MXOIcrM(FG zh-^E9&K&2|ga{9VsY!kKEA#!+=lS=BHXQosV_{Qm5XM^IZDCsA-{G9@8-|wnou4@1 z$LYesgM^(>5e$*-;XW3LBI!^h^?8I6tJi9D09^%f+eC~=qNqfs8ookF588JDG3 zq7XJiq$un2S6W7lbq}^m=@?bmC&^!=3uWvSZCC*9U#`pPGp`R2+{O?hewU9XF(0c( zQ__6Bz3DU6jfI1#hFDQ0>H1}FMy~+|dfC&{)B4MV>)^*pRn}4GjX=q7{zb-2 zwz$^CnKg~ck*&c^xfvd`N6%-8jHU_zLCWJcA+{rMAFtc5|w_o1ZF@6EytA#!~BL%#`V&p zugjhmPFLBrSC}yz#jo$l44{Lxtz~QC_8_Z?Az@*-rGd}-bfTFXv(y`viO+cf3C^b8 zxe4B>L59@RG$Ftuzs;phLnhdAF3iuCRK%?!949nQ&>;XYbdX>$7ZxN~gM7?xMh-{B z=4ijb|$rmHjDZ`{E zYsrnkJ;N0EwzfNIDGnHPO+SYYd@TXj6hw!Bw*UMLr*cC{TcxmDnX-fAiYu6^+aZt@ z$8q7G%xztt8KyD;8RTp1maS#HAZP(y~O7^{fp4!ITSIFfKWG+U}I1fSG7wLWAU5fa1@A z-OSQLJ8NgKclw5)59%r#tHx52vMK546pZ~7af*YL%*q3>Y&`{!!%r>ziPI?R#}2Yz z6rTiu_}SlaMT2H3#&ta_ z@A{#!23pD09tWEkCgP<<^m-e}51EWa4u#*w(?{)xP0cw!MDI+qc**qC8?GHt#(%1d zv>0vMu5==L2xq>TT^-*?NKG}HM*$iUy`(L`{c;52*$={#Hv8!+p$wU>Px=E9cBgtI z?lLLK2aosOSDTHgdC(R0+ovJ{fjUWODL#I%?v$dvq_5Q2jg%njpTvoQYOngIdS<(b z34gdQ-C54|UBzniD*J>f*etaq@3#;pWVNIx*?D2H$UEb8mWA$`9NTcbHO0A zjgHaS^4jB(Ch{E%w|nUq$hfBA3s%R$_sz*Bt(Oam$OIL#3{3Uao9DJ0DV} z&eEK4&wa7OFRavkE2)n>yo6*ytqjmI+Ys;ipWbuA%1^ASEWsp!)`PleJ0JWg(XF`V z`O&u0ZO}#d2%j(nZ zy`0P7;?d4iwWACZ$z%0bla*Cvrh=zQ@DuPlB0Z0egM0t_Kx)kl-RN6`T#cRKuG@Ua z!TyB0X}6C9K+F6TI2tlWI6{RgYr!#d8z0SEgi81|p!a=a1CeEL6mO)%yp9tzSPBuO z^rkqP9wbx-Sr~*4=V5!PBD9TFvo^8Nu?A^>Od8hxD2zyuP^rFXoJ&i>tGRF*)C`)5=7PnvX-g0Xm#8gW9z&m9lUS*83=*h zJ=BeQs|*|PVhcsHb!er(GTMAb8Srk>oZ&g_5I1EEDaqin($f{=o!=;sYchL@_I0um zC}7W!OKyo)6v-y{6bjnNdO3Ml)GfELIFuW|1-zG|ro84xV^x&pW`Kw{v*O^k+hKA3 zw>(lHvcRI(_%8?)ejKzM0v~(!G|AGO;a3O8t>n8xzL4}0^;R;979~wFhiMhUbz+;% z=9Uk`TYil>O<^asX4k>!Adfecu)S}@$seUlb5NI7kl|~l3)Jgh*5_04%2C{^yML6R7>bzn?z}xe7ef=`YjGXY4vgw5`AqlBOzw8Ve;ggw{D;6eH=m) zK?%H3UTQTzIU}RGO}{-t2~6&0v=sYe!;&_E5BOQ`^-);K&F#Mm&UeDF0#+P`7sD_TW$83=xW zx0Xe>cA}4OQ;kXAlYo<59oeoIM?8`W3U&4gGugk1Dcv3|EstkP^{&c%5%pX z2qT>9Ko@`;=?A;Ko{{VBh_rP{UH>T!`|T@|x-b=-$5Z`yppu4vv;0<`EFt z!LSGp_CQoR4K|15YzALik>B|s>tdyVs0lJ(86VY6UXNQMYVPk$6Uv@|z1*qogc{v|19Lc5j=iw#2h2SLyu~LW@Y>yay<_1GY|&yWn-G$WT!v$Y0zPDG8&#VYQy;tfB#$j z6s~TBJY=UN*A;O-Ehq3hm`W;m7-ql%8Lw{1*>z5CrxAP8>!zA8%Tfa(v~&0m@QXy4 zl1ifrPpr)*4=pRyTq9|rQhTr`TAK44!p*jFTm9Q-_hx!4zV;McB%B$ zlQ(KlI;BIk^Rf8$PQU>&idn2Qcpi5+LasVuf%~Ba_@zMHtT; ziT<@X=H;x=21?QQf;A0X5gebP?;(R8R}92Gl_Ui4jbz$s!X~^>~|p5D)&-B z%%Oj+y%`yAEB+&vZfOQRv+{U*)nPL^_G{X`!TnI^g|M+jH#Tr_lwvQQjv0Wq(YTzZ z<(#n_Iv%2`|1iJ42!|=eY}Ob?%;X7O$7#W0dl#6$yQA}kR$M)@@VGY#zwJ#zpV7|kp$qPFgzTauQW z(r*dDht{bA)150@Bv2~dw+XI8)mmtWfb9_s?E#}>$xHlF7B@tn2L<8E;z2KcBb{e4 z<#DrCi;sHVXBqGXhblXnOB@gl$H`bq8ugKM*ukN_Wq$I7XHwnG#TMR}`@}&)#M`<4jI&C43|_5?9wmPtNhKMEe7aQm=YONqo%8pP{R? zsx~Q}P`0rnr2)zo?xB)cs8!i}l<#u&WcbYj*KUDem)y%DB@swr`GcYO+Q__N#r@K; z#a|YbEK%tK9Ewb3>WjE%*5hd!{9QtAic+PAi!f(A;{kVtTO}%gElZmgIW2ga;?^Wz z{{o^F<#65o_1l`E= zgzMvPGRxwkgb)#BDA0VzSO1G?TyDGy5pn1zKAQLyboEh_jHhlKUAxkp{oyQ7<#y+=Q6+T>3%} zajW-p@R*HdoE#Kg)ArR%9r+(g0p=Mf#w161k#MwH$eH-#1{Rd^^ytMOA9pFuf3b9A zKB~EmtuZ@#y=l=6&oic7r+K*$s@af8c3p4HxZz?3scsxiY z9Pm$X$=igPm;dD{a>5N?0`Np1X|#IlP}88)kv?C3(08umEF`zlX%@OYAg_?NIlT5a zaiye8?pht6e`A(l_=QK}x9E8AN5!*qf5zC;XH{De@)KI)*Ax!aGYz}~U1VY8Q|yy{ z<6E8|y$uwQjxQwqZzG=G6@;4!$|ML=Bnb4My8AZ|0+&e-d7H*ifcv^rSFIQwr-d(- z(Vm+cIZvVJX&Uygq#2np7Zs>_Sw|qGjQ()ZCYQli84K zi3y2u1SwNRM}zO}K>dC4*2h%=Yb?n64dM6!T97lvl$E29jOPmbL5Q}>?wqZDMhR*! z{>uR0(oOTv;D}y~Kj{Pa=w-Yo16AwHal3t&=* z5w`Nlw9ahJY4S!I8UM-c>^dd7rMbmEQuU`K$GG#{5{;+}XaBmQXv?zg7uR)l`_kvd zQB=mtO!ebc5g;0g-+Xon%`Jv4i{%ABnIqW8VBS)Qz!FR;Z-sjtd`<`{DhZ~5(r09d z_gwg`qxd^StY0Oa1M>C!)a{q!YN{sK)K=0HTwX$y7Gp5~dIE|$Ola>yq&h)x@9|MU z1`(k`ZOU15g7@CB)3{5aJgy)b_lJZkt{^AAT+3`w9>9$7Em0B_=ZPoE$y-9jHOjO9 z#`Q>_$-g>qvoGYRUu37?;S4INaWkFFvGFDmza^nLTKFHw!-^pUn6yQl?s^dtC*=ZV z|3emSo$29oCN6nh9lfx^w%lg_=lm`WPv&AoG^QTkm?x_d&O-d8z9lMM^uj8Ub;7mB zvO5DZP$xmZKcRbdUTsdVJV16uDYO2KePigjAe6vSWuU1H5@9`Th+Z>@--CkH@b%Ez zT$UXpVDDWQ929@_C!Sw3(+*9G@AH6?x*OYlsr%ERQ$Gq+1=B@q%UsPn4NDZd$UmTw z;;Xm)K?WjElYKaxQciKvTHmzuoN=b-PbFnGU@ENSp=csqD6LRNDYc(^pv2w(qYE5j=J6&9DcOt9h5v*B=gVnUwU!4OZG`ao72>C74JFNwAM~4 zfJ2&-&F%iRXTSZ3D~~DF(*7cse&DcYWc{x%;8Dw23hT_ELm=|#u0~k%VHN}i|-uS*RYBU8P0ru=?|aA;BhL_GC}+ zI6dJA+H0jzlQ1Wq5rz97hR3}{1aqb@}OYlu2QQC2IOfGx0# zqK5M=7HKjK^FFFpM;;iURnM7gDh62$NzpbiHVi|6pN3L!lvdODYESM5e;pz6_m@OxdC$V;3g;(*HssUm0m>i>S3J$e z`TALEU#tT9?5fZxDuJXFZMaDafY_goEK{S=g9w(8OI_GkKqgnYs;XU^v!wD1Z#QTj zee67r(Ba(rfzX(QoRaZeMZ6f;vNPC)>6v|Ti)O$TR$kY^c%QBc0Q*GOHs2t1kKcxw zFf0BX$|63dRiae2XQwTddx+!;!s_~jw0I=_(HLJyxl{iudZbxj?g9%3h?3jcho9Sl z#*eD9%XFkl@W^*vSh2TUa2=dB*;Zun${a4q5<=vQbIzw5mvdBo!Yg&_-C8pCP`yR# z9vzZe*3IN&;g3LO6b)e7xaFYOM~_apwVV8)$D8cE2r<>0X}e73^k{Ew_t^5j+l<0# zvC(yPJqxO-!nfKNj+M#=p3g!u6Xb<(iWqsxXlPPkvyT(-b8OR;kTPl#%Pbr#&Z3j> zImRB@xg?}BX<7Sbgb>}kvZ-?5%8<1R<+KP4p+i`{%gFr}g-^wAeh(v1ci{68l6rRa ztTWrd9em32aqTu|FH);A9*jHBmtX7!!XO~Nd%#A5!TyUN?hUvAxT@Q|Tit(@sN!l6O zv3-4BbmiT)t9prbh8T4+S`K0=HuYp>?3HEvg%T!hxSjH3ujMbtNjD)=~Wc-&gpXXhXGHb1}b>aV2^=N z+f;(UcCz_^!D(YNc_Yx2q#hoHFj?K!;3p$id-Kcm;6w+hqm~)YvxgC*ZCW4SD)!Ec z?~+FS=TlA^Mf`&_J{&^@Y2Qgak>w!AI2LP(kbjm_Ie5Fkt<{T2`-piK{_-j%(H^>$u8a?MkeHYs*O`_@ntHgvF5h9 zju^n>5DV^bF^I*U#>&v!*^*EBVrJWpEt3Jn_GfpW@K59&>Mnc-ZfgPzH)fCLe2rs` z5Z`dUO{{-FTLZx+SVcJ&a*>v_g1f9Y5a)VAb9c5eI7%&x(OTUfGjv1?dtAldbw^-< zPcmmvTuyULVDD(e`*lsHWJ77QISkQw#jPW6ss2}99%mZc8zTmfL_)mCn!1WxZCC_< ziA>wH@9wGK3Uv2vc3+#9)d_= zj1W+n5jUobbV|mMEyLf9a8cA$DuAqllrid#UO6^fd!2hv!)?^>6;?Jn4e<9cMgSRx z^;$7@_&k>pvDI9eoJBIk?3FrZjzqjD6afnCF378}d`0+Rf;L)#|q#_zw#$A%rM-}{UG6(|1O5i=xAg$glXdFeFMJxx6} zQs+v+oYm7g@cL_UkM`0q2a^-k!zbMaJzQpcFgVI7%tunL)_W-Rq|mK-BP&M4XrRvz z`C!-!>US4eJo_|eRGEpkqkOS)$O06$EY&W@+kUt_N&`nFwg0rpk^hJ{StR!ALucjw zAv70s{!sTI<|U)eb-j|~$#R*z(-72Y^Pl=XDNbg&5!{w#ZxqFA7mW-4h~Hx-&&_w^ z^&@Sc@(SHDJR&}(UMyn#p~-8apnp7Hw7woC2Z6hgeS)zrKe@7_ptQX%wFiLNG_ba< z_b{I7Mq|~tU3K#Fa_jx$A9vvcv_=gR>(S2At9vD~y0?BQvF596Iq&V|484y*N8XB; zJWoe|?E#69RNQH}@snt_1!>8uEBfNPUSU;xJrEr>R%9x}aG{T}Tk1a>_aAlidXm_c zyFPtZLshv!*j7hJzfiN2_kh{OOAhFN$f6)C#;isw9D>b46W<8cKbR@RfY1mgdHi_7 zrS&q(J4=#rE)u@BUX-N7%LE2m8tA5jy|9zPKs?ZpmMm|37~Js)_f!fiABa9)IU$55 zUJ`#4;|FFPR6P>b0rUh9M5Iogc-;~cy<0G&o$0nDsaqpgt0uEJE^W0wqf|3DpOW*V zU)*60Echn~sA?Hu2lB}dtoXBE#yL8))HER?heLO zHnt&Glu``Ks@Y!XjSV0)HHsp+(kS-r+nM)%137^tA7f0G$bu$hJiXC1mNd~=myVdE zjJ7&znz+evq+v?4*Pj(M(t_c}E}S+Z-;;cmev7m;VJeE%NXE*x2Z-kg*RW!?Q&tmtm@L*vnIt9<> zFaj7O-wYx1_b`rQ)$>ywlO%>Hepth=%t9q~?H8&QgKs&G<@)+cqL^XEqs@}j$wL)T zUjJ@l0Hh4DJVga4y+#6yi(28!26CX-gU3yx-b2PpFA>Ut0_~Y+jBxy$wZXS+f=IUP)2z4?O(to5TELguy zXm;@$K(}H=;>bOH{XncHVZfM8NI|h0!a;)7YS0T<_nP;r5LxBZ1_mRaI{@3N{j~i`1I7I$zg6a0sk$5c zmjkT}g(>y@GfAUsaK67uT>H08j8IRgC{_DmO-NshZ%%{Z=-&AGMPclHy4}buXvl^G^fhhnqUm!zMG$%eXKjj<>W*9Oofm6yVy}~ zhe*3+KK~GQ+e)!_0{smT$`zaIk!nI7J^Ce?Syc~hrdw`lU&WHiRoL&ab`Gue(oAjQ zWw?%LLJ5zdb`}-UIKwFy#7Rv`BEP*#FyO*J73+FFEQ8Z2ByL{Lg!dV6de)QEqA!e{ z>`}z|xH&3JW_#g%akS(6bibv4eg%Gx0>B&dGQJEMRDj6s!49D7-`D5d?^u#n&lUNJ z`kgZz9S|yI?LYe-Fr>%bHF%!g1Rm@}>osayaQsrB{;1BY%15*)eeU`l<{hd`pwC>( z@ArG6ky3lPU@;Deu0P~D0SVFQPy55upHaIwf4##=vGgq#)A=6qHGA?vIA#7o3Z}9U zqGj+>pAV0QdIV&`FVD(0Md$DSq|m5hSZFF8Cq+_Fqcu!xxEUl2p!8HsjUNxhBwSEQ zJWTfzpAiRIOAkY`Q%j@DFkchEc#^Rh7E1j@G_n5h9qe{{1VG4Xc3sFwEjsELSyk{U z#?tbxuKZ~(vuJc&v{@|j^-JoBKh`+1;TujBTTV0)$^?+lnao}T)Iom*o>N@DD?Uc5 zc$0cSCd$(n&sNq@FI#V|REETNFZ1Zfz0CHf;w#q?{lRt%?~a1UMO4m&k=ybGEx*P? z*%r&h9L^;QRLO(fE6H|fkC87c3A##g@C!y7Wn5QfZv1)I>p}v$6$gHpN?ypIAZ|FH z0DEm_76N>wMNZ^re3_s%%xka%|{0=WBs_+Q? zLcWs}Q&f%im)1EGRj`VY)Op5zQUZcD|NNpti_}LBT(A_LMUsGuL&|!l zM^_X<8KCzCS4^O67OTME@ncS&1GDhY+VMr+x06TaT`qkYe})rAoM6T_H^81&=C3>S z(brbXco!Y4u5*;680Z(qGv}P+cb0Oeb|&E-hX$G1Kht3)1`({PH^gRp2)>8&BBH`| zUId=$f74_Lqu60g#|@sLQoff8)fb3l_JhXP6w-=BS-NWl*|+*$Db!Ji;^zJWf$R<2 zjRt`bn@tTwk6lv>R(dkq1o;LZQ_sINgOC3EtAy51$Zo`Z4JkyAo7j5?$(f8=<}Y=t zpCoH-$`;P%-$LSN?+H)H@?dI(pYE9e1PqWzq!0=hQY=i!`6@`E5HgKJopmd(#=nTH z58@vz%lGRAj~v`k5ib98S!Vh$#p`ncdss@H&HN~~zV(e;nC6ix~u z-s3*yv@DtlSu=H+ks6}>dcGWKf4{I?EH38ysz_=xB{d2wwRAXidNWK3r4rX_nk>-k zKjmGDb_B*rLTV*0^Lk-GDLz*8Iyi2nSJk(eCjkc96cK?1vc#J%JNuy2=M-LgY-XdT zK|rFS{yWTifgE2tRMC9DKT?Qyey%q&c-mo4kzdROong&kwf0wRlXR^i2Z)*+sgZ0t z{^|mJ2@|nuHMjyhZh&8=!;&gJu?P_Dw01&o^VmNZp=emSfmOBDSUhayrbO)>DblVp z5TdL*GNgN++NvD#r<=Y)WpG!yc`IcdcS`wE3wLZ~J@!g@tIAdWoLWscF@meepG;FF zHEF5XTy3=?7U?@u+(kIkY}i^N+~J++oIYG;W7WDoh%%i;A8U!V-k;e*-wOybw9;Q1 zE@f?YK|7LNVyKZmH0pvv#5O*m_GT#>u zsx%Hd?qSa+7_$CS9J<i9*TuVieYplOxVZOeb{lgW|~5N`ru#?CT2??56M}*$~CO@m=@X8 zK&gsw`6%t%G1Hz%=5s56<`4PX&|xp?q@6}T{ZsGnzGnn@+fotq!?`IUeBq&x46tQ| z_QnQZAMgyb^ zL?k2V6=o+H@R|$cvH5UaNpx+ph)e$D=C6+UA)8ncXR?X8nvf2HErHa8Q=@k6>Q24(q5W@Lb7wy@J)+$#n(eXtTbSSt!*mXo6GK396deWh;pyE{ml+^-AgW^0mX&_DMCr zmWGz{ut}pI9eB=#FHNG>cuk~^??jVJ`QG4tgIxS?K{U;h@N530<}a=W5u4f}7t@KW z+lkMZkuYXW4Yk2`L#n|5bo6Qs40H;`YIY5^6HgYWb{NX2io+%01Npe{T0cV{RLg!E{<3I?*kfB zoXFX2LVjZ%TyUCGy6$wU-l>{10_SD7!pk*H=A0F?F)h=@)+p)@``7K1Y+8*29bBQs3efsjEV+ovJkD zma1HN#`U}BoKkhAYpNk%TT^H5U)|`x<8|k?0ein8((Q@o`+yJT5uTfI{DP0|SArrM zi)8T&`GMkf#rLnB&xre$iL#59R=SACUU_bE`arSmW;z^Y$0yRNCjagN3^1YR)OQCQ zuTw$I7+8^%*4m=?X{pQL4%_S9tLtn0Y;Uc$1Q4KGnNPM7DkPpKknRe-M7P-Mw)kNm zV2%cm;_62ZCaN%t3(G=6>!m{7)-koyh*>)UPsHo*ot=}{5X6aG=ZwO*^Spc|L{5bp zYcd#m2TAM=oLym1wuJcZ!^tVbZ1Tpulv$DG0^xt8l2)hPOn+Iq0n3h7$K|CDu%07~ z(!s2Su!j%=M4cXPVvIo>P6!Ll=}c?D;i$G=%++^f{bUi(uJQ_@Fj}KNj%;lMjcw%b zMyA@p)XK)r;st>&?lnGSP97*nm$K{l?H|{0y4`@|4uqV_f#))(h(E6@;9W{ZTmvN* zH>WHwc1v+PBhEqF5jk=?_}BWiUAh?`3XZNIGGu28vFZ^BR3J_2M4^V;&fqctUf&FV ztk$@@zphLX*AFvGkaiXjbqIpu8ptFQ|L7)ExmVG>Q>zin zH$%LSepg3})>lp&)dJ|f%JY9G<>IkYOKntk*+sqX-H^CPqc9vP!vdb3J()ej@-sN@ z$5$|T@A!67{S@ns%M=dr{W#u7}LEcA@m=G5mDN6@_%w!-NG>d^ue-2itx zW+d5y$d@~J)^xw0!03=m&W`USq)>UnYGbe?8^P_(%foTQ%2=8rxHAs#-WzD0*DPoq z>m}4+(>nO%6sFoN0h^7EDb(H$tlNu2UbUKI&hG;F@kRa09JC2q(;=EoS7*#6QM27^ zWR1*jK0d%%%e_X^FiHd9qf*%eP}E-->vMblY(!G0iw|6*88?EY*@-U6MJlJD?jLP& z{T;TFKf-T;<-6`?=l!ZmJX|vHT!FRt9F*F;9GoynDI*`aoVQG-Fg7EzfE7LUz;Qii zl^1=}q{VpoPvO1PG8dpfl>D5Pa^`rjxawk3-GEfpa;@(4aja*3f0Ch?V3iYF`p+lf zGpkBidG?>z>R@~yU#rnKWxmj7vZw>sk>z>o^<3F1;Lk+YO;4BveDweO^SS*%u?1Nf z=$RNy^qCA<4fPF}^$iV-*_hcm^bOb;jP;oqnAmu^|F?;hR6kHddlQ^G+t|pC#aqWCxlXgG;Lyov8YX~=E0vfndR{;hmOjWjsyHQO%vk; z?%&U2Chqe1aWr?OfK`&ayI?aiIq2eV+sIZ}V=jfaO;}@O1?3}B(&KRi_u$cA?LcxoRFQ9@vsND9J-B zI&Y|5A5+g1WYvu$#(EnQM0>@_o{0HERderAb0O+Ee+9VP0qm3!2p!5WGHZLR?n`3^ z21cxu5_dYOP*c~vyUvLTMFS^tTDr4NYGNIq(20uexjL-}XWloH{$neWE!nmdR06Wi zj4(O#XBY$SR}0OsFb!8Uh9s|Zyu6yWim;r4tM*x7lVz|6-Bsv>!8M&)I@^>_U1?uO zLM5^tF)qo@zJ`dYWg|rB+pgwM#@nW;5a3cnn2}(54)?j#m3+~8eueNk(G`6`=lUGn zeWgS2bqwiRutDW`mmH~#mInd1fF1#wzdGXcif#_0ctd;Y6z4J`hP`sh12xj7h{ Sz;Lj$G1J44l8VTQ!u%h0{@j=V diff --git a/spec/tinyspec.tex b/spec/tinyspec.tex index c541657a65..1a31a03eb8 100644 --- a/spec/tinyspec.tex +++ b/spec/tinyspec.tex @@ -23,7 +23,6 @@ \definecolor{axblue}{HTML}{1565C0} % GLOBAL \definecolor{axcyan}{HTML}{00838F} % LOCAL \definecolor{axbrcyan}{HTML}{00ACC1} % WARP -\definecolor{axbrblue}{HTML}{42A5F5} % THREAD \definecolor{axwhite}{HTML}{616161} % LOOP (gray on white paper) \definecolor{axred}{HTML}{C62828} % REDUCE \definecolor{axbrred}{HTML}{E53935} % GROUP_REDUCE @@ -307,7 +306,6 @@ Each kernel's iteration space is a set of \op{Range} axes. Every range has an \t {\color{axblue}\texttt{GLOBAL}} & \texttt{g} & --- & --- & GPU global workgroup dimension. \\ {\color{axcyan}\texttt{LOCAL}} & \texttt{l} & g, L & inner & Workgroup local dimension (shared memory). \\ {\color{axbrcyan}\texttt{WARP}} & \texttt{w} & \multicolumn{2}{l}{(created by \op{TC})} & Warp-level lanes for tensor cores. \\ -{\color{axbrblue}\texttt{THREAD}} & \texttt{t} & g & outer & CPU thread parallelism. \\ {\color{axwhite}\texttt{LOOP}} & \texttt{L} & --- & --- & Generic sequential loop (initial state). \\ {\color{axred}\texttt{REDUCE}} & \texttt{R} & --- & --- & Reduction axis. \\ {\color{axbrred}\texttt{GROUP\_REDUCE}} & \texttt{G} & R & inner/outer & Shared-memory group reduction. \\ diff --git a/test/backend/test_jit.py b/test/backend/test_jit.py index a9c34ad678..920cfa2f3d 100644 --- a/test/backend/test_jit.py +++ b/test/backend/test_jit.py @@ -4,7 +4,7 @@ import numpy as np from test.helpers import assert_jit_cache_len, call_is_graph, not_support_multi_device, needs_second_gpu, KernelCountException from test.unit.test_jit import _simple_test -from tinygrad import Tensor, Variable, TinyJit, Device, dtypes +from tinygrad import Tensor, TinyJit, Device, dtypes from tinygrad.engine.jit import graph_class from tinygrad.helpers import JIT, DEV, GlobalCounters, HCQ2 from tinygrad.uop.ops import Ops @@ -16,19 +16,6 @@ class TestJit(unittest.TestCase): def add(a, b): return (a+b).realize() _simple_test(add) - @unittest.skipUnless(Device.DEFAULT == "CPU", "core_id is a CPU runtimevar") - def test_hcq_core_id_runtimevar_merge(self): - N = 262144 - @TinyJit - def f(x, st): - y = (x + 1).contiguous().realize() - z = x.shrink(((st, st + N),)).contiguous().realize() - return y, z - x = Tensor.arange(2*N).clone().realize() - for _ in range(3): y, z = f(x, Variable("a", 0, N).bind(0)) - self.assertEqual(y.shape, (2*N,)) - self.assertEqual(z.shape, (N,)) - def test_jit_input_view(self): @TinyJit def f(x): return (x[2:5].contiguous() + 1).realize() diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py index 364d02f0ec..002be54572 100644 --- a/test/device/test_hcq2.py +++ b/test/device/test_hcq2.py @@ -12,6 +12,16 @@ class TestHCQ2(unittest.TestCase): with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): np.testing.assert_equal(Tensor(np.arange(61, dtype=np.float32)).to(Device.DEFAULT).contiguous().realize().numpy(), np.arange(61)) + @unittest.skipIf(Device.DEFAULT == "CPU", "ping-pong needs a non-CPU hcq2 device") + def test_cpu_device_ping_pong(self): + # CPU submits run inline, so alternating dependencies must be submitted in schedule order to avoid blocking the host submitter. + x = Tensor.ones(16, device="CPU").contiguous().realize() + a = (x + 1).contiguous() + b = (a.to(Device.DEFAULT).contiguous() + 1).contiguous() + c = (b.to("CPU").contiguous() + 1).contiguous() + out = (c.to(Device.DEFAULT).contiguous() + 1).contiguous().realize() + np.testing.assert_equal(out.numpy(), np.full(16, 5)) + @unittest.skipIf(Device.DEFAULT == "CPU", "staged copies need a non-CPU hcq2 device") def test_staged_copy_slot_reuse(self): # chunks of a staged copy rotate through the staging buffer slots, many rotations must stay bit-exact in both directions diff --git a/test/opt/test_kernel_opts.py b/test/opt/test_kernel_opts.py index 0c347bef21..44150cb8f6 100644 --- a/test/opt/test_kernel_opts.py +++ b/test/opt/test_kernel_opts.py @@ -333,20 +333,6 @@ class TestKernelOpts(unittest.TestCase): #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8), Opt(op=OptOps.UNROLL, axis=1, arg=4)], # noqa: E501 ]) - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_threads, "test requires threads") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.global_max is not None and - Device[Device.DEFAULT].renderer.global_max[0] > 1, "test requires multicore") - def test_thread_opts(self): - a = Tensor.rand(4, 4, 4, 4) - b = Tensor.rand(4, 4, 4) - r = (b.sqrt() + ((a+1).sum(axis=3).exp())) - helper_linearizer_opt(r, [ - [Opt(OptOps.THREAD, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.THREAD, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.THREAD, 0, 2), Opt(OptOps.UNROLL, 0, 2)], - ] + [[Opt(OptOps.THREAD, 0, 4)] if Device[Device.DEFAULT].renderer.global_max[0] >= 4 else []] - + [[Opt(OptOps.THREAD, 0, 8)] if Device[Device.DEFAULT].renderer.global_max[0] >= 8 else []]) - def test_double_sum_group(self): a = Tensor.rand(4, 4, 4) r = a.sum((1, 2)).sum() diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 4f01a8eca5..d555254dca 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -1,7 +1,7 @@ from dataclasses import replace, dataclass import itertools, functools from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC -from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT, TracingKey, Context, panic +from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT, TracingKey, Context, panic from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_weak, pm_commit_weak, pm_cast_const from tinygrad.uop.render import pyrender @@ -497,7 +497,7 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: # config affects generated programs and cache keys; context also carries compile-only behavior to workers to_program_config = (NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, - DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT) + DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT) to_program_context = (*to_program_config, SPEC, DEBUG) def to_program_key(ast:UOp, renderer:Renderer) -> tuple: return (ast.key, type(renderer), renderer.target, *[x.value for x in to_program_config]) diff --git a/tinygrad/codegen/gpudims.py b/tinygrad/codegen/gpudims.py index 10f45d89c8..849061f4b0 100644 --- a/tinygrad/codegen/gpudims.py +++ b/tinygrad/codegen/gpudims.py @@ -1,6 +1,6 @@ import math from tinygrad.uop.ops import UOp, Ops, sint, PatternMatcher, UPat, KernelInfo, ssimplify, AxisType -from tinygrad.dtype import dtypes, AddrSpace +from tinygrad.dtype import AddrSpace from tinygrad.renderer import Renderer def _dim_max(d:sint) -> int: return d if isinstance(d, int) else int(d.vmax) @@ -47,7 +47,7 @@ def add_gpudims(ctx:Renderer, s:UOp): all_ranges = {x.arg[0:-1]:x for x in s_topo if x.op is Ops.RANGE} # extract global/local dims - global_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] in (AxisType.GLOBAL, AxisType.THREAD)]) + global_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] is AxisType.GLOBAL]) local_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] in (AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE)]) if not global_dims and not local_dims: return None @@ -57,8 +57,7 @@ def add_gpudims(ctx:Renderer, s:UOp): # get the idxs ki: KernelInfo = s.arg - if ctx.has_threads: idxs = [UOp.variable("core_id", 0, int(global_shape[0])-1, dtypes.int, param=True).cast(dtypes.weakint)] - elif ki.dont_use_locals: + if ki.dont_use_locals: assert not local_dims, "can't use locals if there's no local dims" idxs = get_grouped_dims("idx", global_shape, ctx.global_max, reverse=True) else: diff --git a/tinygrad/codegen/opt/__init__.py b/tinygrad/codegen/opt/__init__.py index fb4b84ddee..0618cb4d27 100644 --- a/tinygrad/codegen/opt/__init__.py +++ b/tinygrad/codegen/opt/__init__.py @@ -4,7 +4,7 @@ from enum import Enum, auto from dataclasses import dataclass class OptOps(Enum): - TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto(); THREAD = auto() # noqa: E702 + TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto() # noqa: E702 GROUP = auto(); GROUPTOP = auto(); NOLOCALS = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 def __lt__(self, x:OptOps): return self.value < x.value diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index 9ff87f9c09..c0e70443cb 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -191,17 +191,4 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: k.apply_opt(Opt(OptOps.LOCAL, axis, local_sz)) if will_delete_shape: deleted_shape += 1 - # **** threading **** - - if k.ren.has_threads and k.ren.global_max is not None: - for threads in [32,16,12,8,6,5,4,3,2]: - # Skip if too many threads. Heuristic: use about 128K ops per thread - if threads > k.ren.global_max[0] or resolve(prod(k.full_shape) // (128 << 10) < threads): continue - for axis in k.axes_of(AxisType.WEAK): - if k.full_shape[axis] % threads == 0: - try: k.apply_opt(Opt(OptOps.THREAD, axis, threads)) - except KernelOptError: pass - break - if k.applied_opts and k.applied_opts[-1].op is OptOps.THREAD: break - return k diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index cda176f7e7..f15cad181c 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -133,7 +133,7 @@ class Scheduler: opt_to_at = { OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, OptOps.UNROLL: AxisType.UNROLL, OptOps.GROUP: AxisType.GROUP_REDUCE, - OptOps.GROUPTOP: AxisType.GROUP_REDUCE, OptOps.THREAD: AxisType.THREAD} + OptOps.GROUPTOP: AxisType.GROUP_REDUCE} ret = None if opt.op in opt_to_at: @@ -160,16 +160,11 @@ class Scheduler: if opt.op is OptOps.LOCAL: check(not self.dont_use_locals, "can't use locals") check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.WEAK}, "local is for globals") - if opt.op is OptOps.THREAD: - check(self.ren is not None and self.ren.has_threads, "target does not support threads") - check(self.ren is not None and self.ren.global_max is not None and amt <= self.ren.global_max[0], "too many threads") - check(all(x is not AxisType.THREAD for x in self.axis_types), "already threaded") - check(rng in self._globalizable_rngs(), "can't apply range to this dim") if opt.op in {OptOps.GROUP, OptOps.GROUPTOP}: check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? check(not self.dont_use_locals, "can't use locals") check(rng.arg[-1] == AxisType.REDUCE, "group is for reduce") - ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op in {OptOps.GROUPTOP, OptOps.THREAD}) + ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op is OptOps.GROUPTOP) elif opt.op is OptOps.TC: check(len(self.applied_opts) == 0, "tensor core opts must be first") # TODO: remove the need for this by having warps check(opt.axis is not None, "tensor core opts must have an axis") @@ -183,7 +178,6 @@ class Scheduler: elif opt.op is OptOps.PADTO: check(rng.src[0].op is Ops.CONST, "only pad const axes") check(rng.arg[-1] not in {AxisType.UPCAST, AxisType.UNROLL}, "cannot pad upcasted") # TODO: why is this wrong? - check(rng.arg[-1] is not AxisType.THREAD, "cannot pad thread") new_sz = round_up(int(rng.vmax+1), cast(int, opt.arg)) check(rng.vmax+1 > new_sz//4, "pad adds more than quadruple the work") replaced_rng = UOp.range(new_sz, *rng.arg, dtype=rng.dtype) diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index cd7bc59284..9d3d7d3753 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -22,7 +22,6 @@ actions += [Opt(op=OptOps.TC, axis=0, arg=(-1, 0, getenv("TC", 1)))] # covers resnet kernels (3 global * 3 reduce) actions += [Opt(op=OptOps.TC, axis=axis, arg=(-1, getenv("TC_OPT", 2), getenv("TC", 1))) for axis in range(9)] actions += [Opt(op=OptOps.SWAP, axis=axis_0, arg=axis_1) for axis_0 in range(5) for axis_1 in range(axis_0+1, 5)] -actions += [Opt(op=OptOps.THREAD, axis=axis, arg=amt) for amt in [2,3,4,5,8,12,16,24,32,64] for axis in range(3)] if getenv("NOLOCALS"): actions += [Opt(op=OptOps.NOLOCALS)] def get_test_global_size(global_size, max_global_size, var_vals): diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index bfb7f29d35..cbb32b62cc 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -106,13 +106,13 @@ class GraphRunner: def is_sym_dim(dim) -> bool: return not all(isinstance(d, (int, float)) for d in dim) crs = [(j, self.calls[j][1].arg, self.calls[j][3]) for j in range(len(self.calls)) if self.calls[j][1].op is Ops.PROGRAM] - self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv | p.runtimevars}) + self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv}) self.symbolic_dims = dedup(tuple(d) for _,p,_ in crs for d in (p.local_size, p.global_size) if d and is_sym_dim(d)) def find_symbolic_dim(dim): return self.symbolic_dims.index(tuple(dim)) if dim is not None and tuple(dim) in self.symbolic_dims else None for j,p,dv in crs: - if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv | p.runtimevars]): + if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv]): self.var_vals_replace[j] = replace global_dim_idx, local_dim_idx = find_symbolic_dim(p.global_size), find_symbolic_dim(p.local_size) if global_dim_idx is not None or local_dim_idx is not None: diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index b3e7fdded8..a318d5cf8c 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -260,13 +260,13 @@ def _get_cpu_count() -> int: if quota != "max": count = min(count, max(1, int(quota) // int(period))) except (FileNotFoundError, ValueError, ZeroDivisionError): pass return count -NUM_CPU_THREADS = ContextVar("NUM_CPU_THREADS", _get_cpu_count()) +CPU_COUNT = _get_cpu_count() NULL_ALLOW_COPYOUT = ContextVar("NULL_ALLOW_COPYOUT", 0) # VIZ implies PROFILE, but you can run PROFILE without VIZ VIZ = ContextVar("VIZ", 0) # this PARALLEL is for BEAM and compilation, it's currently disabled if you are using VIZ # pytest-xdist workers share the CPU budget, explicit PARALLEL still overrides this default -PARALLEL = ContextVar("PARALLEL", NUM_CPU_THREADS.value // max(1, getenv("PYTEST_XDIST_WORKER_COUNT", 1)) if VIZ == 0 else 0) +PARALLEL = ContextVar("PARALLEL", CPU_COUNT // max(1, getenv("PYTEST_XDIST_WORKER_COUNT", 1)) if VIZ == 0 else 0) PROFILE = ContextVar("PROFILE", abs(VIZ.value)) SPEC = ContextVar("SPEC", 1) # TODO: disable by default due to speed diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index 40682e7ccf..15fde7b193 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -50,7 +50,6 @@ class Estimates: mults = mults.substitute({x:x.const_like(0) for x in mults.toposort() if x.op is Ops.SPECIAL}) if isinstance(mults, UOp) else mults elif u.op is Ops.END: mults = mult_stack.pop(-1) elif u.op is Ops.SPECIAL: mults *= cast(sint, u.src[0].ssimplify()) # NOTE: we don't push to the mult_stack here, you can't end these - elif u.op is Ops.PARAM and u.arg.addrspace == AddrSpace.ALU and u.expr == 'core_id': mults *= int(u.vmax) + 1 elif u.op is Ops.LOAD and u.src[0].addrspace != AddrSpace.REG: lds += u.max_numel() * u.dtype.itemsize * mults elif u.op is Ops.STORE and u.src[0].addrspace != AddrSpace.REG: @@ -67,7 +66,6 @@ class Renderer: # TODO: make this generic with a list of supported types supports_float4: bool = True has_local: bool = True - has_threads: bool = False has_shared: bool = True # NOTE: these two should be in (x,y,z) order to match the max_sizes argument in get_grouped_dims global_max: tuple[int, ...]|None = (0x8FFFFFFF,) * (3) # TODO: Ops.SPECIAL int32 indexes right now diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 680bd4608d..c62a69d14a 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -4,7 +4,7 @@ from collections import defaultdict, Counter from tinygrad.codegen.opt import tc from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str, axis_letters from tinygrad.uop.weak import commit_weak_consts -from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, NUM_CPU_THREADS, IMAGE, FLOAT16, is_image_shape +from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, IMAGE, FLOAT16, is_image_shape from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, float_to_bf16 from tinygrad.renderer import Renderer @@ -263,9 +263,7 @@ class ClangRenderer(CStyleLanguage): float4_style = ('{', '}') gep_arr_threshold = 0 has_local = False - has_threads = bool(getenv("THREADS", 1)) - @property - def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] + global_max = (1, 0, 0) infinity = "__builtin_inff()" nan = '__builtin_nanf("")' diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 042486d3d7..16fcff26cb 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -7,7 +7,7 @@ from tinygrad.dtype import dtypes, DType, truncate, AddrSpace from tinygrad.uop import FastEnum, auto, Ops, GroupOp from tinygrad.uop.ops import UOp, UPat, PatternMatcher, promo_dtype from tinygrad.renderer.isa import ISARenderer, IselContext, Register, PreRegAllocContext, greg -from tinygrad.helpers import getenv, NUM_CPU_THREADS, unwrap, Target +from tinygrad.helpers import unwrap, Target # ***** X86 Ops ***** @@ -791,9 +791,7 @@ encodings = { class X86Renderer(ISARenderer): device = "CPU" has_local = False - has_threads = bool(getenv("THREADS", 1)) - @property - def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] + global_max = (1, 0, 0) extra_matcher = extra_matcher pre_isel_matcher = pre_isel_matcher isel_matcher = isel_matcher diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 2b47bb8ea9..0664d79a17 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -5,7 +5,7 @@ from tinygrad.renderer.cstyle import HIPRenderer, create_non_native_float_pats, from tinygrad.codegen.decomp.transcendental import xexp2, xlog2 from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, GroupOp, range_str from tinygrad.dtype import dtypes, float_to_fp8, DType, truncate, AddrSpace -from tinygrad.helpers import prod, Target, NUM_CPU_THREADS, getenv, OSX +from tinygrad.helpers import prod, Target, OSX def is_volatile(u:UOp) -> bool: return (buf:=u.buf_uop).op is Ops.PARAM and buf.arg.volatile @@ -203,9 +203,7 @@ class LLVMRenderer(Renderer): class CPULLVMRenderer(LLVMRenderer): has_local = False - has_threads = bool(getenv("THREADS", 1)) - @property - def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] + global_max = (1, 0, 0) abi = 'win64cc' if sys.platform == 'win32' else None string_rewrite = base_rewrite def render(self, uops: list[UOp]) -> str: return "\n".join((k:=self._render_kernel(uops))[0] + (k[1], self._render_footer(uops))) diff --git a/tinygrad/runtime/graph/hcq.py b/tinygrad/runtime/graph/hcq.py index 39e68a4430..3ce29acde6 100644 --- a/tinygrad/runtime/graph/hcq.py +++ b/tinygrad/runtime/graph/hcq.py @@ -96,7 +96,6 @@ class HCQGraph(MultiGraphRunner): # set any fixedvars on the device self.device_vars[enqueue_dev] = merge_dicts([self.device_vars.get(enqueue_dev, {}), device_vars]) - if runtime is not None: self.device_vars[enqueue_dev] = merge_dicts([self.device_vars[enqueue_dev], {k: 0 for k in ast.arg.runtimevars}]) if runtime is not None: enqueue_queue = self.comp_queues[enqueue_dev] diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 048c23ce11..ae9967a3d5 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -1,6 +1,6 @@ from __future__ import annotations -import platform, sys, os, ctypes, functools, mmap, threading, array, struct, time -from dataclasses import dataclass, replace +import platform, sys, ctypes, functools, mmap, array, struct, time +from dataclasses import replace from typing import cast, Callable from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device @@ -19,10 +19,10 @@ from tinygrad import UOp, dtypes from tinygrad.dtype import AddrSpace from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher -MAX_ARGS, CMD_SIZE, RING_SLOTS, FUNCS = 63, 64, (16 << 10), (() if WIN else ('clock_gettime', 'sem_wait', 'sem_post')) +MAX_ARGS, CMD_SIZE, FUNCS = 63, 64, (() if WIN else ('clock_gettime',)) # ***************** -# 1. workers +# 1. signal programs def signal_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) @@ -40,21 +40,6 @@ def timestamp_prog(): val = ts.after(call)[0].load() * 1_000_000_000 + ts.after(call)[1].load() return UOp.param(0, dtypes.uint64, 1)[0].store(val) -def worker_prog(): - ring = UOp.param(0, dtypes.uint64, RING_SLOTS * CMD_SIZE, volatile=True) - wait, done = UOp.param(1, dtypes.uint64, 1, volatile=True), UOp.param(2, dtypes.uint64, 1, volatile=True) - sem, cur = UOp.param(3, dtypes.uint64, 1), UOp.range(2**64-1, 0, dtype=dtypes.uint64) # sem is unused on windows, it has to come last - - # spin on windows, sem_wait to sleep on posix - if WIN: ready = (v:=wait.after(lw:=UOp.loop(1), cur)[0].load()).end(lw, v <= cur) - else: ready = (rv:=wait.after(lw:=UOp.loop(1), cur)[0].load().call(sem.after(cur)[0], ret_dtype=dtypes.int)).end(lw, rv != 0) - - entry = [ring.after(ready).index((cur % RING_SLOTS) * CMD_SIZE + i).load() for i in range(CMD_SIZE)] - return done.after(entry[0].call(*entry[1:], ret_dtype=dtypes.void)).index(0).store(cur + 1).end(cur) - -@dataclass -class CPUWorker: ring:Buffer; put:Buffer; sem:Buffer; sys:Buffer; done:Buffer; thread:threading.Thread # noqa: E702 - # ***************** # 2. queue encoders @@ -67,10 +52,7 @@ def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: def cpu_exec(ctx, call:UOp, prg:UOp) -> UOp: devs = ctx.devs args = [get_call_arg_uops(call)[i].getaddr(devs) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] - if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(devs, prg, *args) - - la = [cpu_cmd(devs,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t,dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] - return UOp(Ops.LINEAR, src=tuple(la)) + return cpu_cmd(devs, prg, *args) pm_cpu_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), @@ -85,25 +67,11 @@ pm_cpu_opsel = PatternMatcher([ ]) def cpu_submit(ctx, cmdbuf:UOp) -> UOp: - # copy the cmd entries into the worker ring and post the semaphore once per entry - assert ctx.nbytes % (CMD_SIZE * 8) == 0 and ctx.nbytes // (CMD_SIZE * 8) < RING_SLOTS, f"submit of {ctx.nbytes} bytes doesn't fit the ring" - devs, cnt, cb = ctx.devs, hcq_size_var(cmdbuf) // (CMD_SIZE * 8), cmdbuf.bitcast(dtypes.uint64) - ring, put, done, sem = (make_buf(devs, tag=f"{ctx.queue}_{n}") for n in ("ring", "put", "done", "sem")) - - # submits are serialized on the submitter, so they can bump put without atomics - ran = done.after(l:=UOp.loop(10)).index(0).load() - room = ran.end(l, put.index(0).load() - ran > (RING_SLOTS - cnt).cast(ran.dtype)) # wait until cnt entries fit in the ring - base = ((put.after(room, cmdbuf).index(0).load() % RING_SLOTS) * CMD_SIZE).cast(dtypes.int) - e = UOp.range(cnt, 11, dtype=dtypes.int, src=(cmdbuf, ring)) - # the slot is a multiple of CMD_SIZE, so a word can never wrap on its own: take the modulo once per entry, not per word - slot = (base + e*CMD_SIZE) % (RING_SLOTS * CMD_SIZE) - w = UOp.range(CMD_SIZE, 12, dtype=dtypes.int, src=(cmdbuf, ring)) - copy = ring.index(slot + w).store(cb.index(e*CMD_SIZE + w).load()).end(w) - - bumped = put.after(copy.end(e)).index(0).store(put.index(0).load() + cnt.cast(dtypes.uint64)) - if WIN: return make_buf(devs, tag=f"{ctx.queue}_sys").after(bumped).index(0).store(put.after(bumped).index(0).load()) - e = UOp.range(cnt, 13, dtype=dtypes.int, src=(bumped,)) - return make_buf(devs, tag="func:sem_post").after(e).index(0).load().call(sem.after(e).index(0), ret_dtype=dtypes.void).end(e) + # run the cmd entries inline on the submitting thread, the cpu has no worker threads + cb, cnt = cmdbuf.bitcast(dtypes.uint64), hcq_size_var(cmdbuf) // (CMD_SIZE * 8) + e = UOp.range(cnt, 10, dtype=dtypes.int, src=(cmdbuf,)) + entry = [cb.index(e*CMD_SIZE + i).load() for i in range(CMD_SIZE)] + return entry[0].call(*entry[1:], ret_dtype=dtypes.void).end(e) pm_cpu_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), cpu_submit)]) @@ -119,7 +87,6 @@ class CPUProgram(Program['CPUDevice']): def __init__(self, dev:CPUDevice, obj:TinyELF): self.dev, self.name, self.signature = dev, obj.name, obj.signature - self.runtimevars = {name:slot for name,slot,*_ in obj.signature if name == 'core_id'} self.lvp = obj.target.renderer == "LVP" if sys.platform == "win32": # mypy doesn't understand when WIN is used here @@ -165,9 +132,7 @@ class CPUProgram(Program['CPUDevice']): else: args = [*[cast(int, b.va_addr) for b in bufs], *cast(tuple[int, ...], vals)] assert len(args) <= MAX_ARGS, f"CPU programs support at most {MAX_ARGS} arguments, got {len(args)}" - for tid in range(global_size[0]): - if 'core_id' in self.runtimevars: args[self.runtimevars['core_id']] = tid - self.fxn(*[ctypes.c_uint64(x) for x in args]) + self.fxn(*[ctypes.c_uint64(x) for x in args]) return time.perf_counter() - st if wait else None @suppress_finalizing @@ -198,48 +163,22 @@ class CPUDevice(HCQ2Compiled): pm_encode, pm_lower = {"COMPUTE": pm_cpu_opsel, "SUBMIT": pm_cpu_opsel}, {"COMPUTE": pm_cpu_submit, "SUBMIT": pm_cpu_submit} def __init__(self, device:str=""): - self.workers:list[CPUWorker] = [] super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") self.pm_bufferize = PatternMatcher( - [(UPat(Ops.PARAM, tag=f"{q}_{n}"), lambda ctx, q=q, n=n: getattr(ctx[0].worker(q), n)) - for q in ("COMPUTE:0", "SUBMIT:0") for n in ("ring","put","sem","sys","done")] + [(UPat(Ops.PARAM, tag=f"func:{f}"), lambda ctx, f=f: ctx[0].func_ptr(f)) for f in FUNCS]) + self.pm_bufferize with Context(EMULATED_DTYPES="", TRACK_MATCH_STATS=0): clang = ClangRenderer(replace(self.renderer.target, renderer="CLANG")) self.prgs:dict[Callable, CPUProgram] = {f: CPUProgram(self, do_to_program(f().sink(arg=KernelInfo(f.__name__), tag=1), clang).to_elf()) - for f in (signal_prog, wait_prog, timestamp_prog, worker_prog)} + for f in (signal_prog, wait_prog, timestamp_prog)} def func_ptr(self, name:str) -> Buffer: return self.func_table.view(1, dtypes.uint64, FUNCS.index(name)*8).ensure_allocated() - def synchronize(self, timeout:int|None=None): - for worker in self.workers: - put, done = (getattr(worker, x)._buf.cpu_view().view(fmt='Q') for x in ("put", "done")) - while done[0] < put[0]: self._wait_signal(done, put[0], timeout) - super().synchronize(timeout) - @functools.cached_property def func_table(self) -> Buffer: lib = ctypes.windll.kernel32 if sys.platform == "win32" else libc.dll # type: ignore[attr-defined] (ft:=Buffer(self.device, len(FUNCS), dtypes.uint64, preallocate=True))._buf.cpu_view().view(fmt='Q')[:] = \ array.array('Q', [unwrap(ctypes.cast(getattr(lib, f), ctypes.c_void_p).value) for f in FUNCS]) return ft - - @functools.cache - def worker(self, queue:str) -> CPUWorker: - ring, put, sysbuf, done = (Buffer(self.device, sz, dtypes.uint64, preallocate=True) for sz in (RING_SLOTS*CMD_SIZE, 1, 1, 1)) - addr, hsem = 0, None - - # sem are posix-only - if not WIN: - hsem = libc.sem_open(nm:=f"/tinygrad-{os.getpid()}-{id(ring):x}".encode(), os.O_CREAT|os.O_EXCL, 0o600, 0) # type: ignore[call-arg] - if (addr:=unwrap(ctypes.cast(hsem, ctypes.c_void_p).value)) == ctypes.c_void_p(-1).value or libc.sem_unlink(nm): - raise OSError(ctypes.get_errno(), "semaphore") - sem = Buffer(self.device, 1, dtypes.uint64, options=BufferSpec(external_ptr=addr), preallocate=True) - - worker_args = [ring._buf.va_addr, sysbuf._buf.va_addr if WIN else self.func_ptr('sem_wait')._buf.va_addr, done._buf.va_addr, addr] - (thread:=threading.Thread(target=self.prgs[worker_prog].fxn, daemon=True, args=[ctypes.c_uint64(x) for x in worker_args])).start() - self.workers.append(worker:=CPUWorker(ring, put, sem, sysbuf, done, thread)) - return worker diff --git a/tinygrad/runtime/ops_dsp.py b/tinygrad/runtime/ops_dsp.py index e487f666e7..2de37e791c 100644 --- a/tinygrad/runtime/ops_dsp.py +++ b/tinygrad/runtime/ops_dsp.py @@ -10,7 +10,6 @@ from tinygrad.runtime.autogen import libc, qcom_dsp if getenv("IOCTL"): import extra.dsp.run # noqa: F401 # pylint: disable=unused-import class DSPRenderer(ClangRenderer): - has_threads = False buffer_suffix = " restrict __attribute__((align_value(128)))" kernel_typedef = "__attribute__((noinline)) void" type_map = { **ClangRenderer.type_map, dtypes.uint64: "unsigned long long", dtypes.int64: "long long" } diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index af3ad32a63..5c8519b549 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -167,6 +167,11 @@ def _merge_submits(calls:list[UOp]) -> UOp: estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) def _merge_queues(submits:list[UOp]) -> list[UOp]: + # CPU submits run inline and can block on another queue. Keep multi-queue CPU work in schedule order so every + # producer queue is submitted before a CPU wait; merging by queue can otherwise deadlock alternating dependencies. + keys = [unwrap(get_submit(call)).src[0].arg for call in submits] + if len(set(keys)) > 1 and any(any(d.split(":")[0] == "CPU" for d in devs) for devs, _ in keys): return submits + merged:list[UOp] = [] opened:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> hcq calls in submit order limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) @@ -412,8 +417,8 @@ pm_lower_hcq = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), lower_hcq_call)]) # ***************** -# 6. batch: adjacent hcq calls fold into one submitter on the host SUBMIT:0 ring: a submit whose cmds call the -# compiled piece programs, so the worker runs the batch in fifo order and the python exec is one ring push +# 6. batch: adjacent hcq calls fold into one submitter on the host SUBMIT:0 queue: a submit whose cmds call the +# compiled piece programs, so the batch runs in fifo order and the python exec is one submitter call def _lane_arg(a:UOp, lane:int) -> UOp: return a.mselect(lane) if len(to_tuple(a.device)) > 1 else a diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 196c085133..486f1dbf37 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: class AxisType(Enum): def __repr__(self): return str(self) DEVICE = auto(); GLOBAL = auto(); WARP = auto(); LOCAL = auto(); WEAK = auto(); GROUP_REDUCE = auto(); REDUCE = auto(); UPCAST = auto() # noqa: E702 - UNROLL = auto(); THREAD = auto(); PLACEHOLDER = auto(); LOOP = auto() # noqa: E702 + UNROLL = auto(); PLACEHOLDER = auto(); LOOP = auto() # noqa: E702 @dataclass(frozen=True, order=True) class ParamArg: @@ -39,14 +39,14 @@ class ParamArg: args = [repr(self.slot), repr(self.dtype)] + ([repr(self.size)] if self.size is not None else []) + \ [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] return f"ParamArg({', '.join(args)})" -axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.THREAD: "t", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", +axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", AxisType.LOOP: "L", AxisType.UPCAST: "u", AxisType.GROUP_REDUCE: "G", AxisType.REDUCE: "R", AxisType.UNROLL: "r"} -axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.THREAD: "BLUE", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", +axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", AxisType.WEAK: "WHITE", AxisType.LOOP: "WHITE", AxisType.UPCAST: "yellow", AxisType.GROUP_REDUCE: "RED", AxisType.REDUCE: "red", AxisType.UNROLL: "magenta"} # NOTE: LOCAL and GROUP_REDUCE have the same priority. the order here matters -axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.THREAD: 0, AxisType.GLOBAL: 0, AxisType.WARP: 1, +axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.GLOBAL: 0, AxisType.WARP: 1, AxisType.LOCAL: 2, AxisType.UPCAST: 3, AxisType.GROUP_REDUCE: 2, AxisType.REDUCE: 4, AxisType.UNROLL: 5} range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.LINEAR: 0} @@ -1268,16 +1268,13 @@ class ProgramInfo: @property def function_name(self): return to_function_name(self.name) - @property - def runtimevars(self) -> dict[str, int]: return {v.expr: i for i, v in enumerate(self.vars) if v.expr == 'core_id'} - def launch_dims(self, var_vals:dict[str, int]) -> tuple[tuple[int, ...], tuple[int, ...]|None]: global_size = tuple([sym_infer(sz, var_vals) for sz in self.global_size]) # type: ignore[arg-type] local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) if self.local_size is not None else None return global_size, local_size - def vals(self, var_vals:dict[str, int]) -> tuple[int|None, ...]: - try: return tuple(var_vals[k.expr] if k.expr not in self.runtimevars else None for k in self.vars) + def vals(self, var_vals:dict[str, int]) -> tuple[int, ...]: + try: return tuple(var_vals[k.expr] for k in self.vars) except KeyError as e: raise RuntimeError(f"unbound Variable {e} used by {self.function_name}") from None @staticmethod @@ -1298,7 +1295,6 @@ class ProgramInfo: if u.arg[0] == 'i': local_size = None special_size = local_size if u.arg[0] == 'l' else global_size if special_size is not None: special_size[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) - if u.op is Ops.PARAM and u in _vars and u.expr == 'core_id': global_size[0] = int(u.vmax) + 1 return ProgramInfo(sink.arg.name if isinstance(sink.arg, KernelInfo) else "test", tuple(global_size), tuple(local_size) if local_size is not None else None, tuple(sorted(dedup(_vars), key=lambda v: v.arg.slot)), tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), tuple(sorted(dedup(ins))), target)