From d4889c2a2ec64a50a5db7be19a76ae87bb2560ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 19 May 2026 18:42:32 +0200 Subject: [PATCH] flatpak: run host interpreters via flatpak-spawn; validation --mode flag py_func, lua_func and the run item now reach host binaries through `flatpak-spawn --host` instead of trying to load them under the sandbox runtime (which fails with a glibc ABI mismatch). Adds `--talk-name=org.freedesktop.Flatpak` to the manifest, stages the /app/lib/testium tree under /tmp so the host can read it, and drops the dead `_FLATPAK_HOST_DIRS` / lib-injection code paths that the new approach makes obsolete. Validation suite gains a `--mode source|wheel|pyinstaller|flatpak| appimage` flag so the same item set can run against every packaging channel; per-mode report file names avoid clobbering. Co-Authored-By: Claude Sonnet 4.6 --- doc/manual/testium_manual.pdf | Bin 702478 -> 702473 bytes package/flatpak/org.testium.Testium.yaml | 5 + release_note.txt | 8 + .../interpreter/test_items/test_item_run.py | 10 +- src/testium/interpreter/utils/bins.py | 274 ++++++++++++------ src/testium/interpreter/utils/lua_process.py | 38 ++- src/testium/interpreter/utils/py_process.py | 37 ++- test/validation/README.md | 65 ++++- test/validation/run.bat | 120 ++++++-- test/validation/run.sh | 144 +++++++-- 10 files changed, 530 insertions(+), 171 deletions(-) diff --git a/doc/manual/testium_manual.pdf b/doc/manual/testium_manual.pdf index c03d73c21ebf940d70cd7d1a2ced62055fa1f779..dd483b483f9bdfb1a5d3988dd756efb2df16f0b5 100644 GIT binary patch delta 7156 zcmai$WmFX0y2nMS84!>zL8QAGx+J6-I)tH-W(HwE!J(v^L2@W*kZzEMp(Lc0?(UHE zdhUDIUFWR(;eOj|?fqr{{_#99#~k6u9R9%=!H#fqSq}_@Qxm=EeGf!&RdvS9Tz@#-~K3jGyQO? zR8NUi$J+2W8mxk8<;zMnpiM;8p5~Z8U5d5hvpy(Cxii^0imUUrA13F!f@f8H=}I;M z2Jk;pu;(SGrqp^BEOl#P$+%$ox8eOHVR!i8R)yM+ee-Ja?%!{JNxyM&Futo{{46H=;;ZH^k@HOGoKnI>b zS@wsszT*uac$BRR+SZ&pXi(`8r$(ui@&h}H5y-+x=~VJFNdQT*Cog~vgFiWw7eIqJ zat$IfnjnYGil^C0NYwxEVXEI^3Vpa`HrTOzOhad!08#k}K2&8^K9(ulB#RkSW#%Gy zUrn3S8L4MZ84%~&8sFkG8ns2?Hr!f6*SVCnTd{Tt4VBsVC`qB7HeCp}o)}mH%v@X( zxyo9ov2T^ig}J^tEZO6;f|MA%`s{;v^x*JdNqInKP$3WcUHLo4AUkRIacOf=Mqygi zPNlN2{PIQ_+` z5qC-75U8Q$mG8|iVR7TpEVI8zult*8sUUZ~dtxUg6+u^;!|>~b!`~P*GO;QiHMY^Z zV@&imYNx_+#u8~*AF=y@Pgql*EZqMYGP{c;Eh0@Mf{bC%r{;WOaJAPl(}#4w7zEl3 z@MdF$P&42O2+Ke9zCUVFE*;Uh_{#)6=t4QoqM#b)%Y>GP2{Mh!niMLqjFYpU#@9Oy zw;miK9&EVteclR+Agy%mloTut0&_Qr@N)SSRXLpvevHXB`aV6Y3K*KoU2--X)CleW?}ifPSZ$36jY(id;>!;#HN8 zICCPq{_KHr^E~YSKx2@IECE#-SM6ZZjhwwIZkw+!G^8H=@nnL36YH4s%=hwkbs8;@pXis$ za7oTDMFveT&F1bi99Ivo$p6$R0!nqBxJqgEs{6g(*R(eN z9FuD}BQ4%avo}DuCWDkwaXy^r{`8@56vb^q#IES|ZTu;j8vv>(ON|;sGG#O>Sg}Tq z%sJLp`7_tU`PIrIjL#8Nw=E)h)=E}kWQ-HErSGTx8P!!;@YJ zpoDOd!z|o$uk;-D+86Gt@t90e^xJO88fX%r4qJgY-Q7cAu4(p7&e*GMJDQ|?an12b zA#F5uk94|<;|Xl@9DDc@?;}2QqBS6m?>D%KuCCeVD6+1oe9vO5ma8F$Lmp>>U8ml& z<8gZ0);BA;{*|d`WsT=gYMaU=h!qJK4iUF`i-b4w2x)TO3|{nwchF-2^>1_2(kx z1L(Pj3njm!)VE9*&R-Op02;u|h%QQ?aQ_a=WVAWpAHGAv01ccALfEOt>5m3_5r8@d zCN`0``ahR}pubHloLmY5P$QrgK>>g$q%471L$hre2yGzHZAF6?TR);`$q9YVG&b=e zA@l9Rp{di$vDjr~2ky5j@#Xc+Y$9>mDCROu7rptCj43v$oC~=dxY*0F&EK9ktd(;Amjt4$kIrieGOUa&t_sXT3Eu!Du%kO?F}#vZ zY!rqU|Ez|I$jg;*#9B8bA}2(BiuqnkyjW2LW4NV7WKaEhpPsun6^kF7>}>mazixQ` zsYZ#-(V^ydhO!T5gwTA;kBeWefh-@c*=QPv)~vsc2p1Bo|!F`FqS|*sJne{y?3_B$I;Q zc6-+5PjYhY>+-F2y6+cf)ol6`B!3nh!E#!=N-B6Ly0S;GVBIg0)WmaPA6eiO+@Q>? zqeh><9+lxdJv!a1@`pT#>Amrk_+;0$KA$FWR;>CU~>S3&xTpn7lSI45V z`fL>u^$Y&(TU>9=q_TI4qnkmX%VangfzJlVl*KTWiMAA=Rmk%f7@w^+c*Gs!b18|p z3ST*r=@&Ggj_vyBp4tp;a_7D`IbK8%7*`M@^1A9voLIzTCOxGhe8D_notRpvRZpD3 zYj9Yw&*q5M4DHpdDeD(flJya_Dps==oODw(TW8s`?w3F3XsHhHlkg8X^{SKi44bS% z7^=pBiRgDD5jzD-q8c48k9sT)yeVDP=lQi*f3P)bB#hg^n)V0{w!U)yO9lyq$bVvX z`-}N+dET<0R$G%s|7i}u{i^9c<~iwgWGyM7tWjOWdvj1I_A3dS^kec?W($)&t4~^F z0qbSq=T`sB$><`Xcd#+q*MLGLqe*7{?%UMitOj*suWH-fpgP(8mp-GHa~?pMeJ8e* z?RuU5jdDo@sQn37Y-m(SULdtD0R>hMN=z$KZRS;aq)7r>HEz|F^{4U~ifSXn>fl25 z`U$darHua76jBx)d{tBq);I||!rEH7N|}dqn{+KZRWyK1Y%M|s8O>QQt4DY{*@-cy|rB;~0= z$YXjy5(bAK4!bZq?e?mo@LEQSS|l~a&(;#T;v>zBg3wCdq29FF3aBA~6S&HpU5e^1Hu zk^t<`aVbEG9EkgL;<(M*d(WMb1erRi#Un1w&h+2e{vz0vJ1vrO!$jVpOoOLc^ z_}WWmxQMJQEls3VNqy{H%=J3T#&*avPPFqZeCiwl z#4fZt!2DWuhi)ObI@L(6E*qKP_F1Rk5dR1Q$L^brInE}ReGygX@es-y378HX%5nc*$Br-4f zGFTrwe2V5fm7Vqop3+h6*B3c~AGludX?*DZTsJ+&^nAkL3&!JG{!VT|+EqC`yWm`M z1edZ3+js(%i$hQV6-_c86`>Fo>uX87jR<4*N|pRneOWW;K-tuGpw1qx7hb-I(IXBr z58@9hDpK}avsi@!w+?Q+VxytG6U!|0x%xUzuu zmwPfG*A9saA$n1hb#ye!DMZ6L30}#i$8N8}Hk{y#rMt78}ty5_)O-kmPq1b2ukuM}vR*Yhk@wVt3<1!}S68mZpd6Q&s(` zC#tQ#Mn1xhg_?P>ld|i#$%uQ`GHw0Ao%q{)m`fWncALZcCynZ9(JUkkw%zi=8Sf9Lq2u0klUPIfF)V0t_Wn-6VKNu3XE zbPbO>))CgX7*QTC=~`{daGs9i=MS)8#~MgF#{T@a8Z%NPB7MtFHVEXvTwJk>9tu`T zQWgpbQb-+$d;C>>Zc;3o@?~mJ`-yuXd0QGC;ZaO;Y8)UnO%BgJ?WvYnf?_gTt_N3?At!`oDP8NBW|EieD*@RmIv{xJzM+ zVvny9)A^i|!ip**=D;Az_7S%oKVKVA;4)RV;G!xjR>;`>87fTVnCi>Zbm)7=R$SVZ zo!-2V5Htn5KX3T!L?z4ax<^X0_)tQvN}itx)m|w;x`DR6YM!t+=UYJ|IKiBHcN=ph zSy3v1!_$f3l-fXn8c8!*J`c&mh!pT@vRIT6A!JkI<}oZ)VOPGx!Wamw<5nqFW2z3< zl~&?X;mIslK5@t$bE%b~lF_oCP~>CCn359)WPat(+Ej~{Y;J29DX{`)m+ zUPSh6*GgiVAySmtkUzOh&z$^7GtK@0Z6jGiZfHl5K#TA8ngli#Y*M`FzofxaS-^_~ zc7Kv;8*PDK<&SmJ!wnY*x9)%jD_gyjwuc<)&3N0cRV6tqdDTt7I#tC)3&l4T>fC}dkAMhAsE}Y_Uz0x zGaw@}kUD}bim~QH&p&5K-cxj&G-7^fkRDA9bS3dhUXCTLxpTnfw~bSHUzaHd#BVmY z7<(Vas&I;@05m0XxMxZ~9^gJzaULy%q<_`2hdAvSEyt}_^fQo&G4+;u^tU9DuDXWE zr?log%m0MOVP2cu@}eaHAt54w=Dm=PbdR}yxgiP3`x(d^839^(t@6V-q5nYT^rHI~ zzNG1tm_>GP70-4<7T&c1Q6NlnP$je`%GMixw{!dHd}v^0I*zf72@RuU#pK7stTH9C zX*$(-twVQgoKSS20=qfP_5|C%wbs{5I24wO2&e1)6HSK&u9pKLKK4|}W_8|`WUv3m z)3o)GHHqF2maxo}h;ZvfO_4ZvWmPiOjPriyNeJ8E`*_I3yBxH#z1=2dVZuuPbK=G( z>FZf{Y-pid%*cr5@Z-Ys>RH2@i%Dc7Z7jE2P^O9bvhB3<4fhNg)#I!*cdPerC}|S* zH6e|(uCydi*zk*V5WM@3S$?Yg43ay!Z)1B{7;;l?(OsK{*U?pqw`~B@WVg3o=s5xz zc1dt!O%1pH+&xV+^EG{(b4Nm?8A||(&a6@!`y#3iDaZ2D@RA$OrTCl;${uhUn}XM% zD#V@X?!8W3MvQsd1KCg7X{M%`MrtfIoCec_VzTHl%lDni5oT8Tu`vF6?sl)Qa_{!@ zCgrj7>IeL%hbzGxJ>7xn#OdZ8jzivv^M|{Vi;!KMr8CQuxb!u-Ob}0mdBYh{$aoo& zsvjT~;sNv=ESD)Dvuza=USj#4B2cYKMVzM1GCW@wm17wrMnSv2?`15og{@b7KOAQ( z6(S7MzHhLl>3Y$C1)0|TQN(Xca>>?q%z$_T@_@d3@B2|Q`ByoRj)qmpxIg4vWNz$6 zMZ`+Yhb-+vcnx!-{j2>%%RkFEYXy*6URYE>2qa`^dB48~70@?9f zg7|C&q$U5q7s){tKxKq5=--zX;aOvCy=yY`$?eu9S+nAt4TKcK*oMLu3?{XGgFR1# z_LK!f-(bUuS;8$ZygO4 zYTl3R{AxJ*c|{#!d3TwfZaGcE2n|kOxrAFI@H~mIE!eO*yN?C%LEY|@A=QJYW_2DP zO9k|*M#u(cCSBj2&f(&Bzoylg2_YwV1wWUS5O)`Szluv6ZoVBYy&d^c6nM}7=m|e= z5I^QCe*A9!$F@>v3#OP}wx~Wz+R!S>(36zqdFdzbZ=>vQ_p>#7nAH17)q921(SVcy z1PPSb&Wha5nn1IUU6jlNO09{O%m@(Vr?+2m=F$$iN4)a=E}NOugL9>qcY)3LM2 z)QTMQz}`kuHTP5j<%XSW-joV`yUBLMyXuTQ%~bwzabr1{Z0ZTKxUtF)*>6C@HELTH zq3u-OR55w82-`;T8B4Kp^wY*?obHJNf;#6v+N>PVMGeb#^AWIem+k|ja0zfstyv?@ zl?BOwjq2b~Yc7c`p_AUbxpqvdO@XN`yE2K5OHW^n#F6+gwf>EN9^ z(^IvyG~C0~v82xzm)J{OVRy&xeRNpJlSkDzC*68jFa+vfo0eR)!rz zgFX@zenP9Z!JGDR_w_og1jN_-?&pY}XM@zc6N0B&9Zl;pPa6+U=~3vK)8D z!UQ^I!CvKVjfQJaGvFRs5iWM$QVt8W7aVqf^bHyP8~{Oqq_^?G zx`h87umxO490Fx6+VJXordr1~6VwQ>LYN$X5Q4IK4bZ%yBmNf(U z%454+er$9+Z;xUlW#rDFNz$?=SFb=bbaKyK%C$%1-C8kdjjpVV8p+6tv5~B#-QFjM zJmY`1v_gbP#2%z#HSK$nJI z+*@d`BCL(%@}t zcJDA{t&4>94DimUk8Pm&88O~4nX`C{2jgNdIT3M3CLg23>3#@FSIq5wgDyc#G_v+qqt!~~Rjr0btH-#~RcbIJM~@#Adewl2k8Q7RgR;XyG5 z5iMbgjrhqH{5-wHNH{^M_M%8X4Yp5cCH0I2`{xxCF0?0`8$e`PsjNfePA1uUpU7ZU zWIpH@lyy)SL3a4c=W~U!kCjVTw>K%x34Z;uigPakatrEn(-wH@>-UWJi{$-4Vwof! z+?sGv$;UOxe)I#IR($L+pxtofnGZ1%v85nJvLWe*j(YlZV|z1vz@TXP4=1lx$fnqN_p!yl`U)J{TgridZ8prVTEl2_hI0IQ~t-h%xv0Eqh>1cjD?yB y$S>liGY^AcL5-Q>ksG8_^(L=4RVZ{2$~(>XK=|TK@>&Cs5myYz%&e%T1pF_>kDiSH delta 7123 zcmai3Rag{U*Ol(>M!LJZySt@^&H*F@Mqro`i6I@3l4TQCwB8H@r(1*3t{!5Co7CM;AwkffX(c0eEy;NXoN zvTSQ)>lq_J8gXO3)8Is^`>=@3WKqND^)vH?{`$jFbHE&Bq1@%V{+V~@v;yva$i&FF z?lFsgz<1oSNO4=sFJF$-OwsRrL|E%lXlVfcUn@&#`7nC{+|2;?I-}kZ^33jtTdm*` zCv5qP0G{F__|H=l5bsTA0!DT8$EMED=xlrv`tbqs3fWa1P7ZE@E+C!hNBapcBJ))w zdKVv0E`da;#sN9bWKOjj{i$f`Oea;ML4U5=W3Gc{p064~sN8|2L=6JjYR*xWr%!Ko zq~>;8Mau2;J%)%Ti3-6}*eUFUw^W58v2MS??QrnM$D7s&dx+2kC#NERaGZ|6KOIYK zaK>2tP#jCAHahIMR-c74Lh*abD!b4Iys@0nKX^g{hb=1;r$Om-kVXh?*5*-ua;4$6 z(D%kBX;3F(XyL86lz6Y9-y4CB+0Qup{FAm-6L1H9ENLWRcrHH{4XBRo2P0~Wb1ZF3 zQt64k_>h?ttp7S^XA0D-!Y8X^k6_lm;2zD>{!X0{M)4@Uryh$tPB4xZIMozZq?MTdQTW z#!oQd)x@*>2O?7VngA9%NZ^0inEKTxMgSLbCZn2^TZ)C5PLB0b@kN|+t9w(Yu>jpH zCdZdgzPI8M>5G4t={XgbyIfExQ_S5qWuL`eo@+;Y`yHu0Br z(5`*vmI3}G07FWec;ID(&c^5P{CJ}z{W2$)H`O2rq%Ua!iL%!}k95I5nh&l!8U4k6 zEPS>zDXpVX*W|`E#G;A9G{$2$)?2Sop(GQ!i6fOv@;O1E|HP}Z>d2E4+K6mAibEMy zqCOn01Dkqb`Fww7INx3&rxiQhkkrD{Ly&@+0^0!=dqZ|CH9En=Y+5Ny44kW<_PuXT zV4wx7k5GsCJR91Jv3J7>IHc{`=^I%h1 z13G{-6P$x>nFNa|pMzOxmh#NPSgA>MhF#$zPDZzdWL7qGO-OUIYAbE4`aR>oV}`2# z&u+fDj?2w%yedo1Rhq@>#USmNCgRga8SG0$YV2aY1E9)8f$@Hkc&Ob2k>(nYTl;z16Ml{NekDY&u`U2l%c}_jB9id3^5;tNDo; z_oJT-$g2;P#mrgO1{XOyFa-{8L6jO~Kt!%yJhVowyuL>M>{_88B0hg@R#@|$%4Nv( zkPF~DCN5Skh1tau(3bp~#LCgLP2t;js%{ZVJ@i8ti7cqrS~#rYr%9Z?Pp&lmisF~P zu?k7fJri09(Me4~qyAjXY5p_choaA_I7|bzD&kf~{<`njk!KFTr7WBMti+v`mnq)p zG&mMQ*L-^;<&NZ!uUsBh1&ck^3R_>diNK5Fox0ksHbuRP#us<<7>>sB2>ykWQ3OjB zRKuuL#w~FmBkxakIM16Us*DkJe7Zi?L>rM_s!t^$v4A6vcK0V*~zn{JMjfhl^ zFj9y+y!XTb>*la6HUIpi2UtoLXu@m&$*W)0nwtw*Q|RLBOSKrIm22gV922^-rQ&(N ziK2Gar{#X1K&uKW#0O-u4a(MZ@S%C=?l)u&*yncWu-2 zjcjlW%_)Re$)0e9WzW>Y&Kr^jkYn6{BW%`gVA+-V_{T;Dse1oP=iBE6jpkAI&Mc|iR|UUqLzn+*R}-o;n7k2vOWeiEKGlf zq~@ctlP>BVU5M7tyh7$3t4Yr7!vy>-TC>Js)QDa~h2byO>cx_j7mK#)gW z9X~u^d`-a#c6CORt)i)+4RrA%UZmnnybc*QVZP7$#b@mx+q;N!&t_#G zX}JeIy7I?a-wG%PpI)nF1AIWW9-&gFA$_EC#qw)B^?rLm>(vhWWupa1NySj+v!KuV z)#)Jl;OLnU1f^1=a*bE)h!e!I{-I zA%dp6n@HW-0N!4(a*#}A4C4;f41_}wo4HtmkqMwPD!^_wR$P}1T<1fq7^Te_+wKxeGk_yYQLuc*^mk)jZmvrw1Y|2nm--7UJf=Swr->$7dJLMxekUD}N4{SR6{BcoF?1$!JR8Ket()BOF&!a3En74* zFQD;f35FI6q&ONJq4K`VlQ5&Dk$x`=v{Wpk#2D`*2{P-lL@}wBd)5rZ{WMc3 z$Gyye%Vlam#`S|!4gqK^Y@`c+%yPm&IC>Kvi*PDMsEFdW`s;q|Qfq$yQeqlSX&<`ix1BGDg1K8C5vYXM41H7H~iUY`p#Fqa@MT&44P+WP=WlTd31KM4#wk44>|Wr8pkO91P2Sw> zlYLAiJ1X}xjGK2_`smBg>~T}Go_B1x-RXFo7&T-Y+6Fc2`4GXmIkS0=gsUX zODNbj3Y$k8a4T2 zF03a0f%|@%YVGrzvH7$8*3J=C*TwN0pWi>MCvx{zsM9MeG15a?Pacb(M7sFzG74G_ zk`D+&fD-}Gix%p)Uk|pd;GjpJVgu9B!<73G%U(D6)t$%O{1bU;*vSuQ{xOUW3x*&Q z2OXb&_;yO#t>&3|Ch`=;Ct*zleqs1X!w1@ZSXc!@%V42(k;;AUO$NX@Di@;hw8C8W zSLnBgv-Lj?k!(C37CR<7Q4;PiF9kFtxyFkddikA7dVA~Cd0z}~kU9zHxGa~De}6%5NJ7-lnL!mhU9M%P#tg=tVMDbMY`10)a`DFg zBmC3&r_M?`7T1UQP>>3^aom9;6eWxYnK>gG(QN>Y4d##KnWgmN0NeW=BSuOIo&5a_ zCc$`|Y{SL-yZsREd?{X`Xao`Z_r@w#fL`IRES#MrA(-LU=YFK>>Ob{#B$<{9-EAp2Egrt}DpsQ5i3mvQkJg;4%T-udP zL@Dp42CE8BawBpk7cwf6K3;zD%<~9%Xj;!(8@7}VLIH~p(MsOhMn}YPUdQP$LhaPO*(-s(lWp{FW2g?k zRgBDaUgX1Cn?(5R@bgkfxJOG^%(CLN_fSe=xZmuV45;sUkcAJuY+LPjsK6LQ(wkP* z1kd2yul3Je#=aB_w?4pS-~C6uGoSsoMEe;kxiiPRp&_3qg@#xJyJ#(kzgDdypJhcC z#~OpUo)v7wZO+6d%T~jPVI`SurxL5#s2jaETcn{>TPt?6k6wN!xAWep{AqvHv@EgS zA?JMuK8RywE}044jG2KAISEEs{~Cz9zR+HAbJ`)fJ!04zB(5;}UMG1@o_X&0E8zOQ zV$aZ_`i1A|<~dob&9*ME8-_Q-vJ+zdQk!s3*{fH_N8_P7eoW;T$zgI;5RJgi! zZd>Sn_3k?*en@frATRoo4L4PzZeCa;N^t=P*NWggQzJ4;!vTk>Cf}Jv zu3c3w?&LPmOx`Q6E8A)a8Oj1u#jhBC$!V9XyjD|87%WWMwB3UiN}Whm@zGCR$CvAtev}fu)tu%gk1KqY!eR3S zKTs6bJ#fH9PJf@p7txy>DJi)^4FqTGSm1AGGP*=DzQ&2>G4H|B97U1SCcdK3xVYN+ zvD@knYvJP3y!l8!paR=^R)a-zSQpXAgSIRRCha3l4S)}5WsNe7HEOTNas?*o23j{K zMzO>;^(|7D4&W{nn*BgR`QQ%}zn<(vLe`IjYtmpfnh{;2lde#P+KDyVyd2tWG`&MW zgYW1(iSZUtrm+hz%rP_yZu$AJ)yCWzGu*MS;n3fqTC-6)ViVZrpMX~&P@vTfs>jK6 zV|Ty0AAsE#{xKKtr90J2knXO!daJu_omU_Tj>K0gVIwS*gZw&pY`yb66SVyOgBWx7 z)muMc;2)361lwr(0f##v?}29{jw5+@J!-d{t{mFxrBBOm)1e`ZvT=fd3E(Z6PTccq zc?+9=9-gO_@fvO5!vmuq;8nnhEHQUPdVGWTZL=p{Av1dkxe#NsE_#y-B+=+4J^4!d z;w7?JbAn6J)9KD{n#u*xPR!OzarH-;I?;HavPbyaT5K{6;~w3QM1UO;fQFk+mO#LQ z5CkH*XXBUr`3GPmhsFSKWDVPI9`3|xk#06Ua0-UfkbC!Db(GO6_?f7oEP`~O8 zmwT3juhe4G(+r7N4nr=*mZt7B#T>Om$v$3*7NPFSJK@i%{hOaT>aeMmgv7)g08V0} zPLe``e~+V(q=cxTsDr4eu#>ZcfS9wK%>Q=-4z9yi2Z;&)cSaFiv@{@Fq8mDwyY$|) zGZb4QQYA-PQnfT8Ri%O>iTC@{8JN(*75$NjP|ZhKCS{eEdEx@H25*VszXObl=&K zD}{+7KpKmwBMUytB;})lan0+cpKNH8EM>OXeFc=7KRv{e*YUWu*(fLBT|;k6tgS*|%5{ zj7ujyWG@2L%#11QU-AvIwwYup(uX|=4K5h)4>8$94YV0$Z7*N`;ir5Bsu~*x9XgX~ z<_Z-R1vY-%Nfz3EAJhmbjtb+gMO=+0_)avJwMeqg$jYm3 zy(Y7kR_vy0-NaA#^kzTp!>(4za)0B;e~RE*fdRn;_K89PwFhUi!#8=wwF6Qz;H%NG z=WP0=FT*YXzRrp?1WOsH9JWQ2?gXTzIR^{H3tyJ$T~r9q=`JSla#%|z3YC&gZb{12 zjR^&JH;c8ft>TQXT)VTirb(-t<<5+#rsB-ejmkW@*G4KwrrZ(RnYjeioxYyQcz7@L z0|XrZJ(1S&#&qdDG-!2TSntROV9ika+jpdn%AP>IoRQe&k1A z{!vfpRe!Wtdb%^**B{Z;pJa=Tcm49xC~b8%C;Ow3M2bHnv7-gg=hvvGJj#Gm#Q zNozdPLo-Zm$~qYidQ9g}*V~9QPI9nbGuy~BK+{d{756W~!_`kps|PJ|vv&~TzFcf+ zKhnQ{C?B%ec^`(_!Cm!}ef6r~=gMBNyHrD3=4e=t8aqCS`1V-ivfpr&`gFl*C;x50 zHxOl@>@LV}!gcrjbJ=Ow^pyeqb5{Y0FLCF^pAs8(xj#tvzh25Iy$VmzpOO?`iQNB~ zII&DH=|p4q;ViZAuC#1(`SnpxBo{7##!O9GT|EHKgbDWdPnN6 z!chLuzOn~lLbkNSW9$405x;mNC&jp@n5eX6kyoctP681XbP~j}{vxC24A5=O*A60j z{SqBSb7=xdB!Y3Ls7KB^10!wOJ}g&E541S-gcQPT*hPYna4fz7fQ5?rbtNTt1FH7n zppkFCZW~(bOk&X*J$=vs&vFwJxDI2Y7RO@cdaLzEPTLj*Pe5brC0m=f=OlkHnuJE zf#oW)S}BWNyFNznLZvOie0Uc>fUNVJdNfVxR7U(9J)nXB*V$&M=|;sz-qQdpDx@?;AvPpWNHBgmQw%1tQ~aX7&4eJVXjq_gn<6D z>TTq^Dr-NV@ zHX*PJ)D#fpLf(M$&B0qx?W%!UL7tm3j(UM~SOMK%#_igUyIrdDU#c-&s>;*Ge5HJc zLZE~KWng@Bf{(GY z6T`M~WStE&sF655sL&=3nA_u;fQPqx;4rXL4Tx~&e6y8r%iXr41?Q{{>Xxnf0SIfO zKP)*mj{K=5bQH8zbDR}X%krbyu~8p37s`0Yai6f>d6C1B3oWQ?G^Y#PHDw5rBY#Zq zj;Q4J^_{s50X%QInqN58>M{8z{QccIxFdU6Aq6w%Op2zcEuJ!Fj~g}2E%5~HmS api.testium). + from interpreter.utils.paths import testium_path + tp = testium_path() + + if not tp.startswith("/app/"): + _staged_testium_path = tp + return tp + + staged = tempfile.mkdtemp(prefix="testium_host_", dir="/tmp") + # copytree refuses to write into an existing dir unless dirs_exist_ok=True. + # mkdtemp creates the dir, so we copy *into* it. + for entry in os.listdir(tp): + src = os.path.join(tp, entry) + dst = os.path.join(staged, entry) + if os.path.isdir(src): + shutil.copytree(src, dst, symlinks=True) + else: + shutil.copy2(src, dst, follow_symlinks=False) + _staged_testium_path = staged + atexit.register(shutil.rmtree, staged, ignore_errors=True) + return staged + + +_FORWARDED_ENV_KEYS = ( + "HOME", "USER", "LOGNAME", "TMPDIR", + "XDG_RUNTIME_DIR", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", + "DBUS_SESSION_BUS_ADDRESS", "DISPLAY", "WAYLAND_DISPLAY", + "LANG", "LC_ALL", +) + + +def flatpak_host_spawn(interp_bin, cmd_args, host_cwd, extra_env=None): + """Build a flatpak-spawn --host command vector. + + Args: + interp_bin: absolute path to the host interpreter (e.g. /usr/bin/python3). + cmd_args: list of arguments passed to the interpreter. + host_cwd: working directory on the host (must be reachable from host). + extra_env: optional {name: value} of env vars to set on the host side + in addition to the default forwarded set. Values of "" + unset the variable on the host. + + Returns a list suitable for subprocess.Popen. + """ + spawn = ["flatpak-spawn", "--host", f"--directory={host_cwd}"] + forwarded = {} + for key in _FORWARDED_ENV_KEYS: + val = os.environ.get(key) + if val: + forwarded[key] = val + if extra_env: + forwarded.update(extra_env) + for k, v in forwarded.items(): + if v == "": + spawn.append(f"--unset-env={k}") + else: + spawn.append(f"--env={k}={v}") + spawn.append(interp_bin) + spawn.extend(cmd_args) + return spawn + + +def _which_host_flatpak(name): + """Resolve a binary name (or absolute path) on the host via flatpak-spawn. + + We can't probe /run/host/... because (a) only host-os is mounted there, + not arbitrary paths like /scratch, and (b) returning a /run/host path + would be useless — the host-side spawn sees a different filesystem and + needs the host-native path anyway. + """ + if os.path.isabs(name): + cmd = flatpak_host_spawn("/bin/sh", ["-c", f'test -x "{name}"'], + host_cwd="/tmp") + try: + r = subprocess.run(cmd, capture_output=True, timeout=10) + except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + return "" + return name if r.returncode == 0 else "" + cmd = flatpak_host_spawn("/bin/sh", ["-c", f'command -v "{name}"'], + host_cwd="/tmp") + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + return "" + if r.returncode != 0: + return "" + return r.stdout.strip() def _which(name): if tm.OS() == "Windows": return sys_app_path_win(name) if _in_flatpak(): - for d in _FLATPAK_HOST_DIRS: - p = os.path.join(d, name) - if os.path.isfile(p) and os.access(p, os.X_OK): - return p - return "" + return _which_host_flatpak(name) if _in_appimage(): for d in _APPIMAGE_HOST_DIRS: p = os.path.join(d, name) @@ -146,14 +219,33 @@ def _which(name): def _probe_env(): - """Subprocess env for probing host binaries (adds host libs in Flatpak).""" + """Subprocess env for probing host binaries. + + In AppImage we still need to scrub APPDIR-prefixed entries; in Flatpak we + delegate execution to the host via flatpak-spawn so the sandbox env doesn't + matter, but apply_host_libs is a no-op cost. + """ env = os.environ.copy() apply_host_libs(env) return env -def _python_version(path): - cmd = [path, "-c", "import sys; print(sys.version_info[:3])"] +def _run_probe(cmd): + """Run a probe command, dispatching through flatpak-spawn --host in Flatpak. + + Returns (stdout, stderr) as str, or None on failure. + """ + if _in_flatpak(): + spawn = flatpak_host_spawn(cmd[0], cmd[1:], host_cwd="/tmp") + try: + r = subprocess.run( + spawn, capture_output=True, text=True, timeout=10, + ) + except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + return None + if r.returncode != 0: + return None + return r.stdout, r.stderr try: r = subprocess.run( cmd, capture_output=True, text=True, @@ -161,8 +253,15 @@ def _python_version(path): ) except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): return None + return r.stdout, r.stderr + + +def _python_version(path): + out = _run_probe([path, "-c", "import sys; print(sys.version_info[:3])"]) + if out is None: + return None try: - return eval(r.stdout) + return eval(out[0]) except Exception: return None @@ -173,15 +272,11 @@ def _is_python3(path): def _lua_version(path): - try: - r = subprocess.run( - [path, "-v"], capture_output=True, text=True, timeout=10, - env=_probe_env(), - ) - except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + out = _run_probe([path, "-v"]) + if out is None: return None # On Windows the version banner goes to stderr. - line = r.stdout or r.stderr + line = out[0] or out[1] try: major, minor, _patch = line.split(" ")[1].split(".") return (int(major), int(minor)) @@ -225,7 +320,10 @@ def _resolve(name): # Absolute path: accept as-is (user knows exactly what they want). # Bare name: resolve via _which() so the override stays host-only in # Flatpak/AppImage instead of silently picking the bundled interpreter. - if os.path.isabs(override): + # In Flatpak we always defer to _which() so even absolute paths are + # checked from the host's perspective (the sandbox can't see e.g. + # /scratch/... paths that the user may have configured). + if os.path.isabs(override) and not _in_flatpak(): resolved = override if (os.path.isfile(override) and os.access(override, os.X_OK)) else "" else: diff --git a/src/testium/interpreter/utils/lua_process.py b/src/testium/interpreter/utils/lua_process.py index 5eff8f1..ec21eec 100644 --- a/src/testium/interpreter/utils/lua_process.py +++ b/src/testium/interpreter/utils/lua_process.py @@ -47,9 +47,16 @@ class LuaProcessBase: if self._process is not None: raise ETUMRuntimeError("The function subprocess has already been started.") - func_proc_path = os.path.realpath( - os.path.join(subproc_path(), "lua_func") - ) + # In Flatpak the host can't see /app/lib/testium/lua_func, so use a + # staged copy under /tmp (shared between sandbox and host). + if bins._in_flatpak(): + func_proc_path = os.path.join( + bins._get_host_testium_path(), "lua_func" + ) + else: + func_proc_path = os.path.realpath( + os.path.join(subproc_path(), "lua_func") + ) # POpen config CUST_ENV = { @@ -71,7 +78,6 @@ class LuaProcessBase: env[k] = e else: env[k] = e + ";" + env.get(k, "") - bins.apply_host_lua_paths(env) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("localhost", 0)) @@ -79,8 +85,7 @@ class LuaProcessBase: sock.close() # POpen params - params = [ - self._lbin, + cmd_args = [ "main.lua", "--timeout", f"{self._timeout}", @@ -91,14 +96,31 @@ class LuaProcessBase: ] if tm.debug_enabled() and tm.gd("debug_rpc", False): - params.append("--verbose") + cmd_args.append("--verbose") + + if bins._in_flatpak(): + # Run on the host outside the sandbox: avoids glibc ABI mismatches + # between the Flatpak runtime and host shared libraries. + host_env = { + k: env[k] for k in ("LUA_PATH", "LUA_CPATH", "PATH") + if k in env and env[k] + } + params = bins.flatpak_host_spawn( + self._lbin, cmd_args, host_cwd=func_proc_path, + extra_env=host_env, + ) + popen_kwargs = {} + else: + params = [self._lbin, *cmd_args] + popen_kwargs = {"env": env, "cwd": func_proc_path} self._process = subprocess.Popen( - params, env=env, cwd=func_proc_path, + params, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, restore_signals=False, + **popen_kwargs, ) # Route subprocess stdout/stderr (lua require failures, syntax # errors, anything written to fd 1/2 before the in-script diff --git a/src/testium/interpreter/utils/py_process.py b/src/testium/interpreter/utils/py_process.py index 9d7367f..6ce61d6 100644 --- a/src/testium/interpreter/utils/py_process.py +++ b/src/testium/interpreter/utils/py_process.py @@ -61,14 +61,18 @@ class PyProcessBase: if sock is not None: sock.close() - # Add the path of the subprocess (root sources of testium) - tstium_path = os.path.realpath(testium_path()) - func_proc_path = os.path.realpath(subproc_path()) + # In Flatpak the host can't see /app/lib/testium, so use a staged copy + # under /tmp (shared between sandbox and host) for both cwd and as the + # root in PYTHONPATH. Outside Flatpak the original paths are used. + if bins._in_flatpak(): + tstium_path = bins._get_host_testium_path() + func_proc_path = tstium_path + else: + tstium_path = os.path.realpath(testium_path()) + func_proc_path = os.path.realpath(subproc_path()) env["PYTHONPATH"] = tstium_path + os.pathsep + self._ppath + os.pathsep + env.get("PYTHONPATH", "") - params = [ - self._pbin, - # "-m", + cmd_args = [ "py_func", "-p", f"{self._port}", @@ -77,14 +81,31 @@ class PyProcessBase: ] if tm.debug_enabled() and tm.gd("debug_rpc", False): - params.append("-v") + cmd_args.append("-v") + + if bins._in_flatpak(): + # Run on the host outside the sandbox: avoids glibc ABI mismatches + # between the Flatpak runtime and host shared libraries. + host_env = { + k: env[k] for k in ("PYTHONPATH", "PATH") + if k in env and env[k] + } + params = bins.flatpak_host_spawn( + self._pbin, cmd_args, host_cwd=func_proc_path, + extra_env=host_env, + ) + popen_kwargs = {} + else: + params = [self._pbin, *cmd_args] + popen_kwargs = {"env": env, "cwd": func_proc_path} self._process = subprocess.Popen( - params, env=env, cwd=func_proc_path, + params, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, restore_signals=False, + **popen_kwargs, ) # Route subprocess stdout/stderr (early-startup errors, # unhandled exceptions, anything written to fd 1/2 before the diff --git a/test/validation/README.md b/test/validation/README.md index 3e27e06..a5646cb 100644 --- a/test/validation/README.md +++ b/test/validation/README.md @@ -1,34 +1,67 @@ # Validation -This directory contains the testium validation suite. +This directory contains the testium validation suite. A single set of +items (`items/`), fixtures and post-processing (`post_execution.py`) is +re-used across every packaging channel. ## Running the suite ```sh -./test/validation/run.sh # Linux -test\validation\run.bat # Windows +./test/validation/run.sh # default mode = source +./test/validation/run.sh --mode wheel +./test/validation/run.sh --mode pyinstaller +./test/validation/run.sh --mode flatpak +./test/validation/run.sh --mode appimage ``` -The wrapper creates a dedicated Python venv in the system temp dir -(`${TMPDIR:-/tmp}/testium-validation-venv` on Linux, `%TEMP%\testium-validation-venv` -on Windows), using `--system-site-packages` so existing system packages -stay visible. The validation suite is then run with that venv pinned as -`python_bin`. Every test-execution subprocess (inline `<| ... |>` -evaluation, `py_func`, `cycle`, `post_execution`, ...) runs inside the -venv, while testium itself keeps running in the project's own -environment. +On Windows (only `source`, `wheel`, `pyinstaller` are supported): -Pass `clean` as the first argument to recreate the venv from scratch -(useful after a system Python upgrade): +```bat +test\validation\run.bat --mode pyinstaller +``` + +Pass `clean` as the **first** argument to recreate the validation venv +from scratch (useful after a system Python upgrade): ```sh -./test/validation/run.sh clean +./test/validation/run.sh clean --mode flatpak ``` +Any extra arguments after the mode flag are forwarded to testium. + +## Modes + +| Mode | What it launches | Prerequisite | +|---------------|-------------------------------------------------------------|------------------------------------------------------------------| +| `source` | `python3 src/testium` via the project's `run.sh` | none — works straight out of the repo | +| `wheel` | `python -m testium` inside a dedicated wheel venv | `./build_all.sh` produced `dist/testium--py3-none-any.whl` | +| `pyinstaller` | `dist/testium-` (frozen binary) | `./build_all.sh` produced the PyInstaller binary | +| `flatpak` | `flatpak run --command=testium org.testium.Testium` | the Flatpak bundle is installed (`flatpak install --user dist/testium-.flatpak`) | +| `appimage` | `dist/Testium--x86_64.AppImage` | `./build_all.sh` produced the AppImage | + +Each mode writes its results to a distinct report file +(`validation-.sqlite` / `validation--.xml`), so you +can run several modes in a row without clobbering previous reports. + +## How `python_bin` is pinned + +Every test-execution subprocess (inline `<| ... |>` evaluation, +`py_func`, `cycle`, `post_execution`, …) is routed through a dedicated +venv at `${TMPDIR:-/tmp}/testium-validation-venv`. The venv is created +with `--system-site-packages` so existing system packages stay visible, +then `junit-xml` is pip-installed for `post_execution.py`. + +This is a **host** venv. In every mode (including Flatpak) the +test-execution subprocesses end up running on the host — directly for +source/wheel/pyinstaller/appimage, and via `flatpak-spawn --host` for +Flatpak — so the same venv works across modes. The wheel mode +additionally creates a separate `testium-wheel-venv-` to hold the +installed wheel; that one is only used to launch testium itself. + ## What is checked -The `venv` item under `items/venv/` asserts that the venv is actually -being used: +The `venv` item under `items/venv/` asserts that the validation venv is +actually being used: * `python_bin` is set in the global dict. * The eval subprocess (used for `<| ... |>` expressions) has diff --git a/test/validation/run.bat b/test/validation/run.bat index b481b43..f540504 100644 --- a/test/validation/run.bat +++ b/test/validation/run.bat @@ -1,61 +1,131 @@ @echo off -SETLOCAL EnableExtensions +SETLOCAL EnableExtensions EnableDelayedExpansion -REM Runs the testium validation suite with a dedicated Python venv used -REM by every py_func / cycle / inline-eval subprocess. testium itself -REM keeps running in the project's own environment; the validation venv -REM only isolates *test execution*. +REM Runs the testium validation suite against any installable channel of +REM testium on Windows (source, wheel, pyinstaller). REM -REM test\validation\run.bat [clean] [extra testium args] +REM Usage: +REM test\validation\run.bat [clean] [--mode MODE] [extra testium args] REM -REM Requires the project venv to already exist (run the project's -REM run.bat once first, or any other testium install method). +REM clean remove the validation venv before recreating it +REM (must be the first argument; useful after a Python upgrade) +REM +REM --mode MODE which testium build to validate. One of: +REM source (default) project's run.bat (src\testium) +REM wheel dist\testium--py3-none-any.whl +REM pyinstaller dist\testium-.exe (or dist\testium-) +REM +REM Every test-execution subprocess runs in a dedicated host venv under +REM %TEMP%\testium-validation-venv (created with --system-site-packages, +REM then junit-xml is pip-installed for post_execution.py). +REM +REM The report file is suffixed with the mode so consecutive runs in +REM different modes don't overwrite each other. SET "SCRIPT_DIR=%~dp0" -SET "PROJECT_DIR=%SCRIPT_DIR%..\.." -REM Venv in the user temp dir (Windows equivalent of /tmp). -SET "VENV_DIR=%TEMP%\testium-validation-venv" -SET "PROJECT_VENV=%PROJECT_DIR%\test\tmp\testium_venv" +IF "%SCRIPT_DIR:~-1%"=="\" SET "SCRIPT_DIR=%SCRIPT_DIR:~0,-1%" +SET "PROJECT_DIR=%SCRIPT_DIR%\..\.." +SET /P VERSION=<"%PROJECT_DIR%\src\VERSION" +REM ---------- arg parsing ---------------------------------------------------- + +SET "MODE=source" +SET "CLEAN=0" IF /I "%~1"=="clean" ( - rmdir /s /q "%VENV_DIR%" + SET "CLEAN=1" SHIFT ) -REM Locate a host Python. -SET "PYTHON_EXE=python" +SET "EXTRA=" +:PARSE_ARGS +IF "%~1"=="" GOTO ARGS_DONE +IF /I "%~1"=="--mode" ( + SET "MODE=%~2" + SHIFT + SHIFT + GOTO PARSE_ARGS +) +SET "EXTRA=!EXTRA! "%~1"" +SHIFT +GOTO PARSE_ARGS +:ARGS_DONE + +REM ---------- locate host python --------------------------------------------- + +SET "PYTHON_EXE=" py --version >nul 2>&1 IF %ERRORLEVEL% EQU 0 ( SET "PYTHON_EXE=py" - goto :PYTHON_FOUND + GOTO PYTHON_FOUND ) python --version >nul 2>&1 IF %ERRORLEVEL% EQU 0 ( SET "PYTHON_EXE=python" - goto :PYTHON_FOUND + GOTO PYTHON_FOUND ) -echo ERROR : Python could not be found on this system. +echo ERROR: Python could not be found on this system. exit /b 1 - :PYTHON_FOUND +REM ---------- validation venv ------------------------------------------------- + +SET "VENV_DIR=%TEMP%\testium-validation-venv" +IF "%CLEAN%"=="1" IF EXIST "%VENV_DIR%" rmdir /s /q "%VENV_DIR%" + IF NOT EXIST "%VENV_DIR%" ( echo Creating validation venv at %VENV_DIR% %PYTHON_EXE% -m venv --system-site-packages "%VENV_DIR%" - IF %ERRORLEVEL% NEQ 0 ( + IF !ERRORLEVEL! NEQ 0 ( echo ERROR while creating the validation venv. exit /b 1 ) call "%VENV_DIR%\Scripts\pip" install --quiet --upgrade pip call "%VENV_DIR%\Scripts\pip" install --quiet junit-xml ) - SET "VENV_PYTHON=%VENV_DIR%\Scripts\python.exe" -IF NOT EXIST "%PROJECT_VENV%" ( - echo ERROR : project venv not found at %PROJECT_VENV%. Run the project run.bat once first. +REM ---------- shared "tail" forwarded to every launcher ----------------------- +REM Reports are stamped with the mode so successive runs don't clobber each other. + +SET "TAIL=-b -d "python_bin=%VENV_PYTHON%" -d "validation_report_file=validation-%MODE%" -- "%SCRIPT_DIR%\main.tum"%EXTRA%" + +REM ---------- per-mode launcher ---------------------------------------------- + +echo -- validation mode: %MODE% + +IF /I "%MODE%"=="source" GOTO MODE_SOURCE +IF /I "%MODE%"=="wheel" GOTO MODE_WHEEL +IF /I "%MODE%"=="pyinstaller" GOTO MODE_PYI +echo ERROR: unknown --mode '%MODE%'. Expected: source ^| wheel ^| pyinstaller. +exit /b 1 + +:MODE_SOURCE +call "%PROJECT_DIR%\run.bat" %TAIL% +exit /b %ERRORLEVEL% + +:MODE_WHEEL +SET "WHEEL=%PROJECT_DIR%\dist\testium-%VERSION%-py3-none-any.whl" +IF NOT EXIST "%WHEEL%" ( + echo ERROR: wheel not found at %WHEEL% -- run build_all.sh first. exit /b 1 ) +SET "WHEEL_VENV=%TEMP%\testium-wheel-venv-%VERSION%" +IF "%CLEAN%"=="1" IF EXIST "%WHEEL_VENV%" rmdir /s /q "%WHEEL_VENV%" +IF NOT EXIST "%WHEEL_VENV%" ( + echo Creating wheel venv at %WHEEL_VENV% + %PYTHON_EXE% -m venv --system-site-packages "%WHEEL_VENV%" + call "%WHEEL_VENV%\Scripts\pip" install --quiet --upgrade pip + call "%WHEEL_VENV%\Scripts\pip" install --quiet "%WHEEL%" +) +"%WHEEL_VENV%\Scripts\python.exe" -m testium %TAIL% +exit /b %ERRORLEVEL% -call "%PROJECT_VENV%\Scripts\activate" -python "%PROJECT_DIR%\src\testium" -b -d "python_bin=%VENV_PYTHON%" -- "%SCRIPT_DIR%main.tum" %* +:MODE_PYI +SET "PYI_BIN=%PROJECT_DIR%\dist\testium-%VERSION%.exe" +IF NOT EXIST "%PYI_BIN%" SET "PYI_BIN=%PROJECT_DIR%\dist\testium-%VERSION%" +IF NOT EXIST "%PYI_BIN%" ( + echo ERROR: PyInstaller binary not found in %PROJECT_DIR%\dist -- run build_all.sh first. + exit /b 1 +) +"%PYI_BIN%" %TAIL% +exit /b %ERRORLEVEL% diff --git a/test/validation/run.sh b/test/validation/run.sh index c86bc02..5a1b083 100755 --- a/test/validation/run.sh +++ b/test/validation/run.sh @@ -1,47 +1,143 @@ #!/bin/bash -# Runs the testium validation suite with a dedicated Python venv used by -# every py_func / cycle / inline-eval subprocess (i.e. everything that -# goes through ``bins.python_bin()``). testium itself keeps running in -# the project's own environment — the validation venv only isolates -# *test execution*. +# Runs the testium validation suite against any installable channel of +# testium (source, wheel, pyinstaller, flatpak, appimage). # -# ./test/validation/run.sh [clean] [extra testium args] +# Usage: +# ./test/validation/run.sh [clean] [--mode MODE] [extra testium args] # -# ``clean`` (optional, must be the first arg) removes the venv before -# recreating it; this is the way to refresh the venv after a system -# Python upgrade. +# clean remove the validation venv before recreating it +# (must be the first argument; useful after a Python upgrade) +# +# --mode MODE which testium build to validate. One of: +# source (default) src/testium via project run.sh +# wheel dist/testium--py3-none-any.whl +# pyinstaller dist/testium- +# flatpak installed org.testium.Testium +# appimage dist/Testium--*.AppImage +# +# Every test-execution subprocess (inline <| ... |>, py_func, cycle, +# post_execution, ...) runs in a dedicated host venv under +# /tmp/testium-validation-venv. That venv is shared across modes — +# even Flatpak reaches it via flatpak-spawn --host. The validation venv +# is created with --system-site-packages so existing system packages +# (PySide6, lxml, ...) stay visible, then junit-xml is pip-installed +# for post_execution.py. +# +# The report file is suffixed with the mode (e.g. validation-flatpak.sqlite) +# so consecutive runs in different modes don't overwrite each other. set -e SCRIPT_PATH="$(readlink -f "$0")" SCRIPT_DIR="$(realpath "$(dirname "$SCRIPT_PATH")")" PROJECT_DIR="$(realpath "$SCRIPT_DIR/../..")" -# Venv lives in the system temp dir so it stays out of the project tree -# (and is naturally cleaned up by tmpfiles/reboot on most distros). -VENV_DIR="${TMPDIR:-/tmp}/testium-validation-venv" +VERSION="$(cat "$PROJECT_DIR/src/VERSION")" + +# ---------- arg parsing ------------------------------------------------------- + +MODE=source if [ "${1:-}" = "clean" ]; then - rm -rf "$VENV_DIR" + CLEAN=1 shift +else + CLEAN=0 +fi + +EXTRA=() +while [ $# -gt 0 ]; do + case "$1" in + --mode) + MODE="$2" + shift 2 + ;; + --mode=*) + MODE="${1#--mode=}" + shift + ;; + *) + EXTRA+=("$1") + shift + ;; + esac +done + +# ---------- validation venv --------------------------------------------------- + +VENV_DIR="${TMPDIR:-/tmp}/testium-validation-venv" +if [ "$CLEAN" -eq 1 ]; then + rm -rf "$VENV_DIR" fi if [ ! -d "$VENV_DIR" ]; then echo "Creating validation venv at $VENV_DIR" - # --system-site-packages so we don't have to reinstall pyside6, lxml - # & friends just to support the validation helpers. We still pip - # install junit-xml below because it is the one dep that does *not* - # ship as a system package on most distros and is required by - # post_execution.py. python3 -m venv --system-site-packages "$VENV_DIR" "$VENV_DIR/bin/pip" install --quiet --upgrade pip "$VENV_DIR/bin/pip" install --quiet junit-xml fi - VENV_PYTHON="$VENV_DIR/bin/python3" -# Delegate to the project's run.sh so testium itself still runs in the -# project venv (with pyside6, gitpython, ...). ``-d python_bin=...`` -# pins every test-execution subprocess to the validation venv. -exec "$PROJECT_DIR/run.sh" -b \ +# ---------- per-mode launcher ------------------------------------------------- + +case "$MODE" in + source) + CMD=("$PROJECT_DIR/run.sh") + ;; + wheel) + WHEEL="$PROJECT_DIR/dist/testium-${VERSION}-py3-none-any.whl" + if [ ! -f "$WHEEL" ]; then + echo "ERROR: wheel not found at $WHEEL — run ./build_all.sh first." >&2 + exit 1 + fi + WHEEL_VENV="${TMPDIR:-/tmp}/testium-wheel-venv-${VERSION}" + if [ "$CLEAN" -eq 1 ]; then + rm -rf "$WHEEL_VENV" + fi + if [ ! -d "$WHEEL_VENV" ]; then + echo "Creating wheel venv at $WHEEL_VENV" + python3 -m venv --system-site-packages "$WHEEL_VENV" + "$WHEEL_VENV/bin/pip" install --quiet --upgrade pip + "$WHEEL_VENV/bin/pip" install --quiet "$WHEEL" + fi + CMD=("$WHEEL_VENV/bin/python" -m testium) + ;; + pyinstaller) + PYI_BIN="$PROJECT_DIR/dist/testium-${VERSION}" + if [ ! -x "$PYI_BIN" ]; then + echo "ERROR: PyInstaller binary not found at $PYI_BIN — run ./build_all.sh first." >&2 + exit 1 + fi + CMD=("$PYI_BIN") + ;; + flatpak) + if ! flatpak info --user org.testium.Testium &>/dev/null \ + && ! flatpak info --system org.testium.Testium &>/dev/null; then + echo "ERROR: org.testium.Testium is not installed." >&2 + echo " flatpak install --user $PROJECT_DIR/dist/testium-${VERSION}.flatpak" >&2 + exit 1 + fi + CMD=(flatpak run --command=testium org.testium.Testium) + ;; + appimage) + APPIMAGE=$(ls -1t "$PROJECT_DIR/dist"/Testium-"${VERSION}"-*.AppImage 2>/dev/null | head -1) + if [ -z "$APPIMAGE" ] || [ ! -x "$APPIMAGE" ]; then + echo "ERROR: no AppImage for version $VERSION under $PROJECT_DIR/dist — run ./build_all.sh first." >&2 + exit 1 + fi + CMD=("$APPIMAGE") + ;; + *) + echo "ERROR: unknown --mode '$MODE'. Expected: source|wheel|pyinstaller|flatpak|appimage." >&2 + exit 1 + ;; +esac + +# ---------- launch ------------------------------------------------------------ + +echo "-- validation mode: $MODE" +echo "-- launch: ${CMD[*]}" + +exec "${CMD[@]}" -b \ -d "python_bin=$VENV_PYTHON" \ - -- "$SCRIPT_DIR/main.tum" "$@" + -d "validation_report_file=validation-$MODE" \ + -- "$SCRIPT_DIR/main.tum" "${EXTRA[@]}"