diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a86f4c..c3fc571 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,4 +24,7 @@ jobs: - run: pnpm lint - run: pnpm typecheck - run: pnpm test + # Build the artifact npm actually ships and confirm the compiled bin runs; + # otherwise a build-only breakage merges green and only surfaces at release (F-37). + - run: pnpm build && node dist/cli.js --version - run: pnpm audit --audit-level=high --prod diff --git a/.gitignore b/.gitignore index 73010e9..46994ff 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ node_modules/ .DS_Store dist/ coverage/ -examples/deepseek-support-agent/benchmark-results.md # doctier:begin — managed by 'doctier init', do not edit **/_prototype-* diff --git a/.harness/qa/adversarial-audit-2026-07-03.md b/.harness/qa/adversarial-audit-2026-07-03.md new file mode 100644 index 0000000..1b3d2d2 --- /dev/null +++ b/.harness/qa/adversarial-audit-2026-07-03.md @@ -0,0 +1,680 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IExRMGNCZyBRQ0do +OCtJaXZKR29KQ2czeUNMNXdiekJxUmxXTnl6TXVhZFl4dGJYL0JJCjl2MmdKc3F4 +S0FoU0FIRGkxdENpV1lwRlRiaUtWOHRMUmZmcGgyK3JTdkkKLS0tIEpLWE9aekVD +cHMxWnNlalg0RStRQlZWTGFKakVvRU8yYUxsTmxFZW5FSGMKy0v7sWbraUaxCku/ +wNLmxRBo4+6UV3iLldiK8+8AQYKeUKa6CTUePKLHedcEd5VbKOKT5Qg47GIanF7R +dCAjRTkX8qKOB1MVRm3JVxDLl0WM1txRvwy9gWiZ0qmlunwFg7+YWxB6pKX45VbT +fHgkr2mqswc1P9FGud0Mh6409z/wrFJq89gEsCyBS4QGZ33CrExzinNHq6f52lBT +n4WuWB/rGQ3dcMX5IhCwrQJLNYm0kghLxNTEqGft4CiKRdzOUhlG6JZ2ppCpobVl +kBx0v8MahIdhVa/2TH4+ewSkuocBZpgzkdvQSgXQ0wQlBcYXg/jpUPvMssbP2WBW +BPCny+yEvcTZHfN82CKbgVXoUwJg/W0SKhF6fVDzYivRuqXuL3ht50NhBAox4dg3 +9Lbk2lb6fso2P5HBcYXnz+7Rwh5mDm51f7FLORwQujU4WV+3CAgqWKZC3EosTnXY +3jPY31LF/uPmWE1+0bw3QmBbSpoAg3MTO6t/sd9Bu5W4bdUKD3tpiKNXgcA0bSvI +027ika5PYxNlZvyPvrFs8BlDLyO9d+R9695St9UIQZ6yNh512bPC7kfrNyZrZxX9 +6AsUHMqBx9a2eZiWFP2kpHEc+klNH9zQX6YJHAmv+3xKLdM7eAIqAsVfY2rXY35b +0QWPhZzv0eqbXRv0IubiPU0CiEoT7wz/XL7GD7G0b0moO4YvMGAPY7xPL9snSvRc +vvDCuCoJVPtcCPdB7YEXYxOCMPp/c73ed9lTD9yXUnves4dbKFncLQhu18Ohq2BU +oIbssX1fEc02E75v33FkSXpb6ERrFb22wiKapmtNIO+iNVq1ii/TgR45kBt6uD2c +GfBoCgVJS0PoDLqHcvzA/ZkrFlvoncv6CaDWzgYmTA9uyFYN8QqKXWfRfdbQfmJQ ++5CUAukeRVrWvBQDY3SvFELjsvcJ1u5kt/Yf18K8NmJ836aXGALN6zHg+Gdt24yx +B2+b3lYjQS/zVMckoD7JrU27cCsdCsT9TTlDruEeQ5uUMd/7FAvX8ShzNGSQFBFs +4+QxwfN9Te92V107wgpmWYP+ekrlc/J2BezXWiQcarhQnN8GVJEqH69T+umM1lYJ +ZY/UpQ+Svqct1eQDOtEwUck0Fc3swB9gnx+/5pzsL1/0yCAKv9PaQquqiL/OYwvi +h2v2WJnNwpkJ9WWrvz0e2tdk0N9MEb2ixcpdeUSDH4gNzSQIBo6HEwFnJ7B/glX7 +Y+fShG/FC77BtxuzXQzuVMh5k53vsJnVO/7wPoK3B6KmTexdyOBdGojg5OUBzgvY +jkRWODg2QjCa14r9HiaHPCrXimVvaJXz8n8dML1ZRgxbIEucy23UUdZNJdyrx/Qs +zTC7hGzIO3UQXULqzGkgArPh8X67GDPa9FuTXZ+LQaA891JITZ2cQxZGXGEgaLgu +dEqNysSor5OZTND9I9GhjHqxlXNPJ1WNKRAi2+GF6yeaWB1II+A5tNLmImYHp3G+ +DvHh7Z53w4mZh2PKGC/rw0G9Rjf5y852JRF0MqH32vvFe1tVEoNgcdxbH2Ew6LPg +fW6FlxxHMRWqBSvuCkdN+06LKv1sTgPz20+0FmiU6QOeN8OXun/J2rewcVrZ/RP8 +uW6jR8mO9DFlyPcY5uM9pjbbSfak4mwk1xU0WfNo/kkaq54ub1gfd4LHtCv68PJH +O/r9GlA7LnZt2O6uQLhXLG6LVS7HUBfSF97bqphH1t+DaLDtZBiLuW/vT98dPH9B +towZM2MzPBd3CxgQN1CduAZiUWu0h4e48hIe316LGsh5UjG+h6CJkT8NMRLCwdPz +I1KYd/hnstV2OdVPO92WOehDFbMIkABnwseL77Uj80aWsIErSShuNdQmB20OUh0C +lgQjSrB/xpzfboIV+WP+FsfGnj3Y6ReM469pfO1zI6X6+YOSjGXbI9WQYOTPRbum +PQozWXH0zRxQMEOqYgYLifSnJyEVmHw0LNfvOGYjC2L60y8Iu/51bKBCeALAIYAC +f3PSxbbbLVvIpIsPeVqdmLjBli8kjqWZTghPqKGj/bmtviSOPUylTwq/rp3I1wDJ +MDrY2RkEPGGMoQk/MdCeeuTMOhQgGbS95jTvD5+g/DTeuyScUfCFrNAIBZycGwVJ +2PW3xR/zetuYKwXNZq/c9w5P5/mQvFPtdtY1n/2F9cTkGbecm7te+2cHgRctGhHq +CS+coAgRZAQ8yA/BpX3v99xu+4EwFTNvAcouVT6hBEtJv4KS1Y0kVyj3dSyqZHWL +HuF4BxM8hoOzTaIyvFzbfsHmufYF6PhPUY4XQ1VSD5caqnWgqRp+kofHWXvKMhHO +WMpw4bHwoYyWrDT07bA6ieMoB3U4yBoJKhAAovhn1VKJ7K2I/5aQnQVNTEWkbmSR +r0MoeyNwWyQZGVBBY9+dWqAEmoYu8uR9p+Rtm1YGv3Uv/EtFyW/TQFAvkfaa81Py +Lq5Rw7uqJilo9vepJjaY9OqqAZXGOrlvEo+w6ZMfrjzy6Q+dIiT1R0LvPu4viNoi +953fyj0FrWX6CCXu1femEGubsVzjDdvMgadLKbg1bDAjLtFljuY6ubRHP3ratXw4 +LYVMCCt/2Jou/f0hgF3GNHrvsidjWmsegShg8/FmSGtPkG3LGcALCA1VdPdgt9eq +kMdmYz2u4H2FYNX54appaV1Z1xwg6g15pwzYZFRdnaIh+qmZbwGwpcePz1kh0eOT +Dblm0Ebw6g/Qf5dLL/FLz86X0fmzsnhPTvvwFpthzd8IPcB0kxVbY/DzIoTkDC5v +ssaySxZVNt2uvRxW/aANIQnqGcnYSFbKRJEMfP1MLshlvuuDyFVPkdflcObMUIBm +odaDOAIYteX4r690UZNWBUhZPoI8oSIemGiw/BJf0Lw5/P+PI5uRgqzmv71qlGx3 +XmA42SH3yPLdolNwC/wwbfXiVVLb2JNNvLM1VFYPFHLxnFwPA4j13dpijK7VJjEY +eKDAeGN5PJcvVPrejmXe4R1/5AreEr1J+G9qBNECSjz8FNfOPV8V+bv89tDLMoxJ +3PswOUNUVDa3Zh+4htE9wUWugc8VLLEGR/I5MPw9x8WNniO4Yco2SuU+SAIdk4sU +q9nfJrvigvtTfAHszCJaNsTdm9WiV/n1G/S68AHJ7gFPT8fax/DhhVkXRmqI0BpM +ImnDU0HTp2flX9L2roNLL+FuvvOGGYWojrV/bvd1Qq3Ugcp6FrT2OwIKBnpWxT+F +NGtnyLK4JyTjkQLkRWWby/++mPAuA1EQqSXoGp1NoDmwMZy2m0XZHDz1ab0RDxxT +mwF9lCP2rR3Bin6Lq+6IEiRBQDZgiry4p7320iHcUV4lSJDJxt+B1H8oBxCvhyy3 +awwKttiu/e/4mNq5qFIj/JyzBKvfg24xQ+6+qHa4odPPXdhxAdEdTK6sAoX0eQWp +Nj4vrLNMnBYFfImyzC/D2LxG+rV+izx5HKwZ8UNCEZVVDOpz4G27I52Dp6w/92rN +BRaUcAOBevu7Z0STeuaY3mulcRJSru8mFQ1n1XVtX7LaKrN3biJyKMkdlQJkZzOr +YDDo8Fcw01flFXZ7zsrQ2hKXe3UKRzqFDQG4QMMMa6hle0OTCx115wWxMBjO1gF3 +JJQxm+voeLeaSuhp16Cmt7Bt5+436mPtXxbvoo7XBXK2JJXnTEVqZx1+gAQuD3Ys +75l6vCNHlEWdYdLQXodRMfdQziYjPkogGBXx0NglEpdPohmjXcZlarG5pOfmoMF7 +Q5L1Jf6o6XvQ0ZbDryUIogJKHDGZWGZPHaC3+7rkP3ReAifBmU8a2W4AWh4djOM2 +WOmMy066gBZtdpEmBVR63SM5kQV4pI8MB4BuEAOpKolmmrRZIKyd6UbMUwII06gm +kB2Szb92piMHSSaQNQyi+3gj42uHupltajSGKqrmKNluHOqJjN2pErK7xa1kQVH0 +hs2IKZ8bT16nBzBjb4P0+vASJXW2ytkC2w7y8kboopHwoCM7AmHgpUtiuVSLt1AV +0pUBc3K0xQQLFN98bzKv6mN4bUk9muMNNHByKgHS5VEXnUPf0epiX7lbed60SWaC +HE4dzLLhB2yl35J9soQ4xArcvej6gbuutZwAVvKsDljVs6mYWlAj/NqlCeQDwl8i +FdMz7QchUplQOVwAcLeQaClTrAZeisJ0itR8ZfqkJjPAkdiKogxXsxWikMJoS6dS +78O4hEHbzXjU+oaTAgqnXnlVIVxy+u0E7p2cnL38NAgEJ9Ta8C35jBI8YnpgmGaK +X3KfqJlF8+H51EBY6ZPJ8SJqSnxnkJ35WlfL/XkmrLTXihI6wDBVR8JhSek0mInZ +m0VE3BBMHNj4t7E0+m7m9TQOkPaZ5/ZJH6gNcZdc2icfUSkeAS6kzNqNeJzcRpCL +T+pDzT8hc+Cbfzz1fFmER3VCkC1FmFI+wbisuI44XnENjGmVt9t07cXUsiP51Gqu +hCuew+Jzs2UC7Kt8mSfjMf2qb4mNqcCpZqaqbuSuntsJXtGv8Wugzv9H+XLnAnGJ +7wiErfaCt0+g/8CUsu+hWHSDj2TGuf1zPNDu1/MzaJwmULJ0ZFbFlNxGcxC97i6x +pG0zmo+Qg6u3uXOjCrdGGVvdKOK0zdx/pyYSNop+qGRbghMJsX3uELKgoejm3NYk +kEdMkxnSERKgv0h6tsQ3zkYZ993MctQdQ3OkFm7E+PgrzvL7XZYUsADT4UF/EDNl +JkCAu2bc2aXgNQSue9f0pytjTJdu2koqTxElERDfdOBzond9i+tM86sjhlVDi1am +GjUmE7jZgMqDwNniQe0TqBvndPeNPpiwNr8Ey46VOMkmq7pEo3NSzcKER+/Fbu8v +LvE+hUXSJs8WbwTj1l65aoK46SBBjXAPzlSXhOJW826TeR846CYxtrM8Xs8xSgBe +fVnLHz02NoW865iW4M0O1ZWxtAxcEW/o3yuW6cOXZahzWbBwyOD7wu51Fcu2JHVt +p3AnYRc4TIjSgBsGxmR/jClWH36WmU1rSP3SCkMMMXy9sAPrGe/uT/LOUXm18kSn +MODfqoRG+0k8nNHbfjqcXU4qi3/WVW5z/A+ltRUcSUDPwiomEtIRlyuc/Zucgoz4 +IFrJJHly9ZcSF0FxsN8Qy6Y71ihmvchPyEm4VE/49BD2l6L7SnrwNb9JquGXx1jP +Ozn8IwYT2VO9q6tfwaPRhvODc2jIii+6r5DkuJvrlh7BUwIbDlwIxR6L+xM2AA38 +P+Qi9sorCBP51EQF4R/Jy9qY8/QMC4kbWntR51cqBAwuWAONcIty+whgtpxRHzIT +yBlmH3ooSDPXmG6Hlv2eHSnH912Ej8BhSraskOSgRx2iHuhBZ2KTSNHu86rRCRBB +WhnaGTMJZIH9cEeNy85E6LZfeoUUXKwFI7zdQHpzv9F554VagILIB47TDGIYMVQm +GWyUHWnog2lH3UbeuMWvVkgbBiR6/lIGnUkDnIIoo33lJHoS1xk2wQNrb80uzSvI +PuqkyFW904vze7YWZ+60afoI718U+Vtj9QZa3+kaL/Xc3l+Tbv37zUZbE79yX9dK +Ua72QXQ0EyXIfb0Gdgfs8bNyp7eS4nzT1XnUiy519Mq/rSybEqZzwCTe4h0uJ1Hh +B/SiRRFWF7Pm1UT7PO99Io8HWgsDOMryLUArOer1cMPslOlN6aMLQIfyaxXGmT0a +lLO2FuuomMzeUOBC+E4lNmGCYWLQKbF6+I2AuJussn+Y4u4HHtPiLWcX/NTeCNWd +1cVT8E8O/OOn3ruhmWj+6RTNP79lg8d+7EbGI2jyb0ZXX03cpNbCYVtg9TBQq2RI +xLTeOQoeVsNtkhaAOT999ZPFRjuiJxRHRnT0a5rieU18shfkPvUDnPRAsw86QiA1 +QsUd48qB6SaqDSAdBq1IAVy7P5OtGVc32ZNAUt6/nSV66lhT+N5y6QmrIjiysYZ4 +rNb5u/+1RDc+p9qiIKshkRWm8Jj2ldecwl4VIcSZXI5pkvXpGAF3LSBGstCI24lF +QcRAIzCjezydkTLlOUsad08H7+Gd0rsK92v1VHwdzjblvHxHAjqF7fbmE/2FgZ3b +Q69M6Kzuz40d5wanqcjby3YSZIdTUacukkLpQdVIe4co+Lc5xgu2mJnEwYYbm/lj +6GjicS+9XqTE4HIMoFgD8gPQGP16LbLnBFrWMPHnjCFhf0zmy3/YUwaYF1joFCpv +cnq/SS6YoxgzNkxX7Q8zk6xDIvdJJz0g03WlUrodGz+/I1FpZf7OhUTlgHbP8/2W +eDBYQRvZZDM5AbXfJfsb8pb+jOgIVS3tpJ9DvDm3xcQk4kTpIe/LHuD1a8U1s40T +XBkZRGpoFWMwNzP6YMvpFOu6JfpLBSTHAYC5ZZcMX4irvWf3GiMcq5kg41cEK5xS +Gx9JYjdcJSYVrrFy3R6zOSTNbQ4WVdPABA4NWHBFBnvZsOdZZthyc4+cb0d2unTT +f/XsJvsHbxZ0iXoIfLGxqsai/+TKi7LF1hLiEMQoxg3um5QlC+EAM6S9IhgPJ3cy +Xzhpj/4xJz+2LwzxZdMn0lGfwwJqErOuSWk8NlDP+aLkJgcsDIH8kQJ1ZjWuxxVC +b1f/omQ8D4Caw0pS/gQkqrEx0qqy1S99lxP91B8nfgXYuQiHowlEEUYJVOIu2FVv +ITVwGvksoRulS4XNnw0uCm2b0XYIDD+GxZ7MxzsQryfushWcY+3IqEebAdwAALq7 +LE8XqnTAj1u0O5Q6PAuLG0ucs9JbERSlyfTSaCGrFXFMtOWI07kE1oaXBtVVp+AQ +TMicw2S46tUUjiHdeyDN5xl2/uYeTuNNWDOfC7clZrR0WC4PesJNNZ/hU2GGGLNS +66OU6Xcl/O+3AmmVA42YTaz+k80yFqj/FUTvzb5Jpt0L0yhLTJQdwcIkDftzypWF +NEmNY0GP3uJ81JQq15jgAa13YXDYg8+5q6djtMkxqFS/BNONCVOyoNbUMHfPKnKq +2HnYdW6EynXpmdil03Usd7UK2haVLs++Tezzr/eAxXoBN+eW9x13IYlHKpUYobVg +LfA8iarfibbBsFDV4ZGjHaNW6HflqpKd8vPhl6YloZbXVT3RSNeHGz5vdmGbZpdz +N0XPRDLv1SvZrEEDjga0B+iFkZM+bBflGXfd2LDZLyYvmhz6zxWADbqkRCYx3XB1 +J4Xzkd2X10f6uEU9uLfouDlwv5sAvngBXZU/ae4oqMRZqi9gDGj6NAV3igs4reSV +zZzUZenj+1OMXBrNmuJfb20R6IlReRqrp0hAY/vmQeVhT01u1msHGL1HSkChduMk +Q04TPvSbHHuxoh8RIEt2vZ6bamWiRXdOunquEt4bgW9a5LdTjodwGBfiN+aYoy23 +3FcVPkKV/dH7uCHa0qifzwt3l4vLcrD4dHgW4jldW3JmsCdoDgv1HQ3M8AoyF30N +ClgYXn6/WxQ7ndZ8t192kWnHebLNEvQJaAIpPT15sKWnvQhEy7urN45AxFTD8TUL +lI7powiAJUiZumPo/ilTIAtePIp+pN6JI7Qic4LTCSgw3u4i2VK43mZZNVG9Rt4h +OjGnw7lGvvKHbY6GFCF6YNVy7lNBqFWTcUUTPqAZH81ngdW7c2crPmVO66DqMcYV +5YBDkNOUXYTQvHhDwZg4h3DHmzxjV+YBO8oeJNB5unSpsXR3eD/ev/hzYOwoYaV7 +764KwUYg9HlrGudNW2iQrBHSttpLN6wwSf+x9HPcC/cu8USXTvqhYAkhbVoK+Kjn +JVsEoaP3jrxnF5cApFAqwOoikCkjiwSw4hYY5RUQ5/iOFOqHjsig2c7vX+b9b+ie +gXQGi7VQNSpjnJOHbrp5YtrNr8mQd8jnyrAcLSpSRd3tuzYvbPMyaSxCYi4YHhcz +96VxHIe3mBemFswupxkH18aOEjIXSZHCTK7H5BbI9JdslRlnBuJsolTtHgEauO1O +3Q6hBpMfeNExtZaqQUj9E5hAzFfZEc2/a9IOAYcnsrw2NG0MrzUbxphfTm803yTE +WiNruz27mhYUMy0AvHEqgMd5gHRL3Y/3EvyYQl3u7GWt/exJntDTpc6Lnw5SQgWK +xaKhgmL5WT4Raboiqmz6pLTvCUVtJrMekmK1ylmApfEiGmqrsAfTFjuLlUiKgHPE +rgKsnVF5vB2yyTFD9eps3CJSEm5ABP1Pxmn7ghRjwNwM2kbm/ljrQD6t5dx8rTak +YhTk59h3ugXPDK3EvN40Q4pKZJzENhtI/sMtV8+qpEek4sjYos6vvLu2eAn7phRq +JEjtnIC8Ri6FpG9IXdd2pa4jlmHYbf//bArxQ2261tDISLLUlidc2tpzmLIeY+mF +OFoo0YnD/eRmn7onnpjLy1jxPqTdCe8y9QdFFkT8YzapWuV6yOLReFE30lutHHej +n7JC0CPqczEsF5avBrokKtkSNkVnd/Ntr7fsZAecMYpzrQeUAdAwnXabGhl2CpkN +2amswvX84AF9NbmREUCf16SRo3GeoFfR+E8E21Qk5GK9L+YD+Ze5qh+HtXXHwAz2 +SlZe1h0H7nVId6WeBSwOmQlz3+HaQhA4E2KY3MxiCr/bQl38dUe2TSCYy6TegT0v +4K4WWN/BlNQo7jPma4Qpz3CMJSXLOslElHTrotWTbpejjDUvrJ/hGlwkZYJbYn/q +UzAFNw52pljdMSHDZ78JAMyFEmV6Nmbcg14ekOcS2+HAOZu/GMgRqU1CPygw/VwK +ODfl2eEDWDQ83pIRqtyc2GKeZ6e2rCR++HisVTFZl6v4fXhuJKBkYWvciYjxQVin +2s81Dg/k/nj/mUQN249em45d9taGEvuegRrbrJkNihhOs5+QJrfHDFwU3xlkZMMN +mz0qKeDDRKaklpWL+iKG2c/LQXPtGgaA+nESxZ9nSXCdMxvzpvHu73HEXt/jaVP0 +Gm5DoPS5ResMuZs5aQ3533LTc6pGjRBYo1uXX1gvMQ9NloqLYT5K0/+laX7DFmpw +BzEK1o09/kMbrNqI+SpKmb7FRmoYqT6rmf2f7t3dSbAkwWyaZOyKvCmXjH+9RLVd +O2IkFl9PuZ0NFTfUyEPtetpvpP5tXo2C6Fi4jJdHuDQBQWfKj2C1fsSP9Rk5nDnT +PZIOf1qVaxNBHrDp37+olvqGt2U53RDf2mzgqKEdk5LJ0CwO8E5ozuF2KPjoWxi+ +H++hq1LmJnMfe4CCC+thkJGBqAqBYvgpk8BexzML+NSrETP4QbQRMq8iq86nsKps +i5WKxvFZXtt/B7NT4fw+UY59FRfZDRAGECtAQ4qT0FoH981Qig5BwhEiAKBCLRV6 +WwzYKYizWN2YUK2Dmd//Dh4t2YtuLeGFGFQVvGUV9vJZ3v+vTbeW1UOAmph6kk0O +XjBueXbPXaxHdF2WYSRj6bNW1VKQgLjuJABv8kIeyHs9LObmXHh3m3QbGE0psT/v +D29x+Zg7nYLZqYhj6KKqntiRRd73AYNrKzqkb43iQM4hBx9Ogl7umjzVp7i1x/bH +rNc7l7Kmm1VJMk5z/jBJaIq3Lw9qsuDUgZ9BBasqmCd1XWKMoXXVKyWL99zZA5on +UugXgeICe8CtX32Ny0r+Vu2ZRscDwiXL9YIQYYVaP3OH0mno2NyuPFd5F9MC7zuK +5OdqJ0/sqsPzp/oKN+RRDXcpw+yivY0jaB050KcZxrbcskRE4yg0XkRHrsIYDAfK +y1rv1gzSWcJNub3YFKPXBf1T0phW/yIzxc9gGOnHePnrh9aLoQmz6TNDbNfVAl40 +Gx2c0Xb9WxPGnQJXLMhTnEGWFaIczmDf+nhL0MjRCxjm6NWNyHeNlFj7PvPvcU5B +Dor9uO3MFBGGrDaONPU7efPTntShenxyjOSBfT3mUP/aGDfro0XzCu1RDjgpmuyf +JrestoW1qDotpsUhMtFx3ZtCYez1gh1iKy4ZafWik/bfxo+EjzSa8p+Xi/ue/2uq +4WfMqmNus+Jc6lpQG+jyyuS+nHYf0p0QSsd/Wvm4ljD7E8yoPRLQ041vj7X1SXFt +ABlzT8vQwAiGUCAFj6MZLCyQhsDjC29AuIOZqx308v/O2xvdhjavxJ0Cdn9CtyTo +wq74KhBuoEZwNWWbdXMYVSwtreC3cnraCPU03XMzYGinwEfLfnIlTz55oalAPXDC +XArj4IHum59Yrjb0iJy/75p1jI8Q+B5mFEN4CWGQi/K7FAFB6f7jWvXcyIgl5XS8 +SCxzOc7Xjw+pCkG37+FL6qTFaiofo8RRlnedQvkm53TFxRxrepEZnZWLY39wPSjF +dqU3IoveLD9jIvCnipp3WQKfD1S5Gb0dIikOsNTirnCp/dx0QGJUoWKWDzPZKqQJ +h3aZzUwk6pqk2i/YKanLu0jmtpYmC1HQEDgFshB6s97CRplURJfJQI1TJjl2EHGM +WFPoeohIA/KiIzAo0drLzlFbdO3hZjLCUi4e6VT+Ex3+NY84qY3XkvpBJGoa32jq +Vx4NmSXj91DX2dOVJqG6uO+MwpRIwKmJV9qt8hpKKKMzS0ILinfczgiDXMUf2RSU +00WNlH0F/DTU8GThwASEd2hxbxGaN6DHRF1yzmqTkkfn5mfiU4YJgsqttkj5mbfG +su9O52znaPx6341aWaGD4UOi0+jddI8OsV8zRqn2U6eIPRMFHIvJg22GSe/g+tSN +VLYsqqGyOndWihHBo2Nk4DYvRE/8AQws6AMcBMJBnS//Cl7s6UJc6tLFPML5GLVB +bLoo4cUYEqbMpZvLiOz6AZX0hP7C1ljBQTDd8u000AdMiCDTn6ifNKMMU8Zp6HpG +UitDlqtvYbIcrSyVziNp4wo5Rz2ldjVML+nXnhU7wF1BAqa5a1onGpf80LSVbkVE +Ro/tRQ1K/q+bc5MJIltBCN9e0e60A7Z5bCm8U8leJ09TeckNOx6fpN7UIPVggtSf +X6h/B2tcA2rKhhchEoxHRbKjVGGK4F6gHGIjcoNyx9u3+3YYxZdB6BwZfmE9Hrd1 +w9hCjy01QElq4+V7OsD9HCU/chYn0NMFZCFucm9BvNJ2PRquRjggf16GW6twJvG0 +Y4lRlt9PFwfAdj9DaT1/BxBx4bodLt2ygYq57BLJagaRzyiwRoExll7//EWis/ga +kUtKhn+sHfozkoE1vxBDyWV3An0yv+Uo4nfXVfA7fh+u7s4TufNJtMR82p0SONPR +MI5GggJw3EyvpwClV1BadhwhA7/lWEtauPxZ7YA0gT7/cvF2nTQQi4dTD+/WVSmK +wmX6hUJWsmIihZC5WaKVdxganBcksthMmtTxcaE54hF+RxteaQ7IvE7ysB+8lS0G +GcSumy8MYijY0S9kFgC5H6wbwL5p22VBaJCD3ah+X1StTPc2Z+GgShH2YMk08VRu +iqxjg2IOWq3DBEfy5PWXMqgJn4jPF9eGNoUl7zbcbxefzGnQNFyXVqncet9naFgM +7eZXcbxlEBbUMFNvlHF8X/DUYJ7tu95HRqUlYazgphMcM6nJwEIe1gTW3jbUwpe3 +F6aW64hkWdUyEROufaszHPjdfSEMzwbc6BIAJAOobV3gjOnD2zAe6deU02eft9ZO ++fUN676GQ64RnuwAiZzL9YPdzQZH2r+arUHfoTI5n7sv7lBahgBtE2gquGTKPvcD +lU+vgpWexeyIVITtewSGzeGfvrlpRpG0D1gxdXVQjYIymulGHnzI01fLe1xy2xBA +zM+BfTuj1yvqyBYLs/IVkWp0/Swia4KVIkz7yWiGnFnUDIXWNjT9stb9KQ234tiz +cyCs49DIcZu02h/bygXNPrBNpYjbwIBuCEnf/wzJ4f+47tXlGreldd3LqZ2fo9yT +gAgCaFXX2mCQ643E9GkN4649weKoxOfziQfwmDq2+qOl2MhNxL3F4zLS0TN8x+eG +dY0oDuZICjsvYxh3eAuPZmzloiDFRw+qpe6w1MCQTqc89ufxELlat5C9OclL1C6C +xDlvJI9+GSsRd7OWyBNJgYtqg1un84fyN//XG6l1SK2eEBJE14kfxTt6P5TFKRA8 +tN9MsK6deV/bTmikwoJZUWvObiuGJWLprruJnwd20dfR+aYu+sx0p8/tYIlizqTg +cK27wqBV7HmSjzRQdgghVKlsX3m3qpmJFTJCg+leZ01ge3msBbKHkMzb5sQOMU/K +iMPwguf/I0ZHRR51R+HNpqG3j0X6f/opP1bucF/zBEoptjIIsWG152ovscWWFt7X ++v960KlWxI5rOjpNXGQRmk9iP/bAW7v+rSHkPsXfHT1kytlYGcF4TItMgEQXVe0H +cwaVDqVHk+Nf8ik1ofMpXQI8KmJqhmro7yI9zre/K+WXhD26TmVRiZkYqQdXxdAF +/wXYJ2OWhohrLpiunXvsm7H57fxMV4AC6q8UCZMJTRB2k5qucm7L4ym+EOojSXQV +35nReoI4dOp2bmpFY8Is6XtOKIt33tPcW3583K1RrZTT3yTIyoyj6RvRuLwpL9+v +k0LD+rzRzZTpW++WQzOXn+zmSDwuzuQzDPo41oyyC7xBDxUzpFZbhzHF70pEXHD7 +GLu/ASx2CyfzhSU5udndqCoQD9MQZ6QXhh7Xey6rbVyfuW96IVJRDJMy5r91px1r +KTkbyKZD/nwpK0yCgJHO6vCbbzB1VP8c5fQ8KEsA1rA7+FwXB+61EayxseenDSfc +GPb+M9fyLGEqKKZErDqIsvE6HVcf+IVddKGdHG1QcVj9hPJ7HFv1x/fhuZwbu3u4 +Q9mLztqPJotxU53R+qIYcsOdY3fy75L5yUFO4F1wfB9FCd1xqbjQNwh1/jwOgA3z +t2jjhB8266GDVkwoks3WlcIlkI7fXX/UD3IgFG2SFGKzl2uaOx80BJsiMYz/8wro +ghItQxe75u4mgTOnPdF60WxEcjWT5e7izlrww0UqaQutfZ9hyJzYTNpPR2B7x9I6 +cdNF/7JpNsRIAlkND+g7ryf7fh9ZqnylWpmD2a2qMNsvLft/49ljWn8cvSXkvaUc +319NJN23JIT30Nm9cPMF9y3lCki9cgcyyY60uZTtwI8LOzLlvTwZT3IK9IDPgK8M +fP7qZvWPoKvm29+sgNDP4Mz4K6o7Ixj7YF0kbbm/8ctEQFe0A3RqwO5DIX1IIywq +CROBvLMgmFR5agYzdH+l4UGMyG6aBdiGZKe5v4uWU/4EUGuyCFncbpVp5EJ3QK+t +ehTWNei6Of+xcBNBusSTKWc5IdsB9hV5xWwKTKUWyU2LSSs5Xlaj4G/+WsEsvAHq +Y/1skR3vYAxg4ab/l9fxYfqCxCPTcWtYHmJAcq8iIsiMJNIYYlrUh2ZLuTqOevQJ +WbRcR0YZAdtHpOeHIfauP4PNYjoXsI0o9kTUmmD9kjpZHfrUhIlzNBlw+PCJa/qJ +W8R6YXVD9RdovUtn9GciFzchEe4unUxQxuqMeW/dadZcDbUieDR5+MK4qhAyGiWO +L71UrnPjr0sWGIWjNPpK9Jt2p+rgF5NJ4BYo1cydl7vzLcWnE2u6S3Wss1wTl9li ++lVDuy59EjqX2oDa7VOOISQEbtKSIUYDyIb8WWK6y1r5EY3jElkP5scZm0/CEMiL +PSMbsSZrApMZ34eGQFIhTtjekebbwxkSBpasSZ5+yqArN6eiGLSdSmVwvbe77Erb +zYyVRbCUfyBrCgUL39g/UdtKn3J9QAH+uG4ZLyZtXlJuFPyjw3zYyzsQ1jn3twTO +jRI5Kc+WF2ncab+qqp/r/4iP+C5V0yDFdFCUPvEEwg2a3y1NKs6yT6HGKZs9DBtL +QLqKSB8bnwwDqQI/1UzgwKjbO+gNljaLmr1H1QE4XNKzQjYEyjFeWzKuLOAAsdtt +f/1Za2ms4iffQzLM/NRj5fEidMBO/US0PRj0iUpfTi4FxwH1WcqAi46AkmBI791j +3DYqcU+3XfHWBpSpxlHlG3DowC2WqGcdYs2/dlzZYvZauYIEipkDJNjYHUUOegOe +WFpwu4Ik66lUYfAxwoXD0U/r7pZvoZwpoqhQRTMN3PxgNzJZTjN52R7LihttKTKs +u9Yf748WoVdRd7dgK2a/B58rsIlfmPEH67R8ugNA3ZfftOuPwSRIgYjsDX3VWtO/ +tUxU3W5f6ar02jltQqMkIcl9dOmj7vWkWQr2zqU5eFg9yLgFq3050R2dkuKwuTh1 +WhnhI6QH8SlPutJNFox0XW4URRlciHqf6nc1vWe6Anz1FudwHgNaptoq+7to3Pd0 +Lr7Ih+lV5MlFITvMioqvJTsvJyB2RTKLB5RP9sKLhmbpk/XpQwnNOBEH9SGYiQk7 +Hvvw2YcckEu561D2TYaWy1jReJCs84jEgeyHrPzdnZuAxdgpk6ip1vR09bI1Rh+6 +OtM4KJ9tr/L14QEO+3uUmFG7qCFaPAWT8EnQrfNneGlwVZFR2G5YFRfIG3aI5Abn +2JLC/blUIIXMVkt5m47YI8BL0f2aClP8lr+X8DXSIfucn9A8WACCX32w7GUmmPWP +WMdiUACUWz2sWoH4KH0EY39FApktLpBv0b9mpDnw+EeVh/d4edi47kE5f6uC/0aW +3t3/aQTy/ywIKTTnY8V/LIb8EVpmadza2CpPdGtM8UuckNoIuvBqfKWY7UEnCgxS +LT+F2L5Yj8qZkTBRMf5tanpR0+Uada4xAhIHe/R7/uBqvJvHksyY14op+SwzNxDB +7OEOd38h7urAEMaZ21FFKVlZFXocdIt2n6Z0GXd+14NpJY8FIgzO7T8rKInzHODd +jhZNbZdmvkqdqpJPkkLVpXuLkEmcBORS/YJYlqocs15wB/y938TuFv51iekBZrk+ +/lnxERl2BoOREus85ZQxXynw/3PcFe/TNqcWsF5X283CBuF8LBYFX3aQ5j0xJlql +qm9PJx8COgnfzRQiRY2Q+hT1XfpIvIyo9Kdb/2uPh0q8t4aO0vVjdAqoLItNhmux +hfOCBhofo3zvcp/tvIemjliTwrijDYXKFC5FNVct9iq0ye1LXrk1rjgb2DBAP7g+ +7MBtFtOYs7nxgBdpcT+h3hpzQLsGyUOgT4wacq4Gyk17BSccJgzEKAEn/55x4Ljm +kgG68SNc/w27eKwnbrfOm7sfdiXLBQA7r8jlP/HImmbrQK1EwJRlCx7bJGgEisx8 +nDXKuDlTGTwwjgSOIfeH42I4LDw3dckpHbwSiQHRsc72mHl+mMeYlVfudtsv3Goo +7c/nCC28YQ9MP6NeIOQddn6wKXbRy3hlo+LyRM9/55bs+ghn8orxNJ35YXM3l43V +mAYg0AMcJAC36S4PmB41/TOk2Azs8hu3t1mjaywVHZP4C1PJwClKnn1hno2q6yw0 +0SLjifWpF5vjwemBRhaxpX0Lxd3cRE9ufYhXAVrnJfshHt9qngHkX1JUwTafn41D +SKhlSbGilsRPeAJlBkWkFfrLCwsTMRkXmHRYwelNguzbLXR7owtf65WP/PJOyFRt +jicpdXcr01/XmFhQ82ohhAg36swXJie+flF1HNVMH/242uZvjhU/KuJk6ZTZ2gHS +nMq3TNjFeaNElVERKMJ9tV5xfSvo5ieaesfcgxRHY4P7BmetH7L3i0wNlAaIWzDe +gIGbqT9pXVc+SvicYL7KJU43DiE0Qi0Ovbd3M4xgrWhhp7Py16VArsat3qK7Yu8X +vuZTSJUuKQHRn95lYKWYKRbQ8G5UUU+qegPAt7OOgJPnh4qI9yCCf3VTQDQwdelk +wDNKUA9mWkhLJ0jVES7epfCHPccl+L3WYrUBRQg1r8tiFGIroNtokWasCFIQDw8x +RIO5w8kgMNhyseQWxuqXIGLem+X8896kHhUfIO3PZWAkHC1HHbDQ8dBb+T9xYKVX +8ZNHM0qSqm1KkrC/KeGbYgdDbzfi8Plb6lBNDf8eVeGeorSL6TQ3kPcPPjtNVEPI +ppajKNnj92lTlIikHk4rknzDbHZFtItvcqvotAFfWhQo1qM7phelyaYcEUyuxuUv +6h6NRtoSYHheIYFYkpD+pcNWmYttxT4yq/iZgXwhJIsY9w0zN4NW2o/H4ptk8dxa +IKh6u1N2K1nb5NsaftrAvp0SXmRgY/bbIJ0pX0EXwX/o6H4eueg8N5jGwVxLPl0d +kT6JSwxpnh/+4H5o5WjP1MFbgeIuJS1To6/k5CodX/cZSi9zGL6wnkMnMJ2IVBYW +QaufEwmBiLErjpnWOoUU9SUi2MbVYePQt71eUNvOd64AzMQ6lXrNzrvBrZFGrsmV +t0oVTLQnfKSmbvUgU67RLwup5wHZssVEt8OIYwj4yN7J7fM/ntumci/ofPCh4LRK +ZRqj01oNSRtQlIc2ypN1tZ8JEgpbbVnGOx0j3Sg/cf0JSiYfgNba+8J3ZVx9TPnF +hWM9dBmTokSIGI4vK/2T+iZu/vPxIxL/lUXid9FsLTVpaxDcl9eL14I9tzxhXUpo +oxEHBWHIvUIjKBIsEC6XGi9RS6TmkMT2xkZR5OQSwiNbMUcwze++9a5xX27++s13 +BtUQ3j7EQzhs9nxRkDLfYQbZY/3oiOO2gNzJfDpwuc3O5asPUrTU6uMPm+QNgRLm +KaJNQo+dkl/Al8zMg7AH5TQo7HneXa4dsHu9OAWs7tqO2+DA4w6TZJq/UbdqYPBz +NegtJYMbJOxvp+8z+AOj8vmMknSIU6Oc9d/5fsOOae5z9aoE7E3Jpzetsbr9vWlV +GQVDa2z5zICQELOxVC3PczcW+Gtu2Njbj55nErfgx4fEpg/VYlyEJ+TI8vUy66W/ +jNGzyLBdQ72cbOgK/7/g5o0ysnl2EOj2DIhmWP3FMqIJlpR+zT5HoZj77xgDRx53 +NlA6MCAY9zjpBDHc2WSAGNtVlgkzytcJ1F8yKtf6MvGdoP2Hs8GkP2+idA0UqKRe +82XECMeN/5Ypts4zoXNLxaG+Pxhsbs+/4dWFCgk6ZY71aTzb/vx3fnX0ios+mKj7 +u6lECLObXl/I7t58RDy3fDsykGVNyJxJb+4iDEy6QkExdxHZ81kAQTAT2niJUI8a +a6Szb+5VRD/Wg4HbqWbhCEhuePHCOXgV76R0n6q7ZDvskg9BCSE0PFUEK3pvtJpX +ewNKQ0XiHTp73IcqC9QE14YNShcgDrYWjJ+2rQf6nlqYOyFRZnQM8KBECmEmcWMi +sahTUMOcgJ0cnZ3zKm2uRsqsgDkM3mwwYVF1TpbBqtvBpOEtk5yZwg25aFPwksXt +pJKrVgj56gbs9A8QavZiYn/cT9Imxwf5UCDgpOye+lmMV5hgOyKhYSTkjb6Ap/Ip +5qAfaazOzSsQqsyZplgUF5G2EUtGE9EpxTBtriUYrtiyW2gs6qGwxLGhExebJ9YU +GZ3TvD92MSQtTE4cLHjMeoeY9Zxc+eS+avgQwkXYA12sLRBfBZwl0c/ArBGydGKa +bvstjYz5F+Y9V4V5f57uQNNKgLyVaA7e4O8ciJmVg52RVujNNNmDbCFYzc6aFQBG +ZrBKmfVvtyLIdUJkwKRNG6/amB1EQybhvkG/5b2nHXwlKYHdykk/zzjMC9iM8Z6E +TEZyJvD8mkfWprgYMmacpdzOyeDJtAjDoD81QxRYVd75PIHBcFZ9/qU+fZY997rq +YcP+yl/wGi3ySgTW+yLylLifjA68x8b/JOPAqtkC4pKnNLSaJwRNQNnhBdyxONEC +g2yMoPP2W8H0C8o5Y2ajGHLtepCW00uuxvW2uyNs2QqrkOaWz99dqbM/VABUlsiE +nimOZ9s9ASsNEd1WnleRONHAc0vfeagu2XgIJsDi6Ngscs0zhXLUQ5pBJ+L6he0o +lfdmWj+qXwoxnfWjNSISbZzyH8+4bmqbnnmS+xYtFHT3C18Lq15N2dLeqXvS8MPx +b7BFecA54cZxlZEJw9QPl5rCSV7GLweYjuoB+8nnnIVIN8o/tE69QPGCD7AHOLZ5 +IX2ZGD9qVyPb0kmvJSHr4y/KoY+PbLeZ771zoeUt/Sw6oigduEoxAszSj7kkBoTq +V/D3mR6eNDl2MClDanuihKiHW+j4XbZKPS+tCjrVNZgls6l26e7LtwgzAS32h+Ru +o0ubg0Mzsq2wVNd5RQ/bZCgPNxUDkW+cFtpv+1SAV1cpmnM1yvzbwihV7ZNeKNPR +Hh5Ku0acIcbohEEND8I6rLsfYfREJLjiGhrjORWcAzuw2eynzDKeUVVZA3CZ6jV0 +EcjGBOCVbdY5B0e8C/M1aMTQFe1bE7lnf13yzC9W6NlpOpSeu4IGfiIZBaY6XwZh +GRUAJe0hGcw34s70ORJhgyQEQ6CNJFTeFG8q2lWAULMA9upayt+LQztthaQLUaNz +Jg3eX5fxuMGHHXCdgoYSyEEapqxHGs7psVshanWvX+T5R+1lTVARl84YrvLd6IZt +O3yc3M8GF8JFwlAHW/yXLnCtBFy/CAmaCQqS9Hy4WaUYmuZ9aWEl24H3inzlW21M +xIbMJCWAJANzPS/7SxB6qY1BfkrJ219GxcFJ5Wc2cfl08H6yIQ2KYKB7MMuttTXk +NU2wgSOzmZKMEDFfj0Cfta7UvnoRvBNXarxR3PRb9ARpE5r79Tto/czVPu7SADQk +ad8BWBQefw3uvAZ7jwTYc5Ccu/VEJvd9t7oZ1n0a1vv8gTMVTJu4qa5W5R9yVamP +jTtE+JkbNtLUZqUo1Q6/MTsstJO9ipODpr71azddHBZq+v8AmkBMayVxW4DOpsJw +V58gUbNfvIwAmEADTP+z9aJBma9BujLxPlQLmuI0sA1PVkgaPaKTIp1WLTgxh/2b +lkiwvb7eAHDg3cSxMobeSpEyowBUy7Fud9wPR/BgO18OY0554SM1mAq4TSu1Mx0L +Y/r8mUm7kX6nBZACM+LyM8mJLxqvrkTF7Ug/eM+/rPc9AtOQ2TP/k/mbL/7TSKxu +V7+ESdN6g8kvABPGTsFHmYW+AfYkvf/ve6c49srhOakg/9hRZ3gKHGHx42ZnbYDU +CFN/1I9Dli/vgHaGyX5DFc2WWwi/Z9O2FQrBOlOIzwcaZZ6MA9nm2uRlGrb1IFYD +oKiYeARwvzPCgmnipsK1LTzPKrcnYRsY3dyT9S4nozRIbtwND3oyLCUTVEl5Wnho +ElJiLvDpX6tEZVrJN+2v8DwVOVOpd3BZoljdLIGPZ0JT7qZPgxILg/QHZcIlIbxc +bMmEQFg52iiDDvPaqIEqMJDIqAN0hGNwJlwbTLNCGdoYAQonCXGpKOhN9OZWMGYt +1VN7Fw4zfUy4EJyCpp2s/JRfm60om4daQOar8c007NGF21YZE548Nmmq4v7yJgCW +NdFgBrnetUZm/Foc8hB12mk8wr7k78O40+qWs07p0u1jb1uxRI+sA2UCJQQRIzBV +nE7MwTm6CzaEvzup1KSwvxvnGAgoMzhNxT4sa03fpFCQghoOJZ+1ZaOocTm0QeJ5 +Y4xgVgm2IrXx68X9FnDvR8H1yE3xAODDJCLwe5OxOPg1H/yCP/2w72GxONeWtWyU +4g1YQY6H5lx4vv0lxdpWj+J2S0QI941JVnEwQPh31yY3xN2Xg3yS7dMuvSnDXAro +ZomNsw0clXdbuDDQ+uCKzZxOP+LLm8VketMtdLHhppNfcMFegoMqwnQMCoW8e+nD +lGhrNf7MWEDRqPHp/3VIMyRr+USLE75f7j2cv8uYU2unt9D+a4bVfRhYhjHltCWl +wMAdm9+D7OR5QOr7gTIn48j1evot5tpLEe27LOeQhRgD0K9DMGP+7b69i6l9djYo +95RFVvRU4shKcsrhSIXCM16DM398a22QqVZcjSa+YSKkvN2FBh0q2IrH5JWDiHkn +R4msIjcwKRg51zlfsAjQL0WyMhdXAiiwTmNxksOdN7pwzEQxvRfkwlPdOlZpQof+ +2RFHYzztW83B6pPRl3RoQRvtd67J16j/tghsS3oh8tRjnS4Hp1dRUxy9U2xx8Ydl +zNujsz/ZGc2T4JV7T8CAtH4ImqyQTJqlrmkGrUd95w79zvBhYTchMVWgcctnhVFi +MkSXe/9ARHD6gMc2hPHcGLI2prjcTzSKyvrJcP3dOYoSNpRf9ZcflfX9tACBx6Q2 +u8wncmr2cvj5xUTKpetixFdJRhJRFPH+NBDEGeeBmP+tThLs4F5dNRASH+PEjieV +eijr6Y682jAZ1jTwPA8UGwxcFsK3RFu4meCPUcxN0dhzOyMsNYJWZKlV1pHKN2Lw +3MjINwx9l0jeZDOkjkm3UqtMWB5b6trUXOSm1X9xZzUARjKwmEEWCCge9Q2Q4iE2 +c16N0G05K4eqEPnNnR7H3ok0F8LemBlHfJQwp8e+d/oNUtudVwWvKCg3UlyJWqQi +Apd0zkv0HV3zOtFKjc5FUsOM6XyP9injM3EblulsZopfnaA31BLi2MhxypOsU26i +nrMljoEdXT6VxjjjiC9q5HzNJwvpaW3lWKCyY+xVzaB+yIH527H4vdnRVrrNvaq+ +6g+EljbdezfvQOC2S3u9Dzwj2m/XFLZCZXJN9KGQzSJ479cPj9I5IlDulRVK4EW3 +E3NW6GDoP9fGmrcsJSm/6s3vxuOCWfgYEzBQRO8zfpiNzKj3pPzZyEQPe9HHQ+/3 +U5qKM4Bp7yuNG33QWsdFSiq9103vaiBSueF2lTgDH5myuaIac3Hec90Hv7x5vkKD +t7dyNvX8Rau57evMTFckDbljVEFienqsIIGhlA1a6Pd7XHi1fiqPkubNoRzvPa3g +yrVtpn3PSnRcNat3QR6pc8N3AarmutOC5rBJZEmvhdw2bl2nQFSxA9s+Oe4j+ioy +sPJHTPVRNI6h9rtatbxf684D7abNlg3HS/j0ABBDnP+T5WgjoseAMhE+HYsUvles +EjeKgofenJ7rWFOZquKvgjo7JFIi32JpyKnYXG/E+SdniwgsCAnxJ1dsnVfYYDCI +op+lbp7pwziny94TIojCr+Ac5+W4Z79/ROTBDMFB4j6mritQGHFxCTlTQ8e7TqUT +op1byU2iy/P1nJKwAQ8nG9DZNglsFWW6PtVmn1/Hc4iU5tv6lhRmPH6YEuJvSJIK +rutuWh6S7V/kWgpAfsANUmnyNwhbHHsbCn87P/PTtuOBvwnjTaKUf/NRc46CV/m/ +yYrKKSqlsXdsGfGCIqEy+WaUm/YuTLaX7WCwtarp7hktnY1hIjqkGH8oYv7dLr2M +qddXdbzJDS/WphNhrl1p+3TUpzMAhaxCau8uJ6xAU+sW4d6dFFwg3hObwBi4Sbn5 +piy8OWh5jQYUZBTG/zKlB61ELu4QzmDIwXXLP4QcUe3oyMT57wDLsD+fPCN8DVdh +owHSE0byD1UmiMCm/mp3TzRsGRjqr7uKIGYigi//vrMEyIOjKPZL4bs760mtY6lR +RHoyIPl1PCltJ6sSk/cwWjrHWq/P0SCLd1Ib8zsCc/v91JQGnTxT0q12+Fs6fB36 +Tfks3c/J5V+ERWS+nWqy/tpjM7cFGCpJ0wGjSb8lFdHyQYoAeBw/oMsLQU3q1NvP +pIVUoihoRxqH98UF4/S2J7qs6G/JBss1FuML/zPqjMQlSNB0dDF3dp54zZssV6gK +HPvZCw9PZvbkVBkrSPRWinjJVwWceYdtVrTg/aJoDbVlP8bGv4Pp+hZdV0eyHuBo +noqY9BLmF8t1XUzvfXT0/tkRk7mWHHweX7QymiGNU8Z4ZLzPTq+XFK2S0qSZ1PcU +c8nIsX7sQ2d+LXciqKTLjDysvztJqNsaEgMLNYRlZvVZ4K79wP6xfQOH1BY7bHF+ +3x8C19NtJU1b8FUIfRducS64uc7xvJiBFw/5wdsExmxgGSqDt5mBBXJiglGNVXHw +LMVb35tG904rS5fkcW3xfo1WIr0UQTYFlxaqPhkA4/W3n6nMWbZqpT64u2cMXjpI +3kouIAFCYQKU6DpfCKo9WYrYeqac49ienx8lW0GKyEY+z14g0coE1vWCeEyel8Pg +Egn0A1fnXRVIQBAV7AvKcKQwTFhJZQXGYuNGTx/xT+bwpTJ8giwK+egN3B5jGqr5 +GHEkm7Dv0bBIk5UKKw+vTh6ObSR+tV5KNCl1jlsSp9vROL3/aWzDuM6ZPlSLu2mV +GziwidjNjCunNd3TACxTxXqSJFWaHvrwOdZWyJxCpLpaolSd1jE0fNjdEcLHzSfs +SCVjRAYxauNu3RnGTv07wrle3tlgHJeeQ8tu/E0VGEoZ8+4fNs6MCDrtwJRNpXcq +Hm7sJDPwtO9Vuu8Y42xw//aB6VVUUziwMLRLfM/gtyvKthSuziv8DZx+sgwd1QzA +SQF0yoICwtWE/X68ImbF1LcpPy1wadeqElcKr8nRcgejk54TjDAGsJxwm60WvNJv +q4VG6OeRaaT65enaZO7xpTwRaWY4qAd0f1VKntAPAddhhEpabEtpo4ZP6XTr9NF7 +ob6ur9l4NrQ/GWkA/U8+S3m7vDWXgDeHdVOO4vJa8Qrmxw4tJiH2thy5OatvgJy/ +RxKMZ8Wtdiv1VHg1bysSZCiiJpjz+rxD3xrbLwwKbPaRUC/OCcvgTkpNz+QyZo+W +7iTugEaagjEILC6ZEeYvR3bJO0Oo3NiIMI67uHMTmED0piJ2ms/xiOIKnxAgcW9C +2uHuUhX3yBkkNUqIEp7JKJ4+xHOK+ak6kbLylLzTtZuTTaee7EsEuCYVlrnyEnhO +ytNk6dPX3XvRh6UDyxLF6fvPgAFNFfXKqz1Zpu5IzILKS+zONXo6VU8vrTtGqz5q +oKvz2oVH6XM9xSokB65bBJ0St9YEBAtxtPOGt+xSdZpXmpyRkdo7D3pZDzK0xld/ +sUOVkjrxJiLPA5E2dylY2PXkrGh1kZS6EE9L4twg4FoMDs56wBARLQWf1nWrHWB2 +eRA0K9NaM/refdASr6BYoodKgahCWo7Wog0BVi6305XYQJ/0Lt4RB2i02dSv/58V +z78B4SuOTEoVr9TZ+DBXBkie14rJBpprN4/GtqtZWTk20lBbzxUPrUvc4bI6aCzH +NNu72oG7VhTibbS3QrqgSCVs64FCnUz162KRlOgLfSNnL5eX+pItvQMYpbuxNA2C +xxoFrcfotFt0lDaRPrAN9eR0qfgWShYvw25nvw1Nt6SMUWFzP6eFHvKy7E3yqaBk +KH2Ae01SGyKGSkyFH9t7p2r1Rhy+4QIpXNAzoLMOaXplZfFfLNy/fKEkSmmg4/Um +GGtwmxDZaawxVg3P6nnqWsmF0mDwK5Hynb2MMDz47Fh4HzC3jhHlEmqtsChaFMCM +SxzamfHuchFF7QcD1JlcYJRkcjL8ed1AahBMjtKuN5T95GEp1nXC0vXO1KbdUYdq +wz+GSk4yVfbsj1hiL5z7M1oudAZWnhIy5+OSuYsq0fgMc0tOG7E9yEJhciXGz9cY +ubCzfeKFKTyUHCb/CnL6PvoPgjPzSPOQoFMNax+e4xPx+kgt/aW0hsesh7teOawm +h5qdpyG3cEjepFf4yaMni97WVyP3Fx3RTx+KnYFyqZYWJQXt8pNb71IbS5uGppL5 +JokZWKwVV5qkPdOfzhJBBXnJPL6CnGepMedEJRMCAp8ewsCW1RKP1aZtpqDpoMLw +SKtnzUwq/2CwZ/aI9q2GMwpDZgckiXOh+MO4Wn1GKbRGyJsjhw53h2lwrdenxdLW +TKkwp4jcex/ZPykDU+eep8Bxgy1iS/R8oDZTijrXJmYO66Kt0Zs5LDrfH5sMsf8v +mx4RZ/kIR2wqrFTc9poAVDX92BH73BbZf+T6JBVBIA4nxZYCpq3BnNY4ey5tn6ZU +HlUQvgNDeBJasBNIeCQqe1PkOtHagldixQyiLqeYVcpOS4QUrl6IMEJOjL9dRv9T +4GOhQ9eHH9EAqicQRNkXgMIcjhNgjE+Io8ChAS+K7Dq32TgpZd8YSWExRCR97e7I +B8n22SUZXPSiu1ur5Mb4uGi5BIzFQobTLPf7Il8+AJB34oWtuXnfMf0IPr7L9fGM +Askk8Bp04HBLCiShcCahwKOBBu+TB5KUqsmO9hrcNujCTdYa8jtRR4OtNa5enAm7 +WgDRGmE5fZVvfXNfbITBwGzlWei+SK1oddu3Rk0222fiFpzkl6IYiAYbxWyOgK8c ++mOMH946VlLJ4204ZFI+KusX49UTTt9OBEMzGSUHQ94N0vJhhJu/eVnhSkttJBdk +c9uqkJJBOfi0yNN+IFwrhtxtBg9rCeh7/Z1oH3Pe+3XYRvS/zHBDYjQF8wB/QTT1 +Cv+brIOeMB7unE7THDZTwDavtXjCyWwyzyG/v2DIKmeHJmpkafRRV45yNPpv9sCq +YkonMgpcm0KGmY5kmlmSbeopWqTi/HUl9VPZnam+n1ftunn6dQpgU1VXERfXwUdw +rz6zVieDv3Vuj/RmcaO90TIauiNINkEvwka2yaJYd0F/CTFEJR98NKCZL4Zlhy10 +nvCSb6kJiSJIlSjpjYKSEnAba9rrAjrh+RVZffVIlKvsINl7EeqTwPkQmFINAaKI +65SugRFg0ki8DMAkZAuKs3hwIgSjaNtEWtBOMFp4WDmQNKs6C2DuLg09mHvdnQFF +jdT6FP++6WIUMm73Ii/0rf5KjARELFzfupNQdk4WDDR1W0TIEfmDvuinKrcIHtYz +5AwphehDJaiePr536+aV99uqnatLLFXF4C8T0K+aubbzcHhEn0kD4QEdVV4iTS+f +CvxcN0X+uECga2Lw0k2sasD+fNp9cbmXOwyIACIRTAwTJFsKToamzldZ2KFN3WSo +aRDmVnWKZSbUKqMTNKkWdeTUOq4uX/p/OQ9zcAN1UCLAAAOIrZM0zJrLhuC+t1V8 +zZGwJM78ayvkB2tzeGQhMrRBmUd9anbuXTQcGIXbD4YSo12hpMGuH7TPk65MAFZQ +sIvzw4X76HzTzXvreskIDJQLMbd08i+s9/XlG8ML7WYnHspZtAfPke2LQHM0rxgU +bioJONTnhmXivj4OGKZhjKS0dhS+/PTAqJgqaglYpTiRRVOSP4Nvvn/uaFW6pbbE +SA7tgY8revqcUnT3BbvkGnhPlCG40+upn+TT+hpfaXwDn0BtCo94lUZ1u7H09fb4 +UxatbPGO2/EZ3QyEw6yYpp7OMDCu2NfCAZTbs1nbRtXhK68WOQaX23TwsAIG9IHF +mMpiZc4FRKY4NmpGkDw5LlDbg1K4HgHy05cdaCLNcYk316en0aCXkGAfOAXQjInn +P40JuHaDbYyBaLP9zeZSBa6Nf+IjREtc8GMantGnHOg82Mia0qcKq9jhdpY1oqvP +xfKG6iFva/tbV1TDrmWxPAY5RdBEvVr/ThmTKYC/AgsNm/KCzKn/eQ6YPL4qdqEm +sdHqZ1/unPzKMKG84znbmd1SxABXchAjGaC08esJyMRgZd9eyrXI9Nid1ruIL2jO +dILf+AUfi8b6yaC5lDZf5TlJjINEgxA9CJi0HistDRZKBbG/k/WJkXBPyBsW5scA +hcFUIHwkbGjaX9JYMK6FjwGuTM/WpoZ2pXg/Uv64zX3e8Qd85rvkNY/CXzX5qPFl +Nt8S6bdzG42ngPsR9KuLliO+Zus7H2E9eHVOrBFCreqseAO70aH/iSuR/djAHyp6 +tyranyAymDdMoZSPD59c/Rtce0jMgKioZFK0GyiXiGfddMo+QZpK8BWiTSCoiNqk +nbx0fTZE/vgRuyfEAZBt0hh0roaA/O6fh6xpi08H5Nrhcy8D+PIDwWCK5xLr4QKg +0bT7Q6Ajw82DoFA6vZ843bHF+QVlZjfngVGa0EzMowlSw75TdfyueJ5+O5pLp2QY +OMv1k7bCWXftC1biD9++imSCBPoHBJuDVw5HdddSAwb2UHAedGYSn5N4uVaa/pXh +LaOXrCF7H8gCeamFFBOD3YdT+bv0WdErfjgwcRL+jYLX96gJvj8od6ItwoQsivKB +W8XEnkbVEBCpV8SkR4ORtkidoCaj1Z9ASQsWtQEuOwIdehzzHWzTz6zC/jeOFV9H +t/WdHmExifsE1mjZG37e3cUQUIViQvH19+okCBDQ4Z9miBydeVuPWMorqQYq4eFL +yvaCL9aGw7ywpUMPPvzYRelJtuT+Iaer9+T5e0+KMDlmCg3nR28YFWfQGmRYGOja +3Mg06Q4TLrllaDcmjsR2OPtIY89n+45GPsxNMB6PjvC4Vvyb41aY/8FHOCDpMd9q +XmoyVLUvd8zaI/GaIS0JnT1dhVqsA8aXnus9/MqwxrvFte2IPVYLm44JI5+3byxG +PZhTTfVbHnirPpvLDh57UCExMqZUWg1nqef63ALZ8xuZyszU7MQb3NcufJ7Jdkwo +muQ40soqsQFJ0l5aW4T6kFdGp6SsOKy0fHeJMo7S8AWFeakxnjTApBq73DdXHxxx +kM8375F7ScBy/JeQ3iSMfzwz7/xuCSsoQTOWcFbj0lbntIvEz9prMgBX1aPwNF5c +AYKuGM9qosi0bMI7k7ZXjw4AlQKsr8uKaLmyXr4R3wJcsfNl33hRFscJ+Z6rT6nt +jsWu/jLdomuaIh8U/zPtXxwztCzPUH660LJAv9tT9UqyyqMGrogzBPvghtB/d/aZ +7FIwkqhNgd2geE5qR72pLOnkqcE1wlxTR7+QWYPDvenvwHlO+nxZxiS8nLWKmvET +tHo04rsAw+2aCVfGJ/Z4ku/krEsXBd0qIGjYNqKH5V6et+A1xnn4T+ZiG6ggjKQi +4B9BN52JSyoeljqY3FxVCuKkRne/MnHYq9SksQrTLZFCxXpQwBIO80IubpYDpZJ6 +AQdujk+6UpY0+bFlvi2qVZ4d7mK0Wp3Q9CdylJOhkgdO6jGvhEnZU1HjhkZH4axL +TC2mwE+klPP/ZJo7+2AdecrYriUnqUPFm2HuCHOSoyYks89E4EgZHVjnAZ/ML3Ge +TXryearwzgePaEIWFNruJkva7jqemGokJ1P9K6LLIlhkmbyFKuXC88gmnNo70I+K +1seBYDplbHi1LK7TZuYBjtEEt4xnMfDel+dF6qxR9wwjwWOn9vhkhqG6tdVWRdi6 +2m3/VpTL9/IXlM1wEDlytiv/b+A12QgZhYCbl5E69JU2F5zzBVZ8rmJpqOqE/PlP +R2tl75igxaJNJqqN+7qZO3WgSMvnnTrJEG27UFw1ANPAuxi64tq7KhZ6dt+fcwDe +K7MWvi/XzuUWPE8I1UWHesEawrwOlX3CNnwWnT2CAU2WRtwyQMuAnKGn1f9rmlXX +hGOM/HvZZZcbGGmmlsHVc28xH7oo0x/nVxx/jAfXAmSMcCou4XX94ZBu28e5uoVZ +iGf1pHAEZ72Is/iQ0tT7XOOS60/k/5/hz5VcRJ0T0hHHgu2dC2eOmS9xjwTqAvcb +XE4FiZw/hcFxDAy9hWZa6VWTt8GkJfYhe9FqmIML68P9KvvlZxL4RAtrijOsXwtJ +buBucggHmvXLlTPDjGmXDA1cn5CSlhWxCTmtteupSA0O0O2RS0olnNgm7gYP5uY8 +vBokP31H0GbQqx+EqrPlie9UOVYxO24CZ0Ij28NeXlG+KFFEwwE2HkQVUYm0sPyr +ftLFHQJU6iGyte1K6Br/biIk1cojMiHYxIQ8lutmUICWW/L9gkAGJt4hWNIWr6XN +BonAT1HjJ9LMU65ChgGCFO3vNK37yJk/s5DphHTF153w4QdWWe5bfPRFlRfACiGf +fXzC+PqB4LaE1I3SpWreL3uFSo4HWC8vXOUR7M6OKBahHrGHOwDmFD3Px18jgbr5 +Xly3np5wstehd5JQatAakXm4JK2ftyqrOQdpom4T4oiM2mIgLDpENyUwvjClqh0Q +3UDvXci6OdsefGloixyrLDZ1lqs5B9qIiMgoLofZI+FpnW71nP3LPCaAw0S1KnUg +9otK15pXtjgPh7MQmcLl9W/Xqr0+RvVggRorgixqqgJZedvSZ32K84D6Q6QGtqqp +p0c62VWOfp33mthXOTMfuV5/rdVb9wuehoKRm2so8Grb2NY2jTAlfRx2IlC1DJ91 +vEHrvf4GVjtmsHH6hWUDLpTe/MDn3f9KEehQoj7Uxr58tSHf6uSRCYAIZk6ZyX3K +log78Umu57/yNVnmLAcaVVNiwbrJhHCR1snVb/2YeH78Xn/QK+NI74K3SaAdi3AH +oHDUnxGrfR/O5XMyFjPN6diHU+F4tMp1bDRqurFAqBvEcHOxR+6rKe9RAvT78y/3 +jP/DIA1lekKRxPhX61LiKloLXoklgY1UbBvW3QLBrsXq2pSuH9OjpSw+kvhDgTns +LQc26utS4F/Us2qWC+9xMVn2vPHhMu7PI77uak2uPlQBNG0WlQ4ptkvJTPjFgIJ8 +YNpyz8lg+55c9aYfVWuLgl1dUnjhE/t2xTWwDUihGftRVr6DwFylHhuN+2C2xDbm +QgE2NU0JjZ8R1IFJ3WyUascKsDo9N8VRN8RZ9zPHg9AisOTdcIH7OCRygZHXrglW +GYzt1ZeWrHZ6a/nFykLk+xxrD7wzv9XBuB3dfimdcyiACWuvdyHQtQ0dVVsbT0gu ++s60Kgn3rbGMZ5h3reec/d2e1Y0+eFDfr6AuOq6HfcUCNNSTcROBPoi1J8xHXz7W +stl+My9sHUOfn/QcfHurh8Fi7+odlD4KHygO8seno9G3RsFa2EgZYBcgGXYW1G30 +zkOSK9lTNCx/vdT81ONDX7P4mCuYUa/OsqWK6rBLnx+8hu5v1bWqmpZ9PKuR6oav +MxNJ26mrELwJ9wDScXFVq1ipvwLBscCbk/73vQctzlMAgotpzvj/dHAOPP3cVY01 +W3WuZrsi0aqpN/pqpf2JZM7LU02XrUlr8y+unYZUf1PMOb5v3dm71B1Z5AHmvWOR +82kgHe8jJVEJ6i1QmY0iX6oo9/hbB9+ROD1DgbpxqdP6v+Q5TUW++Ub2L2ycb4bn +rPWS4+5ylghicBwbjZiCToaLGhmXfRu9L4OeFfZQ65x8Kl4oPLRbekNAZSjFobsc +4wPVVX2utMPTyNXI766s+DAmkVlruofyav5C4lLHQlCA8I9mZtfQxYRDAJsLKNTo +s8asz5NN1axPs1RHRa0ZwVetaWQJ21iPPTG8Lxj2d78eg/BsvuHR4XVVKVgAjrC3 +fZXgtXaPE6yObk/QSmBDBD0u5YZa3nI7Bz6H7uSNN7M39bLQs58XJaBz5h/vimsf +HBx/r8NgWNeyGIKv6c/yF7meapSgZNyd6klGsWrGbiAGHQIh1jIvGxnamx0iF9dn +IZvAnTC64VDaVQgT9847rsTKyxerVyswFvyKOSNA5h3I87lQJqQBndoz897NklUe +vm7cGsUOgW7SbGLRJmwutzv9kx41g39eT/cdf+C+w2sgQVJfilRKuUXTqGeRj61L +lsJa4CS2SI+Ibnyn1I3pFgdeWSkpzwMF0FCVNH4/lqFyEvnCp3xPCthhmAte4mLx +0bxO4KIa+ZlmAhTPP0qtTxNwQO/KV4QJxc3BxtnT5OQTuSuOLSU2FO0+KThvCwrZ +tQv7nqMgEwaKypABk1YQLyZYuCiqk/TbyQDUOfX7QgQ19JDgl/B/lUbq40fF30t5 +cY+fk0DRszQVE60CIUXUtopM06bfcen/R8HTyWp6z/ocg2Z8/NoQjyzSPytenw23 +aD0SkXUUyS6udXAevX60HJlbK39MI9Sk9rBraOHLPVfKOI21ywYg5QtGHhff5pZG +frojhLceOEcZMMrbgdlzuAs971ZuyMaqP/QtowzzBKKJ2Z604ZOzDG1Ukc3dNOTc +4eWfRO8FyCAeAFnNmPVYby33Bgy41xm61JEWJfhRuOqNogmpyaVNvr5qPDrgOe0P +lc/eFcAVJ3tZpACsE85ugnqdamShQCAFV+kO7+pDvkJpEEnj8cynA9HMEqIFu6cQ +220GzwSg0LAPR5ve9GwP0vLtvY37nXvpNBWS+Tx2FKo4PHHXv8JzZUxiBy+i9+DF ++//3Fi7k4RY7I/4zdkrWc/4fsZKVpIXe1+6WnXTT4M6O4VqBSUmKxYxaA5Qr+0Tp +mazM1tZaPQaEgynw6ExIhLjJZXmHJ3IC8xxRm3Kqa7Qw3UN2gP4eV3enrYerz0Jc +wmOIx3ZGoBA/5HkYtNuGlQmiEivFwJw6LFt6l0jqVUreMVTr17lc4CGUMrU0L0Pw +Dcq2Bgxmdl85k5FAdVng+1iyOxQZUd7rfHbdojTmV0oDQSNcj/VoWre+31z0NS// +1i3Dw3LBXv+rdUq6JsZpPeRtsvl1Ywp7LMMf9CswdkbcPVyRwaq4nWK//vPc5P2W +8rcVhYRjM6snCEhO4iu3fuPZGlM+HnXPDXBBm+zwi6ddzn48KSR4YUtB+gwktkWr +5iQ2HpZS+N82n4Dz5A7tlBGpwyCSZllWv/5xW/cGdbiPTRoHZOp95p4VQfPMdnvK +w4YSH8ugQ8tU8Oxy+P6eScA3xR9LH3+wWQKBpuGp+xZeLoiufQcofwlLcNec7yXp +bqs0/G0aIFX2c+t+LwC5O+uoJ0Q2CEa/LHoUKM55xgzKj/Z4DUN7u23DpcipxHvq +dh9e4IgLTWnRTO/uLk1GTac/ZSKfkDNwlZCRmy5xoKjd1MS/3i1wFq3sw1izVABh +MU+d0JHFlwNurvxBs4Bhm9TN98XpZ1u0WqsRmQxKaP3Nj7OpGX2lUtdxy07N8aEn +iPrOZA8f1TlakwgZG3naVmEwzEtshQrqLgY0lKDiGgkd8uWovMDvSWO3OVRtgNcF +tpGlXPPRKLOxWyB7jrBaQRtxPJM/Ce3dfzHwWfEXAj6ETbb5Fbjg3YG35ikKWffy +Fv3BT4EVzg/FdUvsSGYMpnap+BRPq7GHq3yG4ubbKxglAblKnvfpiIafo3Zsrtpr +AX9Xg5QvRoThH4qh7kzZo+Kl1MRwMa2RdRbBttDezwjxkG2oIRP6dfWMsm2JPeZ3 +BNq6e6wZu/wn4OfBQ95hezUt+JVXIbyqkPNmlIm61PdcrQWvVuJMqLzSU4iLuTC6 +1pt1ou+CwBP9m2LjZRNPq26hElTHT+aViTL82Flq2S/WjHIAW318BRsSh0BUq37p +naJcTbxsG4LX+2ad49LBkuBLfWgDDTE7ndMw3d5q/zi1RUINDLPH1ipMiGbt8g7G +f1TJM5WvxPwf9HM4RyYecbaZt5J+RQUBfilroqOdwKdUKf2Mw72wSxo8H/0u2Mti +S4QRBKF6KoWTgjc7sGoSMXjdEW0+wDvB6DCcNwwkQGF0XsV2S3myo2dymyQrvxaH +wCF9qr1RQ7nzMoCPkz5qCwZy/AtVBrM3s0LhHcKa/8Kp3ht7qLGI2XHdh5Q85I7P +APkKwDt+ERu1eexYteTOCwg7MWJ44CMqhfX/+jfWg6w9k/Gk55CUP+iJD9CnZT+0 +GQFJ87LJk1twGQj11boPJxURD6/5BmFtGECOC1y3cg703Ycvas+lkxvrpjtM8Ld/ +6ngDRjrntZY1Lj6PeqhouRGCH47AOuASJutVuzXQjtp3zcO0OtPwIWlSIvDy1FyR +kLY7eqxrPVxx+F8u1tNkSplSYy14vFRKqeJACx/RjHkJw6DbDDz5gS8STOxvzc92 +r46L0yvXVW2S78R7kN1+sN0eKzfHFpe+Q4bypvqysDJ/jeMN7xxG064K5NLUI/SP +JWD64hZcqXJz/awdsA3XfL23aqCk4r5fe8S8uYHPn7sZ9VDoATZ8/bYVjZI+JnHT +yKcz1nwSoZuDG+zhYp7Jj/JXAGp8AYpiVGeo13PC/TJLzrcx8vqKxQqkV5KlIDGO +g1PmiWdyPWHnODf+Bh4BTtahpSj1DvhlIOIgSLSXftCFos9pgTs5mF3cdQBTVoxV +iZLvrYF4fl3LNJYpy5qEZPxlJ+UmXqbsQVRVKX5c5vNBeCPamMxmzNtrpWHEb1m1 +hAKU8OEnXJEbC2vuac96pfX4K8KjuSJSTx++sYF4RXIKsDq0CYV8uYmxCIZFEzCy +jDlSQ0yf3bn8nyvqqhL0hCoTkhXXeP96mPDe2ItF2qQ0lP4QxYcWjXLgLzBcjD8B +j39NjbpF7MR6RN+W9macOGoeZZ/5MBHfJa8vvJ3HjHdVdJJo5FrQXTKrnqSkkQ3h +s0LGcrCc4nmfpcxgN5c93TztcvsfMUQ1fpKyJxUgSdPgzkpTZ3E5L4Z6v96n5yJq +lfF1HhosEHCpmb9Paul7XrHf3isRlKo3MgfQBeP8b69eqyJ1hy4V4sc2YgSqOMqn +ok+zEpcnL2Boe1t15sqZFGuuF3pdh9avcFZZVjMyCusPhc6jrqr2RVbrckf5Dhc1 +6hpW++TI/OM5iwbshAGU6+7n1JEfauZQF0/TuKeGw1nmA2AHXqfVYWVgE3YdnyTZ +11knO4UeikZ99JHdtHYWuo+wl9Gr4fJrHLHylwd9r1c+GLQF+/spSLFZU6dXEPq8 +MyDw2Y5z7SOEPbzxV0vJkTOyRJg8g4z0Qny8w1RVqE3twHxxyKe3afp0lJ2FtZqU +jVmGn1bEX2oSLwBgnb2ABsEJ5ppztKIcQMuKGbaJAodQ//CwflqUsv0qojV4IG6v +cwjTXGxneapkIggSG7ONROOWp6fJQskttDWn3cdGtM8VXwa0uxwEE1SdGhTnxVyB +akLCXh+U/ax7pmsfWRvur3rLzrR5fjmYFsJfldTehAUrsSBYgzObRwgwpsx43Uj5 +t5IxT1OPZSU45xYAcyjJClmDEMmnwD5ElxOuriSUTmWiS+3ZpTHAzEDitJ1GE08q +vGnS7D+kV65musNXyGAaf/eLkp8BvGSbDXCPxn5OhB258eyqHPFojlODVa6Ja+43 +VB52+4hrm6RsnWhA4PTtwgvlToKjingRX7VYWQSQpcXzuh+WRjcZyKvhqMVgJXJy +UzAKMSzo1ibbZ9nduCeNYdRTpSLi1AvoqSFbuE2AXgabKRXrK/a5FMvFCSu9EsM3 +GPtTT/Hf8/j7c8CEz7lCkQbSZhPKp714x6Lm3J2r4zI0+wvMjCIxKU6O6nLHT0Qq +93U720jtOx5ITGnAQ9IlmMX9WNoPfQFwlRe1WolYG/XX078MIVbYC2JHq8/xyIFQ +HSWUERCHUF23nKeiyTJJurZalLyzTap5Vt6b7jSXRDLqMP9crF225zeQhfdyuj44 +RLnxHMcqcBh3kdv9292XOvv8SxEh4uyuz5HfHxRXnXpSD58Bg4yBVLUpHlRFyqTL +93VGgy0pZYxnM4c5y+Pk2wxuPIHAJ4rbIq0BdRdJrkcTDiIciWrw6LqcW+PDLR4Z +blJkyerh88y72plb+MIm4cIgSVaKFPOfVa6cvPRaqdHSPdyOhIaGLFMyGgRaPeux +MGohDqzBPOxaP0IdmWYdpA7cI22bRdtOzzzn6vN5kDiwVgKqAI0PmBd+c9dtbCA0 +LMFLPZIOtbaQGL2iMmV2h6AeAn08y7FMDYojsRaZaE5VB/Mo8Ajv7eXpLQbVF49A +9D4a35UmFgewGL0t20CFPbPUIxsFwmxQ5sZpmfkcBPxUN8AHeJ9E7DB/WB8f7Pb4 +EltBHY0+xRXL0UpC6Vp6lR5azpO4ZdaVrEzv/uuPru3aWiXkyx2VkMWZRg4YYljF +e9nlaUBCVLzLBVjD9XKLufeU2P9QGSjkf25L3SABq7iYNqvDynPUWHXlO1/KcqqV +uMwbvFllydJuE44ONpAQBGex5NwM1j+lISwFEcJi+7MAgyphEou0E1BUAdKEu5QC +mGkdJeTR7hr5PQzOHgOwjzyyxZv7qLAezio5wPCz64HalIMmjwcxv0ruTtOmZPsp +lTmRlBsryZ9Zwf1sJDqZbtrgpVqUKQe03Vtv7OparM/0ZWGFnQL3rTLAMBQlduSV +5LnMlLrFfB5UV9eP6W8EcWGIEQFVxNks+wJhDlH6PuQxaIetv6nARc1/+G3BbtX8 +KK9xhwdW4aLTnVGnnH0tgB225gqIo1IRacQ7ADKdugl5X8TtrCbZCZG6SzyRfn4P +q0Y7WrF7oeTKdUU9DcdhF7oYLlD82CXvEcd2MJ1mGsuYibIZT1k1gFkOHqHQLGDZ +jQf9+CWhQoCv1GPB7TKwkObkLJrT4jKX97r0r93Rnxn8zhTQset9yPBWN6S4tMFQ +c8XO3iWS1V4j9ZgXt91eG2EaYHIVSEAAaXj/u3jBjH5EQqLfM3sqYmllPs8uqP37 +bWo9yK4YmjdvH+QEbGm9x1LBllxIipM+62o07/iitvhtfdoWyImITNufjOXDyd3g +m7KOoXe6tSzQviW1kpkRMOB60y6fBC5F1zo4BZ/L7jxXG+fehVptw38jZD+GfwGH +sw7+YhSafQKEI/CdEmAHQoYA64KgCukCL2oijzhRYlrUTTXsu8dn6C1dphU7chzt +4G/9qAsopQxvW5BCsg7su+17yC7uWnpZo+iVEwFai9rEgjIyjJu8jM6eCqUwpIB7 +pu8OKSliozMJ1cidPMXmcb+SDIHl5IuRVhoQx9TGoRGCc6DWsxdo8hyZ331BCPgm +ZhuJf31723SCQ5Iuax+ZqHU30GQOjOfJhzCaleU9zNoyCRTcKf3ejTVfrP1w51qq +QSIq7H7rAxqexD7zYaEtu5LzLBZbAx16oA8TEm0yVFB8aar/+wtdXv/CT2HXJCAs +bYGjrQpK0xAqGsLIPBboedpRNafcN/kQbT/cRZvXTtFr6dd9lMMWUT7otStAPFcv +O9sorqzWkmelXMcaZqS8nEPDVtAllOYjKMpip7Z2h0YBJ5OFZH1U+REBjuzQZXC/ +NrK2vQX33C6m9wPIdjG+8LfjLlZh/P26VbN6Z6QepFIrpSWmgOs33PRkWPBURW8T +Fj/VypR/yNN+ILsBpHFY1TOYuIG0qouz65/+vYYXzwecDeUQdh3VRj+VS0pE2Zri +JtSY55GUO6Z08R/z85fw/6r8ZunDENcY6428He/FBWyWuYzEJZbngVr/GO2x/vja +iBjpAI1XXbdZvLzLXgfVmAwodpHbnACJdv0haRO8TiwQafxkoMTJu9xU+mILzK36 +ts9gJyUnQ6U1gvkpp6hIAm166h4EMs/sNILaF30aKatEPhwG6WHPbGtKOi8zXVxh +qRNYF5ERB1EKwpIyP8ehv7WTX8HGtsp2CpV2QPx2RKA0u0sUKezMlcfxtzy+5tO2 +uX3guMBRKFMyCcYjVBjdD5qLp9sIcVdrl3ZryFERtE+ssq3npOPVxITAq/AVVwja +x0oxPf3EgQ7SynGarcVF65A49EWGFtkRcpIBPTgNfmoEim6B+ht1Nx3tdYKPoFdV +sCQ3fYgVmV2U0k4f54q/r8R9ZAKQQj4DY3WiJ4eDhxkmCIGguPGvF4O7Y+uL1tkC +t5AiTjy8MiyUeCWswOXI+plbffHP7Ech3IW2wEWaOCw/3NmjWnozcX9dzQsRD0ca +2Pde0jNosSVC+LziT/WjTtc9617Xs8xN3DxH7iNoCK6mbp7Xx190eZh3Q548J7EP +2VDnQtMp7wdfk9zGNhTVdPnogRsr0sKvb2DDSIO4vDOPZU9d7j9I13XmIFSwfgKg +9FnBLorIci7nVKsue+YoUYdbC+GI6yMT0jxTg178iriyt/Tz0G09SkGbaOrcoi5+ +8FU09bV/LQFgP+v+iDkPewavWSGcy2mimfZpN8+ylqZ6OBUN1UGaBXcx9RQ182Uy +UFzdro84OHs2EgLleBD3zidLmmUQnFjSniWhEwIbOc/VEcU6luXzdSB8k4SbR+PB +nZFFXzwc2SzfxXos1PAp6UX4TmHjDdap+mepFmOg5GfDKYzau+zB5HFAho+UABvt +m5WWG0LlzfrbYs3TWy0ZxaFzsJhDdlgiBdF6X6i1oMzVxI5w2uR3aGypQFb6dNgR +hQ+Au5r8+rM2AX3MwejjDVEkIjsLFjUchtRm+YHZhFOzW5RYipBbtYuOi7yIhFKt +6og5GJ2D4iJ3BfFmXypoUN2IlVvbR2SSFUjLkaLZ3Eq5hfSOVOfzGFsa8M9jgD5o +kp+Ch5ZOKdG29RsG3Ik2oWrbpbmLexkOPGlgR6p/m96z+o7Yz1x7NTJXCJZrX+Jl +FkifufdoV4U0Ym31KAtYwAx94hoyx7fHBCHqKLbOe89A3csFJLKScDloHWjyDqRU +IxdlaIfrHo/D/q3dGOqfYg/rENBIsG+el1nWxfM/xGHjnJqmMoIYemAlR0Es6sMf +dIQYMbmL/za8KwF2euM97GKqqvseKeQgQqGFD3FEfxHaQ9WcnT288xgWHmlKmJss +wEGydZclen+sB3bNo/zswnSQ2rfYge9kZ479Ot9XgKEmc9Z3PTsZn5FtL2iWw77a +F5IYNdtRivQWHBCUq9EyXzBxAcEPctnqWz2pSfMx9+AKkGgRydGIxPPwY09FtJAE +2OAE98v3DkWWsSQDLzLi4dhENcTUpJ91cM0LqOsLXydd1hFlK76on2oFQEcZKNWo +gzPNsaBKhldwRiU9xy1ZEvVLSa8MZakyfGKtqnHf4gL2Tk64aWcTVMjchc05vau9 +6NwFzRh9eTXs9yatlujeJuXOGabGG96oVpISI5A9+vgNgAQcMlJ41tXP/kOl9VUy +ZmRUauFQpaO2YU/0TZKxkoRK+ddeBDT2mD9FTBC+C1SkGeakOZBQmp8RhBsDQJf7 +ahppMw0xl51FIANRmAByJf6PFCzAXaJP4s3EhlZJp7DoUDw4Ep3isR3NnMEJlEcx +gcByeZy4qcebf3E316U4LCuqGOOTg4LtgXQJTMRZeSzi2mnv1lma8sd5gi9Mm4P3 +WA0XDzNdgYfGdc2n4Gjh00TOfhNXLXRe3uX8KZ7nMaEzcXbA5SM0JPIBo9eSEDO9 +726pZpOmRXtLLQ08hxU+zBzfNBd9gui7SrSiPEG4asfqTTcrNb56z25MVYHMcsV/ +n0D3X1FXgHNgluBV+wjHZ42Bo3eZkkpLAXytoFkCCm1ZlhLN4y7AnFTegJ0Swxfu +uCXfIBjYwgy/Mcg4TG2p182YxmUhBIq1jPpPqUvt6GOkS4ZWL1VCIzWtDmy44Zyn +mjclo+Cmwtx9Xm1GQQ+vpPzuJEoylC/ahe0SlBzUvIExKwwyCXP53EVgwqbEXhMi +J+Ko/8CYU5Ok5wwcFVgPpPrj9lZBBhl/surLtb+7vvckwlPqS8u/MxQ2k7PbfK0n +fUuHAjyVNFJaTmrFqdFtWplvSHsYMewEF9AB1N+uhy2CEfMz1XG8I93zi2o4xF/s +I0GsUfriyGV5jjtJTBNyjAeDsjHO3k1K7GsSbvfL/Efb5p6ULHguiTYiOOwWwm7b ++Ew1QaWw0gwLSuqEf2qWgFtvKB2hWWUPtkScIa3qhH9GB5RbCy//odHzD/gJkZne +82QIhSOZ48GEucyUrzqAmz0K3yYH+H72f1xXBr2G4fr6+GZJyo5k3PFD8A2UpBFe +G08Y2zW77p7IzN+VOcgqpiprcrclYnvcuv08HpXLFeEC5Eav2n3Bq9QXoFD4Q73M +2/Zss9RffQgAAsPeNzqSqgfz8l0uyUAEOBQfpLdI1s1xTEiDTIUHS+UeEjN+Pbq+ +93hfiI4M4zc088j5aD1XilLoHT/xQKHDAsOJXRxNiYuly37JJkTsCnxdY9VKbLL5 +mHiw4rhSicjQOXm8Xja4KcPaWomUXUwVuCmWoJ1khkS0HGLmvgmBZRGS68V/5VoT +6xB2ePVpQ03Z+eZFS8QD2PNGbPRZJquIarLwipWuEUqp6e8KfFF7wP+MEYbi7MqV +gxx/uYvF/KtO7x6tkLS6JG3/PxW+r+gS7taWVpT23h6GEGdacabKngYpk2Qi5JsZ +4eCtKZXr+Q8E8IlvHDaUpW9fzglp8v4GYLV3hwWMq9ckDRqy5pfousKFZZXSZvn6 +NhWHwgZSle07wT0GUblwkHNbMhTAQK5D3zsXXCL/RLrReZF6bZw/YfTUAyIRWIHc +FjHKkgH0KhwhBLL61uG/9G5mXW7/PQXabCC6RiiM9WpViE7dSPxZWs6bTaSTKLMN +Q3tzo7xfAqhk5SOjUA0r4qNrnBl81HWAJLbaQZzdAuzU326sAAWza7vnPcW44SCv +KIhf/uDhrwz2aUDWBrnZWmaB6lc2IDV/kOVyGL3oNvmcO5qncdcX3+yPchiFwll6 +OqWWfH2AFImDl+jhov1ypyiTHCWbu7ZkXdCIhcG3H7Z1m0sYDNtlOS0cQlRc/nUi +F6GxJF6SlBxefsjCAj6/FMziwQAFcXpEHPZYDD2U9G7IS0gzcOFG2qZ+xTE6GvkZ +SNaee2Rlv7Y2DIvah6njoWWIk54UuAIAtZW0HGrXMuVpcJjQbjoSLTB9MMquzzPM +dekONiy5xZgC9dzp36XIe9a3sEcz6NpH4UNIZ29GGqe/7FCjJsSAUsTiCigS5q2a +R7w3vtdV1D2mtQzwnQXveRIvnS6PEaZ9+9LQhcEa+NDi5aAbfRN71m+MROaKIFKU +37JLB76WCFE2sEh6F2AoUvS2iyMJQOW3onqPKrkVX4MgYcofitXZ8mK3zO9nz4Rr +bcxWzZ6XfBIPQhUtwkltNCXCzTpHAjhslzXzq3Ul6UBlIaAB3iuSbL41DXz1ztWd +QZJ7QRBStqNm0nXHZzt35Tcmq5pIA/QaTHWwgVpKLBWrxIt0wDxfzWKKnBs0FmlW +uvpNVelSWLFMhi/u3nJCW7LpS5BrJSkI919yG8G7BOBpMLffnTk1z+8nG6l2C3ju +N1AnirS2W6SROBtJg/xO2DGlnIZqDUTTKM4CuLee6M2i0D10gZaJltdkLSC04vCB +qQZmAbGECIRnM3KJuqV8CkHu1Ar3npd7QTu0qq3RJ/mKJNWkYCiGq7sHDsvMtxLF +GKsewDn8RtSaG8Pk0ubxIvVUMYPXPfxDY6b7LiPpb/7jibpwqTIgkkcLdLHPcM+v +tj8U7G0xJhMiCILWLZa2+AuNaNGXwp5ISqIjv9nui827RTnxCKcSY+4ynnrqMU2h +zPHwowWx3fGSAbXGn2oPo7BI+lWimrFpMiSjW1AEteVlH19/ZMTSq6W9BI6xAhJM +IopZeouaA5Yga8ljxqAZBKPqTsUtEX047qgm50LQeljmNkox8Anf6sGElN7hrFIU +77qJf2wSpNS5red0AG4M/ENOWffvU7ybRxdPgpFAa7TbQFe2QwXEgW5DJTjTZeQP +n34S24MrEVMPAhN88GE41lTVYWQ2xwEJSHzwxDd0ZJDjkq/+cBPRYBoZTnuXpmdi +Uxg7go3n8tIkx9Azv/SXN43nCMdqnJuFgK+/dFMS5uyWPaBgGq/ZMdFPP/yK2ZBH +qqyTrdQ/uGGA/beqJAg1DfSG6dzXcTjNTL0PkGAsVeI+WVHQw3UsPtA3O0rMLX3R +Agv9RU4DM9pq0Q6yZL6X14r+tNqF+3vbRVGgXGxw9fVvZvH5XXkQCiqBJKA43RAs +FkEnk3CWIhyxR+XvVnQbBEX20oWf1nQ55iBETVLll6RiSgaN8boqwbzMR+plQoQH +isWjuezr76ZB+ViwS41WRrmDOSiC1BPWSm+SatBGIQY4v9AkzuU/GqSgLaQovlKe +m5oahjTMoXTKP489N8NXsNZ18dN3K5cLy/IOqg6T5NnaYVpQY4lRm7zXrASHF8K5 +oH9q3VZNdW8QkVw6csBB/2YmibmkqL6WTm3mvfMQQlsNYdt5J4qjdfMiuekUrz1e +v8IFS+aTK1Q3NFFtjL0tzJVHsiaVupGjTtKLNzNhlCQwFhI8t2wvSANyuCvgUEMV +WyKujFCj3HdH3mjJPqtfOGnSnxw1RVRtqG+N2ZKDntttXPFjC4UPQVeFV/UNP+o6 +K8JuvnTM8tgTLE3azDd6PtQLSgD0JIsXgUGjTJwYuYO50jhtCjje7JnaNrVe8t4q +c8Q58H5nR4n1Cr8XzU+uqAMyScZh0rTd4E9hw0udneXC6SJ2trs3xSMkEp7NDZ7A +ickHGsy16VcfKJ7JUKECwxVtwRNaC0t1HqJkbhUpUtJeFD4hQN6GY0NKiSlM7yVj +2PrNsUMxFoyNKf2RM8qiMn1iTUdEvxwBPXGFKW7OJbhmEGSGAmnywBDrGYsiLM18 +rt86YXutYzJOOsDhRxTbCKiWjTDz77UoxAgjCpniJ2KP77+04atY5Nkjcjo+izc6 +d0jgk/edZfjc+E4nKizLEj/xft/bRwG51CGqXQWla1nuoK2eV/a3SoW/eHcGVIB9 +wsShUCMtMpuYf87s2UXJ2oSJjlky4yy5yVWV0yLlc/goYjiNkyjz5YPHgvDVW3QT +NYtLQjFPlAYdU2kN/JWRpvSa7ayu7Qecu8v3/EvLXL9RCqmu/z9xrfGbMD6/N/Gn +hrwiM4MLDDyEcieKrMI5atsbCSdsxyhyCNNUC93S02u/IUbK/ddxj1Il2cAZBikv +FZV0h//OELvnz6BlkRtShjJrNs8nSgMJcnk9kTwbJsvkGhQEaXd447yODgxIPJKh +rJz7J+Q+NpJKaR5vqeMKDLlHdjVhUo1KnzMWgbG2rbytStYGI80wrVhOqfvQZhwD +TpQ6bvAUko5VD6jAgMKSsMB6XWB/dV2VFAKA4JlI/94FLpusc2Q3hF9bKJ21yZaN +hrtBC7Lvh141lS8KqBYmFK+aSbsA/4ZD0whX3Son1teUk7oLswaNap/pQEVN1lHJ +K+85ZlbacuYq7dSZjyDbfLqEDfPAF63CMTx4koQyWUde8qtKmj18jPn9k16TNLRS +SQojYfUylXF7nhmKcxfd3h/JV4kVqZmhPyCuTeaJziF5rXiDpXjtOnf5xPbW5i+C +XEufLT7SfymXA6bRymtLNiZJr2430JT0NL5g26N6zUkkttlMwPLPrxqtXGgchSiA +IhAqbA3hw22SgesZ03+1ggQSrp8mrvTZ8yzsoFNjfLC4Q9yPVlV7ios7Y4jm1Ack +l3tOXC3zUTbFjFn0mKlIrmnfYDQlvXXwKQwGA/Uy8bf/5/3qPGYLXePs5yfFinUy +BB83FSb4cZEcMrKDjk9OhdqKK98mMl3YnvKY7IuqsGRivpxfWkhsMCZl86lMiL+5 +2o2bxD5aAqnEh4pGTxtKv71Ez+vxLNt0ji8vXwI3ylceelPX48mfd9Jye6lOSqtz +2P+QQnM7BJLcPSTXZ9HIzqsALebdXjWUv1E9Dn19PBkSPOJOUkE+3DWXL4BUDfTb +Ubr27BX3cpsrMZChHhUBvnXXssF+s6V++8DKRs4DyM5tGOlu/7QIw4ncGs0NOun/ +xaOggRHjUqFFwcntYKU1DCA+hR/jt95asgG9m9Dip2Xkaeo0epLFw2cNHclGPc7H +o3zXtxaNUKRV1koFcTYOlYd9uEwU3P81zcsWK/vuxOBxa/AgherK5MtN/+MD/huN +Y/GRcopMaXrWFSHetXEjPn9kiZ7pkiq+1y/djSjWSEe/rS9hkBtQHAmpW/te+B0r +KszygiZ8unV6Xy1qA7ufMQrQc1TySa7vX5pSRIU7nvaU/TPVuwbFBl/ePopkS2bz +yT5JwZUg+YSP+9VNk/TCfLLZf/nuNPdVqG5Z3QYM1TN8alTdTJs1z0vF20k2CUIS +AafmJHSvm+2c91Tq/el9Ymo52jvI0v3rZa853N6+G4LtQOexWRL88TpWQfN0xmOk +X95iWrRfuo8cu6g8OS07eMRj1M0x5zUU4c2zsoVNf8VtEq27V+OcSp/7jp8tK4Fv +FeJEB8om74zFwxfvDorLBUV2L2bI8Jt/m4vvgBI/Tx/qD8KC0bpiWBHEy5hWcpOo +n5T9jXlM8WSTGb2PjW06p6PnBHtkOFOoXzYqWQkCuLZ8UzJyMcOe0GOedBmZOqVz +V9xJDmOsSZsV4Bz4iztsg13lTnl7/EGZYu4JjgWELF1rqWSSApSISzzqr1xID+VM +MAK23K/pbZt56TVSDyIaRDIIuH7gRWiWBpeyMct/dT9pjMvlktZkE0/v4Jn9r4pT +hVIsHJig3er2VZ1AjNuNJNa7FWgGtuv0Kr4IDXkNSGh7k0uHH+g0HQDvDTUSBIMS +3CR3TVOsNoZw3wQ7DnJSrrsVwk3XXrXfUhImeZR9HOG6xXvd15V/f+jQ2wUj+iqo +JrvnAD90DVgYrY8g6AKN8FC+rwvKKMmptkcebhWdXn0vC0VUUcqpUXDwz2be+8TV +ZQr2DJM5PuY220ZwQ/6yPorBzbAo411Dnvq9cC882esgE4BRkRSPJyQgx1OD5AAY +LKE5zPrFtAhcqoEYnuRFpx4g4UVyldDyyLU1Yur4dOXcRkaIXRSq2VuetVeb/99u +cKG8FFqQNNcB1GbR8Et6Lz5ESobLk1gymiTeb5lk8kUF0nmP0wN/6C3dMNrJR/s+ +snZCmmYUJi3bvtHJtFpKurZLmNN8a6slI0UYVudC4yHCaI2T98aUHkBxL6gN7+LD +UC3H/b0ZJ0OqfkqzI6twnEVXt2IBf6JxiY3zOUfKbZjXAXgeP6GfHBg2TMvdG9Zx +ot5/icSekzlNQANZME8kMtyYh36wLPSrS9H3pkbO7kt5jsZTXqQClint05U2n9w+ +OzudrFIp/UlN+CUzhakqikexq/vCFFPabs+rpIBhocmKcDSg+Vh9caATCqcS0cm7 +hoCmd0a48bZHwKnjTp8OvW4NY16y3ywe6fF2ezd9YQ6NsHzfDVqAHBr+F1WhDsez +Y5wON/0cJul9Jeg923zEPT9THmeBeJg9tNx89IAouhBjQKDBmShJ+B0R9sIOSoKK +jbTMlrQlMAl1CdDy9udIhBfJyNLHcNaHt/cFnzYwBa0eGxq3/p8MtrulReHMThnS +4Yok6Qe40fjDPPUqCG52lD4xKPIRyRf5wdIFw+69L2j7WP4a6FBwMf0P1qZZitCf +5jieeU0iHRWnoBWzfgvb0yTvP0Kno0fupF3pWTCYTP/SmVPvbYYhbeZqV3Wj7YgN +Xgfb++Ag74ovmNFRg5Xy2NRmXlHDU++nGIK11QGQr+0v8poQbIVMaNNXQxRqjNJQ +6yd7ODzfULIEsPD46WacHcdLO0ZqBB2pEKG7CXH1rPFmPXddXdFlX88HZmrh1hpH +zptCAtujXY8fO/Us69rFSKQU9DMux7F3wCLzpmqxp1s1wXy7IKyzwmvlTDIgzLSp +OVdQLq/fmzXY/A2cJtYpsI4BoTLzN7XWpyxVf/WaGjltu6tGMOdcEIiTnWhrKFjF +hjGuf2+PXZI7aFLCYnJTjqJEoSuf9qlPPFMYxRIGEZg8E9aC+ZFQAFqo6hA7RIxL +FBN/+T3fE+CA/tx1Rgyf8GDVdlg5DBPR7711p9wkjsLiH+oygTvVorn0fkqop1oc +9FzC60Y/mAuKJB4U69nFaBTLD4jfjfLaKNsrfwJbucMr1XQKC8kYqmSKZAwf8FhK +yUo6gap33wp3bqbCqPzRzli9kPYLghHFdIZsAnkXabdpDRWZPiIKTWCTPQfZaSbi +9U5NtQC70Dbos9wMtju5dGv5B9WGiQNMkep40fmy8UeM/8KoCyWVUx95nmL1Ysb/ +CGVA650iSGbWyKwYkrJtPM+eBdO+iQ4whaq+JI4vfDHGnDRNENHynhhqzVS/ATiJ +G/PNlIDMsLzpSUzyx4HlW+H5m41GUjn9v/p52LISfQSDkvXK1SYIIC0C9Yxo/GgW +6zS3/ioeLt/4JTxXQVMpd/3ZDxqtGtr4nhhYqSI8/xxslxgRS6t54jvu +-----END AGE ENCRYPTED FILE----- diff --git a/CHANGELOG.md b/CHANGELOG.md index 82fee3b..066c32c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to this project will be documented in this file. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Added +- `target_error` failure mode: transport failures (network error, non-2xx, malformed or non-JSON response, timeout) are now classified separately from the behavioral `no_tool_call` mode, so an agent outage is no longer counted as locale drift. +- `--timeout MS` flag on `run` (default 30000); a hung locale is recorded as `target_error` instead of stalling the whole run. +- `noToolCall` can forbid multiple tools via `anyOf: [a, b]`. +- `translate` now emits `responseLanguage` for the target locale (when the source asserts one) and warns when the model returns fewer locales than requested. +- `lint` flags duplicate scenario ids across a directory and `responseLanguage` values whose script cannot be determined (the check can never fail). +- The CLI JSON report includes a 95% Wilson confidence interval per locale. + +### Changed +- `responseLanguage`: `ja` now requires kana so pure-Chinese text no longer passes; the non-Latin detector covers every script in the table (including Georgian and Ethiopic); the measured in-script ratio is included in the `detail`. +- Directory input always emits the matrix report shape, even for a single file. +- Markdown run report includes a `Detail` column; the matrix highlights failing cells instead of passing ones and escapes `|` in cell values. +- The `agent:` field is now sent in the POST body to the target as routing metadata. +- Minimum Node version relaxed to `>=22`. + +### Fixed +- Scalar argument assertions no longer let a non-scalar (array/object/null) pass via `String()` coercion. +- Scenario parser: `oneOf` items with quoted commas parse correctly; duplicate locale keys, nameless tool-call list items, tab/odd indentation, and block scalars are rejected with line-numbered errors; an empty `noToolCall:` no longer absorbs a sibling's `name:`. +- `translate` serializes `oneOf` matchers correctly instead of `[object Object]`. +- `--allow-fail` warns when a value matches no locale; `-v`/`-h` no longer hijack a `run` invocation; the target URL is validated. +- CI now builds the compiled artifact and runs it. + ## [0.3.1] - 2026-06-25 ### Fixed diff --git a/README.md b/README.md index 54fb48c..24a0810 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ locales: responseLanguage: fr ``` +Scenario files are a **strict 2-space-indented subset of YAML**, not full YAML. Use exactly two spaces per level (no tabs), and keep every value on one line — block scalars (`input: |`), flow mappings, and multi-line strings are not supported. Out-of-subset input is rejected with a line-numbered error rather than parsed loosely. Run `langdrift lint` to catch these and other issues early. + Run it against your agent: ```bash @@ -150,9 +152,25 @@ expect: `oneOf` is an inline list and must contain at least one value; `langdrift lint` reports an error otherwise. +### Forbidden tools + +`noToolCall` fails the locale if the agent calls a tool it should not. Forbid one tool with `name`, or several with `anyOf`: + +```yaml +expect: + toolCall: + name: create_refund_ticket + noToolCall: + anyOf: [escalate_to_human, contact_seller] +``` + ### Response script -`responseLanguage` is a **script-family check**, not language detection. It confirms a reply uses the script a locale is written in (for example, that an `ar` reply is in Arabic script). It cannot distinguish languages that share a script: a `fr` assertion passes for any Latin-script reply, and `ar` cannot be told apart from `fa` or `ur`. For a locale whose script LangDrift cannot determine, the check passes rather than guessing. +`responseLanguage` is a **script-family check**, not language detection. It confirms a reply uses the script a locale is written in (for example, that an `ar` reply is in Arabic script). Know its limits before relying on it: + +- **It cannot distinguish languages that share a script.** A `fr` assertion passes for any Latin-script reply; `ar` cannot be told apart from `fa` or `ur`; `zh` accepts any Han text. The one Han exception is `ja`, which additionally requires kana, so pure-Chinese text does not pass `responseLanguage: ja`. +- **The thresholds are asymmetric.** A non-Latin locale passes when at least 10% of letters are in its script; a Latin locale fails only when more than 50% of letters are non-Latin. The measured ratio is included in the failure/pass `detail` so near-misses are visible. +- **For a locale whose script LangDrift cannot determine, the check passes** rather than guessing. `langdrift lint` warns when a `responseLanguage` value is not script-determinable, since the check can then never fail. ## HTTP Target Contract @@ -164,10 +182,13 @@ Request: { "locale": "fr", "input": "J'ai été facturé deux fois. Pouvez-vous me rembourser un paiement?", - "scenarioId": "refund_request" + "scenarioId": "refund_request", + "agent": "support" } ``` +`agent` is the scenario's `agent:` field, sent as routing metadata; agents that serve one workflow can ignore it. + Response: ```json @@ -209,8 +230,11 @@ langdrift translate [--locales fr,ar,zh,...] [--write] Useful CI flags: - `--min-pass-rate N`: fail only if the overall pass rate is below `N`. -- `--allow-fail `: keep reporting a known weak locale without letting it fail the build. +- `--allow-fail `: keep reporting a known weak locale without letting it fail the build. A value that matches no locale prints a warning. - `--format markdown`: write a table suitable for GitHub Actions summaries or PR comments. +- `--timeout MS`: per-request timeout (default 30000). A hung locale is recorded as `target_error` instead of stalling the run. + +A transport failure (network error, non-2xx, malformed or non-JSON response, or timeout) is classified as `target_error`, distinct from the behavioral `no_tool_call` mode, so an agent outage is not mistaken for locale drift. See [docs/ci.md](docs/ci.md) for GitHub Actions examples. @@ -246,7 +270,8 @@ langdrift run ./examples/scenarios/support-routing.yaml --target http://127.0.0. - **Behavior over text.** LangDrift checks tool calls and structured behavior, not whether a reply sounds fluent. - **Deterministic assertions first.** No LLM-as-judge in the core loop; failures are explainable and CI-friendly. - **HTTP contract over framework lock-in.** Any agent that can accept one POST request can be tested. -- **Small, inspectable core.** Zero runtime dependencies, TypeScript source, Node >= 24. +- **Small, inspectable core.** Zero runtime dependencies, TypeScript source, Node >= 22. +- **CLI, not a library.** The published package exposes the `langdrift` command only; there is no importable JavaScript API. To reuse the internals, work from a clone of the TypeScript source. - **Demo without API keys.** The fake agent makes the failure mode visible locally before connecting a real model. ## More Context diff --git a/RESEARCH.md b/RESEARCH.md index 4099e15..dab7e94 100644 --- a/RESEARCH.md +++ b/RESEARCH.md @@ -16,6 +16,8 @@ I built a minimal eval harness and ran scenarios across 3 domains and 12 locales - 10 iterations per scenario for each model - English baseline confirmed before any other locale was tested +In the tables below, **Failing locale checks** lists *every* cell below 10/10, ordered worst first (fewest passes first); a cell not listed passed all 10 iterations. Per-cell 95% Wilson confidence intervals for every failing cell are in the [appendix](#appendix-per-cell-confidence-intervals). + **Results — gpt-4o-mini (10 iterations × 12 locales):** | Scenario | Pass rate | Failing locale checks | @@ -31,22 +33,22 @@ I built a minimal eval harness and ran scenarios across 3 domains and 12 locales | Scenario | Pass rate | Failing locale checks | | -------- | --------- | ------------------- | -| support-routing | 59% (71/120) | mn (1/10), sw (3/10), yo (3/10), vi (4/10), cy (5/10), eu (5/10), zh (6/10) | -| support-cancel-subscription | 68% (82/120) | yo (0/10), sw (1/10), mn (2/10), eu (4/10), cy (6/10) | -| ecommerce-cancel-order | 69% (83/120) | yo (0/10), cy (1/10), eu (2/10), zh (6/10), mn (6/10) | -| ecommerce-track-order | 46% (55/120) | mn (0/10), cy (1/10), eu (1/10), en (3/10), sw (4/10) | -| scheduling-reschedule | 88% (106/120) | sw (4/10), eu (7/10), mn (8/10) | -| scheduling-book-new | 40% (48/120) | id (0/10), sw (0/10), cy (0/10), fr (2/10), ar (4/10) | +| support-routing | 59% (71/120) | mn (1/10), sw (3/10), yo (3/10), vi (4/10), cy (5/10), eu (5/10), zh (6/10), id (7/10), ru (8/10), ar (9/10) | +| support-cancel-subscription | 68% (82/120) | yo (0/10), sw (1/10), mn (2/10), eu (4/10), cy (6/10), zh (9/10) | +| ecommerce-cancel-order | 69% (83/120) | yo (0/10), cy (1/10), eu (2/10), zh (6/10), mn (6/10), vi (9/10), sw (9/10) | +| ecommerce-track-order | 46% (55/120) | mn (0/10), cy (1/10), eu (1/10), zh (2/10), en (3/10), sw (4/10), id (5/10), fr (7/10), ar (8/10), ru (8/10), vi (8/10), yo (8/10) | +| scheduling-reschedule | 88% (106/120) | sw (4/10), eu (7/10), mn (8/10), en (9/10), zh (9/10), cy (9/10) | +| scheduling-book-new | 40% (48/120) | id (0/10), sw (0/10), cy (0/10), fr (2/10), ar (4/10), vi (4/10), eu (4/10), yo (4/10), zh (6/10), mn (7/10), ru (8/10), en (9/10) | **Results — DeepSeek deepseek-chat (10 iterations × 12 locales):** | Scenario | Pass rate | Failing locale checks | | -------- | --------- | ------------------- | -| support-routing | 84% (101/120) | mn (1/10), cy (2/10), sw (9/10), zh (9/10) | -| support-cancel-subscription | 62% (74/120) | zh (0/10), eu (0/10), sw (1/10), id (2/10), ar (3/10) | -| ecommerce-cancel-order | 57% (69/120) | zh (0/10), ru (0/10), eu (0/10), yo (0/10), sw (3/10), en (8/10) | -| ecommerce-track-order | 42% (50/120) | ar (0/10), zh (0/10), vi (0/10), sw (0/10), fr (1/10), yo (1/10), mn (3/10) | -| scheduling-reschedule | 64% (77/120) | ar (0/10), zh (0/10), sw (0/10, wrong tool), id (1/10), mn (8/10) | +| support-routing | 84% (101/120) | mn (1/10, wrong_tool), cy (2/10, wrong_tool), sw (9/10, wrong_tool), zh (9/10) | +| support-cancel-subscription | 62% (74/120) | zh (0/10), eu (0/10), sw (1/10), id (2/10), ar (3/10), cy (9/10), yo (9/10) | +| ecommerce-cancel-order | 57% (69/120) | zh (0/10), ru (0/10), eu (0/10), yo (0/10), sw (3/10), en (8/10), cy (9/10), mn (9/10) | +| ecommerce-track-order | 42% (50/120) | ar (0/10), zh (0/10), vi (0/10), sw (0/10), fr (1/10), yo (1/10), mn (3/10), id (6/10), eu (9/10) | +| scheduling-reschedule | 64% (77/120) | ar (0/10), zh (0/10), sw (0/10, wrong_tool), id (1/10), mn (8/10), eu (9/10), yo (9/10) | | scheduling-book-new | 93% (112/120) | eu (2/10) | English is not a perfect baseline on every model. It passes every scenario on gpt-4o-mini except the model-behavior divergence in `scheduling-book-new`, while claude-haiku misses `ecommerce-track-order` in 7/10 runs and DeepSeek misses `ecommerce-cancel-order` in 2/10 runs. That matters: LangDrift surfaces both locale drift and scenario/model reliability issues. @@ -57,7 +59,7 @@ This is an applied experiment, not a scientific claim. **Three models, one architecture.** The benchmark now covers gpt-4o-mini, claude-haiku-4-5-20251001, and DeepSeek deepseek-chat via the same HTTP agent wrapper with the same system prompt and tool set. Cross-model patterns (Basque, Yoruba, low-resource language clusters) are therefore more credible than when a single model was used. However, the agent architecture is still simple: single-turn, 5 tools per domain, no RAG, no multi-turn context. More complex setups may show different failure patterns. -**Small sample, reported with uncertainty.** Each scenario/model/locale cell uses 10 iterations. The agent runs at `temperature 0`, so these iterations are near-deterministic: they capture API-side variance, not a sampling distribution. N=10 is enough to expose repeated failure patterns but is not a large-sample statistical benchmark, and a single 7/10-vs-9/10 difference is well within noise. The benchmark report now prints a 95% Wilson confidence interval per locale, and per-cell pass rates throughout this document should be read as estimates with that uncertainty, not exact rankings. +**Small sample, reported with uncertainty.** Each scenario/model/locale cell uses 10 iterations. The agent runs at `temperature 0`, so these iterations are near-deterministic: they capture API-side variance, not a sampling distribution. N=10 is enough to expose repeated failure patterns but is not a large-sample statistical benchmark, and a single 7/10-vs-9/10 difference is well within noise. The [appendix](#appendix-per-cell-confidence-intervals) gives a 95% Wilson confidence interval for every failing cell, computed directly from the committed pass counts (the CLI's own `--format json` report also emits these intervals per locale). Per-cell pass rates throughout this document should be read as estimates with that uncertainty, not exact rankings. **Unreviewed locale prompts.** The locale inputs were written by one author to preserve intent but were not reviewed by native speakers. Some failures may reflect phrasing gaps rather than model behavior. This is acknowledged as a real threat to validity, but native review at scale is not practical for a solo project. Results should be interpreted with that caveat explicitly in mind. @@ -144,3 +146,128 @@ LangDrift is the harness I used for this experiment, cleaned up and made general - Exits non-zero on failure, so it works in CI The goal is to let any team run localized behavior checks against their own agents, not just refund routing, but any workflow where the right behavior matters across languages. + +## Appendix: per-cell confidence intervals + +95% Wilson score intervals for every failing cell (any cell below 10/10), computed from the committed pass counts in `examples/benchmark/results/`. Cells not listed passed all 10 iterations; a 10/10 cell has the interval [0.72, 1.00] at N=10. These are presentation over the same committed data, not a rerun. + +### gpt-4o-mini + +| Scenario | Locale | Pass | 95% CI | +| -------- | ------ | ---- | ------ | +| ecommerce-cancel-order | eu | 0/10 | [0.00, 0.28] | +| ecommerce-track-order | — | — | — | +| scheduling-book-new | en | 0/10 | [0.00, 0.28] | +| | fr | 0/10 | [0.00, 0.28] | +| | ar | 0/10 | [0.00, 0.28] | +| | zh | 0/10 | [0.00, 0.28] | +| | ru | 0/10 | [0.00, 0.28] | +| | id | 0/10 | [0.00, 0.28] | +| | vi | 0/10 | [0.00, 0.28] | +| | sw | 0/10 | [0.00, 0.28] | +| | cy | 0/10 | [0.00, 0.28] | +| | eu | 0/10 | [0.00, 0.28] | +| | mn | 0/10 | [0.00, 0.28] | +| | yo | 0/10 | [0.00, 0.28] | +| scheduling-reschedule | eu | 0/10 | [0.00, 0.28] | +| support-cancel-subscription | — | — | — | +| support-routing | — | — | — | + +### claude-haiku-4-5-20251001 + +| Scenario | Locale | Pass | 95% CI | +| -------- | ------ | ---- | ------ | +| ecommerce-cancel-order | yo | 0/10 | [0.00, 0.28] | +| | cy | 1/10 | [0.02, 0.40] | +| | eu | 2/10 | [0.06, 0.51] | +| | zh | 6/10 | [0.31, 0.83] | +| | mn | 6/10 | [0.31, 0.83] | +| | vi | 9/10 | [0.60, 0.98] | +| | sw | 9/10 | [0.60, 0.98] | +| ecommerce-track-order | mn | 0/10 | [0.00, 0.28] | +| | cy | 1/10 | [0.02, 0.40] | +| | eu | 1/10 | [0.02, 0.40] | +| | zh | 2/10 | [0.06, 0.51] | +| | en | 3/10 | [0.11, 0.60] | +| | sw | 4/10 | [0.17, 0.69] | +| | id | 5/10 | [0.24, 0.76] | +| | fr | 7/10 | [0.40, 0.89] | +| | ar | 8/10 | [0.49, 0.94] | +| | ru | 8/10 | [0.49, 0.94] | +| | vi | 8/10 | [0.49, 0.94] | +| | yo | 8/10 | [0.49, 0.94] | +| scheduling-book-new | id | 0/10 | [0.00, 0.28] | +| | sw | 0/10 | [0.00, 0.28] | +| | cy | 0/10 | [0.00, 0.28] | +| | fr | 2/10 | [0.06, 0.51] | +| | ar | 4/10 | [0.17, 0.69] | +| | vi | 4/10 | [0.17, 0.69] | +| | eu | 4/10 | [0.17, 0.69] | +| | yo | 4/10 | [0.17, 0.69] | +| | zh | 6/10 | [0.31, 0.83] | +| | mn | 7/10 | [0.40, 0.89] | +| | ru | 8/10 | [0.49, 0.94] | +| | en | 9/10 | [0.60, 0.98] | +| scheduling-reschedule | sw | 4/10 | [0.17, 0.69] | +| | eu | 7/10 | [0.40, 0.89] | +| | mn | 8/10 | [0.49, 0.94] | +| | en | 9/10 | [0.60, 0.98] | +| | zh | 9/10 | [0.60, 0.98] | +| | cy | 9/10 | [0.60, 0.98] | +| support-cancel-subscription | yo | 0/10 | [0.00, 0.28] | +| | sw | 1/10 | [0.02, 0.40] | +| | mn | 2/10 | [0.06, 0.51] | +| | eu | 4/10 | [0.17, 0.69] | +| | cy | 6/10 | [0.31, 0.83] | +| | zh | 9/10 | [0.60, 0.98] | +| support-routing | mn | 1/10 | [0.02, 0.40] | +| | sw | 3/10 | [0.11, 0.60] | +| | yo | 3/10 | [0.11, 0.60] | +| | vi | 4/10 | [0.17, 0.69] | +| | cy | 5/10 | [0.24, 0.76] | +| | eu | 5/10 | [0.24, 0.76] | +| | zh | 6/10 | [0.31, 0.83] | +| | id | 7/10 | [0.40, 0.89] | +| | ru | 8/10 | [0.49, 0.94] | +| | ar | 9/10 | [0.60, 0.98] | + +### DeepSeek deepseek-chat + +| Scenario | Locale | Pass | 95% CI | +| -------- | ------ | ---- | ------ | +| ecommerce-cancel-order | zh | 0/10 | [0.00, 0.28] | +| | ru | 0/10 | [0.00, 0.28] | +| | eu | 0/10 | [0.00, 0.28] | +| | yo | 0/10 | [0.00, 0.28] | +| | sw | 3/10 | [0.11, 0.60] | +| | en | 8/10 | [0.49, 0.94] | +| | cy | 9/10 | [0.60, 0.98] | +| | mn | 9/10 | [0.60, 0.98] | +| ecommerce-track-order | ar | 0/10 | [0.00, 0.28] | +| | zh | 0/10 | [0.00, 0.28] | +| | vi | 0/10 | [0.00, 0.28] | +| | sw | 0/10 | [0.00, 0.28] | +| | fr | 1/10 | [0.02, 0.40] | +| | yo | 1/10 | [0.02, 0.40] | +| | mn | 3/10 | [0.11, 0.60] | +| | id | 6/10 | [0.31, 0.83] | +| | eu | 9/10 | [0.60, 0.98] | +| scheduling-book-new | eu | 2/10 | [0.06, 0.51] | +| scheduling-reschedule | ar | 0/10 | [0.00, 0.28] | +| | zh | 0/10 | [0.00, 0.28] | +| | sw | 0/10 | [0.00, 0.28] | +| | id | 1/10 | [0.02, 0.40] | +| | mn | 8/10 | [0.49, 0.94] | +| | eu | 9/10 | [0.60, 0.98] | +| | yo | 9/10 | [0.60, 0.98] | +| support-cancel-subscription | zh | 0/10 | [0.00, 0.28] | +| | eu | 0/10 | [0.00, 0.28] | +| | sw | 1/10 | [0.02, 0.40] | +| | id | 2/10 | [0.06, 0.51] | +| | ar | 3/10 | [0.11, 0.60] | +| | cy | 9/10 | [0.60, 0.98] | +| | yo | 9/10 | [0.60, 0.98] | +| support-routing | mn | 1/10 | [0.02, 0.40] | +| | cy | 2/10 | [0.06, 0.51] | +| | sw | 9/10 | [0.60, 0.98] | +| | zh | 9/10 | [0.60, 0.98] | diff --git a/docs/ci.md b/docs/ci.md index 8aa7844..c1f6ba5 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -75,8 +75,11 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # Wait on the TCP port, not the POST endpoint: the agent route only + # answers POST, so an HTTP GET probe against it 404s and times out even + # when the agent is up. Use tcp: (or a GET health route if your agent has one). - name: Wait for agent - run: npx wait-on http://127.0.0.1:3010/api/agent --timeout 15000 + run: npx wait-on tcp:127.0.0.1:3010 --timeout 15000 - name: Run locale eval run: | diff --git a/docs/integrations.md b/docs/integrations.md index 12e404b..195ae06 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -171,4 +171,4 @@ app.listen({ port: 3010 }); **`scenarioId` is informational.** LangDrift sends it so your agent can log or branch on it. Most adapters can ignore it. -**Non-2xx responses fail the locale.** If your handler throws or returns a non-2xx status, LangDrift records it as `no_tool_call` with the HTTP status as the detail. +**Non-2xx responses fail the locale.** If your handler throws or returns a non-2xx status, LangDrift records it as `target_error` (a transport failure, kept distinct from behavioral modes like `no_tool_call`) with the HTTP status as the detail. A malformed or non-JSON 2xx body is also a `target_error`. diff --git a/examples/agent/server.ts b/examples/agent/server.ts index 92dbdd3..dc39d16 100644 --- a/examples/agent/server.ts +++ b/examples/agent/server.ts @@ -25,6 +25,13 @@ const apiKey = resolveApiKey(provider); const tools = loadTools(domain); const server = createServer(async (request, response) => { + // Readiness probe: a GET that returns 2xx so `wait-on http://.../health` + // succeeds. The agent endpoint only answers POST, so probing it 404s (F-32). + if (request.method === "GET" && request.url === "/health") { + sendJson(response, 200, { status: "ok" }); + return; + } + if (request.method !== "POST" || request.url !== "/api/agent") { sendJson(response, 404, { error: "not found" }); return; diff --git a/examples/benchmark/run.ts b/examples/benchmark/run.ts index 2a470ba..915b7ae 100644 --- a/examples/benchmark/run.ts +++ b/examples/benchmark/run.ts @@ -3,6 +3,8 @@ import { spawn } from "node:child_process"; import { once } from "node:events"; import { writeFile } from "node:fs/promises"; import { basename } from "node:path"; +import { wilsonInterval } from "../../src/stats.ts"; +import { FAILURE_MODES } from "../../src/types.ts"; const scenarioPath = process.env.SCENARIO ?? "./examples/scenarios/support-routing.yaml"; @@ -55,16 +57,32 @@ try { const startedAt = performance.now(); const run = await runLangDrift(); const durationMs = Math.round(performance.now() - startedAt); - const { summary, log } = parseJsonRun(run.stdout, run.stderr); + // A single malformed run shouldn't discard every completed iteration; skip + // it with a warning and keep going (F-16). + let parsed: { summary: Record; log: string }; + try { + parsed = parseJsonRun(run.stdout, run.stderr); + } catch (error) { + console.error( + `Iteration ${i + 1} produced unparseable output, skipping: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + continue; + } runs.push({ iteration: i + 1, durationMs, exitCode: run.exitCode, - output: log, - summary, + output: parsed.log, + summary: parsed.summary, }); } + if (runs.length === 0) { + throw new Error("No benchmark iterations produced parseable output."); + } + const report = renderReport(runs); const slug = basename(scenarioPath).replace(/\.yaml$/, ""); const modelSlug = modelName.replace(/[^a-z0-9]+/gi, "-").toLowerCase(); @@ -77,11 +95,21 @@ try { console.error(`\nResults written to ${outPath}`); } finally { server.kill("SIGINT"); + // `server.killed` only means a signal was delivered, not that the process + // exited, so it can't gate the SIGKILL fallback. Race the real exit event + // against a timeout and force-kill if the process is still running (F-16). + let exited = false; + const exitPromise = once(server, "exit").then(() => { + exited = true; + }); await Promise.race([ - once(server, "exit"), + exitPromise, new Promise((resolve) => setTimeout(resolve, 3000)), ]); - if (!server.killed) server.kill("SIGKILL"); + if (!exited) { + server.kill("SIGKILL"); + await exitPromise; + } } async function waitForServer(): Promise { @@ -164,27 +192,6 @@ function parseJsonRun( return { summary, log: logLines.join("\n") }; } -// Wilson score interval for a binomial proportion. Reported per locale so -// per-cell pass rates are read as estimates with uncertainty, not exact facts. -// Runs are near-deterministic at temperature 0, so this reflects API-side -// variance over N iterations, not a sampling distribution. -function wilsonInterval( - passes: number, - total: number, - z = 1.96, -): [number, number] | null { - if (total === 0) return null; - const p = passes / total; - const denom = 1 + (z * z) / total; - const center = (p + (z * z) / (2 * total)) / denom; - const margin = - (z / denom) * - Math.sqrt((p * (1 - p)) / total + (z * z) / (4 * total * total)); - const low = Math.max(0, center - margin); - const high = Math.min(1, center + margin); - return [low, high]; -} - function formatCi(passes: number, total: number): string { const ci = wilsonInterval(passes, total); if (!ci) return "—"; @@ -198,14 +205,9 @@ function renderReport(runs: BenchmarkRun[]): string { } const locales = Array.from(localeSet); - const failureModes = [ - "no_tool_call", - "wrong_tool", - "wrong_argument", - "missing_argument", - "forbidden_tool", - "wrong_sequence", - ] as const; + // Derived from the single FailureMode source of truth so a newly added mode + // (wrong_language, target_error) can never silently vanish from the table (F-12). + const failureModes = FAILURE_MODES; type LocaleStats = { passes: number; diff --git a/examples/scenarios/scheduling-book-new.yaml b/examples/scenarios/scheduling-book-new.yaml index e70834a..b6300e6 100644 --- a/examples/scenarios/scheduling-book-new.yaml +++ b/examples/scenarios/scheduling-book-new.yaml @@ -101,7 +101,7 @@ locales: name: reschedule_appointment yo: - input: "Ẹ káàárọ̀, mo jẹ́ oníbàárà tuntun àti pé mo fẹ́ ṣètò ìpàdé àkọ́kọ́ mi ní òwúrọ̀ ọjọ́ Àìkú." + input: "Ẹ káàárọ̀, mo jẹ́ oníbàárà tuntun àti pé mo fẹ́ ṣètò ìpàdé àkọ́kọ́ mi ní òwúrọ̀ ọjọ́ Ajé." expect: toolCall: name: check_availability diff --git a/package.json b/package.json index a195151..0f940ab 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "examples/scenarios/", "examples/benchmark/results/", "RESEARCH.md", + "CHANGELOG.md", "LICENSE" ], "bin": { @@ -57,7 +58,7 @@ "benchmark:all:anthropic": "ITERATIONS=10 MODEL_PROVIDER=anthropic MODEL_NAME=claude-haiku-4-5-20251001 pnpm benchmark:support && ITERATIONS=10 MODEL_PROVIDER=anthropic MODEL_NAME=claude-haiku-4-5-20251001 pnpm benchmark:support-cancel && ITERATIONS=10 MODEL_PROVIDER=anthropic MODEL_NAME=claude-haiku-4-5-20251001 pnpm benchmark:ecommerce && ITERATIONS=10 MODEL_PROVIDER=anthropic MODEL_NAME=claude-haiku-4-5-20251001 pnpm benchmark:ecommerce-track && ITERATIONS=10 MODEL_PROVIDER=anthropic MODEL_NAME=claude-haiku-4-5-20251001 pnpm benchmark:scheduling && ITERATIONS=10 MODEL_PROVIDER=anthropic MODEL_NAME=claude-haiku-4-5-20251001 pnpm benchmark:scheduling-book" }, "engines": { - "node": ">=24" + "node": ">=22" }, "devDependencies": { "@biomejs/biome": "^2.4.15", diff --git a/scripts/release.sh b/scripts/release.sh index d966d14..dfdd046 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -26,6 +26,14 @@ if [ "$BEHIND" != "0" ]; then exit 1 fi +# Being level-with-or-behind isn't enough: unpushed local commits would ride +# into the tag from an unpublished state. Require the tree to be pushed first. +AHEAD=$(git rev-list origin/main..HEAD --count) +if [ "$AHEAD" != "0" ]; then + echo "Branch is $AHEAD commit(s) ahead of origin/main — push first" + exit 1 +fi + pnpm lint pnpm typecheck pnpm test @@ -56,7 +64,15 @@ npm version "$BUMP" --no-git-tag-version VERSION=$(node -p "require('./package.json').version") TAG="v$VERSION" -git add package.json +# The changelog ships in the package (see package.json "files"); make sure it +# actually documents this version before tagging. +if ! grep -q "$VERSION" CHANGELOG.md; then + echo "CHANGELOG.md has no entry for $VERSION — add one before releasing. Reverting version bump." + git checkout package.json + exit 1 +fi + +git add package.json CHANGELOG.md git commit -m "$TAG" git tag "$TAG" git push && git push --tags diff --git a/src/assertions.ts b/src/assertions.ts index aa24bc3..51e949f 100644 --- a/src/assertions.ts +++ b/src/assertions.ts @@ -9,29 +9,43 @@ import type { // responseLanguage is a script-family check, not language detection. It confirms // a response uses the script a locale is written in; it cannot tell apart // languages that share a script (en/fr, ar/fa/ur, ru/uk). Maps BCP-47 base tags -// to their primary non-Latin Unicode script range. -const SCRIPT_PATTERNS: Record = { - ja: /[぀-ヿ一-鿿㐀-䶿]/, - zh: /[一-鿿㐀-䶿]/, - ko: /[가-힯]/, - ar: /[؀-ۿ]/, - fa: /[؀-ۿ]/, - ur: /[؀-ۿ]/, - ru: /[Ѐ-ӿ]/, - uk: /[Ѐ-ӿ]/, - bg: /[Ѐ-ӿ]/, - sr: /[Ѐ-ӿ]/, - mn: /[Ѐ-ӿ]/, - hi: /[ऀ-ॿ]/, - mr: /[ऀ-ॿ]/, - bn: /[ঀ-৿]/, - th: /[฀-๿]/, - he: /[֐-׿]/, - el: /[Ͱ-Ͽ]/, - ka: /[Ⴀ-ჿ]/, - am: /[ሀ-፿]/, +// to their primary non-Latin Unicode script range, expressed as a character-class +// body so SCRIPT_PATTERNS and NON_LATIN_PATTERN are derived from one source and +// can never drift apart (F-2). +const SCRIPT_RANGES: Record = { + ja: "぀-ヿ一-鿿㐀-䶿", // kana + Han; see KANA_PATTERN for the Japanese-specific guard + zh: "一-鿿㐀-䶿", + ko: "가-힯", + ar: "؀-ۿ", + fa: "؀-ۿ", + ur: "؀-ۿ", + ru: "Ѐ-ӿ", + uk: "Ѐ-ӿ", + bg: "Ѐ-ӿ", + sr: "Ѐ-ӿ", + mn: "Ѐ-ӿ", + hi: "ऀ-ॿ", + mr: "ऀ-ॿ", + bn: "ঀ-৿", + th: "฀-๿", + he: "֐-׿", + el: "Ͱ-Ͽ", + ka: "Ⴀ-ჿ", + am: "ሀ-፿", }; +const SCRIPT_PATTERNS: Record = Object.fromEntries( + Object.entries(SCRIPT_RANGES).map(([locale, range]) => [ + locale, + new RegExp(`[${range}]`, "u"), + ]), +); + +// Hiragana + katakana. A reply written entirely in Han (i.e. Chinese) has zero +// kana, so requiring at least one kana character stops pure-Chinese text from +// passing `responseLanguage: ja` (F-3). +const KANA_PATTERN = /[぀-ヿ]/u; + // Locales known to use a Latin script. The non-Latin penalty only fires for // these; a locale that is neither in SCRIPT_PATTERNS nor here is treated as // "script not determinable" and passes rather than being wrongly flagged. @@ -58,8 +72,21 @@ const LATIN_LOCALES = new Set([ "yo", ]); -// All non-Latin script ranges combined; used to detect unexpected non-Latin content in Latin-locale responses. -const NON_LATIN_PATTERN = /[Ͱ-ϿЀ-ӿ֐-׿؀-ۿऀ-৿฀-๿ᄀ-ᇿ぀-ヿ㐀-䶿一-鿿가-힯]/; +// Every non-Latin script range known to SCRIPT_RANGES, combined; used to detect +// unexpected non-Latin content in Latin-locale responses. Derived from the same +// table as SCRIPT_PATTERNS so newly added scripts (e.g. ka, am) are covered here +// automatically. Duplicate ranges in the class are harmless. +const NON_LATIN_PATTERN = new RegExp( + `[${Object.values(SCRIPT_RANGES).join("")}]`, + "u", +); + +// Whether the script check can ever fail for a locale. When false, the check is +// a guaranteed pass (a silent no-op) and lint should say so (F-29). +export function isScriptDeterminable(locale: string): boolean { + const base = locale.split("-")[0].toLowerCase(); + return base in SCRIPT_PATTERNS || LATIN_LOCALES.has(base); +} type Pass = { pass: true; detail: string; failureMode: null }; type Fail = { @@ -74,7 +101,7 @@ export function assertExpectedToolCall( response: TargetResponse, ): AssertionResult { const forbiddenResult = assertForbiddenToolCall( - expected.noToolCall?.name, + expected.noToolCall?.names, response, ); if (!forbiddenResult.pass) return forbiddenResult; @@ -173,6 +200,10 @@ function assertToolCallSequence( ): AssertionResult { // Check that expected tool calls appear as a subsequence in actual tool calls let expectedIndex = 0; + // Best near-miss for the step we're currently waiting on: a call with the + // right name whose arguments didn't match. Surfaced so the failure detail + // doesn't claim a present-but-wrong call is "missing" (F-6). + let nearMiss: string | null = null; for (const actual of response.toolCalls) { if (expectedIndex >= expected.length) break; @@ -184,6 +215,9 @@ function assertToolCallSequence( ); if (argResult.pass) { expectedIndex += 1; + nearMiss = null; + } else { + nearMiss = `${actual.name} called but ${argResult.detail}`; } } } @@ -193,10 +227,13 @@ function assertToolCallSequence( .slice(expectedIndex) .map((e) => e.name) .join(", "); + const detail = nearMiss + ? `sequence incomplete, missing: ${missing} (${nearMiss})` + : `sequence incomplete, missing: ${missing}`; return { pass: false, failureMode: "wrong_sequence", - detail: `sequence incomplete, missing: ${missing}`, + detail, }; } @@ -223,50 +260,70 @@ function assertResponseLanguage( if (scriptPattern) { const scriptCount = letters.filter((c) => scriptPattern.test(c)).length; - if (scriptCount / letters.length < 0.1) { + const ratio = scriptCount / letters.length; + + // Japanese: Han alone is Chinese; require at least one kana character. + if (base === "ja" && !KANA_PATTERN.test(text)) { + return { + pass: false, + failureMode: "wrong_language", + detail: `expected ja script (kana) in response, found none (in-script ratio ${ratio.toFixed(2)})`, + }; + } + + if (ratio < 0.1) { return { pass: false, failureMode: "wrong_language", - detail: `expected ${base} script in response, got mostly other script`, + detail: `expected ${base} script in response, got mostly other script (in-script ratio ${ratio.toFixed(2)})`, }; } - } else if (LATIN_LOCALES.has(base)) { + + return { + pass: true, + detail: `responseLanguage: ${expectedLocale} (in-script ratio ${ratio.toFixed(2)})`, + failureMode: null, + }; + } + + if (LATIN_LOCALES.has(base)) { // Known Latin-script locale: response should not be dominated by a non-Latin script. const nonLatinCount = letters.filter((c) => NON_LATIN_PATTERN.test(c), ).length; - if (nonLatinCount / letters.length > 0.5) { + const nonLatinRatio = nonLatinCount / letters.length; + if (nonLatinRatio > 0.5) { return { pass: false, failureMode: "wrong_language", - detail: `expected Latin-script response (${base}), got mostly non-Latin characters`, + detail: `expected Latin-script response (${base}), got mostly non-Latin characters (non-Latin ratio ${nonLatinRatio.toFixed(2)})`, }; } - } else { - // Script not determinable for this locale: pass rather than guess. + return { pass: true, - detail: `responseLanguage: ${expectedLocale} (script not determinable)`, + detail: `responseLanguage: ${expectedLocale} (non-Latin ratio ${nonLatinRatio.toFixed(2)})`, failureMode: null, }; } + // Script not determinable for this locale: pass rather than guess. return { pass: true, - detail: `responseLanguage: ${expectedLocale}`, + detail: `responseLanguage: ${expectedLocale} (script not determinable)`, failureMode: null, }; } function assertForbiddenToolCall( - forbiddenName: string | undefined, + forbiddenNames: string[] | undefined, response: TargetResponse, ): Pass | Fail { - if (!forbiddenName) { + if (!forbiddenNames || forbiddenNames.length === 0) { return { pass: true, detail: "", failureMode: null }; } - const found = response.toolCalls.find((c) => c.name === forbiddenName); + const found = response.toolCalls.find((c) => forbiddenNames.includes(c.name)); if (!found) { return { pass: true, detail: "", failureMode: null }; @@ -275,18 +332,33 @@ function assertForbiddenToolCall( return { pass: false, failureMode: "forbidden_tool", - detail: `forbidden tool call ${forbiddenName} was called`, + detail: `forbidden tool call ${found.name} was called`, }; } // Scalar-normalized equality: tolerates JSON-type vs YAML-string differences // (number 2 matches "2", boolean true matches "true") so canonical tool args -// are not falsely failed on type alone. -function matchesScalar( - actual: unknown, - expected: string | number | boolean, -): boolean { - return String(actual) === String(expected); +// are not falsely failed on type alone. Non-scalar actuals (arrays, objects, +// null, undefined) never match — `String()` would flatten `["x"]` to `"x"` and +// let a regressed array-of-enums pass a scalar assertion (F-1). +function matchesScalar(actual: unknown, expected: string): boolean { + if ( + typeof actual !== "string" && + typeof actual !== "number" && + typeof actual !== "boolean" + ) { + return false; + } + return String(actual) === expected; +} + +// Human-readable rendering of an actual argument value for failure details, +// naming the shape rather than coercing it (F-1). +function describeActual(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return `array ${JSON.stringify(value)}`; + if (typeof value === "object") return `object ${JSON.stringify(value)}`; + return String(value); } function matchesArg(actual: unknown, expected: ArgMatcher): boolean { @@ -347,7 +419,7 @@ function assertExpectedArguments( return { pass: false, failureMode: "wrong_argument", - detail: `expected argument ${key}=${describeArg(expectedValue)}, got ${String(actual[key])}`, + detail: `expected argument ${key}=${describeArg(expectedValue)}, got ${describeActual(actual[key])}`, }; } } diff --git a/src/cli.ts b/src/cli.ts index 2cfb382..28fc5aa 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,6 +14,9 @@ import { } from "./reportMarkdown.ts"; import { formatLintReport, lintScenarios } from "./lint.ts"; import { DEFAULT_LOCALES, translateScenario } from "./translate.ts"; +import { shouldFail } from "./gate.ts"; + +const COMMANDS = new Set(["run", "init", "lint", "translate"]); type CliOptions = RunOptions | InitOptions | LintOptions | TranslateOptions; @@ -25,6 +28,7 @@ type RunOptions = { format: "text" | "json" | "markdown"; minPassRate: number | null; allowFail: string[]; + timeoutMs: number | null; }; type InitOptions = { @@ -48,14 +52,19 @@ type TranslateOptions = { async function main(): Promise { const args = process.argv.slice(2); - if (args.includes("--help") || args.includes("-h")) { - process.stdout.write(`${usage()}\n`); - return; - } + // Only treat --help/--version as global when they aren't riding inside a real + // command; otherwise `run s.yaml --target x -v` would silently print the + // version and exit 0 in a CI script (F-18). + if (!COMMANDS.has(args[0] ?? "")) { + if (args.includes("--help") || args.includes("-h")) { + process.stdout.write(`${usage()}\n`); + return; + } - if (args.includes("--version") || args.includes("-v")) { - process.stdout.write(`${readPackageVersion()}\n`); - return; + if (args.includes("--version") || args.includes("-v")) { + process.stdout.write(`${readPackageVersion()}\n`); + return; + } } const options = parseArgs(args); @@ -67,7 +76,7 @@ async function main(): Promise { } if (options.command === "lint") { - const paths = await resolveScenarioPaths(options.scenarioPath); + const { paths } = await resolveScenarioPaths(options.scenarioPath); if (paths.length === 0) { throw new Error(`No scenario files found at ${options.scenarioPath}`); } @@ -100,19 +109,25 @@ async function main(): Promise { return; } - const paths = await resolveScenarioPaths(options.scenarioPath); + const { paths, isDirectory } = await resolveScenarioPaths( + options.scenarioPath, + ); if (paths.length === 0) { throw new Error(`No scenario files found at ${options.scenarioPath}`); } - const isMatrix = paths.length > 1; + const timeoutMs = options.timeoutMs ?? undefined; - if (isMatrix) { + // Directory input always emits the matrix shape, even for a single file, so + // downstream tooling parsing `runs[]` doesn't break when a directory shrinks + // to one scenario (F-20). + if (isDirectory) { const matrix = await runScenarios( paths, options.target, options.iterations, + timeoutMs, ); if (options.format === "json") { @@ -124,12 +139,18 @@ async function main(): Promise { } const allResults = matrix.runs.flatMap((run) => run.results); + warnUnknownAllowFail(options.allowFail, allResults); if (shouldFail(allResults, options.minPassRate, options.allowFail)) { process.exitCode = 1; } } else { const scenario = await loadScenario(paths[0]); - const run = await runScenario(scenario, options.target, options.iterations); + const run = await runScenario( + scenario, + options.target, + options.iterations, + timeoutMs, + ); if (options.format === "json") { process.stdout.write(formatJsonReport(run)); @@ -139,27 +160,27 @@ async function main(): Promise { process.stdout.write(formatTerminalReport(run)); } + warnUnknownAllowFail(options.allowFail, run.results); if (shouldFail(run.results, options.minPassRate, options.allowFail)) { process.exitCode = 1; } } } -function shouldFail( - results: import("./types.ts").LocaleResult[], - minPassRate: number | null, +// A mistyped --allow-fail locale (e.g. `ue` for `eu`) filters nothing and lets +// the build fail with no hint; warn when it matches no locale in the run (F-19). +function warnUnknownAllowFail( allowFail: string[], -): boolean { - const counted = results.filter((r) => !allowFail.includes(r.locale)); - - if (minPassRate !== null) { - const totalChecks = counted.reduce((sum, r) => sum + r.total, 0); - const totalPassed = counted.reduce((sum, r) => sum + r.passed, 0); - const rate = totalChecks === 0 ? 100 : (totalPassed / totalChecks) * 100; - return rate < minPassRate; + results: import("./types.ts").LocaleResult[], +): void { + const present = new Set(results.map((r) => r.locale)); + for (const locale of allowFail) { + if (!present.has(locale)) { + process.stderr.write( + `warning: --allow-fail ${locale} matches no locale in the run\n`, + ); + } } - - return counted.some((r) => r.status === "fail"); } function parseArgs(args: string[]): CliOptions { @@ -218,6 +239,10 @@ function parseArgs(args: string[]): CliOptions { minPassRateIndex === -1 ? null : args[minPassRateIndex + 1]; const minPassRate = minPassRateRaw === null ? null : Number(minPassRateRaw); + const timeoutIndex = args.indexOf("--timeout"); + const timeoutRaw = timeoutIndex === -1 ? null : args[timeoutIndex + 1]; + const timeoutMs = timeoutRaw === null ? null : Number(timeoutRaw); + const allowFail: string[] = []; for (let i = 0; i < args.length; i += 1) { if ( @@ -232,12 +257,17 @@ function parseArgs(args: string[]): CliOptions { if ( !scenarioPath || scenarioPath.startsWith("--") || + // A target that is itself a flag means `--target` swallowed the next flag + // (e.g. `--target --format`); reject rather than run against a bogus URL (F-18). !target || + target.startsWith("--") || + !isValidUrl(target) || (format !== "text" && format !== "json" && format !== "markdown") || !Number.isInteger(iterations) || iterations < 1 || (minPassRate !== null && - (Number.isNaN(minPassRate) || minPassRate < 0 || minPassRate > 100)) + (Number.isNaN(minPassRate) || minPassRate < 0 || minPassRate > 100)) || + (timeoutMs !== null && (!Number.isInteger(timeoutMs) || timeoutMs < 1)) ) { throw new Error(usage()); } @@ -250,9 +280,19 @@ function parseArgs(args: string[]): CliOptions { format, minPassRate, allowFail, + timeoutMs, }; } +function isValidUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + function parseInitPath(args: string[]): string { for (let index = 0; index < args.length; index += 1) { if (args[index] === "--template") { @@ -272,7 +312,7 @@ function usage(): string { return [ "Usage:", ` langdrift init [scenario.yaml] [--template ${INIT_TEMPLATES.join("|")}]`, - " langdrift run --target [--iterations N] [--format text|json|markdown] [--min-pass-rate N] [--allow-fail ]", + " langdrift run --target [--iterations N] [--format text|json|markdown] [--min-pass-rate N] [--allow-fail ] [--timeout MS]", " langdrift lint ", " langdrift translate [--locales fr,ar,zh,...] [--write]", ].join("\n"); diff --git a/src/gate.ts b/src/gate.ts new file mode 100644 index 0000000..6870f33 --- /dev/null +++ b/src/gate.ts @@ -0,0 +1,20 @@ +import type { LocaleResult } from "./types.ts"; + +// The CI exit-code decision. Lives here (rather than inline in cli.ts) so the +// tests exercise the real function instead of a copy-pasted duplicate (F-23). +export function shouldFail( + results: LocaleResult[], + minPassRate: number | null, + allowFail: string[], +): boolean { + const counted = results.filter((r) => !allowFail.includes(r.locale)); + + if (minPassRate !== null) { + const totalChecks = counted.reduce((sum, r) => sum + r.total, 0); + const totalPassed = counted.reduce((sum, r) => sum + r.passed, 0); + const rate = totalChecks === 0 ? 100 : (totalPassed / totalChecks) * 100; + return rate < minPassRate; + } + + return counted.some((r) => r.status === "fail"); +} diff --git a/src/httpTarget.ts b/src/httpTarget.ts index 78333e1..68d7f50 100644 --- a/src/httpTarget.ts +++ b/src/httpTarget.ts @@ -5,12 +5,18 @@ export type ExecuteTargetInput = { scenarioId: string; locale: string; input: string; + agent: string; + timeoutMs?: number; }; +// Every failure here is a harness/transport failure, not model behavior; the +// runner maps `ok: false` to the `target_error` failure mode (F-5). export type ExecuteTargetResult = | { ok: true; response: TargetResponse } | { ok: false; detail: string }; +const DEFAULT_TIMEOUT_MS = 30_000; + export async function executeHttpTarget( input: ExecuteTargetInput, ): Promise { @@ -26,9 +32,18 @@ export async function executeHttpTarget( locale: input.locale, input: input.input, scenarioId: input.scenarioId, + agent: input.agent, }), + // Without a timeout one hung locale stalls the whole serial run forever (F-17). + signal: AbortSignal.timeout(input.timeoutMs ?? DEFAULT_TIMEOUT_MS), }); } catch (error) { + if (error instanceof Error && error.name === "TimeoutError") { + return { + ok: false, + detail: `request timed out after ${input.timeoutMs ?? DEFAULT_TIMEOUT_MS}ms`, + }; + } return { ok: false, detail: error instanceof Error ? error.message : "network error", @@ -55,27 +70,54 @@ export async function executeHttpTarget( return { ok: false, detail: "invalid JSON response" }; } - return { ok: true, response: normalizeTargetResponse(body) }; + return normalizeTargetResponse(body); } -function normalizeTargetResponse(body: unknown): TargetResponse { +function normalizeTargetResponse(body: unknown): ExecuteTargetResult { + // A response that isn't a JSON object violates the contract; surface it as a + // transport error rather than silently coercing it to an empty response and + // reporting a behavioral verdict for what is really an integration bug (F-21). if (!body || typeof body !== "object" || Array.isArray(body)) { return { - text: "", - toolCalls: [], - structured: null, + ok: false, + detail: `malformed response: expected a JSON object, got ${describeShape(body)}`, }; } const record = body as Record; + // A field of the wrong *type* is malformed; a `null` (or absent) field is + // benign and falls through to the documented default. + if (record.text != null && typeof record.text !== "string") { + return { + ok: false, + detail: `malformed response: "text" must be a string, got ${describeShape(record.text)}`, + }; + } + + if (record.toolCalls != null && !Array.isArray(record.toolCalls)) { + return { + ok: false, + detail: `malformed response: "toolCalls" must be an array, got ${describeShape(record.toolCalls)}`, + }; + } + return { - text: typeof record.text === "string" ? record.text : "", - toolCalls: normalizeToolCalls(record.toolCalls), - structured: "structured" in record ? record.structured : null, + ok: true, + response: { + text: typeof record.text === "string" ? record.text : "", + toolCalls: normalizeToolCalls(record.toolCalls), + structured: "structured" in record ? record.structured : null, + }, }; } +function describeShape(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + return typeof value; +} + function normalizeToolCalls(value: unknown): TargetResponse["toolCalls"] { if (!Array.isArray(value)) { return []; diff --git a/src/lint.ts b/src/lint.ts index e480a95..016f517 100644 --- a/src/lint.ts +++ b/src/lint.ts @@ -1,3 +1,4 @@ +import { isScriptDeterminable } from "./assertions.ts"; import { loadScenario } from "./scenario.ts"; import type { Scenario } from "./types.ts"; @@ -47,12 +48,48 @@ export async function lintScenarios(paths: string[]): Promise { message: `no "en" locale; English is typically used as the baseline`, }); } + + // A responseLanguage whose script LangDrift can't determine always passes, + // so the assertion can never fail — flag it as a silent no-op (F-29). + for (const [locale, variant] of Object.entries(scenario.locales)) { + const lang = variant.expect.responseLanguage; + if (lang && !isScriptDeterminable(lang)) { + issues.push({ + severity: "warning", + message: `locale "${locale}": responseLanguage "${lang}" is not script-determinable, so the check can never fail`, + }); + } + } } loaded.push({ path, scenario, issues }); } if (paths.length > 1) { + // Two files sharing a scenario id collide in matrix reports keyed by + // scenarioId; flag it as an error (F-24). + const idToPaths = new Map(); + for (const { path, scenario } of loaded) { + if (!scenario) continue; + const existing = idToPaths.get(scenario.id) ?? []; + existing.push(path); + idToPaths.set(scenario.id, existing); + } + for (const [id, idPaths] of idToPaths) { + if (idPaths.length > 1) { + for (const { path, issues } of loaded) { + if (idPaths.includes(path)) { + issues.push({ + severity: "error", + message: `duplicate scenario id "${id}" also defined in ${idPaths + .filter((p) => p !== path) + .join(", ")}`, + }); + } + } + } + } + const localeSetByPath = new Map>(); for (const { path, scenario } of loaded) { if (scenario) { diff --git a/src/reportJson.ts b/src/reportJson.ts index c4b3f83..12978eb 100644 --- a/src/reportJson.ts +++ b/src/reportJson.ts @@ -1,4 +1,13 @@ -import type { MatrixResult, RunResult } from "./types.ts"; +import { wilsonInterval } from "./stats.ts"; +import type { LocaleResult, MatrixResult, RunResult } from "./types.ts"; + +// Strip the raw response and attach a 95% Wilson confidence interval computed +// from this locale's pass count, so the tool's own report carries the CIs the +// research writeup relies on (F-15). +function serializeLocaleResult(result: LocaleResult) { + const { response: _response, ...rest } = result; + return { ...rest, ci: wilsonInterval(result.passed, result.total) }; +} export function formatJsonReport(run: RunResult): string { const failedLocales = run.results.filter((r) => r.status === "fail").length; @@ -18,7 +27,7 @@ export function formatJsonReport(run: RunResult): string { totalRuns: run.results.reduce((sum, r) => sum + r.total, 0), totalPassed: run.results.reduce((sum, r) => sum + r.passed, 0), }, - results: run.results.map(({ response: _response, ...rest }) => rest), + results: run.results.map(serializeLocaleResult), }, null, 2, @@ -57,7 +66,7 @@ export function formatJsonMatrixReport(matrix: MatrixResult): string { status: run.results.every((r) => r.status === "pass") ? "passed" : "failed", - results: run.results.map(({ response: _response, ...rest }) => rest), + results: run.results.map(serializeLocaleResult), })), }, null, diff --git a/src/reportMarkdown.ts b/src/reportMarkdown.ts index 1429548..e245dfa 100644 --- a/src/reportMarkdown.ts +++ b/src/reportMarkdown.ts @@ -1,23 +1,32 @@ import type { MatrixResult, RunResult } from "./types.ts"; +// Table cells can contain `|` (scenario ids, failure details); escape it so the +// markdown table doesn't break into extra columns (F-35). +function escapeCell(value: string): string { + return value.replace(/\|/g, "\\|"); +} + export function formatMarkdownRunReport(run: RunResult): string { const failedLocales = run.results.filter((r) => r.status === "fail").length; const rows = run.results.map((r) => { const rate = `${r.passed}/${r.total}`; const pct = `${Math.round((r.passed / r.total) * 100)}%`; const failure = r.failureMode ?? "—"; - return `| ${r.locale} | ${rate} | ${pct} | ${failure} |`; + // Include the detail so "expected X, got Y" is visible where triage + // happens, not just the mode (F-35). + const detail = r.detail ? escapeCell(r.detail) : "—"; + return `| ${r.locale} | ${rate} | ${pct} | ${failure} | ${detail} |`; }); const lines = [ "# LangDrift Run", "", - `**Scenario:** ${run.scenarioId} `, + `**Scenario:** ${escapeCell(run.scenarioId)} `, `**Target:** ${run.target} `, `**Iterations:** ${run.iterations} `, "", - "| Locale | Passed | Rate | Failure |", - "|--------|--------|------|---------|", + "| Locale | Passed | Rate | Failure | Detail |", + "|--------|--------|------|---------|--------|", ...rows, "", `**Result:** ${failedLocales === 0 ? "passed" : "failed"}, ${failedLocales} of ${run.results.length} locales failed`, @@ -31,7 +40,7 @@ export function formatMarkdownMatrixReport(matrix: MatrixResult): string { const allLocales = collectLocales(matrix); const scenarioIds = matrix.runs.map((r) => r.scenarioId); - const headerRow = `| Locale | ${scenarioIds.join(" | ")} |`; + const headerRow = `| Locale | ${scenarioIds.map(escapeCell).join(" | ")} |`; const separator = `|--------|${scenarioIds.map(() => "--------").join("|")}|`; const dataRows = allLocales.map((locale) => { @@ -39,7 +48,8 @@ export function formatMarkdownMatrixReport(matrix: MatrixResult): string { const result = run.results.find((r) => r.locale === locale); if (!result) return "—"; const cell = `${result.passed}/${result.total}`; - return result.status === "pass" ? `**${cell}**` : cell; + // Highlight failures, not passes — the reader is scanning for problems (F-35). + return result.status === "fail" ? `**${cell}**` : cell; }); return `| ${locale} | ${cells.join(" | ")} |`; }); diff --git a/src/runner.ts b/src/runner.ts index 00a2001..e51453f 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -22,6 +22,7 @@ export async function runScenario( scenario: Scenario, target: string, iterations: number, + timeoutMs?: number, ): Promise { const localeIterations: Record = {}; @@ -36,12 +37,16 @@ export async function runScenario( scenarioId: scenario.id, locale, input: variant.input, + agent: scenario.agent, + timeoutMs, }); if (!targetResult.ok) { + // Transport/harness failures get their own mode so an agent outage or a + // malformed response isn't miscounted as model drift (F-5, F-21). localeIterations[locale].push({ status: "fail", - failureMode: "no_tool_call", + failureMode: "target_error", detail: targetResult.detail, }); continue; @@ -87,27 +92,40 @@ export async function runScenarios( scenarioPaths: string[], target: string, iterations: number, + timeoutMs?: number, ): Promise { const runs: RunResult[] = []; for (const path of scenarioPaths) { const scenario = await loadScenario(path); - runs.push(await runScenario(scenario, target, iterations)); + runs.push(await runScenario(scenario, target, iterations, timeoutMs)); } return { target, iterations, runs }; } -export async function resolveScenarioPaths(path: string): Promise { +// Resolves a run input to the scenario files it covers, and reports whether the +// input was a directory. The directory flag lets the CLI pick the report schema +// by input kind rather than by file count, so a directory that happens to hold +// one file still emits the matrix shape (F-20). +export type ResolvedScenarioPaths = { + paths: string[]; + isDirectory: boolean; +}; + +export async function resolveScenarioPaths( + path: string, +): Promise { try { const entries = await readdir(path); - return entries + const paths = entries .filter((name) => name.endsWith(".yaml") || name.endsWith(".yml")) .map((name) => join(path, name)) .sort(); + return { paths, isDirectory: true }; } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOTDIR") { - return [path]; + return { paths: [path], isDirectory: false }; } throw err; } diff --git a/src/scenario.ts b/src/scenario.ts index 7f0f286..14c890d 100644 --- a/src/scenario.ts +++ b/src/scenario.ts @@ -39,9 +39,17 @@ export function parseScenario(source: string, path = "scenario"): Scenario { throw new Error(`${path}: missing required field "locales"`); } - const locales = parseLocales(lines.slice(localesIndex + 1), path); + const localeLines = lines.slice(localesIndex + 1); + const locales = parseLocales(localeLines, path); if (Object.keys(locales).length === 0) { + // Distinguish "genuinely empty" from "indented wrong" so a 4-space-indented + // file gets an indentation hint rather than a bare "expected a locale" (F-8). + if (localeLines.some((line) => line.indent > 0)) { + throw new Error( + `${path}: no locales found under "locales:"; locale entries must be indented exactly 2 spaces`, + ); + } throw new Error(`${path}: expected at least one locale`); } @@ -75,21 +83,27 @@ function parseToolCallList( const itemLines = lines.slice(i + 1, itemEnd).filter((l) => !l.isList); const name = scalarAt(itemLines, contentIndent, "name"); - if (name) { - const toolArguments = nestedArgsAt( - itemLines, - [contentIndent], - ["arguments"], - contentIndent + 2, + if (!name) { + // A list item with no `name` contributes nothing and silently weakens the + // assertion set (e.g. a `nme:` typo); reject it instead of dropping it (F-11). + throw new Error( + `scenario line ${line.number}: tool-call list item is missing "name"`, ); - assertions.push({ - name, - ...(Object.keys(toolArguments).length > 0 - ? { arguments: toolArguments } - : {}), - }); } + const toolArguments = nestedArgsAt( + itemLines, + [contentIndent], + ["arguments"], + contentIndent + 2, + ); + assertions.push({ + name, + ...(Object.keys(toolArguments).length > 0 + ? { arguments: toolArguments } + : {}), + }); + i = itemEnd - 1; } @@ -114,6 +128,11 @@ function parseLocales( } const locale = line.key; + if (Object.hasOwn(locales, locale)) { + // A duplicate mapping key would silently last-win and halve coverage; a + // copy-pasted `en:` block should be an error, not a quiet overwrite (F-10). + throw new Error(`${path}: duplicate locale "${locale}"`); + } const nextLocaleIndex = lines.findIndex( (candidate, candidateIndex) => candidateIndex > index && @@ -154,11 +173,7 @@ function parseLocales( ? parseToolCallList(block.slice(toolCallsKeyIndex + 1), 8) : []; - const forbiddenToolName = nestedScalarAt( - block, - [4, 6, 8], - ["expect", "noToolCall", "name"], - ); + const forbiddenNames = parseForbiddenNames(block); const responseLanguage = nestedScalarAt( block, @@ -197,9 +212,7 @@ function parseLocales( expect: { ...(toolCall !== undefined ? { toolCall } : {}), ...(toolCallsList.length > 0 ? { toolCalls: toolCallsList } : {}), - ...(forbiddenToolName - ? { noToolCall: { name: forbiddenToolName } } - : {}), + ...(forbiddenNames ? { noToolCall: { names: forbiddenNames } } : {}), ...(responseLanguage ? { responseLanguage } : {}), }, }; @@ -210,6 +223,41 @@ function parseLocales( return locales; } +// Forbidden tool names from a locale's `expect.noToolCall` block, supporting +// either a single `name:` or an `anyOf: [a, b]` inline list (F-28). Scoped to +// the noToolCall block so an empty `noToolCall:` can't absorb a sibling's +// `name:` (F-9). +function parseForbiddenNames(block: Line[]): string[] | undefined { + const expectIndex = block.findIndex( + (l) => l.indent === 4 && l.key === "expect", + ); + if (expectIndex === -1) return undefined; + const expectEnd = blockEndAt(block, expectIndex, 4); + + let noToolCallIndex = -1; + for (let i = expectIndex + 1; i < expectEnd; i += 1) { + if (block[i].indent === 6 && block[i].key === "noToolCall") { + noToolCallIndex = i; + break; + } + } + if (noToolCallIndex === -1) return undefined; + + const end = blockEndAt(block, noToolCallIndex, 6); + const names: string[] = []; + for (let i = noToolCallIndex + 1; i < end; i += 1) { + const l = block[i]; + if (l.indent !== 8) continue; + if (l.key === "name" && l.value !== "") { + names.push(parseScalar(l.value, l.number)); + } else if (l.key === "anyOf") { + names.push(...parseInlineList(l.value, l.number)); + } + } + + return names.length > 0 ? names : undefined; +} + function tokenize(source: string): Line[] { const result: Line[] = []; @@ -217,7 +265,24 @@ function tokenize(source: string): Line[] { const number = index + 1; if (raw.trim() === "" || raw.trimStart().startsWith("#")) continue; - const indent = raw.match(/^ */)?.[0].length ?? 0; + const leading = raw.match(/^[ \t]*/)?.[0] ?? ""; + + // The parser is a strict 2-space-indented YAML subset; catch the common ways + // valid YAML falls outside it and say so, instead of failing later with an + // unrelated message like "expected at least one locale" (F-8). + if (leading.includes("\t")) { + throw new Error( + `scenario line ${number}: tab indentation is not supported; use 2 spaces per level`, + ); + } + + const indent = leading.length; + if (indent % 2 !== 0) { + throw new Error( + `scenario line ${number}: indentation must be a multiple of 2 spaces (got ${indent})`, + ); + } + const trimmed = raw.trim(); if (trimmed.startsWith("- ")) { @@ -225,10 +290,12 @@ function tokenize(source: string): Line[] { const content = trimmed.slice(2); const separator = content.indexOf(":"); if (separator !== -1) { + const value = content.slice(separator + 1).trim(); + rejectBlockScalar(value, number); result.push({ indent: indent + 2, key: content.slice(0, separator).trim(), - value: content.slice(separator + 1).trim(), + value, number, }); } @@ -240,10 +307,13 @@ function tokenize(source: string): Line[] { throw new Error(`scenario line ${number}: expected "key: value"`); } + const value = trimmed.slice(separator + 1).trim(); + rejectBlockScalar(value, number); + result.push({ indent, key: trimmed.slice(0, separator).trim(), - value: trimmed.slice(separator + 1).trim(), + value, number, }); } @@ -251,6 +321,17 @@ function tokenize(source: string): Line[] { return result; } +// Block scalars (`input: |` / `input: >`) span multiple lines, which this +// line-oriented parser can't represent; reject them with a clear message rather +// than swallowing the continuation lines as stray "key: value" errors (F-8). +function rejectBlockScalar(value: string, lineNumber: number): void { + if (/^[|>][+-]?\d*$/.test(value)) { + throw new Error( + `scenario line ${lineNumber}: block scalars (| and >) are not supported; use a quoted single-line string`, + ); + } +} + function scalarAt( lines: Line[], indent: number, @@ -270,14 +351,19 @@ function nestedScalarAt( keys: string[], ): string | undefined { let start = 0; + // Upper bound of the current parent's block. Each descent narrows the window + // to the lines nested under the matched parent, so a key can't be matched in a + // sibling block (e.g. reading toolCall's `name` as noToolCall's) — F-9. + let end = lines.length; for (let index = 0; index < keys.length; index += 1) { - const lineIndex = lines.findIndex( - (line, candidateIndex) => - candidateIndex >= start && - line.indent === indents[index] && - line.key === keys[index], - ); + let lineIndex = -1; + for (let i = start; i < end; i += 1) { + if (lines[i].indent === indents[index] && lines[i].key === keys[index]) { + lineIndex = i; + break; + } + } if (lineIndex === -1) { return undefined; @@ -291,11 +377,25 @@ function nestedScalarAt( } start = lineIndex + 1; + end = blockEndAt(lines, lineIndex, indents[index]); } return undefined; } +// Index of the first line after `parentIndex` whose indent is at or below the +// parent's — i.e. the exclusive end of the parent's nested block. +function blockEndAt( + lines: Line[], + parentIndex: number, + parentIndent: number, +): number { + for (let i = parentIndex + 1; i < lines.length; i += 1) { + if (lines[i].indent <= parentIndent) return i; + } + return lines.length; +} + function nestedArgsAt( lines: Line[], indents: number[], @@ -361,7 +461,9 @@ function parseInlineList(value: string, lineNumber: number): string[] { const items = inner === "" ? [] - : inner.split(",").map((item) => parseScalar(item.trim(), lineNumber)); + : splitTopLevel(inner, lineNumber).map((item) => + parseScalar(item.trim(), lineNumber), + ); if (items.length === 0) { throw new Error( @@ -372,6 +474,42 @@ function parseInlineList(value: string, lineNumber: number): string[] { return items; } +// Split a comma-separated inline list, ignoring commas inside quoted items so +// `["a, b", c]` yields `["a, b", "c"]` instead of mangled fragments (F-7). +function splitTopLevel(inner: string, lineNumber: number): string[] { + const items: string[] = []; + let current = ""; + let quote: '"' | "'" | null = null; + + for (const char of inner) { + if (quote) { + current += char; + if (char === quote) quote = null; + continue; + } + if (char === '"' || char === "'") { + quote = char; + current += char; + continue; + } + if (char === ",") { + items.push(current); + current = ""; + continue; + } + current += char; + } + + if (quote) { + throw new Error( + `scenario line ${lineNumber}: unterminated quote in inline list`, + ); + } + + items.push(current); + return items; +} + function parseScalar(value: string, lineNumber: number): string { if (value.startsWith('"') && value.endsWith('"')) { try { diff --git a/src/stats.ts b/src/stats.ts new file mode 100644 index 0000000..bd498bb --- /dev/null +++ b/src/stats.ts @@ -0,0 +1,20 @@ +// Wilson score interval for a binomial proportion. Reported per locale so +// per-cell pass rates are read as estimates with uncertainty, not exact facts. +// Runs are near-deterministic at temperature 0, so this reflects API-side +// variance over N iterations, not a sampling distribution. +export function wilsonInterval( + passes: number, + total: number, + z = 1.96, +): [number, number] | null { + if (total === 0) return null; + const p = passes / total; + const denom = 1 + (z * z) / total; + const center = (p + (z * z) / (2 * total)) / denom; + const margin = + (z / denom) * + Math.sqrt((p * (1 - p)) / total + (z * z) / (4 * total * total)); + const low = Math.max(0, center - margin); + const high = Math.min(1, center + margin); + return [low, high]; +} diff --git a/src/translate.ts b/src/translate.ts index 80635db..dbd9da4 100644 --- a/src/translate.ts +++ b/src/translate.ts @@ -48,7 +48,16 @@ export async function translateScenario( } const translations = await callLlm(enLocale.input, targetLocales, options); - const enExpect = serializeExpect(scenario.locales.en); + + // The LLM can silently return fewer locales than requested; surface the gap + // instead of quietly producing a shorter scenario (F-30). + const returned = new Set(translations.map((t) => t.locale)); + const dropped = targetLocales.filter((l) => !returned.has(l)); + if (dropped.length > 0) { + process.stderr.write( + `warning: model did not return translations for: ${dropped.join(", ")}\n`, + ); + } const yamlLines: string[] = [ `# Generated locale inputs for: ${scenario.id}`, @@ -62,7 +71,7 @@ export async function translateScenario( yamlLines.push(` ${locale}:`); yamlLines.push(` input: ${yamlQuote(input)}`); yamlLines.push(` expect:`); - for (const line of enExpect) { + for (const line of serializeExpect(scenario.locales.en, locale)) { yamlLines.push(` ${line}`); } yamlLines.push(""); @@ -136,20 +145,23 @@ Target locales: ${locales.join(", ")}`; .map((locale) => ({ locale, input: parsed[locale] })); } -function serializeExpect( +// Serializes an `expect` block back to the scenario's YAML subset. Used to copy +// the English assertions onto each generated locale. `targetLocale` is the +// locale the block is being generated for, so a script check can be rewritten to +// it (F-30). +export function serializeExpect( locale: import("./types.ts").ScenarioLocale, + targetLocale: string, ): string[] { const lines: string[] = []; - const { toolCall, toolCalls, noToolCall } = locale.expect; + const { toolCall, toolCalls, noToolCall, responseLanguage } = locale.expect; if (toolCall !== undefined && !Array.isArray(toolCall)) { lines.push(` toolCall:`); lines.push(` name: ${toolCall.name}`); if (toolCall.arguments) { lines.push(` arguments:`); - for (const [k, v] of Object.entries(toolCall.arguments)) { - lines.push(` ${k}: ${v}`); - } + lines.push(...serializeArgLines(toolCall.arguments, " ")); } } else if (Array.isArray(toolCall)) { lines.push(` toolCall:`); @@ -158,9 +170,7 @@ function serializeExpect( lines.push(` - name: ${option.name}`); if (option.arguments) { lines.push(` arguments:`); - for (const [k, v] of Object.entries(option.arguments)) { - lines.push(` ${k}: ${v}`); - } + lines.push(...serializeArgLines(option.arguments, " ")); } } } @@ -171,21 +181,47 @@ function serializeExpect( lines.push(` - name: ${step.name}`); if (step.arguments) { lines.push(` arguments:`); - for (const [k, v] of Object.entries(step.arguments)) { - lines.push(` ${k}: ${v}`); - } + lines.push(...serializeArgLines(step.arguments, " ")); } } } if (noToolCall) { lines.push(` noToolCall:`); - lines.push(` name: ${noToolCall.name}`); + if (noToolCall.names.length === 1) { + lines.push(` name: ${noToolCall.names[0]}`); + } else { + lines.push(` anyOf: [${noToolCall.names.join(", ")}]`); + } + } + + // If the source asserts a response script, assert the target locale's script + // on the generated block — the one assertion translate can add for free (F-30). + if (responseLanguage) { + lines.push(` responseLanguage: ${targetLocale}`); } return lines; } +// Serializes tool-argument matchers. A `oneOf` matcher becomes a nested inline +// list instead of stringifying to `[object Object]` (F-31). +function serializeArgLines( + args: Record, + keyIndent: string, +): string[] { + const out: string[] = []; + for (const [k, v] of Object.entries(args)) { + if (typeof v === "object" && v !== null && "oneOf" in v) { + out.push(`${keyIndent}${k}:`); + out.push(`${keyIndent} oneOf: [${v.oneOf.join(", ")}]`); + } else { + out.push(`${keyIndent}${k}: ${v}`); + } + } + return out; +} + function appendLocalesToYaml(source: string, snippet: string): string { const trimmed = source.trimEnd(); return `${trimmed}\n\n${snippet}`; diff --git a/src/types.ts b/src/types.ts index 129a3d0..20277f0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,11 +6,9 @@ export type Scenario = { // An expected argument value: a scalar compared with scalar-normalized // equality, or a `oneOf` list that matches if the actual value equals any option. -export type ArgMatcher = - | string - | number - | boolean - | { oneOf: Array }; +// The scenario parser only ever produces strings; the CLI has no programmatic +// surface, so there are no other producers to model here. +export type ArgMatcher = string | { oneOf: string[] }; export type ToolCallAssertion = { name: string; @@ -23,7 +21,7 @@ export type ScenarioLocale = { toolCall?: ToolCallAssertion | ToolCallAssertion[]; // single or anyOf toolCalls?: ToolCallAssertion[]; // ordered sequence noToolCall?: { - name: string; + names: string[]; // one or more forbidden tool names }; responseLanguage?: string; // BCP-47 locale code; checks response text is in that language }; @@ -40,15 +38,21 @@ export type TargetResponse = { structured: unknown; }; -export type FailureMode = - | "no_tool_call" - | "wrong_tool" - | "wrong_argument" - | "missing_argument" - | "forbidden_tool" - | "wrong_sequence" - | "wrong_language" - | null; +// The single source of truth for behavioral failure modes. Anything that +// aggregates by failure mode (the benchmark, reports) derives its column set +// from this array so a new mode can never silently vanish from a table. +export const FAILURE_MODES = [ + "no_tool_call", + "wrong_tool", + "wrong_argument", + "missing_argument", + "forbidden_tool", + "wrong_sequence", + "wrong_language", + "target_error", // the harness/transport failed, not the model — kept out of behavioral stats +] as const; + +export type FailureMode = (typeof FAILURE_MODES)[number] | null; export type LocaleResult = { locale: string; diff --git a/tests/ci.test.ts b/tests/ci.test.ts index 636180b..652d770 100644 --- a/tests/ci.test.ts +++ b/tests/ci.test.ts @@ -1,6 +1,9 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import test from "node:test"; +// Exercise the real gate function, not a copy, so a regression in cli's +// exit-code logic is actually caught (F-23). +import { shouldFail } from "../src/gate.ts"; import type { LocaleResult } from "../src/types.ts"; function makeResult( @@ -19,23 +22,6 @@ function makeResult( }; } -function shouldFail( - results: LocaleResult[], - minPassRate: number | null, - allowFail: string[], -): boolean { - const counted = results.filter((r) => !allowFail.includes(r.locale)); - - if (minPassRate !== null) { - const totalChecks = counted.reduce((sum, r) => sum + r.total, 0); - const totalPassed = counted.reduce((sum, r) => sum + r.passed, 0); - const rate = totalChecks === 0 ? 100 : (totalPassed / totalChecks) * 100; - return rate < minPassRate; - } - - return counted.some((r) => r.status === "fail"); -} - const passing = makeResult("en", 1, 1); const failing = makeResult("eu", 0, 1); const partial = makeResult("sw", 7, 10); @@ -60,7 +46,7 @@ test("allow-fail does not suppress other failing locales", () => { }); test("min-pass-rate passes when rate is above threshold", () => { - // en 1/1 + sw 7/10 = 8/11 = 72.7% → fails at 70 threshold but passes at 70 + // en 1/1 + sw 7/10 = 8/11 = 72.7%, which is above the 70 threshold → pass assert.equal(shouldFail([passing, partial], 70, []), false); }); diff --git a/tests/io.test.ts b/tests/io.test.ts new file mode 100644 index 0000000..6a7e712 --- /dev/null +++ b/tests/io.test.ts @@ -0,0 +1,317 @@ +import assert from "node:assert/strict"; +import { createServer, type Server } from "node:http"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import test from "node:test"; +import { executeHttpTarget } from "../src/httpTarget.ts"; +import { resolveScenarioPaths, runScenario } from "../src/runner.ts"; +import { lintScenarios } from "../src/lint.ts"; +import { serializeExpect } from "../src/translate.ts"; +import { parseScenario } from "../src/scenario.ts"; +import { formatMarkdownRunReport } from "../src/reportMarkdown.ts"; +import { wilsonInterval } from "../src/stats.ts"; +import type { Scenario } from "../src/types.ts"; + +// Spins up a one-off HTTP server that returns whatever the handler decides, so +// the transport half of the product (httpTarget, runner) is tested for real. +async function withServer( + handler: (body: unknown) => { status: number; json: unknown }, + run: (url: string) => Promise, +): Promise { + const server: Server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + let body: unknown = null; + try { + body = JSON.parse(Buffer.concat(chunks).toString("utf8")); + } catch { + body = null; + } + const { status, json } = handler(body); + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(json)); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + try { + await run(`http://127.0.0.1:${port}/`); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } +} + +const baseInput = { + scenarioId: "s", + locale: "en", + input: "hi", + agent: "support", +}; + +test("httpTarget normalizes a well-formed response", async () => { + await withServer( + () => ({ + status: 200, + json: { text: "ok", toolCalls: [{ name: "t", arguments: { a: 1 } }] }, + }), + async (url) => { + const result = await executeHttpTarget({ ...baseInput, target: url }); + assert.ok(result.ok); + assert.equal(result.response.text, "ok"); + assert.deepEqual(result.response.toolCalls, [ + { name: "t", arguments: { a: 1 } }, + ]); + }, + ); +}); + +test("httpTarget sends the agent field in the POST body", async () => { + const received: Record[] = []; + await withServer( + (body) => { + received.push(body as Record); + return { status: 200, json: { text: "", toolCalls: [] } }; + }, + async (url) => { + await executeHttpTarget({ ...baseInput, target: url }); + }, + ); + assert.equal(received[0].agent, "support"); +}); + +test("httpTarget reports a non-object body as malformed", async () => { + await withServer( + () => ({ status: 200, json: "ok" }), + async (url) => { + const result = await executeHttpTarget({ ...baseInput, target: url }); + assert.equal(result.ok, false); + if (!result.ok) assert.match(result.detail, /malformed response/); + }, + ); +}); + +test("httpTarget reports a non-array toolCalls as malformed", async () => { + await withServer( + () => ({ status: 200, json: { toolCalls: "create_refund" } }), + async (url) => { + const result = await executeHttpTarget({ ...baseInput, target: url }); + assert.equal(result.ok, false); + if (!result.ok) assert.match(result.detail, /toolCalls/); + }, + ); +}); + +test("httpTarget surfaces a non-2xx status", async () => { + await withServer( + () => ({ status: 500, json: { error: "boom" } }), + async (url) => { + const result = await executeHttpTarget({ ...baseInput, target: url }); + assert.equal(result.ok, false); + if (!result.ok) assert.match(result.detail, /HTTP 500.*boom/); + }, + ); +}); + +test("runner classifies transport failure as target_error, not no_tool_call", async () => { + const scenario: Scenario = { + id: "s", + agent: "support", + locales: { + en: { input: "hi", expect: { toolCall: { name: "t" } } }, + }, + }; + // Nothing is listening on this port. + const run = await runScenario(scenario, "http://127.0.0.1:1/", 1, 200); + assert.equal(run.results[0].failureMode, "target_error"); +}); + +test("runner aggregation: first failure is the representative", async () => { + const scenario: Scenario = { + id: "s", + agent: "support", + locales: { + en: { input: "hi", expect: { toolCall: { name: "wanted" } } }, + }, + }; + await withServer( + () => ({ status: 200, json: { text: "", toolCalls: [] } }), + async (url) => { + const run = await runScenario(scenario, url, 3); + const r = run.results[0]; + assert.equal(r.status, "fail"); + assert.equal(r.passed, 0); + assert.equal(r.total, 3); + assert.equal(r.failureMode, "no_tool_call"); + }, + ); +}); + +test("resolveScenarioPaths flags a directory and sorts yaml files", async () => { + const dir = await mkdtemp(join(tmpdir(), "langdrift-")); + try { + await writeFile(join(dir, "b.yaml"), ""); + await writeFile(join(dir, "a.yaml"), ""); + await writeFile(join(dir, "note.txt"), ""); + const resolved = await resolveScenarioPaths(dir); + assert.equal(resolved.isDirectory, true); + assert.deepEqual( + resolved.paths.map((p) => p.split("/").pop()), + ["a.yaml", "b.yaml"], + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveScenarioPaths flags a single file as not a directory", async () => { + const dir = await mkdtemp(join(tmpdir(), "langdrift-")); + try { + const file = join(dir, "one.yaml"); + await writeFile(file, ""); + const resolved = await resolveScenarioPaths(file); + assert.equal(resolved.isDirectory, false); + assert.deepEqual(resolved.paths, [file]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +const SCENARIO_SRC = `id: refund +agent: support +locales: + en: + input: "hi" + expect: + toolCall: + name: create_refund + responseLanguage: en +`; + +test("lint warns on single locale and missing en", async () => { + const dir = await mkdtemp(join(tmpdir(), "langdrift-")); + try { + const file = join(dir, "s.yaml"); + await writeFile( + file, + `id: s\nagent: support\nlocales:\n fr:\n input: "salut"\n expect:\n toolCall:\n name: t\n`, + ); + const [result] = await lintScenarios([file]); + const messages = result.issues.map((i) => i.message).join("\n"); + assert.match(messages, /only 1 locale/); + assert.match(messages, /no "en" locale/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("lint errors on duplicate scenario ids across files", async () => { + const dir = await mkdtemp(join(tmpdir(), "langdrift-")); + try { + const a = join(dir, "a.yaml"); + const b = join(dir, "b.yaml"); + await writeFile(a, SCENARIO_SRC); + await writeFile(b, SCENARIO_SRC); + const results = await lintScenarios([a, b]); + const errors = results.flatMap((r) => + r.issues.filter((i) => i.severity === "error"), + ); + assert.ok( + errors.some((e) => /duplicate scenario id "refund"/.test(e.message)), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("lint warns when responseLanguage is not script-determinable", async () => { + const dir = await mkdtemp(join(tmpdir(), "langdrift-")); + try { + const file = join(dir, "s.yaml"); + await writeFile( + file, + `id: s\nagent: support\nlocales:\n en:\n input: "hi"\n expect:\n responseLanguage: tlh\n`, + ); + const [result] = await lintScenarios([file]); + assert.ok( + result.issues.some((i) => /not script-determinable/.test(i.message)), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("serializeExpect round-trips oneOf, noToolCall anyOf, and responseLanguage", () => { + const src = `id: s +agent: support +locales: + en: + input: "hi" + expect: + toolCall: + name: create_refund + arguments: + reason: + oneOf: [duplicate_charge, double_charge] + noToolCall: + anyOf: [escalate, contact_seller] + responseLanguage: en +`; + const scenario = parseScenario(src); + const lines = serializeExpect(scenario.locales.en, "fr"); + const yaml = [ + "id: s2", + "agent: support", + "locales:", + " fr:", + ' input: "salut"', + " expect:", + ...lines.map((l) => ` ${l}`), + ].join("\n"); + + const reparsed = parseScenario(yaml); + const expect = reparsed.locales.fr.expect; + assert.ok(!Array.isArray(expect.toolCall) && expect.toolCall); + assert.deepEqual(expect.toolCall.arguments, { + reason: { oneOf: ["duplicate_charge", "double_charge"] }, + }); + assert.deepEqual(expect.noToolCall?.names, ["escalate", "contact_seller"]); + // responseLanguage is rewritten to the target locale. + assert.equal(expect.responseLanguage, "fr"); +}); + +test("markdown run report escapes pipes and includes a detail column", () => { + const report = formatMarkdownRunReport({ + scenarioId: "a|b", + target: "http://x/", + iterations: 1, + results: [ + { + locale: "en", + status: "fail", + passed: 0, + failed: 1, + total: 1, + failureMode: "wrong_argument", + detail: "expected a|b, got c", + }, + ], + }); + assert.match(report, /Detail/); + assert.match(report, /a\\\|b/); + assert.match(report, /expected a\\\|b, got c/); +}); + +test("wilsonInterval matches reference values", () => { + assert.equal(wilsonInterval(0, 0), null); + const ci7 = wilsonInterval(7, 10); + assert.ok(ci7); + assert.ok(Math.abs(ci7[0] - 0.397) < 0.005); + assert.ok(Math.abs(ci7[1] - 0.892) < 0.005); + const ci0 = wilsonInterval(0, 10); + assert.ok(ci0); + assert.equal(ci0[0], 0); + assert.ok(Math.abs(ci0[1] - 0.278) < 0.005); +}); diff --git a/tests/scenario.test.ts b/tests/scenario.test.ts index c56147e..9f9f6cc 100644 --- a/tests/scenario.test.ts +++ b/tests/scenario.test.ts @@ -33,10 +33,9 @@ locales: assert.ok(!Array.isArray(tc) && tc !== undefined); assert.equal(tc.name, "create_refund_ticket"); assert.deepEqual(tc.arguments, { reason: "duplicate_charge" }); - assert.equal( - scenario.locales.en.expect.noToolCall?.name, + assert.deepEqual(scenario.locales.en.expect.noToolCall?.names, [ "escalate_to_human", - ); + ]); }); test("fails when the expected tool call is missing", () => { @@ -78,7 +77,7 @@ test("fails when a forbidden tool call appears", () => { const result = assertExpectedToolCall( { toolCall: { name: "create_refund_ticket" }, - noToolCall: { name: "escalate_to_human" }, + noToolCall: { names: ["escalate_to_human"] }, }, { text: "I need a person.", @@ -101,7 +100,7 @@ test("passes when a forbidden tool call is absent", () => { const result = assertExpectedToolCall( { toolCall: { name: "create_refund_ticket" }, - noToolCall: { name: "escalate_to_human" }, + noToolCall: { names: ["escalate_to_human"] }, }, { text: "I can help.", @@ -373,11 +372,9 @@ test("responseLanguage fails when English text is expected to be Japanese", () = structured: null, }, ); - assert.deepEqual(result, { - pass: false, - failureMode: "wrong_language", - detail: "expected ja script in response, got mostly other script", - }); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "wrong_language"); + assert.match(result.detail, /expected ja script/); }); test("responseLanguage passes when Latin-script response matches en locale", () => { @@ -408,12 +405,9 @@ test("responseLanguage fails when Japanese text is expected to be English", () = structured: null, }, ); - assert.deepEqual(result, { - pass: false, - failureMode: "wrong_language", - detail: - "expected Latin-script response (en), got mostly non-Latin characters", - }); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "wrong_language"); + assert.match(result.detail, /expected Latin-script response \(en\)/); }); test("responseLanguage passes when Arabic text matches ar locale", () => { @@ -441,11 +435,9 @@ test("responseLanguage can be the sole assertion", () => { structured: null, }, ); - assert.deepEqual(result, { - pass: true, - failureMode: null, - detail: "responseLanguage: ru", - }); + assert.equal(result.pass, true); + assert.equal(result.failureMode, null); + assert.match(result.detail, /responseLanguage: ru/); }); test("responseLanguage tool call failure surfaces before language check", () => { @@ -872,3 +864,190 @@ test("responseLanguage passes for Latin-vs-Latin (script cannot distinguish lang assert.equal(result.pass, true); }); + +// --- Adversarial-audit regression tests ----------------------------------- + +test("F-1: an array argument does not pass a scalar assertion via String() coercion", () => { + const result = assertExpectedToolCall( + { + toolCall: { + name: "create_refund_ticket", + arguments: { reason: "duplicate_charge" }, + }, + }, + { + text: "", + toolCalls: [ + { + name: "create_refund_ticket", + arguments: { reason: ["duplicate_charge"] }, + }, + ], + structured: null, + }, + ); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "wrong_argument"); + assert.match(result.detail, /array/); +}); + +test("F-2: a fully Amharic reply fails responseLanguage: en", () => { + const result = assertExpectedToolCall( + { responseLanguage: "en" }, + { text: "እኔ በዚህ ማገዝ እችላለሁ።", toolCalls: [], structured: null }, + ); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "wrong_language"); +}); + +test("F-2: a fully Georgian reply fails responseLanguage: en", () => { + const result = assertExpectedToolCall( + { responseLanguage: "en" }, + { + text: "შემიძლია დაგეხმაროთ ამ საკითხში.", + toolCalls: [], + structured: null, + }, + ); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "wrong_language"); +}); + +test("F-3: pure-Chinese (Han only) text fails responseLanguage: ja", () => { + const result = assertExpectedToolCall( + { responseLanguage: "ja" }, + { text: "我可以帮您处理退款。", toolCalls: [], structured: null }, + ); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "wrong_language"); +}); + +test("F-3: Japanese text with kana passes responseLanguage: ja", () => { + const result = assertExpectedToolCall( + { responseLanguage: "ja" }, + { text: "払い戻しのお手伝いをします。", toolCalls: [], structured: null }, + ); + assert.equal(result.pass, true); +}); + +test("F-28: noToolCall anyOf fails when any forbidden tool is called", () => { + const scenario = parseScenario(`id: s +agent: support +locales: + en: + input: "hi" + expect: + toolCall: + name: create_refund + noToolCall: + anyOf: [escalate, contact_seller] +`); + assert.deepEqual(scenario.locales.en.expect.noToolCall?.names, [ + "escalate", + "contact_seller", + ]); + const result = assertExpectedToolCall(scenario.locales.en.expect, { + text: "", + toolCalls: [{ name: "create_refund" }, { name: "contact_seller" }], + structured: null, + }); + assert.equal(result.pass, false); + assert.equal(result.failureMode, "forbidden_tool"); +}); + +test("F-7: oneOf items with quoted commas parse as whole items", () => { + const scenario = parseScenario(`id: s +agent: support +locales: + en: + input: "hi" + expect: + toolCall: + name: t + arguments: + reason: + oneOf: ["a, b", c] +`); + const tc = scenario.locales.en.expect.toolCall; + assert.ok(!Array.isArray(tc) && tc?.arguments); + assert.deepEqual(tc.arguments.reason, { oneOf: ["a, b", "c"] }); +}); + +test("F-9: an empty noToolCall does not steal the next block's name", () => { + const scenario = parseScenario(`id: s +agent: support +locales: + en: + input: "hi" + expect: + noToolCall: + toolCall: + name: create_refund +`); + // noToolCall had no name, so it must not exist; the tool call is intact. + assert.equal(scenario.locales.en.expect.noToolCall, undefined); + const tc = scenario.locales.en.expect.toolCall; + assert.ok(!Array.isArray(tc) && tc?.name === "create_refund"); +}); + +test("F-10: duplicate locale keys are rejected", () => { + assert.throws( + () => + parseScenario(`id: s +agent: support +locales: + en: + input: "a" + expect: + toolCall: + name: t + en: + input: "b" + expect: + toolCall: + name: t +`), + /duplicate locale "en"/, + ); +}); + +test("F-11: a tool-call list item without a name is rejected", () => { + assert.throws( + () => + parseScenario(`id: s +agent: support +locales: + en: + input: "hi" + expect: + toolCalls: + - name: first + - nme: typo +`), + /missing "name"/, + ); +}); + +test("F-8: tab indentation is rejected with a clear message", () => { + assert.throws( + () => parseScenario("id: s\nagent: support\nlocales:\n\ten:\n"), + /tab indentation/, + ); +}); + +test("F-8: block scalars are rejected with a clear message", () => { + assert.throws( + () => + parseScenario(`id: s +agent: support +locales: + en: + input: | + multi + expect: + toolCall: + name: t +`), + /block scalars/, + ); +});