From 1f262b4d748b4bf2b1009c50359253940457358d Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 9 May 2022 10:08:51 +0200 Subject: [PATCH 01/25] Bump dev version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 524336a7..d46ef5a2 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { } allprojects { - version = '0.15.0' + version = '0.15.1-SNAPSHOT' group = 'org.radarbase' } From 446621e517e0d713b5646b6e35007e1a89a9874e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 3 Oct 2022 13:20:38 +0200 Subject: [PATCH 02/25] Bumped dependencies --- build.gradle | 4 ++-- gradle.properties | 18 +++++++++--------- gradle/wrapper/gradle-wrapper.jar | Bin 59821 -> 60756 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 6 ++++++ gradlew.bat | 14 ++++++++------ 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index d46ef5a2..2a9d23fd 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ * limitations under the License. */ plugins { - id 'com.github.davidmc24.gradle.plugin.avro' version '1.3.0' + id 'com.github.davidmc24.gradle.plugin.avro' version '1.5.0' id("io.github.gradle-nexus.publish-plugin") version "1.1.0" id("com.github.ben-manes.versions") version "0.42.0" } @@ -121,5 +121,5 @@ nexusPublishing { } wrapper { - gradleVersion '7.4.2' + gradleVersion '7.5.1' } diff --git a/gradle.properties b/gradle.properties index dd09c477..2b70a273 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,12 +1,12 @@ -slf4jVersion=1.7.36 -confluentVersion=7.1.1 -kafkaVersion=3.1.0 -avroVersion=1.11.0 -jacksonVersion=2.13.2.20220328 -okhttpVersion=4.9.3 +slf4jVersion=2.0.3 +confluentVersion=7.2.2 +kafkaVersion=3.3.1 +avroVersion=1.11.1 +jacksonVersion=2.13.4 +okhttpVersion=4.10.0 junitVersion=4.13.2 -mockitoVersion=4.5.1 +mockitoVersion=4.8.0 hamcrestVersion=1.3 radarSchemasVersion=0.7.9 -orgJsonVersion=20220320 -opencsvVersion=5.6 +orgJsonVersion=20220924 +opencsvVersion=5.7.0 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 41d9927a4d4fb3f96a785543079b8df6723c946b..249e5832f090a2944b7473328c07c9755baa3196 100644 GIT binary patch delta 10197 zcmaKS1ymhDwk=#NxVyW%y9U<)A-Dv)xI0|j{UX8L-JRg>5ZnnKAh;%chM6~S-g^K4 z>eZ{yK4;gd>gwvXs=Id8Jk-J}R4pT911;+{Jp9@aiz6!p1Oz9z&_kGLA%J5%3Ih@0 zQ|U}%$)3u|G`jIfPzMVfcWs?jV2BO^*3+q2><~>3j+Z`^Z%=;19VWg0XndJ zwJ~;f4$;t6pBKaWn}UNO-wLCFHBd^1)^v%$P)fJk1PbK5<;Z1K&>k~MUod6d%@Bq9 z>(44uiaK&sdhwTTxFJvC$JDnl;f}*Q-^01T508(8{+!WyquuyB7R!d!J)8Ni0p!cV6$CHsLLy6}7C zYv_$eD;)@L)tLj0GkGpBoa727hs%wH$>EhfuFy{_8Q8@1HI%ZAjlpX$ob{=%g6`Ox zLzM!d^zy`VV1dT9U9(^}YvlTO9Bf8v^wMK37`4wFNFzW?HWDY(U(k6@tp(crHD)X5>8S-# zW1qgdaZa*Sh6i%60e1+hty}34dD%vKgb?QmQiZ=-j+isA4={V_*R$oGN#j|#ia@n6 zuZx4e2Xx?^lUwYFn2&Tmbx0qA3Z8;y+zKoeQu;~k~FZGy!FU_TFxYd!Ck;5QvMx9gj5fI2@BLNp~Ps@ zf@k<&Q2GS5Ia9?_D?v~$I%_CLA4x~eiKIZ>9w^c#r|vB?wXxZ(vXd*vH(Fd%Me8p( z=_0)k=iRh%8i`FYRF>E97uOFTBfajv{IOz(7CU zv0Gd84+o&ciHlVtY)wn6yhZTQQO*4Mvc#dxa>h}82mEKKy7arOqU$enb9sgh#E=Lq zU;_RVm{)30{bw+|056%jMVcZRGEBSJ+JZ@jH#~DvaDQm92^TyUq=bY*+AkEakpK>8 zB{)CkK48&nE5AzTqT;WysOG|!y}5fshxR8Ek(^H6i>|Fd&wu?c&Q@N9ZrJ=?ABHI! z`*z8D`w=~AJ!P-9M=T}f`;76$qZRllB&8#9WgbuO$P7lVqdX1=g*t=7z6!0AQ^ux_ z9rcfUv^t}o_l-ZE+TqvqFsA*~W<^78!k;~!i8(eS+(+@u8FxK+Q7;mHZ<1}|4m<}vh@p`t%|@eM_J(P% zI>M7C)Ir{l|J;$G_EGGEhbP4?6{sYzMqBv+x95N&YWFH6UcE@b}B?q)G*4<4mR@sy1#vPnLMK51tb#ED(8TA1nE zYfhK7bo1!R5WJF$5Y?zG21)6+_(_5oSX9sGIW;(O&S?Rh(nydNQYzKjjJ54aDJ-1F zrJ=np8LsN?%?Rt7f~3aAX!2E{`fh_pb?2(;HOB3W+I*~A>W%iY+v45+^e$cE10fA} zXPvw9=Bd+(;+!rl)pkYj0HGB}+3Z!Mr;zr%gz~c-hFMv8b2VRE2R$8V=_XE zq$3=|Yg05(fmwrJ)QK2ptB4no`Y8Dg_vK2QDc6-6sXRQ5k78-+cPi-fH}vpgs|Ive zE=m*XNVs?EWgiNI!5AcD*3QMW)R`EqT!f0e1%hERO&?AT7HWnSf5@#AR{OGuXG3Zb zCnVWg7h|61lGV3k+>L<#d>)InG>ETn1DbOHCfztqzQ_fBiaUt@q6VMy={Fe-w#~2- z0?*f|z$zgjI9>+JVICObBaK=pU}AEOd@q(8d?j7zQFD@=6t`|KmolTr2MfBI$;EGh zD%W0cA_d#V6Lb$us5yIG(|d>r-QleC4;%hEu5W9hyY zY#+ESY&v`8(&mC~?*|e5WEhC!YU2>m_}`K+q9)a(d$bsS<=YkyZGp}YA%TXw>@abA zS_poVPoN+?<6?DAuCNt&5SHV(hp56PJ})swwVFZFXM->F zc|0c8<$H_OV%DR|y7e+s$12@Ac8SUClPg8_O9sTUjpv%6Jsn5vsZCg>wL+db4c+{+ zsg<#wOuV4jeOq`veckdi-1`dz;gvL)bZeH|D*x=8UwRU5&8W1@l>3$)8WzET0%;1J zM3(X<7tKK&9~kWRI{&FmwY5Gg!b5f4kI_vSm)H1#>l6M+OiReDXC{kPy!`%Ecq-+3yZTk=<` zm)pE6xum5q0Qkd#iny0Q-S}@I0;mDhxf>sX)Oiv)FdsAMnpx%oe8OQ`m%Xeozdzx!C1rQR>m1c_}+J4x)K}k{G zo68;oGG&Ox7w^-m7{g4a7NJu-B|~M;oIH~~#`RyUNm##feZH;E?pf}nshmoiIY52n z%pc%lnU4Q#C=RUz)RU6}E_j4#)jh<&a%JyJj$Fufc#&COaxFHtl}zJUGNLBu3~_@1 zn9F^JO9);Duxo&i@>X(kbYga1i>6p1fca8FzQ0>((Lb-aPUbC*d~a03V$y;*RBY!R ziEJ2IF^FjrvO}0Uy{cMn%u<+P5U!UO>pm9#ZYL5i6|xSC+np7IH$GfXs&uI;y4as@ z&AzJh>(S2?3PKKgab3Z(`xbx(C#46XIvVcW8eG_DjT~}Yz_8PWZ`uf6^Xr=vkvL_` zqmvfgJL+Zc`;iq~iP?%@G7}~fal-zqxa0yNyHBJJ5M)9bI>7S_cg?Ya&p(I)C5Ef4 zZ>YAF6x|U=?ec?g*|f2g5Tw3PgxaM_bi_5Az9MO$;_Byw(2d}2%-|bg4ShdQ;)Z|M z4K|tFv)qx*kKGKoyh!DQY<{n&UmAChq@DJrQP>EY7g1JF(ih*D8wCVWyQ z5Jj^|-NVFSh5T0vd1>hUvPV6?=`90^_)t(L9)XOW7jeP45NyA2lzOn&QAPTl&d#6P zSv%36uaN(9i9WlpcH#}rmiP#=L0q(dfhdxvFVaOwM;pY;KvNQ9wMyUKs6{d}29DZQ z{H3&Sosr6)9Z+C>Q5)iHSW~gGoWGgK-0;k~&dyr-bA3O|3PCNzgC?UKS_B=^i8Ri^ zd_*_qI4B07Cayq|p4{`U_E_P=K`N_~{F|+-+`sCgcNxs`%X!$=(?l2aAW}0M=~COb zf19oe^iuAUuDEf)4tgv<=WRPpK@IjToNNC*#&Ykw!)aqWU4h#|U@(cG_=Qx+&xt~a zvCz~Ds3F71dsjNLkfM%TqdVNu=RNMOzh7?b+%hICbFlOAPphrYy>7D-e7{%o_kPFn z;T!?ilE-LcKM0P(GKMseEeW57Vs`=FF}(y@^pQl;rL3fHs8icmA+!6YJt&8 ztSF?%Un35qkv>drkks&BNTJv~xK?vD;aBkp7eIkDYqn+G0%;sT4FcwAoO+vke{8CO z0d76sgg$CannW5T#q`z~L4id)9BCKRU0A!Z-{HpXr)QJrd9@iJB+l32Ql)Z}*v(St zE)Vp=BB=DDB4Pr}B(UHNe31<@!6d{U?XDoxJ@S)9QM)2L%SA0x^~^fb=bdsBy!uh& zU?M_^kvnt%FZzm+>~bEH{2o?v&Iogs`1t-b+Ml`J!ZPS(46YQJKxWE81O$HE5w;** z|8zM%bp`M7J8)4;%DqH`wVTmM0V@D}xd%tRE3_6>ioMJxyi5Hkb>85muF81&EY!73ei zA3e<#ug||EZJ=1GLXNJ)A z791&ge#lF;GVX6IU?iw0jX^1bYaU?+x{zPlpyX6zijyn*nEdZ$fxxkl!a-~*P3bkf zPd*pzu~3GBYkR_>ET`5UM^>>zTV>5m>)f=az{d0sg6a8VzUtXy$ZS?h#Gk-CA?7)c zI%Vu9DN6XSDQn6;?n9`>l$q&>s?K)R8*OsmI+$L_m z_~E`}w694Z*`Xk3Ne=497Si~=RWRqCM?6=88smrxle#s*W znwhTRsMRmg?37GLJ-)%nDZA7r$YG849j8mJWir1bWBy& zZPneYojSbooC8U@tkO`bWx4%E5*;p#Q^1^S3lsfy7(6A{jL0`A__0vm?>xC%1y8_m z57FfWr^@YG2I1K7MGYuYd>JC}@sT2n^rkrY3w%~$J$Y~HSoOHn?zpR$ zjLj_bq@Yj8kd~DXHh30KVbz@K)0S;hPKm+S&-o%IG+@x@MEcrxW2KFh;z^4dJDZix zGRGe&lQD$p)0JVF4NRgGYuh0bYLy)BCy~sbS3^b3 zHixT<%-Vwbht|25T{3^Hk;qZ^3s!OOgljHs+EIf~C%=_>R5%vQI4mQR9qOXThMXlU zS|oSH>0PjnCakb*js2{ObN`}%HYsT6=%(xA| znpUtG_TJ08kHgm5l@G|t?4E3tG2fq?wNtIp*Vqrb{9@bo^~Rx7+J&OnayrX`LDcF~ zd@0m0ZJ#Z@=T>4kTa5e2FjI&5c(F7S{gnRPoGpu9eIqrtSvnT_tk$8T)r%YwZw!gK zj*k@cG)V&@t+mtDi37#>LhVGTfRA^p%x0d#_P|Mktz3*KOoLIqFm`~KGoDDD4OOxe z?}ag_c08u%vu=5Vx=~uoS8Q;}+R2~?Uh|m-+`-2kDo$d6T!nD*hc#dB(*R{LXV=zo z`PJP0V=O!@3l-bw+d`X6(=@fq=4O#ETa8M^fOvO4qja9o3e8ANc9$sI=A4$zUut~w z4+JryRkI{9qWxU1CCMM$@Aj=6)P+z?vqa=UCv_4XyVNoBD{Xb~Oi4cjjhm8fRD!*U z2)zaS;AI78^Wq+5mDInKiMz|z#K`2emQfNH*U;{9^{NqSMVoq?RSo43<8YpJM^+W$ zxy!A5>5Zl16Vi#?nAYywu3w_=KWnd3*QetocWt`3pK67>)ZVwnT3h zbPdD&MZkD?q=-N`MpCCwpM74L+Tr1aa)zJ)8G;(Pg51@U&5W>aNu9rA`bh{vgfE={ zdJ>aKc|2Ayw_bop+dK?Y5$q--WM*+$9&3Q9BBiwU8L<-`T6E?ZC`mT0b}%HR*LPK} z!MCd_Azd{36?Y_>yN{U1w5yrN8q`z(Vh^RnEF+;4b|2+~lfAvPT!`*{MPiDioiix8 zY*GdCwJ{S(5(HId*I%8XF=pHFz<9tAe;!D5$Z(iN#jzSql4sqX5!7Y?q4_%$lH zz8ehZuyl0K=E&gYhlfFWabnSiGty$>md|PpU1VfaC5~kskDnZX&Yu}?-h;OSav=8u z=e3Yq=mi$4A|sB-J00;1d{Sd1+!v0NtU((Nz2;PFFlC}V{@p&4wGcVhU&nI($RAS! zwXn7)?8~1J3*4+VccRSg5JS<(bBhBM&{ELMD4C_NTpvzboH!{Zr*%HP;{UqxI#g&7 zOAqPSW5Qus$8-xtTvD%h{Tw<2!XR(lU54LZG{)Cah*LZbpJkA=PMawg!O>X@&%+5XiyeIf91n2E*hl$k-Y(3iW*E}Mz-h~H~7S9I1I zR#-j`|Hk?$MqFhE4C@=n!hN*o5+M%NxRqP+aLxDdt=wS6rAu6ECK*;AB%Nyg0uyAv zO^DnbVZZo*|Ef{nsYN>cjZC$OHzR_*g%T#oF zCky9HJS;NCi=7(07tQXq?V8I&OA&kPlJ_dfSRdL2bRUt;tA3yKZRMHMXH&#W@$l%-{vQd7y@~i*^qnj^`Z{)V$6@l&!qP_y zg2oOd!Wit#)2A~w-eqw3*Mbe)U?N|q6sXw~E~&$!!@QYX4b@%;3=>)@Z#K^`8~Aki z+LYKJu~Y$;F5%_0aF9$MsbGS9Bz2~VUG@i@3Fi2q(hG^+Ia44LrfSfqtg$4{%qBDM z_9-O#3V+2~W$dW0G)R7l_R_vw(KSkC--u&%Rs^Io&*?R=`)6BN64>6>)`TxyT_(Rd zUn+aIl1mPa#Jse9B3`!T=|e!pIp$(8ZOe0ao?nS7o?oKlj zypC-fMj1DHIDrh1unUI1vp=-Fln;I9e7Jvs3wj*^_1&W|X} zZSL|S|Bb@CV*YC_-T&2!Ht3b6?)d`tHOP?rA;;t#zaXa0Sc;vGnV0BLIf8f-r{QHh z*Zp`4_ItlOR7{u(K+!p_oLDmaAkNag*l4#29F2b_A*0oz0T|#-&f*;c#<`^)(W@gm z#k9k=t%u8<+C1fNUA{Fh7~wgPrEZZ#(6aBI%6bR4RO(e1(ZocjoDek4#MTgZD>1NG zy9~yoZfWYfwe&S-(zk4o6q6o?2*~DOrJ(%5wSnEJMVOKCzHd z=Yhm+HLzoDl{P*Ybro7@sk1!Ez3`hE+&qr7Rw^2glw^M(b(NS2!F|Q!mi|l~lF94o z!QiV)Q{Z>GO5;l1y!$O)=)got;^)%@v#B!ZEVQy1(BJApHr5%Zh&W|gweD+%Ky%CO ztr45vR*y(@*Dg_Qw5v~PJtm^@Lyh*zRuT6~(K+^HWEF{;R#L$vL2!_ndBxCtUvZ(_ zauI7Qq}ERUWjr&XW9SwMbU>*@p)(cuWXCxRK&?ZoOy>2VESII53iPDP64S1pl{NsC zD;@EGPxs&}$W1;P6BB9THF%xfoLX|4?S;cu@$)9OdFst-!A7T{(LXtdNQSx!*GUSIS_lyI`da8>!y_tpJb3Zuf0O*;2y?HCfH z5QT6@nL|%l3&u4;F!~XG9E%1YwF*Fgs5V&uFsx52*iag(?6O|gYCBY3R{qhxT-Etb zq(E%V=MgQnuDGEKOGsmBj9T0-nmI%zys8NSO>gfJT4bP>tI>|ol@ zDt(&SUKrg%cz>AmqtJKEMUM;f47FEOFc%Bbmh~|*#E zDd!Tl(wa)ZZIFwe^*)4>{T+zuRykc3^-=P1aI%0Mh}*x7%SP6wD{_? zisraq`Las#y-6{`y@CU3Ta$tOl|@>4qXcB;1bb)oH9kD6 zKym@d$ zv&PZSSAV1Gwwzqrc?^_1+-ZGY+3_7~a(L+`-WdcJMo>EWZN3%z4y6JyF4NR^urk`c z?osO|J#V}k_6*9*n2?j+`F{B<%?9cdTQyVNm8D}H~T}?HOCXt%r7#2hz97Gx#X%62hyaLbU z_ZepP0<`<;eABrHrJAc!_m?kmu#7j}{empH@iUIEk^jk}^EFwO)vd7NZB=&uk6JG^ zC>xad8X$h|eCAOX&MaX<$tA1~r|hW?-0{t4PkVygTc`yh39c;&efwY(-#;$W)+4Xb z$XFsdG&;@^X`aynAMxsq)J#KZXX!sI@g~YiJdHI~r z$4mj_?S29sIa4c$z)19JmJ;Uj?>Kq=0XuH#k#};I&-6zZ_&>)j>UR0XetRO!-sjF< zd_6b1A2vfi++?>cf}s{@#BvTD|a%{9si7G}T+8ZnwuA z1k8c%lgE<-7f~H`cqgF;qZ|$>R-xNPA$25N1WI3#n%gj}4Ix}vj|e=x)B^roGQpB) zO+^#nO2 zjzJ9kHI6nI5ni&V_#5> z!?<7Qd9{|xwIf4b0bRc;zb}V4>snRg6*wl$Xz`hRDN8laL5tg&+@Dv>U^IjGQ}*=XBnXWrwTy;2nX?<1rkvOs#u(#qJ=A zBy>W`N!?%@Ay=upXFI}%LS9bjw?$h)7Dry0%d}=v0YcCSXf9nnp0tBKT1eqZ-4LU` zyiXglKRX)gtT0VbX1}w0f2ce8{$WH?BQm@$`ua%YP8G@<$n13D#*(Yd5-bHfI8!on zf5q4CPdgJLl;BqIo#>CIkX)G;rh|bzGuz1N%rr+5seP${mEg$;uQ3jC$;TsR&{IX< z;}7j3LnV+xNn^$F1;QarDf6rNYj7He+VsjJk6R@0MAkcwrsq4?(~`GKy|mgkfkd1msc2>%B!HpZ~HOzj}kl|ZF(IqB=D6ZTVcKe=I7)LlAI=!XU?J*i#9VXeKeaG zwx_l@Z(w`)5Cclw`6kQKlS<;_Knj)^Dh2pL`hQo!=GPOMR0iqEtx12ORLpN(KBOm5 zontAH5X5!9WHS_=tJfbACz@Dnkuw|^7t=l&x8yb2a~q|aqE_W&0M|tI7@ilGXqE)MONI8p67OiQGqKEQWw;LGga=ZM1;{pSw1jJK_y$vhY6 ztFrV7-xf>lbeKH1U)j3R=?w*>(Yh~NNEPVmeQ8n}0x01$-o z2Jyjn+sXhgOz>AzcZ zAbJZ@f}MBS0lLKR=IE{z;Fav%tcb+`Yi*!`HTDPqSCsFr>;yt^^&SI2mhKJ8f*%ji zz%JkZGvOn{JFn;)5jf^21AvO-9nRzsg0&CPz;OEn07`CfT@gK4abFBT$Mu?8fCcscmRkK+ zbAVJZ~#_a z{|(FFX}~8d3;DW8zuY9?r#Dt>!aD>} zlYw>D7y#eDy+PLZ&XKIY&Df0hsLDDi(Yrq8O==d30RchrUw8a=Eex>Dd?)3+k=}Q> z-b85lun-V$I}86Vg#l1S@1%=$2BQD5_waAZKQfJ${3{b2SZ#w1u+jMr{dJMvI|Og= zpQ9D={XK|ggbe04zTUd}iF{`GO1dV%zWK~?sM9OM(= zVK9&y4F^w1WFW{$qi|xQk0F`@HG8oLI5|5$j~ci9xTMT69v5KS-Yym--raU5kn2#C z<~5q^Bf0rTXVhctG2%&MG(cUGaz(gC(rcG~>qgO$W6>!#NOVQJ;pIYe-lLy(S=HgI zPh;lkL$l+FfMHItHnw_^bj8}CKM19t(C_2vSrhX2$K@-gFlH};#C?1;kk&U1L%4S~ zR^h%h+O1WE7DI$~dly?-_C7>(!E`~#REJ~Xa7lyrB$T!`&qYV5QreAa^aKr%toUJR zPWh)J3iD`(P6BI5k$oE$us#%!4$>`iH2p-88?WV0M$-K)JDibvA4 zpef%_*txN$Ei3=Lt(BBxZ&mhl|mUz-z*OD1=r9nfN zc5vOMFWpi>K=!$6f{eb?5Ru4M3o;t9xLpry|C%j~`@$f)OFB5+xo8XM8g&US@UU-sB|dAoc20y(F@=-2Ggp_`SWjEb#>IG^@j zuQK}e^>So#W2%|-)~K!+)wdU#6l>w5wnZt2pRL5Dz#~N`*UyC9tYechBTc2`@(OI# zNvcE*+zZZjU-H`QOITK^tZwOyLo)ZCLk>>Wm+flMsr5X{A<|m`Y281n?8H_2Fkz5}X?i%Rfm5s+n`J zDB&->=U+LtOIJ|jdYXjQWSQZFEs>Rm{`knop4Sq)(}O_@gk{14y51)iOcGQ5J=b#e z2Yx^6^*F^F7q_m-AGFFgx5uqyw6_4w?yKCJKDGGprWyekr;X(!4CnM5_5?KgN=3qCm03 z##6k%kIU5%g!cCL(+aK>`Wd;dZ4h$h_jb7n?nqx5&o9cUJfr%h#m4+Bh)>HodKcDcsXDXwzJ3jR(sSFqWV(OKHC*cV8;;&bH=ZI0YbW3PgIHwTjiWy z?2MXWO2u0RAEEq(zv9e%Rsz|0(OKB?_3*kkXwHxEuazIZ7=JhaNV*P~hv57q55LoebmJpfHXA@yuS{Esg+ z*C}0V-`x^=0nOa@SPUJek>td~tJ{U1T&m)~`FLp*4DF77S^{|0g%|JIqd-=5)p6a` zpJOsEkKT(FPS@t^80V!I-YJbLE@{5KmVXjEq{QbCnir%}3 zB)-J379=wrBNK6rbUL7Mh^tVmQYn-BJJP=n?P&m-7)P#OZjQoK0{5?}XqJScV6>QX zPR>G{xvU_P;q!;S9Y7*07=Z!=wxIUorMQP(m?te~6&Z0PXQ@I=EYhD*XomZ^z;`Os z4>Uh4)Cg2_##mUa>i1Dxi+R~g#!!i{?SMj%9rfaBPlWj_Yk)lCV--e^&3INB>I?lu z9YXCY5(9U`3o?w2Xa5ErMbl5+pDVpu8v+KJzI9{KFk1H?(1`_W>Cu903Hg81vEX32l{nP2vROa1Fi!Wou0+ZX7Rp`g;B$*Ni3MC-vZ`f zFTi7}c+D)!4hz6NH2e%%t_;tkA0nfkmhLtRW%){TpIqD_ev>}#mVc)<$-1GKO_oK8 zy$CF^aV#x7>F4-J;P@tqWKG0|D1+7h+{ZHU5OVjh>#aa8+V;6BQ)8L5k9t`>)>7zr zfIlv77^`Fvm<)_+^z@ac%D&hnlUAFt8!x=jdaUo{)M9Ar;Tz5Dcd_|~Hl6CaRnK3R zYn${wZe8_BZ0l0c%qbP}>($jsNDay>8+JG@F!uV4F;#zGsBP0f$f3HqEHDz_sCr^q z1;1}7KJ9&`AX2Qdav1(nNzz+GPdEk5K3;hGXe{Hq13{)c zZy%fFEEH#nlJoG{f*M^#8yXuW%!9svN8ry-Vi7AOFnN~r&D`%6d#lvMXBgZkX^vFj z;tkent^62jUr$Cc^@y31Lka6hS>F?1tE8JW$iXO*n9CQMk}D*At3U(-W1E~z>tG?> z5f`5R5LbrhRNR8kv&5d9SL7ke2a*Xr)Qp#75 z6?-p035n2<7hK;sb>t9GAwG4{9v~iEIG>}7B5zcCgZhu$M0-z8?eUO^E?g)md^XT_ z2^~-u$yak>LBy(=*GsTj6p<>b5PO&un@5hGCxpBQlOB3DpsItKZRC*oXq-r{u}Wb; z&ko>#fbnl2Z;o@KqS-d6DTeCG?m1 z&E>p}SEc*)SD&QjZbs!Csjx~0+$@ekuzV_wAalnQvX3a^n~3ui)|rDO+9HW|JPEeBGP4 z)?zcZ<8qv47`EWA*_X~H^vr(lP|f%=%cWFM;u)OFHruKT<~?>5Y8l?56>&;=WdZU# zZEK4-C8s-3zPMA^&y~e*9z)!ZJghr3N^pJa2A$??Xqx-BR*TytGYor&l8Q+^^r%Yq02xay^f#;;wO6K7G!v>wRd6531WnDI~h$PN( z+4#08uX?r&zVKsQ;?5eBX=FxsXaGyH4Gth4a&L|{8LnNCHFr1M{KjJ!BfBS_aiy-E zxtmNcXq3}WTwQ7Dq-9YS5o758sT(5b`Sg-NcH>M9OH1oW6&sZ@|GYk|cJI`vm zO<$~q!3_$&GfWetudRc*mp8)M)q7DEY-#@8w=ItkApfq3sa)*GRqofuL7)dafznKf zLuembr#8gm*lIqKH)KMxSDqbik*B(1bFt%3Vv|ypehXLCa&wc7#u!cJNlUfWs8iQ` z$66(F=1fkxwg745-8_eqV>nWGY3DjB9gE23$R5g&w|C{|xvT@7j*@aZNB199scGchI7pINb5iyqYn)O=yJJX)Ca3&Ca+{n<=1w|(|f0)h<9gs$pVSV<<9Og-V z8ki@nKwE)x)^wmHBMk?mpMT=g{S#^8W|>&rI#Ceh;9za}io0k@0JxiCqi-jHlxbt3 zjJA?RihhRvhk6%G5-D{ePh1jare*fQS<328P-DcVAxPTrw=n6k?C6EV75f}cnBRPT zMYDqqKu(ND&aOtc!QRV`vzJSVxx8i~WB#5Ml{b#eQqNnSi7l-bS-`ITW<^zyYQA(b zbj4SuRK>q9o`_v%+C=S?h>2e4!66Ij(P5{7Uz$3u6YJJC$W%EoBa{-(=tQ|y1vov%ZkXVOV z##_UVg4V^4ne#4~<-1DkJqkKqgT+E_=&4Ue&eQ-JC+gi?7G@d6= zximz{zE)WW{b@QCJ!7l&N5x=dXS?$5RBU-VvN4Uec-GHK&jPa&P2z+qDdLhIB+HU) zu0CW&uLvE^4I5xtK-$+oe|58)7m6*PO%Xt<+-XEA%jG_BEachkF3e@pn?tl!`8lOF zbi2QOuNXX)YT*MCYflILO{VZ*9GiC%R4FO20zMK?p+&aCMm2oeMK7(aW=UDzr=AO0 z$5mJ%=qRsR8rZ>_YsL+vi{3*J_9Kzq(;ZwRj+4_f0-*wbkSMPWahX#Fj_a8BnrhJ6 zo^ZZ?Vah1@&6#r=JkuaYDBdp;J3@ii+CHM&@9*er&#P}$@wI$bfrH)&c!*|nkvhf%^*Y6b%dKz%QBSIo@U z{?V^qEs4`q<8@n+u8YiB^sc@6g>TncG<|GsmC3egwE6aO=EwLr~3-2 zNr`+)`i+-83?|1Xy0^8ps&pb}YT?w1eWVnC9Ps1=KM;Rw)bH6O!7Did1NwpnqVPZc z*%Qo~qkDL>@^<^fmIBtx$WUWQiNtAB2x-LO^BB=|w~-zTnJNEdm1Ou(?8PF&U88X@ z#8rdaTd||)dG^uJw~N_-%!XNbuAyh4`>Shea=pSj0TqP+w4!`nxsmVSv02kb`DBr% zyX=e>5IJ3JYPtdbCHvKMdhXUO_*E9jc_?se7%VJF#&ZaBD;7+eFN3x+hER7!u&`Wz z7zMvBPR4y`*$a250KYjFhAKS%*XG&c;R-kS0wNY1=836wL6q02mqx;IPcH(6ThA@2 zXKQF|9H>6AW$KUF#^A%l6y5{fel77_+cR_zZ0(7=6bmNXABv}R!B-{(E^O6Y?ZS)n zs1QEmh_Fm7p}oRyT3zxUNr4UV8NGs+2b8|4shO$OGFj3D&7_e?#yDi=TTe%$2QbG5 zk<;q7aQ;p!M-Osm{vFdmXZ@!z9uWh!;*%>(vTRggufuUGP9Hols@vhx z73pn$3u2;vzRvnXuT&$Os7J@6y12*j!{ix%3B4YU1466ItmJs0NsU(4ZYRYh7wEA6q{b*Hs6@k~ zi7Yq@Ax!et0cUMTvk7P%ym){MHpcliHEI~e3HP0NV=}7;xFv#IC?a<=`>~j_sk{e> z7vg-tK*p83HZ0=QK@ zRIHo^r{D8&Ms-^WZp+6US_Quqjh$Q66W^1}=Uz&XJ8AQE9&2}P zY|FXZzZ|0IiaBd2qdt6dIjQr(ZMIOU%NG1F&fu6Po9m^?BvLhI6T0R!H2d8;U(&p2 zYA|MFscMqcO(ye~Jp?F;0>Ke+5hzVr?aBNe>GsGgr$XrpS9uajN2kNQ3o$V5rp0T( z0$6TJC;3)26SNG#XcX7l^MKTn$ga?6r4Jzfb%ZgA(Zbwit0$kY=avSnI$@Gk%+^pu zS5mHrcRS8LFPC*uVWH4DDD1pY$H8N>X?KIJZuZ2SvTqc5Nr0GHdD8TCJcd$zIhOdC zZX0ErnsozQh;t^==4zTfrZO421AL?)O)l#GSxU#|LTTg4#&yeK=^w#;q63!Nv~1(@ zs^-RNRuF&qgcr+bIzc@7$h9L;_yjdifE*$j0Q&Np=1AuHL--zdkv@}`1 zo~LlDl_YAq*z?vmr4M`GjDkl9?p|-tl(DtX76oZv25_DtZutLS9Ez!5~p?th@4 zyc_uax4W#<(#)LMkvo)yp|5tKsC2=p#6PyhpH|449T<9Zdk|%CAb5cw?fhvQtBO&7 zpQ9$24yLqPHP;$N&fe2wm%8qdctwIna<3SwGtQA3{C77s%CW%LYxtK(SBGustL0<( zu~U9r0UOkr(c{OJxZS0Ntu3+cJlF7R`7k-Bsa&q?9Ae5{{|o~?cM+T7{lB1^#vT8R z?>c9fNWey`1dKDY%F3d2O*8^qYhjlB8*7HMKE<*=(A`{>=1%s1}Pm&#_t1xy!FkPk@%SMEka2@*= zxDuM|vJJ5s+xgDls{>*o!7eOcs|xuVBPWX&+y5vEiADK%hi`#Dbd>;;Pbk2H4*-X&R?_-6ZEutSd8hC+sSjhIo z;D(j4P;2EVpEj#UF7IjM6PC+X$C5T&=nL`*!*hm9U)#O?>wqOgC>jXKN3Slk_yaQX zLf|4D8T4k|wHW`;#ZQVocNF|3izi0sOqXzi7@KlYC3CXBG`94wD;tMI1bj|8Vm zY}9`VI9!plSfhAal$M_HlaYOVNU?9Z#0<$o?lXXbX3O(l_?f)i3_~r+GcO-x#+x^X zfsZl0>Rj2iP1rsT;+b;Mr? z4Vu&O)Q5ru4j;qaSP5gA{az@XTS1NpT0d9Xhl_FkkRpcEGA0(QQ~YMh#&zwDUkNzm z6cgkdgl9W{iL6ArJ1TQHqnQ^SQ1WGu?FT|93$Ba}mPCH~!$3}0Y0g zcoG%bdTd$bmBx9Y<`Jc+=Cp4}c@EUfjiz;Rcz101p z=?#i$wo>gBE9|szaZMt-d4nUIhBnYRuBVyx+p?5#aZQgUe(!ah`J#l1$%bl5avL27 zU2~@V`3Ic&!?FhDX@Cw!R4%xtWark#p8DLT)HCZ?VJxf^yr@AD*!ERK3#L$E^*Yr? zzN&uF9Roh4rP+r`Z#7U$tzl6>k!b~HgM$C<_crP=vC>6=q{j?(I}!9>g3rJU(&){o z`R^E*9%+kEa8H_fkD9VT7(Fks&Y-RcHaUJYf-|B+eMXMaRM;{FKRiTB>1(=Iij4k1(X__|WqAd-~t#2@UQ}Z&<1Th0azdXfoll!dd)6>1miA z!&=6sDJm=e$?L&06+Q3`D-HNSkK-3$3DdZMX-6Xjn;wd#9A{~ur!2NcX>(qY_oZL0~H7dnQ9sgLe!W>~2|RSW7|hWn<({Pg*xF$%B-!rKe^_R_vc z(LO!0agxxP;FWPV({8#lEv$&&GVakGus=@!3YVG`y^AO1m{2%Np;>HNA1e{=?ra1C}H zAwT0sbwG|!am;fl?*_t^^#yLDXZ*Nx)_FqueZi0c-G~omtpHW0Cu)mEJ`Z1X8brq$ z%vK##b~o*^b&Hz!hgrD=^6P8}aW40lhzMLB5T5*v`1QH?+L~-@CDi3+C@nRf2{7UE zyDIe{@LKw`Eu=Z%6<<_=#V|yxJIKiq_N?ZJ_v0$c)N4l07ZV_mIXG}glfBSPivOhw z-~+9GdckSpMBNR9eR`Y|9_)sXS+u_OiQ%!9rE(2AFjoxN8lk16Sb~^Sq6kRoEp3yD(mm`HsYIXcag_EAB8MHc}nahxVVUTts~U9P|f;7Ul$_` zStR4v&P4q_$KXOEni$lkxy8=9w8G&47VY0oDb^+jT+>ARe3NHUg~St`$RDxY)?;_F znqTujR&chZd2qHF7y8D$4&E3+e@J~!X3&BW4BF(Ebp#TEjrd+9SU!)j;qH+ZkL@AW z?J6Mj}v0_+D zH0qlbzCkHf|EZ`6c>5ig5NAFF%|La%M-}g(7&}Vx8K)qg30YD;H!S!??{;YivzrH0 z(M%2*b_S-)yh&Aiqai)GF^c!<1Xemj|13>dZ_M#)41SrP;OEMaRJ)bCeX*ZT7W`4Y zQ|8L@NHpD@Tf(5>1U(s5iW~Zdf7$@pAL`a3X@YUv1J>q-uJ_(Dy5nYTCUHC}1(dlI zt;5>DLcHh&jbysqt?G01MhXI3!8wgf){Hv}=0N|L$t8M#L7d6WscO8Om2|NBz2Ga^ zs86y%x$H18)~akOWD7@em7)ldlWgb?_sRN>-EcYQO_}aX@+b$dR{146>{kXWP4$nN{V0_+|3{Lt|8uX_fhKh~i{(x%cj*PU$i{PO(5$uA? zQzO>a6oPj-TUk&{zq?JD2MNb6Mf~V3g$ra+PB;ujLJ2JM(a7N*b`y{MX--!fAd}5C zF$D_b8S;+Np(!cW)(hnv5b@@|EMt*RLKF*wy>ykFhEhlPN~n_Bj>LT9B^_yj>z#fx z3JuE4H&?Cc!;G@}E*3k`HK#8ag`yE3Z1)5JUlSua%qkF zkTu|<9{w9OSi$qr)WD#7EzITnch=xnR63E*d~WGvi*Co9BBE?ETHud;!Z)7&wz+l6 zuKODYG1>I1U#a%&(GNJ`AqRfg=H!BtSl+_;CEeufF-#+*2EMMz-22@>18=8PH{PHd z);mN=aR0MPF>eutLiS#-AOX>#2%+pTGEOj!j4L(m0~&xR=0+g#HNpno6@veLhJp}e zyNVC$a>4;!9&iGvU_dj&xbKt@^t6r%f^)+}eV^suRTLP52+BVs0kOLwg6n`=NUv50E7My8XQUh?y%mW62OT1pMrKI3Q(r`7vU&@93=G~A?b(^pvC-8x=bSk zZ60BQR96WB1Z@9Df(M1IQh+YrU8sEjB=Tc2;(zBn-pete*icZE|M&Uc+oHg`|1o`g zH~m+k=D$o);{Rs)b<9Zo|9_Z6L6QHLNki(N>Dw^^i1LITprZeeqIaT#+)fw)PlllU zldphHC)t!0Gf(i9zgVm>`*TbmITF zH1FZ4{wrjRCx{t^26VK_2srZuWuY*EMAsMrJYFFCH35Ky7bq8<0K|ey2wHnrFMZyr z&^yEgX{{3i@&iE5>xKZ{Ads36G3a!i50D!C4?^~cLB<<|fc1!XN(HJRM)H^21sEs%vv+Mu0h*HkLHaEffMwc0n6)JhNXY#M5w@iO@dfXY z0c6dM2a4Hd1SA*#qYj@jK}uVgAZdaBj8t6uuhUNe>)ne9vfd#C6qLV9+@Q7{MnF#0 zJ7fd-ivG_~u3bVvOzpcw1u~ZSp8-kl(sunnX>L~*K-ByWDM2E8>;Si6kn^58AZQxI xVa^It*?521mj4+UJO?7%w*+`EfEcU=@KhDx-s^WzP+ae~{CgHDE&XryzW}Nww%-5% diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aa991fce..ae04661e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c7873..a69d9cb6 100755 --- a/gradlew +++ b/gradlew @@ -205,6 +205,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f9..53a6b238 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal From d2332a23232c1378a3afd5747937244854ea64cd Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 15 Nov 2022 15:41:47 +0100 Subject: [PATCH 03/25] Misc Snyk GA fixes --- .github/workflows/scheduled_snyk.yaml | 11 +++++++++-- .github/workflows/snyk.yaml | 6 +++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/scheduled_snyk.yaml b/.github/workflows/scheduled_snyk.yaml index 965cbfb1..b4b2e617 100644 --- a/.github/workflows/scheduled_snyk.yaml +++ b/.github/workflows/scheduled_snyk.yaml @@ -2,16 +2,21 @@ name: Snyk scheduled test on: schedule: - cron: '0 2 * * 1' + push: + branches: + - master + jobs: security: runs-on: ubuntu-latest env: REPORT_FILE: test.json + steps: - uses: actions/checkout@v3 - uses: snyk/actions/setup@master with: - snyk-version: v1.931.0 + snyk-version: v1.1032.0 - uses: actions/setup-java@v3 with: @@ -28,13 +33,15 @@ jobs: snyk test --all-sub-projects --configuration-matching='^runtimeClasspath$' + --fail-on=upgradable --json-file-output=${{ env.REPORT_FILE }} --org=radar-base + --policy-path=$PWD/.snyk - name: Report new vulnerabilities uses: thehyve/report-vulnerability@master + if: success() || failure() with: report-file: ${{ env.REPORT_FILE }} env: TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: ${{ failure() }} diff --git a/.github/workflows/snyk.yaml b/.github/workflows/snyk.yaml index de55c9f2..6169f72d 100644 --- a/.github/workflows/snyk.yaml +++ b/.github/workflows/snyk.yaml @@ -3,6 +3,7 @@ on: pull_request: branches: - master + jobs: security: runs-on: ubuntu-latest @@ -10,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - uses: snyk/actions/setup@master with: - snyk-version: v1.931.0 + snyk-version: v1.1032.0 - uses: actions/setup-java@v3 with: @@ -27,6 +28,5 @@ jobs: snyk test --all-sub-projects --configuration-matching='^runtimeClasspath$' - --fail-on=upgradable --org=radar-base - --severity-threshold=high + --policy-path=$PWD/.snyk From 17cbb6e43697e33ad331fbb5da8f4c096c104132 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 22 Nov 2022 16:56:03 +0100 Subject: [PATCH 04/25] Convert radar-commons to Kotlin --- README.md | 20 +- build.gradle | 125 ----- build.gradle.kts | 250 +++++++++ config/checkstyle/checkstyle.xml | 233 -------- config/pmd/ruleset.xml | 98 ---- gradle.properties | 15 +- gradle/codestyle.gradle | 24 - radar-commons-server/build.gradle | 51 -- radar-commons-server/build.gradle.kts | 57 ++ .../org/radarbase/config/AvroTopicConfig.java | 2 +- .../producer/direct/DirectSender.java | 95 ---- .../collector/NumericAggregateCollector.java | 4 +- .../radarbase/util/RollingTimeAverage.java | 110 ---- .../org/radarbase/util/RollingTimeAverage.kt | 92 ++++ .../NumericAggregateCollectorTest.java | 1 - radar-commons-testing/build.gradle | 60 --- radar-commons-testing/build.gradle.kts | 65 +++ .../java/org/radarbase/mock/MockDevice.java | 2 +- .../org/radarbase/mock/MockFileSender.java | 2 +- .../java/org/radarbase/mock/MockProducer.java | 69 +-- .../mock/config/BasicMockConfig.java | 4 - .../radarbase/mock/data/MockCsvParser.java | 2 +- .../mock/data/MockRecordValidator.java | 4 +- .../radarbase/mock/data/RecordGenerator.java | 8 +- .../radarbase/mock/model/ExpectedValue.java | 4 +- .../org/radarbase/mock/CsvGeneratorTest.java | 4 +- .../radarbase/mock/RecordGeneratorTest.java | 14 +- .../mock/data/MockRecordValidatorTest.java | 2 +- .../org/radarbase/util/MetronomeTest.java | 2 - radar-commons/build.gradle | 39 -- radar-commons/build.gradle.kts | 43 ++ .../org/radarbase/config/ServerConfig.java | 281 ---------- .../java/org/radarbase/config/ServerConfig.kt | 206 ++++++++ .../org/radarbase/data/AvroDatumDecoder.java | 82 --- .../org/radarbase/data/AvroDatumDecoder.kt | 73 +++ .../org/radarbase/data/AvroDatumEncoder.java | 47 -- .../org/radarbase/data/AvroDatumEncoder.kt | 40 ++ .../data/{AvroDecoder.java => AvroDecoder.kt} | 25 +- .../data/{AvroEncoder.java => AvroEncoder.kt} | 44 +- .../org/radarbase/data/AvroRecordData.java | 59 --- .../java/org/radarbase/data/AvroRecordData.kt | 32 ++ .../org/radarbase/data/AvroRecordWriter.java | 77 --- .../org/radarbase/data/AvroRecordWriter.kt | 69 +++ .../radarbase/data/{Record.java => Record.kt} | 25 +- .../data/{RecordData.java => RecordData.kt} | 16 +- .../radarbase/data/RemoteSchemaEncoder.java | 88 ---- .../org/radarbase/data/RemoteSchemaEncoder.kt | 83 +++ .../org/radarbase/data/StringEncoder.java | 65 --- .../java/org/radarbase/data/TimedInt.java | 75 --- ...eption.java => AuthenticationException.kt} | 28 +- .../producer/BatchedKafkaSender.java | 169 ------ .../radarbase/producer/BatchedKafkaSender.kt | 151 ++++++ .../{KafkaSender.java => KafkaSender.kt} | 35 +- ...kaTopicSender.java => KafkaTopicSender.kt} | 25 +- ...{AvroDataMapper.java => AvroDataMapper.kt} | 10 +- .../producer/rest/AvroDataMapperFactory.java | 497 ------------------ .../producer/rest/AvroDataMapperFactory.kt | 399 ++++++++++++++ .../producer/rest/BinaryRecordRequest.java | 133 ----- .../producer/rest/BinaryRecordRequest.kt | 137 +++++ .../producer/rest/ConnectionState.java | 95 ---- .../producer/rest/ConnectionState.kt | 94 ++++ .../producer/rest/GzipRequestInterceptor.java | 75 --- .../producer/rest/GzipRequestInterceptor.kt | 61 +++ .../producer/rest/JsonRecordRequest.java | 126 ----- .../producer/rest/JsonRecordRequest.kt | 131 +++++ .../producer/rest/ParsedSchemaMetadata.java | 56 -- .../producer/rest/ParsedSchemaMetadata.kt | 33 ++ .../{RecordRequest.java => RecordRequest.kt} | 40 +- .../radarbase/producer/rest/RestClient.java | 310 ----------- .../org/radarbase/producer/rest/RestClient.kt | 281 ++++++++++ .../producer/rest/RestException.java | 59 --- .../radarbase/producer/rest/RestException.kt | 43 ++ .../radarbase/producer/rest/RestSender.java | 333 ------------ .../org/radarbase/producer/rest/RestSender.kt | 278 ++++++++++ .../producer/rest/RestTopicSender.java | 209 -------- .../producer/rest/RestTopicSender.kt | 209 ++++++++ .../producer/rest/SchemaRestClient.java | 117 ----- .../producer/rest/SchemaRestClient.kt | 103 ++++ .../producer/rest/SchemaRetriever.java | 273 ---------- .../producer/rest/SchemaRetriever.kt | 235 +++++++++ .../producer/rest/TopicRequestBody.java | 54 -- .../producer/rest/TopicRequestBody.kt | 49 ++ .../rest/UncheckedRequestException.java | 92 ---- .../rest/UncheckedRequestException.kt | 91 ++++ .../java/org/radarbase/topic/AvroTopic.java | 160 ------ .../java/org/radarbase/topic/AvroTopic.kt | 121 +++++ .../java/org/radarbase/topic/KafkaTopic.java | 82 --- .../java/org/radarbase/topic/KafkaTopic.kt | 55 ++ .../java/org/radarbase/topic/SensorTopic.java | 78 --- .../java/org/radarbase/topic/SensorTopic.kt | 70 +++ .../main/java/org/radarbase/util/Base64.java | 143 ----- .../main/java/org/radarbase/util/Base64.kt | 179 +++++++ .../java/org/radarbase/util/CacheConfig.kt | 13 + .../java/org/radarbase/util/RestUtils.java | 121 ----- .../main/java/org/radarbase/util/RestUtils.kt | 96 ++++ .../org/radarbase/util/Serialization.java | 138 ----- .../main/java/org/radarbase/util/Strings.java | 96 ---- .../main/java/org/radarbase/util/Strings.kt | 47 ++ .../java/org/radarbase/util/TimedInt.java | 34 -- .../main/java/org/radarbase/util/TimedInt.kt | 19 + .../java/org/radarbase/util/TimedValue.java | 36 -- .../java/org/radarbase/util/TimedValue.kt | 19 + .../org/radarbase/util/TimedVariable.java | 5 - .../java/org/radarbase/util/TimedVariable.kt | 21 + .../radarbase/data/AvroDatumDecoderTest.java | 2 +- .../org/radarbase/data/StringEncoderTest.java | 44 -- .../rest/AvroDataMapperFactoryTest.java | 215 -------- .../rest/AvroDataMapperFactoryTest.kt | 231 ++++++++ .../rest/BinaryRecordRequestTest.java | 139 ----- .../producer/rest/BinaryRecordRequestTest.kt | 160 ++++++ .../producer/rest/ConnectionStateTest.java | 50 -- .../producer/rest/ConnectionStateTest.kt | 45 ++ .../producer/rest/RestClientTest.java | 105 ---- .../radarbase/producer/rest/RestClientTest.kt | 110 ++++ .../producer/rest/RestSenderTest.java | 284 ---------- .../radarbase/producer/rest/RestSenderTest.kt | 282 ++++++++++ .../producer/rest/SchemaRestClientTest.java | 80 --- .../producer/rest/SchemaRestClientTest.kt | 71 +++ .../producer/rest/SchemaRetrieverTest.java | 140 ----- .../producer/rest/SchemaRetrieverTest.kt | 165 ++++++ .../org/radarbase/topic/KafkaTopicTest.java | 48 -- .../org/radarbase/topic/KafkaTopicTest.kt | 36 ++ .../org/radarbase/topic/SensorTopicTest.java | 151 ------ .../org/radarbase/topic/SensorTopicTest.kt | 134 +++++ .../java/org/radarbase/util/Base64Test.java | 25 - .../java/org/radarbase/util/Base64Test.kt | 25 + .../org/radarbase/util/SerializationTest.java | 93 ---- settings.gradle => settings.gradle.kts | 6 +- 128 files changed, 5389 insertions(+), 6704 deletions(-) delete mode 100644 build.gradle create mode 100644 build.gradle.kts delete mode 100644 config/checkstyle/checkstyle.xml delete mode 100644 config/pmd/ruleset.xml delete mode 100644 gradle/codestyle.gradle delete mode 100644 radar-commons-server/build.gradle create mode 100644 radar-commons-server/build.gradle.kts delete mode 100644 radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java delete mode 100644 radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java create mode 100644 radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt delete mode 100644 radar-commons-testing/build.gradle create mode 100644 radar-commons-testing/build.gradle.kts delete mode 100644 radar-commons/build.gradle create mode 100644 radar-commons/build.gradle.kts delete mode 100644 radar-commons/src/main/java/org/radarbase/config/ServerConfig.java create mode 100644 radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java create mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java create mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt rename radar-commons/src/main/java/org/radarbase/data/{AvroDecoder.java => AvroDecoder.kt} (66%) rename radar-commons/src/main/java/org/radarbase/data/{AvroEncoder.java => AvroEncoder.kt} (64%) delete mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java create mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java create mode 100644 radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt rename radar-commons/src/main/java/org/radarbase/data/{Record.java => Record.kt} (68%) rename radar-commons/src/main/java/org/radarbase/data/{RecordData.java => RecordData.kt} (67%) delete mode 100644 radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java create mode 100644 radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/data/StringEncoder.java delete mode 100644 radar-commons/src/main/java/org/radarbase/data/TimedInt.java rename radar-commons/src/main/java/org/radarbase/producer/{AuthenticationException.java => AuthenticationException.kt} (57%) delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt rename radar-commons/src/main/java/org/radarbase/producer/{KafkaSender.java => KafkaSender.kt} (55%) rename radar-commons/src/main/java/org/radarbase/producer/{KafkaTopicSender.java => KafkaTopicSender.kt} (62%) rename radar-commons/src/main/java/org/radarbase/producer/rest/{AvroDataMapper.java => AvroDataMapper.kt} (52%) delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt rename radar-commons/src/main/java/org/radarbase/producer/rest/{RecordRequest.java => RecordRequest.kt} (57%) delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java create mode 100644 radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java create mode 100644 radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java create mode 100644 radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/Base64.java create mode 100644 radar-commons/src/main/java/org/radarbase/util/Base64.kt create mode 100644 radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/RestUtils.java create mode 100644 radar-commons/src/main/java/org/radarbase/util/RestUtils.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/Serialization.java delete mode 100644 radar-commons/src/main/java/org/radarbase/util/Strings.java create mode 100644 radar-commons/src/main/java/org/radarbase/util/Strings.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedInt.java create mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedInt.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedValue.java create mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedValue.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedVariable.java create mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java create mode 100644 radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/util/Base64Test.java create mode 100644 radar-commons/src/test/java/org/radarbase/util/Base64Test.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/util/SerializationTest.java rename settings.gradle => settings.gradle.kts (86%) diff --git a/README.md b/README.md index d2cae214..e9be27fe 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ # RADAR-Commons -[![Build Status](https://travis-ci.org/RADAR-base/radar-commons.svg?branch=master)](https://travis-ci.org/RADAR-base/radar-commons) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9fe7a419c83e4798af671e468c7e91cf)](https://www.codacy.com/app/RADAR-base/radar-commons?utm_source=github.com&utm_medium=referral&utm_content=RADAR-base/radar-commons&utm_campaign=Badge_Grade) Common utilities library containing basic schemas, streaming features, testing bridges and utils. # Usage -Add the RADAR-Commons library to your project with Gradle by updating your `build.gradle` file with: +Add the RADAR-Commons library to your project with Gradle by updating your `build.gradle.kts` file with: ```gradle repositories { @@ -14,7 +12,7 @@ repositories { } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons', version: '0.15.0' + implementation("org.radarbase:radar-commons:0.15.0") } ``` @@ -65,11 +63,11 @@ For server utilities, include `radar-commons-server`: ```gradle repositories { mavenCentral() - maven { url 'https://packages.confluent.io/maven/' } + maven(url = "https://packages.confluent.io/maven/") } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons-server', version: '0.15.0' + implementation("org.radarbase:radar-commons-server:0.15.0") } ``` @@ -78,11 +76,11 @@ For mocking clients of the RADAR-base infrastructure, use that 'radar-commons-te ```gradle repositories { mavenCentral() - maven { url 'https://packages.confluent.io/maven/' } + maven(url = "https://packages.confluent.io/maven/") } dependencies { - testImplementation group: 'org.radarbase', name: 'radar-commons-testing', version: '0.15.0' + testImplementation("org.radarbase:radar-commons-testing:0.15.0") } ``` @@ -98,16 +96,16 @@ For latest code use `dev` branch. This is released on JFrog's OSS Artifactory. T ```gradle repositories { - maven { url 'https://oss.sonatype.org/content/repositories/snapshots' } + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") } configurations.all { // Check for updates every build - resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' + resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") } dependencies { - implementation group: 'org.radarbase', name: 'radar-commons', version: '0.15.1-SNAPSHOT' + implementation("org.radarbase:radar-commons:0.15.1-SNAPSHOT") } ``` diff --git a/build.gradle b/build.gradle deleted file mode 100644 index 2a9d23fd..00000000 --- a/build.gradle +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -plugins { - id 'com.github.davidmc24.gradle.plugin.avro' version '1.5.0' - id("io.github.gradle-nexus.publish-plugin") version "1.1.0" - id("com.github.ben-manes.versions") version "0.42.0" -} - -allprojects { - version = '0.15.1-SNAPSHOT' - group = 'org.radarbase' -} - -subprojects { - // Apply the plugins - apply plugin: 'java' - apply plugin: 'java-library' - apply plugin: 'idea' - - targetCompatibility = '11' - sourceCompatibility = '11' - - //---------------------------------------------------------------------------// - // Configuration // - //---------------------------------------------------------------------------// - ext.githubRepoName = 'RADAR-base/radar-commons' - ext.githubUrl = "https://github.com/$githubRepoName" - ext.issueUrl = "https://github.com/$githubRepoName/issues" - ext.website = 'https://radar-base.org' - - //---------------------------------------------------------------------------// - // Dependencies // - //---------------------------------------------------------------------------// - repositories { - mavenCentral() - maven { url 'https://packages.confluent.io/maven/' } - flatDir { - dirs "${project.rootDir}/libs" - } - } - - idea { - module { - downloadSources = true - } - } - - //---------------------------------------------------------------------------// - // Style checking // - //---------------------------------------------------------------------------// - - tasks.matching { it instanceof Test }.all { - def stdout = new LinkedList() - beforeTest { TestDescriptor td -> - stdout.clear() - } - - onOutput { TestDescriptor td, TestOutputEvent toe -> - stdout.addAll(Arrays.asList(toe.getMessage().split('(?m)$'))) - while (stdout.size() > 100) { - stdout.remove() - } - } - - afterTest { TestDescriptor td, TestResult tr -> - if (tr.resultType == TestResult.ResultType.FAILURE) { - println() - print("${td.className}.${td.name} FAILED") - if (stdout.empty) { - println(" without any output") - } else { - println(" with last 100 lines of output:") - println('=' * 100) - stdout.each { print(it) } - println('=' * 100) - } - } - } - - testLogging { - showExceptions = true - showCauses = true - showStackTraces = true - exceptionFormat "full" - } - } -} - -def isNonStable = { String version -> - def stableKeyword = ["RELEASE", "FINAL", "GA"].any { version.toUpperCase().contains(it) } - def regex = /^[0-9,.v-]+(-r)?$/ - return !stableKeyword && !(version ==~ regex) -} - -tasks.named("dependencyUpdates").configure { - rejectVersionIf { - isNonStable(it.candidate.version) - } -} - -nexusPublishing { - repositories { - sonatype { - username = project.hasProperty("ossrh.user") ? project.property("ossrh.user") : System.getenv("OSSRH_USER") - password = project.hasProperty("ossrh.password") ? project.property("ossrh.password") : System.getenv("OSSRH_PASSWORD") - } - } -} - -wrapper { - gradleVersion '7.5.1' -} diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 00000000..bb70bcc7 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,250 @@ +import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask +import java.util.LinkedList +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + kotlin("jvm") version "1.7.21" apply false + id("com.github.davidmc24.gradle.plugin.avro") version "1.5.0" apply false + id("io.github.gradle-nexus.publish-plugin") version "1.1.0" + id("com.github.ben-manes.versions") version "0.44.0" + id("org.jetbrains.dokka") version "1.7.20" apply false + idea + `maven-publish` + signing +} + +val githubRepoName = "RADAR-base/radar-commons" +val githubUrl = "https://github.com/$githubRepoName" +val githubIssueUrl = "https://github.com/$githubRepoName/issues" +val website = "https://radar-base.org" + +allprojects { + version = "0.15.1-SNAPSHOT" + group = "org.radarbase" +} + +subprojects { + val myProject = this + + // Apply the plugins + apply(plugin = "java") + apply(plugin = "java-library") + apply(plugin = "org.jetbrains.kotlin.jvm") + apply(plugin = "idea") + + tasks.withType { + targetCompatibility = JavaVersion.VERSION_11.toString() + sourceCompatibility = JavaVersion.VERSION_11.toString() + } + tasks.withType { + kotlinOptions { + jvmTarget = "11" + languageVersion = "1.7" + apiVersion = "1.7" + } + } + + //---------------------------------------------------------------------------// + // Dependencies // + //---------------------------------------------------------------------------// + repositories { + mavenCentral() + mavenLocal() + maven(url = "https://packages.confluent.io/maven/") + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") + } + + afterEvaluate { + configurations { + named("implementation") { + resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") + } + } + } + + idea { + module { + isDownloadSources = true + } + } + + val sourcesJar by tasks.registering(Jar::class) { + from(myProject.the()["main"].allSource) + archiveClassifier.set("sources") + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + val classes by tasks + dependsOn(classes) + } + + apply(plugin = "org.jetbrains.dokka") + val dokkaJar by tasks.registering(Jar::class) { + from("$buildDir/dokka/javadoc") + archiveClassifier.set("javadoc") + val dokkaJavadoc by tasks + dependsOn(dokkaJavadoc) + } + + tasks.withType { + compression = Compression.GZIP + archiveExtension.set("tar.gz") + } + + tasks.withType { + manifest { + attributes( + "Implementation-Title" to project.name, + "Implementation-Version" to project.version + ) + } + } + + apply(plugin = "maven-publish") + apply(plugin = "signing") + + val assemble by tasks + assemble.dependsOn(sourcesJar) + assemble.dependsOn(dokkaJar) + + val mavenJar by publishing.publications.creating(MavenPublication::class) { + from(components["java"]) + + artifact(sourcesJar) + artifact(dokkaJar) + + afterEvaluate { + pom { + name.set(myProject.name) + description.set(myProject.description) + url.set(githubUrl) + licenses { + license { + name.set("The Apache Software License, Version 2.0") + url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") + distribution.set("repo") + } + } + developers { + developer { + id.set("blootsvoets") + name.set("Joris Borgdorff") + email.set("joris@thehyve.nl") + organization.set("The Hyve") + } + developer { + id.set("nivemaham") + name.set("Nivethika Mahasivam") + email.set("nivethika@thehyve.nl") + organization.set("The Hyve") + } + } + issueManagement { + system.set("GitHub") + url.set(githubIssueUrl) + } + organization { + name.set("RADAR-base") + url.set("https://radar-base.org") + } + scm { + connection.set("scm:git:$githubUrl") + url.set(githubUrl) + } + } + } + } + + signing { + useGpgCmd() + isRequired = true + sign(tasks["sourcesJar"], tasks["dokkaJar"]) + sign(mavenJar) + } + + tasks.withType { + onlyIf { gradle.taskGraph.hasTask(myProject.tasks["publish"]) } + } + + //---------------------------------------------------------------------------// + // Style checking // + //---------------------------------------------------------------------------// + + tasks.withType { + val stdout = LinkedList() + beforeTest(closureOf { + stdout.clear() + }) + + onOutput(KotlinClosure2({ td, toe -> + stdout.addAll(toe.getMessage().split("(?m)$").toList()) + while (stdout.size > 100) { + stdout.remove() + } + })) + + afterTest(KotlinClosure2({ td, tr -> + if (tr.resultType == TestResult.ResultType.FAILURE) { + println() + print("${td.className}.${td.name} FAILED") + if (stdout.isEmpty()) { + println(" without any output") + } else { + println(" with last 100 lines of output:") + println("=".repeat(100)) + stdout.forEach { print(it) } + println("=".repeat(100)) + } + } + })) + + testLogging { + showExceptions = true + showCauses = true + showStackTraces = true + setExceptionFormat("full") + } + } +} + +tasks.withType { + val acceptedVersion = "(RELEASE|FINAL|GA|-ce|^[0-9,.v-]+(-r)?)$" + .toRegex(RegexOption.IGNORE_CASE) + rejectVersionIf { + !acceptedVersion.containsMatchIn(candidate.version) + } +} + +fun Project.propertyOrEnv(propertyName: String, envName: String): String? { + return if (hasProperty(propertyName)) { + property(propertyName)?.toString() + } else { + System.getenv(envName) + } +} + +nexusPublishing { + repositories { + sonatype { + username.set(propertyOrEnv("ossrh.user", "OSSRH_USER")) + password.set(propertyOrEnv("ossrh.password", "OSSRH_PASSWORD")) + } + } +} + +tasks.wrapper { + gradleVersion = "7.5.1" +} diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml deleted file mode 100644 index a252c58c..00000000 --- a/config/checkstyle/checkstyle.xml +++ /dev/null @@ -1,233 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/pmd/ruleset.xml b/config/pmd/ruleset.xml deleted file mode 100644 index 1157371b..00000000 --- a/config/pmd/ruleset.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - - This ruleset was parsed from the Codacy default codestyle. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/gradle.properties b/gradle.properties index 2b70a273..742b3b99 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,12 +1,13 @@ slf4jVersion=2.0.3 -confluentVersion=7.2.2 -kafkaVersion=3.3.1 +confluentVersion=7.3.0 +kafkaVersion=7.3.0-ce avroVersion=1.11.1 -jacksonVersion=2.13.4 +jacksonVersion=2.14.0 okhttpVersion=4.10.0 junitVersion=4.13.2 -mockitoVersion=4.8.0 -hamcrestVersion=1.3 -radarSchemasVersion=0.7.9 +mockitoVersion=4.9.0 +hamcrestVersion=2.2 +radarSchemasVersion=0.8.1 orgJsonVersion=20220924 -opencsvVersion=5.7.0 +opencsvVersion=5.7.1 +ktorVersion=2.1.3 diff --git a/gradle/codestyle.gradle b/gradle/codestyle.gradle deleted file mode 100644 index 9a24dbb0..00000000 --- a/gradle/codestyle.gradle +++ /dev/null @@ -1,24 +0,0 @@ -apply plugin: 'checkstyle' -apply plugin: 'pmd' - -checkstyle { - toolVersion = '9.2' - ignoreFailures = false - - configFile = rootProject.file('config/checkstyle/checkstyle.xml') - // ignore tests - sourceSets = [sourceSets.main] -} - -pmd { - toolVersion = '6.41.0' - ignoreFailures = false - - consoleOutput = true - - ruleSets = [] - ruleSetFiles = rootProject.files("config/pmd/ruleset.xml") - // ignore tests - sourceSets = [sourceSets.main] -} - diff --git a/radar-commons-server/build.gradle b/radar-commons-server/build.gradle deleted file mode 100644 index f1f85ade..00000000 --- a/radar-commons-server/build.gradle +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -apply plugin: 'com.github.davidmc24.gradle.plugin.avro' - -description = 'RADAR Common server library utilities.' - -dependencies { - api project(':radar-commons') - - // For POJO classes and ConfigLoader - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml' - - api group: 'org.apache.avro', name: 'avro', version: avroVersion - - implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion - - testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion - // Direct producer uses KafkaAvroSerializer if initialized - testImplementation group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion - testImplementation group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion - // Direct producer uses KafkaAvroSerializer if initialized - testImplementation group: 'junit', name: 'junit', version: junitVersion - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion -} - -apply from: '../gradle/publishing.gradle' -apply from: '../gradle/codestyle.gradle' - -tasks.withType(Checkstyle) { - exclude '**/org/radarbase/stream/collector/*State.*' -} - -tasks.withType(Pmd) { - exclude '**/org/radarbase/stream/collector/*State.*' -} diff --git a/radar-commons-server/build.gradle.kts b/radar-commons-server/build.gradle.kts new file mode 100644 index 00000000..029a61ad --- /dev/null +++ b/radar-commons-server/build.gradle.kts @@ -0,0 +1,57 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +apply(plugin = "com.github.davidmc24.gradle.plugin.avro") + +description = "RADAR Common server library utilities." + +dependencies { + api(project(":radar-commons")) + + // For POJO classes and ConfigLoader + val jacksonVersion: String by project + implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") + implementation("com.fasterxml.jackson.core:jackson-databind") + + val avroVersion: String by project + api("org.apache.avro:avro:$avroVersion") + + val kafkaVersion: String by project + implementation("org.apache.kafka:kafka-clients:$kafkaVersion") + + val mockitoVersion: String by project + testImplementation("org.mockito:mockito-core:$mockitoVersion") + // Direct producer uses KafkaAvroSerializer if initialized + val confluentVersion: String by project + testImplementation("io.confluent:kafka-avro-serializer:$confluentVersion") + val radarSchemasVersion: String by project + testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion") + // Direct producer uses KafkaAvroSerializer if initialized + val junitVersion: String by project + testImplementation("junit:junit:$junitVersion") + val slf4jVersion: String by project + testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") +} + +tasks.withType { + dependsOn(tasks.named("generateAvroJava")) +} + +tasks.withType { + dependsOn(tasks.named("generateAvroJava")) +} diff --git a/radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java b/radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java index 6e916f6d..1a14e1b4 100644 --- a/radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java +++ b/radar-commons-server/src/main/java/org/radarbase/config/AvroTopicConfig.java @@ -40,7 +40,7 @@ public class AvroTopicConfig { */ public AvroTopic parseAvroTopic() { try { - return AvroTopic.parse(topic, keySchema, valueSchema); + return AvroTopic.Companion.parse(topic, keySchema, valueSchema); } catch (IllegalArgumentException ex) { throw new IllegalStateException("Topic " + topic + " schema cannot be instantiated", ex); diff --git a/radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java b/radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java deleted file mode 100644 index 4617c574..00000000 --- a/radar-commons-server/src/main/java/org/radarbase/producer/direct/DirectSender.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.direct; - -import java.util.Properties; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.radarbase.data.RecordData; -import org.radarbase.producer.KafkaSender; -import org.radarbase.producer.KafkaTopicSender; -import org.radarbase.topic.AvroTopic; - -/** - * Directly sends a message to Kafka using a KafkaProducer. - */ -public class DirectSender implements KafkaSender { - private final KafkaProducer producer; - - public DirectSender(Properties properties) { - producer = new KafkaProducer(properties); - } - - @Override - public KafkaTopicSender sender(final AvroTopic topic) { - return new DirectTopicSender<>(topic); - } - - @Override - public boolean resetConnection() { - return true; - } - - @Override - public boolean isConnected() { - return true; - } - - @Override - public void close() { - producer.flush(); - producer.close(); - } - - @SuppressWarnings("unchecked") - private class DirectTopicSender implements KafkaTopicSender { - private final String name; - - private DirectTopicSender(AvroTopic topic) { - name = topic.getName(); - } - - @Override - public void send(K key, V value) { - producer.send(new ProducerRecord<>(name, key, value)); - producer.flush(); - } - - @Override - public void send(RecordData records) { - for (V record : records) { - producer.send(new ProducerRecord<>(name, records.getKey(), record)); - } - producer.flush(); - } - - @Override - public void clear() { - // noop - } - - @Override - public void flush() { - // noop - } - - @Override - public void close() { - // noop - } - } -} diff --git a/radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java b/radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java index 4077c9b8..70fff99f 100644 --- a/radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java +++ b/radar-commons-server/src/main/java/org/radarbase/stream/collector/NumericAggregateCollector.java @@ -16,8 +16,6 @@ package org.radarbase.stream.collector; -import static org.radarbase.util.Serialization.floatToDouble; - import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -124,7 +122,7 @@ public NumericAggregateCollector add(IndexedRecord record) { /** Add a single sample. */ public NumericAggregateCollector add(float value) { - return this.add(floatToDouble(value)); + return this.add(Double.parseDouble(String.valueOf(value))); } /** diff --git a/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java deleted file mode 100644 index a4515c96..00000000 --- a/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -import java.util.Deque; -import java.util.LinkedList; - -/** - * Get the average of a set of values collected in a sliding time window of fixed duration. At least - * one value is needed to get an average. - */ -public class RollingTimeAverage { - private final long window; - private TimeCount firstTime; - private double total; - private final Deque deque; - - /** - * A rolling time average with a sliding time window of fixed duration. - * @param timeWindowMillis duration of the time window. - */ - public RollingTimeAverage(long timeWindowMillis) { - this.window = timeWindowMillis; - this.total = 0d; - this.firstTime = null; - this.deque = new LinkedList<>(); - } - - /** Whether values have already been added. */ - public boolean hasAverage() { - return firstTime != null; - } - - /** Add a new value. */ - public void add(double x) { - if (firstTime == null) { - firstTime = new TimeCount(x); - } else { - deque.addLast(new TimeCount(x)); - } - total += x; - } - - /** Add a value of one. */ - public void increment() { - add(1d); - } - - /** - * Get the average value per second over a sliding time window of fixed size. - * - *

It takes one value before the window started as a baseline, and adds all values in the - * window. It then divides by the total time window from the first value (outside/before the - * window) to the last value (at the end of the window). - * @return average value per second - */ - public double getAverage() { - if (!hasAverage()) { - throw new IllegalStateException("Cannot get average without values"); - } - - long now = System.currentTimeMillis(); - long currentWindowStart = now - window; - while (!this.deque.isEmpty() && this.deque.getFirst().time < currentWindowStart) { - total -= this.firstTime.value; - this.firstTime = this.deque.removeFirst(); - } - if (this.deque.isEmpty() || this.firstTime.time >= currentWindowStart) { - return 1000d * total / (now - this.firstTime.time); - } else { - long time = this.deque.getLast().time - currentWindowStart; - double removedRate = (currentWindowStart - this.firstTime.time) - / (this.deque.getFirst().time - firstTime.time); - double removedValue = this.firstTime.value + this.deque.getFirst().value * removedRate; - double value = (total - removedValue) / time; - return 1000d * value; - } - } - - /** - * Rounded {@link #getAverage()}. - */ - public int getCount() { - return (int)Math.round(getAverage()); - } - - private static class TimeCount { - private final double value; - private final long time; - - TimeCount(double value) { - this.value = value; - this.time = System.currentTimeMillis(); - } - } -} diff --git a/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt new file mode 100644 index 00000000..6b298097 --- /dev/null +++ b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.util + +import java.time.Duration +import java.time.Instant +import java.util.* +import kotlin.math.roundToInt + +/** + * Get the average of a set of values collected in a sliding time window of fixed duration. At least + * one value is needed to get an average. + * @param window duration of the time window. + */ +class RollingTimeAverage( + private val window: Duration, +) { + private var firstTime: TimeCount? = null + private val deque: Deque = LinkedList() + + /** Whether values have already been added. */ + val hasAverage: Boolean + get() = firstTime != null + + /** Add a new value. */ + fun add(x: Double) { + if (firstTime == null) { + firstTime = TimeCount(x) + } else { + deque.addLast(TimeCount(x)) + } + } + + /** Add a value of one. */ + fun increment() { + add(1.0) + } + + /** + * Get the average value per second over a sliding time window of fixed size. + * + * It takes one value before the window started as a baseline, and adds all values in the + * window. It then divides by the total time window from the first value (outside/before the + * window) to the last value (at the end of the window). + * @return average value per second + */ + val average: Double + get() { + var localFirstTime = checkNotNull(firstTime) { "Cannot get average without values" } + val now = Instant.now() + val windowStart = now - window + while (!deque.isEmpty() && deque.first.time < windowStart) { + localFirstTime = deque.removeFirst() + } + val total = localFirstTime.value + deque.sumOf { it.value } + firstTime = localFirstTime + return if (deque.isEmpty() || localFirstTime.time >= windowStart) { + 1000.0 * total / Duration.between(localFirstTime.time, now).toMillis() + } else { + val time = Duration.between(windowStart, deque.last.time) + val removedRate = Duration.between(localFirstTime.time, windowStart).toMillis() / + Duration.between(localFirstTime.time, deque.first.time).toMillis().toDouble() + val removedValue = localFirstTime.value + deque.first.value * removedRate + 1000.0 * (total - removedValue) / time.toMillis() + } + } + + /** + * Rounded [.getAverage]. + */ + val count: Int + get() = average.roundToInt() + + private class TimeCount( + val value: Double + ) { + val time: Instant = Instant.now() + } +} diff --git a/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java index 13e60a86..f64160ef 100644 --- a/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; -import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Arrays; diff --git a/radar-commons-testing/build.gradle b/radar-commons-testing/build.gradle deleted file mode 100644 index b4bc1b31..00000000 --- a/radar-commons-testing/build.gradle +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -apply plugin: 'application' - -mainClassName = 'org.radarbase.mock.MockProducer' - -configurations { - applicationRuntimeOnly -} - -run { - classpath += configurations.applicationRuntimeOnly - if (project.hasProperty("mockConfig")) { - args project.property("mockConfig") - } else { - args 'mock.yml' - } -} - -description = 'RADAR Common testing library mocking code and utilities.' - -dependencies { - api project(':radar-commons') - api project(':radar-commons-server') - api group: 'org.apache.avro', name: 'avro', version: avroVersion - api group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion - - implementation group: 'com.opencsv', name: 'opencsv', version: opencsvVersion - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' - implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion - implementation (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { - exclude group: 'com.101tec' - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - } - - applicationRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion - - // Direct producer uses KafkaAvroSerializer if initialized - testImplementation group: 'junit', name: 'junit', version: junitVersion - testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: hamcrestVersion - testImplementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion -} - -apply from: '../gradle/publishing.gradle' -apply from: '../gradle/codestyle.gradle' diff --git a/radar-commons-testing/build.gradle.kts b/radar-commons-testing/build.gradle.kts new file mode 100644 index 00000000..d95386fc --- /dev/null +++ b/radar-commons-testing/build.gradle.kts @@ -0,0 +1,65 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + application +} + +val applicationRuntimeOnly by configurations.creating + +application { + mainClass.set("org.radarbase.mock.MockProducer") +} + +tasks.named("run") { + classpath += applicationRuntimeOnly + if (project.hasProperty("mockConfig")) { + args(project.property("mockConfig")) + } else { + args("mock.yml") + } +} + +description = "RADAR Common testing library mocking code and utilities." + +dependencies { + api(project(":radar-commons")) + api(project(":radar-commons-server")) + val avroVersion: String by project + api("org.apache.avro:avro:$avroVersion") + val radarSchemasVersion: String by project + api("org.radarbase:radar-schemas-commons:$radarSchemasVersion") + + val opencsvVersion: String by project + implementation("com.opencsv:opencsv:$opencsvVersion") + val jacksonVersion: String by project + implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + implementation("com.fasterxml.jackson.core:jackson-databind") + val kafkaVersion: String by project + implementation("org.apache.kafka:kafka-clients:$kafkaVersion") + val confluentVersion: String by project + implementation("io.confluent:kafka-avro-serializer:$confluentVersion") + + val slf4jVersion: String by project + applicationRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") + + // Direct producer uses KafkaAvroSerializer if initialized + val junitVersion: String by project + testImplementation("junit:junit:$junitVersion") + val hamcrestVersion: String by project + testImplementation("org.hamcrest:hamcrest:$hamcrestVersion") + testImplementation("org.slf4j:slf4j-simple:$slf4jVersion") +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java index f2592fb2..4b96760d 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java @@ -87,7 +87,7 @@ public void run() { int frequency = generators.get(i).getConfig().getFrequency(); if (frequency > 0 && beat % (baseFrequency / frequency) == 0) { Record record = recordIterators.get(i).next(); - topicSenders.get(i).send(record.key, record.value); + topicSenders.get(i).send(record.getKey(), record.getValue()); } } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java index 668fd151..fa9fce8d 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java @@ -47,7 +47,7 @@ public void send() throws IOException { try (KafkaTopicSender topicSender = sender.sender(parser.getTopic())) { while (parser.hasNext()) { Record record = parser.next(); - topicSender.send(record.key, record.value); + topicSender.send(record.getKey(), record.getValue()); } } catch (SchemaValidationException e) { throw new IOException("Failed to match schemas", e); diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java index 0452c861..89827831 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java @@ -16,13 +16,7 @@ package org.radarbase.mock; -import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; - import com.opencsv.exceptions.CsvValidationException; -import io.confluent.kafka.serializers.KafkaAvroSerializer; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -30,9 +24,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import kotlin.Unit; import okhttp3.Credentials; import okhttp3.FormBody; import okhttp3.Headers; @@ -46,18 +40,17 @@ import org.radarbase.config.ServerConfig; import org.radarbase.config.YamlConfigLoader; import org.radarbase.mock.config.AuthConfig; +import org.radarbase.mock.config.BasicMockConfig; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.MockCsvParser; import org.radarbase.mock.data.RecordGenerator; import org.radarbase.producer.BatchedKafkaSender; import org.radarbase.producer.KafkaSender; -import org.radarbase.producer.direct.DirectSender; import org.radarbase.producer.rest.ConnectionState; import org.radarbase.producer.rest.RestClient; import org.radarbase.producer.rest.RestSender; import org.radarbase.producer.rest.SchemaRetriever; import org.radarcns.kafka.ObservationKey; -import org.radarbase.mock.config.BasicMockConfig; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; @@ -100,7 +93,12 @@ public MockProducer(BasicMockConfig mockConfig) throws IOException { public MockProducer(BasicMockConfig mockConfig, Path root) throws IOException { int numDevices = mockConfig.getNumberOfDevices(); - retriever = new SchemaRetriever(mockConfig.getSchemaRegistry(), 10); + RestClient restClient = RestClient.Companion.globalRestClient(builder -> { + builder.setServer(mockConfig.getSchemaRegistry()); + builder.timeout(10, TimeUnit.SECONDS); + return Unit.INSTANCE; + }); + retriever = new SchemaRetriever(restClient); List tmpSenders = null; try { @@ -155,29 +153,8 @@ public MockProducer(BasicMockConfig mockConfig, Path root) throws IOException { private List createSenders( BasicMockConfig mockConfig, int numDevices, AuthConfig authConfig) throws IOException { - if (mockConfig.isDirectProducer()) { - return createDirectSenders(numDevices, mockConfig.getSchemaRegistry().getUrlString(), - mockConfig.getBrokerPaths()); - } else { - return createRestSenders(numDevices, retriever, mockConfig.getRestProxy(), - mockConfig.hasCompression(), authConfig); - } - } - - /** Create senders that directly produce data to Kafka. */ - private List createDirectSenders(int numDevices, - String retrieverUrl, String brokerPaths) { - List result = new ArrayList<>(numDevices); - for (int i = 0; i < numDevices; i++) { - Properties properties = new Properties(); - properties.put(KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); - properties.put(VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); - properties.put(SCHEMA_REGISTRY_URL_CONFIG, retrieverUrl); - properties.put(BOOTSTRAP_SERVERS_CONFIG, brokerPaths); - - result.add(new DirectSender(properties)); - } - return result; + return createRestSenders(numDevices, retriever, mockConfig.getRestProxy(), + mockConfig.hasCompression(), authConfig); } private String requestAccessToken(OkHttpClient okHttpClient, AuthConfig authConfig) @@ -224,18 +201,20 @@ private List createRestSenders(int numDevices, } for (int i = 0; i < numDevices; i++) { - RestClient httpClient = RestClient.newClient() - .server(restProxy) - .gzipCompression(useCompression) - .timeout(10, TimeUnit.SECONDS) - .build(); - - RestSender restSender = new RestSender.Builder() - .schemaRetriever(retriever) - .httpClient(httpClient) - .connectionState(sharedState) - .headers(headers) - .build(); + RestClient httpClient = RestClient.Companion.newRestClient(builder -> { + builder.setServer(restProxy); + builder.gzipCompression(useCompression); + builder.timeout(10, TimeUnit.SECONDS); + return Unit.INSTANCE; + }); + + RestSender restSender = RestSender.Companion.restSender(builder -> { + builder.setSchemaRetriever(retriever); + builder.setHttpClient(httpClient); + builder.setConnectionState(sharedState); + builder.setHeaders(headers.newBuilder()); + return Unit.INSTANCE; + }); result.add(new BatchedKafkaSender(restSender, 1000, 1000)); } return result; diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java b/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java index 7adebb64..ad0ddf5a 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/config/BasicMockConfig.java @@ -97,10 +97,6 @@ public void setRestProxy(ServerConfig restProxy) { this.restProxy = restProxy; } - public boolean isDirectProducer() { - return this.producerMode.trim().equalsIgnoreCase("direct"); - } - public boolean isUnsafeProducer() { return this.producerMode.trim().equalsIgnoreCase("unsafe"); } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java index 005c5668..96d9b7ba 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java @@ -77,7 +77,7 @@ public MockCsvParser(MockDataConfig config, Path root, Instant startTime, } catch (IllegalStateException ex) { Objects.requireNonNull(retriever, "Cannot instantiate value schema without " + "schema retriever."); - keySchema = AvroTopic.parseSpecificRecord(config.getKeySchema()).getSchema(); + keySchema = AvroTopic.Companion.parseSpecificRecord(config.getKeySchema()).getSchema(); valueSchema = retriever.getBySubjectAndVersion( config.getTopic(), true, 0).getSchema(); } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java index 55886bde..01b3ce70 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java @@ -98,12 +98,12 @@ private void checkFrequency(long line) { private void checkRecord(Record record, Record last, long line) { double previousTime = time; - time = (Double) record.value.get(timePos); + time = (Double) record.getValue().get(timePos); if (last == null) { // no checks, only update initial time stamp startTime = time; - } else if (!last.key.equals(record.key)) { + } else if (!last.getKey().equals(record.getKey())) { error("It is possible to test only one user/source at time.", line, null); } else if (time < previousTime) { error("Time must increase row by row.", line, null); diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java index 60f4cddf..2bdc0b72 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/RecordGenerator.java @@ -191,15 +191,15 @@ public boolean hasNext() { public String[] next() { Record record = baseIterator.next(); - int keyFieldsSize = record.key.getSchema().getFields().size(); - int valueFieldsSize = record.value.getSchema().getFields().size(); + int keyFieldsSize = record.getKey().getSchema().getFields().size(); + int valueFieldsSize = record.getValue().getSchema().getFields().size(); String[] result = new String[keyFieldsSize + valueFieldsSize]; for (int i = 0; i < keyFieldsSize; i++) { - result[i] = record.key.get(i).toString(); + result[i] = record.getKey().get(i).toString(); } for (int i = 0; i < valueFieldsSize; i++) { - result[i + keyFieldsSize] = record.value.get(i).toString(); + result[i + keyFieldsSize] = record.getValue().get(i).toString(); } return result; } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java index dae02afc..233202b5 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/model/ExpectedValue.java @@ -70,12 +70,12 @@ public void add(Record record) { if (timeReceivedPos == -1) { throw new IllegalStateException("Cannot parse record without a schema."); } - long timeMillis = (long) ((Double) record.value.get(timeReceivedPos) * 1000d); + long timeMillis = (long) ((Double) record.getValue().get(timeReceivedPos) * 1000d); if (timeMillis >= lastTimestamp + DURATION || lastCollector == null) { lastTimestamp = timeMillis - (timeMillis % DURATION); lastCollector = createCollector(); getSeries().put(lastTimestamp, lastCollector); } - lastCollector.add(record.value); + lastCollector.add(record.getValue()); } } diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java index d0aaba2f..06c7df1e 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java @@ -31,11 +31,11 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.radarbase.mock.data.MockRecordValidatorTest; -import org.radarcns.kafka.ObservationKey; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.CsvGenerator; +import org.radarbase.mock.data.MockRecordValidatorTest; import org.radarbase.mock.data.RecordGenerator; +import org.radarcns.kafka.ObservationKey; public class CsvGeneratorTest { @Rule diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java index 783b39d4..95aed120 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java @@ -24,9 +24,9 @@ import org.apache.avro.specific.SpecificRecord; import org.junit.Test; import org.radarbase.data.Record; -import org.radarcns.kafka.ObservationKey; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.RecordGenerator; +import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; /** @@ -49,19 +49,19 @@ public void generate() throws Exception { Iterator> iter = generator .iterateValues(new ObservationKey("test", "a", "b"), 0); Record record = iter.next(); - assertEquals(new ObservationKey("test", "a", "b"), record.key); - float x = ((EmpaticaE4Acceleration)record.value).getX(); + assertEquals(new ObservationKey("test", "a", "b"), record.getKey()); + float x = ((EmpaticaE4Acceleration)record.getValue()).getX(); assertTrue(x >= 0.1f && x < 9.9f); - float y = ((EmpaticaE4Acceleration)record.value).getX(); + float y = ((EmpaticaE4Acceleration)record.getValue()).getX(); assertTrue(y >= 0.1f && y < 9.9f); - float z = ((EmpaticaE4Acceleration)record.value).getX(); + float z = ((EmpaticaE4Acceleration)record.getValue()).getX(); assertTrue(z >= 0.1f && z < 9.9f); - double time = ((EmpaticaE4Acceleration)record.value).getTime(); + double time = ((EmpaticaE4Acceleration)record.getValue()).getTime(); assertTrue(time > System.currentTimeMillis() / 1000d - 1d && time <= System.currentTimeMillis() / 1000d); Record nextRecord = iter.next(); - assertEquals(time + 0.1d, (Double)nextRecord.value.get(0), 1e-6); + assertEquals(time + 0.1d, (Double)nextRecord.getValue().get(0), 1e-6); } @Test diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java index a3a3569e..3c09709a 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java @@ -29,8 +29,8 @@ import org.junit.rules.TemporaryFolder; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.producer.rest.SchemaRetriever; -import org.radarcns.monitor.application.ApplicationServerStatus; import org.radarcns.kafka.ObservationKey; +import org.radarcns.monitor.application.ApplicationServerStatus; import org.radarcns.passive.phone.PhoneAcceleration; import org.radarcns.passive.phone.PhoneLight; diff --git a/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java b/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java index 1a5f3343..17b17dca 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java @@ -22,9 +22,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThrows; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; public class MetronomeTest { private void check(Metronome it, long expectedMin) { diff --git a/radar-commons/build.gradle b/radar-commons/build.gradle deleted file mode 100644 index 4249a990..00000000 --- a/radar-commons/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -description = 'RADAR Common utilities library.' - -//---------------------------------------------------------------------------// -// Sources and classpath configurations // -//---------------------------------------------------------------------------// - -configurations { - implementation { - resolutionStrategy.cacheChangingModulesFor 0, 'SECONDS' - } -} - -// In this section you declare where to find the dependencies of your project -repositories { - maven { url 'https://jitpack.io' } -} - -// In this section you declare the dependencies for your production and test code -dependencies { - api (group: 'org.apache.avro', name: 'avro', version: avroVersion) - - // to implement producers and consumers - api group: 'com.squareup.okhttp3', name: 'okhttp', version: okhttpVersion - api group: 'org.json', name: 'json', version: orgJsonVersion - - // The production code uses the SLF4J logging API at compile time - implementation group: 'org.slf4j', name:'slf4j-api', version: slf4jVersion - - testImplementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) - testImplementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind' - testImplementation group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion - testImplementation group: 'junit', name: 'junit', version: junitVersion - testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion - testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: okhttpVersion - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion -} - -apply from: '../gradle/publishing.gradle' -apply from: '../gradle/codestyle.gradle' diff --git a/radar-commons/build.gradle.kts b/radar-commons/build.gradle.kts new file mode 100644 index 00000000..b8d95e70 --- /dev/null +++ b/radar-commons/build.gradle.kts @@ -0,0 +1,43 @@ +description = "RADAR Common utilities library." + +//---------------------------------------------------------------------------// +// Sources and classpath configurations // +//---------------------------------------------------------------------------// + +// In this section you declare where to find the dependencies of your project +repositories { + maven(url = "https://jitpack.io") +} + +// In this section you declare the dependencies for your production and test code +dependencies { + val avroVersion: String by project + api("org.apache.avro:avro:$avroVersion") + + val ktorVersion: String by project + + implementation("io.ktor:ktor-client-core:$ktorVersion") + implementation("io.ktor:ktor-client-cio:$ktorVersion") + + // to implement producers and consumers + val okhttpVersion: String by project + api("com.squareup.okhttp3:okhttp:$okhttpVersion") + val orgJsonVersion: String by project + api("org.json:json:$orgJsonVersion") + + // The production code uses the SLF4J logging API at compile time + val slf4jVersion: String by project + implementation("org.slf4j:slf4j-api:$slf4jVersion") + + val jacksonVersion: String by project + testImplementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + testImplementation("com.fasterxml.jackson.core:jackson-databind") + val radarSchemasVersion: String by project + testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion") + val junitVersion: String by project + testImplementation("junit:junit:$junitVersion") + val mockitoVersion: String by project + testImplementation("org.mockito:mockito-core:$mockitoVersion") + testImplementation("com.squareup.okhttp3:mockwebserver:$okhttpVersion") + testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") +} diff --git a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java deleted file mode 100644 index 2050b376..00000000 --- a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.config; - -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.Proxy; -import java.net.Proxy.Type; -import java.net.URL; -import java.util.List; -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import okhttp3.HttpUrl; - -/** - * POJO representing a ServerConfig configuration. - */ -@SuppressWarnings("PMD.GodClass") -public class ServerConfig { - private static final Pattern URL_PATTERN = Pattern.compile( - "(?:(?\\w+)://)?(?[^:/]+)(?::(?\\d+))?(?/.*)?"); - - private String host; - private int port = -1; - private String protocol; - private String path = null; - private String proxyHost; - private int proxyPort = -1; - private boolean unsafe = false; - - /** Pojo initializer. */ - public ServerConfig() { - // POJO initializer - } - - /** Parses the config from a URL. */ - public ServerConfig(URL url) { - host = url.getHost(); - port = url.getPort(); - protocol = url.getProtocol(); - setPath(url.getFile()); - } - - /** Parses the config from a URL string. */ - public ServerConfig(String urlString) throws MalformedURLException { - Matcher matcher = URL_PATTERN.matcher(urlString); - if (!matcher.matches()) { - throw new MalformedURLException("Cannot create URL from string " + urlString); - } - protocol = matcher.group("protocol"); - host = matcher.group("host"); - String portString = matcher.group("port"); - if (portString != null && !portString.isEmpty()) { - port = Integer.parseInt(portString); - } - setPath(matcher.group("path")); - } - - /** Get the path of the server as a string. This does not include proxyHost information. */ - public String getUrlString() { - StringBuilder builder = new StringBuilder(host.length() - + (path != null ? path.length() : 0) + 20); - appendUrlString(builder); - return builder.toString(); - } - - /** Get the path of the server as a string. This does not include proxyHost information. */ - private void appendUrlString(StringBuilder builder) { - if (protocol != null) { - builder.append(protocol).append("://"); - } - builder.append(host); - if (port != -1) { - builder.append(':').append(port); - } - if (path != null) { - builder.append(path); - } - } - - /** Get the paths of a list of servers, concatenated with commas. */ - public static String getPaths(List configList) { - StringBuilder builder = new StringBuilder(configList.size() * 40); - boolean first = true; - for (ServerConfig server : configList) { - if (first) { - first = false; - } else { - builder.append(','); - } - server.appendUrlString(builder); - } - return builder.toString(); - } - - /** - * Get the server as a URL. - * - * @return URL to the server. - * @throws IllegalStateException if the URL is invalid - */ - public URL getUrl() { - if (protocol == null || host == null) { - throw new IllegalStateException("Cannot create URL without protocol and host"); - } - try { - return new URL(protocol, host, port, path == null ? "" : path); - } catch (MalformedURLException ex) { - throw new IllegalStateException("Already parsed a URL but it turned out invalid", ex); - } - } - - /** - * Get the server as an HttpUrl. - * @return HttpUrl to the server - * @throws IllegalStateException if the URL is invalid - */ - public HttpUrl getHttpUrl() { - if (protocol == null) { - protocol = "http"; - } - return HttpUrl.get(getUrlString()); - } - - /** - * Get the HTTP proxyHost associated to given server. - * @return http proxyHost if specified, or null if none is specified. - * @throws IllegalStateException if proxyHost is set but proxyPort is not or if the server - * protocol is not HTTP(s) - */ - public Proxy getHttpProxy() { - if (proxyHost == null) { - return null; - } else if (proxyPort == -1) { - throw new IllegalStateException("proxy_port is not specified for server " - + getUrlString() + " with proxyHost"); - } - if (protocol != null - && !protocol.equalsIgnoreCase("http") - && !protocol.equalsIgnoreCase("https")) { - throw new IllegalStateException( - "Server is not an HTTP(S) server, so it cannot use a HTTP proxyHost."); - } - return new Proxy(Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); - } - - /** Server host name or IP address. */ - public String getHost() { - return host; - } - - /** Set server host name or IP address. */ - public void setHost(String host) { - this.host = host; - } - - /** Server port. Defaults to -1. */ - public int getPort() { - return port; - } - - /** Set server port. */ - public void setPort(int port) { - this.port = port; - } - - /** Server protocol. */ - public String getProtocol() { - return protocol; - } - - /** Set server protocol. */ - public void setProtocol(String protocol) { - this.protocol = protocol; - } - - /** Proxy host name. Null if not set. */ - public String getProxyHost() { - return proxyHost; - } - - /** Set proxyHost host name. */ - public void setProxyHost(String proxyHost) { - this.proxyHost = proxyHost; - } - - /** Proxy port. Defaults to -1. */ - public int getProxyPort() { - return proxyPort; - } - - /** Set proxyHost port. */ - public void setProxyPort(int proxyPort) { - this.proxyPort = proxyPort; - } - - public String getPath() { - return path; - } - - /** - * Set the absolute path. If the path is empty, it will be set to the root. The path - * will be ended with a single slash. The path will be prepended with a single slash if needed. - * @param path path string - * @throws IllegalArgumentException if the path contains a question mark. - */ - public final void setPath(String path) { - this.path = cleanPath(path); - } - - @SuppressWarnings("PMD.UseStringBufferForStringAppends") - private static String cleanPath(String path) { - if (path == null) { - return null; - } - if (path.contains("?") || path.contains("#")) { - throw new IllegalArgumentException("Cannot set server path with query string"); - } - String newPath = path.trim(); - if (newPath.isEmpty()) { - return "/"; - } - if (newPath.charAt(0) != '/') { - newPath = '/' + newPath; - } - if (newPath.charAt(newPath.length() - 1) != '/') { - newPath += '/'; - } - return newPath; - } - - @Override - public String toString() { - return getUrlString(); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - ServerConfig otherConfig = (ServerConfig) other; - return Objects.equals(host, otherConfig.host) - && port == otherConfig.port - && unsafe == otherConfig.unsafe - && Objects.equals(protocol, otherConfig.protocol) - && Objects.equals(proxyHost, otherConfig.proxyHost) - && proxyPort == otherConfig.proxyPort; - } - - @Override - public int hashCode() { - return Objects.hash(protocol, host, port); - } - - public boolean isUnsafe() { - return unsafe; - } - - public void setUnsafe(boolean unsafe) { - this.unsafe = unsafe; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt new file mode 100644 index 00000000..f819f54e --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt @@ -0,0 +1,206 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.config + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.net.InetSocketAddress +import java.net.MalformedURLException +import java.net.Proxy +import java.net.URL +import java.util.* + +/** + * POJO representing a ServerConfig configuration. + */ +class ServerConfig { + /** Server host name or IP address. */ + /** Set server host name or IP address. */ + var host: String? = null + /** Server port. Defaults to -1. */ + /** Set server port. */ + var port = -1 + /** Server protocol. */ + /** Set server protocol. */ + var protocol: String? = null + /** + * Set the absolute path. If the path is empty, it will be set to the root. The path + * will be ended with a single slash. The path will be prepended with a single slash if needed. + * @throws IllegalArgumentException if the path contains a question mark. + */ + var path: String = "" + set(value) { + field = value.toUrlPath() + } + /** Proxy host name. Null if not set. */ + /** Set proxyHost host name. */ + var proxyHost: String? = null + /** Proxy port. Defaults to -1. */ + /** Set proxyHost port. */ + var proxyPort = -1 + var isUnsafe = false + + /** Pojo initializer. */ + constructor() { + // POJO initializer + } + + /** Parses the config from a URL. */ + constructor(url: URL) { + host = url.host + port = url.port + protocol = url.protocol + path = url.file + } + + /** Parses the config from a URL string. */ + constructor(urlString: String) { + val matcher = URL_PATTERN.matchEntire(urlString) + ?: throw MalformedURLException("Cannot create URL from string $this") + val groups = matcher.groups + protocol = groups[1]?.value ?: "https" + host = requireNotNull(groups[2]?.value) { "Cannot create URL without host name from $this" } + port = groups[3]?.value?.toIntOrNull() ?: -1 + path = groups[4]?.value.toUrlPath() + } + + /** Get the path of the server as a string. This does not include proxyHost information. */ + val urlString: String + get() = buildString(host!!.length + path.length + 20) { + appendUrlString(this) + } + + /** Get the path of the server as a string. This does not include proxyHost information. */ + private fun appendUrlString(builder: StringBuilder) = builder.run { + if (protocol != null) { + append(protocol) + append("://") + } + append(host) + if (port != -1) { + append(':') + append(port) + } + append(path) + } + + /** + * Get the server as a URL. + * + * @return URL to the server. + * @throws IllegalStateException if the URL is invalid + */ + val url: URL + get() { + checkNotNull(protocol) { "Cannot create URL without protocol" } + checkNotNull(host) { "Cannot create URL without host" } + return try { + URL(protocol, host, port, path) + } catch (ex: MalformedURLException) { + throw IllegalStateException("Already parsed a URL but it turned out invalid", ex) + } + } + + /** + * Get the server as an HttpUrl. + * @return HttpUrl to the server + * @throws IllegalStateException if the URL is invalid + */ + val httpUrl: HttpUrl + get() { + if (protocol == null) { + protocol = "http" + } + return urlString.toHttpUrl() + } + + /** + * Get the HTTP proxyHost associated to given server. + * @return http proxyHost if specified, or null if none is specified. + * @throws IllegalStateException if proxyHost is set but proxyPort is not or if the server + * protocol is not HTTP(s) + */ + val httpProxy: Proxy? + get() { + proxyHost ?: return null + check(proxyPort != -1) { "proxy_port is not specified for server $urlString with proxyHost" } + + check( + protocol == null || + protocol.equals("http", ignoreCase = true) || + protocol.equals("https", ignoreCase = true) + ) { "Server is not an HTTP(S) server, so it cannot use a HTTP proxyHost." } + return Proxy(Proxy.Type.HTTP, InetSocketAddress(proxyHost, proxyPort)) + } + + override fun toString(): String = urlString + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (other == null || javaClass != other.javaClass) { + return false + } + other as ServerConfig + return host == other.host && + port == other.port && + isUnsafe == other.isUnsafe && + protocol == other.protocol && + proxyHost == other.proxyHost && + proxyPort == other.proxyPort + } + + override fun hashCode(): Int { + return Objects.hash(host, path) + } + + companion object { + private val URL_PATTERN = "(?:(\\w+)://)?([^:/]+)(?::(\\d+))?(/.*)?".toRegex() + private val BAD_SLASHES_REGEX = "/(\\.*/)+".toRegex() + + /** Get the paths of a list of servers, concatenated with commas. */ + @JvmStatic + fun getPaths(configList: List): String = buildString(configList.size * 40) { + var first = true + for (server in configList) { + if (first) { + first = false + } else { + append(',') + } + server.appendUrlString(this) + } + } + + private fun String?.toUrlPath(): String { + this ?: return "" + require(!contains("?")) { "Cannot set server path with query string" } + require(!contains("#")) { "Cannot set server path with location string" } + var newPath = trim { it <= ' ' } + if (newPath.isEmpty()) { + return "/" + } + if (newPath.first() != '/') { + newPath = "/$newPath" + } + if (newPath.last() != '/') { + newPath += '/' + } + return newPath.replace(BAD_SLASHES_REGEX, "/") + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java deleted file mode 100644 index 21417ec1..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.data; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.Decoder; -import org.apache.avro.io.DecoderFactory; - -/** An AvroDecoder to decode known SpecificRecord classes. */ -public class AvroDatumDecoder implements AvroDecoder { - private final DecoderFactory decoderFactory; - private final boolean binary; - private final GenericData genericData; - - /** - * Decoder for Avro data. - * @param genericData instance of GenericData or SpecificData that should implement - * {@link GenericData#createDatumReader(Schema)}. - * @param binary true if the read data has Avro binary encoding, false if it has Avro JSON - * encoding. - */ - public AvroDatumDecoder(GenericData genericData, boolean binary) { - this.genericData = genericData; - this.decoderFactory = DecoderFactory.get(); - this.binary = binary; - } - - @Override - public AvroReader reader(Schema schema, Class clazz) { - @SuppressWarnings("unchecked") - DatumReader reader = genericData.createDatumReader(schema); - return new AvroRecordReader<>(schema, reader); - } - - private class AvroRecordReader implements AvroReader { - private final DatumReader reader; - private final Schema schema; - private Decoder decoder; - - private AvroRecordReader(Schema schema, DatumReader reader) { - this.reader = reader; - this.schema = schema; - this.decoder = null; - } - - @Override - public T decode(byte[] record) throws IOException { - return decode(record, 0); - } - - @Override - public T decode(byte[] record, int offset) throws IOException { - if (binary) { - decoder = decoderFactory.binaryDecoder(record, offset, record.length - offset, - (BinaryDecoder) decoder); - } else { - decoder = decoderFactory.jsonDecoder(schema, - new ByteArrayInputStream(record, offset, record.length - offset)); - } - return reader.read(null, decoder); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt new file mode 100644 index 00000000..e1aa04e1 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.data + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.avro.io.BinaryDecoder +import org.apache.avro.io.DatumReader +import org.apache.avro.io.Decoder +import org.apache.avro.io.DecoderFactory +import org.radarbase.data.AvroDecoder.AvroReader +import java.io.ByteArrayInputStream +import java.io.IOException + +/** An AvroDecoder to decode known SpecificRecord classes. */ +/** + * Decoder for Avro data. + * @param genericData instance of GenericData or SpecificData that should implement + * [GenericData.createDatumReader]. + * @param binary true if the read data has Avro binary encoding, false if it has Avro JSON + * encoding. + */ +class AvroDatumDecoder( + private val genericData: GenericData, + private val binary: Boolean, +) : AvroDecoder { + private val decoderFactory: DecoderFactory = DecoderFactory.get() + + override fun reader(schema: Schema, clazz: Class): AvroReader { + val reader = genericData.createDatumReader(schema) as DatumReader + return AvroRecordReader(schema, reader) + } + + private inner class AvroRecordReader( + private val schema: Schema, + private val reader: DatumReader + ) : AvroReader { + private var decoder: Decoder? = null + @Throws(IOException::class) + override fun decode(`object`: ByteArray): T { + return decode(`object`, 0) + } + + @Throws(IOException::class) + override fun decode(`object`: ByteArray, offset: Int): T { + decoder = if (binary) { + decoderFactory.binaryDecoder( + `object`, offset, `object`.size - offset, + decoder as? BinaryDecoder + ) + } else { + decoderFactory.jsonDecoder( + schema, + ByteArrayInputStream(`object`, offset, `object`.size - offset) + ) + } + return reader.read(null, decoder) + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java deleted file mode 100644 index fc364a05..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.data; - -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.EncoderFactory; - -/** An AvroEncoder to encode known SpecificRecord classes. */ -public class AvroDatumEncoder implements AvroEncoder { - private final EncoderFactory encoderFactory; - private final boolean binary; - private final GenericData genericData; - - /** - * Create a SpecificRecordEncoder. - * @param binary whether to use binary encoding or JSON. - */ - public AvroDatumEncoder(GenericData genericData, boolean binary) { - this.genericData = genericData; - this.encoderFactory = EncoderFactory.get(); - this.binary = binary; - } - - @Override - public AvroWriter writer(Schema schema, Class clazz) throws IOException { - @SuppressWarnings("unchecked") - DatumWriter writer = (DatumWriter)genericData.createDatumWriter(schema); - return new AvroRecordWriter<>(encoderFactory, schema, writer, binary); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt new file mode 100644 index 00000000..7502adff --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.data + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.avro.io.DatumWriter +import org.apache.avro.io.EncoderFactory +import org.radarbase.data.AvroEncoder.AvroWriter +import java.io.IOException + +/** + * An AvroEncoder to encode known SpecificRecord classes. + * @param binary whether to use binary encoding or JSON. + */ +class AvroDatumEncoder( + private val genericData: GenericData, + private val binary: Boolean, +) : AvroEncoder { + private val encoderFactory: EncoderFactory = EncoderFactory.get() + + @Throws(IOException::class) + override fun writer(schema: Schema, clazz: Class): AvroWriter { + val writer = genericData.createDatumWriter(schema) as DatumWriter + return AvroRecordWriter(encoderFactory, schema, writer, binary) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDecoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroDecoder.kt similarity index 66% rename from radar-commons/src/main/java/org/radarbase/data/AvroDecoder.java rename to radar-commons/src/main/java/org/radarbase/data/AvroDecoder.kt index bf411098..0b48bc07 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDecoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDecoder.kt @@ -13,29 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.data -package org.radarbase.data; - -import java.io.IOException; -import org.apache.avro.Schema; - -/** Decode Avro values with a given encoder. */ -public interface AvroDecoder { - /** Create a new reader. This method is thread-safe, but the class it returns is not. */ - AvroReader reader(Schema schema, Class clazz) throws IOException; +import org.apache.avro.Schema +import java.io.IOException +/** Decode Avro values with a given encoder. */ +interface AvroDecoder { + /** Create a new reader. This method is thread-safe, but the class it returns is not. */ + @Throws(IOException::class) + fun reader(schema: Schema, clazz: Class): AvroReader interface AvroReader { /** * Decode an object from bytes. This method is not thread-safe. Equivalent to calling * decode(object, 0). */ - T decode(byte[] object) throws IOException; + @Throws(IOException::class) + fun decode(`object`: ByteArray): T /** * Decode an object from bytes. This method is not thread-safe. * @param object bytes to decode from - * @param start start offset to decode from. + * @param offset start offset to decode from. */ - T decode(byte[] object, int start) throws IOException; + @Throws(IOException::class) + fun decode(`object`: ByteArray, offset: Int): T } } diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.java b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt similarity index 64% rename from radar-commons/src/main/java/org/radarbase/data/AvroEncoder.java rename to radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt index fcfc9fab..5f573ab6 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.java +++ b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt @@ -13,41 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.data -package org.radarbase.data; - -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.SchemaValidationException; -import org.radarbase.producer.rest.ParsedSchemaMetadata; +import org.apache.avro.Schema +import org.apache.avro.SchemaValidationException +import org.radarbase.producer.rest.ParsedSchemaMetadata +import java.io.IOException /** Encode Avro values with a given encoder. The encoder may take into account the schema - * that the schema registry has listed for a given topic. */ -public interface AvroEncoder { - /** Create a new writer. This method is thread-safe, but the class it returns is not. */ - AvroWriter writer(Schema schema, Class clazz) throws IOException; - - interface AvroWriter { + * that the schema registry has listed for a given topic. */ +interface AvroEncoder { + /** Create a new writer. This method is thread-safe, but the class it returns is not. */ + @Throws(IOException::class) + fun writer(schema: Schema, clazz: Class): AvroWriter + interface AvroWriter { /** * Encode an object. This method is not thread-safe. Call - * {@link #setReaderSchema(ParsedSchemaMetadata)} before calling encode. + * [.setReaderSchema] before calling encode. * @param object object to encode * @return byte array with serialized object. */ - byte[] encode(T object) throws IOException; - + @Throws(IOException::class) + fun encode(`object`: T): ByteArray + /** + * Get the schema that the server lists. + * @return schema as set by setReaderSchema or null if not called yet. + */ /** * Update the schema that the server is lists for the current topic. * @param readerSchema schema listed by the schema registry. * @throws SchemaValidationException if the server schema is incompatible with the writer - * schema. - */ - void setReaderSchema(ParsedSchemaMetadata readerSchema) throws SchemaValidationException; - - /** - * Get the schema that the server lists. - * @return schema as set by setReaderSchema or null if not called yet. + * schema. */ - ParsedSchemaMetadata getReaderSchema(); + @set:Throws(SchemaValidationException::class) + var readerSchema: ParsedSchemaMetadata? } } diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java deleted file mode 100644 index 9180e7a6..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.radarbase.data; - -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import org.radarbase.topic.AvroTopic; - -/** - * Avro record data belonging to a single key. - * @param key type - * @param value type - */ -public class AvroRecordData implements RecordData { - private final AvroTopic topic; - private final K key; - private final List records; - - /** - * Data from a topic. - * @param topic data topic - * @param key data key - * @param values non-empty data values. - * @throws IllegalArgumentException if the values are empty. - * @throws NullPointerException if any of the parameters are null. - */ - public AvroRecordData(AvroTopic topic, K key, List values) { - this.topic = Objects.requireNonNull(topic); - this.key = Objects.requireNonNull(key); - this.records = Objects.requireNonNull(values); - if (this.records.isEmpty()) { - throw new IllegalArgumentException("Records should not be empty."); - } - } - - @Override - public AvroTopic getTopic() { - return topic; - } - - @Override - public K getKey() { - return key; - } - - @Override - public Iterator iterator() { - return records.iterator(); - } - - @Override - public boolean isEmpty() { - return false; - } - - @Override - public int size() { - return records.size(); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt new file mode 100644 index 00000000..4e0dc39c --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt @@ -0,0 +1,32 @@ +package org.radarbase.data + +import org.radarbase.topic.AvroTopic + +/** + * Avro record data belonging to a single key. + * @param key type + * @param value type + */ +/** + * Data from a topic. + * @param topic data topic + * @param key data key + * @param records non-empty data values. + * @throws IllegalArgumentException if the values are empty. + * @throws NullPointerException if any of the parameters are null. + */ +class AvroRecordData( + override val topic: AvroTopic, + override val key: K, + private val records: List, +) : RecordData { + init { + require(records.isNotEmpty()) { "Records should not be empty." } + } + + override fun iterator(): Iterator = records.iterator() + + override val isEmpty: Boolean = records.isEmpty() + + override fun size(): Int = records.size +} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java deleted file mode 100644 index bec2b3ba..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.data; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.radarbase.producer.rest.ParsedSchemaMetadata; -import org.radarbase.data.AvroEncoder.AvroWriter; - -/** - * Encodes Avro records to bytes. - */ -public class AvroRecordWriter implements AvroWriter { - private final Encoder encoder; - private final ByteArrayOutputStream out; - private final DatumWriter writer; - private ParsedSchemaMetadata serverSchema; - - /** - * Writer for a given encoder, schema and writer. - * @param encoderFactory encoder factory to use. - * @param schema schema to write records with. - * @param writer data writer - * @param binary true if the data should be serialized with binary Avro encoding, false if it - * should be with JSON encoding. - * @throws IOException if an encoder cannot be constructed. - */ - public AvroRecordWriter(EncoderFactory encoderFactory, Schema schema, DatumWriter writer, - boolean binary) throws IOException { - this.writer = writer; - out = new ByteArrayOutputStream(); - if (binary) { - encoder = encoderFactory.binaryEncoder(out, null); - } else { - encoder = encoderFactory.jsonEncoder(schema, out); - } - } - - @Override - public byte[] encode(T record) throws IOException { - try { - writer.write(record, encoder); - encoder.flush(); - return out.toByteArray(); - } finally { - out.reset(); - } - } - - @Override - public void setReaderSchema(ParsedSchemaMetadata readerSchema) { - this.serverSchema = readerSchema; - } - - @Override - public ParsedSchemaMetadata getReaderSchema() { - return serverSchema; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt new file mode 100644 index 00000000..df5c2d00 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.data + +import org.apache.avro.Schema +import org.apache.avro.io.DatumWriter +import org.apache.avro.io.Encoder +import org.apache.avro.io.EncoderFactory +import org.radarbase.data.AvroEncoder.AvroWriter +import org.radarbase.producer.rest.ParsedSchemaMetadata +import java.io.ByteArrayOutputStream +import java.io.IOException + +/** + * Encodes Avro records to bytes. + * + * @param encoderFactory encoder factory to use. + * @param schema schema to write records with. + * @param writer data writer + * @param binary true if the data should be serialized with binary Avro encoding, false if it + * should be with JSON encoding. + * @throws IOException if an encoder cannot be constructed. + */ +class AvroRecordWriter( + encoderFactory: EncoderFactory, + schema: Schema, + private val writer: DatumWriter, + binary: Boolean, +) : AvroWriter { + private val out: ByteArrayOutputStream = ByteArrayOutputStream() + private var encoder: Encoder = if (binary) { + encoderFactory.binaryEncoder(out, null) + } else { + encoderFactory.jsonEncoder(schema, out) + } + override var readerSchema: ParsedSchemaMetadata? = null + + init { + encoder = if (binary) { + encoderFactory.binaryEncoder(out, null) + } else { + encoderFactory.jsonEncoder(schema, out) + } + } + + @Throws(IOException::class) + override fun encode(`object`: T): ByteArray { + return try { + writer.write(`object`, encoder) + encoder.flush() + out.toByteArray() + } finally { + out.reset() + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/data/Record.java b/radar-commons/src/main/java/org/radarbase/data/Record.kt similarity index 68% rename from radar-commons/src/main/java/org/radarbase/data/Record.java rename to radar-commons/src/main/java/org/radarbase/data/Record.kt index 91f526a5..4c3d4c8e 100644 --- a/radar-commons/src/main/java/org/radarbase/data/Record.java +++ b/radar-commons/src/main/java/org/radarbase/data/Record.kt @@ -13,26 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.radarbase.data; +package org.radarbase.data /** * A single data record. * * @param key type * @param value type - */ -public class Record { - public final K key; - public final V value; - - /** - * Single record, with current time as time added. - * @param key key - * @param value value - */ - public Record(K key, V value) { - this.key = key; - this.value = value; - } -} + */ +class Record +/** + * Single record, with current time as time added. + * @param key key + * @param value value + */(val key: K, val value: V) diff --git a/radar-commons/src/main/java/org/radarbase/data/RecordData.java b/radar-commons/src/main/java/org/radarbase/data/RecordData.kt similarity index 67% rename from radar-commons/src/main/java/org/radarbase/data/RecordData.java rename to radar-commons/src/main/java/org/radarbase/data/RecordData.kt index 54b0cb5c..8e5c2a5a 100644 --- a/radar-commons/src/main/java/org/radarbase/data/RecordData.java +++ b/radar-commons/src/main/java/org/radarbase/data/RecordData.kt @@ -1,34 +1,34 @@ -package org.radarbase.data; +package org.radarbase.data -import org.radarbase.topic.AvroTopic; +import org.radarbase.topic.AvroTopic /** * Record data belonging to a single key. * @param key type * @param value type - */ -public interface RecordData extends Iterable { + */ +interface RecordData : Iterable { /** * Topic that the data belongs to. * @return Avro topic. */ - AvroTopic getTopic(); + val topic: AvroTopic /** * Key of each of the entries in the data set. * @return key */ - K getKey(); + val key: K /** * Whether the list of values is empty. * @return true if empty, false otherwise. */ - boolean isEmpty(); + val isEmpty: Boolean /** * The size of the value list. * @return size. */ - int size(); + fun size(): Int } diff --git a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java deleted file mode 100644 index bbd65b48..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.radarbase.data; - -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.generic.GenericData; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificRecord; -import org.radarbase.producer.rest.AvroDataMapper; -import org.radarbase.producer.rest.AvroDataMapperFactory; -import org.radarbase.producer.rest.ParsedSchemaMetadata; - -/** - * Encodes data according to an Avro schema to the format and schema of the server. - */ -public class RemoteSchemaEncoder implements AvroEncoder { - private final boolean binary; - - /** - * Schema encoder. - * @param binary true if the server wants binary encoding, false if it wants JSON encoding. - */ - public RemoteSchemaEncoder(boolean binary) { - this.binary = binary; - } - - @Override - public AvroWriter writer(Schema schema, Class clazz) { - return new SchemaEncoderWriter<>(schema, clazz); - } - - private class SchemaEncoderWriter implements AvroWriter { - private final AvroEncoder recordEncoder; - private AvroEncoder.AvroWriter encoder; - private final boolean isGeneric; - private ParsedSchemaMetadata serverSchema; - private AvroDataMapper mapper; - private final Schema schema; - - SchemaEncoderWriter(Schema schema, Class clazz) { - this.schema = schema; - - GenericData genericData; - ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - if (SpecificRecord.class.isAssignableFrom(clazz)) { - genericData = new SpecificData(classLoader); - isGeneric = false; - } else { - genericData = new GenericData(classLoader); - isGeneric = true; - } - recordEncoder = new AvroDatumEncoder(genericData, binary); - } - - @Override - public byte[] encode(T object) throws IOException { - return encoder.encode(mapper.convertAvro(object)); - } - - @Override - public final void setReaderSchema(ParsedSchemaMetadata readerSchema) - throws SchemaValidationException { - if (this.serverSchema != null - && readerSchema.getSchema().equals(this.serverSchema.getSchema())) { - return; - } - try { - if (!isGeneric) { - this.mapper = AvroDataMapperFactory.IDENTITY_MAPPER; - encoder = recordEncoder.writer(schema, Object.class); - } else { - this.mapper = AvroDataMapperFactory.get() - .createMapper(schema, readerSchema.getSchema(), - null); - encoder = recordEncoder.writer(readerSchema.getSchema(), Object.class); - } - this.serverSchema = readerSchema; - } catch (IOException ex) { - throw new IllegalStateException("Cannot construct Avro writer", ex); - } - } - - @Override - public ParsedSchemaMetadata getReaderSchema() { - return serverSchema; - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt new file mode 100644 index 00000000..410b0329 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt @@ -0,0 +1,83 @@ +package org.radarbase.data + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.avro.specific.SpecificData +import org.apache.avro.specific.SpecificRecord +import org.radarbase.data.AvroEncoder.AvroWriter +import org.radarbase.producer.rest.AvroDataMapper +import org.radarbase.producer.rest.AvroDataMapperFactory +import org.radarbase.producer.rest.ParsedSchemaMetadata +import java.io.IOException + +/** + * Encodes data according to an Avro schema to the format and schema of the server. + * + * @param binary true if the server wants binary encoding, false if it wants JSON encoding. + */ +class RemoteSchemaEncoder( + private val binary: Boolean, +) : AvroEncoder { + override fun writer(schema: Schema, clazz: Class): AvroWriter { + return SchemaEncoderWriter(schema, clazz) + } + + private inner class SchemaEncoderWriter( + private val schema: Schema, + clazz: Class + ) : AvroWriter { + private val recordEncoder: AvroEncoder + private var encoder: AvroWriter? = null + private var isGeneric = false + override var readerSchema: ParsedSchemaMetadata? = null + set(value) { + value ?: return + val currentField = field + if (currentField != null && value.schema == currentField.schema) { + return + } + try { + if (!isGeneric) { + mapper = AvroDataMapperFactory.IDENTITY_MAPPER + encoder = recordEncoder.writer(schema, Any::class.java) + } else { + mapper = AvroDataMapperFactory.instance.createMapper( + schema, + value.schema, + null + ) + encoder = recordEncoder.writer(value.schema, Any::class.java) + } + field = value + } catch (ex: IOException) { + throw IllegalStateException("Cannot construct Avro writer", ex) + } + } + + private var mapper: AvroDataMapper? = null + + init { + val genericData: GenericData + val classLoader = Thread.currentThread().contextClassLoader + if (SpecificRecord::class.java.isAssignableFrom(clazz)) { + genericData = SpecificData(classLoader) + isGeneric = false + } else { + genericData = GenericData(classLoader) + isGeneric = true + } + recordEncoder = AvroDatumEncoder(genericData, binary) + } + + @Throws(IOException::class) + override fun encode(`object`: T): ByteArray { + val localEncoder = checkNotNull(encoder) { "Did not initialize reader schema yet " } + val localMapper = checkNotNull(mapper) { "Did not initialize reader schema yet" } + return localEncoder.encode( + requireNotNull(localMapper.convertAvro(`object`)) { + "Cannot map $`object` to Avro" + } + ) + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/data/StringEncoder.java b/radar-commons/src/main/java/org/radarbase/data/StringEncoder.java deleted file mode 100644 index 0169be8d..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/StringEncoder.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.data; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.SchemaValidationException; -import org.radarbase.producer.rest.ParsedSchemaMetadata; - -/** Encodes a String as Avro. */ -public class StringEncoder implements AvroEncoder, AvroEncoder.AvroWriter { - private static final ObjectWriter JSON_ENCODER = new ObjectMapper().writer(); - private ParsedSchemaMetadata readerSchema; - - @SuppressWarnings("unchecked") - @Override - public AvroWriter writer(Schema schema, Class clazz) { - if (schema.getType() != Schema.Type.STRING || !clazz.equals(String.class)) { - throw new IllegalArgumentException( - "Cannot encode String with a different type than STRING."); - } - - return (AvroWriter) this; - } - - @Override - public byte[] encode(String object) throws IOException { - return JSON_ENCODER.writeValueAsBytes(object); - } - - @Override - public void setReaderSchema(ParsedSchemaMetadata readerSchema) - throws SchemaValidationException { - if (readerSchema.getSchema().getType() != Type.STRING) { - throw new SchemaValidationException( - Schema.create(Type.STRING), - readerSchema.getSchema(), - new IllegalArgumentException("Cannot convert type to STRING")); - } - this.readerSchema = readerSchema; - - } - - @Override - public ParsedSchemaMetadata getReaderSchema() { - return readerSchema; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/data/TimedInt.java b/radar-commons/src/main/java/org/radarbase/data/TimedInt.java deleted file mode 100644 index 7c065bba..00000000 --- a/radar-commons/src/main/java/org/radarbase/data/TimedInt.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.data; - -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -/** - * A single int, with modification times timed with system milliseconds time. - * This class can be used from multiple threads. - */ -public class TimedInt { - private final AtomicInteger value = new AtomicInteger(); - private final AtomicLong time = new AtomicLong(-1L); - - /** - * Value of the int. - */ - public int getValue() { - return value.get(); - } - - /** - * Time that the int got modified. - */ - public long getTime() { - return time.get(); - } - - /** - * Add value to the int. This updates the time variable to now. - * @param delta value to add. - */ - public void add(int delta) { - value.addAndGet(delta); - time.set(System.currentTimeMillis()); - } - - /** - * Set value to the int. This updates the time variable to now. - * @param value new value - */ - public void set(int value) { - this.value.set(value); - time.set(System.currentTimeMillis()); - } - - @Override - public synchronized boolean equals(Object other) { - if (other == null || !getClass().equals(other.getClass())) { - return false; - } - TimedInt timedOther = (TimedInt)other; - return value.equals(timedOther.value) && time.equals(timedOther.time); - } - - @Override - public int hashCode() { - return 31 * value.hashCode() + time.hashCode(); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.java b/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt similarity index 57% rename from radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.java rename to radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt index dcc81fab..7531ddd3 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.java +++ b/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt @@ -13,30 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.producer -package org.radarbase.producer; - -import java.io.IOException; +import java.io.IOException /** * Failed to authenticate to server. */ -public class AuthenticationException extends IOException { - private static final long serialVersionUID = 1; - - public AuthenticationException() { - super(); - } - - public AuthenticationException(String message, Throwable cause) { - super(message, cause); - } - - public AuthenticationException(String message) { - super(message); - } +class AuthenticationException : IOException { + constructor() : super() {} + constructor(message: String?, cause: Throwable?) : super(message, cause) {} + constructor(message: String?) : super(message) {} + constructor(cause: Throwable?) : super(cause) {} - public AuthenticationException(Throwable cause) { - super(cause); + companion object { + private const val serialVersionUID: Long = 1 } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java deleted file mode 100644 index 8bc7f22c..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import org.apache.avro.SchemaValidationException; -import org.radarbase.data.AvroRecordData; -import org.radarbase.data.RecordData; -import org.radarbase.topic.AvroTopic; - -/** - * A Kafka REST Proxy sender that batches up records. It will send data once the batch size is - * exceeded, or when at a send call the first record in the batch is older than given age. If send, - * flush or close are not called within this given age, the data will also not be sent. Calling - * {@link #close()} will not flush or close the KafkaTopicSender that were created. That must be - * done separately. - */ -public class BatchedKafkaSender implements KafkaSender { - private final KafkaSender wrappedSender; - private final long ageNanos; - private final int maxBatchSize; - - /** - * Kafka sender that sends data along. - * @param sender kafka sender to send data with. - * @param ageMillis threshold time after which a record should be sent. - * @param maxBatchSize threshold batch size over which records should be sent. - */ - public BatchedKafkaSender(KafkaSender sender, int ageMillis, int maxBatchSize) { - this.wrappedSender = sender; - this.ageNanos = TimeUnit.MILLISECONDS.toNanos(ageMillis); - this.maxBatchSize = maxBatchSize; - } - - @Override - public KafkaTopicSender sender(final AvroTopic topic) - throws IOException, SchemaValidationException { - return new BatchedKafkaTopicSender<>(topic); - } - - @Override - public boolean isConnected() throws AuthenticationException { - return wrappedSender.isConnected(); - } - - @Override - public boolean resetConnection() throws AuthenticationException { - return wrappedSender.resetConnection(); - } - - @Override - public synchronized void close() throws IOException { - wrappedSender.close(); - } - - /** Batched kafka topic sender. This does the actual data batching. */ - private class BatchedKafkaTopicSender implements KafkaTopicSender { - private long nanoAdded; - private K cachedKey; - private final List cache; - private final KafkaTopicSender topicSender; - private final AvroTopic topic; - - private BatchedKafkaTopicSender(AvroTopic topic) - throws IOException, SchemaValidationException { - cache = new ArrayList<>(); - this.topic = topic; - topicSender = wrappedSender.sender(topic); - } - - @Override - public void send(K key, V value) throws IOException, SchemaValidationException { - if (!isConnected()) { - throw new IOException("Cannot send records to unconnected producer."); - } - trySend(key, value); - } - - @Override - public void send(RecordData records) throws IOException, SchemaValidationException { - if (records.isEmpty()) { - return; - } - K key = records.getKey(); - for (V value : records) { - trySend(key, value); - } - } - - private void trySend(K key, V record) throws IOException, SchemaValidationException { - boolean keysMatch; - - if (cache.isEmpty()) { - cachedKey = key; - nanoAdded = System.nanoTime(); - keysMatch = true; - } else { - keysMatch = Objects.equals(key, cachedKey); - } - - if (keysMatch) { - cache.add(record); - if (exceedsBuffer(cache)) { - doSend(); - } - } else { - doSend(); - trySend(key, record); - } - } - - private void doSend() throws IOException, SchemaValidationException { - topicSender.send(new AvroRecordData<>(topic, cachedKey, cache)); - cache.clear(); - cachedKey = null; - } - - @Override - public void clear() { - cache.clear(); - topicSender.clear(); - } - - @Override - public void flush() throws IOException { - if (!cache.isEmpty()) { - try { - doSend(); - } catch (SchemaValidationException ex) { - throw new IOException("Schemas do not match", ex); - } - } - topicSender.flush(); - } - - @Override - @SuppressWarnings("PMD.UseTryWithResources") - public void close() throws IOException { - try { - flush(); - } finally { - wrappedSender.close(); - } - } - - private boolean exceedsBuffer(List records) { - return records.size() >= maxBatchSize - || System.nanoTime() - nanoAdded >= ageNanos; - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt new file mode 100644 index 00000000..ac4dea11 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt @@ -0,0 +1,151 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer + +import org.apache.avro.SchemaValidationException +import org.radarbase.data.AvroRecordData +import org.radarbase.data.RecordData +import org.radarbase.topic.AvroTopic +import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * A Kafka REST Proxy sender that batches up records. It will send data once the batch size is + * exceeded, or when at a send call the first record in the batch is older than given age. If send, + * flush or close are not called within this given age, the data will also not be sent. Calling + * [.close] will not flush or close the KafkaTopicSender that were created. That must be + * done separately. + * + * @param wrappedSender kafka sender to send data with. + * @param ageMillis threshold time after which a record should be sent. + * @param maxBatchSize threshold batch size over which records should be sent. +*/ +class BatchedKafkaSender( + private val wrappedSender: KafkaSender, + ageMillis: Int, + private val maxBatchSize: Int +) : KafkaSender { + private val ageNanos: Long + + init { + ageNanos = TimeUnit.MILLISECONDS.toNanos(ageMillis.toLong()) + } + + @Throws(IOException::class, SchemaValidationException::class) + override fun sender(topic: AvroTopic): KafkaTopicSender { + return BatchedKafkaTopicSender(topic) + } + + @get:Throws(AuthenticationException::class) + override val isConnected: Boolean + get() = wrappedSender.isConnected + + @Throws(AuthenticationException::class) + override fun resetConnection(): Boolean { + return wrappedSender.resetConnection() + } + + @Synchronized + @Throws(IOException::class) + override fun close() { + wrappedSender.close() + } + + /** Batched kafka topic sender. This does the actual data batching. */ + private inner class BatchedKafkaTopicSender( + private val topic: AvroTopic + ) : KafkaTopicSender { + private var nanoAdded: Long = 0 + private var cachedKey: K? = null + private val cache: MutableList = ArrayList() + private val topicSender: KafkaTopicSender = wrappedSender.sender(topic) + + @Throws(IOException::class, SchemaValidationException::class) + override fun send(key: K, value: V) { + if (!isConnected) { + throw IOException("Cannot send records to unconnected producer.") + } + trySend(key, value) + } + + @Throws(IOException::class, SchemaValidationException::class) + override fun send(records: RecordData) { + if (records.isEmpty) return + val key = records.key + for (value in records) { + trySend(key, value) + } + } + + @Throws(IOException::class, SchemaValidationException::class) + private fun trySend(key: K, record: V) { + val keysMatch: Boolean + if (cache.isEmpty()) { + cachedKey = key + nanoAdded = System.nanoTime() + keysMatch = true + } else { + keysMatch = key == cachedKey + } + if (keysMatch) { + cache.add(record) + if (exceedsBuffer(cache)) { + doSend() + } + } else { + doSend() + trySend(key, record) + } + } + + @Throws(IOException::class, SchemaValidationException::class) + private fun doSend() { + val key = checkNotNull(cachedKey) { "Cached key should not be null in this function" } + topicSender.send(AvroRecordData(topic, key, cache)) + cache.clear() + cachedKey = null + } + + override fun clear() { + cache.clear() + topicSender.clear() + } + + @Throws(IOException::class) + override fun flush() { + if (cache.isNotEmpty()) { + try { + doSend() + } catch (ex: SchemaValidationException) { + throw IOException("Schemas do not match", ex) + } + } + topicSender.flush() + } + + @Throws(IOException::class) + override fun close() { + wrappedSender.use { + flush() + } + } + + private fun exceedsBuffer(records: List<*>): Boolean { + return records.size >= maxBatchSize || + System.nanoTime() - nanoAdded >= ageNanos + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.java b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt similarity index 55% rename from radar-commons/src/main/java/org/radarbase/producer/KafkaSender.java rename to radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt index f5e3ac70..53fea6c6 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt @@ -13,38 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.producer -package org.radarbase.producer; - -import java.io.Closeable; -import java.io.IOException; -import org.apache.avro.SchemaValidationException; -import org.radarbase.topic.AvroTopic; +import org.apache.avro.SchemaValidationException +import org.radarbase.topic.AvroTopic +import java.io.Closeable +import java.io.IOException /** - * Thread-safe sender. Calling {@link #close()} must be done after all {@link KafkaTopicSender} - * senders created with {@link #sender(AvroTopic)} have been called. + * Thread-safe sender. Calling [.close] must be done after all [KafkaTopicSender] + * senders created with [.sender] have been called. */ -public interface KafkaSender extends Closeable { - /** Get a non thread-safe sender instance. */ - KafkaTopicSender sender(AvroTopic topic) - throws IOException, SchemaValidationException; +interface KafkaSender : Closeable { + /** Get a non thread-safe sender instance. */ + @Throws(IOException::class, SchemaValidationException::class) + fun sender(topic: AvroTopic): KafkaTopicSender /** * If the sender is no longer connected, try to reconnect. * @return whether the connection has been restored. * @throws AuthenticationException if the headers caused an authentication error - * in the current request or in a previous one. + * in the current request or in a previous one. */ - boolean resetConnection() throws AuthenticationException; + @Throws(AuthenticationException::class) + fun resetConnection(): Boolean /** * Get the current connection state to Kafka. If the connection state is unknown, this will * trigger a connection check. * @return true if connected, false if not connected. * @throws AuthenticationException if the headers caused an authentication error - * in a previous request or during an additional connection - * check. + * in a previous request or during an additional connection + * check. */ - boolean isConnected() throws AuthenticationException; + @get:Throws(AuthenticationException::class) + val isConnected: Boolean } diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.java b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt similarity index 62% rename from radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.java rename to radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt index 21e26a32..7eb8e79f 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.java +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt @@ -1,14 +1,14 @@ -package org.radarbase.producer; +package org.radarbase.producer -import java.io.Closeable; -import java.io.IOException; -import org.apache.avro.SchemaValidationException; -import org.radarbase.data.RecordData; +import org.apache.avro.SchemaValidationException +import org.radarbase.data.RecordData +import java.io.Closeable +import java.io.IOException /** - * Sender for a single topic. Should be created through a {@link KafkaSender}. + * Sender for a single topic. Should be created through a [KafkaSender]. */ -public interface KafkaTopicSender extends Closeable { +interface KafkaTopicSender : Closeable { /** * Send a message to Kafka eventually. * @@ -17,7 +17,8 @@ public interface KafkaTopicSender extends Closeable { * @throws AuthenticationException if the client failed to authenticate itself * @throws IOException if the client could not send a message */ - void send(K key, V value) throws IOException, SchemaValidationException; + @Throws(IOException::class, SchemaValidationException::class) + fun send(key: K, value: V) /** * Send a message to Kafka eventually. Contained offsets must be strictly monotonically @@ -27,12 +28,13 @@ public interface KafkaTopicSender extends Closeable { * @throws AuthenticationException if the client failed to authenticate itself * @throws IOException if the client could not send a message */ - void send(RecordData records) throws IOException, SchemaValidationException; + @Throws(IOException::class, SchemaValidationException::class) + fun send(records: RecordData) /** * Clears any messages still in cache. */ - void clear(); + fun clear() /** * Flush all remaining messages. @@ -40,5 +42,6 @@ public interface KafkaTopicSender extends Closeable { * @throws AuthenticationException if the client failed to authenticate itself * @throws IOException if the client could not send a message */ - void flush() throws IOException; + @Throws(IOException::class) + fun flush() } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.java b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.kt similarity index 52% rename from radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.kt index f2027223..d0e450db 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.kt @@ -1,16 +1,14 @@ -package org.radarbase.producer.rest; - -import org.apache.avro.Schema; +package org.radarbase.producer.rest /** * Maps data from one avro record schema to another. Create it by calling - * {@link AvroDataMapperFactory#createMapper(Schema, Schema, Object)}. + * [AvroDataMapperFactory.createMapper]. */ -public interface AvroDataMapper { +fun interface AvroDataMapper { /** * Convert an Avro GenericData to another Avro GenericData representation. * @param object Avro object * @return Avro object */ - Object convertAvro(Object object); + fun convertAvro(`object`: Any?): Any? } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java deleted file mode 100644 index 6ff189be..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.java +++ /dev/null @@ -1,497 +0,0 @@ -package org.radarbase.producer.rest; - -import static org.apache.avro.JsonProperties.NULL_VALUE; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericData.Fixed; -import org.apache.avro.generic.GenericEnumSymbol; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.generic.IndexedRecord; -import org.radarbase.util.Base64; -import org.radarbase.util.Base64.Encoder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings({"PMD"}) -public final class AvroDataMapperFactory { - private static final Logger logger = LoggerFactory.getLogger(AvroDataMapperFactory.class); - - public static final AvroDataMapper IDENTITY_MAPPER = new AvroDataMapper() { - @Override - public Object convertAvro(Object obj) { - return obj; - } - - @Override - public String toString() { - return "Identity"; - } - }; - private static final AvroDataMapperFactory INSTANCE = new AvroDataMapperFactory(); - - public static AvroDataMapperFactory get() { - return INSTANCE; - } - - /** - * Create a mapper for data in one Avro schema to that in another Avro schema. - * @param from originating Avro schema - * @param to resulting Avro schema - * @param defaultVal default value as defined in an Avro record field, - * may be null if there is no default value. - * @return Avro data mapper - * @throws SchemaValidationException if the given schemas are incompatible. - */ - public AvroDataMapper createMapper(Schema from, Schema to, final Object defaultVal) - throws SchemaValidationException { - if (from.equals(to)) { - logger.debug("Using identity schema mapping from {} to {}", from, to); - return IDENTITY_MAPPER; - } - - logger.debug("Computing custom mapping from {} to {}", from, to); - try { - if (to.getType() == Schema.Type.UNION || from.getType() == Schema.Type.UNION) { - return mapUnion(from, to, defaultVal); - } - if (to.getType() == Schema.Type.ENUM || to.getType() == Schema.Type.ENUM) { - return mapEnum(from, to, defaultVal); - } - - switch (to.getType()) { - case INT: - case LONG: - case DOUBLE: - case FLOAT: - return mapNumber(from, to, defaultVal); - default: - break; - } - switch (from.getType()) { - case RECORD: - return mapRecord(from, to); - case ARRAY: - return mapArray(from, to); - case MAP: - return mapMap(from, to); - case FIXED: - case BYTES: - return mapBytes(from, to, defaultVal); - case INT: - case LONG: - case DOUBLE: - case FLOAT: - return mapNumber(from, to, defaultVal); - default: - if (from.getType() != to.getType()) { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Schema types of from and to don't match")); - } - return IDENTITY_MAPPER; - } - } catch (SchemaValidationException ex) { - if (defaultVal != null) { - if (defaultVal == NULL_VALUE) { - return obj -> null; - } else { - return obj -> defaultVal; - } - } else { - throw ex; - } - } - } - - /** Map one enum to another or to String. */ - private static AvroDataMapper mapEnum(Schema from, final Schema to, Object defaultVal) - throws SchemaValidationException { - if (to.getType() == Schema.Type.ENUM) { - boolean containsAll = true; - if (from.getType() == Schema.Type.ENUM) { - for (String s : from.getEnumSymbols()) { - if (!to.hasEnumSymbol(s)) { - containsAll = false; - break; - } - } - } else if (from.getType() == Schema.Type.STRING) { - containsAll = false; - } else { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map enum from non-string or enum type")); - } - if (containsAll) { - return obj -> new GenericData.EnumSymbol(to, obj.toString()); - } else { - String defaultString = (String) defaultVal; - if (defaultString == null && to.hasEnumSymbol("UNKNOWN")) { - defaultString = "UNKNOWN"; - } - if (defaultString == null) { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map enum symbols without default value")); - } else { - GenericEnumSymbol symbol = new GenericData.EnumSymbol(to, defaultString); - return obj -> { - String value = obj.toString(); - if (to.hasEnumSymbol(value)) { - return new GenericData.EnumSymbol(to, value); - } else { - return symbol; - } - }; - } - } - } else if (from.getType() == Schema.Type.ENUM && to.getType() == Schema.Type.STRING) { - return Object::toString; - } else { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map unknown type with enum.")); - } - } - - /** Get the default value as a Generic type. */ - private static Object getDefaultValue(Object defaultVal, Schema schema) { - if (defaultVal == null) { - return null; - } else if (schema.getType() == Schema.Type.ENUM) { - return new GenericData.EnumSymbol(schema, defaultVal); - } else { - return defaultVal; - } - } - - /** Maps one number type to another or parses/converts to a string. */ - private static AvroDataMapper mapNumber(Schema from, Schema to, final Object defaultVal) - throws SchemaValidationException { - if (from.getType() == to.getType()) { - return IDENTITY_MAPPER; - } - - if (from.getType() == Schema.Type.STRING) { - if (defaultVal == null) { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map string to number without default value.")); - } else { - switch (to.getType()) { - case INT: - return new StringToNumberMapper(defaultVal) { - @Override - public Number stringToNumber(String obj) { - return Integer.valueOf(obj); - } - }; - case LONG: - return new StringToNumberMapper(defaultVal) { - @Override - public Number stringToNumber(String obj) { - return Long.valueOf(obj); - } - }; - case DOUBLE: - return new StringToNumberMapper(defaultVal) { - @Override - public Number stringToNumber(String obj) { - return Double.valueOf(obj); - } - }; - case FLOAT: - return new StringToNumberMapper(defaultVal) { - @Override - public Number stringToNumber(String obj) { - return Float.valueOf(obj); - } - }; - default: - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map numeric type with non-numeric type")); - } - } - } else { - switch (to.getType()) { - case INT: - return obj -> ((Number) obj).intValue(); - case LONG: - return obj -> ((Number) obj).longValue(); - case DOUBLE: - return obj -> Double.valueOf(obj.toString()); - case FLOAT: - return obj -> ((Number) obj).floatValue(); - case STRING: - return Object::toString; - default: - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map numeric type with non-numeric type")); - } - } - } - - /** Get the non-null union type of a nullable/optional union field. */ - private static Schema nonNullUnionSchema(Schema schema) throws SchemaValidationException { - List types = schema.getTypes(); - - if (types.size() != 2) { - throw new SchemaValidationException(schema, schema, - new IllegalArgumentException("Types must denote optionals")); - } - - if (types.get(0).getType() == Schema.Type.NULL) { - if (types.get(1).getType() != Schema.Type.NULL) { - return types.get(1); - } else { - throw new SchemaValidationException(schema, schema, - new IllegalArgumentException("Types must denote optionals")); - } - } else if (types.get(1).getType() == Schema.Type.NULL) { - return types.get(0); - } else { - throw new SchemaValidationException(schema, schema, - new IllegalArgumentException("Types must denote optionals.")); - } - } - - /** Map one union to another, or a union to non-union, or non-union to union. */ - private AvroDataMapper mapUnion(Schema from, Schema to, Object defaultVal) - throws SchemaValidationException { - - // Do not create a custom mapper for trivial changes. - if (from.getType() == Schema.Type.UNION && to.getType() == Schema.Type.UNION - && from.getTypes().size() == from.getTypes().size()) { - boolean matches = true; - for (int i = 0; i < from.getTypes().size(); i++) { - Schema.Type fromType = from.getTypes().get(i).getType(); - Schema.Type toType = to.getTypes().get(i).getType(); - - if (fromType != toType || !isPrimitive(fromType)) { - matches = false; - break; - } - } - if (matches) { - return IDENTITY_MAPPER; - } - } - - Schema resolvedFrom = from.getType() == Schema.Type.UNION ? nonNullUnionSchema(from) : from; - - if (from.getType() == Schema.Type.UNION && to.getType() != Schema.Type.UNION) { - if (defaultVal != null) { - final Object actualDefault = getDefaultValue(defaultVal, to); - final AvroDataMapper subMapper = createMapper(resolvedFrom, to, defaultVal); - return obj -> { - if (obj == null) { - return actualDefault; - } else { - return subMapper.convertAvro(obj); - } - }; - } else { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map union to non-union without a default value")); - } - } else { - Schema toNonNull = nonNullUnionSchema(to); - final AvroDataMapper unionMapper = createMapper(resolvedFrom, toNonNull, defaultVal); - return obj -> { - if (obj == null) { - return null; - } else { - return unionMapper.convertAvro(obj); - } - }; - } - } - - /** Map an array to another. */ - private AvroDataMapper mapArray(Schema from, Schema to) - throws SchemaValidationException { - if (to.getType() != Schema.Type.ARRAY) { - throw new SchemaValidationException(to, from, - new IllegalArgumentException("Cannot map array to non-array")); - } - final AvroDataMapper subMapper = createMapper(from.getElementType(), to.getElementType(), - null); - return obj -> { - List array = (List) obj; - List toArray = new ArrayList<>(array.size()); - for (Object val : array) { - toArray.add(subMapper.convertAvro(val)); - } - return toArray; - }; - } - - /** Map a map to another. */ - private AvroDataMapper mapMap(Schema from, Schema to) throws SchemaValidationException { - if (to.getType() != Schema.Type.MAP) { - throw new SchemaValidationException(to, from, - new IllegalArgumentException("Cannot map array to non-array")); - } - final AvroDataMapper subMapper = createMapper(from.getValueType(), to.getValueType(), - null); - return obj -> { - @SuppressWarnings("unchecked") - Map map = (Map) obj; - Map toMap = new HashMap<>(map.size() * 4 / 3 + 1); - for (Map.Entry entry : map.entrySet()) { - toMap.put(entry.getKey().toString(), subMapper.convertAvro(entry.getValue())); - } - return toMap; - }; - } - - private AvroDataMapper mapBytes(Schema from, final Schema to, final Object defaultVal) - throws SchemaValidationException { - if (from.getType() == to.getType() - && (from.getType() == Type.BYTES - || (from.getType() == Type.FIXED && from.getFixedSize() == to.getFixedSize()))) { - return IDENTITY_MAPPER; - } else if (from.getType() == Type.FIXED && to.getType() == Schema.Type.BYTES) { - return object -> ByteBuffer.wrap(((Fixed)object).bytes()); - } else if (from.getType() == Type.BYTES && to.getType() == Type.FIXED) { - if (defaultVal == null) { - throw new SchemaValidationException(to, from, new IllegalArgumentException( - "Cannot map bytes to fixed without default value")); - } - return object -> { - byte[] bytes = ((ByteBuffer) object).array(); - if (bytes.length == to.getFixedSize()) { - return GenericData.get().createFixed(null, bytes, to); - } else { - return GenericData.get().createFixed(null, (byte[]) defaultVal, to); - } - }; - } else if (to.getType() == Type.STRING) { - final Encoder encoder = Base64.getEncoder(); - if (from.getType() == Type.FIXED) { - return object -> encoder.encode(((Fixed) object).bytes()); - } else { - return object -> encoder.encode(((ByteBuffer) object).array()); - } - } else { - throw new SchemaValidationException(to, from, - new IllegalArgumentException( - "Fixed type must be mapped to comparable byte size")); - } - } - - - private AvroDataMapper mapRecord(Schema from, Schema to) - throws SchemaValidationException { - if (to.getType() != Schema.Type.RECORD) { - throw new SchemaValidationException(to, from, - new IllegalArgumentException("From and to schemas must be records.")); - } - List fromFields = from.getFields(); - Schema.Field[] toFields = new Schema.Field[fromFields.size()]; - AvroDataMapper[] fieldMappers = new AvroDataMapper[fromFields.size()]; - - boolean[] filledPositions = new boolean[to.getFields().size()]; - - for (int i = 0; i < fromFields.size(); i++) { - Schema.Field fromField = fromFields.get(i); - Schema.Field toField = to.getField(fromField.name()); - if (toField == null) { - continue; - } - - filledPositions[toField.pos()] = true; - - Schema fromSchema = fromField.schema(); - Schema toSchema = toField.schema(); - - toFields[i] = toField; - fieldMappers[i] = createMapper(fromSchema, toSchema, toField.defaultVal()); - } - - for (int i = 0; i < filledPositions.length; i++) { - if (!filledPositions[i] && to.getFields().get(i).defaultVal() == null) { - throw new SchemaValidationException(to, from, - new IllegalArgumentException("Cannot map to record without default value" - + " for new field " + to.getFields().get(i).name())); - } - } - - return new RecordMapper(to, toFields, fieldMappers); - } - - /** Maps one record to another. */ - private static class RecordMapper implements AvroDataMapper { - private final AvroDataMapper[] fieldMappers; - private final Schema.Field[] toFields; - private final Schema toSchema; - - RecordMapper(Schema toSchema, Schema.Field[] toFields, AvroDataMapper[] fieldMappers) { - this.toSchema = toSchema; - this.fieldMappers = fieldMappers; - this.toFields = toFields; - } - - - @Override - public GenericRecord convertAvro(Object obj) { - GenericRecordBuilder builder = new GenericRecordBuilder(toSchema); - IndexedRecord record = (IndexedRecord) obj; - for (int i = 0; i < toFields.length; i++) { - Schema.Field field = toFields[i]; - if (field == null) { - continue; - } - builder.set(field, fieldMappers[i].convertAvro(record.get(i))); - } - return builder.build(); - } - - @Override - public String toString() { - return "RecordMapper{" - + "fieldMappers=" + Arrays.toString(fieldMappers) - + ", toFields=" + Arrays.toString(toFields) + '}'; - } - } - - private abstract static class StringToNumberMapper implements AvroDataMapper { - private final Object defaultVal; - - StringToNumberMapper(Object defaultVal) { - this.defaultVal = defaultVal; - } - - @Override - public Object convertAvro(Object object) { - try { - return stringToNumber(object.toString()); - } catch (NumberFormatException ex) { - return defaultVal; - } - } - - abstract Number stringToNumber(String toString); - } - - private static boolean isPrimitive(Schema.Type type) { - switch (type) { - case INT: - case LONG: - case BYTES: - case FLOAT: - case DOUBLE: - case NULL: - case BOOLEAN: - case STRING: - return true; - default: - return false; - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt new file mode 100644 index 00000000..16ef050c --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt @@ -0,0 +1,399 @@ +package org.radarbase.producer.rest + +import org.apache.avro.JsonProperties +import org.apache.avro.Schema +import org.apache.avro.SchemaValidationException +import org.apache.avro.generic.* +import org.radarbase.util.Base64Encoder +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.nio.ByteBuffer +import java.util.* + +class AvroDataMapperFactory { + /** + * Create a mapper for data in one Avro schema to that in another Avro schema. + * @param from originating Avro schema + * @param to resulting Avro schema + * @param defaultVal default value as defined in an Avro record field, + * may be null if there is no default value. + * @return Avro data mapper + * @throws SchemaValidationException if the given schemas are incompatible. + */ + @Throws(SchemaValidationException::class) + fun createMapper(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + if (from == to) { + logger.debug("Using identity schema mapping from {} to {}", from, to) + return IDENTITY_MAPPER + } + logger.debug("Computing custom mapping from {} to {}", from, to) + try { + if (to.type == Schema.Type.UNION || from.type == Schema.Type.UNION) { + return mapUnion(from, to, defaultVal) + } + if (to.type == Schema.Type.ENUM || from.type == Schema.Type.ENUM) { + return mapEnum(from, to, defaultVal) + } + when (to.type) { + Schema.Type.INT, Schema.Type.LONG, Schema.Type.DOUBLE, Schema.Type.FLOAT -> + return mapNumber(from, to, defaultVal) + else -> {} + } + return when (from.type) { + Schema.Type.RECORD -> mapRecord(from, to) + Schema.Type.ARRAY -> mapArray(from, to) + Schema.Type.MAP -> mapMap(from, to) + Schema.Type.FIXED, Schema.Type.BYTES -> mapBytes(from, to, defaultVal) + Schema.Type.INT, Schema.Type.LONG, Schema.Type.DOUBLE, Schema.Type.FLOAT -> + mapNumber(from, to, defaultVal) + to.type -> IDENTITY_MAPPER + else -> throw validationException(to, from, "Schema types of from and to don't match") + } + } catch (ex: SchemaValidationException) { + return if (defaultVal != null) { + if (defaultVal === JsonProperties.NULL_VALUE) { + AvroDataMapper { null } + } else { + AvroDataMapper { defaultVal } + } + } else { + throw ex + } + } + } + + /** Map one union to another, or a union to non-union, or non-union to union. */ + @Throws(SchemaValidationException::class) + private fun mapUnion(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + // Do not create a custom mapper for trivial changes. + if ( + from.type == Schema.Type.UNION && + to.type == Schema.Type.UNION && + from.types.size == from.types.size + ) { + val matches = from.types.indices.all { i -> + val fromType = from.types[i].type + val toType = to.types[i].type + fromType == toType && fromType.isPrimitive() + } + if (matches) { + return IDENTITY_MAPPER + } + } + val resolvedFrom = if (from.type == Schema.Type.UNION) { + nonNullUnionSchema(from) + } else from + + return if (from.type == Schema.Type.UNION && to.type != Schema.Type.UNION) { + defaultVal ?: throw validationException(to, from, "Cannot map union to non-union without a default value") + val actualDefault = getDefaultValue(defaultVal, to) + val subMapper = createMapper(resolvedFrom, to, defaultVal) + AvroDataMapper { obj -> + if (obj == null) { + actualDefault + } else { + subMapper.convertAvro(obj) + } + } + } else { + val toNonNull = nonNullUnionSchema(to) + val unionMapper = createMapper(resolvedFrom, toNonNull, defaultVal) + AvroDataMapper { obj -> + obj ?: return@AvroDataMapper null + unionMapper.convertAvro(obj) + } + } + } + + /** Map an array to another. */ + @Throws(SchemaValidationException::class) + private fun mapArray(from: Schema, to: Schema): AvroDataMapper { + if (to.type != Schema.Type.ARRAY) { + throw validationException(to, from, "Cannot map array to non-array") + } + val subMapper = createMapper(from.elementType, to.elementType, null) + return AvroDataMapper { obj -> + obj.asAvroType>(from, to).map { subMapper.convertAvro(it) } + } + } + + /** Map a map to another. */ + @Throws(SchemaValidationException::class) + private fun mapMap(from: Schema, to: Schema): AvroDataMapper { + if (to.type != Schema.Type.MAP) { + throw validationException(to, from, "Cannot map map to non-map") + } + val subMapper = createMapper(from.valueType, to.valueType, null) + return AvroDataMapper { obj -> + buildMap { + obj.asAvroType>(from, to).forEach { (k, v) -> + put(k.toString(), subMapper.convertAvro(v)) + } + } + } + } + + @Throws(SchemaValidationException::class) + private fun mapBytes(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + return if (from.type == Schema.Type.BYTES && to.type == Schema.Type.BYTES) { + IDENTITY_MAPPER + } else if (from.type == Schema.Type.FIXED && to.type == Schema.Type.FIXED + && from.fixedSize == to.fixedSize + ) { + IDENTITY_MAPPER + } else if (from.type == Schema.Type.FIXED && to.type == Schema.Type.BYTES) { + AvroDataMapper { `object` -> + ByteBuffer.wrap(`object`.asAvroType(from, to).bytes()) + } + } else if (from.type == Schema.Type.BYTES && to.type == Schema.Type.FIXED) { + defaultVal ?: throw validationException(to, from, "Cannot map bytes to fixed without default value") + + AvroDataMapper { `object`: Any? -> + val bytes = `object`.asAvroType(from, to).array() + val value = if (bytes.size == to.fixedSize) { + bytes + } else { + defaultVal as? ByteArray + } + GenericData.get().createFixed(null, value, to) + } + } else if (to.type == Schema.Type.STRING) { + val encoder = Base64Encoder + if (from.type == Schema.Type.FIXED) { + AvroDataMapper { `object` -> + encoder.encode(`object`.asAvroType(from, to).bytes()) + } + } else { + AvroDataMapper { `object` -> + encoder.encode(`object`.asAvroType(from, to).array()) + } + } + } else { + throw validationException(to, from, "Fixed type must be mapped to comparable byte size") + } + } + + @Throws(SchemaValidationException::class) + private fun mapRecord(from: Schema, to: Schema): AvroDataMapper { + if (to.type != Schema.Type.RECORD) { + throw validationException(to, from, "From and to schemas must be records.") + } + val fromFields = from.fields + val toFields = arrayOfNulls( + fromFields.size + ) + val fieldMappers = arrayOfNulls( + fromFields.size + ) + val filledPositions = BooleanArray(to.fields.size) + for (i in fromFields.indices) { + val fromField = fromFields[i] + val toField = to.getField(fromField.name()) ?: continue + filledPositions[toField.pos()] = true + toFields[i] = toField + fieldMappers[i] = createMapper( + fromField.schema(), + toField.schema(), + toField.defaultVal() + ) + } + filledPositions.forEachIndexed { i, isFilled -> + if (!isFilled && to.fields[i].defaultVal() == null) { + throw validationException(to, from, + "Cannot map to record without default value for new field ${to.fields[i].name()}" + ) + } + } + return RecordMapper(to, toFields, fieldMappers) + } + + /** Maps one record to another. */ + private class RecordMapper constructor( + private val toSchema: Schema, + private val toFields: Array, + private val fieldMappers: Array + ) : AvroDataMapper { + override fun convertAvro(`object`: Any?): GenericRecord { + val builder = GenericRecordBuilder(toSchema) + val record = `object`.asAvroType(toSchema, toSchema) + for (i in toFields.indices) { + val field = toFields[i] ?: continue + val mapper = fieldMappers[i] ?: continue + builder[field] = mapper.convertAvro(record[i]) + } + return builder.build() + } + + override fun toString(): String { + return ("RecordMapper{" + + "fieldMappers=" + fieldMappers.contentToString() + + ", toFields=" + toFields.contentToString() + '}') + } + } + + private class StringToNumberMapper( + private val defaultVal: Any?, + private val mapping: (String) -> Number, + ) : + AvroDataMapper { + override fun convertAvro(`object`: Any?): Any? { + `object` ?: return defaultVal + return try { + mapping(`object`.toString()) + } catch (ex: NumberFormatException) { + defaultVal + } + } + } + + companion object { + private val logger: Logger = LoggerFactory.getLogger( + AvroDataMapperFactory::class.java + ) + val IDENTITY_MAPPER: AvroDataMapper = object : AvroDataMapper { + override fun convertAvro(`object`: Any?): Any? = `object` + + override fun toString(): String = "Identity" + } + val instance: AvroDataMapperFactory by lazy { AvroDataMapperFactory() } + + private inline fun Any?.asAvroType(from: Schema, to: Schema): T { + if (this !is T) { + throw validationException( + to, from, "${to.type} type cannot be mapped from ${this?.javaClass?.name} Java type." + ) + } + return this + } + + private val PRIMITIVE_TYPES = EnumSet.of( + Schema.Type.INT, + Schema.Type.LONG, + Schema.Type.BYTES, + Schema.Type.FLOAT, + Schema.Type.DOUBLE, + Schema.Type.NULL, + Schema.Type.BOOLEAN, + Schema.Type.STRING, + ) + + /** Map one enum to another or to String. */ + @Throws(SchemaValidationException::class) + private fun mapEnum(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + return if (to.type == Schema.Type.ENUM) { + var containsAll = true + if (from.type == Schema.Type.ENUM) { + for (s in from.enumSymbols) { + if (!to.hasEnumSymbol(s)) { + containsAll = false + break + } + } + } else if (from.type == Schema.Type.STRING) { + containsAll = false + } else { + throw validationException(to, from, "Cannot map enum from non-string or enum type") + } + if (containsAll) { + AvroDataMapper { obj -> GenericData.EnumSymbol(to, obj.toString()) } + } else { + var defaultString = defaultVal as? String + if (defaultString == null) { + if (to.hasEnumSymbol("UNKNOWN")) { + defaultString = "UNKNOWN" + } else { + throw validationException(to, from, + "Cannot map enum symbols without default value" + ) + } + } + val symbol: GenericEnumSymbol<*> = GenericData.EnumSymbol(to, defaultString) + AvroDataMapper { obj: Any? -> + val value = obj.toString() + if (to.hasEnumSymbol(value)) { + GenericData.EnumSymbol(to, value) + } else { + symbol + } + } + } + } else if (from.type == Schema.Type.ENUM && to.type == Schema.Type.STRING) { + AvroDataMapper { it.toString() } + } else { + throw validationException(to, from, "Cannot map unknown type with enum.") + } + } + + /** Get the default value as a Generic type. */ + private fun getDefaultValue(defaultVal: Any?, schema: Schema): Any? { + return if (defaultVal == null) { + null + } else if (schema.type == Schema.Type.ENUM) { + GenericData.EnumSymbol(schema, defaultVal) + } else { + defaultVal + } + } + + /** Maps one number type to another or parses/converts to a string. */ + @Throws(SchemaValidationException::class) + private fun mapNumber(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + if (from.type == to.type) { + return IDENTITY_MAPPER + } + return if (from.type == Schema.Type.STRING) { + defaultVal ?: throw validationException(to, from, "Cannot map string to number without default value.") + when (to.type) { + Schema.Type.INT -> StringToNumberMapper(defaultVal, Integer::valueOf) + Schema.Type.LONG -> StringToNumberMapper(defaultVal, String::toLong) + Schema.Type.DOUBLE -> StringToNumberMapper(defaultVal, String::toDouble) + Schema.Type.FLOAT -> StringToNumberMapper(defaultVal, String::toFloat) + else -> throw validationException( + to, from, "Cannot map numeric type with non-numeric type" + ) + } + } else { + when (to.type) { + Schema.Type.INT -> AvroDataMapper { it.asAvroType(from, to).toInt() } + Schema.Type.LONG -> AvroDataMapper { it.asAvroType(from, to).toLong() } + Schema.Type.DOUBLE -> AvroDataMapper { it.toString().toDouble() } + Schema.Type.FLOAT -> AvroDataMapper { it.asAvroType(from, to).toFloat() } + Schema.Type.STRING -> AvroDataMapper { it.toString() } + else -> throw validationException( + to, from, "Cannot map numeric type with non-numeric type" + ) + } + } + } + + /** Get the non-null union type of a nullable/optional union field. */ + @Throws(SchemaValidationException::class) + private fun nonNullUnionSchema(schema: Schema): Schema { + val types = checkNotNull(schema.types) { "Union does not have subtypes" } + if (types.size != 2) { + throw validationException(schema, schema, "Types must denote optionals.") + } + return if (types[0].type == Schema.Type.NULL) { + if (types[1].type != Schema.Type.NULL) { + types[1] + } else { + throw validationException(schema, schema, "Types must denote optionals.") + } + } else if (types[1].type == Schema.Type.NULL) { + types[0] + } else { + throw validationException(schema, schema, "Types must denote optionals.") + } + } + + private fun Schema.Type.isPrimitive(): Boolean = this in PRIMITIVE_TYPES + + internal fun validationException( + from: Schema, + to: Schema, + message: String, + ): SchemaValidationException = SchemaValidationException( + to, from, IllegalArgumentException(message) + ) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java deleted file mode 100644 index 41c56377..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import okio.Buffer; -import okio.BufferedSink; -import org.apache.avro.Schema; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.generic.IndexedRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; -import org.radarbase.data.AvroEncoder.AvroWriter; -import org.radarbase.data.RecordData; -import org.radarbase.data.RemoteSchemaEncoder; -import org.radarbase.topic.AvroTopic; -import org.radarbase.util.Strings; - -/** - * Encodes a record request as binary data, in the form of a RecordSet. - * @param record key type - * @param record value type - */ -public class BinaryRecordRequest implements RecordRequest { - private int keyVersion; - private int valueVersion; - private RecordData records; - private BinaryEncoder binaryEncoder; - private final AvroWriter valueEncoder; - private final int sourceIdPos; - - /** - * Binary record request for given topic. - * @param topic topic to send data for. - * @throws SchemaValidationException if the key schema does not contain a - * {@code sourceId} field. - * @throws IllegalArgumentException if the topic cannot be used to make a AvroWriter. - */ - public BinaryRecordRequest(AvroTopic topic) throws SchemaValidationException { - if (topic.getKeySchema() == null || topic.getKeySchema().getType() != Schema.Type.RECORD) { - Schema keySchema = topic.getKeySchema(); - if (keySchema == null) { - keySchema = Schema.create(Schema.Type.NULL); - } - throw new SchemaValidationException(keySchema, keySchema, - new IllegalArgumentException("Cannot use non-record key schema")); - } - Schema.Field sourceIdField = topic.getKeySchema().getField("sourceId"); - if (sourceIdField == null) { - throw new SchemaValidationException(topic.getKeySchema(), topic.getKeySchema(), - new IllegalArgumentException("Cannot use binary encoder without a source ID.")); - } else { - sourceIdPos = sourceIdField.pos(); - } - valueEncoder = new RemoteSchemaEncoder(true) - .writer(topic.getValueSchema(), topic.getValueClass()); - } - - @Override - public void writeToSink(BufferedSink sink) throws IOException { - writeToSink(sink, Integer.MAX_VALUE); - } - - private void writeToSink(BufferedSink sink, int maxLength) throws IOException { - binaryEncoder = EncoderFactory.get().directBinaryEncoder( - sink.outputStream(), binaryEncoder); - binaryEncoder.startItem(); - binaryEncoder.writeInt(keyVersion); - binaryEncoder.writeInt(valueVersion); - - // do not send project ID; it is encoded in the serialization - binaryEncoder.writeIndex(0); - // do not send user ID; it is encoded in the serialization - binaryEncoder.writeIndex(0); - String sourceId = ((IndexedRecord) records.getKey()).get(sourceIdPos).toString(); - binaryEncoder.writeString(sourceId); - binaryEncoder.writeArrayStart(); - binaryEncoder.setItemCount(records.size()); - - int curLength = 18 + sourceId.length(); - - for (V record : records) { - if (curLength >= maxLength) { - return; - } - binaryEncoder.startItem(); - byte[] valueBytes = valueEncoder.encode(record); - binaryEncoder.writeBytes(valueBytes); - curLength += 4 + valueBytes.length; - } - binaryEncoder.writeArrayEnd(); - binaryEncoder.flush(); - } - - @Override - public void reset() { - records = null; - } - - @Override - public void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSchema, - RecordData records) throws SchemaValidationException { - keyVersion = keySchema.getVersion() == null ? 0 : keySchema.getVersion(); - valueVersion = valueSchema.getVersion() == null ? 0 : valueSchema.getVersion(); - - valueEncoder.setReaderSchema(valueSchema); - - this.records = records; - } - - @Override - public String content(int maxLength) throws IOException { - try (Buffer buffer = new Buffer()) { - writeToSink(buffer, maxLength / 2 - 2); - return "0x" + Strings.bytesToHex( - buffer.readByteArray(Math.min(buffer.size(), maxLength - 2))); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt new file mode 100644 index 00000000..0a16602d --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt @@ -0,0 +1,137 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okio.Buffer +import okio.BufferedSink +import org.apache.avro.Schema +import org.apache.avro.SchemaValidationException +import org.apache.avro.generic.IndexedRecord +import org.apache.avro.io.BinaryEncoder +import org.apache.avro.io.EncoderFactory +import org.radarbase.data.AvroEncoder.AvroWriter +import org.radarbase.data.RecordData +import org.radarbase.data.RemoteSchemaEncoder +import org.radarbase.producer.rest.AvroDataMapperFactory.Companion.validationException +import org.radarbase.topic.AvroTopic +import org.radarbase.util.Strings.toHexString +import java.io.IOException +import java.lang.IllegalArgumentException + +/** + * Encodes a record request as binary data, in the form of a RecordSet. + * @param record key type + * @param record value type + */ +class BinaryRecordRequest(topic: AvroTopic) : RecordRequest { + private var keyVersion = 0 + private var valueVersion = 0 + private var records: RecordData? = null + private var binaryEncoder: BinaryEncoder? = null + private val valueEncoder: AvroWriter + private var sourceIdPos = 0 + + /** + * Binary record request for given topic. + * @param topic topic to send data for. + * @throws SchemaValidationException if the key schema does not contain a + * `sourceId` field. + * @throws IllegalArgumentException if the topic cannot be used to make a AvroWriter. + */ + init { + if (topic.keySchema.type != Schema.Type.RECORD) { + throw validationException( + topic.keySchema, topic.keySchema, + "Cannot use non-record key schema" + ) + } + val sourceIdField = topic.keySchema.getField("sourceId") + sourceIdPos = sourceIdField?.pos() + ?: throw validationException( + topic.keySchema, topic.keySchema, + "Cannot use binary encoder without a source ID." + ) + valueEncoder = RemoteSchemaEncoder(true) + .writer(topic.valueSchema, topic.valueClass) + } + + @Throws(IOException::class) + override fun writeToSink(sink: BufferedSink) { + writeToSink(sink, Int.MAX_VALUE) + } + + @Throws(IOException::class) + private fun writeToSink(sink: BufferedSink, maxLength: Int) { + binaryEncoder = EncoderFactory.get().directBinaryEncoder( + sink.outputStream(), binaryEncoder + ) + binaryEncoder?.writeRecords( + records ?: return, + maxLength + ) + } + + private fun BinaryEncoder.writeRecords(records: RecordData, maxLength: Int) { + startItem() + writeInt(keyVersion) + writeInt(valueVersion) + + // do not send project ID; it is encoded in the serialization + writeIndex(0) + // do not send user ID; it is encoded in the serialization + writeIndex(0) + val sourceId = (records.key as IndexedRecord)[sourceIdPos].toString() + writeString(sourceId) + writeArrayStart() + setItemCount(records.size().toLong()) + var curLength = 18 + sourceId.length + for (record in records) { + if (curLength >= maxLength) { + return + } + startItem() + val valueBytes = valueEncoder.encode(record) + writeBytes(valueBytes) + curLength += 4 + valueBytes.size + } + writeArrayEnd() + flush() + } + + override fun reset() { + records = null + } + + @Throws(SchemaValidationException::class) + override fun prepare( + keySchema: ParsedSchemaMetadata, valueSchema: ParsedSchemaMetadata, + records: RecordData + ) { + keyVersion = if (keySchema.version == null) 0 else keySchema.version + valueVersion = if (valueSchema.version == null) 0 else valueSchema.version + valueEncoder.readerSchema = valueSchema + this.records = records + } + + @Throws(IOException::class) + override fun content(maxLength: Int): String { + Buffer().use { buffer -> + writeToSink(buffer, maxLength / 2 - 2) + val printSize = buffer.size.coerceAtMost((maxLength - 2).toLong()) + return "0x" + buffer.readByteArray(printSize).toHexString() + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java deleted file mode 100644 index cb911f07..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.util.concurrent.TimeUnit; - -/** - * Current connection status of a KafkaSender. After a timeout occurs this will turn to - * disconnected. When the connection is dropped, the associated KafkaSender should set this to - * disconnected, when it successfully connects, it should set it to connected. This class is - * thread-safe. The state transition diagram is CONNECTED to and from DISCONNECTED with - * {@link #didConnect()} and {@link #didDisconnect()}; CONNECTED to and from UNKNOWN with - * {@link #getState()} after a timeout occurs and {@link #didConnect()}; and UNKNOWN to DISCONNECTED - * with {@link #didDisconnect()}. - * - *

A connection state could be shared with multiple HTTP clients if they are talking to the same - * server. - */ -public final class ConnectionState { - - /** State symbols of the connection. */ - public enum State { - CONNECTED, DISCONNECTED, UNKNOWN, UNAUTHORIZED - } - - private long timeout; - private long lastConnection; - private State state; - - /** - * Connection state with given timeout. The state will start as connected. - * @param timeout timeout - * @param unit unit of the timeout - * @throws IllegalArgumentException if the timeout is not strictly positive. - */ - public ConnectionState(long timeout, TimeUnit unit) { - lastConnection = -1L; - state = State.UNKNOWN; - setTimeout(timeout, unit); - } - - /** Current state of the connection. */ - public synchronized State getState() { - if (state == State.CONNECTED && System.currentTimeMillis() - lastConnection >= timeout) { - state = State.UNKNOWN; - } - return state; - } - - /** For a sender to indicate that a connection attempt succeeded. */ - public synchronized void didConnect() { - state = State.CONNECTED; - lastConnection = System.currentTimeMillis(); - } - - /** For a sender to indicate that a connection attempt failed. */ - public synchronized void didDisconnect() { - state = State.DISCONNECTED; - } - - public synchronized void wasUnauthorized() { - state = State.UNAUTHORIZED; - } - - public synchronized void reset() { - state = State.UNKNOWN; - } - - /** - * Set the timeout after which the state will go from CONNECTED to UNKNOWN. - * @param timeout timeout - * @param unit unit of the timeout - * @throws IllegalArgumentException if the timeout is not strictly positive - */ - public synchronized void setTimeout(long timeout, TimeUnit unit) { - if (timeout <= 0) { - throw new IllegalArgumentException("Timeout must be strictly positive"); - } - this.timeout = TimeUnit.MILLISECONDS.convert(timeout, unit); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt new file mode 100644 index 00000000..0f627326 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt @@ -0,0 +1,94 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import java.util.concurrent.TimeUnit + +/** + * Current connection status of a KafkaSender. After a timeout occurs this will turn to + * disconnected. When the connection is dropped, the associated KafkaSender should set this to + * disconnected, when it successfully connects, it should set it to connected. This class is + * thread-safe. The state transition diagram is CONNECTED to and from DISCONNECTED with + * [.didConnect] and [.didDisconnect]; CONNECTED to and from UNKNOWN with + * [.getState] after a timeout occurs and [.didConnect]; and UNKNOWN to DISCONNECTED + * with [.didDisconnect]. + * + * + * A connection state could be shared with multiple HTTP clients if they are talking to the same + * server. + * + * @param timeout timeout + * @param unit unit of the timeout + * @throws IllegalArgumentException if the timeout is not strictly positive. + */ +class ConnectionState( + timeout: Long, + unit: TimeUnit, +) { + /** State symbols of the connection. */ + enum class State { + CONNECTED, DISCONNECTED, UNKNOWN, UNAUTHORIZED + } + + private var timeout: Long = TimeUnit.MILLISECONDS.convert(timeout, unit) + private var lastConnection: Long = -1L + + /** Current state of the connection. */ + @get:Synchronized + var state: State = State.UNKNOWN + get() { + if (field == State.CONNECTED && System.currentTimeMillis() - lastConnection >= timeout) { + field = State.UNKNOWN + } + return field + } + private set + + /** For a sender to indicate that a connection attempt succeeded. */ + @Synchronized + fun didConnect() { + state = State.CONNECTED + lastConnection = System.currentTimeMillis() + } + + /** For a sender to indicate that a connection attempt failed. */ + @Synchronized + fun didDisconnect() { + state = State.DISCONNECTED + } + + @Synchronized + fun wasUnauthorized() { + state = State.UNAUTHORIZED + } + + @Synchronized + fun reset() { + state = State.UNKNOWN + } + + /** + * Set the timeout after which the state will go from CONNECTED to UNKNOWN. + * @param timeout timeout + * @param unit unit of the timeout + * @throws IllegalArgumentException if the timeout is not strictly positive + */ + @Synchronized + fun setTimeout(timeout: Long, unit: TimeUnit) { + require(timeout > 0) { "Timeout must be strictly positive" } + this.timeout = TimeUnit.MILLISECONDS.convert(timeout, unit) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java deleted file mode 100644 index 6ce5f98b..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import okhttp3.Interceptor; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okio.BufferedSink; -import okio.GzipSink; -import okio.Okio; - -/** This interceptor compresses the HTTP request body. Many webservers can't handle this! */ -public class GzipRequestInterceptor implements Interceptor { - @Override - public Response intercept(Interceptor.Chain chain) throws IOException { - Request originalRequest = chain.request(); - if (originalRequest.body() == null || originalRequest.header("Content-Encoding") != null) { - return chain.proceed(originalRequest); - } - - Request compressedRequest = originalRequest.newBuilder() - .header("Content-Encoding", "gzip") - .method(originalRequest.method(), gzip(originalRequest.body())) - .build(); - return chain.proceed(compressedRequest); - } - - private RequestBody gzip(final RequestBody body) { - return new RequestBody() { - @Override - public MediaType contentType() { - return body.contentType(); - } - - @Override - public long contentLength() { - return -1; // We don't know the compressed length in advance! - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - try (BufferedSink gzipSink = Okio.buffer(new GzipSink(sink))) { - body.writeTo(gzipSink); - } - } - }; - } - - @Override - public int hashCode() { - return 1; - } - - @Override - public boolean equals(Object obj) { - return this == obj || obj != null && getClass() == obj.getClass(); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt new file mode 100644 index 00000000..2be26e8d --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.* +import okio.BufferedSink +import okio.GzipSink +import okio.buffer +import java.io.IOException + +/** This interceptor compresses the HTTP request body. Many webservers can't handle this! */ +class GzipRequestInterceptor : Interceptor { + @Throws(IOException::class) + override fun intercept(chain: Interceptor.Chain): Response { + val originalRequest: Request = chain.request() + if (originalRequest.body == null || originalRequest.header("Content-Encoding") != null) { + return chain.proceed(originalRequest) + } + + return chain.proceed( + originalRequest.newBuilder() + .header("Content-Encoding", "gzip") + .method(originalRequest.method, gzip(originalRequest.body)) + .build() + ) + } + + private fun gzip(body: RequestBody?): RequestBody { + return object : RequestBody() { + override fun contentType(): MediaType? = body?.contentType() + + override fun contentLength(): Long { + return -1 // We don't know the compressed length in advance! + } + + @Throws(IOException::class) + override fun writeTo(sink: BufferedSink) { + GzipSink(sink).buffer().use { gzipSink -> body?.writeTo(gzipSink) } + } + } + } + + override fun hashCode(): Int = 1 + + override fun equals(other: Any?): Boolean { + return this === other || other != null && javaClass == other.javaClass + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java deleted file mode 100644 index 2ae287ce..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import okio.Buffer; -import okio.BufferedSink; -import org.apache.avro.SchemaValidationException; -import org.json.JSONObject; -import org.radarbase.data.AvroEncoder.AvroWriter; -import org.radarbase.data.RecordData; -import org.radarbase.data.RemoteSchemaEncoder; -import org.radarbase.topic.AvroTopic; -import org.radarbase.util.Strings; - -/** - * Request data to submit records to the Kafka REST proxy. - */ -public class JsonRecordRequest implements RecordRequest { - public static final byte[] KEY_SCHEMA_ID = Strings.utf8("\"key_schema_id\":"); - public static final byte[] VALUE_SCHEMA_ID = Strings.utf8(",\"value_schema_id\":"); - public static final byte[] RECORDS = Strings.utf8(",\"records\":["); - public static final byte[] KEY = Strings.utf8("{\"key\":"); - public static final byte[] VALUE = Strings.utf8(",\"value\":"); - public static final byte[] END = Strings.utf8("]}"); - - private final AvroWriter keyEncoder; - private final AvroWriter valueEncoder; - - private RecordData records; - - /** - * Generate a record request for given topic. - * @param topic topic to use. - * @throws IllegalStateException if key or value encoders could not be made. - */ - public JsonRecordRequest(AvroTopic topic) { - RemoteSchemaEncoder schemaEncoder = new RemoteSchemaEncoder(false); - - this.keyEncoder = schemaEncoder.writer(topic.getKeySchema(), topic.getKeyClass()); - this.valueEncoder = schemaEncoder.writer(topic.getValueSchema(), topic.getValueClass()); - } - - /** - * Writes the current topic to a stream. This implementation does not use any JSON writers to - * write the data, but writes it directly to a stream. {@link JSONObject#quote(String)} - * is used to get the correct formatting. This makes the method as lean as possible. - * @param sink buffered sink to write to. - * @throws IOException if a superimposing stream could not be created - */ - @Override - public void writeToSink(BufferedSink sink) throws IOException { - writeToSink(sink, Integer.MAX_VALUE); - } - - private void writeToSink(BufferedSink sink, int maxLength) throws IOException { - sink.writeByte('{'); - sink.write(KEY_SCHEMA_ID); - sink.write(Strings.utf8(String.valueOf(keyEncoder.getReaderSchema().getId()))); - sink.write(VALUE_SCHEMA_ID); - sink.write(Strings.utf8(String.valueOf(valueEncoder.getReaderSchema().getId()))); - - sink.write(RECORDS); - - byte[] key = keyEncoder.encode(records.getKey()); - - int curLength = KEY_SCHEMA_ID.length + VALUE_SCHEMA_ID.length + 7; - - boolean first = true; - for (V record : records) { - if (curLength >= maxLength) { - return; - } - if (first) { - first = false; - } else { - sink.writeByte(','); - } - sink.write(KEY); - sink.write(key); - - sink.write(VALUE); - byte[] valueBytes = valueEncoder.encode(record); - sink.write(valueBytes); - sink.writeByte('}'); - curLength += 2 + key.length + KEY.length + VALUE.length + valueBytes.length; - } - sink.write(END); - } - - @Override - public void reset() { - records = null; - } - - @Override - public void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSchema, - RecordData records) throws SchemaValidationException { - keyEncoder.setReaderSchema(keySchema); - valueEncoder.setReaderSchema(valueSchema); - this.records = records; - } - - @Override - public String content(int maxLength) throws IOException { - try (Buffer buffer = new Buffer()) { - writeToSink(buffer, maxLength); - return buffer.readString(Math.min(buffer.size(), maxLength), StandardCharsets.UTF_8); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt new file mode 100644 index 00000000..a47f95aa --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt @@ -0,0 +1,131 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okio.Buffer +import okio.BufferedSink +import org.apache.avro.SchemaValidationException +import org.radarbase.data.AvroEncoder.AvroWriter +import org.radarbase.data.RecordData +import org.radarbase.data.RemoteSchemaEncoder +import org.radarbase.topic.AvroTopic +import java.io.IOException +import java.nio.charset.StandardCharsets + +/** + * Request data to submit records to the Kafka REST proxy. + */ +class JsonRecordRequest(topic: AvroTopic) : RecordRequest { + private val keyEncoder: AvroWriter + private val valueEncoder: AvroWriter + private var records: RecordData? = null + + /** + * Generate a record request for given topic. + * @param topic topic to use. + * @throws IllegalStateException if key or value encoders could not be made. + */ + init { + val schemaEncoder = RemoteSchemaEncoder(false) + keyEncoder = schemaEncoder.writer(topic.keySchema, topic.keyClass) + valueEncoder = schemaEncoder.writer(topic.valueSchema, topic.valueClass) + } + + /** + * Writes the current topic to a stream. This implementation does not use any JSON writers to + * write the data, but writes it directly to a stream. [JSONObject.quote] + * is used to get the correct formatting. This makes the method as lean as possible. + * @param sink buffered sink to write to. + * @throws IOException if a superimposing stream could not be created + */ + @Throws(IOException::class) + override fun writeToSink(sink: BufferedSink) { + writeToSink(sink, Int.MAX_VALUE) + } + + @Throws(IOException::class) + private fun writeToSink(sink: BufferedSink, maxLength: Int) { + val keySchema = checkNotNull(keyEncoder.readerSchema) { + "Record request has not been prepared with the proper reader schemas" + } + val valueSchema = checkNotNull(valueEncoder.readerSchema) { + "Record request has not been prepared with the proper reader schemas" + } + + sink.writeByte('{'.code) + sink.write(KEY_SCHEMA_ID) + sink.write(keySchema.id.toString().toByteArray()) + sink.write(VALUE_SCHEMA_ID) + sink.write(valueSchema.id.toString().toByteArray()) + sink.write(RECORDS) + val key = keyEncoder.encode(records!!.key) + var curLength = KEY_SCHEMA_ID.size + VALUE_SCHEMA_ID.size + 7 + var first = true + for (record in records!!) { + if (curLength >= maxLength) { + return + } + if (first) { + first = false + } else { + sink.writeByte(','.code) + } + sink.write(KEY) + sink.write(key) + sink.write(VALUE) + val valueBytes = valueEncoder.encode(record) + sink.write(valueBytes) + sink.writeByte('}'.code) + curLength += 2 + key.size + KEY.size + VALUE.size + valueBytes.size + } + sink.write(END) + } + + override fun reset() { + records = null + } + + @Throws(SchemaValidationException::class) + override fun prepare( + keySchema: ParsedSchemaMetadata, + valueSchema: ParsedSchemaMetadata, + records: RecordData + ) { + keyEncoder.readerSchema = keySchema + valueEncoder.readerSchema = valueSchema + this.records = records + } + + @Throws(IOException::class) + override fun content(maxLength: Int): String { + Buffer().use { buffer -> + writeToSink(buffer, maxLength) + return buffer.readString( + buffer.size.coerceAtMost(maxLength.toLong()), + StandardCharsets.UTF_8 + ) + } + } + + companion object { + val KEY_SCHEMA_ID = "\"key_schema_id\":".toByteArray() + val VALUE_SCHEMA_ID = ",\"value_schema_id\":".toByteArray() + val RECORDS = ",\"records\":[".toByteArray() + val KEY = "{\"key\":".toByteArray() + val VALUE = ",\"value\":".toByteArray() + val END = "]}".toByteArray() + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java b/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java deleted file mode 100644 index d0009e72..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import org.apache.avro.Schema; - -/** - * Parsed schema metadata from a Schema Registry. - */ -public class ParsedSchemaMetadata { - private final Integer version; - private Integer id; - private final Schema schema; - - /** - * Schema metadata. - * @param id schema ID, may be null. - * @param version schema version, may be null. - * @param schema parsed schema. - */ - public ParsedSchemaMetadata(Integer id, Integer version, Schema schema) { - this.id = id; - this.version = version; - this.schema = schema; - } - - public Integer getId() { - return id; - } - - public Schema getSchema() { - return schema; - } - - public Integer getVersion() { - return version; - } - - public void setId(Integer id) { - this.id = id; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt new file mode 100644 index 00000000..1b5808ba --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import org.apache.avro.Schema + +/** + * Parsed schema metadata from a Schema Registry. + */ +data class ParsedSchemaMetadata +/** + * Schema metadata. + * @param id schema ID, may be null. + * @param version schema version, may be null. + * @param schema parsed schema. + */( + val id: Int, + val version: Int?, + val schema: Schema, +) diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt similarity index 57% rename from radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.java rename to radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt index 5c737f35..048a15ce 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.java +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt @@ -13,32 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.radarbase.producer.rest -package org.radarbase.producer.rest; - -import java.io.IOException; -import okio.BufferedSink; -import org.apache.avro.SchemaValidationException; -import org.radarbase.data.RecordData; +import okio.BufferedSink +import org.apache.avro.SchemaValidationException +import org.radarbase.data.RecordData +import java.io.IOException /** - * Record request contents. Before {@link #writeToSink(BufferedSink)} is called, first - * {@link #prepare(ParsedSchemaMetadata, ParsedSchemaMetadata, RecordData)} should be called. This + * Record request contents. Before [.writeToSink] is called, first + * [.prepare] should be called. This * class may be reused by calling prepare and reset alternatively. * * @param record key type. * @param record content type. - */ -public interface RecordRequest { - /** Write the current records to a stream as a request. */ - void writeToSink(BufferedSink sink) throws IOException; + */ +interface RecordRequest { + /** Write the current records to a stream as a request. */ + @Throws(IOException::class) + fun writeToSink(sink: BufferedSink) - /** Reset the contents. This may free up some memory because the recordrequest may be stored. */ - void reset(); + /** Reset the contents. This may free up some memory because the recordrequest may be stored. */ + fun reset() - /** Set the records to be sent. */ - void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSchema, - RecordData records) throws IOException, SchemaValidationException; + /** Set the records to be sent. */ + @Throws(IOException::class, SchemaValidationException::class) + fun prepare( + keySchema: ParsedSchemaMetadata, valueSchema: ParsedSchemaMetadata, + records: RecordData + ) /** * Return the content of the record as a string. To avoid dual reading of data for RecordData @@ -47,5 +50,6 @@ void prepare(ParsedSchemaMetadata keySchema, ParsedSchemaMetadata valueSchema, * @return the content. * @throws IOException if the content cannot be written. */ - String content(int maxLength) throws IOException; + @Throws(IOException::class) + fun content(maxLength: Int): String } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java deleted file mode 100644 index df1cf8f6..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.net.MalformedURLException; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.X509TrustManager; -import okhttp3.Callback; -import okhttp3.Headers; -import okhttp3.HttpUrl; -import okhttp3.Interceptor; -import okhttp3.OkHttpClient; -import okhttp3.Protocol; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; -import org.radarbase.config.ServerConfig; -import org.radarbase.util.RestUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** REST client using OkHttp3. This class is not thread-safe. */ -public class RestClient { - private static final Logger logger = LoggerFactory.getLogger(RestClient.class); - - public static final long DEFAULT_TIMEOUT = 30; - private static WeakReference globalHttpClient = new WeakReference<>(null); - - private final ServerConfig server; - private final OkHttpClient httpClient; - private final Headers headers; - - private RestClient(Builder builder) { - this.server = Objects.requireNonNull(builder.serverConfig); - this.httpClient = builder.client.build(); - this.headers = builder.requestHeaders; - } - - /** OkHttp client. */ - public OkHttpClient getHttpClient() { - return httpClient; - } - - /** Configured connection timeout in seconds. */ - public long getTimeout() { - return httpClient.connectTimeoutMillis() / 1000; - } - - /** Configured server. */ - public ServerConfig getServer() { - return server; - } - - /** - * Make a blocking request. - * @param request request, possibly built with {@link #requestBuilder(String)} - * @return response to the request - * @throws IOException if the request fails - * @throws NullPointerException if the request is null - */ - public Response request(Request request) throws IOException { - Objects.requireNonNull(request); - return httpClient.newCall(request).execute(); - } - - /** - * Make an asynchronous request. - * @param request request, possibly built with {@link #requestBuilder(String)} - * @param callback callback to activate once the request is done. - */ - public void request(Request request, Callback callback) { - Objects.requireNonNull(request); - Objects.requireNonNull(callback); - httpClient.newCall(request).enqueue(callback); - } - - /** - * Make a request to given relative path. This does not set any request properties except the - * URL. - * @param relativePath relative path to request - * @return response to the request - * @throws IOException if the path is invalid or the request failed. - */ - public Response request(String relativePath) throws IOException { - return request(requestBuilder(relativePath).build()); - } - - /** - * Make a blocking request and return the body. - * @param request request to make. - * @return response body string. - * @throws RestException if no body was returned or an HTTP status code indicating error was - * returned. - * @throws IOException if the request cannot be completed or the response cannot be read. - * - */ - public String requestString(Request request) throws IOException { - try (Response response = request(request)) { - String bodyString = responseBody(response); - - if (!response.isSuccessful() || bodyString == null) { - throw new RestException(response.code(), bodyString); - } - - return bodyString; - } - } - - /** - * Create a OkHttp3 request builder with {@link Request.Builder#url(HttpUrl)} set. - * Call{@link Request.Builder#build()} to make the actual request with - * {@link #request(Request)}. - * - * @param relativePath relative path from the server serverConfig - * @return request builder. - * @throws MalformedURLException if the path not valid - */ - public Request.Builder requestBuilder(String relativePath) throws MalformedURLException { - return new Request.Builder().url(getRelativeUrl(relativePath)).headers(headers); - } - - /** - * Get a URL relative to the configured server. - * @param path relative path - * @return URL - * @throws MalformedURLException if the path is malformed - */ - public HttpUrl getRelativeUrl(String path) throws MalformedURLException { - String strippedPath = path; - while (!strippedPath.isEmpty() && strippedPath.charAt(0) == '/') { - strippedPath = strippedPath.substring(1); - } - HttpUrl.Builder builder = getServer().getHttpUrl().newBuilder(strippedPath); - if (builder == null) { - throw new MalformedURLException(); - } - return builder.build(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - RestClient that = (RestClient) o; - - return this.server.equals(that.server) && this.httpClient.equals(that.httpClient); - } - - @Override - public int hashCode() { - return Objects.hash(server, httpClient); - } - - @Override - public String toString() { - return "RestClient{serverConfig=" + server + ", httpClient=" + httpClient + '}'; - } - - /** Get the response body of a response as a String. - * Will return null if the response body is null. - * @param response call response - * @return body contents as a String. - * @throws IOException if the body could not be read as a String. - */ - public static String responseBody(Response response) throws IOException { - try (ResponseBody body = response.body()) { - if (body == null) { - return null; - } - return body.string(); - } - } - - /** Create a new builder with the settings of the current client. */ - public Builder newBuilder() { - return new Builder(httpClient) - .server(server); - } - - /** Builder. */ - public static class Builder { - private ServerConfig serverConfig; - private final OkHttpClient.Builder client; - private Headers requestHeaders = Headers.of(); - - public Builder(OkHttpClient client) { - this(client.newBuilder()); - } - - public Builder(OkHttpClient.Builder client) { - this.client = client; - } - - /** Server configuration. */ - public Builder server(ServerConfig config) { - this.serverConfig = Objects.requireNonNull(config); - - if (config.isUnsafe()) { - this.client.sslSocketFactory(RestUtils.UNSAFE_SSL_FACTORY, - (X509TrustManager) RestUtils.UNSAFE_TRUST_MANAGER[0]); - this.client.hostnameVerifier(RestUtils.UNSAFE_HOSTNAME_VERIFIER); - } else { - X509TrustManager trustManager = RestUtils.systemDefaultTrustManager(); - SSLSocketFactory socketFactory = RestUtils.systemDefaultSslSocketFactory( - trustManager); - this.client.sslSocketFactory(socketFactory, trustManager); - this.client.hostnameVerifier(RestUtils.DEFAULT_HOSTNAME_VERIFIER); - } - return this; - } - - /** Allowed protocols. */ - public Builder protocols(List protocols) { - this.client.protocols(protocols); - return this; - } - - /** Builder to extend the HTTP client with. */ - public OkHttpClient.Builder httpClientBuilder() { - return client; - } - - public Builder headers(Headers headers) { - this.requestHeaders = headers; - return this; - } - - /** Whether to enable GZIP compression. */ - public Builder gzipCompression(boolean compression) { - GzipRequestInterceptor gzip = null; - for (Interceptor interceptor : client.interceptors()) { - if (interceptor instanceof GzipRequestInterceptor) { - gzip = (GzipRequestInterceptor) interceptor; - break; - } - } - if (compression && gzip == null) { - logger.debug("Enabling GZIP compression"); - client.addInterceptor(new GzipRequestInterceptor()); - } else if (!compression && gzip != null) { - logger.debug("Disabling GZIP compression"); - client.interceptors().remove(gzip); - } - return this; - } - - /** Timeouts for connecting, reading and writing. */ - public Builder timeout(long timeout, TimeUnit unit) { - client.connectTimeout(timeout, unit) - .readTimeout(timeout, unit) - .writeTimeout(timeout, unit); - return this; - } - - /** Build a new RestClient. */ - public RestClient build() { - return new RestClient(this); - } - } - - /** Create a builder with a global shared OkHttpClient. */ - public static synchronized RestClient.Builder global() { - OkHttpClient client = globalHttpClient.get(); - if (client == null) { - client = createDefaultClient().build(); - globalHttpClient = new WeakReference<>(client); - } - return new RestClient.Builder(client); - } - - /** Create a builder with a new OkHttpClient using default settings. */ - public static synchronized RestClient.Builder newClient() { - return new RestClient.Builder(createDefaultClient()); - } - - /** - * Create a new OkHttpClient. The timeouts are set to the default. - * @return new OkHttpClient. - */ - private static OkHttpClient.Builder createDefaultClient() { - return new OkHttpClient.Builder() - .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) - .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) - .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt new file mode 100644 index 00000000..1997c344 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt @@ -0,0 +1,281 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.* +import okhttp3.Headers.Companion.headersOf +import org.radarbase.config.ServerConfig +import org.radarbase.util.RestUtils +import org.slf4j.LoggerFactory +import java.io.IOException +import java.lang.ref.WeakReference +import java.net.MalformedURLException +import java.util.* +import java.util.concurrent.TimeUnit +import javax.net.ssl.X509TrustManager + +/** REST client using OkHttp3. This class is not thread-safe. */ +class RestClient private constructor(builder: Builder) { + /** Configured server. */ + val server: ServerConfig = requireNotNull(builder.server) { "Missing server configuration" } + /** OkHttp client. */ + val httpClient: OkHttpClient = builder.httpClientBuilder.build() + private val headers: Headers = builder.headers + private val relativeUrl = server.httpUrl + + /** Configured connection timeout in seconds. */ + val timeout: Long + get() = (httpClient.connectTimeoutMillis / 1000).toLong() + + /** + * Make a blocking request. + * @param request request, possibly built with [.requestBuilder] + * @return response to the request + * @throws IOException if the request fails + * @throws NullPointerException if the request is null + */ + @Throws(IOException::class) + fun request(request: Request): Response { + return httpClient.newCall(request).execute() + } + + /** + * Make a blocking request. + * @param request request, possibly built with [.requestBuilder] + * @return response to the request + * @throws IOException if the request fails + * @throws NullPointerException if the request is null + */ + @Throws(IOException::class) + fun request(builder: Request.Builder.() -> Unit): Response = httpClient.newCall( + Request.Builder().apply { + headers(headers) + builder() + }.build() + ).execute() + + /** + * Make a blocking request. + * @param request request, possibly built with [.requestBuilder] + * @return response to the request + * @throws IOException if the request fails + * @throws NullPointerException if the request is null + */ + @Throws(IOException::class) + inline fun request( + relativePath: String, + crossinline builder: Request.Builder.() -> Unit, + ): Response = request { + url(relativeUrl(relativePath)) + builder() + } + + /** + * Make an asynchronous request. + * @param request request, possibly built with [.requestBuilder] + * @param callback callback to activate once the request is done. + */ + fun request(request: Request, callback: Callback) = + httpClient.newCall(request).enqueue(callback) + + /** + * Make a request to given relative path. This does not set any request properties except the + * URL. + * @param relativePath relative path to request + * @return response to the request + * @throws IOException if the path is invalid or the request failed. + */ + @Throws(IOException::class) + fun request(relativePath: String): Response = request(buildRequest(relativePath)) + + /** + * Make a blocking request and return the body. + * @param request request to make. + * @return response body string. + * @throws RestException if no body was returned or an HTTP status code indicating error was + * returned. + * @throws IOException if the request cannot be completed or the response cannot be read. + */ + @Throws(IOException::class) + fun requestString(request: Request): String { + request(request).use { response -> + val bodyString = response.bodyString() + if (!response.isSuccessful || bodyString == null) { + throw RestException(response.code, bodyString) + } + return bodyString + } + } + + /** + * Create a OkHttp3 request builder with [Request.Builder.url] set. + * Call[Request.Builder.build] to make the actual request with + * [.request]. + * + * @param relativePath relative path from the server serverConfig + * @return request builder. + * @throws MalformedURLException if the path not valid + */ + @Throws(MalformedURLException::class) + fun requestBuilder(relativePath: String): Request.Builder = Request.Builder() + .url(relativeUrl(relativePath)) + .headers(headers) + + fun buildRequest( + relativePath: String, + builder: Request.Builder.() -> Unit = {}, + ): Request = Request.Builder().apply { + url(relativeUrl(relativePath)) + headers(headers) + builder() + }.build() + + /** + * Get a URL relative to the configured server. + * @param path relative path + * @return URL + * @throws MalformedURLException if the path is malformed + */ + @Throws(MalformedURLException::class) + fun relativeUrl(path: String): HttpUrl { + val urlBuilder = relativeUrl.newBuilder(path.trimStart { it == '/' }) + ?: throw MalformedURLException() + return urlBuilder.build() + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (other == null || javaClass != other.javaClass) { + return false + } + other as RestClient + return server == other.server && httpClient == other.httpClient + } + + override fun hashCode(): Int = Objects.hash(server, httpClient) + + override fun toString(): String = "RestClient{serverConfig=$server, httpClient=$httpClient}" + + /** Create a new builder with the settings of the current client. */ + fun withConfiguration(builder: Builder.() -> Unit): RestClient = Builder(httpClient).apply { + this.server = this@RestClient.server + builder() + }.build() + + /** Builder. */ + class Builder internal constructor( + val httpClientBuilder: OkHttpClient.Builder + ) { + var server: ServerConfig? = null + set(value) { + if (value != null) { + if (value.isUnsafe) { + checkNotNull(RestUtils.UNSAFE_SSL_FACTORY) { + "Cannot use unsafe connection, it is disallowed by the runtime environment." + } + httpClientBuilder.sslSocketFactory( + RestUtils.UNSAFE_SSL_FACTORY, + RestUtils.UNSAFE_TRUST_MANAGER[0] as X509TrustManager + ) + httpClientBuilder.hostnameVerifier(RestUtils.UNSAFE_HOSTNAME_VERIFIER) + } else { + val trustManager = RestUtils.systemDefaultTrustManager() + val socketFactory = RestUtils.systemDefaultSslSocketFactory(trustManager) + httpClientBuilder.sslSocketFactory(socketFactory, trustManager) + httpClientBuilder.hostnameVerifier(RestUtils.DEFAULT_HOSTNAME_VERIFIER) + } + } + field = value + } + var headers: Headers = headersOf() + + constructor(client: OkHttpClient) : this(client.newBuilder()) + + /** Allowed protocols. */ + fun protocols(protocols: List) { + httpClientBuilder.protocols(protocols) + } + + /** Whether to enable GZIP compression. */ + fun gzipCompression(compression: Boolean) { + val gzip = httpClientBuilder.interceptors() + .find { it is GzipRequestInterceptor} as GzipRequestInterceptor? + if (compression && gzip == null) { + logger.debug("Enabling GZIP compression") + httpClientBuilder.addInterceptor(GzipRequestInterceptor()) + } else if (!compression && gzip != null) { + logger.debug("Disabling GZIP compression") + httpClientBuilder.interceptors().remove(gzip) + } + } + + /** Timeouts for connecting, reading and writing. */ + fun timeout(timeout: Long, unit: TimeUnit) { + httpClientBuilder.connectTimeout(timeout, unit) + .readTimeout(timeout, unit) + .writeTimeout(timeout, unit) + } + + /** Build a new RestClient. */ + fun build(): RestClient { + return RestClient(this) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(RestClient::class.java) + const val DEFAULT_TIMEOUT: Long = 30 + private var globalHttpClientRef = WeakReference(null) + + val globalHttpClient: OkHttpClient + @Synchronized + get() = globalHttpClientRef.get() + ?: createDefaultClient().build() + .also { globalHttpClientRef = WeakReference(it) } + + /** Get the response body of a response as a String. + * Will return null if the response body is null. + * @return body contents as a String. + * @throws IOException if the body could not be read as a String. + */ + @JvmStatic + @Throws(IOException::class) + fun Response.bodyString(): String? { + return body?.use { it.string() } + } + + /** Create a builder with a global shared OkHttpClient. */ + @JvmStatic + fun globalRestClient(builder: Builder.() -> Unit = {}): RestClient = + Builder(globalHttpClient).apply(builder).build() + + /** Create a builder with a new OkHttpClient using default settings. */ + @JvmStatic + fun newRestClient(builder: Builder.() -> Unit): RestClient = + Builder(createDefaultClient()).apply(builder).build() + + /** + * Create a new OkHttpClient. The timeouts are set to the default. + * @return new OkHttpClient. + */ + private fun createDefaultClient() = OkHttpClient.Builder() + .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java deleted file mode 100644 index 0c8429f6..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; - -/** - * Exception when a HTTP REST request fails. - */ -public class RestException extends IOException { - private static final long serialVersionUID = 1; - - private final int statusCode; - private final String body; - - /** - * Request with status code and response body. - * @param statusCode HTTP status code - * @param body response body. - */ - public RestException(int statusCode, String body) { - this(statusCode, body, null); - } - - /** - * Request with status code, response body and cause. - * @param statusCode HTTP status code - * @param body response body. - * @param cause causing exception. - */ - public RestException(int statusCode, String body, Throwable cause) { - super("REST call failed (HTTP code " + statusCode + "): " - + body.substring(0, Math.min(512, body.length())), cause); - this.statusCode = statusCode; - this.body = body; - } - - public int getStatusCode() { - return statusCode; - } - - public String getBody() { - return body; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt new file mode 100644 index 00000000..86871b16 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import java.io.IOException + +/** + * Exception when a HTTP REST request fails. + */ +class RestException +/** + * Request with status code and response body. + * @param statusCode HTTP status code + * @param body response body. + */( + val statusCode: Int, + val body: String?, + cause: Throwable? = null, +) : IOException( + buildString(150) { + append("REST call failed (HTTP code ") + append(statusCode) + if (body == null) { + append(')') + } else { + append(body.substring(0, body.length.coerceAtMost(512))) + } + }, + cause, +) diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java deleted file mode 100644 index 59556f90..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.radarbase.producer.rest.RestClient.DEFAULT_TIMEOUT; -import static org.radarbase.producer.rest.RestClient.responseBody; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.Collections; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import okhttp3.Headers; -import okhttp3.MediaType; -import okhttp3.Protocol; -import okhttp3.Request; -import okhttp3.Response; -import org.apache.avro.SchemaValidationException; -import org.radarbase.config.ServerConfig; -import org.radarbase.producer.AuthenticationException; -import org.radarbase.producer.KafkaSender; -import org.radarbase.producer.KafkaTopicSender; -import org.radarbase.producer.rest.ConnectionState.State; -import org.radarbase.topic.AvroTopic; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new - * sender must be constructed with {@link #sender(AvroTopic)} per AvroTopic. This implementation is - * blocking and unbuffered, so flush, clear and close do not do anything. - */ -public class RestSender implements KafkaSender { - private static final Logger logger = LoggerFactory.getLogger(RestSender.class); - - public static final String KAFKA_REST_ACCEPT_ENCODING = - "application/vnd.kafka.v2+json, application/vnd.kafka+json, application/json"; - public static final String KAFKA_REST_ACCEPT_LEGACY_ENCODING = - "application/vnd.kafka.v1+json, application/vnd.kafka+json, application/json"; - public static final MediaType KAFKA_REST_BINARY_ENCODING = - MediaType.parse("application/vnd.radarbase.avro.v1+binary"); - public static final MediaType KAFKA_REST_AVRO_ENCODING = - MediaType.parse("application/vnd.kafka.avro.v2+json; charset=utf-8"); - public static final MediaType KAFKA_REST_AVRO_LEGACY_ENCODING = - MediaType.parse("application/vnd.kafka.avro.v1+json; charset=utf-8"); - private RequestProperties requestProperties; - - private Request.Builder connectionTestRequest; - private SchemaRetriever schemaRetriever; - private RestClient httpClient; - private final ConnectionState state; - - /** - * Construct a RestSender. - */ - private RestSender(Builder builder) { - this.schemaRetriever = Objects.requireNonNull(builder.retriever); - this.requestProperties = new RequestProperties( - KAFKA_REST_ACCEPT_ENCODING, - builder.binary ? KAFKA_REST_BINARY_ENCODING : KAFKA_REST_AVRO_ENCODING, - builder.additionalHeaders.build(), - builder.binary); - this.state = builder.state; - setRestClient(Objects.requireNonNull(builder.client).newBuilder() - .protocols(Collections.singletonList(Protocol.HTTP_1_1)) - .build()); - } - - /** - * Set the connection timeout. This affects both the connection state as the HTTP client - * setting. - * @param connectionTimeout timeout - * @param unit time unit - */ - public synchronized void setConnectionTimeout(long connectionTimeout, TimeUnit unit) { - if (connectionTimeout != httpClient.getTimeout()) { - httpClient = httpClient.newBuilder().timeout(connectionTimeout, unit).build(); - state.setTimeout(connectionTimeout, unit); - } - } - - /** - * Set the Kafka REST Proxy settings. This affects the REST client. - * @param kafkaConfig server configuration of the Kafka REST proxy. - */ - public synchronized void setKafkaConfig(ServerConfig kafkaConfig) { - Objects.requireNonNull(kafkaConfig); - if (kafkaConfig.equals(httpClient.getServer())) { - return; - } - setRestClient(httpClient.newBuilder().server(kafkaConfig).build()); - } - - /** - * Set the REST client. This will reset the connection state. - */ - private void setRestClient(RestClient newClient) { - try { - connectionTestRequest = newClient.requestBuilder("").head(); - } catch (MalformedURLException ex) { - throw new IllegalArgumentException("Schemaless topics do not have a valid URL", ex); - } - httpClient = newClient; - state.reset(); - } - - /** Set the schema retriever. */ - public final synchronized void setSchemaRetriever(SchemaRetriever retriever) { - this.schemaRetriever = retriever; - } - - /** Get the current REST client. */ - public synchronized RestClient getRestClient() { - return httpClient; - } - - /** Get the schema retriever. */ - public synchronized SchemaRetriever getSchemaRetriever() { - return this.schemaRetriever; - } - - /** Get a request to check the connection status. */ - private synchronized Request getConnectionTestRequest() { - return connectionTestRequest.headers(requestProperties.headers).build(); - } - - /** Set the compression of the REST client. */ - public synchronized void setCompression(boolean useCompression) { - httpClient = httpClient.newBuilder().gzipCompression(useCompression).build(); - } - - /** Get the headers used in requests. */ - public synchronized Headers getHeaders() { - return requestProperties.headers; - } - - /** Set the headers used in requests. */ - public synchronized void setHeaders(Headers additionalHeaders) { - this.requestProperties = new RequestProperties(requestProperties.acceptType, - requestProperties.contentType, additionalHeaders, - requestProperties.binary); - this.state.reset(); - } - - @Override - public KafkaTopicSender sender(AvroTopic topic) - throws SchemaValidationException { - return new RestTopicSender<>(topic, this, state); - } - - /** - * Get the current request properties. - */ - public synchronized RequestProperties getRequestProperties() { - return requestProperties; - } - - /** - * Get the current request context. - */ - public synchronized RequestContext getRequestContext() { - return new RequestContext(httpClient, requestProperties); - } - - @Override - public boolean resetConnection() throws AuthenticationException { - if (state.getState() == State.CONNECTED) { - return true; - } - try (Response response = httpClient.request(getConnectionTestRequest())) { - if (response.isSuccessful()) { - state.didConnect(); - } else if (response.code() == 401) { - state.wasUnauthorized(); - } else { - state.didDisconnect(); - String bodyString = responseBody(response); - logger.warn("Failed to make heartbeat request to {} (HTTP status code {}): {}", - httpClient, response.code(), bodyString); - } - } catch (IOException ex) { - // no stack trace is needed - state.didDisconnect(); - logger.warn("Failed to make heartbeat request to {}: {}", httpClient, ex.toString()); - } - - if (state.getState() == State.UNAUTHORIZED) { - throw new AuthenticationException("HEAD request unauthorized"); - } - - return state.getState() == State.CONNECTED; - } - - @Override - public boolean isConnected() throws AuthenticationException { - switch (state.getState()) { - case CONNECTED: - return true; - case DISCONNECTED: - return false; - case UNAUTHORIZED: - throw new AuthenticationException("Unauthorized"); - case UNKNOWN: - return resetConnection(); - default: - throw new IllegalStateException("Illegal connection state"); - } - } - - @Override - public void close() { - // noop - } - - /** - * Revert to a legacy connection if the server does not support the latest protocols. - * @param acceptEncoding accept encoding to use in the legacy connection. - * @param contentEncoding content encoding to use in the legacy connection. - * @param binary whether to send the data as binary. - */ - public synchronized void useLegacyEncoding(String acceptEncoding, - MediaType contentEncoding, boolean binary) { - logger.debug("Reverting to encoding {} -> {} (binary: {})", - contentEncoding, acceptEncoding, binary); - this.requestProperties = new RequestProperties(acceptEncoding, - contentEncoding, - requestProperties.headers, binary); - } - - public static class Builder { - private SchemaRetriever retriever; - private ConnectionState state; - private RestClient client; - private Headers.Builder additionalHeaders = new Headers.Builder(); - private boolean binary = false; - - public Builder schemaRetriever(SchemaRetriever schemaRetriever) { - this.retriever = schemaRetriever; - return this; - } - - /** - * Whether to try to send binary content. This only works if the server supports it. If not, - * there may be an additional round-trip. - * @param binary true if attempt to send binary content, false otherwise - */ - public Builder useBinaryContent(boolean binary) { - this.binary = binary; - return this; - } - - /** - * Whether to try to send binary content. This only works if the server supports it. If not, - * there may be an additional round-trip. - * @param binary true if attempt to send binary content, false otherwise - * @deprecated use {@link #useBinaryContent(boolean)} instead - */ - @Deprecated - @SuppressWarnings("PMD.LinguisticNaming") - public Builder hasBinaryContent(boolean binary) { - this.binary = binary; - return this; - } - - public Builder connectionState(ConnectionState state) { - this.state = state; - return this; - } - - public Builder httpClient(RestClient client) { - this.client = client; - return this; - } - - public Builder headers(Headers headers) { - additionalHeaders = headers.newBuilder(); - return this; - } - - public Builder addHeader(String header, String value) { - additionalHeaders.add(header + ": " + value); - return this; - } - - /** Build a new RestSender. */ - public RestSender build() { - if (state == null) { - state = new ConnectionState(DEFAULT_TIMEOUT, TimeUnit.SECONDS); - } - - return new RestSender(this); - } - } - - static final class RequestContext { - final RequestProperties properties; - final RestClient client; - - RequestContext(RestClient client, RequestProperties properties) { - this.properties = properties; - this.client = client; - } - } - - static final class RequestProperties { - final String acceptType; - final MediaType contentType; - final Headers headers; - final boolean binary; - - RequestProperties(String acceptType, MediaType contentType, Headers headers, - boolean binary) { - this.acceptType = acceptType; - this.contentType = contentType; - this.headers = headers; - this.binary = binary; - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt new file mode 100644 index 00000000..a9ad76a7 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt @@ -0,0 +1,278 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.Headers +import okhttp3.MediaType +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol +import okhttp3.Request +import org.apache.avro.SchemaValidationException +import org.radarbase.config.ServerConfig +import org.radarbase.producer.AuthenticationException +import org.radarbase.producer.KafkaSender +import org.radarbase.producer.KafkaTopicSender +import org.radarbase.producer.rest.RestClient.Companion.bodyString +import org.radarbase.producer.rest.RestClient.Companion.globalRestClient +import org.radarbase.topic.AvroTopic +import org.slf4j.LoggerFactory +import java.io.IOException +import java.net.MalformedURLException +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new + * sender must be constructed with [.sender] per AvroTopic. This implementation is + * blocking and unbuffered, so flush, clear and close do not do anything. + */ +class RestSender private constructor(builder: Builder) : KafkaSender { + /** + * Get the current request properties. + */ + @get:Synchronized + var requestProperties: RequestProperties + private set + private var connectionTestRequest: Request.Builder + /** Get the schema retriever. */ + /** Set the schema retriever. */ + @get:Synchronized + @set:Synchronized + var schemaRetriever: SchemaRetriever? + + /** Get the current REST client. */ + @get:Synchronized + var restClient: RestClient = requireNotNull(builder.httpClient).withConfiguration { + protocols(listOf(Protocol.HTTP_1_1)) + } + private set(value) { + try { + connectionTestRequest = value.requestBuilder("").head() + } catch (ex: MalformedURLException) { + throw IllegalArgumentException("Schemaless topics do not have a valid URL", ex) + } + field = value + state.reset() + } + private val state: ConnectionState + + /** + * Construct a RestSender. + */ + init { + schemaRetriever = Objects.requireNonNull(builder.schemaRetriever) + requestProperties = RequestProperties( + KAFKA_REST_ACCEPT_ENCODING, + if (builder.useBinaryContent) KAFKA_REST_BINARY_ENCODING else KAFKA_REST_AVRO_ENCODING, + builder.headers.build(), + builder.useBinaryContent + ) + state = builder.connectionState + ?: ConnectionState(RestClient.DEFAULT_TIMEOUT, TimeUnit.SECONDS) + restClient = builder.httpClient?.withConfiguration { + protocols(listOf(Protocol.HTTP_1_1)) + } ?: globalRestClient { + protocols(listOf(Protocol.HTTP_1_1)) + } + try { + connectionTestRequest = restClient.requestBuilder("").head() + } catch (ex: MalformedURLException) { + throw IllegalArgumentException("Schemaless topics do not have a valid URL", ex) + } + } + + /** + * Set the connection timeout. This affects both the connection state as the HTTP client + * setting. + * @param connectionTimeout timeout + * @param unit time unit + */ + @Synchronized + fun setConnectionTimeout(connectionTimeout: Long, unit: TimeUnit) { + if (connectionTimeout == restClient.timeout) return + restClient = restClient.withConfiguration { + timeout(connectionTimeout, unit) + } + state.setTimeout(connectionTimeout, unit) + } + + /** + * Set the Kafka REST Proxy settings. This affects the REST client. + * @param kafkaConfig server configuration of the Kafka REST proxy. + */ + @Synchronized + fun setKafkaConfig(kafkaConfig: ServerConfig) { + if (kafkaConfig == restClient.server) return + restClient = restClient.withConfiguration { + server = kafkaConfig + } + } + + /** Get a request to check the connection status. */ + @Synchronized + private fun getConnectionTestRequest(): Request { + return connectionTestRequest.headers(requestProperties.headers).build() + } + + /** Set the compression of the REST client. */ + @Synchronized + fun setCompression(useCompression: Boolean) { + restClient = restClient.withConfiguration { + gzipCompression(useCompression) + } + } + /** Get the headers used in requests. */ + /** Set the headers used in requests. */ + @get:Synchronized + @set:Synchronized + var headers: Headers + get() = requestProperties.headers + set(additionalHeaders) { + requestProperties = RequestProperties( + requestProperties.acceptType, + requestProperties.contentType, additionalHeaders, + requestProperties.binary + ) + state.reset() + } + + @Throws(SchemaValidationException::class) + override fun sender(topic: AvroTopic): KafkaTopicSender { + return RestTopicSender(topic, this, state) + } + + /** + * Get the current request context. + */ + @get:Synchronized + val requestContext: RequestContext + get() = RequestContext(restClient, requestProperties) + + @Throws(AuthenticationException::class) + override fun resetConnection(): Boolean { + if (state.state === ConnectionState.State.CONNECTED) { + return true + } + try { + restClient.request(getConnectionTestRequest()).use { response -> + if (response.isSuccessful) { + state.didConnect() + } else if (response.code == 401) { + state.wasUnauthorized() + } else { + state.didDisconnect() + val bodyString = response.bodyString() + logger.warn( + "Failed to make heartbeat request to {} (HTTP status code {}): {}", + restClient, response.code, bodyString + ) + } + } + } catch (ex: IOException) { + // no stack trace is needed + state.didDisconnect() + logger.warn("Failed to make heartbeat request to {}: {}", restClient, ex.toString()) + } + if (state.state === ConnectionState.State.UNAUTHORIZED) { + throw AuthenticationException("HEAD request unauthorized") + } + return state.state === ConnectionState.State.CONNECTED + } + + @get:Throws(AuthenticationException::class) + override val isConnected: Boolean + get() = when (state.state) { + ConnectionState.State.CONNECTED -> true + ConnectionState.State.DISCONNECTED -> false + ConnectionState.State.UNAUTHORIZED -> throw AuthenticationException("Unauthorized") + ConnectionState.State.UNKNOWN -> resetConnection() + } + + override fun close() { + // noop + } + + /** + * Revert to a legacy connection if the server does not support the latest protocols. + * @param acceptEncoding accept encoding to use in the legacy connection. + * @param contentEncoding content encoding to use in the legacy connection. + * @param binary whether to send the data as binary. + */ + @Synchronized + fun useLegacyEncoding( + acceptEncoding: String, + contentEncoding: MediaType, + binary: Boolean, + ) { + logger.debug( + "Reverting to encoding {} -> {} (binary: {})", + contentEncoding, acceptEncoding, binary + ) + requestProperties = RequestProperties( + acceptEncoding, + contentEncoding, + requestProperties.headers, binary + ) + } + + class Builder internal constructor(){ + var schemaRetriever: SchemaRetriever? = null + var connectionState: ConnectionState? = null + var httpClient: RestClient? = null + var headers = Headers.Builder() + /** + * Whether to try to send binary content. This only works if the server supports it. If not, + * there may be an additional round-trip. + */ + var useBinaryContent = false + + /** Build a new RestSender. */ + fun build(): RestSender { + return RestSender(this) + } + } + + class RequestContext( + val client: RestClient, + val properties: RequestProperties, + ) + class RequestProperties( + val acceptType: String, + val contentType: MediaType, + val headers: Headers, + val binary: Boolean, + ) + + companion object { + private val logger = LoggerFactory.getLogger(RestSender::class.java) + const val KAFKA_REST_ACCEPT_ENCODING = + "application/vnd.kafka.v2+json, application/vnd.kafka+json, application/json" + const val KAFKA_REST_ACCEPT_LEGACY_ENCODING = + "application/vnd.kafka.v1+json, application/vnd.kafka+json, application/json" + @JvmField + val KAFKA_REST_BINARY_ENCODING: MediaType = + "application/vnd.radarbase.avro.v1+binary".toMediaType() + @JvmField + val KAFKA_REST_AVRO_ENCODING: MediaType = + "application/vnd.kafka.avro.v2+json; charset=utf-8".toMediaType() + @JvmField + val KAFKA_REST_AVRO_LEGACY_ENCODING: MediaType = + "application/vnd.kafka.avro.v1+json; charset=utf-8".toMediaType() + + fun restSender(builder: Builder.() -> Unit): RestSender = + Builder().apply(builder).build() + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java b/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java deleted file mode 100644 index 147ce268..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.radarbase.producer.rest.UncheckedRequestException.fail; - -import java.io.IOException; -import java.util.Collections; -import java.util.Objects; -import okhttp3.HttpUrl; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.apache.avro.SchemaValidationException; -import org.json.JSONException; -import org.radarbase.data.AvroRecordData; -import org.radarbase.data.RecordData; -import org.radarbase.producer.AuthenticationException; -import org.radarbase.producer.KafkaTopicSender; -import org.radarbase.topic.AvroTopic; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class RestTopicSender - implements KafkaTopicSender { - private static final Logger logger = LoggerFactory.getLogger(RestTopicSender.class); - - private final AvroTopic topic; - private RecordRequest requestData; - private final RestSender sender; - private final ConnectionState state; - - RestTopicSender(AvroTopic topic, RestSender sender, ConnectionState state) - throws SchemaValidationException { - this.topic = topic; - this.sender = sender; - this.state = state; - - if (sender.getRequestContext().properties.binary) { - try { - requestData = new BinaryRecordRequest<>(topic); - } catch (IllegalArgumentException ex) { - logger.warn("Cannot use Binary encoding for incompatible topic {}: {}", - topic, ex.toString()); - } - } - - if (requestData == null) { - requestData = new JsonRecordRequest<>(topic); - } - } - - @Override - public void send(K key, V value) throws IOException, SchemaValidationException { - send(new AvroRecordData<>(topic, key, Collections.singletonList(value))); - } - - /** - * Actually make a REST request to the Kafka REST server and Schema Registry. - * - * @param records values to send - * @throws IOException if records could not be sent - */ - @Override - public void send(RecordData records) throws IOException, SchemaValidationException { - RestSender.RequestContext context = sender.getRequestContext(); - Request request = buildRequest(context, records); - - boolean doResend = false; - try (Response response = context.client.request(request)) { - if (response.isSuccessful()) { - state.didConnect(); - logger.debug("Added message to topic {}", topic); - } else if (response.code() == 401 || response.code() == 403) { - state.wasUnauthorized(); - } else if (response.code() == 415) { - downgradeConnection(request, response); - doResend = true; - } else { - throw fail(request, response, null); - } - } catch (IOException ex) { - state.didDisconnect(); - fail(request, null, ex).rethrow(); - } catch (UncheckedRequestException ex) { - state.didDisconnect(); - ex.rethrow(); - } finally { - requestData.reset(); - } - - if (state.getState() == ConnectionState.State.UNAUTHORIZED) { - throw new AuthenticationException("Request unauthorized"); - } - - if (doResend) { - send(records); - } - } - - private void updateRecords(RestSender.RequestContext context, RecordData records) - throws IOException, SchemaValidationException { - if (!context.properties.binary && requestData instanceof BinaryRecordRequest) { - requestData = new JsonRecordRequest<>(topic); - } - - String sendTopic = topic.getName(); - SchemaRetriever retriever = sender.getSchemaRetriever(); - - ParsedSchemaMetadata keyMetadata; - ParsedSchemaMetadata valueMetadata; - - try { - keyMetadata = retriever.getOrSetSchemaMetadata( - sendTopic, false, topic.getKeySchema(), -1); - valueMetadata = retriever.getOrSetSchemaMetadata( - sendTopic, true, topic.getValueSchema(), -1); - } catch (JSONException | IOException ex) { - throw new IOException("Failed to get schemas for topic " + topic, ex); - } - - requestData.prepare(keyMetadata, valueMetadata, records); - } - - private void downgradeConnection(Request request, Response response) throws IOException { - if (this.requestData instanceof BinaryRecordRequest) { - state.didConnect(); - logger.warn("Binary Avro encoding is not supported." - + " Switching to JSON encoding."); - sender.useLegacyEncoding( - RestSender.KAFKA_REST_ACCEPT_ENCODING, RestSender.KAFKA_REST_AVRO_ENCODING, - false); - requestData = new JsonRecordRequest<>(topic); - } else if (Objects.equals(request.header("Accept"), - RestSender.KAFKA_REST_ACCEPT_ENCODING)) { - state.didConnect(); - logger.warn("Latest Avro encoding is not supported. Switching to legacy " - + "encoding."); - sender.useLegacyEncoding( - RestSender.KAFKA_REST_ACCEPT_LEGACY_ENCODING, - RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING, - false); - } else { - RequestBody body = request.body(); - MediaType contentType = body != null ? body.contentType() : null; - if (contentType == null - || contentType.equals(RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING)) { - throw fail(request, response, - new IOException("Content-Type " + contentType + " not accepted by server.")); - } else { - // the connection may have been downgraded already - state.didConnect(); - logger.warn("Content-Type changed during request"); - } - } - } - - private Request buildRequest(RestSender.RequestContext context, RecordData records) - throws IOException, SchemaValidationException { - updateRecords(context, records); - - HttpUrl sendToUrl = context.client.getRelativeUrl("topics/" + topic.getName()); - - TopicRequestBody requestBody; - Request.Builder requestBuilder = new Request.Builder() - .url(sendToUrl) - .headers(context.properties.headers) - .header("Accept", context.properties.acceptType); - - MediaType contentType = context.properties.contentType; - if (contentType.equals(RestSender.KAFKA_REST_BINARY_ENCODING) - && !(requestData instanceof BinaryRecordRequest)) { - contentType = RestSender.KAFKA_REST_AVRO_ENCODING; - } - requestBody = new TopicRequestBody(requestData, contentType); - - return requestBuilder.post(requestBody).build(); - } - - @Override - public void clear() { - // nothing - } - - @Override - public void flush() { - // nothing - } - - @Override - public void close() { - // noop - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt new file mode 100644 index 00000000..435b5a3a --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt @@ -0,0 +1,209 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.Request +import okhttp3.Response +import org.apache.avro.SchemaValidationException +import org.json.JSONException +import org.radarbase.data.AvroRecordData +import org.radarbase.data.RecordData +import org.radarbase.producer.AuthenticationException +import org.radarbase.producer.KafkaTopicSender +import org.radarbase.topic.AvroTopic +import org.slf4j.LoggerFactory +import java.io.IOException + +internal class RestTopicSender( + topic: AvroTopic, + sender: RestSender, + state: ConnectionState +) : KafkaTopicSender { + private val topic: AvroTopic + private var requestData: RecordRequest + private val sender: RestSender + private val state: ConnectionState + + init { + this.topic = topic + this.sender = sender + this.state = state + requestData = if (sender.requestContext.properties.binary) { + try { + BinaryRecordRequest(topic) + } catch (ex: IllegalArgumentException) { + logger.warn( + "Cannot use Binary encoding for incompatible topic {}: {}", + topic, ex.toString() + ) + JsonRecordRequest(topic) + } + } else { + JsonRecordRequest(topic) + } + } + + @Throws(IOException::class, SchemaValidationException::class) + override fun send(key: K, value: V) { + send(AvroRecordData(topic, key, listOf(value))) + } + + /** + * Actually make a REST request to the Kafka REST server and Schema Registry. + * + * @param records values to send + * @throws IOException if records could not be sent + */ + @Throws(IOException::class, SchemaValidationException::class) + override fun send(records: RecordData) { + val context = sender.requestContext + val request = buildRequest(context, records) + var doResend = false + try { + context.client.request(request).use { response -> + if (response.isSuccessful) { + state.didConnect() + logger.debug("Added message to topic {}", topic) + } else if (response.code == 401 || response.code == 403) { + state.wasUnauthorized() + } else if (response.code == 415) { + downgradeConnection(request, response) + doResend = true + } else { + throw UncheckedRequestException.fail(request, response, null) + } + } + } catch (ex: IOException) { + state.didDisconnect() + UncheckedRequestException.fail(request, null, ex).rethrow() + } catch (ex: UncheckedRequestException) { + state.didDisconnect() + ex.rethrow() + } finally { + requestData.reset() + } + if (state.state === ConnectionState.State.UNAUTHORIZED) { + throw AuthenticationException("Request unauthorized") + } + if (doResend) { + send(records) + } + } + + @Throws(IOException::class, SchemaValidationException::class) + private fun updateRecords(context: RestSender.RequestContext, records: RecordData) { + if (!context.properties.binary && requestData is BinaryRecordRequest<*, *>) { + requestData = JsonRecordRequest(topic) + } + val sendTopic = topic.name + val retriever = sender.schemaRetriever + val keyMetadata: ParsedSchemaMetadata + val valueMetadata: ParsedSchemaMetadata + try { + keyMetadata = retriever!!.getOrSetSchemaMetadata( + sendTopic, false, topic.keySchema, -1 + ) + valueMetadata = retriever.getOrSetSchemaMetadata( + sendTopic, true, topic.valueSchema, -1 + ) + } catch (ex: JSONException) { + throw IOException("Failed to get schemas for topic $topic", ex) + } catch (ex: IOException) { + throw IOException("Failed to get schemas for topic $topic", ex) + } + requestData.prepare(keyMetadata, valueMetadata, records) + } + + @Throws(IOException::class) + private fun downgradeConnection(request: Request, response: Response) { + if (requestData is BinaryRecordRequest<*, *>) { + state.didConnect() + logger.warn( + "Binary Avro encoding is not supported." + + " Switching to JSON encoding." + ) + sender.useLegacyEncoding( + RestSender.KAFKA_REST_ACCEPT_ENCODING, RestSender.KAFKA_REST_AVRO_ENCODING, + false + ) + requestData = JsonRecordRequest(topic) + } else if (request.header("Accept") == RestSender.KAFKA_REST_ACCEPT_ENCODING) { + state.didConnect() + logger.warn( + "Latest Avro encoding is not supported. Switching to legacy " + + "encoding." + ) + sender.useLegacyEncoding( + RestSender.KAFKA_REST_ACCEPT_LEGACY_ENCODING, + RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING, + false + ) + } else { + val body = request.body + val contentType = body?.contentType() + if (contentType == null + || contentType == RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING + ) { + throw UncheckedRequestException.fail( + request, response, + IOException("Content-Type $contentType not accepted by server.") + ) + } else { + // the connection may have been downgraded already + state.didConnect() + logger.warn("Content-Type changed during request") + } + } + } + + @Throws(IOException::class, SchemaValidationException::class) + private fun buildRequest( + context: RestSender.RequestContext, + records: RecordData + ): Request { + updateRecords(context, records) + val sendToUrl = context.client.relativeUrl("topics/" + topic.name) + val requestBody: TopicRequestBody + val requestBuilder = Request.Builder() + .url(sendToUrl) + .headers(context.properties.headers) + .header("Accept", context.properties.acceptType) + var contentType = context.properties.contentType + if (contentType == RestSender.KAFKA_REST_BINARY_ENCODING + && requestData !is BinaryRecordRequest<*, *> + ) { + contentType = RestSender.KAFKA_REST_AVRO_ENCODING + } + requestBody = TopicRequestBody(requestData, contentType) + return requestBuilder.post(requestBody).build() + } + + override fun clear() { + // nothing + } + + override fun flush() { + // nothing + } + + override fun close() { + // noop + } + + companion object { + private val logger = LoggerFactory.getLogger(RestTopicSender::class.java) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.java b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.java deleted file mode 100644 index ae1a3343..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.radarbase.producer.rest; - -import java.io.IOException; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okio.BufferedSink; -import org.apache.avro.Schema; -import org.json.JSONException; -import org.json.JSONObject; -import org.radarbase.util.Strings; - -/** REST client for Confluent schema registry. */ -public class SchemaRestClient { - private final RestClient client; - - public SchemaRestClient(RestClient client) { - this.client = client; - } - - /** Retrieve schema metadata from server. */ - public ParsedSchemaMetadata retrieveSchemaMetadata(String subject, int version) - throws JSONException, IOException { - boolean isLatest = version <= 0; - - StringBuilder pathBuilder = new StringBuilder(50) - .append("/subjects/") - .append(subject) - .append("/versions/"); - - if (isLatest) { - pathBuilder.append("latest"); - } else { - pathBuilder.append(version); - } - - JSONObject node = requestJson(pathBuilder.toString()); - int newVersion = isLatest ? node.getInt("version") : version; - int schemaId = node.getInt("id"); - Schema schema = parseSchema(node.getString("schema")); - return new ParsedSchemaMetadata(schemaId, newVersion, schema); - } - - private JSONObject requestJson(String path) throws IOException { - Request request = client.requestBuilder(path) - .addHeader("Accept", "application/json") - .build(); - - String response = client.requestString(request); - return new JSONObject(response); - } - - - /** Parse a schema from string. */ - public Schema parseSchema(String schemaString) { - Schema.Parser parser = new Schema.Parser(); - return parser.parse(schemaString); - } - - /** Add a schema to a subject. */ - public int addSchema(String subject, Schema schema) throws IOException { - Request request = client.requestBuilder("/subjects/" + subject + "/versions") - .addHeader("Accept", "application/json") - .post(new SchemaRequestBody(schema)) - .build(); - - String response = client.requestString(request); - JSONObject node = new JSONObject(response); - return node.getInt("id"); - } - - /** Request metadata for a schema on a subject. */ - public ParsedSchemaMetadata requestMetadata(String subject, Schema schema) - throws IOException { - Request request = client.requestBuilder("/subjects/" + subject) - .addHeader("Accept", "application/json") - .post(new SchemaRequestBody(schema)) - .build(); - - String response = client.requestString(request); - JSONObject node = new JSONObject(response); - - return new ParsedSchemaMetadata(node.getInt("id"), - node.getInt("version"), schema); - } - - /** Retrieve schema metadata from server. */ - public Schema retrieveSchemaById(int id) - throws JSONException, IOException { - JSONObject node = requestJson("/schemas/ids/" + id); - return parseSchema(node.getString("schema")); - } - - private static class SchemaRequestBody extends RequestBody { - private static final byte[] SCHEMA = Strings.utf8("{\"schema\":"); - private static final MediaType CONTENT_TYPE = MediaType.parse( - "application/vnd.schemaregistry.v1+json; charset=utf-8"); - - private final Schema schema; - - private SchemaRequestBody(Schema schema) { - this.schema = schema; - } - - @Override - public MediaType contentType() { - return CONTENT_TYPE; - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - sink.write(SCHEMA); - sink.writeUtf8(JSONObject.quote(schema.toString())); - sink.writeByte('}'); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt new file mode 100644 index 00000000..8a8630a7 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt @@ -0,0 +1,103 @@ +package org.radarbase.producer.rest + +import okhttp3.MediaType +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody +import okio.BufferedSink +import org.apache.avro.Schema +import org.json.JSONException +import org.json.JSONObject +import java.io.IOException + +/** REST client for Confluent schema registry. */ +class SchemaRestClient( + private val client: RestClient, +) { + /** Retrieve schema metadata from server. */ + @Throws(JSONException::class, IOException::class) + fun retrieveSchemaMetadata(subject: String, version: Int): ParsedSchemaMetadata { + val isLatest = version <= 0 + val node = requestJson(buildString(60) { + append("/subjects/") + append(subject) + append("/versions/") + if (isLatest) { + append("latest") + } else { + append(version) + } + }) + val newVersion = if (isLatest) node.getInt("version") else version + val schemaId = node.getInt("id") + val schema = parseSchema(node.getString("schema")) + return ParsedSchemaMetadata(schemaId, newVersion, schema) + } + + @Throws(IOException::class) + private fun requestJson(path: String): JSONObject { + val request: Request = client.requestBuilder(path) + .addHeader("Accept", "application/json") + .build() + val response = client.requestString(request) + return JSONObject(response) + } + + /** Parse a schema from string. */ + fun parseSchema(schemaString: String): Schema { + return Schema.Parser().parse(schemaString) + } + + /** Add a schema to a subject. */ + @Throws(IOException::class) + fun addSchema(subject: String, schema: Schema): Int { + val request: Request = client.requestBuilder("/subjects/$subject/versions") + .addHeader("Accept", "application/json") + .post(SchemaRequestBody(schema)) + .build() + val response = client.requestString(request) + val node = JSONObject(response) + return node.getInt("id") + } + + /** Request metadata for a schema on a subject. */ + @Throws(IOException::class) + fun requestMetadata(subject: String, schema: Schema): ParsedSchemaMetadata { + val request: Request = client.requestBuilder("/subjects/$subject") + .addHeader("Accept", "application/json") + .post(SchemaRequestBody(schema)) + .build() + val response = client.requestString(request) + val node = JSONObject(response) + return ParsedSchemaMetadata( + node.getInt("id"), + node.getInt("version"), schema + ) + } + + /** Retrieve schema metadata from server. */ + @Throws(JSONException::class, IOException::class) + fun retrieveSchemaById(id: Int): Schema { + val node = requestJson("/schemas/ids/$id") + return parseSchema(node.getString("schema")) + } + + private class SchemaRequestBody( + private val schema: Schema + ) : RequestBody() { + override fun contentType(): MediaType = CONTENT_TYPE + + @Throws(IOException::class) + override fun writeTo(sink: BufferedSink) { + sink.write(SCHEMA) + sink.writeUtf8(JSONObject.quote(schema.toString())) + sink.writeByte('}'.code) + } + + companion object { + private val SCHEMA = "{\"schema\":".toByteArray() + private val CONTENT_TYPE: MediaType = "application/vnd.schemaregistry.v1+json; charset=utf-8" + .toMediaType() + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java deleted file mode 100644 index 074a1ac0..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; -import org.apache.avro.Schema; -import org.json.JSONException; -import org.json.JSONObject; -import org.radarbase.config.ServerConfig; -import org.radarbase.util.TimedInt; -import org.radarbase.util.TimedValue; -import org.radarbase.util.TimedVariable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Retriever of an Avro Schema. Internally, only {@link JSONObject} is used to manage JSON data, - * to keep the class as lean as possible. - */ -@SuppressWarnings("PMD.GodClass") -public class SchemaRetriever { - private static final Logger logger = LoggerFactory.getLogger(SchemaRetriever.class); - private static final long MAX_VALIDITY = 86400L; - - private final ConcurrentMap> idCache = - new ConcurrentHashMap<>(); - private final ConcurrentMap schemaCache = new ConcurrentHashMap<>(); - private final ConcurrentMap> subjectVersionCache = - new ConcurrentHashMap<>(); - - private final SchemaRestClient restClient; - private final long cacheValidity; - - public SchemaRetriever(RestClient client, long cacheValidity) { - restClient = new SchemaRestClient(client); - this.cacheValidity = cacheValidity; - } - - public SchemaRetriever(RestClient client) { - this(client, MAX_VALIDITY); - } - - /** - * Schema retriever for a Confluent Schema Registry. - * @param config schema registry configuration. - * @param connectionTimeout timeout in seconds. - */ - public SchemaRetriever(ServerConfig config, long connectionTimeout) { - this(RestClient.global() - .server(Objects.requireNonNull(config)) - .timeout(connectionTimeout, TimeUnit.SECONDS) - .build()); - } - - /** - * Schema retriever for a Confluent Schema Registry. - * @param config schema registry configuration. - * @param connectionTimeout timeout in seconds. - * @param cacheValidity timeout in seconds for considering a schema stale. - */ - public SchemaRetriever(ServerConfig config, long connectionTimeout, long cacheValidity) { - this(RestClient.global() - .server(Objects.requireNonNull(config)) - .timeout(connectionTimeout, TimeUnit.SECONDS) - .build(), cacheValidity); - } - - /** - * Add schema metadata to the retriever. This implementation only adds it to the cache. - * @return schema ID - */ - public int addSchema(String topic, boolean ofValue, Schema schema) - throws JSONException, IOException { - String subject = subject(topic, ofValue); - int id = restClient.addSchema(subject, schema); - cache(new ParsedSchemaMetadata(id, null, schema), subject, false); - return id; - } - - /** - * Get schema metadata, and if none is found, add a new schema. - * - * @param version version to get or 0 if the latest version can be used. - */ - public ParsedSchemaMetadata getOrSetSchemaMetadata(String topic, boolean ofValue, Schema schema, - int version) throws JSONException, IOException { - try { - return getBySubjectAndVersion(topic, ofValue, version); - } catch (RestException ex) { - if (ex.getStatusCode() == 404) { - logger.warn("Schema for {} value was not yet added to the schema registry.", topic); - addSchema(topic, ofValue, schema); - return getMetadata(topic, ofValue, schema, version <= 0); - } else { - throw ex; - } - } - } - - /** Get a schema by its ID. */ - public Schema getById(int id) throws IOException { - TimedValue value = idCache.get(id); - if (value == null || value.isExpired()) { - value = new TimedValue<>(restClient.retrieveSchemaById(id), cacheValidity); - idCache.put(id, value); - schemaCache.put(value.value, new TimedInt(id, cacheValidity)); - } - return value.value; - } - - /** Gets a schema by ID and check that it is present in the given topic. */ - public ParsedSchemaMetadata getBySubjectAndId(String topic, boolean ofValue, int id) - throws IOException { - Schema schema = getById(id); - String subject = subject(topic, ofValue); - ParsedSchemaMetadata metadata = getCachedVersion(subject, id, null, schema); - return metadata != null ? metadata : getMetadata(topic, ofValue, schema); - } - - /** Get schema metadata. Cached schema metadata will be used if present. */ - public ParsedSchemaMetadata getBySubjectAndVersion(String topic, boolean ofValue, int version) - throws JSONException, IOException { - String subject = subject(topic, ofValue); - ConcurrentMap versionMap = computeIfAbsent(subjectVersionCache, subject, - new ConcurrentHashMap<>()); - TimedInt id = versionMap.get(Math.max(version, 0)); - if (id == null || id.isExpired()) { - ParsedSchemaMetadata metadata = restClient.retrieveSchemaMetadata(subject, version); - cache(metadata, subject, version <= 0); - return metadata; - } else { - Schema schema = getById(id.value); - ParsedSchemaMetadata metadata = getCachedVersion(subject, id.value, version, schema); - return metadata != null ? metadata : getMetadata(topic, ofValue, schema, version <= 0); - } - } - - /** Get all schema versions in a subject. */ - public ParsedSchemaMetadata getMetadata(String topic, boolean ofValue, Schema schema) - throws IOException { - return getMetadata(topic, ofValue, schema, false); - } - - - /** Get the metadata of a specific schema in a topic. */ - public ParsedSchemaMetadata getMetadata(String topic, boolean ofValue, Schema schema, - boolean ofLatestVersion) throws IOException { - TimedInt id = schemaCache.get(schema); - String subject = subject(topic, ofValue); - - if (id != null && !id.isExpired()) { - ParsedSchemaMetadata metadata = getCachedVersion(subject, id.value, null, schema); - if (metadata != null) { - return metadata; - } - } - - ParsedSchemaMetadata metadata = restClient.requestMetadata(subject, schema); - cache(metadata, subject, ofLatestVersion); - return metadata; - } - - - /** - * Get cached metadata. - * @param subject schema registry subject - * @param id schema ID. - * @param reportedVersion version requested by the client. Null if no version was requested. - * This version will be used if the actual version was not cached. - * @param schema schema to use. - * @return metadata if present. Returns null if no metadata is cached or if no version is cached - * and the reportedVersion is null. - */ - protected ParsedSchemaMetadata getCachedVersion(String subject, int id, - Integer reportedVersion, Schema schema) { - Integer version = reportedVersion; - if (version == null || version <= 0) { - ConcurrentMap versions = subjectVersionCache.get(subject); - version = findCachedVersion(id, versions); - if (version == null || version <= 0) { - return null; - } - } - return new ParsedSchemaMetadata(id, version, schema); - } - - private Integer findCachedVersion(int id, ConcurrentMap cache) { - if (cache == null) { - return null; - } - for (Map.Entry entry : cache.entrySet()) { - if (!entry.getValue().isExpired() - && entry.getKey() != 0 - && entry.getValue().value == id) { - return entry.getKey(); - } - } - return null; - } - - protected void cache(ParsedSchemaMetadata metadata, String subject, boolean latest) { - TimedInt id = new TimedInt(metadata.getId(), cacheValidity); - schemaCache.put(metadata.getSchema(), id); - if (metadata.getVersion() != null) { - ConcurrentMap versionCache = computeIfAbsent(subjectVersionCache, - subject, new ConcurrentHashMap<>()); - - versionCache.put(metadata.getVersion(), id); - if (latest) { - versionCache.put(0, id); - } - } - idCache.put(metadata.getId(), new TimedValue<>(metadata.getSchema(), cacheValidity)); - } - - /** - * Remove expired entries from cache. - */ - public void pruneCache() { - prune(schemaCache); - prune(idCache); - for (ConcurrentMap versionMap : subjectVersionCache.values()) { - prune(versionMap); - } - } - - /** - * Remove all entries from cache. - */ - public void clearCache() { - subjectVersionCache.clear(); - idCache.clear(); - schemaCache.clear(); - } - - /** The subject in the Avro Schema Registry, given a Kafka topic. */ - protected static String subject(String topic, boolean ofValue) { - return topic + (ofValue ? "-value" : "-key"); - } - - private static void prune(Map map) { - for (Entry entry : map.entrySet()) { - if (entry.getValue().isExpired()) { - map.remove(entry.getKey(), entry.getValue()); - } - } - } - - private static V computeIfAbsent(ConcurrentMap original, K key, V newValue) { - V existingValue = original.putIfAbsent(key, newValue); - return existingValue != null ? existingValue : newValue; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt new file mode 100644 index 00000000..49f93280 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt @@ -0,0 +1,235 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import org.apache.avro.Schema +import org.json.JSONException +import org.radarbase.util.CacheConfig +import org.radarbase.util.TimedInt +import org.radarbase.util.TimedValue +import org.radarbase.util.TimedVariable.Companion.prune +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Duration +import java.util.* +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap + +/** + * Retriever of an Avro Schema. + */ +open class SchemaRetriever @JvmOverloads constructor( + client: RestClient, + private val cacheConfig: CacheConfig = CacheConfig(MAX_VALIDITY), +) { + private val idCache: ConcurrentMap> = ConcurrentHashMap() + private val schemaCache: ConcurrentMap = ConcurrentHashMap() + private val subjectVersionCache: ConcurrentMap> = + ConcurrentHashMap() + private val restClient: SchemaRestClient = SchemaRestClient(client) + + /** + * Add schema metadata to the retriever. This implementation only adds it to the cache. + * @return schema ID + */ + @Throws(JSONException::class, IOException::class) + fun addSchema(topic: String, ofValue: Boolean, schema: Schema): Int { + val subject = subject(topic, ofValue) + val id = restClient.addSchema(subject, schema) + cache(ParsedSchemaMetadata(id, null, schema), subject, false) + return id + } + + /** + * Get schema metadata, and if none is found, add a new schema. + * + * @param version version to get or 0 if the latest version can be used. + */ + @Throws(JSONException::class, IOException::class) + open fun getOrSetSchemaMetadata( + topic: String, ofValue: Boolean, schema: Schema, + version: Int + ): ParsedSchemaMetadata { + return try { + getBySubjectAndVersion(topic, ofValue, version) + } catch (ex: RestException) { + if (ex.statusCode == 404) { + logger.warn("Schema for {} value was not yet added to the schema registry.", topic) + addSchema(topic, ofValue, schema) + getMetadata(topic, ofValue, schema, version <= 0) + } else { + throw ex + } + } + } + + /** Get a schema by its ID. */ + @Throws(IOException::class) + fun getById(id: Int): Schema { + var value = idCache[id] + if (value == null || value.isExpired) { + value = TimedValue(restClient.retrieveSchemaById(id), cacheConfig) + idCache[id] = value + schemaCache[value.value] = TimedInt(id, cacheConfig) + } + return value.value + } + + /** Gets a schema by ID and check that it is present in the given topic. */ + @Throws(IOException::class) + fun getBySubjectAndId(topic: String, ofValue: Boolean, id: Int): ParsedSchemaMetadata { + val schema = getById(id) + val subject = subject(topic, ofValue) + val metadata = getCachedVersion(subject, id, null, schema) + return metadata ?: getMetadata(topic, ofValue, schema) + } + + /** Get schema metadata. Cached schema metadata will be used if present. */ + @Throws(JSONException::class, IOException::class) + fun getBySubjectAndVersion( + topic: String, + ofValue: Boolean, + version: Int + ): ParsedSchemaMetadata { + val subject = subject(topic, ofValue) + val versionMap = subjectVersionCache.computeIfAbsent( + subject, + ::ConcurrentHashMap + ) + val id = versionMap[version.coerceAtLeast(0)] + return if (id == null || id.isExpired) { + val metadata = restClient.retrieveSchemaMetadata(subject, version) + cache(metadata, subject, version <= 0) + metadata + } else { + val schema = getById(id.value) + val metadata = getCachedVersion(subject, id.value, version, schema) + metadata ?: getMetadata(topic, ofValue, schema, version <= 0) + } + } + + /** Get all schema versions in a subject. */ + @Throws(IOException::class) + fun getMetadata(topic: String, ofValue: Boolean, schema: Schema): ParsedSchemaMetadata { + return getMetadata(topic, ofValue, schema, false) + } + + /** Get the metadata of a specific schema in a topic. */ + @Throws(IOException::class) + fun getMetadata( + topic: String, ofValue: Boolean, schema: Schema, + ofLatestVersion: Boolean + ): ParsedSchemaMetadata { + val id = schemaCache[schema] + val subject = subject(topic, ofValue) + if (id != null && !id.isExpired) { + val metadata = getCachedVersion(subject, id.value, null, schema) + if (metadata != null) { + return metadata + } + } + val metadata = restClient.requestMetadata(subject, schema) + cache(metadata, subject, ofLatestVersion) + return metadata + } + + /** + * Get cached metadata. + * @param subject schema registry subject + * @param id schema ID. + * @param reportedVersion version requested by the client. Null if no version was requested. + * This version will be used if the actual version was not cached. + * @param schema schema to use. + * @return metadata if present. Returns null if no metadata is cached or if no version is cached + * and the reportedVersion is null. + */ + protected fun getCachedVersion( + subject: String, id: Int, + reportedVersion: Int?, schema: Schema? + ): ParsedSchemaMetadata? { + var version = reportedVersion + if (version == null || version <= 0) { + version = findCachedVersion(id, subjectVersionCache[subject]) + if (version == null || version <= 0) { + return null + } + } + return ParsedSchemaMetadata(id, version, schema!!) + } + + private fun findCachedVersion(id: Int, cache: ConcurrentMap?): Int? { + cache ?: return null + return cache.entries.find { (k, v) -> !v.isExpired && k != 0 && v.value == id } + ?.key + } + + protected fun cache(metadata: ParsedSchemaMetadata, subject: String, latest: Boolean) { + val id = TimedInt(metadata.id, cacheConfig) + schemaCache[metadata.schema] = id + if (metadata.version != null) { + val versionCache = subjectVersionCache.computeIfAbsent( + subject, + ::ConcurrentHashMap + ) + versionCache[metadata.version] = id + if (latest) { + versionCache[0] = id + } + } + idCache[metadata.id] = TimedValue(metadata.schema, cacheConfig) + } + + /** + * Remove expired entries from cache. + */ + fun pruneCache() { + schemaCache.prune() + idCache.prune() + for (versionMap in subjectVersionCache.values) { + versionMap.prune() + } + } + + /** + * Remove all entries from cache. + */ + fun clearCache() { + subjectVersionCache.clear() + idCache.clear() + schemaCache.clear() + } + + companion object { + private val logger = LoggerFactory.getLogger(SchemaRetriever::class.java) + private val MAX_VALIDITY = Duration.ofDays(1) + + /** The subject in the Avro Schema Registry, given a Kafka topic. */ + @JvmStatic + fun subject(topic: String, ofValue: Boolean): String = if (ofValue) "$topic-value" else "$topic-key" + + private fun MutableMap.computeIfAbsent( + key: K, + newValueGenerator: () -> V, + ): V { + return get(key) + ?: run { + val newValue = newValueGenerator() + putIfAbsent(key, newValue) + ?: newValue + } + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java b/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java deleted file mode 100644 index 353afb9c..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okio.BufferedSink; - -/** - * TopicRequestData in a RequestBody. - */ -class TopicRequestBody extends RequestBody { - protected final RecordRequest data; - private final MediaType mediaType; - - TopicRequestBody(RecordRequest requestData, MediaType mediaType) { - this.data = requestData; - this.mediaType = mediaType; - } - - @Override - public MediaType contentType() { - return mediaType; - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - data.writeToSink(sink); - } - - static String topicRequestContent(Request request, int maxLength) throws IOException { - TopicRequestBody body = (TopicRequestBody) request.body(); - if (body == null) { - return null; - } - return body.data.content(maxLength); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt new file mode 100644 index 00000000..d8e7935b --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.MediaType +import okhttp3.Request +import okhttp3.RequestBody +import okio.BufferedSink +import java.io.IOException + +/** + * TopicRequestData in a RequestBody. + */ +internal class TopicRequestBody( + protected val data: RecordRequest<*, *>, + private val mediaType: MediaType +) : RequestBody() { + override fun contentType(): MediaType? { + return mediaType + } + + @Throws(IOException::class) + override fun writeTo(sink: BufferedSink) { + data.writeToSink(sink) + } + + companion object { + @JvmStatic + @Throws(IOException::class) + fun topicRequestContent(request: Request?, maxLength: Int): String? { + request ?: return null + val body = request.body as TopicRequestBody? ?: return null + return body.data.content(maxLength) + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java b/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java deleted file mode 100644 index a1524427..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import java.io.IOException; -import okhttp3.Request; -import okhttp3.Response; - -/** Unchecked exception for failures during request handling. */ -public class UncheckedRequestException extends RuntimeException { - private static final long serialVersionUID = 1; - private static final int LOG_CONTENT_LENGTH = 1024; - - /** - * Unchecked exception. - * @param message exception message. - * @param cause cause of this exception, may be null - */ - public UncheckedRequestException(String message, IOException cause) { - super(message, cause); - } - - /** - * Rethrow this exception using either its cause, if that is an IOException, or using - * the current exception. - * @throws IOException if the cause of the exception was an IOException. - * @throws UncheckedRequestException if the cause of the exception was not an IOException. - */ - public void rethrow() throws IOException { - if (getCause() instanceof IOException) { - throw (IOException)getCause(); - } else { - throw new IOException(this); - } - } - - /** - * Create a new UncheckedRequestException based on given call. - * - * @param request call request - * @param response call response, may be null - * @param cause exception cause, may be null - * @return new exception - * @throws IOException if the request or response cannot be constructed into a message. - */ - public static UncheckedRequestException fail(Request request, - Response response, IOException cause) throws IOException { - - StringBuilder message = new StringBuilder(128); - message.append("FAILED to transmit message"); - String content; - if (response != null) { - message.append(" (HTTP status code ") - .append(response.code()) - .append(')'); - content = RestClient.responseBody(response); - } else { - content = null; - } - - String requestContent = TopicRequestBody.topicRequestContent(request, LOG_CONTENT_LENGTH); - if (requestContent != null || content != null) { - message.append(':'); - } - - if (requestContent != null) { - message.append("\n ") - .append(requestContent); - } - - if (content != null) { - message.append("\n ") - .append(content); - } - - return new UncheckedRequestException(message.toString(), cause); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt new file mode 100644 index 00000000..a8f597e8 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.Request +import okhttp3.Response +import org.radarbase.producer.rest.RestClient.Companion.bodyString +import org.radarbase.producer.rest.TopicRequestBody.Companion.topicRequestContent +import java.io.IOException + +/** Unchecked exception for failures during request handling. */ +class UncheckedRequestException +/** + * Unchecked exception. + * @param message exception message. + * @param cause cause of this exception, may be null + */ + (message: String?, cause: IOException?) : RuntimeException(message, cause) { + /** + * Rethrow this exception using either its cause, if that is an IOException, or using + * the current exception. + * @throws IOException if the cause of the exception was an IOException. + * @throws UncheckedRequestException if the cause of the exception was not an IOException. + */ + @Throws(IOException::class) + fun rethrow() { + if (cause is IOException) { + throw (cause as IOException?)!! + } else { + throw IOException(this) + } + } + + companion object { + private const val serialVersionUID: Long = 1 + private const val LOG_CONTENT_LENGTH = 1024 + + /** + * Create a new UncheckedRequestException based on given call. + * + * @param request call request + * @param response call response, may be null + * @param cause exception cause, may be null + * @return new exception + * @throws IOException if the request or response cannot be constructed into a message. + */ + @Throws(IOException::class) + fun fail( + request: Request?, + response: Response?, cause: IOException? + ): UncheckedRequestException { + val message = buildString(128) { + append("FAILED to transmit message") + val content: String? = if (response != null) { + append(" (HTTP status code ") + append(response.code) + append(')') + response.bodyString() + } else { + null + } + val requestContent = topicRequestContent(request, LOG_CONTENT_LENGTH) + if (requestContent != null || content != null) { + append(':') + } + if (requestContent != null) { + append("\n ") + append(requestContent) + } + if (content != null) { + append("\n ") + append(content) + } + } + return UncheckedRequestException(message, cause) + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java deleted file mode 100644 index 01d515f7..00000000 --- a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.topic; - -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.specific.SpecificData; -import org.apache.avro.specific.SpecificRecord; - -/** Kafka topic with schema. */ -public class AvroTopic extends KafkaTopic { - private final Schema valueSchema; - private final Schema keySchema; - private final Schema.Type[] valueFieldTypes; - private final Class valueClass; - private final Class keyClass; - - /** - * Kafka topic with Avro schema. - * @param name topic name - * @param keySchema Avro schema for keys - * @param valueSchema Avro schema for values - * @param keyClass Java class for keys - * @param valueClass Java class for values - */ - public AvroTopic(String name, - Schema keySchema, Schema valueSchema, - Class keyClass, Class valueClass) { - super(name); - - if (keySchema == null || valueSchema == null || keyClass == null || valueClass == null) { - throw new IllegalArgumentException("Topic values may not be null"); - } - - this.keySchema = keySchema; - this.valueSchema = valueSchema; - this.valueClass = valueClass; - this.keyClass = keyClass; - - if (valueSchema.getType() == Type.RECORD) { - List fields = valueSchema.getFields(); - this.valueFieldTypes = new Schema.Type[fields.size()]; - for (int i = 0; i < fields.size(); i++) { - valueFieldTypes[i] = fields.get(i).schema().getType(); - } - } else { - this.valueFieldTypes = null; - } - } - - /** Avro schema used for keys. */ - public Schema getKeySchema() { - return keySchema; - } - - /** Avro schema used for values. */ - public Schema getValueSchema() { - return valueSchema; - } - - /** Java class used for keys. */ - public Class getKeyClass() { - return keyClass; - } - - /** Java class used for values. */ - public Class getValueClass() { - return valueClass; - } - - /** - * Tries to construct a new SpecificData instance of the value. - * @return new empty SpecificData class - * @throws ClassCastException Value class is not a SpecificData class - */ - @SuppressWarnings("unchecked") - public V newValueInstance() throws ClassCastException { - return (V)SpecificData.newInstance(valueClass, valueSchema); - } - - public Schema.Type[] getValueFieldTypes() { - return Arrays.copyOf(valueFieldTypes, valueFieldTypes.length); - } - - /** - * Parse an AvroTopic. - * - * @throws IllegalArgumentException if the key_schema or value_schema properties are not valid - * Avro SpecificRecord classes - */ - @SuppressWarnings({"unchecked"}) - public static AvroTopic parse( - String topic, String keySchema, String valueSchema) { - Objects.requireNonNull(topic, "topic needs to be specified"); - K key = parseSpecificRecord(keySchema); - V value = parseSpecificRecord(valueSchema); - return new AvroTopic<>(topic, - key.getSchema(), value.getSchema(), - (Class) key.getClass(), (Class) value.getClass()); - } - - /** - * Parse the schema of a single specific record. - * - * @param schemaClass class name of the SpecificRecord to use - * @param class type to return - * @return Instantiated class of given specific record class - */ - @SuppressWarnings("unchecked") - public static K parseSpecificRecord(String schemaClass) { - try { - Objects.requireNonNull(schemaClass, "schema needs to be specified"); - - Class keyClass = (Class) Class.forName(schemaClass); - Schema keyAvroSchema = (Schema) keyClass - .getMethod("getClassSchema").invoke(null); - // check instantiation - return (K) SpecificData.newInstance(keyClass, keyAvroSchema); - } catch (ClassCastException | ReflectiveOperationException ex) { - throw new IllegalArgumentException("Schema " + schemaClass + " cannot be instantiated", - ex); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!super.equals(o)) { - return false; - } - - AvroTopic topic = (AvroTopic) o; - - return keyClass == topic.getKeyClass() && valueClass == topic.getValueClass(); - } - - @Override - public int hashCode() { - return Objects.hash(getName(), keyClass, valueClass); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt new file mode 100644 index 00000000..443c8243 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.topic + +import org.apache.avro.Schema +import org.apache.avro.specific.SpecificData +import org.apache.avro.specific.SpecificRecord +import java.util.* + +/** + * Kafka topic with Avro schema. + * @param name topic name + * @param keySchema Avro schema for keys + * @param valueSchema Avro schema for values + * @param keyClass Java class for keys + * @param valueClass Java class for values + */ +open class AvroTopic( + name: String, + val keySchema: Schema, + val valueSchema: Schema, + val keyClass: Class, + val valueClass: Class, +) : KafkaTopic(name) { + val valueFieldTypes: Array? = if (valueSchema.type == Schema.Type.RECORD) { + val fields = valueSchema.fields + Array(fields.size) { i -> + fields[i].schema().type + } + } else null + get() = field?.copyOf() + + /** + * Tries to construct a new SpecificData instance of the value. + * @return new empty SpecificData class + * @throws ClassCastException Value class is not a SpecificData class + */ + @Suppress("UNCHECKED_CAST") + @Throws(ClassCastException::class) + fun newValueInstance(): V { + return SpecificData.newInstance(valueClass, valueSchema) as V + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (!super.equals(other)) { + return false + } + other as AvroTopic<*, *> + return keyClass == other.keyClass && valueClass == other.valueClass + } + + override fun hashCode(): Int = name.hashCode() + + companion object { + /** + * Parse an AvroTopic. + * + * @throws IllegalArgumentException if the key_schema or value_schema properties are not valid + * Avro SpecificRecord classes + */ + fun parse( + topic: String, + keySchema: String, + valueSchema: String, + ): AvroTopic { + val key = parseSpecificRecord(keySchema) + val value = parseSpecificRecord(valueSchema) + return AvroTopic( + topic, + key.schema, + value.schema, + key.javaClass, + value.javaClass, + ) + } + + /** + * Parse the schema of a single specific record. + * + * @param schemaClass class name of the SpecificRecord to use + * @param class type to return + * @return Instantiated class of given specific record class + */ + @Suppress("UNCHECKED_CAST") + fun parseSpecificRecord(schemaClass: String): K { + return try { + val keyClass = Class.forName(schemaClass) + val keyAvroSchema = keyClass + .getMethod("getClassSchema").invoke(null) as Schema + // check instantiation + SpecificData.newInstance(keyClass, keyAvroSchema) as K + } catch (ex: ClassCastException) { + throw IllegalArgumentException( + "Schema $schemaClass cannot be instantiated", + ex + ) + } catch (ex: ReflectiveOperationException) { + throw IllegalArgumentException( + "Schema $schemaClass cannot be instantiated", + ex + ) + } + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java deleted file mode 100644 index 61b40e26..00000000 --- a/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.topic; - -import java.util.regex.Pattern; - -/** - * A topic that used by Apache Kafka. - */ -public class KafkaTopic implements Comparable { - private final String name; - private static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("[a-zA-Z][a-zA-Z0-9_]*"); - - /** - * Kafka topic with given name. - * @param name topic name inside the Kafka cluster - * @throws IllegalArgumentException if the topic name is null or is not ASCII-alphanumeric with - * possible underscores. - */ - public KafkaTopic(String name) { - if (name == null) { - throw new IllegalArgumentException("Kafka topic name may not be null"); - } - if (!TOPIC_NAME_PATTERN.matcher(name).matches()) { - throw new IllegalArgumentException("Kafka topic " + name + " is not ASCII-alphanumeric " - + "with possible underscores."); - } - this.name = name; - } - - /** - * Get the topic name. - * @return topic name - */ - public String getName() { - return this.name; - } - - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - KafkaTopic topic = (KafkaTopic) o; - - return name.equals(topic.name); - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName() + "<" + name + ">"; - } - - @Override - public int compareTo(KafkaTopic o) { - return name.compareTo(o.name); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt new file mode 100644 index 00000000..6258261e --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.topic + +/** + * A topic that used by Apache Kafka. + * + * @param name topic name inside the Kafka cluster + * @throws IllegalArgumentException if the topic name is null or is not ASCII-alphanumeric with + * possible underscores. +*/ +open class KafkaTopic( + val name: String +) : Comparable { + init { + require(name.matches(TOPIC_NAME_PATTERN)) { + ("Kafka topic " + name + " is not ASCII-alphanumeric " + + "with possible underscores.") + } + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (other == null || javaClass != other.javaClass) { + return false + } + other as KafkaTopic + return name == other.name + } + + override fun hashCode(): Int = name.hashCode() + + override fun toString(): String = javaClass.simpleName + "<" + name + ">" + + override fun compareTo(other: KafkaTopic): Int = name.compareTo(other.name) + + companion object { + private val TOPIC_NAME_PATTERN = "[a-zA-Z][a-zA-Z0-9_]*".toRegex() + } +} diff --git a/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java deleted file mode 100644 index b675dd19..00000000 --- a/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.topic; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.specific.SpecificRecord; - -/** - * AvroTopic used by sensors. This has additional verification on the schemas that are used compared - * to AvroTopic. - */ -@SuppressWarnings("PMD.UseUtilityClass") -public class SensorTopic extends AvroTopic { - /** - * AvroTopic that does additional validation on the keys and values. - * @param name topic name - * @param keySchema key schema - * @param valueSchema value schema - * @param keyClass actual key class - * @param valueClass actual value class - */ - public SensorTopic(String name, Schema keySchema, Schema valueSchema, - Class keyClass, Class valueClass) { - super(name, keySchema, valueSchema, keyClass, valueClass); - - if (keySchema.getType() != Type.RECORD) { - throw new IllegalArgumentException("Sensors must send records as keys"); - } - if (valueSchema.getType() != Type.RECORD) { - throw new IllegalArgumentException("Sensors must send records as values"); - } - - if (keySchema.getField("projectId") == null) { - throw new IllegalArgumentException("Key schema must have a project ID"); - } - if (keySchema.getField("userId") == null) { - throw new IllegalArgumentException("Key schema must have a user ID"); - } - if (keySchema.getField("sourceId") == null) { - throw new IllegalArgumentException("Key schema must have a source ID"); - } - if (valueSchema.getField("time") == null) { - throw new IllegalArgumentException("Schema must have time as its first field"); - } - if (valueSchema.getField("timeReceived") == null) { - throw new IllegalArgumentException("Schema must have timeReceived as a field"); - } - } - - /** - * Parse a SensorTopic. - * - * @throws IllegalArgumentException if the key_schema or value_schema properties are not valid - * Avro SpecificRecord classes - */ - public static SensorTopic parse( - String topic, String keySchema, String valueSchema) { - AvroTopic parseAvro = AvroTopic.parse(topic, keySchema, valueSchema); - return new SensorTopic<>(parseAvro.getName(), - parseAvro.getKeySchema(), parseAvro.getValueSchema(), - parseAvro.getKeyClass(), parseAvro.getValueClass()); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt new file mode 100644 index 00000000..01b130b8 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.topic + +import org.apache.avro.Schema +import org.apache.avro.specific.SpecificRecord + +/** + * AvroTopic used by sensors. This has additional verification on the schemas that are used compared + * to AvroTopic. + * + * @param name topic name + * @param keySchema key schema + * @param valueSchema value schema + * @param keyClass actual key class + * @param valueClass actual value class +*/ +class SensorTopic( + name: String, + keySchema: Schema, + valueSchema: Schema, + keyClass: Class, + valueClass: Class, +) : AvroTopic(name, keySchema, valueSchema, keyClass, valueClass) { + + init { + require(keySchema.type == Schema.Type.RECORD) { "Sensors must send records as keys" } + require(valueSchema.type == Schema.Type.RECORD) { "Sensors must send records as values" } + requireNotNull(keySchema.getField("projectId")) { "Key schema must have a project ID" } + requireNotNull(keySchema.getField("userId")) { "Key schema must have a user ID" } + requireNotNull(keySchema.getField("sourceId")) { "Key schema must have a source ID" } + requireNotNull(valueSchema.getField("time")) { "Schema must have time as its first field" } + requireNotNull(valueSchema.getField("timeReceived")) { "Schema must have timeReceived as a field" } + } + + companion object { + /** + * Parse a SensorTopic. + * + * @throws IllegalArgumentException if the key_schema or value_schema properties are not valid + * Avro SpecificRecord classes + */ + @JvmStatic + inline fun parse( + topic: String, + keySchema: String, + valueSchema: String, + ): SensorTopic { + val parseAvro = AvroTopic.parse(topic, keySchema, valueSchema) + return SensorTopic( + parseAvro.name, + parseAvro.keySchema, parseAvro.valueSchema, + parseAvro.keyClass, parseAvro.valueClass + ) + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/util/Base64.java b/radar-commons/src/main/java/org/radarbase/util/Base64.java deleted file mode 100644 index 3f23e4a3..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/Base64.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -package org.radarbase.util; - -import static java.nio.charset.StandardCharsets.UTF_8; // Since Android API 19 - -/** - * This class consists exclusively of static methods for obtaining - * encoders and decoders for the Base64 encoding scheme. The - * implementation of this class supports the following types of Base64 - * as specified in - * RFC 4648 and - * RFC 2045. - * - *

Uses "The Base64 Alphabet" as specified in Table 1 of - * RFC 4648 and RFC 2045 for encoding and decoding operation. - * The encoder does not add any line feed (line separator) - * character. The decoder rejects data that contains characters - * outside the base64 alphabet.

- * - *

Unless otherwise noted, passing a {@code null} argument to a - * method of this class will cause a {@link java.lang.NullPointerException - * NullPointerException} to be thrown. - * - * @author Xueming Shen - * @since 1.8 - */ - -@SuppressWarnings("PMD.ClassNamingConventions") -public class Base64 { - - private Base64() { - } - - /** - * Returns a {@link Encoder} that encodes using the - * Basic type base64 encoding scheme. - * - * @return A Base64 encoder. - */ - public static Encoder getEncoder() { - return Encoder.RFC4648; - } - - /** - * This class implements an encoder for encoding byte data using - * the Base64 encoding scheme as specified in RFC 4648 and RFC 2045. - * - *

Instances of {@link Encoder} class are safe for use by - * multiple concurrent threads. - * - *

Unless otherwise noted, passing a {@code null} argument to - * a method of this class will cause a - * {@link java.lang.NullPointerException NullPointerException} to - * be thrown. - * - * @since 1.8 - */ - public static class Encoder { - /** - * This array is a lookup table that translates 6-bit positive integer - * index values into their "Base64 Alphabet" equivalents as specified - * in "Table 1: The Base64 Alphabet" of RFC 2045 (and RFC 4648). - */ - private static final byte[] BASE_64_CHAR = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' - }; - - static final Encoder RFC4648 = new Encoder(); - - private Encoder() { - } - - /** - * Encodes all bytes from the specified byte array into a newly-allocated - * byte array using the {@link Base64} encoding scheme. The returned byte - * array is of the length of the resulting bytes. - * - * @param src - * the byte array to encode - * @return A newly-allocated byte array containing the resulting - * encoded bytes. - */ - public String encode(byte[] src) { - int srcLen = src.length; - byte[] dst = new byte[4 * ((srcLen + 2) / 3)]; - int fullDataLen = srcLen / 3 * 3; - int dstP = 0; - int srcP = 0; - for (; srcP < fullDataLen; srcP += 3) { - int bits = (src[srcP] & 0xff) << 16 - | (src[srcP + 1] & 0xff) << 8 - | (src[srcP + 2] & 0xff); - dst[dstP++] = BASE_64_CHAR[(bits >>> 18) & 0x3f]; - dst[dstP++] = BASE_64_CHAR[(bits >>> 12) & 0x3f]; - dst[dstP++] = BASE_64_CHAR[(bits >>> 6) & 0x3f]; - dst[dstP++] = BASE_64_CHAR[bits & 0x3f]; - } - if (srcP < srcLen) { // 1 or 2 leftover bytes - int b0 = src[srcP++] & 0xff; - dst[dstP++] = BASE_64_CHAR[b0 >> 2]; - if (srcP == srcLen) { - dst[dstP++] = BASE_64_CHAR[(b0 << 4) & 0x3f]; - dst[dstP++] = '='; - } else { - int b1 = src[srcP] & 0xff; - dst[dstP++] = BASE_64_CHAR[(b0 << 4) & 0x3f | (b1 >> 4)]; - dst[dstP++] = BASE_64_CHAR[(b1 << 2) & 0x3f]; - } - dst[dstP] = '='; - } - - return new String(dst, UTF_8); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/Base64.kt b/radar-commons/src/main/java/org/radarbase/util/Base64.kt new file mode 100644 index 00000000..9f78000a --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/Base64.kt @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.radarbase.util + +// Since Android API 19 +/** + * This class consists exclusively of static methods for obtaining + * encoders and decoders for the Base64 encoding scheme. The + * implementation of this class supports the following types of Base64 + * as specified in + * [RFC 4648](http://www.ietf.org/rfc/rfc4648.txt) and + * [RFC 2045](http://www.ietf.org/rfc/rfc2045.txt). + * + * + * Uses "The Base64 Alphabet" as specified in Table 1 of + * RFC 4648 and RFC 2045 for encoding and decoding operation. + * The encoder does not add any line feed (line separator) + * character. The decoder rejects data that contains characters + * outside the base64 alphabet. + * + * + * Unless otherwise noted, passing a `null` argument to a + * method of this class will cause a [ NullPointerException][java.lang.NullPointerException] to be thrown. + * + * Note: needed because it is only included in Android API level 26. + * + * @author Xueming Shen + * @since 1.8 + */ +object Base64Encoder { + /** + * This class implements an encoder for encoding byte data using + * the Base64 encoding scheme as specified in RFC 4648 and RFC 2045. + * + * + * Instances of [Encoder] class are safe for use by + * multiple concurrent threads. + * + * + * Unless otherwise noted, passing a `null` argument to + * a method of this class will cause a + * [NullPointerException][java.lang.NullPointerException] to + * be thrown. + * + * Encodes all bytes from the specified byte array into a newly-allocated + * byte array using the [Base64] encoding scheme. The returned byte + * array is of the length of the resulting bytes. + * + * @param src + * the byte array to encode + * @return A newly-allocated byte array containing the resulting + * encoded bytes. + */ + fun encode(src: ByteArray): String { + val srcLen = src.size + val dst = ByteArray(4 * ((srcLen + 2) / 3)) + val fullDataLen = srcLen / 3 * 3 + var dstP = 0 + var srcP = 0 + while (srcP < fullDataLen) { + val bits = (src[srcP].toInt() and 0xff).shl(16) or + (src[srcP + 1].toInt() and 0xff).shl(8) or + (src[srcP + 2].toInt() and 0xff) + dst[dstP++] = BASE_64_CHAR[bits.ushr(18) and 0x3f] + dst[dstP++] = BASE_64_CHAR[bits.ushr(12) and 0x3f] + dst[dstP++] = BASE_64_CHAR[bits.ushr(6) and 0x3f] + dst[dstP++] = BASE_64_CHAR[bits and 0x3f] + srcP += 3 + } + if (srcP < srcLen) { // 1 or 2 leftover bytes + val b0 = src[srcP++].toInt() and 0xff + dst[dstP++] = BASE_64_CHAR[b0 shr 2] + if (srcP == srcLen) { + dst[dstP++] = BASE_64_CHAR[b0 shl 4 and 0x3f] + dst[dstP++] = '='.code.toByte() + } else { + val b1 = src[srcP].toInt() and 0xff + dst[dstP++] = BASE_64_CHAR[b0 shl 4 and 0x3f or (b1 shr 4)] + dst[dstP++] = BASE_64_CHAR[b1 shl 2 and 0x3f] + } + dst[dstP] = '='.code.toByte() + } + return String(dst) + } + + /** + * This array is a lookup table that translates 6-bit positive integer + * index values into their "Base64 Alphabet" equivalents as specified + * in "Table 1: The Base64 Alphabet" of RFC 2045 (and RFC 4648). + */ + private val BASE_64_CHAR = byteArrayOf( + 'A'.code.toByte(), + 'B'.code.toByte(), + 'C'.code.toByte(), + 'D'.code.toByte(), + 'E'.code.toByte(), + 'F'.code.toByte(), + 'G'.code.toByte(), + 'H'.code.toByte(), + 'I'.code.toByte(), + 'J'.code.toByte(), + 'K'.code.toByte(), + 'L'.code.toByte(), + 'M'.code.toByte(), + 'N'.code.toByte(), + 'O'.code.toByte(), + 'P'.code.toByte(), + 'Q'.code.toByte(), + 'R'.code.toByte(), + 'S'.code.toByte(), + 'T'.code.toByte(), + 'U'.code.toByte(), + 'V'.code.toByte(), + 'W'.code.toByte(), + 'X'.code.toByte(), + 'Y'.code.toByte(), + 'Z'.code.toByte(), + 'a'.code.toByte(), + 'b'.code.toByte(), + 'c'.code.toByte(), + 'd'.code.toByte(), + 'e'.code.toByte(), + 'f'.code.toByte(), + 'g'.code.toByte(), + 'h'.code.toByte(), + 'i'.code.toByte(), + 'j'.code.toByte(), + 'k'.code.toByte(), + 'l'.code.toByte(), + 'm'.code.toByte(), + 'n'.code.toByte(), + 'o'.code.toByte(), + 'p'.code.toByte(), + 'q'.code.toByte(), + 'r'.code.toByte(), + 's'.code.toByte(), + 't'.code.toByte(), + 'u'.code.toByte(), + 'v'.code.toByte(), + 'w'.code.toByte(), + 'x'.code.toByte(), + 'y'.code.toByte(), + 'z'.code.toByte(), + '0'.code.toByte(), + '1'.code.toByte(), + '2'.code.toByte(), + '3'.code.toByte(), + '4'.code.toByte(), + '5'.code.toByte(), + '6'.code.toByte(), + '7'.code.toByte(), + '8'.code.toByte(), + '9'.code.toByte(), + '+'.code.toByte(), + '/'.code.toByte(), + ) +} diff --git a/radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt b/radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt new file mode 100644 index 00000000..0610e99b --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt @@ -0,0 +1,13 @@ +package org.radarbase.util + +import java.time.Duration + +data class CacheConfig( + val validity: Duration, + val timeSource: () -> Long = System::currentTimeMillis, +) { + val currentExpiryTime: Long + get() = timeSource() + validity.toMillis() + + fun isPassed(timeMillis: Long): Boolean = timeMillis < timeSource() +} diff --git a/radar-commons/src/main/java/org/radarbase/util/RestUtils.java b/radar-commons/src/main/java/org/radarbase/util/RestUtils.java deleted file mode 100644 index 6b90181a..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/RestUtils.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -import java.security.GeneralSecurityException; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509TrustManager; -import okhttp3.internal.platform.Platform; -import okhttp3.internal.tls.OkHostnameVerifier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Utility methods and variables for OkHttp initialization. */ -public final class RestUtils { - private static final Logger logger = LoggerFactory.getLogger(RestUtils.class); - - /** OkHttp3 default hostname verifier. */ - public static final HostnameVerifier DEFAULT_HOSTNAME_VERIFIER = OkHostnameVerifier.INSTANCE; - /** OkHttp3 hostname verifier for unsafe connections. */ - public static final HostnameVerifier UNSAFE_HOSTNAME_VERIFIER = (hostname, session) -> true; - - /** Unsafe OkHttp3 trust manager that trusts all certificates. */ - public static final TrustManager[] UNSAFE_TRUST_MANAGER = { - new X509TrustManager() { - @Override - public void checkClientTrusted(java.security.cert.X509Certificate[] chain, - String authType) { - //Nothing to do - } - - @Override - public void checkServerTrusted(java.security.cert.X509Certificate[] chain, - String authType) { - //Nothing to do - } - - @Override - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return new java.security.cert.X509Certificate[]{}; - } - } - }; - - /** Unsafe OkHttp3 SSLSocketFactory that trusts all certificates. */ - public static final SSLSocketFactory UNSAFE_SSL_FACTORY; - - static { - SSLSocketFactory factory; - try { - final SSLContext sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, UNSAFE_TRUST_MANAGER, new java.security.SecureRandom()); - - factory = sslContext.getSocketFactory(); - } catch (NoSuchAlgorithmException | KeyManagementException e) { - logger.error("Failed to initialize unsafe SSL factory", e); - factory = null; - } - UNSAFE_SSL_FACTORY = factory; - } - - - private RestUtils() { - // utility class - } - - /** - * Default OkHttp3 trust manager that trusts all certificates. - * Copied from private method in OkHttpClient. - */ - public static X509TrustManager systemDefaultTrustManager() { - try { - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance( - TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init((KeyStore) null); - TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); - if (trustManagers.length != 1 || !(trustManagers[0] instanceof X509TrustManager)) { - throw new IllegalStateException("Unexpected default trust managers:" - + Arrays.toString(trustManagers)); - } - return (X509TrustManager) trustManagers[0]; - } catch (GeneralSecurityException e) { - throw new IllegalStateException("No System TLS", e); - } - } - - /** - * Default OkHttp3 SSLSocketFactory that trusts all certificates. - * Copied from private method in OkHttpClient. - */ - public static SSLSocketFactory systemDefaultSslSocketFactory(X509TrustManager trustManager) { - try { - SSLContext sslContext = Platform.get().newSSLContext(); - sslContext.init(null, new TrustManager[] { trustManager }, null); - return sslContext.getSocketFactory(); - } catch (GeneralSecurityException e) { - throw new IllegalStateException("No System TLS", e); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/RestUtils.kt b/radar-commons/src/main/java/org/radarbase/util/RestUtils.kt new file mode 100644 index 00000000..816dbdf1 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/RestUtils.kt @@ -0,0 +1,96 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.util + +import okhttp3.internal.platform.Platform +import okhttp3.internal.tls.OkHostnameVerifier +import org.slf4j.LoggerFactory +import java.security.GeneralSecurityException +import java.security.KeyStore +import java.security.SecureRandom +import java.security.cert.X509Certificate +import javax.net.ssl.* + +/** Utility methods and variables for OkHttp initialization. */ +object RestUtils { + private val logger = LoggerFactory.getLogger(RestUtils::class.java) + + /** OkHttp3 default hostname verifier. */ + val DEFAULT_HOSTNAME_VERIFIER: HostnameVerifier = OkHostnameVerifier + + /** OkHttp3 hostname verifier for unsafe connections. */ + val UNSAFE_HOSTNAME_VERIFIER = HostnameVerifier { _, _ -> true } + + /** Unsafe OkHttp3 trust manager that trusts all certificates. */ + val UNSAFE_TRUST_MANAGER = arrayOf( + object : X509TrustManager { + override fun checkClientTrusted(chain: Array, authType: String) = Unit + + override fun checkServerTrusted(chain: Array, authType: String) = Unit + + override fun getAcceptedIssuers(): Array = arrayOf() + } + ) + + /** Unsafe OkHttp3 SSLSocketFactory that trusts all certificates. */ + val UNSAFE_SSL_FACTORY: SSLSocketFactory? + + init { + val factory: SSLSocketFactory? = try { + SSLContext.getInstance("SSL").apply { + init(null, UNSAFE_TRUST_MANAGER, SecureRandom()) + }.socketFactory + } catch (e: GeneralSecurityException) { + logger.error("Failed to initialize unsafe SSL factory", e) + null + } + UNSAFE_SSL_FACTORY = factory + } + + /** + * Default OkHttp3 trust manager that trusts all certificates. + * Copied from private method in OkHttpClient. + */ + fun systemDefaultTrustManager(): X509TrustManager { + return try { + val trustManagerFactory = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm() + ) + trustManagerFactory.init(null as KeyStore?) + val trustManagers = trustManagerFactory.trustManagers + check(trustManagers.size == 1 && trustManagers[0] is X509TrustManager) { + "Unexpected default trust managers:" + trustManagers.contentToString() + } + trustManagers[0] as X509TrustManager + } catch (e: GeneralSecurityException) { + throw IllegalStateException("No System TLS", e) + } + } + + /** + * Default OkHttp3 SSLSocketFactory that trusts all certificates. + * Copied from private method in OkHttpClient. + */ + fun systemDefaultSslSocketFactory(trustManager: X509TrustManager): SSLSocketFactory { + return try { + val sslContext: SSLContext = Platform.get().newSSLContext() + sslContext.init(null, arrayOf(trustManager), null) + sslContext.socketFactory + } catch (e: GeneralSecurityException) { + throw IllegalStateException("No System TLS", e) + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/util/Serialization.java b/radar-commons/src/main/java/org/radarbase/util/Serialization.java deleted file mode 100644 index 2726f21f..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/Serialization.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** Serialization utility class. */ -@SuppressWarnings("PMD.ClassNamingConventions") -public final class Serialization { - - private Serialization() { - // utility class - } - - /** Read a little-endian encoded long from given bytes, starting from startIndex. */ - public static long bytesToLong(byte[] b, int startIndex) { - long result = 0; - for (int i = 0; i < 8; i++) { - result <<= 8; - result |= b[i + startIndex] & 0xFF; - } - return result; - } - - /** Write a long to given bytes with little-endian encoding, starting from startIndex. */ - public static void longToBytes(long value, byte[] b, int startIndex) { - b[startIndex] = (byte)((value >> 56) & 0xFF); - b[startIndex + 1] = (byte)((value >> 48) & 0xFF); - b[startIndex + 2] = (byte)((value >> 40) & 0xFF); - b[startIndex + 3] = (byte)((value >> 32) & 0xFF); - b[startIndex + 4] = (byte)((value >> 24) & 0xFF); - b[startIndex + 5] = (byte)((value >> 16) & 0xFF); - b[startIndex + 6] = (byte)((value >> 8) & 0xFF); - b[startIndex + 7] = (byte)(value & 0xFF); - } - - /** Write an int to given bytes with little-endian encoding, starting from startIndex. */ - public static void intToBytes(int value, byte[] b, int startIndex) { - b[startIndex] = (byte)((value >> 24) & 0xFF); - b[startIndex + 1] = (byte)((value >> 16) & 0xFF); - b[startIndex + 2] = (byte)((value >> 8) & 0xFF); - b[startIndex + 3] = (byte)(value & 0xFF); - } - - /** Read a little-endian encoded int from given bytes, starting from startIndex. */ - public static int bytesToInt(byte[] b, int startIndex) { - int result = 0; - for (int i = 0; i < 4; i++) { - result <<= 8; - result |= b[i + startIndex] & 0xFF; - } - return result; - } - - /** Read a little-endian encoded short from given bytes, starting from startIndex. */ - public static short bytesToShort(byte[] b, int startIndex) { - short result = 0; - for (int i = 0; i < 2; i++) { - result <<= 8; - result |= b[i + startIndex] & 0xFF; - } - return result; - } - - /** - * Convert a boolean to a byte. - * @return -1 if b is null, 1 if b, and 0 if not b - */ - public static byte booleanToByte(Boolean b) { - if (b == null) { - return -1; - } else if (b.equals(Boolean.TRUE)) { - return 1; - } else { - return 0; - } - } - - /** - * Read a boolean from a byte. - * @return null if b == -1, true if b == 1, false otherwise. - */ - public static Boolean byteToBoolean(byte b) { - if (b == -1) { - return null; - } else if (b == 1) { - return Boolean.TRUE; - } else { - return Boolean.FALSE; - } - } - - /** - * Convert a float to a double using its apparent value. This avoids casting to the double - * value closest to the mathematical value of a float. - */ - public static double floatToDouble(float value) { - return Double.parseDouble(Float.toString(value)); - } - - /** - * Copy a stream using a buffer. - * - * @param buffer non-empty, non-null buffer for the copy operations. - * @param in input stream to read data from - * @param out output stream to write data to - * @throws IOException if the streams cannot be read from or written to. - * @throws IllegalArgumentException if the buffer has size 0 - * @throws NullPointerException if buffer, in or out are null. - */ - public static void copyStream(byte[] buffer, InputStream in, OutputStream out) - throws IOException { - if (buffer.length == 0) { - throw new IllegalArgumentException("Cannot copy with empty buffer."); - } - int len = in.read(buffer); - while (len != -1) { - out.write(buffer, 0, len); - len = in.read(buffer); - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/Strings.java b/radar-commons/src/main/java/org/radarbase/util/Strings.java deleted file mode 100644 index e613eb8f..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/Strings.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Iterator; -import java.util.regex.Pattern; - -/** - * String utilities. - */ -@SuppressWarnings("PMD.ClassNamingConventions") -public final class Strings { - private static final Charset UTF_8 = StandardCharsets.UTF_8; - private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray(); - - private Strings() { - // utility class - } - - /** - * For each string, compiles a pattern that checks if it is contained in another string in a - * case-insensitive way. - */ - public static Pattern[] containsPatterns(Collection contains) { - Pattern[] patterns = new Pattern[contains.size()]; - Iterator containsIterator = contains.iterator(); - for (int i = 0; i < patterns.length; i++) { - patterns[i] = containsIgnoreCasePattern(containsIterator.next()); - } - return patterns; - } - - /** - * Compiles a pattern that checks if it is contained in another string in a case-insensitive - * 7way. - */ - public static Pattern containsIgnoreCasePattern(String containsString) { - int flags = Pattern.CASE_INSENSITIVE // case insensitive - | Pattern.LITERAL // do not compile special characters - | Pattern.UNICODE_CASE; // case insensitive even for Unicode (special) characters. - return Pattern.compile(containsString, flags); - } - - /** - * Whether any of the patterns matches given value. - */ - public static boolean findAny(Pattern[] patterns, CharSequence value) { - for (Pattern pattern : patterns) { - if (pattern.matcher(value).find()) { - return true; - } - } - return false; - } - - public static byte[] utf8(String value) { - return value.getBytes(UTF_8); - } - - /** Whether given value is null or empty. */ - public static boolean isNullOrEmpty(String value) { - return value == null || value.isEmpty(); - } - - /** - * Converts given bytes to a hex string. - * @param bytes bytes to read. - * @return String with hex values. - */ - public static String bytesToHex(byte[] bytes) { - char[] hexChars = new char[bytes.length * 2]; - for (int i = 0; i < bytes.length; i++) { - int value = bytes[i] & 0xFF; - hexChars[i * 2] = HEX_ARRAY[value >>> 4]; - hexChars[i * 2 + 1] = HEX_ARRAY[value & 0x0F]; - } - return new String(hexChars); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/Strings.kt b/radar-commons/src/main/java/org/radarbase/util/Strings.kt new file mode 100644 index 00000000..7cee213a --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/Strings.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.util + +/** + * String utilities. + */ +object Strings { + private val HEX_ARRAY = "0123456789ABCDEF".toCharArray() + + /** + * Compiles a pattern that checks if it is contained in another string in a case-insensitive + * way. + */ + fun String.toIgnoreCaseRegex(): Regex = toRegex(setOf( + RegexOption.IGNORE_CASE, + RegexOption.LITERAL + )) + + /** + * Converts given bytes to a hex string. + * @param bytes bytes to read. + * @return String with hex values. + */ + fun ByteArray.toHexString(): String { + val hexChars = CharArray(size * 2) + for (i in indices) { + val value = get(i).toInt() and 0xFF + hexChars[i * 2] = HEX_ARRAY[value ushr 4] + hexChars[i * 2 + 1] = HEX_ARRAY[value and 0x0F] + } + return String(hexChars) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedInt.java b/radar-commons/src/main/java/org/radarbase/util/TimedInt.java deleted file mode 100644 index 108bbc54..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimedInt.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.radarbase.util; - -public class TimedInt implements TimedVariable { - public final int value; - private final long expiry; - - public TimedInt(int value, long validity) { - expiry = System.currentTimeMillis() + validity * 1000L; - this.value = value; - } - - @Override - public boolean isExpired() { - return expiry < System.currentTimeMillis(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TimedInt other = (TimedInt)o; - return value == other.value - && expiry == other.expiry; - } - - @Override - public int hashCode() { - return value; - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt b/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt new file mode 100644 index 00000000..a39989c2 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt @@ -0,0 +1,19 @@ +package org.radarbase.util + +class TimedInt( + val value: Int, + cacheConfig: CacheConfig, +) : TimedVariable(cacheConfig) { + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (other == null || javaClass != other.javaClass) { + return false + } + other as TimedInt + return value == other.value && expiry == other.expiry + } + + override fun hashCode(): Int = value +} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedValue.java b/radar-commons/src/main/java/org/radarbase/util/TimedValue.java deleted file mode 100644 index cf186055..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimedValue.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.radarbase.util; - -import java.util.Objects; - -public class TimedValue implements TimedVariable { - public final T value; - private final long expiry; - - public TimedValue(T value, long validity) { - expiry = System.currentTimeMillis() + validity * 1000L; - this.value = Objects.requireNonNull(value); - } - - @Override - public boolean isExpired() { - return expiry < System.currentTimeMillis(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - TimedValue other = (TimedValue)o; - return value.equals(other.value) - && expiry == other.expiry; - } - - @Override - public int hashCode() { - return value.hashCode(); - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt b/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt new file mode 100644 index 00000000..91806990 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt @@ -0,0 +1,19 @@ +package org.radarbase.util + +class TimedValue( + val value: T, + private val cacheConfig: CacheConfig, +) : TimedVariable(cacheConfig) { + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (other == null || javaClass != other.javaClass) { + return false + } + other as TimedValue<*> + return value == other.value && expiry == other.expiry + } + + override fun hashCode(): Int = value.hashCode() +} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.java b/radar-commons/src/main/java/org/radarbase/util/TimedVariable.java deleted file mode 100644 index c4affa99..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.radarbase.util; - -public interface TimedVariable { - boolean isExpired(); -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt b/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt new file mode 100644 index 00000000..99f3fc9b --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt @@ -0,0 +1,21 @@ +package org.radarbase.util + +open class TimedVariable( + private val cacheConfig: CacheConfig, +) { + protected val expiry: Long = cacheConfig.currentExpiryTime + + val isExpired: Boolean + get() = cacheConfig.isPassed(expiry) + + companion object { + internal fun MutableMap<*, out TimedVariable>.prune() { + val iterator = values.iterator() + for (value in iterator) { + if (value.isExpired) { + iterator.remove() + } + } + } + } +} diff --git a/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java b/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java index e19ec3e5..68cd2c87 100644 --- a/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java +++ b/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java @@ -21,9 +21,9 @@ import java.io.IOException; import org.apache.avro.specific.SpecificData; import org.junit.Test; +import org.radarbase.topic.AvroTopic; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; -import org.radarbase.topic.AvroTopic; /** * Created by nivethika on 24-2-17. diff --git a/radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java b/radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java deleted file mode 100644 index 10cacf6b..00000000 --- a/radar-commons/src/test/java/org/radarbase/data/StringEncoderTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.data; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.junit.Test; - -/** - * Created by nivethika on 24-2-17. - */ -public class StringEncoderTest { - - @Test - public void encodeString() throws IOException { - StringEncoder encoder = new StringEncoder(); - Schema schema = Schema.create(Type.STRING); - - AvroEncoder.AvroWriter keyEncoder = encoder.writer(schema, String.class); - - - byte[] key = keyEncoder.encode("{\"userId\":\"a\",\"sourceId\":\"b\"}"); - assertTrue( new String(key).contains("userId")); - assertTrue( new String(key).contains("sourceId")); - } - -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java deleted file mode 100644 index 8d11e655..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.java +++ /dev/null @@ -1,215 +0,0 @@ -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.avro.Schema; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.io.JsonDecoder; -import org.apache.avro.io.JsonEncoder; -import org.junit.Before; -import org.junit.Test; -import org.radarcns.kafka.ObservationKey; - -public class AvroDataMapperFactoryTest { - private static final Schema MEASUREMENT_KEY_SCHEMA = new Schema.Parser().parse("{"+ - " \"namespace\": \"org.radarcns.key\","+ - " \"type\": \"record\","+ - " \"name\": \"MeasurementKey\","+ - " \"doc\": \"Measurement key in the RADAR-base project\","+ - " \"fields\": ["+ - " {\"name\": \"userId\", \"type\": \"string\", \"doc\": \"user ID\"},"+ - " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}"+ - " ]"+ - "}"); - - private static final Schema INCOMPLETE_MEASUREMENT_KEY_SCHEMA = new Schema.Parser().parse("{"+ - " \"namespace\": \"org.radarcns.key\","+ - " \"type\": \"record\","+ - " \"name\": \"MeasurementKey\","+ - " \"doc\": \"Measurement key in the RADAR-base project\","+ - " \"fields\": ["+ - " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}"+ - " ]"+ - "}"); - - private static final Schema SMALL_ENUM_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}}" - + "]}"); - - private static final Schema LARGE_ENUM_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - +"{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"C\"]}}" - + "]}"); - - private static final Schema UNKNOWN_ENUM_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"UNKNOWN\"]}}" - + "]}"); - - private static final Schema DEFAULT_ENUM_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}" - + "]}"); - - private static final Schema ALL_TYPES_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," - + "{\"name\": \"i\", \"type\": \"int\"}," - + "{\"name\": \"l\", \"type\": \"long\"}," - + "{\"name\": \"d\", \"type\": \"double\"}," - + "{\"name\": \"f\", \"type\": \"float\"}," - + "{\"name\": \"sI\", \"type\": \"string\"}," - + "{\"name\": \"sD\", \"type\": \"string\"}," - + "{\"name\": \"sU\", \"type\": [\"null\", \"string\"]}," - + "{\"name\": \"sUi\", \"type\": [\"null\", \"string\"]}," - + "{\"name\": \"sUe\", \"type\": [\"null\", {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}]}," - + "{\"name\": \"uS\", \"type\": \"string\"}," - + "{\"name\": \"se2\", \"type\": \"string\"}," - + "{\"name\": \"se3\", \"type\": \"string\"}," - + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"int\"}}}," - + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"int\"}}}," - + "{\"name\": \"fS\", \"type\": {\"name\": \"f1\", \"type\":\"fixed\", \"size\": 2}}," - + "{\"name\": \"bS\", \"type\": \"bytes\"}," - + "{\"name\": \"fb\", \"type\": {\"name\": \"f2\",\"type\": \"fixed\", \"size\": 2}}," - + "{\"name\": \"bf\", \"type\": \"bytes\"}," - + "{\"name\": \"bfd\", \"type\": \"bytes\"}," - + "{\"name\": \"unmapped\", \"type\": \"int\"}" - + "]}"); - private static final Schema ALL_TYPES_ALT_SCHEMA = new Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," - + "{\"name\": \"i\", \"type\": \"long\"}," - + "{\"name\": \"l\", \"type\": \"double\"}," - + "{\"name\": \"d\", \"type\": \"float\"}," - + "{\"name\": \"f\", \"type\": \"double\"}," - + "{\"name\": \"sI\", \"type\": \"int\", \"default\": 0}," - + "{\"name\": \"sD\", \"type\": \"double\", \"default\": 0.0}," - + "{\"name\": \"sU\", \"type\": \"string\", \"default\": \"\"}," - + "{\"name\": \"sUi\", \"type\": [\"null\", \"int\"], \"default\":null}," - + "{\"name\": \"sUe\", \"type\": {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," - + "{\"name\": \"uS\", \"type\": [\"null\", \"string\"]}," - + "{\"name\": \"se2\", \"type\": {\"name\": \"SE2\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," - + "{\"name\": \"se3\", \"type\": {\"name\": \"SE3\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," - + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"float\"}}}," - + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"float\"}}}," - + "{\"name\": \"fS\", \"type\": \"string\"}," - + "{\"name\": \"bS\", \"type\": \"string\"}," - + "{\"name\": \"fb\", \"type\": \"bytes\"}," - + "{\"name\": \"bf\", \"type\": {\"name\": \"f3\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}," - + "{\"name\": \"bfd\", \"type\": {\"name\": \"f4\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}" - + "]}"); - private AvroDataMapperFactory factory; - - @Before - public void setUp() { - this.factory = new AvroDataMapperFactory(); - } - - @Test - public void mapRecord() throws SchemaValidationException, IOException { - String actual = doMap(MEASUREMENT_KEY_SCHEMA, ObservationKey.getClassSchema(), - "{\"userId\":\"u\", \"sourceId\": \"s\"}"); - assertEquals("{\"projectId\":null,\"userId\":\"u\",\"sourceId\":\"s\"}", actual); - } - - @Test(expected = SchemaValidationException.class) - public void mapRecordIncomplete() throws SchemaValidationException { - factory.createMapper(INCOMPLETE_MEASUREMENT_KEY_SCHEMA, ObservationKey.getClassSchema(), null); - } - - @Test - public void mapEnumLarger() throws SchemaValidationException, IOException { - String actual = doMap(SMALL_ENUM_SCHEMA, LARGE_ENUM_SCHEMA, "{\"e\":\"A\"}"); - assertEquals("{\"e\":\"A\"}", actual); - } - - @Test(expected = SchemaValidationException.class) - public void mapEnumSmaller() throws SchemaValidationException { - factory.createMapper(LARGE_ENUM_SCHEMA, SMALL_ENUM_SCHEMA, null); - } - - @Test - public void mapEnumSmallerUnknown() throws SchemaValidationException, IOException { - String actual = doMap(LARGE_ENUM_SCHEMA, UNKNOWN_ENUM_SCHEMA, "{\"e\":\"C\"}"); - assertEquals("{\"e\":\"UNKNOWN\"}", actual); - } - - - @Test - public void mapEnumSmallerDefault() throws SchemaValidationException, IOException { - String actual = doMap(LARGE_ENUM_SCHEMA, DEFAULT_ENUM_SCHEMA, "{\"e\":\"C\"}"); - assertEquals("{\"e\":\"A\"}", actual); - } - - @Test - public void mapAll() throws SchemaValidationException, IOException { - String actual = doMap(ALL_TYPES_SCHEMA, ALL_TYPES_ALT_SCHEMA, "{" + - "\"e\":\"A\"," + - "\"i\":1," + - "\"l\":2," + - "\"d\":3.0," + - "\"f\":4.0," + - "\"sI\":\"5\"," + - "\"sD\":\"6.5\"," + - "\"sU\":null," + - "\"sUi\":{\"string\":\"7\"}," + - "\"sUe\":null," + - "\"uS\":\"s\"," + - "\"se2\":\"B\"," + - "\"se3\":\"g\"," + - "\"a\":[1,2]," + - "\"m\":{\"a\":9}," + - "\"fS\":\"ab\"," + - "\"bS\":\"ab\"," + - "\"fb\":\"ab\"," + - "\"bf\":\"ab\"," + - "\"bfd\":\"abc\"," + - "\"unmapped\":10}"); - - assertEquals("{" + - "\"e\":\"A\"," + - "\"i\":1," + - "\"l\":2.0," + - "\"d\":3.0," + - "\"f\":4.0," + - "\"sI\":5," + - "\"sD\":6.5," + - "\"sU\":\"\"," + - "\"sUi\":{\"int\":7}," + - "\"sUe\":\"A\"," + - "\"uS\":{\"string\":\"s\"}," + - "\"se2\":\"B\"," + - "\"se3\":\"A\"," + - "\"a\":[1.0,2.0]," + - "\"m\":{\"a\":9.0}," + - "\"fS\":\"YWI=\"," + - "\"bS\":\"YWI=\"," + - "\"fb\":\"ab\"," + - "\"bf\":\"ab\"," + - "\"bfd\":\"aa\"" + - "}", actual); - } - - private String doMap(Schema from, Schema to, String value) - throws IOException, SchemaValidationException { - AvroDataMapper mapper = factory.createMapper(from, to, null); - - GenericDatumReader reader = new GenericDatumReader<>(from); - JsonDecoder decoder = DecoderFactory.get().jsonDecoder(from, value); - Object readValue = reader.read(null, decoder); - - GenericDatumWriter writer = new GenericDatumWriter<>(to); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - JsonEncoder encoder = EncoderFactory.get().jsonEncoder(to, out); - writer.write(mapper.convertAvro(readValue), encoder); - encoder.flush(); - return out.toString("utf-8"); - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt new file mode 100644 index 00000000..0717c5a9 --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt @@ -0,0 +1,231 @@ +package org.radarbase.producer.rest + +import org.apache.avro.Schema +import org.apache.avro.SchemaValidationException +import org.apache.avro.generic.GenericDatumReader +import org.apache.avro.generic.GenericDatumWriter +import org.apache.avro.io.DecoderFactory +import org.apache.avro.io.EncoderFactory +import org.junit.Assert +import org.junit.Before +import org.junit.Test +import org.radarcns.kafka.ObservationKey +import java.io.ByteArrayOutputStream +import java.io.IOException + +class AvroDataMapperFactoryTest { + private lateinit var factory: AvroDataMapperFactory + @Before + fun setUp() { + factory = AvroDataMapperFactory() + } + + @Test + @Throws(SchemaValidationException::class, IOException::class) + fun mapRecord() { + val actual = doMap( + MEASUREMENT_KEY_SCHEMA, ObservationKey.getClassSchema(), + "{\"userId\":\"u\", \"sourceId\": \"s\"}" + ) + Assert.assertEquals("{\"projectId\":null,\"userId\":\"u\",\"sourceId\":\"s\"}", actual) + } + + @Test(expected = SchemaValidationException::class) + @Throws(SchemaValidationException::class) + fun mapRecordIncomplete() { + factory.createMapper( + INCOMPLETE_MEASUREMENT_KEY_SCHEMA, + ObservationKey.getClassSchema(), + null + ) + } + + @Test + @Throws(SchemaValidationException::class, IOException::class) + fun mapEnumLarger() { + val actual = doMap(SMALL_ENUM_SCHEMA, LARGE_ENUM_SCHEMA, "{\"e\":\"A\"}") + Assert.assertEquals("{\"e\":\"A\"}", actual) + } + + @Test(expected = SchemaValidationException::class) + @Throws(SchemaValidationException::class) + fun mapEnumSmaller() { + factory.createMapper(LARGE_ENUM_SCHEMA, SMALL_ENUM_SCHEMA, null) + } + + @Test + @Throws(SchemaValidationException::class, IOException::class) + fun mapEnumSmallerUnknown() { + val actual = doMap(LARGE_ENUM_SCHEMA, UNKNOWN_ENUM_SCHEMA, "{\"e\":\"C\"}") + Assert.assertEquals("{\"e\":\"UNKNOWN\"}", actual) + } + + @Test + @Throws(SchemaValidationException::class, IOException::class) + fun mapEnumSmallerDefault() { + val actual = doMap(LARGE_ENUM_SCHEMA, DEFAULT_ENUM_SCHEMA, "{\"e\":\"C\"}") + Assert.assertEquals("{\"e\":\"A\"}", actual) + } + + @Test + @Throws(SchemaValidationException::class, IOException::class) + fun mapAll() { + val actual = doMap( + ALL_TYPES_SCHEMA, ALL_TYPES_ALT_SCHEMA, "{" + + "\"e\":\"A\"," + + "\"i\":1," + + "\"l\":2," + + "\"d\":3.0," + + "\"f\":4.0," + + "\"sI\":\"5\"," + + "\"sD\":\"6.5\"," + + "\"sU\":null," + + "\"sUi\":{\"string\":\"7\"}," + + "\"sUe\":null," + + "\"uS\":\"s\"," + + "\"se2\":\"B\"," + + "\"se3\":\"g\"," + + "\"a\":[1,2]," + + "\"m\":{\"a\":9}," + + "\"fS\":\"ab\"," + + "\"bS\":\"ab\"," + + "\"fb\":\"ab\"," + + "\"bf\":\"ab\"," + + "\"bfd\":\"abc\"," + + "\"unmapped\":10}" + ) + Assert.assertEquals( + "{" + + "\"e\":\"A\"," + + "\"i\":1," + + "\"l\":2.0," + + "\"d\":3.0," + + "\"f\":4.0," + + "\"sI\":5," + + "\"sD\":6.5," + + "\"sU\":\"\"," + + "\"sUi\":{\"int\":7}," + + "\"sUe\":\"A\"," + + "\"uS\":{\"string\":\"s\"}," + + "\"se2\":\"B\"," + + "\"se3\":\"A\"," + + "\"a\":[1.0,2.0]," + + "\"m\":{\"a\":9.0}," + + "\"fS\":\"YWI=\"," + + "\"bS\":\"YWI=\"," + + "\"fb\":\"ab\"," + + "\"bf\":\"ab\"," + + "\"bfd\":\"aa\"" + + "}", actual + ) + } + + @Throws(IOException::class, SchemaValidationException::class) + private fun doMap(from: Schema, to: Schema, value: String): String { + val mapper = factory.createMapper(from, to, null) + val reader = GenericDatumReader(from) + val decoder = DecoderFactory.get().jsonDecoder(from, value) + val readValue = reader.read(null, decoder) + val writer = GenericDatumWriter(to) + val out = ByteArrayOutputStream() + val encoder = EncoderFactory.get().jsonEncoder(to, out) + writer.write(mapper.convertAvro(readValue), encoder) + encoder.flush() + return out.toString("utf-8") + } + + companion object { + private val MEASUREMENT_KEY_SCHEMA = Schema.Parser().parse( + "{" + + " \"namespace\": \"org.radarcns.key\"," + + " \"type\": \"record\"," + + " \"name\": \"MeasurementKey\"," + + " \"doc\": \"Measurement key in the RADAR-base project\"," + + " \"fields\": [" + + " {\"name\": \"userId\", \"type\": \"string\", \"doc\": \"user ID\"}," + + " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}" + + " ]" + + "}" + ) + private val INCOMPLETE_MEASUREMENT_KEY_SCHEMA = Schema.Parser().parse( + "{" + + " \"namespace\": \"org.radarcns.key\"," + + " \"type\": \"record\"," + + " \"name\": \"MeasurementKey\"," + + " \"doc\": \"Measurement key in the RADAR-base project\"," + + " \"fields\": [" + + " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}" + + " ]" + + "}" + ) + private val SMALL_ENUM_SCHEMA = Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}}" + + "]}" + ) + private val LARGE_ENUM_SCHEMA = Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"C\"]}}" + + "]}" + ) + private val UNKNOWN_ENUM_SCHEMA = Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"UNKNOWN\"]}}" + + "]}" + ) + private val DEFAULT_ENUM_SCHEMA = Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}" + + "]}" + ) + private val ALL_TYPES_SCHEMA = Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," + + "{\"name\": \"i\", \"type\": \"int\"}," + + "{\"name\": \"l\", \"type\": \"long\"}," + + "{\"name\": \"d\", \"type\": \"double\"}," + + "{\"name\": \"f\", \"type\": \"float\"}," + + "{\"name\": \"sI\", \"type\": \"string\"}," + + "{\"name\": \"sD\", \"type\": \"string\"}," + + "{\"name\": \"sU\", \"type\": [\"null\", \"string\"]}," + + "{\"name\": \"sUi\", \"type\": [\"null\", \"string\"]}," + + "{\"name\": \"sUe\", \"type\": [\"null\", {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}]}," + + "{\"name\": \"uS\", \"type\": \"string\"}," + + "{\"name\": \"se2\", \"type\": \"string\"}," + + "{\"name\": \"se3\", \"type\": \"string\"}," + + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"int\"}}}," + + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"int\"}}}," + + "{\"name\": \"fS\", \"type\": {\"name\": \"f1\", \"type\":\"fixed\", \"size\": 2}}," + + "{\"name\": \"bS\", \"type\": \"bytes\"}," + + "{\"name\": \"fb\", \"type\": {\"name\": \"f2\",\"type\": \"fixed\", \"size\": 2}}," + + "{\"name\": \"bf\", \"type\": \"bytes\"}," + + "{\"name\": \"bfd\", \"type\": \"bytes\"}," + + "{\"name\": \"unmapped\", \"type\": \"int\"}" + + "]}" + ) + private val ALL_TYPES_ALT_SCHEMA = Schema.Parser().parse( + "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," + + "{\"name\": \"i\", \"type\": \"long\"}," + + "{\"name\": \"l\", \"type\": \"double\"}," + + "{\"name\": \"d\", \"type\": \"float\"}," + + "{\"name\": \"f\", \"type\": \"double\"}," + + "{\"name\": \"sI\", \"type\": \"int\", \"default\": 0}," + + "{\"name\": \"sD\", \"type\": \"double\", \"default\": 0.0}," + + "{\"name\": \"sU\", \"type\": \"string\", \"default\": \"\"}," + + "{\"name\": \"sUi\", \"type\": [\"null\", \"int\"], \"default\":null}," + + "{\"name\": \"sUe\", \"type\": {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," + + "{\"name\": \"uS\", \"type\": [\"null\", \"string\"]}," + + "{\"name\": \"se2\", \"type\": {\"name\": \"SE2\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," + + "{\"name\": \"se3\", \"type\": {\"name\": \"SE3\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," + + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"float\"}}}," + + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"float\"}}}," + + "{\"name\": \"fS\", \"type\": \"string\"}," + + "{\"name\": \"bS\", \"type\": \"string\"}," + + "{\"name\": \"fb\", \"type\": \"bytes\"}," + + "{\"name\": \"bf\", \"type\": {\"name\": \"f3\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}," + + "{\"name\": \"bfd\", \"type\": {\"name\": \"f4\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}" + + "]}" + ) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java deleted file mode 100644 index 36e888b8..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertArrayEquals; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.zip.GZIPOutputStream; -import okio.Buffer; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumWriter; -import org.junit.Test; -import org.radarbase.data.AvroRecordData; -import org.radarbase.topic.AvroTopic; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.kafka.RecordSet; -import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; -import org.radarcns.passive.phone.PhoneAcceleration; - -public class BinaryRecordRequestTest { - - // note that positive numbers are multiplied by two in avro binary encoding, due to the - // zig-zag encoding schema used. - // See http://avro.apache.org/docs/1.8.1/spec.html#binary_encoding - private static final byte[] EXPECTED = { - 2, // key version x2 - 4, // value version x2 - 0, // null project ID - 0, // null user ID - 2, (byte)'b', // string length x2, sourceId - 2, // number of records x2 - 40, // number of bytes in the first value x2 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // value - 0 // end of array - }; - - @Test - public void writeToStream() throws SchemaValidationException, IOException { - ObservationKey k = new ObservationKey("test", "a", "b"); - EmpaticaE4BloodVolumePulse v = new EmpaticaE4BloodVolumePulse(0.0, 0.0, - 0.0f); - - AvroTopic t = new AvroTopic<>( - "t", k.getSchema(), v.getSchema(), k.getClass(), v.getClass()); - - BinaryRecordRequest request = new BinaryRecordRequest<>(t); - request.prepare( - new ParsedSchemaMetadata(2, 1, k.getSchema()), - new ParsedSchemaMetadata(4, 2, v.getSchema()), - new AvroRecordData<>(t, k, Collections.singletonList(v))); - - - Buffer buffer = new Buffer(); - request.writeToSink(buffer); - assertArrayEquals(EXPECTED, buffer.readByteArray()); - } - - @Test - public void expectedMatchesRecordSet() throws IOException { - RecordSet recordSet = RecordSet.newBuilder() - .setKeySchemaVersion(1) - .setValueSchemaVersion(2) - .setData(Collections.singletonList(ByteBuffer.wrap(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}))) - .setProjectId(null) - .setUserId(null) - .setSourceId("b") - .build(); - - SpecificDatumWriter writer = new SpecificDatumWriter<>(RecordSet.SCHEMA$); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); - writer.write(recordSet, encoder); - encoder.flush(); - - assertArrayEquals(EXPECTED, out.toByteArray()); - } - - @Test - public void testSize() throws IOException { - - - SpecificDatumWriter writer = new SpecificDatumWriter<>(PhoneAcceleration.SCHEMA$); - - List records = new ArrayList<>(540); - try (InputStream stream = BinaryRecordRequestTest.class.getResourceAsStream("android_phone_acceleration.csv"); - InputStreamReader reader = new InputStreamReader(stream); - BufferedReader br = new BufferedReader(reader)) { - - String line = br.readLine(); - BinaryEncoder encoder = null; - - while (line != null) { - String[] values = line.split(","); - PhoneAcceleration acc = new PhoneAcceleration(Double.parseDouble(values[0]), - Double.parseDouble(values[1]), Float.parseFloat(values[2]), - Float.parseFloat(values[3]), Float.parseFloat(values[4])); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - encoder = EncoderFactory.get().binaryEncoder(out, encoder); - writer.write(acc, encoder); - encoder.flush(); - records.add(ByteBuffer.wrap(out.toByteArray())); - - line = br.readLine(); - } - } - - RecordSet recordSet = RecordSet.newBuilder() - .setKeySchemaVersion(1) - .setValueSchemaVersion(2) - .setData(records) - .setProjectId(null) - .setUserId(null) - .setSourceId("596740ca-5875-4c97-87ab-a08405f36aff") - .build(); - - SpecificDatumWriter recordWriter = new SpecificDatumWriter<>(RecordSet.SCHEMA$); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); - recordWriter.write(recordSet, encoder); - encoder.flush(); - - System.out.println("Size of record set with " + records.size() + " entries: " + out.size()); - - ByteArrayOutputStream gzippedOut = new ByteArrayOutputStream(); - GZIPOutputStream gzipOut = new GZIPOutputStream(gzippedOut); - gzipOut.write(out.size()); - gzipOut.close(); - - System.out.println("Gzipped size of record set with " + records.size() + " entries: " + gzippedOut.size()); - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt new file mode 100644 index 00000000..279b8c7c --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt @@ -0,0 +1,160 @@ +package org.radarbase.producer.rest + +import okio.Buffer +import org.apache.avro.SchemaValidationException +import org.apache.avro.io.BinaryEncoder +import org.apache.avro.io.EncoderFactory +import org.apache.avro.specific.SpecificDatumWriter +import org.junit.Assert +import org.junit.Test +import org.radarbase.data.AvroRecordData +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.ObservationKey +import org.radarcns.kafka.RecordSet +import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse +import org.radarcns.passive.phone.PhoneAcceleration +import java.io.BufferedReader +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.io.InputStreamReader +import java.nio.ByteBuffer +import java.util.zip.GZIPOutputStream + +class BinaryRecordRequestTest { + @Test + @Throws(SchemaValidationException::class, IOException::class) + fun writeToStream() { + val k = ObservationKey("test", "a", "b") + val v = EmpaticaE4BloodVolumePulse( + 0.0, 0.0, + 0.0f + ) + val t = AvroTopic( + "t", k.schema, v.schema, k.javaClass, v.javaClass + ) + val request = BinaryRecordRequest(t) + request.prepare( + ParsedSchemaMetadata(2, 1, k.schema), + ParsedSchemaMetadata(4, 2, v.schema), + AvroRecordData(t, k, listOf(v)) + ) + val buffer = Buffer() + request.writeToSink(buffer) + Assert.assertArrayEquals(EXPECTED, buffer.readByteArray()) + } + + @Test + @Throws(IOException::class) + fun expectedMatchesRecordSet() { + val recordSet = RecordSet.newBuilder() + .setKeySchemaVersion(1) + .setValueSchemaVersion(2) + .setData( + listOf( + ByteBuffer.wrap( + byteArrayOf( + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ) + ) + ) + ) + .setProjectId(null) + .setUserId(null) + .setSourceId("b") + .build() + val writer = SpecificDatumWriter(RecordSet.`SCHEMA$`) + val out = ByteArrayOutputStream() + val encoder = EncoderFactory.get().binaryEncoder(out, null) + writer.write(recordSet, encoder) + encoder.flush() + Assert.assertArrayEquals(EXPECTED, out.toByteArray()) + } + + @Test + @Throws(IOException::class) + fun testSize() { + val writer = SpecificDatumWriter(PhoneAcceleration.`SCHEMA$`) + val records: MutableList = ArrayList(540) + requireNotNull(BinaryRecordRequestTest::class.java.getResourceAsStream("android_phone_acceleration.csv")) + .use { stream -> + InputStreamReader(stream).use { reader -> + BufferedReader(reader).use { br -> + var line = br.readLine() + var encoder: BinaryEncoder? = null + while (line != null) { + val values = line.split(",".toRegex()).dropLastWhile { it.isEmpty() } + .toTypedArray() + val acc = PhoneAcceleration( + values[0].toDouble(), + values[1].toDouble(), + values[2].toFloat(), + values[3].toFloat(), + values[4].toFloat() + ) + val out = ByteArrayOutputStream() + encoder = EncoderFactory.get().binaryEncoder(out, encoder) + writer.write(acc, encoder) + encoder.flush() + records.add(ByteBuffer.wrap(out.toByteArray())) + line = br.readLine() + } + } + } + } + val recordSet = RecordSet.newBuilder() + .setKeySchemaVersion(1) + .setValueSchemaVersion(2) + .setData(records) + .setProjectId(null) + .setUserId(null) + .setSourceId("596740ca-5875-4c97-87ab-a08405f36aff") + .build() + val recordWriter = SpecificDatumWriter(RecordSet.`SCHEMA$`) + val out = ByteArrayOutputStream() + val encoder = EncoderFactory.get().binaryEncoder(out, null) + recordWriter.write(recordSet, encoder) + encoder.flush() + println("Size of record set with " + records.size + " entries: " + out.size()) + val gzippedOut = ByteArrayOutputStream() + val gzipOut = GZIPOutputStream(gzippedOut) + gzipOut.write(out.size()) + gzipOut.close() + println("Gzipped size of record set with " + records.size + " entries: " + gzippedOut.size()) + } + + companion object { + // note that positive numbers are multiplied by two in avro binary encoding, due to the + // zig-zag encoding schema used. + // See http://avro.apache.org/docs/1.8.1/spec.html#binary_encoding + private val EXPECTED = byteArrayOf( + 2, // key version x2 + 4, // value version x2 + 0, // null project ID + 0, // null user ID + 2, 'b'.code.toByte(), // string length x2, sourceId + 2, // number of records x2 + 40, // number of bytes in the first value x2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // value + 0 // end of array + ) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java deleted file mode 100644 index 2dd6f271..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertEquals; -import static org.radarbase.producer.rest.ConnectionState.State.CONNECTED; -import static org.radarbase.producer.rest.ConnectionState.State.DISCONNECTED; -import static org.radarbase.producer.rest.ConnectionState.State.UNKNOWN; - -import java.util.concurrent.TimeUnit; -import org.junit.Test; - -public class ConnectionStateTest { - @Test - public void getState() throws Exception { - ConnectionState state = new ConnectionState(10, TimeUnit.MILLISECONDS); - assertEquals(UNKNOWN, state.getState()); - state.didConnect(); - assertEquals(CONNECTED, state.getState()); - state.didDisconnect(); - assertEquals(DISCONNECTED, state.getState()); - Thread.sleep(15); - assertEquals(DISCONNECTED, state.getState()); - state.didConnect(); - assertEquals(CONNECTED, state.getState()); - Thread.sleep(10); - assertEquals(UNKNOWN, state.getState()); - state.setTimeout(25, TimeUnit.MILLISECONDS); - state.didConnect(); - assertEquals(CONNECTED, state.getState()); - Thread.sleep(10); - assertEquals(CONNECTED, state.getState()); - Thread.sleep(15); - assertEquals(UNKNOWN, state.getState()); - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt new file mode 100644 index 00000000..df66fb9b --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import org.junit.Assert +import org.junit.Test +import java.util.concurrent.TimeUnit + +class ConnectionStateTest { + @Test + fun testState() { + val state = ConnectionState(10, TimeUnit.MILLISECONDS) + Assert.assertEquals(ConnectionState.State.UNKNOWN, state.state) + state.didConnect() + Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) + state.didDisconnect() + Assert.assertEquals(ConnectionState.State.DISCONNECTED, state.state) + Thread.sleep(15) + Assert.assertEquals(ConnectionState.State.DISCONNECTED, state.state) + state.didConnect() + Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) + Thread.sleep(10) + Assert.assertEquals(ConnectionState.State.UNKNOWN, state.state) + state.setTimeout(25, TimeUnit.MILLISECONDS) + state.didConnect() + Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) + Thread.sleep(10) + Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) + Thread.sleep(15) + Assert.assertEquals(ConnectionState.State.UNKNOWN, state.state) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java deleted file mode 100644 index 0c983019..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.net.URL; -import java.util.concurrent.TimeUnit; -import okhttp3.HttpUrl; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.junit.Before; -import org.junit.Test; -import org.radarbase.config.ServerConfig; - -public class RestClientTest { - private MockWebServer server; - private ServerConfig config; - private RestClient client; - - @Before - public void setUp() { - server = new MockWebServer(); - config = new ServerConfig(server.url("base").url()); - client = RestClient.newClient() - .server(config) - .timeout(1, TimeUnit.SECONDS) - .build(); - } - - @Test - public void request() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - Request request = client.requestBuilder("myPath").build(); - try (Response response = client.request(request)) { - assertTrue(response.isSuccessful()); - assertEquals("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", response.body().string()); - } - RecordedRequest recordedRequest = server.takeRequest(); - assertEquals("GET", recordedRequest.getMethod()); - assertEquals("/base/myPath", recordedRequest.getPath()); - } - - @Test - public void requestStringPath() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - try (Response response = client.request("myPath")) { - assertTrue(response.isSuccessful()); - assertEquals("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", response.body().string()); - } - RecordedRequest recordedRequest = server.takeRequest(); - assertEquals("GET", recordedRequest.getMethod()); - assertEquals("/base/myPath", recordedRequest.getPath()); - } - - @Test - public void requestString() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - String response = client.requestString(client.requestBuilder("myPath").build()); - assertEquals("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", response); - RecordedRequest recordedRequest = server.takeRequest(); - assertEquals("GET", recordedRequest.getMethod()); - assertEquals("/base/myPath", recordedRequest.getPath()); - } - - @Test(expected = RestException.class) - public void requestStringEmpty() throws Exception { - server.enqueue(new MockResponse().setResponseCode(500)); - client.requestString(client.requestBuilder("myPath").build()); - } - - @Test - public void requestBuilder() throws Exception { - Request.Builder builder = client.requestBuilder("myPath"); - Request request = builder.build(); - assertEquals(request.url(), HttpUrl.get(new URL(config.getUrl(), "myPath"))); - } - - @Test - public void getRelativeUrl() throws Exception { - HttpUrl url = client.getRelativeUrl("myPath"); - assertEquals(server.getHostName(), url.host()); - assertEquals(server.getPort(), url.port()); - assertEquals("http", url.scheme()); - assertEquals("/base/myPath", url.encodedPath()); - } -} \ No newline at end of file diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt new file mode 100644 index 00000000..c91e7e9c --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt @@ -0,0 +1,110 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Request +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.Assert +import org.junit.Before +import org.junit.Test +import org.radarbase.config.ServerConfig +import java.net.URL +import java.util.concurrent.TimeUnit + +class RestClientTest { + private lateinit var server: MockWebServer + private lateinit var config: ServerConfig + private lateinit var client: RestClient + @Before + fun setUp() { + server = MockWebServer() + config = ServerConfig(server.url("base").toUrl()) + client = RestClient.newRestClient { + server = config + timeout(1, TimeUnit.SECONDS) + } + } + + @Test + @Throws(Exception::class) + fun request() { + server.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + val request = client.buildRequest("myPath") + client.request(request).use { response -> + Assert.assertTrue(response.isSuccessful) + Assert.assertEquals( + "{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", + response.body!!.string() + ) + } + val recordedRequest = server.takeRequest() + Assert.assertEquals("GET", recordedRequest.method) + Assert.assertEquals("/base/myPath", recordedRequest.path) + } + + @Test + @Throws(Exception::class) + fun requestStringPath() { + server.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + client.request("myPath").use { response -> + Assert.assertTrue(response.isSuccessful) + Assert.assertEquals( + "{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", + response.body!!.string() + ) + } + val recordedRequest = server.takeRequest() + Assert.assertEquals("GET", recordedRequest.method) + Assert.assertEquals("/base/myPath", recordedRequest.path) + } + + @Test + @Throws(Exception::class) + fun requestString() { + server.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + val response = client.requestString(client.buildRequest("myPath")) + Assert.assertEquals("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", response) + val recordedRequest = server.takeRequest() + Assert.assertEquals("GET", recordedRequest.method) + Assert.assertEquals("/base/myPath", recordedRequest.path) + } + + @Test(expected = RestException::class) + @Throws(Exception::class) + fun requestStringEmpty() { + server.enqueue(MockResponse().setResponseCode(500)) + client.requestString(client.buildRequest("myPath")) + } + + @Test + @Throws(Exception::class) + fun requestBuilder() { + val request: Request = client.buildRequest("myPath") + Assert.assertEquals(request.url, URL(config.url, "myPath").toHttpUrlOrNull()) + } + + @Throws(Exception::class) + @Test + fun testRelativeUrl() { + val url = client.relativeUrl("myPath") + Assert.assertEquals(server.hostName, url.host) + Assert.assertEquals(server.port.toLong(), url.port.toLong()) + Assert.assertEquals("http", url.scheme) + Assert.assertEquals("/base/myPath", url.encodedPath) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java deleted file mode 100644 index b244f41b..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.node.JsonNodeType; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.zip.GZIPInputStream; -import okhttp3.Headers; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.apache.avro.Schema; -import org.apache.avro.SchemaValidationException; -import org.json.JSONException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.radarbase.config.ServerConfig; -import org.radarbase.data.AvroRecordData; -import org.radarbase.producer.AuthenticationException; -import org.radarbase.producer.KafkaTopicSender; -import org.radarbase.topic.AvroTopic; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.passive.phone.PhoneLight; - -public class RestSenderTest { - private static final JsonFactory FACTORY = new JsonFactory(); - private static final ObjectReader READER = new ObjectMapper(FACTORY).reader(); - private SchemaRetriever retriever; - private RestSender sender; - - @Rule - public MockWebServer webServer = new MockWebServer(); - - @Before - public void setUp() { - this.retriever = mock(SchemaRetriever.class); - - ServerConfig config = new ServerConfig(webServer.url("/").url()); - - RestClient client = RestClient.newClient() - .server(config) - .build(); - - this.sender = new RestSender.Builder() - .httpClient(client) - .schemaRetriever(retriever) - .build(); - } - - @Test - public void sender() throws Exception { - Schema keySchema = ObservationKey.getClassSchema(); - Schema valueSchema = PhoneLight.getClassSchema(); - AvroTopic topic = new AvroTopic<>("test", - keySchema, valueSchema, ObservationKey.class, PhoneLight.class); - Headers headers = new Headers.Builder() - .add("Cookie: ab") - .add("Cookie: bc") - .build(); - sender.setHeaders(headers); - KafkaTopicSender topicSender = sender.sender(topic); - - ObservationKey key = new ObservationKey("test","a", "b"); - PhoneLight value = new PhoneLight(0.1, 0.2, 0.3f); - ParsedSchemaMetadata keySchemaMetadata = new ParsedSchemaMetadata(10, 2, keySchema); - ParsedSchemaMetadata valueSchemaMetadata = new ParsedSchemaMetadata(10, 2, valueSchema); - - when(retriever - .getOrSetSchemaMetadata("test", false, keySchema, -1)) - .thenReturn(keySchemaMetadata); - when(retriever - .getOrSetSchemaMetadata("test", true, valueSchema, -1)) - .thenReturn(valueSchemaMetadata); - - webServer.enqueue(new MockResponse() - .setHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"offset\": 100}")); - - topicSender.send(key, value); - - verify(retriever, times(1)) - .getOrSetSchemaMetadata("test", false, keySchema, -1); - verify(retriever, times(1)) - .getOrSetSchemaMetadata("test", true, valueSchema, -1); - - RecordedRequest request = webServer.takeRequest(); - assertEquals("/topics/test", request.getPath()); - JsonNode body = READER.readTree(request.getBody().inputStream()); - assertEquals(10, body.get("key_schema_id").asInt()); - assertEquals(10, body.get("value_schema_id").asInt()); - JsonNode records = body.get("records"); - assertEquals(JsonNodeType.ARRAY, records.getNodeType()); - assertEquals(1, records.size()); - checkChildren(records); - Headers receivedHeaders = request.getHeaders(); - assertEquals(Arrays.asList("ab", "bc"), receivedHeaders.values("Cookie")); - } - - @Test - public void sendTwo() throws Exception { - Schema keySchema = ObservationKey.getClassSchema(); - Schema valueSchema = PhoneLight.getClassSchema(); - AvroTopic topic = new AvroTopic<>("test", - keySchema, valueSchema, ObservationKey.class, PhoneLight.class); - KafkaTopicSender topicSender = sender.sender(topic); - - ObservationKey key = new ObservationKey("test", "a", "b"); - PhoneLight value = new PhoneLight(0.1, 0.2, 0.3f); - ParsedSchemaMetadata keySchemaMetadata = new ParsedSchemaMetadata(10, 2, keySchema); - ParsedSchemaMetadata valueSchemaMetadata = new ParsedSchemaMetadata(10, 2, valueSchema); - - when(retriever - .getOrSetSchemaMetadata("test", false, keySchema, -1)) - .thenReturn(keySchemaMetadata); - when(retriever - .getOrSetSchemaMetadata("test", true, valueSchema, -1)) - .thenReturn(valueSchemaMetadata); - - webServer.enqueue(new MockResponse() - .setHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"offset\": 100}")); - - topicSender.send(new AvroRecordData<>(topic, key, Arrays.asList(value, value))); - - verify(retriever, times(1)) - .getOrSetSchemaMetadata("test", false, keySchema, -1); - verify(retriever, times(1)) - .getOrSetSchemaMetadata("test", true, valueSchema, -1); - - RecordedRequest request = webServer.takeRequest(); - assertEquals("/topics/test", request.getPath()); - JsonNode body = READER.readTree(request.getBody().inputStream()); - assertEquals(10, body.get("key_schema_id").asInt()); - assertEquals(10, body.get("value_schema_id").asInt()); - JsonNode records = body.get("records"); - assertEquals(JsonNodeType.ARRAY, records.getNodeType()); - assertEquals(2, records.size()); - checkChildren(records); - } - - @Test - public void resetConnection() throws Exception { - int n_requests = 0; - - webServer.enqueue(new MockResponse().setResponseCode(500)); - assertFalse(sender.isConnected()); - assertEquals(++n_requests, webServer.getRequestCount()); - RecordedRequest request = webServer.takeRequest(); - assertEquals("/", request.getPath()); - assertEquals("HEAD", request.getMethod()); - webServer.enqueue(new MockResponse().setResponseCode(500)); - assertFalse(sender.resetConnection()); - assertEquals(++n_requests, webServer.getRequestCount()); - request = webServer.takeRequest(); - assertEquals("/", request.getPath()); - assertEquals("HEAD", request.getMethod()); - webServer.enqueue(new MockResponse()); - assertFalse(sender.isConnected()); - assertEquals(n_requests, webServer.getRequestCount()); - assertTrue(sender.resetConnection()); - assertEquals(++n_requests, webServer.getRequestCount()); - request = webServer.takeRequest(); - assertEquals("/", request.getPath()); - assertEquals("HEAD", request.getMethod()); - } - - @Test - public void resetConnectionUnauthorized() throws Exception { - webServer.enqueue(new MockResponse().setResponseCode(401)); - try { - sender.isConnected(); - fail("Authentication exception expected"); - } catch (AuthenticationException ex) { - // success - } - try { - sender.isConnected(); - fail("Authentication exception expected"); - } catch (AuthenticationException ex) { - // success - } - webServer.enqueue(new MockResponse().setResponseCode(401)); - try { - sender.resetConnection(); - fail("Authentication exception expected"); - } catch (AuthenticationException ex) { - assertEquals(2, webServer.getRequestCount()); - // success - } - webServer.enqueue(new MockResponse().setResponseCode(200)); - try { - assertTrue(sender.resetConnection()); - } catch (AuthenticationException ex) { - assertEquals(3, webServer.getRequestCount()); - fail("Unexpected authentication failure"); - } - } - - @Test - public void withCompression() - throws IOException, InterruptedException, SchemaValidationException, JSONException { - sender.setCompression(true); - webServer.enqueue(new MockResponse() - .setHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"offset\": 100}")); - Schema keySchema = ObservationKey.getClassSchema(); - Schema valueSchema = PhoneLight.getClassSchema(); - AvroTopic topic = new AvroTopic<>("test", - keySchema, valueSchema, ObservationKey.class, PhoneLight.class); - KafkaTopicSender topicSender = sender.sender(topic); - - ObservationKey key = new ObservationKey("test", "a", "b"); - PhoneLight value = new PhoneLight(0.1, 0.2, 0.3f); - ParsedSchemaMetadata keySchemaMetadata = new ParsedSchemaMetadata(10, 2, keySchema); - ParsedSchemaMetadata valueSchemaMetadata = new ParsedSchemaMetadata(10, 2, valueSchema); - - when(retriever - .getOrSetSchemaMetadata("test", false, keySchema, -1)) - .thenReturn(keySchemaMetadata); - when(retriever - .getOrSetSchemaMetadata("test", true, valueSchema, -1)) - .thenReturn(valueSchemaMetadata); - - topicSender.send(key, value); - - RecordedRequest request = webServer.takeRequest(); - assertEquals("gzip", request.getHeader("Content-Encoding")); - - try (InputStream in = request.getBody().inputStream(); - GZIPInputStream gzipIn = new GZIPInputStream(in)) { - JsonNode body = READER.readTree(gzipIn); - assertEquals(10, body.get("key_schema_id").asInt()); - assertEquals(10, body.get("value_schema_id").asInt()); - JsonNode records = body.get("records"); - assertEquals(JsonNodeType.ARRAY, records.getNodeType()); - assertEquals(1, records.size()); - checkChildren(records); - } - } - - private static void checkChildren(JsonNode records) { - for (JsonNode child : records) { - JsonNode jsonKey = child.get("key"); - assertEquals(JsonNodeType.OBJECT, jsonKey.getNodeType()); - assertEquals("a", jsonKey.get("userId").asText()); - assertEquals("b", jsonKey.get("sourceId").asText()); - JsonNode jsonValue = child.get("value"); - assertEquals(JsonNodeType.OBJECT, jsonValue.getNodeType()); - assertEquals(0.1, jsonValue.get("time").asDouble(), 0); - assertEquals(0.2, jsonValue.get("timeReceived").asDouble(), 0); - assertEquals(0.3f, (float)jsonValue.get("light").asDouble(), 0); - } - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt new file mode 100644 index 00000000..797175db --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt @@ -0,0 +1,282 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeType +import okhttp3.Headers.Companion.headersOf +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.apache.avro.SchemaValidationException +import org.json.JSONException +import org.junit.Assert +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.Mockito +import org.radarbase.config.ServerConfig +import org.radarbase.data.AvroRecordData +import org.radarbase.producer.AuthenticationException +import org.radarbase.producer.rest.RestSender.Companion.restSender +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.ObservationKey +import org.radarcns.passive.phone.PhoneLight +import java.io.IOException +import java.util.* +import java.util.zip.GZIPInputStream + +class RestSenderTest { + private lateinit var retriever: SchemaRetriever + private lateinit var sender: RestSender + + @Rule + @JvmField + var webServer = MockWebServer() + + @Before + fun setUp() { + retriever = Mockito.mock(SchemaRetriever::class.java) + sender = restSender { + httpClient = RestClient.newRestClient { + server = ServerConfig(webServer.url("/").toUrl()) + } + schemaRetriever = retriever + } + } + + @Test + @Throws(Exception::class) + fun sender() { + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + sender.headers = headersOf("Cookie", "ab", "Cookie", "bc") + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + Mockito.`when`( + retriever + .getOrSetSchemaMetadata("test", false, keySchema, -1) + ) + .thenReturn(keySchemaMetadata) + Mockito.`when`( + retriever + .getOrSetSchemaMetadata("test", true, valueSchema, -1) + ) + .thenReturn(valueSchemaMetadata) + webServer.enqueue( + MockResponse() + .setHeader("Content-Type", "application/json; charset=utf-8") + .setBody("{\"offset\": 100}") + ) + topicSender.send(key, value) + Mockito.verify(retriever, Mockito.times(1)) + .getOrSetSchemaMetadata("test", false, keySchema, -1) + Mockito.verify(retriever, Mockito.times(1)) + .getOrSetSchemaMetadata("test", true, valueSchema, -1) + val request = webServer.takeRequest() + Assert.assertEquals("/topics/test", request.path) + val body = READER.readTree(request.body.inputStream()) + Assert.assertEquals(10, body["key_schema_id"].asInt().toLong()) + Assert.assertEquals(10, body["value_schema_id"].asInt().toLong()) + val records = body["records"] + Assert.assertEquals(JsonNodeType.ARRAY, records.nodeType) + Assert.assertEquals(1, records.size().toLong()) + checkChildren(records) + val receivedHeaders = request.headers + Assert.assertEquals(listOf("ab", "bc"), receivedHeaders.values("Cookie")) + } + + @Test + @Throws(Exception::class) + fun sendTwo() { + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + Mockito.`when`( + retriever + .getOrSetSchemaMetadata("test", false, keySchema, -1) + ) + .thenReturn(keySchemaMetadata) + Mockito.`when`( + retriever + .getOrSetSchemaMetadata("test", true, valueSchema, -1) + ) + .thenReturn(valueSchemaMetadata) + webServer.enqueue( + MockResponse() + .setHeader("Content-Type", "application/json; charset=utf-8") + .setBody("{\"offset\": 100}") + ) + topicSender.send(AvroRecordData(topic, key, Arrays.asList(value, value))) + Mockito.verify(retriever, Mockito.times(1)) + .getOrSetSchemaMetadata("test", false, keySchema, -1) + Mockito.verify(retriever, Mockito.times(1)) + .getOrSetSchemaMetadata("test", true, valueSchema, -1) + val request = webServer.takeRequest() + Assert.assertEquals("/topics/test", request.path) + val body = READER.readTree(request.body.inputStream()) + Assert.assertEquals(10, body["key_schema_id"].asInt().toLong()) + Assert.assertEquals(10, body["value_schema_id"].asInt().toLong()) + val records = body["records"] + Assert.assertEquals(JsonNodeType.ARRAY, records.nodeType) + Assert.assertEquals(2, records.size().toLong()) + checkChildren(records) + } + + @Test + @Throws(Exception::class) + fun resetConnection() { + var nRequests = 0 + webServer.enqueue(MockResponse().setResponseCode(500)) + Assert.assertFalse(sender.isConnected) + Assert.assertEquals(++nRequests, webServer.requestCount) + var request = webServer.takeRequest() + Assert.assertEquals("/", request.path) + Assert.assertEquals("HEAD", request.method) + webServer.enqueue(MockResponse().setResponseCode(500)) + Assert.assertFalse(sender.resetConnection()) + Assert.assertEquals(++nRequests, webServer.requestCount) + request = webServer.takeRequest() + Assert.assertEquals("/", request.path) + Assert.assertEquals("HEAD", request.method) + webServer.enqueue(MockResponse()) + Assert.assertFalse(sender.isConnected) + Assert.assertEquals(nRequests, webServer.requestCount) + Assert.assertTrue(sender.resetConnection()) + Assert.assertEquals(++nRequests, webServer.requestCount) + request = webServer.takeRequest() + Assert.assertEquals("/", request.path) + Assert.assertEquals("HEAD", request.method) + } + + @Test + @Throws(Exception::class) + fun resetConnectionUnauthorized() { + webServer.enqueue(MockResponse().setResponseCode(401)) + try { + sender.isConnected + Assert.fail("Authentication exception expected") + } catch (ex: AuthenticationException) { + // success + } + try { + sender.isConnected + Assert.fail("Authentication exception expected") + } catch (ex: AuthenticationException) { + // success + } + webServer.enqueue(MockResponse().setResponseCode(401)) + try { + sender.resetConnection() + Assert.fail("Authentication exception expected") + } catch (ex: AuthenticationException) { + Assert.assertEquals(2, webServer.requestCount.toLong()) + // success + } + webServer.enqueue(MockResponse().setResponseCode(200)) + try { + Assert.assertTrue(sender.resetConnection()) + } catch (ex: AuthenticationException) { + Assert.assertEquals(3, webServer.requestCount.toLong()) + Assert.fail("Unexpected authentication failure") + } + } + + @Test + @Throws( + IOException::class, + InterruptedException::class, + SchemaValidationException::class, + JSONException::class + ) + fun withCompression() { + sender.setCompression(true) + webServer.enqueue( + MockResponse() + .setHeader("Content-Type", "application/json; charset=utf-8") + .setBody("{\"offset\": 100}") + ) + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + Mockito.`when`( + retriever + .getOrSetSchemaMetadata("test", false, keySchema, -1) + ) + .thenReturn(keySchemaMetadata) + Mockito.`when`( + retriever + .getOrSetSchemaMetadata("test", true, valueSchema, -1) + ) + .thenReturn(valueSchemaMetadata) + topicSender.send(key, value) + val request = webServer.takeRequest() + Assert.assertEquals("gzip", request.getHeader("Content-Encoding")) + request.body.inputStream().use { `in` -> + GZIPInputStream(`in`).use { gzipIn -> + val body = READER.readTree(gzipIn) + Assert.assertEquals(10, body["key_schema_id"].asInt().toLong()) + Assert.assertEquals(10, body["value_schema_id"].asInt().toLong()) + val records = body["records"] + Assert.assertEquals(JsonNodeType.ARRAY, records.nodeType) + Assert.assertEquals(1, records.size().toLong()) + checkChildren(records) + } + } + } + + companion object { + private val FACTORY = JsonFactory() + private val READER = ObjectMapper(FACTORY).reader() + private fun checkChildren(records: JsonNode) { + for (child in records) { + val jsonKey = child["key"] + Assert.assertEquals(JsonNodeType.OBJECT, jsonKey.nodeType) + Assert.assertEquals("a", jsonKey["userId"].asText()) + Assert.assertEquals("b", jsonKey["sourceId"].asText()) + val jsonValue = child["value"] + Assert.assertEquals(JsonNodeType.OBJECT, jsonValue.nodeType) + Assert.assertEquals(0.1, jsonValue["time"].asDouble(), 0.0) + Assert.assertEquals(0.2, jsonValue["timeReceived"].asDouble(), 0.0) + Assert.assertEquals(0.3f, jsonValue["light"].asDouble().toFloat(), 0f) + } + } + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.java deleted file mode 100644 index fd10e838..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.radarbase.config.ServerConfig; - -public class SchemaRestClientTest { - private MockWebServer server; - private SchemaRestClient retriever; - - @Before - public void setUp() { - server = new MockWebServer(); - ServerConfig config = new ServerConfig(); - config.setProtocol("http"); - config.setHost(server.getHostName()); - config.setPort(server.getPort()); - config.setPath("base"); - retriever = new SchemaRestClient(RestClient.global() - .server(Objects.requireNonNull(config)) - .timeout(1L, TimeUnit.SECONDS) - .build()); - } - - @After - public void tearDown() throws IOException { - server.close(); - } - - @Test - public void retrieveSchemaMetadata() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - ParsedSchemaMetadata metadata = retriever.retrieveSchemaMetadata("bla-value", -1); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Integer.valueOf(2), metadata.getVersion()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - assertEquals("/base/subjects/bla-value/versions/latest", server.takeRequest().getPath()); - } - - - @Test - public void retrieveSchemaMetadataVersion() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - ParsedSchemaMetadata metadata = retriever.retrieveSchemaMetadata("bla-value", 2); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Integer.valueOf(2), metadata.getVersion()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - assertEquals("/base/subjects/bla-value/versions/2", server.takeRequest().getPath()); - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt new file mode 100644 index 00000000..06f3dffb --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.apache.avro.Schema +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.radarbase.config.ServerConfig +import java.io.IOException +import java.util.* +import java.util.concurrent.TimeUnit + +class SchemaRestClientTest { + private lateinit var mockServer: MockWebServer + private lateinit var retriever: SchemaRestClient + @Before + fun setUp() { + mockServer = MockWebServer() + retriever = SchemaRestClient( + RestClient.globalRestClient { + server = ServerConfig("http://${mockServer.hostName}:${mockServer.port}/base") + timeout(1L, TimeUnit.SECONDS) + } + ) + } + + @After + @Throws(IOException::class) + fun tearDown() { + mockServer.close() + } + + @Test + @Throws(Exception::class) + fun retrieveSchemaMetadata() { + mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + val (id, version, schema) = retriever.retrieveSchemaMetadata("bla-value", -1) + assertEquals(10, id) + assertEquals(2, version) + assertEquals(Schema.create(Schema.Type.STRING), schema) + assertEquals("/base/subjects/bla-value/versions/latest", mockServer.takeRequest().path) + } + + @Test + @Throws(Exception::class) + fun retrieveSchemaMetadataVersion() { + mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + val (id, version, schema) = retriever.retrieveSchemaMetadata("bla-value", 2) + assertEquals(10, id) + assertEquals(2, version) + assertEquals(Schema.create(Schema.Type.STRING), schema) + assertEquals("/base/subjects/bla-value/versions/2", mockServer.takeRequest().path) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java deleted file mode 100644 index d78f9211..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.producer.rest; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.radarbase.config.ServerConfig; - -public class SchemaRetrieverTest { - private MockWebServer server; - private SchemaRetriever retriever; - - @Before - public void setUp() { - server = new MockWebServer(); - ServerConfig config = new ServerConfig(); - config.setProtocol("http"); - config.setHost(server.getHostName()); - config.setPort(server.getPort()); - config.setPath("base"); - retriever = new SchemaRetriever(config, 1L); - } - - @After - public void tearDown() throws IOException { - server.close(); - } - - @Test - public void subject() { - assertEquals("bla-value", SchemaRetriever.subject("bla", true)); - assertEquals("bla-key", SchemaRetriever.subject("bla", false)); - } - - @Test - public void getSchemaMetadata() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - ParsedSchemaMetadata metadata = retriever.getBySubjectAndVersion("bla", true, 2); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Integer.valueOf(2), metadata.getVersion()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - assertEquals("/base/subjects/bla-value/versions/2", server.takeRequest().getPath()); - - // Already queried schema is cached and does not need another request - ParsedSchemaMetadata metadata2 = retriever.getBySubjectAndVersion("bla", true, 2); - assertEquals(Integer.valueOf(10), metadata2.getId()); - assertEquals(Integer.valueOf(2), metadata2.getVersion()); - assertEquals(Schema.create(Schema.Type.STRING), metadata2.getSchema()); - assertEquals(1, server.getRequestCount()); - - // Not yet queried schema needs a new request, so if the server does not respond, an - // IOException is thrown. - server.enqueue(new MockResponse().setResponseCode(500)); - assertThrows(IOException.class, () -> retriever.getBySubjectAndVersion("bla", false, 2)); - } - - @Test - public void addSchemaMetadata() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10}")); - int id = retriever.addSchema("bla", true, Schema.create(Schema.Type.STRING)); - assertEquals(10, id); - - assertEquals(1, server.getRequestCount()); - RecordedRequest request = server.takeRequest(); - assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.getBody().readUtf8()); - - List schemaFields = Collections.singletonList( - new Field("a", Schema.create(Schema.Type.INT), "that a", 10)); - - Schema record = Schema.createRecord("C", "that C", "org.radarcns", false, schemaFields); - server.enqueue(new MockResponse().setBody("{\"id\":11}")); - id = retriever.addSchema("bla", true, record); - assertEquals(11, id); - request = server.takeRequest(); - assertEquals("{\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"C\\\",\\\"namespace\\\":\\\"org.radarcns\\\",\\\"doc\\\":\\\"that C\\\",\\\"fields\\\":[{\\\"name\\\":\\\"a\\\",\\\"type\\\":\\\"int\\\",\\\"doc\\\":\\\"that a\\\",\\\"default\\\":10}]}\"}", request.getBody().readUtf8()); - } - - @Test - public void getOrSetSchemaMetadataSet() throws Exception { - server.enqueue(new MockResponse().setResponseCode(404)); - server.enqueue(new MockResponse().setBody("{\"id\":10}")); - server.enqueue(new MockResponse().setBody("{\"id\":10, \"version\": 2}")); - ParsedSchemaMetadata metadata = retriever.getOrSetSchemaMetadata("bla", true, Schema.create(Schema.Type.STRING), -1); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - - assertEquals(3, server.getRequestCount()); - server.takeRequest(); - RecordedRequest request = server.takeRequest(); - assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.getBody().readUtf8()); - assertEquals("/base/subjects/bla-value/versions", request.getPath()); - - metadata = retriever.getOrSetSchemaMetadata("bla", true, Schema.create(Schema.Type.STRING), -1); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - } - - @Test - public void getOrSetSchemaMetadataGet() throws Exception { - server.enqueue(new MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")); - ParsedSchemaMetadata metadata = retriever.getOrSetSchemaMetadata("bla", true, Schema.create(Schema.Type.STRING), 2); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Integer.valueOf(2), metadata.getVersion()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - - assertEquals(1, server.getRequestCount()); - RecordedRequest request = server.takeRequest(); - assertEquals("/base/subjects/bla-value/versions/2", request.getPath()); - - metadata = retriever.getOrSetSchemaMetadata("bla", true, Schema.create(Schema.Type.STRING), 2); - assertEquals(Integer.valueOf(10), metadata.getId()); - assertEquals(Schema.create(Schema.Type.STRING), metadata.getSchema()); - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt new file mode 100644 index 00000000..1f3ba57e --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt @@ -0,0 +1,165 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.apache.avro.Schema +import org.junit.After +import org.junit.Assert +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.radarbase.config.ServerConfig +import org.radarbase.producer.rest.RestClient.Companion.globalRestClient +import org.radarbase.producer.rest.SchemaRetriever.Companion.subject +import java.io.IOException +import java.util.concurrent.TimeUnit + +class SchemaRetrieverTest { + private lateinit var mockServer: MockWebServer + private lateinit var retriever: SchemaRetriever + + @Suppress("HttpUrlsUsage") + @Before + fun setUp() { + mockServer = MockWebServer() + val restClient = globalRestClient { + server = ServerConfig("http://${mockServer.hostName}:${mockServer.port}/base") + timeout(1L, TimeUnit.SECONDS) + } + retriever = SchemaRetriever(restClient) + } + + @After + @Throws(IOException::class) + fun tearDown() { + mockServer.close() + } + + @Test + fun subject() { + assertEquals("bla-value", subject("bla", true)) + assertEquals("bla-key", subject("bla", false)) + } + + // Already queried schema is cached and does not need another request + @Test + fun testSchemaMetadata() { + // Not yet queried schema needs a new request, so if the server does not respond, an + // IOException is thrown. + mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + val (id, version, schema) = retriever.getBySubjectAndVersion("bla", true, 2) + assertEquals(10, id) + assertEquals(2, version) + assertEquals(Schema.create(Schema.Type.STRING), schema) + assertEquals("/base/subjects/bla-value/versions/2", mockServer.takeRequest().path) + + // Already queried schema is cached and does not need another request + val (id1, version1, schema1) = retriever.getBySubjectAndVersion("bla", true, 2) + assertEquals(10, id1) + assertEquals(2, version1) + assertEquals(Schema.create(Schema.Type.STRING), schema1) + assertEquals(1, mockServer.requestCount.toLong()) + + // Not yet queried schema needs a new request, so if the server does not respond, an + // IOException is thrown. + mockServer.enqueue(MockResponse().setResponseCode(500)) + Assert.assertThrows(IOException::class.java) { + retriever.getBySubjectAndVersion( + "bla", + false, + 2 + ) + } + } + + @Test + @Throws(Exception::class) + fun addSchemaMetadata() { + mockServer.enqueue(MockResponse().setBody("{\"id\":10}")) + var id = retriever.addSchema("bla", true, Schema.create(Schema.Type.STRING)) + assertEquals(10, id.toLong()) + assertEquals(1, mockServer.requestCount.toLong()) + var request = mockServer.takeRequest() + assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.body.readUtf8()) + val schemaFields = listOf( + Schema.Field("a", Schema.create(Schema.Type.INT), "that a", 10) + ) + val record = Schema.createRecord("C", "that C", "org.radarcns", false, schemaFields) + mockServer.enqueue(MockResponse().setBody("{\"id\":11}")) + id = retriever.addSchema("bla", true, record) + assertEquals(11, id.toLong()) + request = mockServer.takeRequest() + assertEquals( + "{\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"C\\\",\\\"namespace\\\":\\\"org.radarcns\\\",\\\"doc\\\":\\\"that C\\\",\\\"fields\\\":[{\\\"name\\\":\\\"a\\\",\\\"type\\\":\\\"int\\\",\\\"doc\\\":\\\"that a\\\",\\\"default\\\":10}]}\"}", + request.body.readUtf8() + ) + } + + @Test + fun getOrSetSchemaMetadataSet() { + mockServer.enqueue(MockResponse().setResponseCode(404)) + mockServer.enqueue(MockResponse().setBody("{\"id\":10}")) + mockServer.enqueue(MockResponse().setBody("{\"id\":10, \"version\": 2}")) + var metadata = retriever.getOrSetSchemaMetadata( + "bla", + true, + Schema.create(Schema.Type.STRING), + -1 + ) + assertEquals(10, metadata.id) + assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) + assertEquals(3, mockServer.requestCount.toLong()) + mockServer.takeRequest() + val request = mockServer.takeRequest() + assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.body.readUtf8()) + assertEquals("/base/subjects/bla-value/versions", request.path) + metadata = retriever.getOrSetSchemaMetadata( + "bla", + true, + Schema.create(Schema.Type.STRING), + -1 + ) + assertEquals(10, metadata.id) + assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) + } + + @Test + fun getOrSetSchemaMetadataGet() { + mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + var metadata = retriever.getOrSetSchemaMetadata( + "bla", + true, + Schema.create(Schema.Type.STRING), + 2 + ) + assertEquals(10, metadata.id) + assertEquals(2, metadata.version) + assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) + assertEquals(1, mockServer.requestCount.toLong()) + val request = mockServer.takeRequest() + assertEquals("/base/subjects/bla-value/versions/2", request.path) + metadata = retriever.getOrSetSchemaMetadata( + "bla", + true, + Schema.create(Schema.Type.STRING), + 2 + ) + assertEquals(10, metadata.id) + assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java deleted file mode 100644 index cf9fac39..00000000 --- a/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.radarbase.topic; - -import static org.junit.Assert.assertEquals; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; -import org.junit.Test; - -public class KafkaTopicTest { - @Test(expected = IllegalArgumentException.class) - public void nullArguments() { - new KafkaTopic(null); - } - - @Test(expected = IllegalArgumentException.class) - public void invalidTopicName() { - new KafkaTopic("bla$"); - } - - - @Test - public void getName() { - KafkaTopic topic = new KafkaTopic("aba"); - assertEquals("aba", topic.getName()); - } - - - @Test - public void compare() throws Exception { - final int randomSize = 100; - List randomString = new ArrayList<>(randomSize); - List randomTopic = new ArrayList<>(randomSize); - for (int i = 0; i < randomSize; i++) { - String str = 'a' + UUID.randomUUID().toString().replace('-', '_'); - randomString.add(str); - randomTopic.add(new KafkaTopic(str)); - } - - Collections.sort(randomString); - Collections.sort(randomTopic); - - for (int i = 0; i < randomSize; i++) { - assertEquals(randomString.get(i), randomTopic.get(i).getName()); - } - } -} diff --git a/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt new file mode 100644 index 00000000..2422d5a2 --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt @@ -0,0 +1,36 @@ +package org.radarbase.topic + +import org.junit.Assert +import org.junit.Test +import java.util.* + +class KafkaTopicTest { + @Test(expected = IllegalArgumentException::class) + fun invalidTopicName() { + KafkaTopic("bla$") + } + + @Test + fun testName() { + val topic = KafkaTopic("aba") + Assert.assertEquals("aba", topic.name) + } + + @Test + @Throws(Exception::class) + fun compare() { + val randomSize = 100 + val randomString: MutableList = ArrayList(randomSize) + val randomTopic: MutableList = ArrayList(randomSize) + for (i in 0 until randomSize) { + val str = 'a'.toString() + UUID.randomUUID().toString().replace('-', '_') + randomString.add(str) + randomTopic.add(KafkaTopic(str)) + } + randomString.sort() + randomTopic.sort() + for (i in 0 until randomSize) { + Assert.assertEquals(randomString[i], randomTopic[i].name) + } + } +} diff --git a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java deleted file mode 100644 index 681f94f0..00000000 --- a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.topic; - -import static org.junit.Assert.assertEquals; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericRecord; -import org.junit.Test; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.passive.phone.PhoneAcceleration; - -/** - * Created by joris on 05/07/2017. - */ -public class SensorTopicTest { - - @Test - public void workingConstructor() { - Schema keySchema = SchemaBuilder.record("key").fields() - .name("projectId").type(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING))).withDefault(null) - .name("userId").type(Schema.create(Type.STRING)).noDefault() - .name("sourceId").type(Schema.create(Type.STRING)).noDefault() - .endRecord(); - Schema valueSchema = SchemaBuilder.record("value").fields() - .name("time").type(Schema.create(Type.DOUBLE)).noDefault() - .name("timeReceived").type(Schema.create(Type.DOUBLE)).noDefault() - .name("value").type(Schema.create(Type.DOUBLE)).noDefault() - .endRecord(); - - new SensorTopic<>("test", - keySchema, valueSchema, - GenericRecord.class, GenericRecord.class); - } - - @Test(expected = IllegalArgumentException.class) - public void nullArguments() { - new SensorTopic<>(null, null, null, null, null); - } - - @Test(expected = IllegalArgumentException.class) - public void nullArgumentsExceptName() { - new SensorTopic<>("test", null, null, null, null); - } - - @Test(expected = IllegalArgumentException.class) - public void nullName() { - Schema keySchema = SchemaBuilder.record("key").fields() - .name("userId").type(Schema.create(Type.STRING)).noDefault() - .name("sourceId").type(Schema.create(Type.STRING)).noDefault() - .endRecord(); - Schema valueSchema = SchemaBuilder.record("value").fields() - .name("time").type(Schema.create(Type.DOUBLE)).noDefault() - .name("timeReceived").type(Schema.create(Type.DOUBLE)).noDefault() - .name("value").type(Schema.create(Type.DOUBLE)).noDefault() - .endRecord(); - - new SensorTopic<>(null, - keySchema, valueSchema, - GenericRecord.class, GenericRecord.class); - } - - - @Test(expected = IllegalArgumentException.class) - public void missingUserId() { - Schema keySchema = SchemaBuilder.record("key").fields() - .name("sourceId").type(Schema.create(Type.STRING)).noDefault() - .endRecord(); - Schema valueSchema = SchemaBuilder.record("value").fields() - .name("time").type(Schema.create(Type.DOUBLE)).noDefault() - .name("timeReceived").type(Schema.create(Type.DOUBLE)).noDefault() - .name("value").type(Schema.create(Type.DOUBLE)).noDefault() - .endRecord(); - - new SensorTopic<>("test", - keySchema, valueSchema, - GenericRecord.class, GenericRecord.class); - } - - @Test(expected = IllegalArgumentException.class) - public void missingTime() { - Schema keySchema = SchemaBuilder.record("key").fields() - .name("userId").type(Schema.create(Type.STRING)).noDefault() - .name("sourceId").type(Schema.create(Type.STRING)).noDefault() - .endRecord(); - Schema valueSchema = SchemaBuilder.record("value").fields() - .name("timeReceived").type(Schema.create(Type.DOUBLE)).noDefault() - .name("value").type(Schema.create(Type.DOUBLE)).noDefault() - .endRecord(); - - new SensorTopic<>("test", - keySchema, valueSchema, - GenericRecord.class, GenericRecord.class); - } - - @Test(expected = IllegalArgumentException.class) - public void notARecord() { - Schema keySchema = Schema.create(Type.STRING); - Schema valueSchema = SchemaBuilder.record("value").fields() - .name("timeReceived").type(Schema.create(Type.DOUBLE)).noDefault() - .name("value").type(Schema.create(Type.DOUBLE)).noDefault() - .endRecord(); - - new SensorTopic<>("test", - keySchema, valueSchema, - GenericRecord.class, GenericRecord.class); - } - - @Test - public void parseTopic() { - SensorTopic topic = SensorTopic.parse("test", - ObservationKey.class.getName(), PhoneAcceleration.class.getName()); - - SensorTopic expected = new SensorTopic<>("test", - ObservationKey.getClassSchema(), PhoneAcceleration.getClassSchema(), - ObservationKey.class, PhoneAcceleration.class); - - assertEquals(expected, topic); - } - - @Test(expected = IllegalArgumentException.class) - public void parseUnexistingKey() { - SensorTopic.parse("test", - "unexisting." + ObservationKey.class.getName(), - PhoneAcceleration.class.getName()); - } - - - @Test(expected = IllegalArgumentException.class) - public void parseUnexistingValue() { - SensorTopic.parse("test", - ObservationKey.class.getName(), - "unexisting." + PhoneAcceleration.class.getName()); - } -} diff --git a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt new file mode 100644 index 00000000..55dd4c72 --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt @@ -0,0 +1,134 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.topic + +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder +import org.apache.avro.generic.GenericRecord +import org.junit.Assert +import org.junit.Test +import org.radarcns.kafka.ObservationKey +import org.radarcns.passive.phone.PhoneAcceleration + +/** + * Created by joris on 05/07/2017. + */ +class SensorTopicTest { + @Test + fun workingConstructor() { + val keySchema = SchemaBuilder.record("key").fields() + .name("projectId").type( + Schema.createUnion( + Schema.create(Schema.Type.NULL), Schema.create( + Schema.Type.STRING + ) + ) + ).withDefault(null) + .name("userId").type(Schema.create(Schema.Type.STRING)).noDefault() + .name("sourceId").type(Schema.create(Schema.Type.STRING)).noDefault() + .endRecord() + val valueSchema = SchemaBuilder.record("value").fields() + .name("time").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .endRecord() + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } + + @Test(expected = IllegalArgumentException::class) + fun missingUserId() { + val keySchema = SchemaBuilder.record("key").fields() + .name("sourceId").type(Schema.create(Schema.Type.STRING)).noDefault() + .endRecord() + val valueSchema = SchemaBuilder.record("value").fields() + .name("time").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .endRecord() + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } + + @Test(expected = IllegalArgumentException::class) + fun missingTime() { + val keySchema = SchemaBuilder.record("key").fields() + .name("userId").type(Schema.create(Schema.Type.STRING)).noDefault() + .name("sourceId").type(Schema.create(Schema.Type.STRING)).noDefault() + .endRecord() + val valueSchema = SchemaBuilder.record("value").fields() + .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .endRecord() + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } + + @Test(expected = IllegalArgumentException::class) + fun notARecord() { + val keySchema = Schema.create(Schema.Type.STRING) + val valueSchema = SchemaBuilder.record("value").fields() + .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() + .endRecord() + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } + + @Test + fun parseTopic() { + val topic: SensorTopic = SensorTopic.parse( + "test", + ObservationKey::class.java.name, PhoneAcceleration::class.java.name + ) + val expected = SensorTopic( + "test", + ObservationKey.getClassSchema(), PhoneAcceleration.getClassSchema(), + ObservationKey::class.java, PhoneAcceleration::class.java + ) + Assert.assertEquals(expected, topic) + } + + @Test(expected = IllegalArgumentException::class) + fun parseUnexistingKey() { + SensorTopic.parse( + "test", + "unexisting." + ObservationKey::class.java.name, + PhoneAcceleration::class.java.name + ) + } + + @Test(expected = IllegalArgumentException::class) + fun parseUnexistingValue() { + SensorTopic.parse( + "test", + ObservationKey::class.java.name, + "unexisting." + PhoneAcceleration::class.java.name + ) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/util/Base64Test.java b/radar-commons/src/test/java/org/radarbase/util/Base64Test.java deleted file mode 100644 index a22f6bf9..00000000 --- a/radar-commons/src/test/java/org/radarbase/util/Base64Test.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.radarbase.util; - -import static org.junit.Assert.*; - -import java.util.concurrent.ThreadLocalRandom; -import kotlin.text.Charsets; -import org.junit.Test; -import org.radarbase.util.Base64.Encoder; - -public class Base64Test { - @Test - public void encoderTest() { - Encoder encoder = Base64.getEncoder(); - java.util.Base64.Encoder javaEncoder = java.util.Base64.getEncoder(); - - ThreadLocalRandom random = ThreadLocalRandom.current(); - for (int i = 0; i < 2_000; i += 7) { - byte[] src = new byte[i]; - random.nextBytes(src); - String actual = encoder.encode(src); - String expected = new String(javaEncoder.encode(src), Charsets.UTF_8); - assertEquals(expected, actual); - } - } -} diff --git a/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt b/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt new file mode 100644 index 00000000..5800541d --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt @@ -0,0 +1,25 @@ +package org.radarbase.util + +import org.junit.Assert +import org.junit.Test +import java.util.* +import java.util.concurrent.ThreadLocalRandom +import kotlin.text.Charsets.UTF_8 + +class Base64Test { + @Test + fun encoderTest() { + val javaEncoder = Base64.getEncoder() + val random = ThreadLocalRandom.current() + var i = 0 + while (i < 2000) { + val src = ByteArray(i) + random.nextBytes(src) + Assert.assertEquals( + Base64Encoder.encode(src), + String(javaEncoder.encode(src), UTF_8), + ) + i += 7 + } + } +} diff --git a/radar-commons/src/test/java/org/radarbase/util/SerializationTest.java b/radar-commons/src/test/java/org/radarbase/util/SerializationTest.java deleted file mode 100644 index d8081002..00000000 --- a/radar-commons/src/test/java/org/radarbase/util/SerializationTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.util.Random; -import org.junit.Test; - -/** - * Created by joris on 22/02/2017. - */ -public class SerializationTest { - @Test - public void bytesToLong() throws Exception { - byte[] input = {0, 0, 0, 0, 0, 0, 0, 1}; - assertEquals(1L, Serialization.bytesToLong(input, 0)); - } - - @Test - public void longToBytes() throws Exception { - byte[] buffer = new byte[8]; - Serialization.longToBytes(1L, buffer, 0); - assertArrayEquals(new byte[] {0, 0, 0, 0, 0, 0, 0, 1}, buffer); - } - - @Test - public void longToBytesAndBack() throws Exception { - byte[] buffer = new byte[8]; - Random random = new Random(); - for (int i = 0; i < 10; i++) { - long value = random.nextLong(); - Serialization.longToBytes(value, buffer, 0); - assertEquals(value, Serialization.bytesToLong(buffer, 0)); - } - } - - @Test - public void longToBytesAndOffset() throws Exception { - Random random = new Random(); - byte[] buffer = new byte[8 + 256]; - random.nextBytes(buffer); - - for (int i = 0; i < 10; i++) { - int offset = random.nextInt(256); - long value = random.nextLong(); - Serialization.longToBytes(value, buffer, offset); - assertEquals(value, Serialization.bytesToLong(buffer, offset)); - } - } - - @Test - public void intToBytes() throws Exception { - byte[] buffer = new byte[4]; - Serialization.intToBytes(1, buffer, 0); - assertArrayEquals(new byte[] {0, 0, 0, 1}, buffer); - } - - @Test - public void bytesToInt() throws Exception { - byte[] input = {0, 0, 0, 1}; - assertEquals(1, Serialization.bytesToInt(input, 0)); - } - - @Test - public void intToBytesAndOffset() throws Exception { - Random random = new Random(); - byte[] buffer = new byte[4 + 256]; - random.nextBytes(buffer); - - for (int i = 0; i < 10; i++) { - int offset = random.nextInt(256); - int value = random.nextInt(); - Serialization.intToBytes(value, buffer, offset); - assertEquals(value, Serialization.bytesToInt(buffer, offset)); - } - } -} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle.kts similarity index 86% rename from settings.gradle rename to settings.gradle.kts index 2c61dca1..2fdfd50d 100644 --- a/settings.gradle +++ b/settings.gradle.kts @@ -14,6 +14,6 @@ * limitations under the License. */ -include ':radar-commons' -include ':radar-commons-testing' -include ':radar-commons-server' +include(":radar-commons") +include(":radar-commons-testing") +include(":radar-commons-server") From 9ec3cf48aa0a830cb40677607a5e69cf55a84657 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 22 Nov 2022 21:27:23 +0100 Subject: [PATCH 05/25] Use ktor client with coroutines --- build.gradle.kts | 14 +- gradle.properties | 4 +- radar-commons-server/build.gradle.kts | 3 - .../radarbase/config/ServerConfigTest.java | 28 +- .../collector/AggregateListCollectorTest.java | 4 +- .../NumericAggregateCollectorTest.java | 31 +- .../UniformSamplingReservoirTest.java | 8 +- radar-commons-testing/build.gradle.kts | 11 +- .../java/org/radarbase/mock/MockDevice.java | 144 ------ .../java/org/radarbase/mock/MockDevice.kt | 124 ++++++ .../org/radarbase/mock/MockFileSender.java | 65 --- .../java/org/radarbase/mock/MockFileSender.kt | 61 +++ .../java/org/radarbase/mock/MockProducer.java | 409 ------------------ .../java/org/radarbase/mock/MockProducer.kt | 347 +++++++++++++++ .../radarbase/mock/data/HeaderHierarchy.java | 91 ---- .../radarbase/mock/data/HeaderHierarchy.kt | 64 +++ .../radarbase/mock/data/MockCsvParser.java | 287 ------------ .../org/radarbase/mock/data/MockCsvParser.kt | 278 ++++++++++++ .../mock/data/MockRecordValidator.java | 144 ------ .../mock/data/MockRecordValidator.kt | 143 ++++++ .../radarbase/mock/model/MockAggregator.java | 2 +- .../java/org/radarbase/util/Oscilloscope.java | 70 --- .../java/org/radarbase/util/Oscilloscope.kt | 61 +++ .../org/radarbase/mock/CsvGeneratorTest.java | 26 +- .../radarbase/mock/RecordGeneratorTest.java | 10 +- .../mock/data/MockRecordValidatorTest.java | 224 ---------- .../mock/data/MockRecordValidatorTest.kt | 229 ++++++++++ .../org/radarbase/util/MetronomeTest.java | 4 +- .../org/radarbase/util/OscilloscopeTest.java | 56 --- .../org/radarbase/util/OscilloscopeTest.kt | 52 +++ radar-commons/build.gradle.kts | 27 +- .../java/org/radarbase/config/ServerConfig.kt | 15 - .../org/radarbase/data/AvroDatumDecoder.kt | 1 + .../org/radarbase/data/AvroDatumEncoder.kt | 3 +- .../java/org/radarbase/data/AvroEncoder.kt | 20 +- .../java/org/radarbase/data/AvroRecordData.kt | 7 + .../org/radarbase/data/AvroRecordWriter.kt | 2 - .../java/org/radarbase/data/RecordData.kt | 3 + .../org/radarbase/data/RemoteSchemaEncoder.kt | 69 +-- .../producer/AuthenticationException.kt | 12 +- .../radarbase/producer/BatchedKafkaSender.kt | 151 ------- .../org/radarbase/producer/KafkaSender.kt | 25 +- .../radarbase/producer/KafkaTopicSender.kt | 25 +- .../producer/{rest => avro}/AvroDataMapper.kt | 2 +- .../{rest => avro}/AvroDataMapperFactory.kt | 268 ++++++------ .../radarbase/producer/io/BinaryEncoder.kt | 136 ++++++ .../producer/io/DirectBinaryEncoder.kt | 129 ++++++ .../java/org/radarbase/producer/io/Encoder.kt | 342 +++++++++++++++ .../io/FunctionalWriteChannelContent.kt | 10 + .../producer/io/GzipContentEncoding.kt | 95 ++++ .../producer/io/HttpClientExtensions.kt | 36 ++ .../io/UnsupportedMediaTypeException.kt | 11 + .../producer/rest/AvroContentConverter.kt | 63 +++ .../producer/rest/AvroRecordContent.kt | 7 + .../producer/rest/BinaryRecordContent.kt | 67 +++ .../producer/rest/BinaryRecordRequest.kt | 137 ------ .../producer/rest/ConnectionState.kt | 69 ++- .../producer/rest/GzipRequestInterceptor.kt | 61 --- .../producer/rest/JsonRecordContent.kt | 66 +++ .../producer/rest/JsonRecordRequest.kt | 131 ------ .../radarbase/producer/rest/RecordRequest.kt | 55 --- .../org/radarbase/producer/rest/RestClient.kt | 281 ------------ .../radarbase/producer/rest/RestException.kt | 9 +- .../producer/rest/RestKafkaSender.kt | 257 +++++++++++ .../org/radarbase/producer/rest/RestSender.kt | 278 ------------ .../producer/rest/RestTopicSender.kt | 209 --------- .../producer/rest/SchemaRestClient.kt | 103 ----- .../producer/rest/TopicRequestBody.kt | 49 --- .../rest/UncheckedRequestException.kt | 91 ---- .../{rest => schema}/ParsedSchemaMetadata.kt | 2 +- .../producer/schema/SchemaMetadata.kt | 43 ++ .../producer/schema/SchemaRestClient.kt | 105 +++++ .../{rest => schema}/SchemaRetriever.kt | 151 +++++-- .../java/org/radarbase/util/Annotations.kt | 8 + .../main/java/org/radarbase/util/RestUtils.kt | 96 ---- .../main/java/org/radarbase/util/TimedInt.kt | 2 +- .../java/org/radarbase/util/TimedValue.kt | 2 +- .../java/org/radarbase/util/TimedVariable.kt | 2 +- .../util/{CacheConfig.kt => TimeoutConfig.kt} | 2 +- .../radarbase/data/AvroDatumDecoderTest.java | 4 +- .../AvroDataMapperFactoryTest.kt | 88 ++-- ...uestTest.kt => BinaryRecordContentTest.kt} | 39 +- .../producer/rest/ConnectionStateTest.kt | 56 ++- .../radarbase/producer/rest/RestClientTest.kt | 110 ----- .../producer/rest/RestKafkaSenderTest.kt | 327 ++++++++++++++ .../radarbase/producer/rest/RestSenderTest.kt | 282 ------------ .../{rest => schema}/SchemaRestClientTest.kt | 42 +- .../{rest => schema}/SchemaRetrieverTest.kt | 80 ++-- .../org/radarbase/topic/KafkaTopicTest.kt | 15 +- .../org/radarbase/topic/SensorTopicTest.kt | 78 ++-- .../java/org/radarbase/util/Base64Test.kt | 6 +- 91 files changed, 3720 insertions(+), 4136 deletions(-) delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt delete mode 100644 radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java create mode 100644 radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt delete mode 100644 radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java create mode 100644 radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.kt delete mode 100644 radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java create mode 100644 radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt rename radar-commons/src/main/java/org/radarbase/producer/{rest => avro}/AvroDataMapper.kt (90%) rename radar-commons/src/main/java/org/radarbase/producer/{rest => avro}/AvroDataMapperFactory.kt (60%) create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/BinaryEncoder.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/FunctionalWriteChannelContent.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/AvroRecordContent.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt rename radar-commons/src/main/java/org/radarbase/producer/{rest => schema}/ParsedSchemaMetadata.kt (96%) create mode 100644 radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt create mode 100644 radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt rename radar-commons/src/main/java/org/radarbase/producer/{rest => schema}/SchemaRetriever.kt (60%) create mode 100644 radar-commons/src/main/java/org/radarbase/util/Annotations.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/RestUtils.kt rename radar-commons/src/main/java/org/radarbase/util/{CacheConfig.kt => TimeoutConfig.kt} (92%) rename radar-commons/src/test/java/org/radarbase/producer/{rest => avro}/AvroDataMapperFactoryTest.kt (84%) rename radar-commons/src/test/java/org/radarbase/producer/rest/{BinaryRecordRequestTest.kt => BinaryRecordContentTest.kt} (84%) delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt create mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt delete mode 100644 radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt rename radar-commons/src/test/java/org/radarbase/producer/{rest => schema}/SchemaRestClientTest.kt (62%) rename radar-commons/src/test/java/org/radarbase/producer/{rest => schema}/SchemaRetrieverTest.kt (69%) diff --git a/build.gradle.kts b/build.gradle.kts index bb70bcc7..ac8b9e50 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,6 +19,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile */ plugins { kotlin("jvm") version "1.7.21" apply false + kotlin("plugin.serialization") version "1.7.21" apply false id("com.github.davidmc24.gradle.plugin.avro") version "1.5.0" apply false id("io.github.gradle-nexus.publish-plugin") version "1.1.0" id("com.github.ben-manes.versions") version "0.44.0" @@ -34,7 +35,7 @@ val githubIssueUrl = "https://github.com/$githubRepoName/issues" val website = "https://radar-base.org" allprojects { - version = "0.15.1-SNAPSHOT" + version = "0.16.0-SNAPSHOT" group = "org.radarbase" } @@ -69,6 +70,15 @@ subprojects { maven(url = "https://oss.sonatype.org/content/repositories/snapshots") } + dependencies { + val coroutinesVersion: String by project + configurations["testImplementation"]("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion") + + val junitVersion: String by project + configurations["testImplementation"]("org.junit.jupiter:junit-jupiter-api:$junitVersion") + configurations["testRuntimeOnly"]("org.junit.jupiter:junit-jupiter-engine:$junitVersion") + } + afterEvaluate { configurations { named("implementation") { @@ -184,6 +194,8 @@ subprojects { //---------------------------------------------------------------------------// tasks.withType { + useJUnitPlatform() + val stdout = LinkedList() beforeTest(closureOf { stdout.clear() diff --git a/gradle.properties b/gradle.properties index 742b3b99..68349877 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,10 +4,12 @@ kafkaVersion=7.3.0-ce avroVersion=1.11.1 jacksonVersion=2.14.0 okhttpVersion=4.10.0 -junitVersion=4.13.2 +junitVersion=5.9.1 mockitoVersion=4.9.0 +mockitoKotlinVersion=4.0.0 hamcrestVersion=2.2 radarSchemasVersion=0.8.1 orgJsonVersion=20220924 opencsvVersion=5.7.1 ktorVersion=2.1.3 +coroutinesVersion=1.6.4 diff --git a/radar-commons-server/build.gradle.kts b/radar-commons-server/build.gradle.kts index 029a61ad..79c59a62 100644 --- a/radar-commons-server/build.gradle.kts +++ b/radar-commons-server/build.gradle.kts @@ -41,9 +41,6 @@ dependencies { testImplementation("io.confluent:kafka-avro-serializer:$confluentVersion") val radarSchemasVersion: String by project testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion") - // Direct producer uses KafkaAvroSerializer if initialized - val junitVersion: String by project - testImplementation("junit:junit:$junitVersion") val slf4jVersion: String by project testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") } diff --git a/radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java b/radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java index 11cc14bc..c5b1ba00 100644 --- a/radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/config/ServerConfigTest.java @@ -16,16 +16,14 @@ package org.radarbase.config; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; -import okhttp3.HttpUrl; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * Created by joris on 01/05/2017. @@ -61,26 +59,4 @@ public void jacksonUrl() throws IOException { + "path: /schema")) .getUrlString()); } - - @Test - public void getHttpUrl() throws MalformedURLException { - ServerConfig config = new ServerConfig("http://something.else/that"); - HttpUrl url = config.getHttpUrl(); - assertEquals("http://something.else/that/", url.toString()); - assertEquals("something.else", url.host()); - assertEquals("http", url.scheme()); - assertEquals(80, url.port()); - assertEquals("/that/", url.encodedPath()); - } - - @Test - public void getHttpUrlWitoutRoot() throws MalformedURLException { - ServerConfig config = new ServerConfig("http://something.else"); - HttpUrl url = config.getHttpUrl(); - assertEquals("http://something.else/", url.toString()); - assertEquals("something.else", url.host()); - assertEquals("http", url.scheme()); - assertEquals(80, url.port()); - assertEquals("/", url.encodedPath()); - } } diff --git a/radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java index 3645a24d..282bc7fa 100644 --- a/radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/AggregateListCollectorTest.java @@ -16,9 +16,9 @@ package org.radarbase.stream.collector; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; /** diff --git a/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java index f64160ef..9dbfedad 100644 --- a/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/NumericAggregateCollectorTest.java @@ -16,13 +16,14 @@ package org.radarbase.stream.collector; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Arrays; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.radarcns.kafka.AggregateKey; import org.radarcns.monitor.application.ApplicationRecordCounts; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; @@ -35,7 +36,7 @@ public class NumericAggregateCollectorTest { private NumericAggregateCollector valueCollector; - @Before + @BeforeEach public void setUp() { this.valueCollector = new NumericAggregateCollector("test", true); } @@ -105,21 +106,27 @@ public void testAverageFloat() { assertEquals(36.8508954, valueCollector.getMean(), 0); } - @Test(expected = IllegalStateException.class) + @Test public void testAddRecordWithoutSchema() { - valueCollector.add(new EmpaticaE4BloodVolumePulse(0d, 0d, 0f)); + assertThrows(IllegalStateException.class, () -> + valueCollector.add(new EmpaticaE4BloodVolumePulse(0d, 0d, 0f)) + ); } - @Test(expected = IllegalArgumentException.class) + @Test public void testWrongRecordType() { - this.valueCollector = new NumericAggregateCollector("isPlugged", - PhoneBatteryLevel.getClassSchema(), false); + assertThrows(IllegalArgumentException.class, () -> + this.valueCollector = new NumericAggregateCollector("isPlugged", + PhoneBatteryLevel.getClassSchema(), false) + ); } - @Test(expected = IllegalArgumentException.class) + @Test public void testWrongFieldName() { - this.valueCollector = new NumericAggregateCollector("doesNotExist", - PhoneBatteryLevel.getClassSchema(), false); + assertThrows(IllegalArgumentException.class, () -> + this.valueCollector = new NumericAggregateCollector("doesNotExist", + PhoneBatteryLevel.getClassSchema(), false) + ); } @Test diff --git a/radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java b/radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java index 359289b8..30ac161b 100644 --- a/radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java +++ b/radar-commons-server/src/test/java/org/radarbase/stream/collector/UniformSamplingReservoirTest.java @@ -1,12 +1,12 @@ package org.radarbase.stream.collector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Iterator; import java.util.List; import java.util.concurrent.ThreadLocalRandom; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class UniformSamplingReservoirTest { @Test @@ -66,4 +66,4 @@ private static > boolean isOrdered(List list) { } return true; } -} \ No newline at end of file +} diff --git a/radar-commons-testing/build.gradle.kts b/radar-commons-testing/build.gradle.kts index d95386fc..14515706 100644 --- a/radar-commons-testing/build.gradle.kts +++ b/radar-commons-testing/build.gradle.kts @@ -18,7 +18,7 @@ plugins { application } -val applicationRuntimeOnly by configurations.creating +val applicationRuntimeOnly: Configuration by configurations.creating application { mainClass.set("org.radarbase.mock.MockProducer") @@ -53,13 +53,16 @@ dependencies { val confluentVersion: String by project implementation("io.confluent:kafka-avro-serializer:$confluentVersion") + val ktorVersion: String by project + implementation(platform("io.ktor:ktor-bom:$ktorVersion")) + implementation("io.ktor:ktor-serialization-kotlinx-json") + val slf4jVersion: String by project applicationRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") - // Direct producer uses KafkaAvroSerializer if initialized - val junitVersion: String by project - testImplementation("junit:junit:$junitVersion") val hamcrestVersion: String by project testImplementation("org.hamcrest:hamcrest:$hamcrestVersion") testImplementation("org.slf4j:slf4j-simple:$slf4jVersion") + val mockitoVersion: String by project + testImplementation("org.mockito:mockito-core:$mockitoVersion") } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java deleted file mode 100644 index 4b96760d..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.mock; - -import java.io.IOException; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.specific.SpecificRecord; -import org.radarbase.data.Record; -import org.radarbase.mock.data.RecordGenerator; -import org.radarbase.producer.KafkaSender; -import org.radarbase.producer.KafkaTopicSender; -import org.radarbase.util.Oscilloscope; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Mock device that sends data for given topics at a given rate. This can be used to simulate - * any number of real devices. - * @param record key type - */ -public class MockDevice extends Thread { - private static final Logger logger = LoggerFactory.getLogger(MockDevice.class); - private final int baseFrequency; - private final KafkaSender sender; - private final AtomicBoolean stopping; - private final List> generators; - private final K key; - - private Exception exception; - - /** - * Basic constructor. - * @param sender sender to send data with - * @param key key to send all messages with - * @param generators data generators that produce the data we send - */ - public MockDevice(KafkaSender sender, K key, List> generators) { - this.generators = generators; - this.key = key; - baseFrequency = computeBaseFrequency(generators); - this.sender = sender; - this.stopping = new AtomicBoolean(false); - exception = null; - } - - @Override - @SuppressWarnings("PMD.CloseResource") - public void run() { - List> topicSenders = - new ArrayList<>(generators.size()); - List>> recordIterators = - new ArrayList<>(generators.size()); - - try { - for (RecordGenerator generator : generators) { - topicSenders.add(sender.sender(generator.getTopic())); - recordIterators.add(generator.iterateValues(key, 0)); - } - Oscilloscope oscilloscope = new Oscilloscope(baseFrequency); - - try { - while (!stopping.get()) { - // The time keeping is regulated with beats, with baseFrequency beats per - // second. - int beat = oscilloscope.beat(); - - for (int i = 0; i < generators.size(); i++) { - int frequency = generators.get(i).getConfig().getFrequency(); - if (frequency > 0 && beat % (baseFrequency / frequency) == 0) { - Record record = recordIterators.get(i).next(); - topicSenders.get(i).send(record.getKey(), record.getValue()); - } - } - } - } catch (InterruptedException ex) { - // do nothing, just exit the loop - } - - for (KafkaTopicSender topicSender : topicSenders) { - topicSender.close(); - } - } catch (SchemaValidationException | IOException e) { - synchronized (this) { - this.exception = e; - } - logger.error("MockDevice {} failed to send message", key, e); - } - } - - /** - * Shut down the device eventually. - */ - public void shutdown() { - stopping.set(true); - } - - /** Get the exception that occurred in the thread. Returns null if no exception occurred. */ - public synchronized Exception getException() { - return exception; - } - - /** Check whether an exception occurred, and rethrow the exception if that is the case. */ - public synchronized void checkException() throws IOException, SchemaValidationException { - if (exception != null) { - if (exception instanceof IOException) { - throw (IOException) exception; - } else if (exception instanceof SchemaValidationException) { - throw (SchemaValidationException) exception; - } else if (exception instanceof RuntimeException) { - throw (RuntimeException) exception; - } else { - throw new IllegalStateException("Unknown exception occurred", exception); - } - } - } - - private int computeBaseFrequency(List> generators) { - BigInteger lcm = BigInteger.ONE; - for (RecordGenerator generator : generators) { - BigInteger freq = BigInteger.valueOf(generator.getConfig().getFrequency()); - lcm = lcm.multiply(freq.divide(lcm.gcd(freq))); // a * (b / gcd(a, b)); - } - return lcm.intValue(); - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt new file mode 100644 index 00000000..bede832c --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt @@ -0,0 +1,124 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.mock + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.isActive +import org.apache.avro.SchemaValidationException +import org.apache.avro.specific.SpecificRecord +import org.radarbase.mock.data.RecordGenerator +import org.radarbase.producer.KafkaSender +import org.radarbase.util.Oscilloscope +import org.slf4j.LoggerFactory +import java.io.IOException +import java.math.BigInteger +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Mock device that sends data for given topics at a given rate. This can be used to simulate + * any number of real devices. + * @param record key type + */ +class MockDevice( + sender: KafkaSender, + private val key: K, + private val generators: List> +) { + private val baseFrequency: Int + private val sender: KafkaSender + private val stopping: AtomicBoolean + + /** Get the exception that occurred in the thread. Returns null if no exception occurred. */ + @get:Synchronized + var exception: Exception? + private set + + /** + * Basic constructor. + * @param sender sender to send data with + * @param key key to send all messages with + * @param generators data generators that produce the data we send + */ + init { + baseFrequency = computeBaseFrequency(generators) + this.sender = sender + stopping = AtomicBoolean(false) + exception = null + } + + suspend fun CoroutineScope.run() { + try { + val topicSenders = generators.map { sender.sender(it.topic) } + val recordIterators = generators.map { it.iterateValues(key, 0) } + val oscilloscope = Oscilloscope(baseFrequency) + try { + while (isActive) { + // The time keeping is regulated with beats, with baseFrequency beats per + // second. + val beat = oscilloscope.beat() + for (i in generators.indices) { + val frequency = generators[i].config.frequency + if (frequency > 0 && beat % (baseFrequency / frequency) == 0) { + val record = recordIterators[i].next() + topicSenders[i].send(record.key, record.value) + } + } + } + } catch (ex: InterruptedException) { + // do nothing, just exit the loop + } + } catch (e: SchemaValidationException) { + synchronized(this) { exception = e } + logger.error("MockDevice {} failed to send message", key, e) + } catch (e: IOException) { + synchronized(this) { exception = e } + logger.error("MockDevice {} failed to send message", key, e) + } + } + + /** + * Shut down the device eventually. + */ + fun shutdown() { + stopping.set(true) + } + + /** Check whether an exception occurred, and rethrow the exception if that is the case. */ + @Synchronized + @Throws(IOException::class, SchemaValidationException::class) + fun checkException() { + when (exception) { + null -> {} + is IOException -> throw exception as IOException + is SchemaValidationException -> throw exception as SchemaValidationException + is RuntimeException -> throw exception as RuntimeException + else -> throw IllegalStateException("Unknown exception occurred", exception) + } + } + + private fun computeBaseFrequency(generators: List>): Int { + var lcm = BigInteger.ONE + for (generator in generators) { + val freq = BigInteger.valueOf(generator.config.frequency.toLong()) + lcm = lcm.multiply(freq.divide(lcm.gcd(freq))) // a * (b / gcd(a, b)); + } + return lcm.toInt() + } + + companion object { + private val logger = LoggerFactory.getLogger(MockDevice::class.java) + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java deleted file mode 100644 index fa9fce8d..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.mock; - -import com.opencsv.exceptions.CsvValidationException; -import java.io.IOException; -import org.apache.avro.SchemaValidationException; -import org.radarbase.data.Record; -import org.radarbase.mock.data.MockCsvParser; -import org.radarbase.producer.KafkaSender; -import org.radarbase.producer.KafkaTopicSender; - -/** - * Send mock data from a CSV file. - * - *

The value type is dynamic, so we will not check any of the generics. - */ -public class MockFileSender { - private final KafkaSender sender; - private final MockCsvParser parser; - - public MockFileSender(KafkaSender sender, MockCsvParser parser) { - this.parser = parser; - this.sender = sender; - } - - /** - * Send data from the configured CSV file synchronously. - * @throws IOException if data could not be read or sent. - */ - @SuppressWarnings("unchecked") - public void send() throws IOException { - try (KafkaTopicSender topicSender = sender.sender(parser.getTopic())) { - while (parser.hasNext()) { - Record record = parser.next(); - topicSender.send(record.getKey(), record.getValue()); - } - } catch (SchemaValidationException e) { - throw new IOException("Failed to match schemas", e); - } catch (CsvValidationException e) { - throw new IOException("Failed to read CSV file", e); - } - } - - @Override - public String toString() { - return "MockFileSender{" - + "parser=" + parser - + '}'; - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt new file mode 100644 index 00000000..e79b0b0d --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.mock + +import com.opencsv.exceptions.CsvValidationException +import org.apache.avro.SchemaValidationException +import org.apache.avro.generic.GenericRecord +import org.radarbase.data.Record +import org.radarbase.mock.data.MockCsvParser +import org.radarbase.producer.KafkaSender +import java.io.IOException + +/** + * Send mock data from a CSV file. + * + * + * The value type is dynamic, so we will not check any of the generics. + */ +class MockFileSender( + private val sender: KafkaSender, + private val parser: MockCsvParser, +) { + /** + * Send data from the configured CSV file synchronously. + * @throws IOException if data could not be read or sent. + */ + @Throws(IOException::class) + suspend fun send() { + parser.initialize() + try { + val topicSender = sender.sender(parser.topic) + while (parser.hasNext()) { + val record: Record<*, *> = parser.next() + topicSender.send(record.key as GenericRecord, record.value as GenericRecord) + } + } catch (e: SchemaValidationException) { + throw IOException("Failed to match schemas", e) + } catch (e: CsvValidationException) { + throw IOException("Failed to read CSV file", e) + } + } + + override fun toString(): String { + return ("MockFileSender{" + + "parser=" + parser + + '}') + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java deleted file mode 100644 index 89827831..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.java +++ /dev/null @@ -1,409 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.mock; - -import com.opencsv.exceptions.CsvValidationException; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import kotlin.Unit; -import okhttp3.Credentials; -import okhttp3.FormBody; -import okhttp3.Headers; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Request.Builder; -import okhttp3.Response; -import okhttp3.ResponseBody; -import org.apache.avro.SchemaValidationException; -import org.json.JSONObject; -import org.radarbase.config.ServerConfig; -import org.radarbase.config.YamlConfigLoader; -import org.radarbase.mock.config.AuthConfig; -import org.radarbase.mock.config.BasicMockConfig; -import org.radarbase.mock.config.MockDataConfig; -import org.radarbase.mock.data.MockCsvParser; -import org.radarbase.mock.data.RecordGenerator; -import org.radarbase.producer.BatchedKafkaSender; -import org.radarbase.producer.KafkaSender; -import org.radarbase.producer.rest.ConnectionState; -import org.radarbase.producer.rest.RestClient; -import org.radarbase.producer.rest.RestSender; -import org.radarbase.producer.rest.SchemaRetriever; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.passive.empatica.EmpaticaE4Acceleration; -import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; -import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; -import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; -import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; -import org.radarcns.passive.empatica.EmpaticaE4Temperature; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A Mock Producer class that can be used to stream data. It can use MockFileSender and MockDevice - * for testing purposes, with direct or indirect streaming. - */ -@SuppressWarnings("PMD") -public class MockProducer { - - private static final Logger logger = LoggerFactory.getLogger(MockProducer.class); - - private final List> devices; - private final List files; - private final List senders; - private final SchemaRetriever retriever; - - /** - * MockProducer with files from current directory. The data root directory will be the current - * directory. - * @param mockConfig configuration to mock - * @throws IOException if the data could not be read or sent - */ - public MockProducer(BasicMockConfig mockConfig) throws IOException { - this(mockConfig, null); - } - - /** - * Basic constructor. - * @param mockConfig configuration to mock - * @param root root directory of where mock files are located - * @throws IOException if data could not be sent - */ - public MockProducer(BasicMockConfig mockConfig, Path root) throws IOException { - int numDevices = mockConfig.getNumberOfDevices(); - - RestClient restClient = RestClient.Companion.globalRestClient(builder -> { - builder.setServer(mockConfig.getSchemaRegistry()); - builder.timeout(10, TimeUnit.SECONDS); - return Unit.INSTANCE; - }); - retriever = new SchemaRetriever(restClient); - List tmpSenders = null; - - try { - devices = new ArrayList<>(numDevices); - files = new ArrayList<>(numDevices); - - List dataConfigs = mockConfig.getData(); - if (dataConfigs == null) { - dataConfigs = defaultDataConfig(); - } - - List> generators; - List mockFiles; - generators = createGenerators(dataConfigs); - mockFiles = createMockFiles(dataConfigs, root); - - tmpSenders = createSenders(mockConfig, numDevices + mockFiles.size(), - mockConfig.getAuthConfig()); - - if (!generators.isEmpty()) { - String userId = "UserID_"; - String sourceId = "SourceID_"; - - for (int i = 0; i < numDevices; i++) { - ObservationKey key = new ObservationKey("test", userId + i, sourceId + i); - devices.add(new MockDevice<>(tmpSenders.get(i), key, generators)); - } - } - - for (int i = 0; i < mockFiles.size(); i++) { - files.add(new MockFileSender(tmpSenders.get(i + numDevices), mockFiles.get(i))); - } - } catch (CsvValidationException ex) { - if (tmpSenders != null) { - for (KafkaSender sender : tmpSenders) { - sender.close(); - } - } - throw new IOException("Cannot read CSV file", ex); - } catch (Exception ex) { - if (tmpSenders != null) { - for (KafkaSender sender : tmpSenders) { - sender.close(); - } - } - throw ex; - } - - senders = tmpSenders; - } - - private List createSenders( - BasicMockConfig mockConfig, int numDevices, AuthConfig authConfig) throws IOException { - - return createRestSenders(numDevices, retriever, mockConfig.getRestProxy(), - mockConfig.hasCompression(), authConfig); - } - - private String requestAccessToken(OkHttpClient okHttpClient, AuthConfig authConfig) - throws IOException { - Request request = new Builder() - .url(authConfig.getTokenUrl()) - .post(new FormBody.Builder() - .add("grant_type", "client_credentials") - .add("client_id", authConfig.getClientId()) - .add("client_secret", authConfig.getClientSecret()) - .build()) - .addHeader("Authorization", Credentials - .basic(authConfig.getClientId(), authConfig.getClientSecret())) - .build(); - - try (Response response = okHttpClient.newCall(request).execute()) { - ResponseBody responseBody = response.body(); - if (responseBody == null) { - throw new IOException("Cannot request token at " + request.url() - + " (" + response.code() + ") returned no body"); - } - if (!response.isSuccessful()) { - throw new IOException("Cannot request token: at " + request.url() - + " (" + response.code() + "): " + responseBody.string()); - } - return new JSONObject(responseBody.string()).getString("access_token"); - } - } - - /** Create senders that produce data to Kafka via the REST proxy. */ - private List createRestSenders(int numDevices, - SchemaRetriever retriever, ServerConfig restProxy, boolean useCompression, - AuthConfig authConfig) throws IOException { - List result = new ArrayList<>(numDevices); - ConnectionState sharedState = new ConnectionState(10, TimeUnit.SECONDS); - - Headers headers; - if (authConfig == null) { - headers = Headers.of(); - } else { - OkHttpClient okHttpClient = new OkHttpClient(); - String token = requestAccessToken(okHttpClient, authConfig); - headers = Headers.of("Authorization", "Bearer " + token); - } - - for (int i = 0; i < numDevices; i++) { - RestClient httpClient = RestClient.Companion.newRestClient(builder -> { - builder.setServer(restProxy); - builder.gzipCompression(useCompression); - builder.timeout(10, TimeUnit.SECONDS); - return Unit.INSTANCE; - }); - - RestSender restSender = RestSender.Companion.restSender(builder -> { - builder.setSchemaRetriever(retriever); - builder.setHttpClient(httpClient); - builder.setConnectionState(sharedState); - builder.setHeaders(headers.newBuilder()); - return Unit.INSTANCE; - }); - result.add(new BatchedKafkaSender(restSender, 1000, 1000)); - } - return result; - } - - /** Start sending data. */ - public void start() throws IOException { - for (MockDevice device : devices) { - device.start(); - } - for (MockFileSender file : files) { - file.send(); - logger.info("Sent data {}", file); - } - } - - /** Stop sending data and clean up all resources. */ - public void shutdown() throws IOException, InterruptedException, SchemaValidationException { - if (!devices.isEmpty()) { - logger.info("Shutting down mock devices"); - for (MockDevice device : devices) { - device.shutdown(); - } - logger.info("Waiting for mock devices to finish..."); - for (MockDevice device : devices) { - device.join(5_000L); - } - } - logger.info("Closing channels"); - for (KafkaSender sender : senders) { - sender.close(); - } - - for (MockDevice device : devices) { - device.checkException(); - } - } - - /** - * Runs the MockProducer with given YAML mock config file. - */ - public static void main(String[] args) { - if (args.length != 1) { - logger.error("This command needs a mock file argument"); - System.exit(1); - } - - Path mockFile = Paths.get(args[0]).toAbsolutePath(); - BasicMockConfig config = null; - try { - config = new YamlConfigLoader().load(mockFile, BasicMockConfig.class); - } catch (IOException ex) { - logger.error("Failed to load given mock file {}: {}", mockFile, ex.getMessage()); - System.exit(1); - } - - try { - MockProducer producer = new MockProducer(config, mockFile.getParent()); - producer.start(); - if (!producer.devices.isEmpty()) { - waitForProducer(producer, config.getDuration()); - } - } catch (IllegalArgumentException ex) { - logger.error("{}", ex.getMessage()); - System.exit(1); - } catch (InterruptedException e) { - // during shutdown, not that important. Will shutdown again. - } catch (Exception ex) { - logger.error("Failed to start mock producer", ex); - System.exit(1); - } - } - - /** Wait for given duration and then stop the producer. */ - private static void waitForProducer(final MockProducer producer, long duration) - throws IOException, InterruptedException, SchemaValidationException { - final AtomicBoolean isShutdown = new AtomicBoolean(false); - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - if (!isShutdown.get()) { - producer.shutdown(); - } - } catch (InterruptedException ex) { - logger.warn("Shutdown interrupted", ex); - } catch (Exception ex) { - logger.warn("Failed to shutdown producer", ex); - } - })); - - if (duration <= 0L) { - try { - logger.info("Producing data until interrupted"); - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException ex) { - // this is intended - } - } else { - try { - logger.info("Producing data for {} seconds", duration / 1000d); - Thread.sleep(duration); - } catch (InterruptedException ex) { - logger.warn("Data producing interrupted"); - } - producer.shutdown(); - isShutdown.set(true); - logger.info("Producing data done."); - } - } - - private List defaultDataConfig() { - MockDataConfig acceleration = new MockDataConfig(); - acceleration.setTopic("android_empatica_e4_acceleration"); - acceleration.setFrequency(32); - acceleration.setValueSchema(EmpaticaE4Acceleration.class.getName()); - acceleration.setInterval(-2d, 2d); - acceleration.setValueFields(Arrays.asList("x", "y", "z")); - - MockDataConfig battery = new MockDataConfig(); - battery.setTopic("android_empatica_e4_battery_level"); - battery.setValueSchema(EmpaticaE4BatteryLevel.class.getName()); - battery.setFrequency(1); - battery.setInterval(0d, 1d); - battery.setValueField("batteryLevel"); - - MockDataConfig bvp = new MockDataConfig(); - bvp.setTopic("android_empatica_e4_blood_volume_pulse"); - bvp.setValueSchema(EmpaticaE4BloodVolumePulse.class.getName()); - bvp.setFrequency(64); - bvp.setInterval(60d, 90d); - bvp.setValueField("bloodVolumePulse"); - - MockDataConfig eda = new MockDataConfig(); - eda.setTopic("android_empatica_e4_electrodermal_activity"); - eda.setValueSchema(EmpaticaE4ElectroDermalActivity.class.getName()); - eda.setValueField("electroDermalActivity"); - eda.setFrequency(4); - eda.setInterval(0.01d, 0.05d); - - MockDataConfig ibi = new MockDataConfig(); - ibi.setTopic("android_empatica_e4_inter_beat_interval"); - ibi.setValueSchema(EmpaticaE4InterBeatInterval.class.getName()); - ibi.setValueField("interBeatInterval"); - ibi.setFrequency(1); - ibi.setInterval(40d, 150d); - - MockDataConfig temperature = new MockDataConfig(); - temperature.setTopic("android_empatica_e4_temperature"); - temperature.setValueSchema(EmpaticaE4Temperature.class.getName()); - temperature.setFrequency(4); - temperature.setInterval(20d, 60d); - temperature.setValueField("temperature"); - - return Arrays.asList(acceleration, battery, bvp, eda, ibi, temperature); - } - - private List> createGenerators(List configs) { - - List> result = new ArrayList<>(configs.size()); - - for (MockDataConfig config : configs) { - if (config.getDataFile() == null) { - result.add(new RecordGenerator<>(config, ObservationKey.class)); - } - } - - return result; - } - - private List createMockFiles(List configs, - Path dataRoot) throws IOException, CsvValidationException { - - List result = new ArrayList<>(configs.size()); - - Instant now = Instant.now(); - Path parent = dataRoot; - if (parent == null) { - parent = Paths.get(".").toAbsolutePath(); - } - - for (MockDataConfig config : configs) { - if (config.getDataFile() != null) { - logger.info("Reading mock data from {}", config.getDataFile()); - result.add(new MockCsvParser(config, parent, now, retriever)); - } else { - logger.info("Generating mock data from {}", config); - } - } - - return result; - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt new file mode 100644 index 00000000..ff24faa1 --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt @@ -0,0 +1,347 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.mock + +import com.opencsv.exceptions.CsvValidationException +import io.ktor.client.call.* +import io.ktor.client.plugins.* +import io.ktor.client.plugins.auth.* +import io.ktor.client.plugins.auth.providers.* +import io.ktor.client.request.forms.* +import io.ktor.http.* +import kotlinx.coroutines.* +import org.apache.avro.SchemaValidationException +import org.radarbase.config.ServerConfig +import org.radarbase.config.YamlConfigLoader +import org.radarbase.management.auth.ClientCredentialsConfig +import org.radarbase.management.auth.clientCredentials +import org.radarbase.mock.config.AuthConfig +import org.radarbase.mock.config.BasicMockConfig +import org.radarbase.mock.config.MockDataConfig +import org.radarbase.mock.data.MockCsvParser +import org.radarbase.mock.data.RecordGenerator +import org.radarbase.producer.KafkaSender +import org.radarbase.producer.io.GzipContentEncoding +import org.radarbase.producer.io.timeout +import org.radarbase.producer.rest.ConnectionState +import org.radarbase.producer.rest.RestKafkaSender.Companion.restKafkaSender +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever +import org.radarbase.util.TimeoutConfig +import org.radarcns.kafka.ObservationKey +import org.radarcns.passive.empatica.* +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Path +import java.nio.file.Paths +import java.time.Duration +import java.time.Instant +import java.util.* +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.system.exitProcess + +/** + * A Mock Producer class that can be used to stream data. It can use MockFileSender and MockDevice + * for testing purposes, with direct or indirect streaming. + * @param mockConfig configuration to mock + * @param root root directory of where mock files are located + * @throws IOException if the data could not be read or sent + */ +class MockProducer @JvmOverloads constructor( + mockConfig: BasicMockConfig, + root: Path? = null, +) { + private var devices: MutableList> + private var files: MutableList + private val senders: List + private val retriever: SchemaRetriever + private var job: Job? = null + + init { + val numDevices = mockConfig.numberOfDevices + retriever = schemaRetriever(mockConfig.schemaRegistry.urlString) { + httpClient { + timeout(Duration.ofSeconds(10)) + } + } + val dataConfigs = mockConfig.data + ?: defaultDataConfig() + try { + val generators: List> = createGenerators(dataConfigs) + val mockFiles: List = createMockFiles(dataConfigs, root) + senders = createSenders( + mockConfig, numDevices + mockFiles.size, + mockConfig.authConfig + ) + + devices = ArrayList(numDevices) + files = ArrayList(numDevices) + + if (generators.isNotEmpty()) { + val userId = "UserID_" + val sourceId = "SourceID_" + for (i in 0 until numDevices) { + val key = ObservationKey("test", userId + i, sourceId + i) + devices.add(MockDevice(senders[i], key, generators)) + } + } + for (i in mockFiles.indices) { + files.add(MockFileSender(senders[i + numDevices], mockFiles[i])) + } + } catch (ex: CsvValidationException) { + throw IOException("Cannot read CSV file", ex) + } catch (ex: Exception) { + throw ex + } + } + + @Throws(IOException::class) + private fun createSenders( + mockConfig: BasicMockConfig, numDevices: Int, authConfig: AuthConfig + ): List = createRestSenders( + numDevices, + retriever, + mockConfig.restProxy, + mockConfig.hasCompression(), + authConfig + ) + + /** Create senders that produce data to Kafka via the REST proxy. */ + @Throws(IOException::class) + private fun createRestSenders( + numDevices: Int, + retriever: SchemaRetriever, + restProxy: ServerConfig, + useCompression: Boolean, + authConfig: AuthConfig? + ): List { + val sharedState = ConnectionState(TimeoutConfig(Duration.ofSeconds(10))) + return (0 until numDevices) + .map { + restKafkaSender { + schemaRetriever = retriever + connectionState = sharedState + + httpClient { + defaultRequest { + url(restProxy.urlString) + } + if (authConfig != null) { + install(Auth) { + clientCredentials( + ClientCredentialsConfig( + authConfig.tokenUrl, + authConfig.clientId, + authConfig.clientSecret + ).copyWithEnv(), + restProxy.host, + ) + } + } + if (useCompression) { + install(GzipContentEncoding) + } + timeout(Duration.ofSeconds(10)) + } + } + } + } + + /** Start sending data. */ + @Throws(IOException::class) + suspend fun start() { + job = SupervisorJob() + withContext(Dispatchers.Default + job!!) { + for (device in devices) { + launch { + with(device) { + run() + } + } + } + for (file in files) { + launch { + file.send() + logger.info("Sent data {}", file) + } + } + } + job = null + } + + /** Stop sending data and clean up all resources. */ + @Throws(IOException::class, InterruptedException::class, SchemaValidationException::class) + suspend fun shutdown() { + job?.run { + logger.info("Shutting down mock devices") + cancel() + join() + } + for (device in devices) { + device.checkException() + } + } + + private fun defaultDataConfig(): List { + val acceleration = MockDataConfig() + acceleration.topic = "android_empatica_e4_acceleration" + acceleration.frequency = 32 + acceleration.valueSchema = EmpaticaE4Acceleration::class.java.name + acceleration.setInterval(-2.0, 2.0) + acceleration.valueFields = listOf("x", "y", "z") + val battery = MockDataConfig() + battery.topic = "android_empatica_e4_battery_level" + battery.valueSchema = EmpaticaE4BatteryLevel::class.java.name + battery.frequency = 1 + battery.setInterval(0.0, 1.0) + battery.setValueField("batteryLevel") + val bvp = MockDataConfig() + bvp.topic = "android_empatica_e4_blood_volume_pulse" + bvp.valueSchema = EmpaticaE4BloodVolumePulse::class.java.name + bvp.frequency = 64 + bvp.setInterval(60.0, 90.0) + bvp.setValueField("bloodVolumePulse") + val eda = MockDataConfig() + eda.topic = "android_empatica_e4_electrodermal_activity" + eda.valueSchema = EmpaticaE4ElectroDermalActivity::class.java.name + eda.setValueField("electroDermalActivity") + eda.frequency = 4 + eda.setInterval(0.01, 0.05) + val ibi = MockDataConfig() + ibi.topic = "android_empatica_e4_inter_beat_interval" + ibi.valueSchema = EmpaticaE4InterBeatInterval::class.java.name + ibi.setValueField("interBeatInterval") + ibi.frequency = 1 + ibi.setInterval(40.0, 150.0) + val temperature = MockDataConfig() + temperature.topic = "android_empatica_e4_temperature" + temperature.valueSchema = EmpaticaE4Temperature::class.java.name + temperature.frequency = 4 + temperature.setInterval(20.0, 60.0) + temperature.setValueField("temperature") + return listOf(acceleration, battery, bvp, eda, ibi, temperature) + } + + private fun createGenerators( + configs: List + ): List> = configs.mapNotNull { config -> + if (config.dataFile == null) { + RecordGenerator(config, ObservationKey::class.java) + } else { + null + } + } + + @Throws(IOException::class, CsvValidationException::class) + private fun createMockFiles( + configs: List, + dataRoot: Path? + ): List { + val now = Instant.now() + var parent = dataRoot + if (parent == null) { + parent = Paths.get(".").toAbsolutePath() + } + return configs.mapNotNull { config -> + if (config.dataFile != null) { + logger.info("Reading mock data from {}", config.dataFile) + MockCsvParser(config, parent, now, retriever) + } else { + logger.info("Generating mock data from {}", config) + null + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(MockProducer::class.java) + + /** + * Runs the MockProducer with given YAML mock config file. + */ + @JvmStatic + fun main(args: Array) { + if (args.size != 1) { + logger.error("This command needs a mock file argument") + exitProcess(1) + } + val mockFile = Paths.get(args[0]).toAbsolutePath() + val config: BasicMockConfig + try { + config = YamlConfigLoader().load(mockFile, BasicMockConfig::class.java) + } catch (ex: IOException) { + logger.error("Failed to load given mock file {}: {}", mockFile, ex.message) + exitProcess(1) + } + try { + val producer = MockProducer(config, mockFile.parent) + runBlocking { + producer.start() + } + if (producer.devices.isNotEmpty()) { + waitForProducer(producer, config.duration) + } + } catch (ex: IllegalArgumentException) { + logger.error("{}", ex.message) + exitProcess(1) + } catch (e: InterruptedException) { + // during shutdown, not that important. Will shutdown again. + } catch (ex: Exception) { + logger.error("Failed to start mock producer", ex) + exitProcess(1) + } + } + + /** Wait for given duration and then stop the producer. */ + @Throws(IOException::class, InterruptedException::class, SchemaValidationException::class) + private fun waitForProducer(producer: MockProducer, duration: Long) { + val isShutdown = AtomicBoolean(false) + Runtime.getRuntime().addShutdownHook(Thread { + try { + if (!isShutdown.get()) { + runBlocking { + producer.shutdown() + } + } + } catch (ex: InterruptedException) { + logger.warn("Shutdown interrupted", ex) + } catch (ex: Exception) { + logger.warn("Failed to shutdown producer", ex) + } + }) + if (duration <= 0L) { + try { + logger.info("Producing data until interrupted") + Thread.sleep(Long.MAX_VALUE) + } catch (ex: InterruptedException) { + // this is intended + } + } else { + try { + logger.info("Producing data for {} seconds", duration / 1000.0) + Thread.sleep(duration) + } catch (ex: InterruptedException) { + logger.warn("Data producing interrupted") + } + runBlocking { + producer.shutdown() + } + isShutdown.set(true) + logger.info("Producing data done.") + } + } + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.java deleted file mode 100644 index 976d0950..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.radarbase.mock.data; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -public class HeaderHierarchy { - private final int index; - private final Map children; - private final HeaderHierarchy parent; - private final String name; - - /** Root node. */ - public HeaderHierarchy() { - this(null, -1, null); - } - - /** - * Header hierarchy child node. Usually accessed via {@link #add(int, List)} - * - * @param name name of the node - * @param index index in the csv file. -1 if not a leaf node. - * @param parent parent node. - */ - public HeaderHierarchy(String name, int index, HeaderHierarchy parent) { - this.name = name; - this.index = index; - this.children = new HashMap<>(); - this.parent = parent; - } - - /** - * Add child nodes to this node. Each item in the list will become a new level down, and the - * last item will become a leaf node with given index. - * - * @param i index if the item. - * @param item list of item elements, each one level deeper than the previous. - */ - public void add(int i, List item) { - Objects.requireNonNull(item); - if (item.isEmpty()) { - return; - } - HeaderHierarchy child = this.children.computeIfAbsent(item.get(0), - k -> new HeaderHierarchy(k, item.size() == 1 ? i : -1, this)); - child.add(i, item.subList(1, item.size())); - } - - /** - * Get the index of current element. - * - * @return index - * @throws IllegalStateException if current node is not a leaf. - */ - public int getIndex() { - if (index == -1) { - throw new IllegalStateException("Header does not exist"); - } - return index; - } - - public Map getChildren() { - return children; - } - - private void appendTo(StringBuilder builder) { - if (parent != null) { - parent.appendTo(builder); - } - if (name != null) { - builder.append('.').append(name); - } - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(50); - appendTo(builder); - if (index >= 0) { - builder.append('[') - .append(index) - .append(']'); - } - return builder.toString(); - } - - public String getName() { - return name; - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt new file mode 100644 index 00000000..deb18f3d --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt @@ -0,0 +1,64 @@ +package org.radarbase.mock.data + +import java.util.* + +/** + * Header hierarchy child node. Usually accessed via [.add] + * + * @param name name of the node + * @param index index in the csv file. -1 if not a leaf node. + * @param parent parent node. + */ +class HeaderHierarchy( + val name: String? = null, + index: Int = -1, + private val parent: HeaderHierarchy? = null +) { + /** The index of current element. */ + val index: Int = index + get() { + check(field != -1) { "Header does not exist" } + return field + } + + private val _children: MutableMap = HashMap() + val children: Map + get() = _children.toMap() + + /** + * Add child nodes to this node. Each item in the list will become a new level down, and the + * last item will become a leaf node with given index. + * + * @param index index of the item. + * @param item list of item elements, each one level deeper than the previous. + */ + fun add(index: Int, item: List) { + Objects.requireNonNull(item) + if (item.isEmpty()) { + return + } + val child = _children.computeIfAbsent(item[0]) { k -> + HeaderHierarchy(k, if (item.size == 1) index else -1, this) + } + child.add(index, item.subList(1, item.size)) + } + + private fun StringBuilder.appendHeader() { + parent?.run { + appendHeader() + } + if (name != null) { + append('.') + append(name) + } + } + + override fun toString(): String = buildString(50) { + appendHeader() + if (index >= 0) { + append('[') + append(index) + append(']') + } + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java deleted file mode 100644 index 96d9b7ba..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.mock.data; - -import com.opencsv.CSVReader; -import com.opencsv.exceptions.CsvValidationException; -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.util.Base64; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.specific.SpecificRecord; -import org.radarbase.data.Record; -import org.radarbase.mock.config.MockDataConfig; -import org.radarbase.producer.rest.SchemaRetriever; -import org.radarbase.topic.AvroTopic; - -/** - * Parse mock data from a CSV file. - */ -@SuppressWarnings("PMD.GodClass") -public class MockCsvParser implements Closeable { - private final AvroTopic topic; - private final CSVReader csvReader; - private final BufferedReader bufferedReader; - private final Instant startTime; - private final Duration rowDuration; - private final HeaderHierarchy headers; - private String[] currentLine; - private int row; - private long rowTime; - - /** - * Base constructor. - * @param config configuration of the stream. - * @param root parent directory of the data file. - * @param retriever schema retriever to fetch schema with if none is supplied. - * @throws IllegalArgumentException if the second row has the wrong number of columns - */ - public MockCsvParser(MockDataConfig config, Path root, Instant startTime, - SchemaRetriever retriever) - throws IOException, CsvValidationException { - Schema keySchema; - Schema valueSchema; - try { - AvroTopic specificTopic = config.parseAvroTopic(); - keySchema = specificTopic.getKeySchema(); - valueSchema = specificTopic.getValueSchema(); - } catch (IllegalStateException ex) { - Objects.requireNonNull(retriever, "Cannot instantiate value schema without " - + "schema retriever."); - keySchema = AvroTopic.Companion.parseSpecificRecord(config.getKeySchema()).getSchema(); - valueSchema = retriever.getBySubjectAndVersion( - config.getTopic(), true, 0).getSchema(); - } - topic = new AvroTopic<>(config.getTopic(), - keySchema, valueSchema, - GenericRecord.class, GenericRecord.class); - - this.startTime = startTime; - row = 0; - rowDuration = Duration.ofMillis((long)(1.0 / config.getFrequency())); - rowTime = this.startTime.toEpochMilli(); - - Path dataFile = config.getDataFile(root); - bufferedReader = Files.newBufferedReader(dataFile); - csvReader = new CSVReader(bufferedReader); - headers = new HeaderHierarchy(); - String[] header = csvReader.readNext(); - for (int i = 0; i < header.length; i++) { - headers.add(i, List.of(header[i].split("\\."))); - } - currentLine = csvReader.readNext(); - } - - public AvroTopic getTopic() { - return topic; - } - - /** - * Read the next record in the file. - * @throws NullPointerException if a field from the Avro schema is missing as a column - * @throws IllegalArgumentException if the row has the wrong number of columns - * @throws IllegalStateException if a next row is not available - * @throws IOException if the next row could not be read - */ - public Record next() throws IOException, CsvValidationException { - if (!hasNext()) { - throw new IllegalStateException("No next record available"); - } - - GenericRecord key = parseRecord(currentLine, topic.getKeySchema(), - headers.getChildren().get("key")); - GenericRecord value = parseRecord(currentLine, topic.getValueSchema(), - headers.getChildren().get("value")); - incrementRow(); - return new Record<>(key, value); - } - - private void incrementRow() throws CsvValidationException, IOException { - currentLine = csvReader.readNext(); - row++; - rowTime = startTime - .plus(rowDuration.multipliedBy(row)) - .toEpochMilli(); - } - - /** - * Whether there is a next record in the file. - */ - public boolean hasNext() { - return currentLine != null; - } - - private GenericRecord parseRecord(String[] rawValues, Schema schema, HeaderHierarchy headers) { - GenericRecordBuilder record = new GenericRecordBuilder(schema); - Map children = headers.getChildren(); - - for (Field field : schema.getFields()) { - HeaderHierarchy child = children.get(field.name()); - if (child != null) { - record.set(field, parseValue(rawValues, field.schema(), child)); - } - } - - return record.build(); - } - - /** Parse value from Schema. */ - public Object parseValue(String[] rawValues, Schema schema, HeaderHierarchy headers) { - switch (schema.getType()) { - case NULL: - case INT: - case LONG: - case FLOAT: - case DOUBLE: - case BOOLEAN: - case STRING: - case ENUM: - case BYTES: - return parseScalar(rawValues, schema, headers); - case UNION: - return parseUnion(rawValues, schema, headers); - case RECORD: - return parseRecord(rawValues, schema, headers); - case ARRAY: - return parseArray(rawValues, schema, headers); - case MAP: - return parseMap(rawValues, schema, headers); - default: - throw new IllegalArgumentException("Cannot handle schemas of type " - + schema.getType() + " in " + headers); - } - } - - private Object parseScalar(String[] rawValues, Schema schema, HeaderHierarchy headers) { - int fieldHeader = headers.getIndex(); - if (fieldHeader >= rawValues.length) { - throw new IllegalArgumentException("Row is missing value for " + headers.getName()); - } - String fieldString = rawValues[fieldHeader] - .replace("${timeSeconds}", Double.toString(rowTime / 1000.0)) - .replace("${timeMillis}", Long.toString(rowTime)); - - return parseScalar(fieldString, schema, headers); - } - - private static Object parseScalar(String fieldString, Schema schema, HeaderHierarchy headers) { - switch (schema.getType()) { - case NULL: - if (fieldString == null || fieldString.isEmpty() || fieldString.equals("null")) { - return null; - } else { - throw new IllegalArgumentException("Cannot parse " + fieldString + " as null"); - } - case INT: - return Integer.parseInt(fieldString); - case LONG: - return Long.parseLong(fieldString); - case FLOAT: - return Float.parseFloat(fieldString); - case DOUBLE: - return Double.parseDouble(fieldString); - case BOOLEAN: - return Boolean.parseBoolean(fieldString); - case STRING: - return fieldString; - case ENUM: - return parseEnum(schema, fieldString); - case BYTES: - return parseBytes(fieldString); - default: - throw new IllegalArgumentException("Cannot handle scalar schema of type " - + schema.getType() + " in " + headers); - } - } - - private Map parseMap(String[] rawValues, Schema schema, - HeaderHierarchy headers) { - Map children = headers.getChildren(); - Map map = new LinkedHashMap<>(children.size() * 4 / 3); - for (HeaderHierarchy child : children.values()) { - map.put(child.getName(), parseValue(rawValues, schema.getValueType(), child)); - } - return map; - } - - private static ByteBuffer parseBytes(String fieldString) { - byte[] result = Base64.getDecoder() - .decode(fieldString.getBytes(StandardCharsets.UTF_8)); - return ByteBuffer.wrap(result); - } - - private Object parseUnion(String[] rawValues, Schema schema, HeaderHierarchy headers) { - for (Schema subschema : schema.getTypes()) { - try { - return parseValue(rawValues, subschema, headers); - } catch (IllegalArgumentException ex) { - // skip bad union member - } - } - throw new IllegalArgumentException("Cannot handle union types " - + schema.getTypes() + " in " + headers); - } - - private List parseArray(String[] rawValues, Schema schema, HeaderHierarchy headers) { - Map children = headers.getChildren(); - int arrayLength = children.keySet().stream() - .mapToInt(headerName -> Integer.parseInt(headerName) + 1) - .max() - .orElse(0); - - GenericData.Array array = new GenericData.Array<>(arrayLength, schema); - for (int i = 0; i < arrayLength; i++) { - HeaderHierarchy child = children.get(String.valueOf(i)); - if (child != null) { - array.add(i, parseValue(rawValues, schema.getElementType(), child)); - } else { - array.add(i, null); - } - } - return array; - } - - private static GenericData.EnumSymbol parseEnum(Schema schema, String fieldString) { - return new GenericData.EnumSymbol(schema, fieldString); - } - - @Override - public void close() throws IOException { - csvReader.close(); - bufferedReader.close(); - } - - @Override - public String toString() { - return "MockCsvParser{" + "topic=" + topic + '}'; - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt new file mode 100644 index 00000000..3c135962 --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt @@ -0,0 +1,278 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.mock.data + +import com.opencsv.CSVReader +import com.opencsv.exceptions.CsvValidationException +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecord +import org.apache.avro.generic.GenericRecordBuilder +import org.apache.avro.specific.SpecificRecord +import org.radarbase.data.Record +import org.radarbase.mock.config.MockDataConfig +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.topic.AvroTopic +import org.radarbase.topic.AvroTopic.Companion.parseSpecificRecord +import java.io.BufferedReader +import java.io.Closeable +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import java.util.* +import kotlin.io.path.bufferedReader + +/** + * Parse mock data from a CSV file. + * + * @param config configuration of the stream. + * @param root parent directory of the data file. + * @param retriever schema retriever to fetch schema with if none is supplied. + * @throws IllegalArgumentException if the second row has the wrong number of columns + */ +class MockCsvParser constructor( + private val config: MockDataConfig, + root: Path?, + private val startTime: Instant, + private val retriever: SchemaRetriever +) : Closeable { + lateinit var topic: AvroTopic + private val csvReader: CSVReader + private val bufferedReader: BufferedReader + private val rowDuration: Duration = Duration.ofMillis((1.0 / config.frequency).toLong()) + private val headers: HeaderHierarchy + private var currentLine: Array? + private var row: Int = 0 + private var rowTime: Long = this.startTime.toEpochMilli() + + init { + bufferedReader = config.getDataFile(root).bufferedReader() + csvReader = CSVReader(bufferedReader) + headers = HeaderHierarchy() + val header = csvReader.readNext() + for (i in header.indices) { + headers.add( + i, + header[i].split("\\.".toRegex()).dropLastWhile { it.isEmpty() } + ) + } + currentLine = csvReader.readNext() + } + + suspend fun initialize() { + val (keySchema, valueSchema) = try { + val specificTopic = config.parseAvroTopic() + Pair(specificTopic.keySchema, specificTopic.valueSchema) + } catch (ex: IllegalStateException) { + Pair( + parseSpecificRecord(config.keySchema).schema, + retriever.getByVersion(config.topic, true, 0).schema, + ) + } + + topic = AvroTopic( + config.topic, + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } + + /** + * Read the next record in the file. + * @throws NullPointerException if a field from the Avro schema is missing as a column + * @throws IllegalArgumentException if the row has the wrong number of columns + * @throws IllegalStateException if a next row is not available + * @throws IOException if the next row could not be read + */ + @Throws(IOException::class, CsvValidationException::class) + operator fun next(): Record { + check(hasNext()) { "No next record available" } + val key = parseRecord( + currentLine, + topic.keySchema, + checkNotNull(headers.children["key"]) { "Missing key fields" } + ) + val value = parseRecord( + currentLine, + topic.valueSchema, + checkNotNull(headers.children["value"]) { "Missing value fields" } + ) + incrementRow() + return Record(key, value) + } + + @Throws(CsvValidationException::class, IOException::class) + private fun incrementRow() { + currentLine = csvReader.readNext() + row++ + rowTime = startTime + .plus(rowDuration.multipliedBy(row.toLong())) + .toEpochMilli() + } + + /** + * Whether there is a next record in the file. + */ + operator fun hasNext(): Boolean { + return currentLine != null + } + + private fun parseRecord( + rawValues: Array?, + schema: Schema, + headers: HeaderHierarchy + ): GenericRecord { + val record = GenericRecordBuilder(schema) + val children = headers.children + for (field in schema.fields) { + val child = children[field.name()] + if (child != null) { + record[field] = parseValue(rawValues, field.schema(), child) + } + } + return record.build() + } + + /** Parse value from Schema. */ + fun parseValue(rawValues: Array?, schema: Schema, headers: HeaderHierarchy): Any? { + return when (schema.type) { + Schema.Type.NULL, Schema.Type.INT, Schema.Type.LONG, Schema.Type.FLOAT, Schema.Type.DOUBLE, Schema.Type.BOOLEAN, Schema.Type.STRING, Schema.Type.ENUM, Schema.Type.BYTES -> parseScalar( + rawValues, + schema, + headers + ) + + Schema.Type.UNION -> parseUnion(rawValues, schema, headers) + Schema.Type.RECORD -> parseRecord(rawValues, schema, headers) + Schema.Type.ARRAY -> parseArray(rawValues, schema, headers) + Schema.Type.MAP -> parseMap(rawValues, schema, headers) + else -> throw IllegalArgumentException( + "Cannot handle schemas of type " + + schema.type + " in " + headers + ) + } + } + + private fun parseScalar( + rawValues: Array?, + schema: Schema, + headers: HeaderHierarchy + ): Any? { + val fieldHeader = headers.index + require(fieldHeader < rawValues!!.size) { "Row is missing value for " + headers.name } + val fieldString = rawValues[fieldHeader] + .replace("\${timeSeconds}", java.lang.Double.toString(rowTime / 1000.0)) + .replace("\${timeMillis}", java.lang.Long.toString(rowTime)) + return parseScalar(fieldString, schema, headers) + } + + private fun parseMap( + rawValues: Array?, schema: Schema, + headers: HeaderHierarchy + ): Map = buildMap { + for (child in headers.children.values) { + put(child.name!!, parseValue(rawValues, schema.valueType, child)) + } + } + + private fun parseUnion( + rawValues: Array?, + schema: Schema, + headers: HeaderHierarchy + ): Any = requireNotNull( + schema.types.firstNotNullOfOrNull { subSchema -> + try { + parseValue(rawValues, subSchema, headers) + } catch (ex: IllegalArgumentException) { + // skip bad union member + null + } + } + ) { "Cannot handle union types ${schema.types} in $headers" } + + private fun parseArray( + rawValues: Array?, + schema: Schema, + headers: HeaderHierarchy + ): List { + val children = headers.children + val arrayLength = children.keys.stream() + .mapToInt { headerName: String -> headerName.toInt() + 1 } + .max() + .orElse(0) + val array = GenericData.Array(arrayLength, schema) + for (i in 0 until arrayLength) { + val child = children[i.toString()] + if (child != null) { + array.add(i, parseValue(rawValues, schema.elementType, child)) + } else { + array.add(i, null) + } + } + return array + } + + @Throws(IOException::class) + override fun close() { + csvReader.close() + bufferedReader.close() + } + + override fun toString(): String { + return "MockCsvParser{topic=$topic}" + } + + companion object { + private fun parseScalar( + fieldString: String?, + schema: Schema, + headers: HeaderHierarchy + ): Any? { + return when (schema.type) { + Schema.Type.NULL -> if (fieldString.isNullOrEmpty() || fieldString == "null") { + null + } else { + throw IllegalArgumentException("Cannot parse $fieldString as null") + } + Schema.Type.INT -> fieldString!!.toInt() + Schema.Type.LONG -> fieldString!!.toLong() + Schema.Type.FLOAT -> fieldString!!.toFloat() + Schema.Type.DOUBLE -> fieldString!!.toDouble() + Schema.Type.BOOLEAN -> java.lang.Boolean.parseBoolean(fieldString) + Schema.Type.STRING -> fieldString + Schema.Type.ENUM -> parseEnum(schema, fieldString) + Schema.Type.BYTES -> parseBytes(fieldString) + else -> throw IllegalArgumentException( + "Cannot handle scalar schema of type " + + schema.type + " in " + headers + ) + } + } + + private fun parseBytes(fieldString: String?): ByteBuffer { + val result = Base64.getDecoder() + .decode(fieldString!!.toByteArray(StandardCharsets.UTF_8)) + return ByteBuffer.wrap(result) + } + + private fun parseEnum(schema: Schema, fieldString: String?): GenericData.EnumSymbol { + return GenericData.EnumSymbol(schema, fieldString) + } + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java deleted file mode 100644 index 01b3ce70..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.mock.data; - -import com.opencsv.exceptions.CsvValidationException; -import java.io.IOException; -import java.nio.file.Path; -import java.time.Instant; -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.generic.GenericRecord; -import org.radarbase.data.Record; -import org.radarbase.mock.config.MockDataConfig; -import org.radarbase.producer.rest.SchemaRetriever; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * CSV files must be validated before using since MockAggregator can handle only files containing - * unique User_ID and Source_ID and having increasing timestamp at each raw. - */ -public class MockRecordValidator { - private static final Logger logger = LoggerFactory.getLogger(MockRecordValidator.class); - private final MockDataConfig config; - private final long duration; - private final Path root; - private int timePos; - private double time; - private double startTime; - private final SchemaRetriever retriever; - - /** Create a new validator for given configuration. */ - public MockRecordValidator(MockDataConfig config, long duration, Path root, - SchemaRetriever retriever) { - this.config = config; - this.duration = duration; - this.root = root; - this.retriever = retriever; - this.time = Double.NaN; - this.startTime = Double.NaN; - } - - /** - * Verify whether the CSV file can be used or not. - * @throws IllegalArgumentException if the CSV file does not respect the constraints. - */ - public void validate() { - Instant now = Instant.now(); - try (MockCsvParser parser = new MockCsvParser(config, root, now, retriever)) { - if (!parser.hasNext()) { - throw new IllegalArgumentException("CSV file is empty"); - } - - Schema valueSchema = config.parseAvroTopic().getValueSchema(); - Field timeField = valueSchema.getField("timeReceived"); - if (timeField == null) { - timeField = valueSchema.getField("time"); - } - timePos = timeField.pos(); - - Record last = null; - long line = 1L; - - while (parser.hasNext()) { - Record record = parser.next(); - checkRecord(record, last, line++); - last = record; - } - - checkDuration(); - checkFrequency(line); - } catch (IOException | CsvValidationException e) { - error("Cannot open file", -1, e); - } - } - - private void checkFrequency(long line) { - long expected = config.getFrequency() * duration / 1000L + 1L; - if (line != config.getFrequency() * duration / 1000L + 1L) { - error("CSV contains fewer messages " + line + " than expected " + expected, -1L, null); - } - } - - private void checkRecord(Record record, - Record last, long line) { - double previousTime = time; - time = (Double) record.getValue().get(timePos); - - if (last == null) { - // no checks, only update initial time stamp - startTime = time; - } else if (!last.getKey().equals(record.getKey())) { - error("It is possible to test only one user/source at time.", line, null); - } else if (time < previousTime) { - error("Time must increase row by row.", line, null); - } - } - - private void error(String message, long line, Exception ex) { - StringBuilder messageBuilder = new StringBuilder(150); - messageBuilder - .append(config.getDataFile()) - .append(" with topic ") - .append(config.getTopic()) - .append(" is invalid"); - if (line > 0L) { - messageBuilder - .append(" on line ") - .append(line); - } - String fullMessage = messageBuilder - .append(". ") - .append(message) - .toString(); - logger.error(fullMessage); - throw new IllegalArgumentException(fullMessage, ex); - } - - private void checkDuration() { - long interval = (long)(time * 1000d) - (long)(startTime * 1000d); - - // add a margin of 50 for clock error purposes - long margin = 50L; - - if (duration <= interval - margin || duration > interval + 1000L + margin) { - error("Data does not cover " + duration + " milliseconds but " - + interval + " instead.", -1L, null); - } - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt new file mode 100644 index 00000000..0e7b4b87 --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt @@ -0,0 +1,143 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.mock.data + +import com.opencsv.exceptions.CsvValidationException +import org.apache.avro.generic.GenericRecord +import org.apache.avro.specific.SpecificRecord +import org.radarbase.data.Record +import org.radarbase.mock.config.MockDataConfig +import org.radarbase.producer.schema.SchemaRetriever +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Path +import java.time.Instant + +/** + * CSV files must be validated before using since MockAggregator can handle only files containing + * unique User_ID and Source_ID and having increasing timestamp at each raw. + */ +class MockRecordValidator( + private val config: MockDataConfig, private val duration: Long, private val root: Path, + private val retriever: SchemaRetriever +) { + private var timePos = 0 + private var time: Double + private var startTime: Double + + /** Create a new validator for given configuration. */ + init { + time = Double.NaN + startTime = Double.NaN + } + + /** + * Verify whether the CSV file can be used or not. + * @throws IllegalArgumentException if the CSV file does not respect the constraints. + */ + suspend fun validate() { + val now = Instant.now() + try { + MockCsvParser(config, root, now, retriever).use { parser -> + parser.initialize() + require(parser.hasNext()) { "CSV file is empty" } + val valueSchema = + config.parseAvroTopic().valueSchema + var timeField = valueSchema.getField("timeReceived") + if (timeField == null) { + timeField = valueSchema.getField("time") + } + timePos = timeField!!.pos() + var last: Record? = null + var line = 1L + while (parser.hasNext()) { + val record = parser.next() + checkRecord(record, last, line++) + last = record + } + checkDuration() + checkFrequency(line) + } + } catch (e: IOException) { + error("Cannot open file", -1, e) + } catch (e: CsvValidationException) { + error("Cannot open file", -1, e) + } + } + + private fun checkFrequency(line: Long) { + val expected = config.frequency * duration / 1000L + 1L + if (line != config.frequency * duration / 1000L + 1L) { + error("CSV contains fewer messages $line than expected $expected", -1L, null) + } + } + + private fun checkRecord( + record: Record, + last: Record?, + line: Long + ) { + val previousTime = time + time = record.value[timePos] as Double + if (last == null) { + // no checks, only update initial time stamp + startTime = time + } else if (last.key != record.key) { + error("It is possible to test only one user/source at time.", line, null) + } else if (time < previousTime) { + error("Time must increase row by row.", line, null) + } + } + + private fun error(message: String, line: Long, ex: Exception?) { + val messageBuilder = StringBuilder(150) + messageBuilder + .append(config.dataFile) + .append(" with topic ") + .append(config.topic) + .append(" is invalid") + if (line > 0L) { + messageBuilder + .append(" on line ") + .append(line) + } + val fullMessage = messageBuilder + .append(". ") + .append(message) + .toString() + logger.error(fullMessage) + throw IllegalArgumentException(fullMessage, ex) + } + + private fun checkDuration() { + val interval = (time * 1000.0).toLong() - (startTime * 1000.0).toLong() + + // add a margin of 50 for clock error purposes + val margin = 50L + if (duration <= interval - margin || duration > interval + 1000L + margin) { + error( + "Data does not cover " + duration + " milliseconds but " + + interval + " instead.", -1L, null + ) + } + } + + companion object { + private val logger = LoggerFactory.getLogger( + MockRecordValidator::class.java + ) + } +} diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java b/radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java index 933884b3..a4d6d27c 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/model/MockAggregator.java @@ -26,7 +26,7 @@ import org.apache.avro.Schema; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.MockCsvParser; -import org.radarbase.producer.rest.SchemaRetriever; +import org.radarbase.producer.schema.SchemaRetriever; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java deleted file mode 100644 index 28492cd2..00000000 --- a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -/** - * Oscilloscope gives out a regular beat, at a given frequency per second. The intended way to use - * this is with a do-while loop, with the {@link #beat()} retrieved at the start of the loop, and - * {@link #willRestart()} in the condition of the loop. - */ -public class Oscilloscope { - private final int frequency; - private final long timeStep; - - private int iteration; - private long baseTime; - - /** - * Frequency which will give beat at given frequency per second. - * @param frequency number of beats per second. - */ - public Oscilloscope(int frequency) { - this.frequency = frequency; - this.baseTime = System.nanoTime(); - this.timeStep = 1_000_000_000L / this.frequency; - this.iteration = 0; - } - - /** Restart the oscilloscope at zero. */ - public void reset() { - this.baseTime = System.nanoTime(); - this.iteration = 0; - } - - /** Whether the next beat will restart at one. */ - public boolean willRestart() { - return iteration % frequency == 0; - } - - /** - * One oscilloscope beat, sleeping if necessary to not exceed the frequency per second. The beat - * number starts at one, increases to the frequency, and then goes to one again. - * @return one up to the given frequency - * @throws InterruptedException when the sleep was interrupted. - */ - public int beat() throws InterruptedException { - long currentTime = System.nanoTime(); - long nextSend = baseTime + iteration * timeStep; - if (currentTime < nextSend) { - long timeToSleep = nextSend - currentTime; - Thread.sleep(timeToSleep / 1_000_000L, ((int) timeToSleep) % 1_000_000); - } - int beatNumber = iteration % frequency + 1; - iteration++; - return beatNumber; - } -} diff --git a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt new file mode 100644 index 00000000..0ce21e25 --- /dev/null +++ b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.util + +import kotlinx.coroutines.delay +import org.slf4j.LoggerFactory +import java.util.concurrent.atomic.AtomicInteger + +/** + * Oscilloscope gives out a regular beat, at a given frequency per second. The intended way to use + * this is with a do-while loop, with the [.beat] retrieved at the start of the loop, and + * [.willRestart] in the condition of the loop. + */ +class Oscilloscope( + private val frequency: Int +) { + private val timeStep: Long = 1_000_000_000L / frequency + private val baseTime: Long = System.nanoTime() + private var iteration: AtomicInteger = AtomicInteger(0) + + /** Whether the next beat will restart at one. */ + fun willRestart(): Boolean { + return iteration.get() % frequency == 0 + } + + /** + * One oscilloscope beat, sleeping if necessary to not exceed the frequency per second. The beat + * number starts at one, increases to the frequency, and then goes to one again. + * @return one up to the given frequency + * @throws InterruptedException when the sleep was interrupted. + */ + @Throws(InterruptedException::class) + suspend fun beat(): Int { + val currentTime = System.nanoTime() + val currentIteration = iteration.getAndIncrement() + val timeToSleep = baseTime + currentIteration * timeStep - currentTime + if (timeToSleep > 0) { + val millis = (timeToSleep / 1_000_000L).coerceAtLeast(1L) + logger.info("delaying {} millis", millis) + delay(millis) + } + return currentIteration % frequency + 1 + } + + companion object { + private val logger = LoggerFactory.getLogger(Oscilloscope::class.java) + } +} diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java index 06c7df1e..9420f175 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/CsvGeneratorTest.java @@ -16,9 +16,9 @@ package org.radarbase.mock; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvValidationException; @@ -28,9 +28,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.CsvGenerator; import org.radarbase.mock.data.MockRecordValidatorTest; @@ -38,19 +37,16 @@ import org.radarcns.kafka.ObservationKey; public class CsvGeneratorTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private MockDataConfig makeConfig() throws IOException { + private MockDataConfig makeConfig(Path folder) throws IOException { return MockRecordValidatorTest.makeConfig(folder); } @Test - public void generateMockConfig() throws IOException, CsvValidationException { + public void generateMockConfig(@TempDir Path folder) throws IOException, CsvValidationException { CsvGenerator generator = new CsvGenerator(); - MockDataConfig config = makeConfig(); - generator.generate(config, 100_000L, folder.getRoot().toPath()); + MockDataConfig config = makeConfig(folder); + generator.generate(config, 100_000L, folder.getRoot()); Path p = Paths.get(config.getDataFile()); try (Reader reader = Files.newBufferedReader(p); @@ -74,10 +70,10 @@ public void generateMockConfig() throws IOException, CsvValidationException { } @Test - public void generateGenerator() throws IOException, CsvValidationException { + public void generateGenerator(@TempDir Path folder) throws IOException, CsvValidationException { CsvGenerator generator = new CsvGenerator(); - MockDataConfig config = makeConfig(); + MockDataConfig config = makeConfig(folder); final String time = Double.toString(System.currentTimeMillis() / 1000d); diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java index 95aed120..d392586b 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/RecordGeneratorTest.java @@ -16,13 +16,13 @@ package org.radarbase.mock; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.Iterator; import org.apache.avro.specific.SpecificRecord; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.radarbase.data.Record; import org.radarbase.mock.config.MockDataConfig; import org.radarbase.mock.data.RecordGenerator; @@ -35,7 +35,7 @@ public class RecordGeneratorTest { @Test - public void generate() throws Exception { + public void generate() { MockDataConfig config = new MockDataConfig(); config.setTopic("test"); config.setFrequency(10); @@ -65,7 +65,7 @@ public void generate() throws Exception { } @Test - public void getHeaders() throws Exception { + public void getHeaders() { MockDataConfig config = new MockDataConfig(); config.setTopic("test"); config.setValueSchema(EmpaticaE4Acceleration.class.getName()); diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java deleted file mode 100644 index 3c09709a..00000000 --- a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.mock.data; - -import static org.junit.Assert.assertThrows; - -import java.io.IOException; -import java.io.Writer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.radarbase.mock.config.MockDataConfig; -import org.radarbase.producer.rest.SchemaRetriever; -import org.radarcns.kafka.ObservationKey; -import org.radarcns.monitor.application.ApplicationServerStatus; -import org.radarcns.passive.phone.PhoneAcceleration; -import org.radarcns.passive.phone.PhoneLight; - -public class MockRecordValidatorTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private Path root; - private SchemaRetriever retriever; - - private MockDataConfig makeConfig() throws IOException { - return makeConfig(folder); - } - - @Before - public void setUp() { - root = folder.getRoot().toPath(); - retriever = null; - } - - public static MockDataConfig makeConfig(TemporaryFolder folder) throws IOException { - MockDataConfig config = new MockDataConfig(); - config.setKeySchema(ObservationKey.class.getName()); - config.setDataFile(folder.newFile().getAbsolutePath()); - config.setValueSchema(PhoneLight.class.getName()); - config.setValueField("light"); - config.setTopic("test"); - return config; - } - - @Test - public void validateEnum() throws IOException { - MockDataConfig config = makeConfig(); - config.setValueSchema(ApplicationServerStatus.class.getName()); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.serverStatus,value.ipAddress\n"); - writer.append("test,a,b,1,UNKNOWN,\n"); - writer.append("test,a,b,2,CONNECTED,\n"); - } - - new MockRecordValidator(config, 2_000L, root, retriever).validate(); - } - - - @Test - public void validateEnumGenerated() throws IOException { - MockDataConfig config = makeConfig(); - config.setValueSchema(ApplicationServerStatus.class.getName()); - config.setValueField("serverStatus"); - CsvGenerator generator = new CsvGenerator(); - generator.generate(config, 2_000L, root); - new MockRecordValidator(config, 2_000L, root, retriever).validate(); - } - - @Test - public void validate() throws Exception { - CsvGenerator generator = new CsvGenerator(); - - MockDataConfig config = makeConfig(); - generator.generate(config, 100_000L, root); - - new MockRecordValidator(config, 100_000L, root, retriever).validate(); - } - - @Test - public void validateWrongDuration() throws Exception { - CsvGenerator generator = new CsvGenerator(); - - MockDataConfig config = makeConfig(); - generator.generate(config, 100_000L, root); - - assertValidateThrows(IllegalArgumentException.class, config); - } - - @Test - public void validateCustom() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,b,1,1,1\n"); - writer.append("test,a,b,1,2,1\n"); - } - - assertValidate(config); - } - - @Test - public void validateWrongKey() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,b,1,1,1\n"); - writer.append("test,a,c,1,2,1\n"); - } - - assertValidateThrows(IllegalArgumentException.class, config); - } - - @Test - public void validateWrongTime() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,b,1,1,1\n"); - writer.append("test,a,b,1,0,1\n"); - } - - assertValidateThrows(IllegalArgumentException.class, config); - } - - - @Test - public void validateMissingKeyField() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,1,1,1\n"); - writer.append("test,a,1,2,1\n"); - } - - assertValidateThrows(IllegalArgumentException.class, config); - } - - @Test - public void validateMissingValueField() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,b,1,1\n"); - writer.append("test,a,b,1,2\n"); - } - - assertValidateThrows(IllegalArgumentException.class, config); - } - - @Test - public void validateMissingValue() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,b,1,1\n"); - writer.append("test,a,b,1,2,1\n"); - } - - assertValidateThrows(IllegalArgumentException.class, config); - } - - @Test - public void validateWrongValueType() throws Exception { - MockDataConfig config = makeConfig(); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n"); - writer.append("test,a,b,1,1,a\n"); - writer.append("test,a,b,1,2,b\n"); - } - - assertValidateThrows(NumberFormatException.class, config); - } - - @Test - public void validateMultipleFields() throws Exception { - MockDataConfig config = makeConfig(); - config.setValueSchema(PhoneAcceleration.class.getName()); - config.setValueFields(Arrays.asList("x", "y", "z")); - - try (Writer writer = Files.newBufferedWriter(config.getDataFile(root))) { - writer.append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.x,value.y,value.z\n"); - writer.append("test,a,b,1,1,1,1,1\n"); - writer.append("test,a,b,1,2,1,1,1\n"); - } - - assertValidate(config); - } - - private void assertValidateThrows(Class ex, MockDataConfig config) { - MockRecordValidator validator = new MockRecordValidator(config, 2_000L, root, retriever); - assertThrows(ex, validator::validate); - } - - private void assertValidate(MockDataConfig config) { - new MockRecordValidator(config, 2_000L, root, retriever).validate(); - } -} diff --git a/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.kt b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.kt new file mode 100644 index 00000000..ce7b1cfa --- /dev/null +++ b/radar-commons-testing/src/test/java/org/radarbase/mock/data/MockRecordValidatorTest.kt @@ -0,0 +1,229 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.mock.data + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withContext +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.mockito.Mockito.mock +import org.radarbase.mock.config.MockDataConfig +import org.radarbase.producer.schema.SchemaRetriever +import org.radarcns.kafka.ObservationKey +import org.radarcns.monitor.application.ApplicationServerStatus +import org.radarcns.passive.phone.PhoneAcceleration +import org.radarcns.passive.phone.PhoneLight +import java.io.IOException +import java.io.Writer +import java.nio.file.Files +import java.nio.file.Path +import java.util.* +import kotlin.io.path.bufferedWriter + +@OptIn(ExperimentalCoroutinesApi::class) +class MockRecordValidatorTest { + @TempDir + lateinit var folder: Path + private lateinit var root: Path + private lateinit var retriever: SchemaRetriever + + @Throws(IOException::class) + private fun makeConfig(): MockDataConfig { + return makeConfig(folder) + } + + @BeforeEach + fun setUp(@TempDir folder: Path) { + root = folder.root + retriever = mock(SchemaRetriever::class.java) + } + + @Test + @Throws(IOException::class) + fun validateEnum() = runTest { + val config = makeConfig() + config.valueSchema = ApplicationServerStatus::class.java.name + withContext(Dispatchers.IO) { + config.getDataFile(root).bufferedWriter().use { writer -> + writer.append("key.projectId,key.userId,key.sourceId,value.time,value.serverStatus,value.ipAddress\n") + writer.append("test,a,b,1,UNKNOWN,\n") + writer.append("test,a,b,2,CONNECTED,\n") + } + } + MockRecordValidator(config, 2000L, root, retriever).validate() + } + + @Test + @Throws(IOException::class) + fun validateEnumGenerated() = runTest { + val config = makeConfig() + config.valueSchema = ApplicationServerStatus::class.java.name + config.setValueField("serverStatus") + val generator = CsvGenerator() + generator.generate(config, 2000L, root) + MockRecordValidator(config, 2000L, root, retriever).validate() + } + + @Test + @Throws(Exception::class) + fun validate() = runTest { + val generator = CsvGenerator() + val config = makeConfig() + generator.generate(config, 100000L, root) + MockRecordValidator(config, 100000L, root, retriever).validate() + } + + @Test + @Throws(Exception::class) + fun validateWrongDuration() = runTest { + val generator = CsvGenerator() + val config = makeConfig() + generator.generate(config, 100000L, root) + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateCustom() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,b,1,1,1\n") + append("test,a,b,1,2,1\n") + } + assertValidate(config) + } + + @Test + @Throws(Exception::class) + fun validateWrongKey() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,b,1,1,1\n") + append("test,a,c,1,2,1\n") + } + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateWrongTime() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,b,1,1,1\n") + append("test,a,b,1,0,1\n") + } + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateMissingKeyField() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,1,1,1\n") + append("test,a,1,2,1\n") + } + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateMissingValueField() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,b,1,1\n") + append("test,a,b,1,2\n") + } + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateMissingValue() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,b,1,1\n") + append("test,a,b,1,2,1\n") + } + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateWrongValueType() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.light\n") + append("test,a,b,1,1,a\n") + append("test,a,b,1,2,b\n") + } + assertValidateThrows(config) + } + + @Test + @Throws(Exception::class) + fun validateMultipleFields() = runTest { + val config = writeConfig { + append("key.projectId,key.userId,key.sourceId,value.time,value.timeReceived,value.x,value.y,value.z\n") + append("test,a,b,1,1,1,1,1\n") + append("test,a,b,1,2,1,1,1\n") + } + config.valueSchema = PhoneAcceleration::class.java.name + config.valueFields = listOf("x", "y", "z") + assertValidate(config) + } + + private suspend inline fun assertValidateThrows(config: MockDataConfig) { + val validator = MockRecordValidator(config, 2000L, root, retriever) + try { + validator.validate() + throw AssertionError("No exception thrown (expected ${T::class.java})") + } catch (ex: Throwable) { + if (!ex.javaClass.isAssignableFrom(T::class.java)) { + throw AssertionError("Another exception than ${T::class.java} thrown", ex) + } + } + } + + private suspend fun writeConfig(write: Writer.() -> Unit): MockDataConfig { + val config = makeConfig() + withContext(Dispatchers.IO) { + config.getDataFile(root).bufferedWriter().use(write) + } + return config + } + + private suspend fun assertValidate(config: MockDataConfig) { + MockRecordValidator(config, 2000L, root, retriever).validate() + } + + companion object { + @JvmStatic + @Throws(IOException::class) + fun makeConfig(folder: Path): MockDataConfig { + val config = MockDataConfig() + val dataFile = Files.createTempFile(folder, "datafile", ".csv") + config.keySchema = ObservationKey::class.java.name + config.dataFile = dataFile.toAbsolutePath().toString() + config.valueSchema = PhoneLight::class.java.name + config.setValueField("light") + config.topic = "test" + return config + } + } +} diff --git a/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java b/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java index 17b17dca..024dbebe 100644 --- a/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java +++ b/radar-commons-testing/src/test/java/org/radarbase/util/MetronomeTest.java @@ -20,9 +20,9 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class MetronomeTest { private void check(Metronome it, long expectedMin) { diff --git a/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java deleted file mode 100644 index 3941f7de..00000000 --- a/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarbase.util; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.core.Is.is; - -import org.junit.Test; - -public class OscilloscopeTest { - @Test - public void beat() throws Exception { - Oscilloscope oscilloscope = new Oscilloscope(128); - - long time = System.currentTimeMillis(); - int iteration = 1; - do { - int beat = oscilloscope.beat(); - assertThat(beat, is(iteration++)); - if (beat == 2) { - // time of one beat is about 1/128 seconds = 7.8125 milliseconds - long beatDiff = System.currentTimeMillis() - time; - assertThat(beatDiff, greaterThanOrEqualTo(7L)); - assertThat(beatDiff, lessThanOrEqualTo(14L)); - } - } while (!oscilloscope.willRestart()); - - // frequency must match - assertThat(iteration, is(129)); - // restarts every frequency, the willRestart function does not reset - assertThat(oscilloscope.willRestart(), is(true)); - // beat starts at 1 again - assertThat(oscilloscope.beat(), is(1)); - // total time, from one cycle to the next, is about 1 second - long cycleDiff = System.currentTimeMillis() - time; - assertThat(cycleDiff, greaterThanOrEqualTo(996L)); - assertThat(cycleDiff, lessThan(1020L)); - } -} diff --git a/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.kt b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.kt new file mode 100644 index 00000000..6272b22a --- /dev/null +++ b/radar-commons-testing/src/test/java/org/radarbase/util/OscilloscopeTest.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.util + +import kotlinx.coroutines.runBlocking +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.* +import org.junit.jupiter.api.Test + +class OscilloscopeTest { + @Test + @Throws(Exception::class) + fun beat() = runBlocking { + val oscilloscope = Oscilloscope(128) + val time = System.currentTimeMillis() + var iteration = 1 + do { + val beat = oscilloscope.beat() + assertThat(beat, `is`(iteration++)) + if (beat == 2) { + // time of one beat is about 1/128 seconds = 7.8125 milliseconds + val beatDiff = System.currentTimeMillis() - time + assertThat(beatDiff, greaterThanOrEqualTo(5L)) + assertThat(beatDiff, lessThanOrEqualTo(14L)) + } + } while (!oscilloscope.willRestart()) + + // frequency must match + assertThat(iteration, `is`(129)) + // restarts every frequency, the willRestart function does not reset + assertThat(oscilloscope.willRestart(), `is`(true)) + // beat starts at 1 again + assertThat(oscilloscope.beat(), `is`(1)) + // total time, from one cycle to the next, is about 1 second + val cycleDiff = System.currentTimeMillis() - time + assertThat(cycleDiff, greaterThanOrEqualTo(996L)) + assertThat(cycleDiff, lessThan(1020L)) + } +} diff --git a/radar-commons/build.gradle.kts b/radar-commons/build.gradle.kts index b8d95e70..177be4b3 100644 --- a/radar-commons/build.gradle.kts +++ b/radar-commons/build.gradle.kts @@ -1,3 +1,7 @@ +plugins { + kotlin("plugin.serialization") +} + description = "RADAR Common utilities library." //---------------------------------------------------------------------------// @@ -13,17 +17,21 @@ repositories { dependencies { val avroVersion: String by project api("org.apache.avro:avro:$avroVersion") + api(kotlin("reflect")) val ktorVersion: String by project - implementation("io.ktor:ktor-client-core:$ktorVersion") - implementation("io.ktor:ktor-client-cio:$ktorVersion") + api(platform("io.ktor:ktor-bom:$ktorVersion")) + api("io.ktor:ktor-client-core") + api("io.ktor:ktor-client-cio") + api("io.ktor:ktor-client-auth") + implementation("io.ktor:ktor-client-content-negotiation") + implementation("io.ktor:ktor-serialization-kotlinx-json") - // to implement producers and consumers - val okhttpVersion: String by project - api("com.squareup.okhttp3:okhttp:$okhttpVersion") - val orgJsonVersion: String by project - api("org.json:json:$orgJsonVersion") + api("org.radarbase:managementportal-client:0.9.0-SNAPSHOT") + + val coroutinesVersion: String by project + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") // The production code uses the SLF4J logging API at compile time val slf4jVersion: String by project @@ -34,10 +42,11 @@ dependencies { testImplementation("com.fasterxml.jackson.core:jackson-databind") val radarSchemasVersion: String by project testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion") - val junitVersion: String by project - testImplementation("junit:junit:$junitVersion") val mockitoVersion: String by project testImplementation("org.mockito:mockito-core:$mockitoVersion") + val mockitoKotlinVersion: String by project + testImplementation("org.mockito.kotlin:mockito-kotlin:$mockitoKotlinVersion") + val okhttpVersion: String by project testImplementation("com.squareup.okhttp3:mockwebserver:$okhttpVersion") testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") } diff --git a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt index f819f54e..db720151 100644 --- a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt +++ b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt @@ -15,8 +15,6 @@ */ package org.radarbase.config -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl import java.net.InetSocketAddress import java.net.MalformedURLException import java.net.Proxy @@ -114,19 +112,6 @@ class ServerConfig { } } - /** - * Get the server as an HttpUrl. - * @return HttpUrl to the server - * @throws IllegalStateException if the URL is invalid - */ - val httpUrl: HttpUrl - get() { - if (protocol == null) { - protocol = "http" - } - return urlString.toHttpUrl() - } - /** * Get the HTTP proxyHost associated to given server. * @return http proxyHost if specified, or null if none is specified. diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt index e1aa04e1..d52185cc 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt @@ -39,6 +39,7 @@ class AvroDatumDecoder( ) : AvroDecoder { private val decoderFactory: DecoderFactory = DecoderFactory.get() + @Suppress("UNCHECKED_CAST") override fun reader(schema: Schema, clazz: Class): AvroReader { val reader = genericData.createDatumReader(schema) as DatumReader return AvroRecordReader(schema, reader) diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt index 7502adff..00abe2b3 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt @@ -32,8 +32,9 @@ class AvroDatumEncoder( ) : AvroEncoder { private val encoderFactory: EncoderFactory = EncoderFactory.get() + @Suppress("UNCHECKED_CAST") @Throws(IOException::class) - override fun writer(schema: Schema, clazz: Class): AvroWriter { + override fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter { val writer = genericData.createDatumWriter(schema) as DatumWriter return AvroRecordWriter(encoderFactory, schema, writer, binary) } diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt index 5f573ab6..62e18200 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt @@ -16,8 +16,6 @@ package org.radarbase.data import org.apache.avro.Schema -import org.apache.avro.SchemaValidationException -import org.radarbase.producer.rest.ParsedSchemaMetadata import java.io.IOException /** Encode Avro values with a given encoder. The encoder may take into account the schema @@ -25,7 +23,11 @@ import java.io.IOException interface AvroEncoder { /** Create a new writer. This method is thread-safe, but the class it returns is not. */ @Throws(IOException::class) - fun writer(schema: Schema, clazz: Class): AvroWriter + fun writer(schema: Schema, clazz: Class): AvroWriter = + writer(schema, clazz, schema) + @Throws(IOException::class) + fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter + interface AvroWriter { /** * Encode an object. This method is not thread-safe. Call @@ -35,17 +37,5 @@ interface AvroEncoder { */ @Throws(IOException::class) fun encode(`object`: T): ByteArray - /** - * Get the schema that the server lists. - * @return schema as set by setReaderSchema or null if not called yet. - */ - /** - * Update the schema that the server is lists for the current topic. - * @param readerSchema schema listed by the schema registry. - * @throws SchemaValidationException if the server schema is incompatible with the writer - * schema. - */ - @set:Throws(SchemaValidationException::class) - var readerSchema: ParsedSchemaMetadata? } } diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt index 4e0dc39c..77d0047f 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt @@ -1,5 +1,6 @@ package org.radarbase.data +import org.apache.avro.generic.IndexedRecord import org.radarbase.topic.AvroTopic /** @@ -24,6 +25,12 @@ class AvroRecordData( require(records.isNotEmpty()) { "Records should not be empty." } } + override val sourceId: String? by lazy { + val sourceIdField = topic.keySchema.getField("sourceId") ?: return@lazy null + if (key !is IndexedRecord) return@lazy null + key.get(sourceIdField.pos()).toString() + } + override fun iterator(): Iterator = records.iterator() override val isEmpty: Boolean = records.isEmpty() diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt index df5c2d00..84d30966 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt @@ -20,7 +20,6 @@ import org.apache.avro.io.DatumWriter import org.apache.avro.io.Encoder import org.apache.avro.io.EncoderFactory import org.radarbase.data.AvroEncoder.AvroWriter -import org.radarbase.producer.rest.ParsedSchemaMetadata import java.io.ByteArrayOutputStream import java.io.IOException @@ -46,7 +45,6 @@ class AvroRecordWriter( } else { encoderFactory.jsonEncoder(schema, out) } - override var readerSchema: ParsedSchemaMetadata? = null init { encoder = if (binary) { diff --git a/radar-commons/src/main/java/org/radarbase/data/RecordData.kt b/radar-commons/src/main/java/org/radarbase/data/RecordData.kt index 8e5c2a5a..7acd8100 100644 --- a/radar-commons/src/main/java/org/radarbase/data/RecordData.kt +++ b/radar-commons/src/main/java/org/radarbase/data/RecordData.kt @@ -20,6 +20,9 @@ interface RecordData : Iterable { */ val key: K + /** Source ID linked to record data, if any. */ + val sourceId: String? + /** * Whether the list of values is empty. * @return true if empty, false otherwise. diff --git a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt index 410b0329..f6f11e2a 100644 --- a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt @@ -5,9 +5,9 @@ import org.apache.avro.generic.GenericData import org.apache.avro.specific.SpecificData import org.apache.avro.specific.SpecificRecord import org.radarbase.data.AvroEncoder.AvroWriter -import org.radarbase.producer.rest.AvroDataMapper -import org.radarbase.producer.rest.AvroDataMapperFactory -import org.radarbase.producer.rest.ParsedSchemaMetadata +import org.radarbase.producer.avro.AvroDataMapper +import org.radarbase.producer.avro.AvroDataMapperFactory +import org.radarbase.producer.avro.AvroDataMapperFactory.validationException import java.io.IOException /** @@ -18,66 +18,45 @@ import java.io.IOException class RemoteSchemaEncoder( private val binary: Boolean, ) : AvroEncoder { - override fun writer(schema: Schema, clazz: Class): AvroWriter { - return SchemaEncoderWriter(schema, clazz) + override fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter { + return SchemaEncoderWriter(binary, schema, clazz, readerSchema) } - private inner class SchemaEncoderWriter( - private val schema: Schema, - clazz: Class + class SchemaEncoderWriter( + binary: Boolean, + schema: Schema, + clazz: Class, + readerSchema: Schema, ) : AvroWriter { private val recordEncoder: AvroEncoder - private var encoder: AvroWriter? = null - private var isGeneric = false - override var readerSchema: ParsedSchemaMetadata? = null - set(value) { - value ?: return - val currentField = field - if (currentField != null && value.schema == currentField.schema) { - return - } - try { - if (!isGeneric) { - mapper = AvroDataMapperFactory.IDENTITY_MAPPER - encoder = recordEncoder.writer(schema, Any::class.java) - } else { - mapper = AvroDataMapperFactory.instance.createMapper( - schema, - value.schema, - null - ) - encoder = recordEncoder.writer(value.schema, Any::class.java) - } - field = value - } catch (ex: IOException) { - throw IllegalStateException("Cannot construct Avro writer", ex) - } - } - - private var mapper: AvroDataMapper? = null + private val encoder: AvroWriter + private val isGeneric: Boolean + private val mapper: AvroDataMapper init { + if (schema.type !== Schema.Type.RECORD) throw validationException(schema, readerSchema, "Can only map records.") val genericData: GenericData val classLoader = Thread.currentThread().contextClassLoader + val useReaderSchema: Schema if (SpecificRecord::class.java.isAssignableFrom(clazz)) { genericData = SpecificData(classLoader) + useReaderSchema = schema isGeneric = false } else { genericData = GenericData(classLoader) + useReaderSchema = readerSchema isGeneric = true } recordEncoder = AvroDatumEncoder(genericData, binary) + mapper = AvroDataMapperFactory.createMapper(schema, useReaderSchema, null) + encoder = recordEncoder.writer(useReaderSchema, Any::class.java) } @Throws(IOException::class) - override fun encode(`object`: T): ByteArray { - val localEncoder = checkNotNull(encoder) { "Did not initialize reader schema yet " } - val localMapper = checkNotNull(mapper) { "Did not initialize reader schema yet" } - return localEncoder.encode( - requireNotNull(localMapper.convertAvro(`object`)) { - "Cannot map $`object` to Avro" - } - ) - } + override fun encode(`object`: T): ByteArray = encoder.encode( + requireNotNull(mapper.convertAvro(`object`)) { + "Cannot map $`object` to Avro" + } + ) } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt b/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt index 7531ddd3..7899de5d 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/AuthenticationException.kt @@ -15,18 +15,10 @@ */ package org.radarbase.producer -import java.io.IOException - /** * Failed to authenticate to server. */ -class AuthenticationException : IOException { - constructor() : super() {} - constructor(message: String?, cause: Throwable?) : super(message, cause) {} +class AuthenticationException : RuntimeException { constructor(message: String?) : super(message) {} - constructor(cause: Throwable?) : super(cause) {} - - companion object { - private const val serialVersionUID: Long = 1 - } + constructor(message: String?, cause: Throwable?) : super(message, cause) {} } diff --git a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt deleted file mode 100644 index ac4dea11..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/BatchedKafkaSender.kt +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer - -import org.apache.avro.SchemaValidationException -import org.radarbase.data.AvroRecordData -import org.radarbase.data.RecordData -import org.radarbase.topic.AvroTopic -import java.io.IOException -import java.util.concurrent.TimeUnit - -/** - * A Kafka REST Proxy sender that batches up records. It will send data once the batch size is - * exceeded, or when at a send call the first record in the batch is older than given age. If send, - * flush or close are not called within this given age, the data will also not be sent. Calling - * [.close] will not flush or close the KafkaTopicSender that were created. That must be - * done separately. - * - * @param wrappedSender kafka sender to send data with. - * @param ageMillis threshold time after which a record should be sent. - * @param maxBatchSize threshold batch size over which records should be sent. -*/ -class BatchedKafkaSender( - private val wrappedSender: KafkaSender, - ageMillis: Int, - private val maxBatchSize: Int -) : KafkaSender { - private val ageNanos: Long - - init { - ageNanos = TimeUnit.MILLISECONDS.toNanos(ageMillis.toLong()) - } - - @Throws(IOException::class, SchemaValidationException::class) - override fun sender(topic: AvroTopic): KafkaTopicSender { - return BatchedKafkaTopicSender(topic) - } - - @get:Throws(AuthenticationException::class) - override val isConnected: Boolean - get() = wrappedSender.isConnected - - @Throws(AuthenticationException::class) - override fun resetConnection(): Boolean { - return wrappedSender.resetConnection() - } - - @Synchronized - @Throws(IOException::class) - override fun close() { - wrappedSender.close() - } - - /** Batched kafka topic sender. This does the actual data batching. */ - private inner class BatchedKafkaTopicSender( - private val topic: AvroTopic - ) : KafkaTopicSender { - private var nanoAdded: Long = 0 - private var cachedKey: K? = null - private val cache: MutableList = ArrayList() - private val topicSender: KafkaTopicSender = wrappedSender.sender(topic) - - @Throws(IOException::class, SchemaValidationException::class) - override fun send(key: K, value: V) { - if (!isConnected) { - throw IOException("Cannot send records to unconnected producer.") - } - trySend(key, value) - } - - @Throws(IOException::class, SchemaValidationException::class) - override fun send(records: RecordData) { - if (records.isEmpty) return - val key = records.key - for (value in records) { - trySend(key, value) - } - } - - @Throws(IOException::class, SchemaValidationException::class) - private fun trySend(key: K, record: V) { - val keysMatch: Boolean - if (cache.isEmpty()) { - cachedKey = key - nanoAdded = System.nanoTime() - keysMatch = true - } else { - keysMatch = key == cachedKey - } - if (keysMatch) { - cache.add(record) - if (exceedsBuffer(cache)) { - doSend() - } - } else { - doSend() - trySend(key, record) - } - } - - @Throws(IOException::class, SchemaValidationException::class) - private fun doSend() { - val key = checkNotNull(cachedKey) { "Cached key should not be null in this function" } - topicSender.send(AvroRecordData(topic, key, cache)) - cache.clear() - cachedKey = null - } - - override fun clear() { - cache.clear() - topicSender.clear() - } - - @Throws(IOException::class) - override fun flush() { - if (cache.isNotEmpty()) { - try { - doSend() - } catch (ex: SchemaValidationException) { - throw IOException("Schemas do not match", ex) - } - } - topicSender.flush() - } - - @Throws(IOException::class) - override fun close() { - wrappedSender.use { - flush() - } - } - - private fun exceedsBuffer(records: List<*>): Boolean { - return records.size >= maxBatchSize || - System.nanoTime() - nanoAdded >= ageNanos - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt index 53fea6c6..eb2965c4 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt @@ -15,16 +15,24 @@ */ package org.radarbase.producer +import io.ktor.client.* +import io.ktor.client.engine.* +import io.ktor.client.engine.cio.* +import io.ktor.client.plugins.* +import kotlinx.coroutines.channels.BroadcastChannel +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.SharedFlow import org.apache.avro.SchemaValidationException +import org.radarbase.producer.rest.ConnectionState import org.radarbase.topic.AvroTopic -import java.io.Closeable import java.io.IOException /** * Thread-safe sender. Calling [.close] must be done after all [KafkaTopicSender] * senders created with [.sender] have been called. */ -interface KafkaSender : Closeable { +interface KafkaSender { /** Get a non thread-safe sender instance. */ @Throws(IOException::class, SchemaValidationException::class) fun sender(topic: AvroTopic): KafkaTopicSender @@ -36,16 +44,7 @@ interface KafkaSender : Closeable { * in the current request or in a previous one. */ @Throws(AuthenticationException::class) - fun resetConnection(): Boolean + suspend fun resetConnection(): Boolean - /** - * Get the current connection state to Kafka. If the connection state is unknown, this will - * trigger a connection check. - * @return true if connected, false if not connected. - * @throws AuthenticationException if the headers caused an authentication error - * in a previous request or during an additional connection - * check. - */ - @get:Throws(AuthenticationException::class) - val isConnected: Boolean + val connectionState: Flow } diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt index 7eb8e79f..2a98fda0 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt @@ -1,14 +1,17 @@ package org.radarbase.producer import org.apache.avro.SchemaValidationException +import org.radarbase.data.AvroRecordData import org.radarbase.data.RecordData -import java.io.Closeable +import org.radarbase.topic.AvroTopic import java.io.IOException /** * Sender for a single topic. Should be created through a [KafkaSender]. */ -interface KafkaTopicSender : Closeable { +interface KafkaTopicSender { + val topic: AvroTopic + /** * Send a message to Kafka eventually. * @@ -18,7 +21,7 @@ interface KafkaTopicSender : Closeable { * @throws IOException if the client could not send a message */ @Throws(IOException::class, SchemaValidationException::class) - fun send(key: K, value: V) + suspend fun send(key: K, value: V) = send(AvroRecordData(topic, key, listOf(value))) /** * Send a message to Kafka eventually. Contained offsets must be strictly monotonically @@ -29,19 +32,5 @@ interface KafkaTopicSender : Closeable { * @throws IOException if the client could not send a message */ @Throws(IOException::class, SchemaValidationException::class) - fun send(records: RecordData) - - /** - * Clears any messages still in cache. - */ - fun clear() - - /** - * Flush all remaining messages. - * - * @throws AuthenticationException if the client failed to authenticate itself - * @throws IOException if the client could not send a message - */ - @Throws(IOException::class) - fun flush() + suspend fun send(records: RecordData) } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.kt b/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapper.kt similarity index 90% rename from radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.kt rename to radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapper.kt index d0e450db..e502eec7 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapper.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapper.kt @@ -1,4 +1,4 @@ -package org.radarbase.producer.rest +package org.radarbase.producer.avro /** * Maps data from one avro record schema to another. Create it by calling diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt b/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt similarity index 60% rename from radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt rename to radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt index 16ef050c..b460b4dd 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroDataMapperFactory.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt @@ -1,4 +1,4 @@ -package org.radarbase.producer.rest +package org.radarbase.producer.avro import org.apache.avro.JsonProperties import org.apache.avro.Schema @@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory import java.nio.ByteBuffer import java.util.* -class AvroDataMapperFactory { +object AvroDataMapperFactory { /** * Create a mapper for data in one Avro schema to that in another Avro schema. * @param from originating Avro schema @@ -27,7 +27,7 @@ class AvroDataMapperFactory { return IDENTITY_MAPPER } logger.debug("Computing custom mapping from {} to {}", from, to) - try { + return try { if (to.type == Schema.Type.UNION || from.type == Schema.Type.UNION) { return mapUnion(from, to, defaultVal) } @@ -39,7 +39,7 @@ class AvroDataMapperFactory { return mapNumber(from, to, defaultVal) else -> {} } - return when (from.type) { + when (from.type) { Schema.Type.RECORD -> mapRecord(from, to) Schema.Type.ARRAY -> mapArray(from, to) Schema.Type.MAP -> mapMap(from, to) @@ -50,14 +50,11 @@ class AvroDataMapperFactory { else -> throw validationException(to, from, "Schema types of from and to don't match") } } catch (ex: SchemaValidationException) { - return if (defaultVal != null) { - if (defaultVal === JsonProperties.NULL_VALUE) { - AvroDataMapper { null } - } else { - AvroDataMapper { defaultVal } - } + defaultVal ?: throw ex + if (defaultVal === JsonProperties.NULL_VALUE) { + AvroDataMapper { null } } else { - throw ex + AvroDataMapper { defaultVal } } } } @@ -246,154 +243,149 @@ class AvroDataMapperFactory { } } - companion object { - private val logger: Logger = LoggerFactory.getLogger( - AvroDataMapperFactory::class.java - ) - val IDENTITY_MAPPER: AvroDataMapper = object : AvroDataMapper { - override fun convertAvro(`object`: Any?): Any? = `object` + private val logger: Logger = LoggerFactory.getLogger(AvroDataMapperFactory::class.java) + val IDENTITY_MAPPER: AvroDataMapper = object : AvroDataMapper { + override fun convertAvro(`object`: Any?): Any? = `object` - override fun toString(): String = "Identity" - } - val instance: AvroDataMapperFactory by lazy { AvroDataMapperFactory() } + override fun toString(): String = "Identity" + } - private inline fun Any?.asAvroType(from: Schema, to: Schema): T { - if (this !is T) { - throw validationException( - to, from, "${to.type} type cannot be mapped from ${this?.javaClass?.name} Java type." - ) - } - return this + private inline fun Any?.asAvroType(from: Schema, to: Schema): T { + if (this !is T) { + throw validationException( + to, from, "${to.type} type cannot be mapped from ${this?.javaClass?.name} Java type." + ) } + return this + } - private val PRIMITIVE_TYPES = EnumSet.of( - Schema.Type.INT, - Schema.Type.LONG, - Schema.Type.BYTES, - Schema.Type.FLOAT, - Schema.Type.DOUBLE, - Schema.Type.NULL, - Schema.Type.BOOLEAN, - Schema.Type.STRING, - ) + private val PRIMITIVE_TYPES = EnumSet.of( + Schema.Type.INT, + Schema.Type.LONG, + Schema.Type.BYTES, + Schema.Type.FLOAT, + Schema.Type.DOUBLE, + Schema.Type.NULL, + Schema.Type.BOOLEAN, + Schema.Type.STRING, + ) - /** Map one enum to another or to String. */ - @Throws(SchemaValidationException::class) - private fun mapEnum(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { - return if (to.type == Schema.Type.ENUM) { - var containsAll = true - if (from.type == Schema.Type.ENUM) { - for (s in from.enumSymbols) { - if (!to.hasEnumSymbol(s)) { - containsAll = false - break - } + /** Map one enum to another or to String. */ + @Throws(SchemaValidationException::class) + private fun mapEnum(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + return if (to.type == Schema.Type.ENUM) { + var containsAll = true + if (from.type == Schema.Type.ENUM) { + for (s in from.enumSymbols) { + if (!to.hasEnumSymbol(s)) { + containsAll = false + break } - } else if (from.type == Schema.Type.STRING) { - containsAll = false - } else { - throw validationException(to, from, "Cannot map enum from non-string or enum type") } - if (containsAll) { - AvroDataMapper { obj -> GenericData.EnumSymbol(to, obj.toString()) } - } else { - var defaultString = defaultVal as? String - if (defaultString == null) { - if (to.hasEnumSymbol("UNKNOWN")) { - defaultString = "UNKNOWN" - } else { - throw validationException(to, from, - "Cannot map enum symbols without default value" - ) - } - } - val symbol: GenericEnumSymbol<*> = GenericData.EnumSymbol(to, defaultString) - AvroDataMapper { obj: Any? -> - val value = obj.toString() - if (to.hasEnumSymbol(value)) { - GenericData.EnumSymbol(to, value) - } else { - symbol - } - } - } - } else if (from.type == Schema.Type.ENUM && to.type == Schema.Type.STRING) { - AvroDataMapper { it.toString() } + } else if (from.type == Schema.Type.STRING) { + containsAll = false } else { - throw validationException(to, from, "Cannot map unknown type with enum.") + throw validationException(to, from, "Cannot map enum from non-string or enum type") } + if (containsAll) { + AvroDataMapper { obj -> GenericData.EnumSymbol(to, obj.toString()) } + } else { + var defaultString = defaultVal as? String + if (defaultString == null) { + if (to.hasEnumSymbol("UNKNOWN")) { + defaultString = "UNKNOWN" + } else { + throw validationException(to, from, + "Cannot map enum symbols without default value" + ) + } + } + val symbol: GenericEnumSymbol<*> = GenericData.EnumSymbol(to, defaultString) + AvroDataMapper { obj: Any? -> + val value = obj.toString() + if (to.hasEnumSymbol(value)) { + GenericData.EnumSymbol(to, value) + } else { + symbol + } + } + } + } else if (from.type == Schema.Type.ENUM && to.type == Schema.Type.STRING) { + AvroDataMapper { it.toString() } + } else { + throw validationException(to, from, "Cannot map unknown type with enum.") } + } - /** Get the default value as a Generic type. */ - private fun getDefaultValue(defaultVal: Any?, schema: Schema): Any? { - return if (defaultVal == null) { - null - } else if (schema.type == Schema.Type.ENUM) { - GenericData.EnumSymbol(schema, defaultVal) - } else { - defaultVal - } + /** Get the default value as a Generic type. */ + private fun getDefaultValue(defaultVal: Any?, schema: Schema): Any? { + return if (defaultVal == null) { + null + } else if (schema.type == Schema.Type.ENUM) { + GenericData.EnumSymbol(schema, defaultVal) + } else { + defaultVal } + } - /** Maps one number type to another or parses/converts to a string. */ - @Throws(SchemaValidationException::class) - private fun mapNumber(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { - if (from.type == to.type) { - return IDENTITY_MAPPER + /** Maps one number type to another or parses/converts to a string. */ + @Throws(SchemaValidationException::class) + private fun mapNumber(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { + if (from.type == to.type) { + return IDENTITY_MAPPER + } + return if (from.type == Schema.Type.STRING) { + defaultVal ?: throw validationException(to, from, "Cannot map string to number without default value.") + when (to.type) { + Schema.Type.INT -> StringToNumberMapper(defaultVal, Integer::valueOf) + Schema.Type.LONG -> StringToNumberMapper(defaultVal, String::toLong) + Schema.Type.DOUBLE -> StringToNumberMapper(defaultVal, String::toDouble) + Schema.Type.FLOAT -> StringToNumberMapper(defaultVal, String::toFloat) + else -> throw validationException( + to, from, "Cannot map numeric type with non-numeric type" + ) } - return if (from.type == Schema.Type.STRING) { - defaultVal ?: throw validationException(to, from, "Cannot map string to number without default value.") - when (to.type) { - Schema.Type.INT -> StringToNumberMapper(defaultVal, Integer::valueOf) - Schema.Type.LONG -> StringToNumberMapper(defaultVal, String::toLong) - Schema.Type.DOUBLE -> StringToNumberMapper(defaultVal, String::toDouble) - Schema.Type.FLOAT -> StringToNumberMapper(defaultVal, String::toFloat) - else -> throw validationException( - to, from, "Cannot map numeric type with non-numeric type" - ) - } - } else { - when (to.type) { - Schema.Type.INT -> AvroDataMapper { it.asAvroType(from, to).toInt() } - Schema.Type.LONG -> AvroDataMapper { it.asAvroType(from, to).toLong() } - Schema.Type.DOUBLE -> AvroDataMapper { it.toString().toDouble() } - Schema.Type.FLOAT -> AvroDataMapper { it.asAvroType(from, to).toFloat() } - Schema.Type.STRING -> AvroDataMapper { it.toString() } - else -> throw validationException( - to, from, "Cannot map numeric type with non-numeric type" - ) - } + } else { + when (to.type) { + Schema.Type.INT -> AvroDataMapper { it.asAvroType(from, to).toInt() } + Schema.Type.LONG -> AvroDataMapper { it.asAvroType(from, to).toLong() } + Schema.Type.DOUBLE -> AvroDataMapper { it.toString().toDouble() } + Schema.Type.FLOAT -> AvroDataMapper { it.asAvroType(from, to).toFloat() } + Schema.Type.STRING -> AvroDataMapper { it.toString() } + else -> throw validationException( + to, from, "Cannot map numeric type with non-numeric type" + ) } } + } - /** Get the non-null union type of a nullable/optional union field. */ - @Throws(SchemaValidationException::class) - private fun nonNullUnionSchema(schema: Schema): Schema { - val types = checkNotNull(schema.types) { "Union does not have subtypes" } - if (types.size != 2) { - throw validationException(schema, schema, "Types must denote optionals.") - } - return if (types[0].type == Schema.Type.NULL) { - if (types[1].type != Schema.Type.NULL) { - types[1] - } else { - throw validationException(schema, schema, "Types must denote optionals.") - } - } else if (types[1].type == Schema.Type.NULL) { - types[0] + /** Get the non-null union type of a nullable/optional union field. */ + @Throws(SchemaValidationException::class) + private fun nonNullUnionSchema(schema: Schema): Schema { + val types = checkNotNull(schema.types) { "Union does not have subtypes" } + if (types.size != 2) { + throw validationException(schema, schema, "Types must denote optionals.") + } + return if (types[0].type == Schema.Type.NULL) { + if (types[1].type != Schema.Type.NULL) { + types[1] } else { throw validationException(schema, schema, "Types must denote optionals.") } + } else if (types[1].type == Schema.Type.NULL) { + types[0] + } else { + throw validationException(schema, schema, "Types must denote optionals.") } + } - private fun Schema.Type.isPrimitive(): Boolean = this in PRIMITIVE_TYPES + private fun Schema.Type.isPrimitive(): Boolean = this in PRIMITIVE_TYPES - internal fun validationException( - from: Schema, - to: Schema, - message: String, - ): SchemaValidationException = SchemaValidationException( - to, from, IllegalArgumentException(message) - ) - } + internal fun validationException( + from: Schema, + to: Schema, + message: String, + ): SchemaValidationException = SchemaValidationException( + to, from, IllegalArgumentException(message) + ) } diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/BinaryEncoder.kt b/radar-commons/src/main/java/org/radarbase/producer/io/BinaryEncoder.kt new file mode 100644 index 00000000..86502180 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/BinaryEncoder.kt @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.io + +import org.apache.avro.util.Utf8 +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets + +/** + * An abstract [Encoder] for Avro's binary encoding. + * + * + * To construct and configure instances, use [EncoderFactory] + * + * @see EncoderFactory + * + * @see BufferedBinaryEncoder + * + * @see DirectBinaryEncoder + * + * @see BlockingBinaryEncoder + * + * @see Encoder + * + * @see Decoder + */ +abstract class BinaryEncoder : Encoder { + @Throws(IOException::class) + override suspend fun writeNull() { + } + + @Throws(IOException::class) + override suspend fun writeString(utf8: Utf8) { + this.writeBytes(utf8.bytes, 0, utf8.byteLength) + } + + @Throws(IOException::class) + override suspend fun writeString(str: String) { + if (str.isEmpty()) { + writeZero() + return + } + val bytes = str.toByteArray(StandardCharsets.UTF_8) + writeInt(bytes.size) + writeFixed(bytes, 0, bytes.size) + } + + @Throws(IOException::class) + override suspend fun writeBytes(bytes: ByteBuffer) { + val len = bytes.limit() - bytes.position() + if (0 == len) { + writeZero() + } else { + writeInt(len) + writeFixed(bytes) + } + } + + @Throws(IOException::class) + override suspend fun writeBytes(bytes: ByteArray, start: Int, len: Int) { + if (0 == len) { + writeZero() + return + } + writeInt(len) + this.writeFixed(bytes, start, len) + } + + @Throws(IOException::class) + override suspend fun writeEnum(e: Int) { + writeInt(e) + } + + @Throws(IOException::class) + override suspend fun writeArrayStart() { + } + + @Throws(IOException::class) + override suspend fun setItemCount(itemCount: Long) { + if (itemCount > 0) { + writeLong(itemCount) + } + } + + @Throws(IOException::class) + override suspend fun startItem() { + } + + @Throws(IOException::class) + override suspend fun writeArrayEnd() { + writeZero() + } + + @Throws(IOException::class) + override suspend fun writeMapStart() { + } + + @Throws(IOException::class) + override suspend fun writeMapEnd() { + writeZero() + } + + @Throws(IOException::class) + override suspend fun writeIndex(unionIndex: Int) { + writeInt(unionIndex) + } + + /** Write a zero byte to the underlying output. */ + @Throws(IOException::class) + protected abstract suspend fun writeZero() + + /** + * Returns the number of bytes currently buffered by this encoder. If this + * Encoder does not buffer, this will always return zero. + * + * + * Call [.flush] to empty the buffer to the underlying output. + */ + open val bytesBuffered: Int = 0 +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt b/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt new file mode 100644 index 00000000..62ea0241 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.io + +import io.ktor.utils.io.* +import org.apache.avro.io.BinaryData +import java.io.IOException +import java.util.* + +/** + * An [Encoder] for Avro's binary encoding that does not buffer output. + * + * + * This encoder does not buffer writes, and as a result is slower than + * [BufferedBinaryEncoder]. However, it is lighter-weight and useful when + * the buffering in BufferedBinaryEncoder is not desired and/or the Encoder is + * very short lived. + * + * + * To construct, use + * [EncoderFactory.directBinaryEncoder] + * + * + * DirectBinaryEncoder is not thread-safe + * + * @see BinaryEncoder + * + * @see EncoderFactory + * + * @see Encoder + * + * @see Decoder + */ +class DirectBinaryEncoder( + var out: ByteWriteChannel +) : BinaryEncoder() { + // the buffer is used for writing floats, doubles, and large longs. + private val buf = ByteArray(12) + + @Throws(IOException::class) + override suspend fun flush() { + out.flush() + } + + override fun close() { + out.close() + } + + @Throws(IOException::class) + override suspend fun writeBoolean(b: Boolean) { + out.writeByte(if (b) 1 else 0) + } + + /* + * buffering is slower for ints that encode to just 1 or two bytes, and and + * faster for large ones. (Sun JRE 1.6u22, x64 -server) + */ + @Throws(IOException::class) + override suspend fun writeInt(n: Int) { + val `val` = n shl 1 xor (n shr 31) + if (`val` and 0x7F.inv() == 0) { + out.writeByte(`val`) + return + } else if (`val` and 0x3FFF.inv() == 0) { + out.writeByte(0x80 or `val`) + out.writeByte(`val` ushr 7) + return + } + val len = BinaryData.encodeInt(n, buf, 0) + out.writeFully(buf, 0, len) + } + + /* + * buffering is slower for writeLong when the number is small enough to fit in + * an int. (Sun JRE 1.6u22, x64 -server) + */ + @Throws(IOException::class) + override suspend fun writeLong(n: Long) { + val `val` = n shl 1 xor (n shr 63) // move sign to low-order bit + if (`val` and 0x7FFFFFFFL.inv() == 0L) { + var i = `val`.toInt() + while (i and 0x7F.inv() != 0) { + out.writeByte((0x80 or i and 0xFF).toByte().toInt()) + i = i ushr 7 + } + out.writeByte(i.toByte().toInt()) + return + } + val len = BinaryData.encodeLong(n, buf, 0) + out.writeFully(buf, 0, len) + } + + @Throws(IOException::class) + override suspend fun writeFloat(f: Float) { + val len = BinaryData.encodeFloat(f, buf, 0) + out.writeFully(buf, 0, len) + } + + @Throws(IOException::class) + override suspend fun writeDouble(d: Double) { + val len = BinaryData.encodeDouble(d, buf, 0) + out.writeFully(buf, 0, len) + } + + @Throws(IOException::class) + override suspend fun writeFixed(bytes: ByteArray, start: Int, len: Int) { + out.writeFully(bytes, start, len) + } + + @Throws(IOException::class) + override suspend fun writeZero() { + out.writeByte(0) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt b/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt new file mode 100644 index 00000000..a1de5621 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.io + +import org.apache.avro.util.Utf8 +import java.io.Closeable +import java.io.IOException +import java.nio.ByteBuffer + +/** + * Low-level support for serializing Avro values. + * + * + * This class has two types of methods. One type of methods support the writing + * of leaf values (for example, [.writeLong] and [.writeString]). + * These methods have analogs in [Decoder]. + * + * + * The other type of methods support the writing of maps and arrays. These + * methods are [.writeArrayStart], [.startItem], and + * [.writeArrayEnd] (and similar methods for maps). Some implementations + * of [Encoder] handle the buffering required to break large maps and + * arrays into blocks, which is necessary for applications that want to do + * streaming. (See [.writeArrayStart] for details on these methods.) + * + * + * [EncoderFactory] contains Encoder construction and configuration + * facilities. + * + * @see EncoderFactory + * + * @see Decoder + */ +interface Encoder: Closeable { + /** + * "Writes" a null value. (Doesn't actually write anything, but advances the + * state of the parser if this class is stateful.) + * + * @throws AvroTypeException If this is a stateful writer and a null is not + * expected + */ + @Throws(IOException::class) + suspend fun writeNull() + + /** + * Write a boolean value. + * + * @throws AvroTypeException If this is a stateful writer and a boolean is not + * expected + */ + @Throws(IOException::class) + suspend fun writeBoolean(b: Boolean) + + /** + * Writes a 32-bit integer. + * + * @throws AvroTypeException If this is a stateful writer and an integer is not + * expected + */ + @Throws(IOException::class) + suspend fun writeInt(n: Int) + + /** + * Write a 64-bit integer. + * + * @throws AvroTypeException If this is a stateful writer and a long is not + * expected + */ + @Throws(IOException::class) + suspend fun writeLong(n: Long) + + /** + * Write a float. + * + * @throws IOException + * @throws AvroTypeException If this is a stateful writer and a float is not + * expected + */ + @Throws(IOException::class) + suspend fun writeFloat(f: Float) + + /** + * Write a double. + * + * @throws AvroTypeException If this is a stateful writer and a double is not + * expected + */ + @Throws(IOException::class) + suspend fun writeDouble(d: Double) + + /** + * Write a Unicode character string. + * + * @throws AvroTypeException If this is a stateful writer and a char-string is + * not expected + */ + @Throws(IOException::class) + suspend fun writeString(utf8: Utf8) + + /** + * Write a Unicode character string. The default implementation converts the + * String to a [Utf8]. Some Encoder implementations + * may want to do something different as a performance optimization. + * + * @throws AvroTypeException If this is a stateful writer and a char-string is + * not expected + */ + @Throws(IOException::class) + suspend fun writeString(str: String) { + writeString(Utf8(str)) + } + + /** + * Write a Unicode character string. If the CharSequence is an + * [Utf8] it writes this directly, otherwise the + * CharSequence is converted to a String via toString() and written. + * + * @throws AvroTypeException If this is a stateful writer and a char-string is + * not expected + */ + @Throws(IOException::class) + suspend fun writeString(charSequence: CharSequence) { + if (charSequence is Utf8) writeString(charSequence) else writeString(charSequence.toString()) + } + + /** + * Write a byte string. + * + * @throws AvroTypeException If this is a stateful writer and a byte-string is + * not expected + */ + @Throws(IOException::class) + suspend fun writeBytes(bytes: ByteBuffer) + + /** + * Write a byte string. + * + * @throws AvroTypeException If this is a stateful writer and a byte-string is + * not expected + */ + @Throws(IOException::class) + suspend fun writeBytes(bytes: ByteArray, start: Int, len: Int) + + /** + * Writes a byte string. Equivalent to + * writeBytes(bytes, 0, bytes.length) + * + * @throws IOException + * @throws AvroTypeException If this is a stateful writer and a byte-string is + * not expected + */ + @Throws(IOException::class) + suspend fun writeBytes(bytes: ByteArray) { + writeBytes(bytes, 0, bytes.size) + } + + /** + * Writes a fixed size binary object. + * + * @param bytes The contents to write + * @param start The position within bytes where the contents start. + * @param len The number of bytes to write. + * @throws AvroTypeException If this is a stateful writer and a byte-string is + * not expected + * @throws IOException + */ + @Throws(IOException::class) + suspend fun writeFixed(bytes: ByteArray, start: Int, len: Int) + + /** + * A shorthand for writeFixed(bytes, 0, bytes.length) + * + * @param bytes + */ + @Throws(IOException::class) + suspend fun writeFixed(bytes: ByteArray) { + writeFixed(bytes, 0, bytes.size) + } + + /** Writes a fixed from a ByteBuffer. */ + @Throws(IOException::class) + suspend fun writeFixed(bytes: ByteBuffer) { + val pos = bytes.position() + val len = bytes.limit() - pos + if (bytes.hasArray()) { + writeFixed(bytes.array(), bytes.arrayOffset() + pos, len) + } else { + val b = ByteArray(len) + bytes.duplicate()[b, 0, len] + writeFixed(b, 0, len) + } + } + + /** + * Writes an enumeration. + * + * @param e + * @throws AvroTypeException If this is a stateful writer and an enumeration is + * not expected or the e is out of range. + * @throws IOException + */ + @Throws(IOException::class) + suspend fun writeEnum(e: Int) + + /** + * Call this method to start writing an array. + * + * When starting to serialize an array, call [.writeArrayStart]. Then, + * before writing any data for any item call [.setItemCount] followed by a + * sequence of [.startItem] and the item itself. The number of + * [.startItem] should match the number specified in + * [.setItemCount]. When actually writing the data of the item, you can + * call any [Encoder] method (e.g., [.writeLong]). When all items of + * the array have been written, call [.writeArrayEnd]. + * + * As an example, let's say you want to write an array of records, the record + * consisting of an Long field and a Boolean field. Your code would look + * something like this: + * + * 
+     * out.writeArrayStart();
+     * out.setItemCount(list.size());
+     * for (Record r : list) {
+     * out.startItem();
+     * out.writeLong(r.longField);
+     * out.writeBoolean(r.boolField);
+     * }
+     * out.writeArrayEnd();
+
* + * + * @throws AvroTypeException If this is a stateful writer and an array is not + * expected + */ + @Throws(IOException::class) + suspend fun writeArrayStart() + + /** + * Call this method before writing a batch of items in an array or a map. Then + * for each item, call [.startItem] followed by any of the other write + * methods of [Encoder]. The number of calls to [.startItem] must + * be equal to the count specified in [.setItemCount]. Once a batch is + * completed you can start another batch with [.setItemCount]. + * + * @param itemCount The number of [.startItem] calls to follow. + * @throws IOException + */ + @Throws(IOException::class) + suspend fun setItemCount(itemCount: Long) + + /** + * Start a new item of an array or map. See [.writeArrayStart] for usage + * information. + * + * @throws AvroTypeException If called outside of an array or map context + */ + @Throws(IOException::class) + suspend fun startItem() + + /** + * Call this method to finish writing an array. See [.writeArrayStart] for + * usage information. + * + * @throws AvroTypeException If items written does not match count provided to + * [.writeArrayStart] + * @throws AvroTypeException If not currently inside an array + */ + @Throws(IOException::class) + suspend fun writeArrayEnd() + + /** + * Call this to start a new map. See [.writeArrayStart] for details on + * usage. + * + * As an example of usage, let's say you want to write a map of records, the + * record consisting of an Long field and a Boolean field. Your code would look + * something like this: + * + * 
+     * out.writeMapStart();
+     * out.setItemCount(list.size());
+     * for (Map.Entry, Record> entry : map.entrySet()) {
+     * out.startItem();
+     * out.writeString(entry.getKey());
+     * out.writeLong(entry.getValue().longField);
+     * out.writeBoolean(entry.getValue().boolField);
+     * }
+     * out.writeMapEnd();
+
* + * + * @throws AvroTypeException If this is a stateful writer and a map is not + * expected + */ + @Throws(IOException::class) + suspend fun writeMapStart() + + /** + * Call this method to terminate the inner-most, currently-opened map. See + * [.writeArrayStart] for more details. + * + * @throws AvroTypeException If items written does not match count provided to + * [.writeMapStart] + * @throws AvroTypeException If not currently inside a map + */ + @Throws(IOException::class) + suspend fun writeMapEnd() + + /** + * Call this method to write the tag of a union. + * + * As an example of usage, let's say you want to write a union, whose second + * branch is a record consisting of an Long field and a Boolean field. Your code + * would look something like this: + * + * 
+     * out.writeIndex(1);
+     * out.writeLong(record.longField);
+     * out.writeBoolean(record.boolField);
+
* + * + * @throws AvroTypeException If this is a stateful writer and a map is not + * expected + */ + @Throws(IOException::class) + suspend fun writeIndex(unionIndex: Int) + + suspend fun flush() +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/FunctionalWriteChannelContent.kt b/radar-commons/src/main/java/org/radarbase/producer/io/FunctionalWriteChannelContent.kt new file mode 100644 index 00000000..2bf60d01 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/FunctionalWriteChannelContent.kt @@ -0,0 +1,10 @@ +package org.radarbase.producer.io + +import io.ktor.http.content.* +import io.ktor.utils.io.* + +class FunctionalWriteChannelContent( + private val writeAction: suspend (ByteWriteChannel) -> Unit, +) : OutgoingContent.WriteChannelContent() { + override suspend fun writeTo(channel: ByteWriteChannel) = writeAction(channel) +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt b/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt new file mode 100644 index 00000000..4aaecf5b --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt @@ -0,0 +1,95 @@ +package org.radarbase.producer.io + +import io.ktor.client.* +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.http.* +import io.ktor.http.content.* +import io.ktor.util.* +import io.ktor.util.cio.* +import io.ktor.utils.io.* +import kotlinx.coroutines.coroutineScope + + +/** + * A plugin that allows you to enable specified compression algorithms (such as `gzip` and `deflate`) and configure their settings. + * This plugin serves two primary purposes: + * - Sets the `Accept-Encoding` header with the specified quality value. + * - Decodes content received from a server to obtain the original payload. + * + * You can learn more from [Content encoding](https://ktor.io/docs/content-encoding.html). + */ +class GzipContentEncoding private constructor() { + private fun setRequestHeaders(headers: HeadersBuilder) { + if (headers.contains(HttpHeaders.ContentEncoding)) return + headers[HttpHeaders.ContentEncoding] = "gzip" + } + + private fun encode(headers: Headers, content: OutgoingContent): OutgoingContent { + val encodingHeader = (headers[HttpHeaders.ContentEncoding] ?: return content).split(",") + if (!encodingHeader.containsIgnoreCase("gzip")) return content + + return when (content) { + is OutgoingContent.ProtocolUpgrade, is OutgoingContent.NoContent -> content + is OutgoingContent.ReadChannelContent -> GzipReadChannel(content.readFrom()) + is OutgoingContent.ByteArrayContent -> GzipReadChannel(ByteReadChannel(content.bytes())) + is OutgoingContent.WriteChannelContent -> GzipWriteChannel(content) + } + } + + /** + * A configuration for the [GzipContentEncoding] plugin. + */ + @KtorDsl + class Config + + companion object : HttpClientPlugin { + override val key: AttributeKey = AttributeKey("GzipHttpEncoding") + + override fun prepare(block: Config.() -> Unit): GzipContentEncoding { + return GzipContentEncoding() + } + + override fun install(plugin: GzipContentEncoding, scope: HttpClient) { + scope.requestPipeline.intercept(HttpRequestPipeline.State) { + plugin.setRequestHeaders(context.headers) + } + + scope.requestPipeline.intercept(HttpRequestPipeline.Transform) { call -> + val method = this.context.method + val contentLength = context.contentLength() + + if (contentLength == 0L) return@intercept + if (contentLength == null && (method == HttpMethod.Head || method == HttpMethod.Options)) return@intercept + + if (call !is OutgoingContent) return@intercept + + proceedWith(plugin.encode(context.headers.build(), call)) + } + } + + private fun List.containsIgnoreCase(value: String): Boolean { + return any { el -> el.trim { it <= ' ' }.equals(value, ignoreCase = true) } + } + } + + + private class GzipReadChannel( + private val original: ByteReadChannel, + ) : OutgoingContent.ReadChannelContent() { + override fun readFrom(): ByteReadChannel = + original.deflated(gzip = true) + } + + private class GzipWriteChannel( + private val content: WriteChannelContent, + ) : OutgoingContent.WriteChannelContent() { + override suspend fun writeTo(channel: ByteWriteChannel) { + coroutineScope { + channel.deflated(gzip = true, coroutineContext = coroutineContext).use { + content.writeTo(this) + } + } + } + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt b/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt new file mode 100644 index 00000000..64950963 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt @@ -0,0 +1,36 @@ +package org.radarbase.producer.io + +import io.ktor.client.* +import io.ktor.client.engine.cio.* +import io.ktor.client.plugins.* +import java.security.cert.X509Certificate +import java.time.Duration +import javax.net.ssl.X509TrustManager + +fun HttpClientConfig<*>.timeout(duration: Duration) { + install(HttpTimeout) { + val millis = duration.toMillis() + connectTimeoutMillis = millis + socketTimeoutMillis = millis + requestTimeoutMillis = millis + } +} + +fun HttpClientConfig<*>.unsafeSsl() { + engine { + if (this is CIOEngineConfig) { + https { + trustManager = UNSAFE_TRUST_MANAGER + } + } + } +} + +/** Unsafe trust manager that trusts all certificates. */ +private val UNSAFE_TRUST_MANAGER = object : X509TrustManager { + override fun checkClientTrusted(chain: Array, authType: String) = Unit + + override fun checkServerTrusted(chain: Array, authType: String) = Unit + + override fun getAcceptedIssuers(): Array = arrayOf() +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt b/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt new file mode 100644 index 00000000..0efeddc3 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt @@ -0,0 +1,11 @@ +package org.radarbase.producer.io + +import io.ktor.http.* +import java.io.IOException + +class UnsupportedMediaTypeException( + contentType: ContentType?, + contentEncoding: String?, +) : IOException( + "Unsupported media type ${contentType ?: "unknown"} with ${contentEncoding ?: "no"} encoding" +) diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt new file mode 100644 index 00000000..68e4bc3e --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt @@ -0,0 +1,63 @@ +package org.radarbase.producer.rest + +import io.ktor.http.* +import io.ktor.http.content.* +import io.ktor.serialization.* +import io.ktor.util.reflect.* +import io.ktor.utils.io.* +import io.ktor.utils.io.charsets.* +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.radarbase.data.RecordData +import org.radarbase.producer.schema.SchemaRetriever + +class AvroContentConverter( + private val schemaRetriever: SchemaRetriever, + private val binary: Boolean, +) : ContentConverter { + override suspend fun serializeNullable( + contentType: ContentType, + charset: Charset, + typeInfo: TypeInfo, + value: Any? + ): OutgoingContent? { + if (value !is RecordData<*, *>) return null + + return coroutineScope { + val keySchema = async { + schemaRetriever.metadata( + topic = value.topic.name, + ofValue = false, + schema = value.topic.keySchema + ) + } + val valueSchema = async { + schemaRetriever.metadata( + topic = value.topic.name, + ofValue = true, + schema = value.topic.valueSchema + ) + } + val maker = if (binary) { + BinaryRecordContent( + records = value, + keySchemaMetadata = keySchema.await(), + valueSchemaMetadata = valueSchema.await(), + ) + } else { + JsonRecordContent( + records = value, + keySchemaMetadata = keySchema.await(), + valueSchemaMetadata = valueSchema.await() + ) + } + maker.createContent() + } + } + + override suspend fun deserialize( + charset: Charset, + typeInfo: TypeInfo, + content: ByteReadChannel, + ): Any? = null +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroRecordContent.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroRecordContent.kt new file mode 100644 index 00000000..5bcea266 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroRecordContent.kt @@ -0,0 +1,7 @@ +package org.radarbase.producer.rest + +import io.ktor.http.content.* + +interface AvroRecordContent { + fun createContent(): OutgoingContent +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt new file mode 100644 index 00000000..e17b4e9e --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt @@ -0,0 +1,67 @@ +package org.radarbase.producer.rest + +import io.ktor.http.content.* +import org.radarbase.data.RecordData +import org.radarbase.data.RemoteSchemaEncoder +import org.radarbase.producer.avro.AvroDataMapperFactory +import org.radarbase.producer.io.BinaryEncoder +import org.radarbase.producer.io.DirectBinaryEncoder +import org.radarbase.producer.io.FunctionalWriteChannelContent +import org.radarbase.producer.schema.ParsedSchemaMetadata +import org.slf4j.LoggerFactory + +class BinaryRecordContent( + private val records: RecordData<*, V>, + keySchemaMetadata: ParsedSchemaMetadata, + valueSchemaMetadata: ParsedSchemaMetadata, +): AvroRecordContent { + private val valueEncoder = RemoteSchemaEncoder.SchemaEncoderWriter( + binary = true, + schema = records.topic.valueSchema, + clazz = records.topic.valueClass, + readerSchema = valueSchemaMetadata.schema + ) + private val sourceId = records.sourceId + ?: throw AvroDataMapperFactory.validationException( + records.topic.keySchema, + keySchemaMetadata.schema, + "Cannot map record without source ID" + ) + + private val keySchemaVersion = requireNotNull(keySchemaMetadata.version) { + "missing key schema version" + } + private val valueSchemaVersion = requireNotNull(valueSchemaMetadata.version) { + "missing key schema version" + } + + override fun createContent(): OutgoingContent = FunctionalWriteChannelContent { channel -> + DirectBinaryEncoder(channel).use { + it.writeRecords() + } + } + + private suspend fun BinaryEncoder.writeRecords() { + startItem() + writeInt(keySchemaVersion) + writeInt(valueSchemaVersion) + + // do not send project ID; it is encoded in the serialization + writeIndex(0) + // do not send user ID; it is encoded in the serialization + writeIndex(0) + writeString(sourceId) + writeArrayStart() + setItemCount(records.size().toLong()) + for (record in records) { + startItem() + writeBytes(valueEncoder.encode(record)) + } + writeArrayEnd() + flush() + } + + companion object { + private val logger = LoggerFactory.getLogger(BinaryRecordContent::class.java) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt deleted file mode 100644 index 0a16602d..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordRequest.kt +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okio.Buffer -import okio.BufferedSink -import org.apache.avro.Schema -import org.apache.avro.SchemaValidationException -import org.apache.avro.generic.IndexedRecord -import org.apache.avro.io.BinaryEncoder -import org.apache.avro.io.EncoderFactory -import org.radarbase.data.AvroEncoder.AvroWriter -import org.radarbase.data.RecordData -import org.radarbase.data.RemoteSchemaEncoder -import org.radarbase.producer.rest.AvroDataMapperFactory.Companion.validationException -import org.radarbase.topic.AvroTopic -import org.radarbase.util.Strings.toHexString -import java.io.IOException -import java.lang.IllegalArgumentException - -/** - * Encodes a record request as binary data, in the form of a RecordSet. - * @param record key type - * @param record value type - */ -class BinaryRecordRequest(topic: AvroTopic) : RecordRequest { - private var keyVersion = 0 - private var valueVersion = 0 - private var records: RecordData? = null - private var binaryEncoder: BinaryEncoder? = null - private val valueEncoder: AvroWriter - private var sourceIdPos = 0 - - /** - * Binary record request for given topic. - * @param topic topic to send data for. - * @throws SchemaValidationException if the key schema does not contain a - * `sourceId` field. - * @throws IllegalArgumentException if the topic cannot be used to make a AvroWriter. - */ - init { - if (topic.keySchema.type != Schema.Type.RECORD) { - throw validationException( - topic.keySchema, topic.keySchema, - "Cannot use non-record key schema" - ) - } - val sourceIdField = topic.keySchema.getField("sourceId") - sourceIdPos = sourceIdField?.pos() - ?: throw validationException( - topic.keySchema, topic.keySchema, - "Cannot use binary encoder without a source ID." - ) - valueEncoder = RemoteSchemaEncoder(true) - .writer(topic.valueSchema, topic.valueClass) - } - - @Throws(IOException::class) - override fun writeToSink(sink: BufferedSink) { - writeToSink(sink, Int.MAX_VALUE) - } - - @Throws(IOException::class) - private fun writeToSink(sink: BufferedSink, maxLength: Int) { - binaryEncoder = EncoderFactory.get().directBinaryEncoder( - sink.outputStream(), binaryEncoder - ) - binaryEncoder?.writeRecords( - records ?: return, - maxLength - ) - } - - private fun BinaryEncoder.writeRecords(records: RecordData, maxLength: Int) { - startItem() - writeInt(keyVersion) - writeInt(valueVersion) - - // do not send project ID; it is encoded in the serialization - writeIndex(0) - // do not send user ID; it is encoded in the serialization - writeIndex(0) - val sourceId = (records.key as IndexedRecord)[sourceIdPos].toString() - writeString(sourceId) - writeArrayStart() - setItemCount(records.size().toLong()) - var curLength = 18 + sourceId.length - for (record in records) { - if (curLength >= maxLength) { - return - } - startItem() - val valueBytes = valueEncoder.encode(record) - writeBytes(valueBytes) - curLength += 4 + valueBytes.size - } - writeArrayEnd() - flush() - } - - override fun reset() { - records = null - } - - @Throws(SchemaValidationException::class) - override fun prepare( - keySchema: ParsedSchemaMetadata, valueSchema: ParsedSchemaMetadata, - records: RecordData - ) { - keyVersion = if (keySchema.version == null) 0 else keySchema.version - valueVersion = if (valueSchema.version == null) 0 else valueSchema.version - valueEncoder.readerSchema = valueSchema - this.records = records - } - - @Throws(IOException::class) - override fun content(maxLength: Int): String { - Buffer().use { buffer -> - writeToSink(buffer, maxLength / 2 - 2) - val printSize = buffer.size.coerceAtMost((maxLength - 2).toLong()) - return "0x" + buffer.readByteArray(printSize).toHexString() - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt index 0f627326..35eb66dc 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt @@ -15,7 +15,11 @@ */ package org.radarbase.producer.rest -import java.util.concurrent.TimeUnit +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.map +import org.radarbase.util.TimedValue +import org.radarbase.util.TimeoutConfig /** * Current connection status of a KafkaSender. After a timeout occurs this will turn to @@ -30,65 +34,46 @@ import java.util.concurrent.TimeUnit * A connection state could be shared with multiple HTTP clients if they are talking to the same * server. * - * @param timeout timeout - * @param unit unit of the timeout + * @param timeoutConfig timeout config * @throws IllegalArgumentException if the timeout is not strictly positive. */ class ConnectionState( - timeout: Long, - unit: TimeUnit, + private val timeoutConfig: TimeoutConfig, ) { /** State symbols of the connection. */ enum class State { CONNECTED, DISCONNECTED, UNKNOWN, UNAUTHORIZED } - private var timeout: Long = TimeUnit.MILLISECONDS.convert(timeout, unit) - private var lastConnection: Long = -1L - - /** Current state of the connection. */ - @get:Synchronized - var state: State = State.UNKNOWN - get() { - if (field == State.CONNECTED && System.currentTimeMillis() - lastConnection >= timeout) { - field = State.UNKNOWN - } - return field + val state: Flow + get() = mutableState.map { + if (it.value === State.CONNECTED && it.isExpired) + State.UNKNOWN + else + it.value } - private set - /** For a sender to indicate that a connection attempt succeeded. */ - @Synchronized - fun didConnect() { - state = State.CONNECTED - lastConnection = System.currentTimeMillis() + private val mutableState = MutableStateFlow(TimedValue(State.UNKNOWN, timeoutConfig)) + + init { + mutableState.tryEmit(TimedValue(State.UNKNOWN, timeoutConfig)) } - /** For a sender to indicate that a connection attempt failed. */ - @Synchronized - fun didDisconnect() { - state = State.DISCONNECTED + /** For a sender to indicate that a connection attempt succeeded. */ + suspend fun didConnect() { + mutableState.emit(TimedValue(State.CONNECTED, timeoutConfig)) } - @Synchronized - fun wasUnauthorized() { - state = State.UNAUTHORIZED + /** For a sender to indicate that a connection attempt failed. */ + suspend fun didDisconnect() { + mutableState.emit(TimedValue(State.DISCONNECTED, timeoutConfig)) } - @Synchronized - fun reset() { - state = State.UNKNOWN + suspend fun wasUnauthorized() { + mutableState.emit(TimedValue(State.UNAUTHORIZED, timeoutConfig)) } - /** - * Set the timeout after which the state will go from CONNECTED to UNKNOWN. - * @param timeout timeout - * @param unit unit of the timeout - * @throws IllegalArgumentException if the timeout is not strictly positive - */ - @Synchronized - fun setTimeout(timeout: Long, unit: TimeUnit) { - require(timeout > 0) { "Timeout must be strictly positive" } - this.timeout = TimeUnit.MILLISECONDS.convert(timeout, unit) + suspend fun reset() { + mutableState.emit(TimedValue(State.UNKNOWN, timeoutConfig)) } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt deleted file mode 100644 index 2be26e8d..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/GzipRequestInterceptor.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.* -import okio.BufferedSink -import okio.GzipSink -import okio.buffer -import java.io.IOException - -/** This interceptor compresses the HTTP request body. Many webservers can't handle this! */ -class GzipRequestInterceptor : Interceptor { - @Throws(IOException::class) - override fun intercept(chain: Interceptor.Chain): Response { - val originalRequest: Request = chain.request() - if (originalRequest.body == null || originalRequest.header("Content-Encoding") != null) { - return chain.proceed(originalRequest) - } - - return chain.proceed( - originalRequest.newBuilder() - .header("Content-Encoding", "gzip") - .method(originalRequest.method, gzip(originalRequest.body)) - .build() - ) - } - - private fun gzip(body: RequestBody?): RequestBody { - return object : RequestBody() { - override fun contentType(): MediaType? = body?.contentType() - - override fun contentLength(): Long { - return -1 // We don't know the compressed length in advance! - } - - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - GzipSink(sink).buffer().use { gzipSink -> body?.writeTo(gzipSink) } - } - } - } - - override fun hashCode(): Int = 1 - - override fun equals(other: Any?): Boolean { - return this === other || other != null && javaClass == other.javaClass - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt new file mode 100644 index 00000000..23ffa513 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt @@ -0,0 +1,66 @@ +package org.radarbase.producer.rest + +import io.ktor.http.content.* +import io.ktor.utils.io.* +import org.radarbase.data.RecordData +import org.radarbase.data.RemoteSchemaEncoder +import org.radarbase.producer.io.FunctionalWriteChannelContent +import org.radarbase.producer.schema.ParsedSchemaMetadata +import org.slf4j.LoggerFactory + +class JsonRecordContent( + private val records: RecordData, + private val keySchemaMetadata: ParsedSchemaMetadata, + private val valueSchemaMetadata: ParsedSchemaMetadata, +) : AvroRecordContent { + private val keyEncoder = RemoteSchemaEncoder.SchemaEncoderWriter( + binary = false, + schema = records.topic.keySchema, + clazz = records.topic.keyClass, + readerSchema = keySchemaMetadata.schema + ) + private val valueEncoder = RemoteSchemaEncoder.SchemaEncoderWriter( + binary = false, + schema = records.topic.valueSchema, + clazz = records.topic.valueClass, + readerSchema = valueSchemaMetadata.schema + ) + + override fun createContent(): OutgoingContent = + FunctionalWriteChannelContent { it.writeRecords() } + + private suspend fun ByteWriteChannel.writeRecords() { + writeByte('{'.code) + writeFully(KEY_SCHEMA_ID) + writeFully(keySchemaMetadata.id.toString().toByteArray()) + writeFully(VALUE_SCHEMA_ID) + writeFully(valueSchemaMetadata.id.toString().toByteArray()) + writeFully(RECORDS) + val key = keyEncoder.encode(records.key) + var first = true + for (record in records) { + if (first) { + first = false + } else { + writeByte(','.code) + } + writeFully(KEY) + writeFully(key) + writeFully(VALUE) + writeFully(valueEncoder.encode(record)) + writeByte('}'.code) + } + writeFully(END) + } + + companion object { + val KEY_SCHEMA_ID = "\"key_schema_id\":".toByteArray() + val VALUE_SCHEMA_ID = ",\"value_schema_id\":".toByteArray() + val RECORDS = ",\"records\":[".toByteArray() + val KEY = "{\"key\":".toByteArray() + val VALUE = ",\"value\":".toByteArray() + val END = "]}".toByteArray() + + private val logger = LoggerFactory.getLogger(JsonRecordContent::class.java) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt deleted file mode 100644 index a47f95aa..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordRequest.kt +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okio.Buffer -import okio.BufferedSink -import org.apache.avro.SchemaValidationException -import org.radarbase.data.AvroEncoder.AvroWriter -import org.radarbase.data.RecordData -import org.radarbase.data.RemoteSchemaEncoder -import org.radarbase.topic.AvroTopic -import java.io.IOException -import java.nio.charset.StandardCharsets - -/** - * Request data to submit records to the Kafka REST proxy. - */ -class JsonRecordRequest(topic: AvroTopic) : RecordRequest { - private val keyEncoder: AvroWriter - private val valueEncoder: AvroWriter - private var records: RecordData? = null - - /** - * Generate a record request for given topic. - * @param topic topic to use. - * @throws IllegalStateException if key or value encoders could not be made. - */ - init { - val schemaEncoder = RemoteSchemaEncoder(false) - keyEncoder = schemaEncoder.writer(topic.keySchema, topic.keyClass) - valueEncoder = schemaEncoder.writer(topic.valueSchema, topic.valueClass) - } - - /** - * Writes the current topic to a stream. This implementation does not use any JSON writers to - * write the data, but writes it directly to a stream. [JSONObject.quote] - * is used to get the correct formatting. This makes the method as lean as possible. - * @param sink buffered sink to write to. - * @throws IOException if a superimposing stream could not be created - */ - @Throws(IOException::class) - override fun writeToSink(sink: BufferedSink) { - writeToSink(sink, Int.MAX_VALUE) - } - - @Throws(IOException::class) - private fun writeToSink(sink: BufferedSink, maxLength: Int) { - val keySchema = checkNotNull(keyEncoder.readerSchema) { - "Record request has not been prepared with the proper reader schemas" - } - val valueSchema = checkNotNull(valueEncoder.readerSchema) { - "Record request has not been prepared with the proper reader schemas" - } - - sink.writeByte('{'.code) - sink.write(KEY_SCHEMA_ID) - sink.write(keySchema.id.toString().toByteArray()) - sink.write(VALUE_SCHEMA_ID) - sink.write(valueSchema.id.toString().toByteArray()) - sink.write(RECORDS) - val key = keyEncoder.encode(records!!.key) - var curLength = KEY_SCHEMA_ID.size + VALUE_SCHEMA_ID.size + 7 - var first = true - for (record in records!!) { - if (curLength >= maxLength) { - return - } - if (first) { - first = false - } else { - sink.writeByte(','.code) - } - sink.write(KEY) - sink.write(key) - sink.write(VALUE) - val valueBytes = valueEncoder.encode(record) - sink.write(valueBytes) - sink.writeByte('}'.code) - curLength += 2 + key.size + KEY.size + VALUE.size + valueBytes.size - } - sink.write(END) - } - - override fun reset() { - records = null - } - - @Throws(SchemaValidationException::class) - override fun prepare( - keySchema: ParsedSchemaMetadata, - valueSchema: ParsedSchemaMetadata, - records: RecordData - ) { - keyEncoder.readerSchema = keySchema - valueEncoder.readerSchema = valueSchema - this.records = records - } - - @Throws(IOException::class) - override fun content(maxLength: Int): String { - Buffer().use { buffer -> - writeToSink(buffer, maxLength) - return buffer.readString( - buffer.size.coerceAtMost(maxLength.toLong()), - StandardCharsets.UTF_8 - ) - } - } - - companion object { - val KEY_SCHEMA_ID = "\"key_schema_id\":".toByteArray() - val VALUE_SCHEMA_ID = ",\"value_schema_id\":".toByteArray() - val RECORDS = ",\"records\":[".toByteArray() - val KEY = "{\"key\":".toByteArray() - val VALUE = ",\"value\":".toByteArray() - val END = "]}".toByteArray() - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt deleted file mode 100644 index 048a15ce..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RecordRequest.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okio.BufferedSink -import org.apache.avro.SchemaValidationException -import org.radarbase.data.RecordData -import java.io.IOException - -/** - * Record request contents. Before [.writeToSink] is called, first - * [.prepare] should be called. This - * class may be reused by calling prepare and reset alternatively. - * - * @param record key type. - * @param record content type. - */ -interface RecordRequest { - /** Write the current records to a stream as a request. */ - @Throws(IOException::class) - fun writeToSink(sink: BufferedSink) - - /** Reset the contents. This may free up some memory because the recordrequest may be stored. */ - fun reset() - - /** Set the records to be sent. */ - @Throws(IOException::class, SchemaValidationException::class) - fun prepare( - keySchema: ParsedSchemaMetadata, valueSchema: ParsedSchemaMetadata, - records: RecordData - ) - - /** - * Return the content of the record as a string. To avoid dual reading of data for RecordData - * that does not store the results, prepare and reset may be called around this method. - * @param maxLength maximum returned length - * @return the content. - * @throws IOException if the content cannot be written. - */ - @Throws(IOException::class) - fun content(maxLength: Int): String -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt deleted file mode 100644 index 1997c344..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestClient.kt +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.* -import okhttp3.Headers.Companion.headersOf -import org.radarbase.config.ServerConfig -import org.radarbase.util.RestUtils -import org.slf4j.LoggerFactory -import java.io.IOException -import java.lang.ref.WeakReference -import java.net.MalformedURLException -import java.util.* -import java.util.concurrent.TimeUnit -import javax.net.ssl.X509TrustManager - -/** REST client using OkHttp3. This class is not thread-safe. */ -class RestClient private constructor(builder: Builder) { - /** Configured server. */ - val server: ServerConfig = requireNotNull(builder.server) { "Missing server configuration" } - /** OkHttp client. */ - val httpClient: OkHttpClient = builder.httpClientBuilder.build() - private val headers: Headers = builder.headers - private val relativeUrl = server.httpUrl - - /** Configured connection timeout in seconds. */ - val timeout: Long - get() = (httpClient.connectTimeoutMillis / 1000).toLong() - - /** - * Make a blocking request. - * @param request request, possibly built with [.requestBuilder] - * @return response to the request - * @throws IOException if the request fails - * @throws NullPointerException if the request is null - */ - @Throws(IOException::class) - fun request(request: Request): Response { - return httpClient.newCall(request).execute() - } - - /** - * Make a blocking request. - * @param request request, possibly built with [.requestBuilder] - * @return response to the request - * @throws IOException if the request fails - * @throws NullPointerException if the request is null - */ - @Throws(IOException::class) - fun request(builder: Request.Builder.() -> Unit): Response = httpClient.newCall( - Request.Builder().apply { - headers(headers) - builder() - }.build() - ).execute() - - /** - * Make a blocking request. - * @param request request, possibly built with [.requestBuilder] - * @return response to the request - * @throws IOException if the request fails - * @throws NullPointerException if the request is null - */ - @Throws(IOException::class) - inline fun request( - relativePath: String, - crossinline builder: Request.Builder.() -> Unit, - ): Response = request { - url(relativeUrl(relativePath)) - builder() - } - - /** - * Make an asynchronous request. - * @param request request, possibly built with [.requestBuilder] - * @param callback callback to activate once the request is done. - */ - fun request(request: Request, callback: Callback) = - httpClient.newCall(request).enqueue(callback) - - /** - * Make a request to given relative path. This does not set any request properties except the - * URL. - * @param relativePath relative path to request - * @return response to the request - * @throws IOException if the path is invalid or the request failed. - */ - @Throws(IOException::class) - fun request(relativePath: String): Response = request(buildRequest(relativePath)) - - /** - * Make a blocking request and return the body. - * @param request request to make. - * @return response body string. - * @throws RestException if no body was returned or an HTTP status code indicating error was - * returned. - * @throws IOException if the request cannot be completed or the response cannot be read. - */ - @Throws(IOException::class) - fun requestString(request: Request): String { - request(request).use { response -> - val bodyString = response.bodyString() - if (!response.isSuccessful || bodyString == null) { - throw RestException(response.code, bodyString) - } - return bodyString - } - } - - /** - * Create a OkHttp3 request builder with [Request.Builder.url] set. - * Call[Request.Builder.build] to make the actual request with - * [.request]. - * - * @param relativePath relative path from the server serverConfig - * @return request builder. - * @throws MalformedURLException if the path not valid - */ - @Throws(MalformedURLException::class) - fun requestBuilder(relativePath: String): Request.Builder = Request.Builder() - .url(relativeUrl(relativePath)) - .headers(headers) - - fun buildRequest( - relativePath: String, - builder: Request.Builder.() -> Unit = {}, - ): Request = Request.Builder().apply { - url(relativeUrl(relativePath)) - headers(headers) - builder() - }.build() - - /** - * Get a URL relative to the configured server. - * @param path relative path - * @return URL - * @throws MalformedURLException if the path is malformed - */ - @Throws(MalformedURLException::class) - fun relativeUrl(path: String): HttpUrl { - val urlBuilder = relativeUrl.newBuilder(path.trimStart { it == '/' }) - ?: throw MalformedURLException() - return urlBuilder.build() - } - - override fun equals(other: Any?): Boolean { - if (this === other) { - return true - } - if (other == null || javaClass != other.javaClass) { - return false - } - other as RestClient - return server == other.server && httpClient == other.httpClient - } - - override fun hashCode(): Int = Objects.hash(server, httpClient) - - override fun toString(): String = "RestClient{serverConfig=$server, httpClient=$httpClient}" - - /** Create a new builder with the settings of the current client. */ - fun withConfiguration(builder: Builder.() -> Unit): RestClient = Builder(httpClient).apply { - this.server = this@RestClient.server - builder() - }.build() - - /** Builder. */ - class Builder internal constructor( - val httpClientBuilder: OkHttpClient.Builder - ) { - var server: ServerConfig? = null - set(value) { - if (value != null) { - if (value.isUnsafe) { - checkNotNull(RestUtils.UNSAFE_SSL_FACTORY) { - "Cannot use unsafe connection, it is disallowed by the runtime environment." - } - httpClientBuilder.sslSocketFactory( - RestUtils.UNSAFE_SSL_FACTORY, - RestUtils.UNSAFE_TRUST_MANAGER[0] as X509TrustManager - ) - httpClientBuilder.hostnameVerifier(RestUtils.UNSAFE_HOSTNAME_VERIFIER) - } else { - val trustManager = RestUtils.systemDefaultTrustManager() - val socketFactory = RestUtils.systemDefaultSslSocketFactory(trustManager) - httpClientBuilder.sslSocketFactory(socketFactory, trustManager) - httpClientBuilder.hostnameVerifier(RestUtils.DEFAULT_HOSTNAME_VERIFIER) - } - } - field = value - } - var headers: Headers = headersOf() - - constructor(client: OkHttpClient) : this(client.newBuilder()) - - /** Allowed protocols. */ - fun protocols(protocols: List) { - httpClientBuilder.protocols(protocols) - } - - /** Whether to enable GZIP compression. */ - fun gzipCompression(compression: Boolean) { - val gzip = httpClientBuilder.interceptors() - .find { it is GzipRequestInterceptor} as GzipRequestInterceptor? - if (compression && gzip == null) { - logger.debug("Enabling GZIP compression") - httpClientBuilder.addInterceptor(GzipRequestInterceptor()) - } else if (!compression && gzip != null) { - logger.debug("Disabling GZIP compression") - httpClientBuilder.interceptors().remove(gzip) - } - } - - /** Timeouts for connecting, reading and writing. */ - fun timeout(timeout: Long, unit: TimeUnit) { - httpClientBuilder.connectTimeout(timeout, unit) - .readTimeout(timeout, unit) - .writeTimeout(timeout, unit) - } - - /** Build a new RestClient. */ - fun build(): RestClient { - return RestClient(this) - } - } - - companion object { - private val logger = LoggerFactory.getLogger(RestClient::class.java) - const val DEFAULT_TIMEOUT: Long = 30 - private var globalHttpClientRef = WeakReference(null) - - val globalHttpClient: OkHttpClient - @Synchronized - get() = globalHttpClientRef.get() - ?: createDefaultClient().build() - .also { globalHttpClientRef = WeakReference(it) } - - /** Get the response body of a response as a String. - * Will return null if the response body is null. - * @return body contents as a String. - * @throws IOException if the body could not be read as a String. - */ - @JvmStatic - @Throws(IOException::class) - fun Response.bodyString(): String? { - return body?.use { it.string() } - } - - /** Create a builder with a global shared OkHttpClient. */ - @JvmStatic - fun globalRestClient(builder: Builder.() -> Unit = {}): RestClient = - Builder(globalHttpClient).apply(builder).build() - - /** Create a builder with a new OkHttpClient using default settings. */ - @JvmStatic - fun newRestClient(builder: Builder.() -> Unit): RestClient = - Builder(createDefaultClient()).apply(builder).build() - - /** - * Create a new OkHttpClient. The timeouts are set to the default. - * @return new OkHttpClient. - */ - private fun createDefaultClient() = OkHttpClient.Builder() - .connectTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) - .readTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) - .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.SECONDS) - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt index 86871b16..9048105b 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestException.kt @@ -15,6 +15,7 @@ */ package org.radarbase.producer.rest +import io.ktor.http.* import java.io.IOException /** @@ -23,16 +24,16 @@ import java.io.IOException class RestException /** * Request with status code and response body. - * @param statusCode HTTP status code + * @param status HTTP status code * @param body response body. */( - val statusCode: Int, - val body: String?, + val status: HttpStatusCode, + body: String? = null, cause: Throwable? = null, ) : IOException( buildString(150) { append("REST call failed (HTTP code ") - append(statusCode) + append(status) if (body == null) { append(')') } else { diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt new file mode 100644 index 00000000..419effb6 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt @@ -0,0 +1,257 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.engine.cio.* +import io.ktor.client.plugins.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.util.reflect.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withContext +import org.apache.avro.SchemaValidationException +import org.radarbase.data.RecordData +import org.radarbase.producer.AuthenticationException +import org.radarbase.producer.KafkaSender +import org.radarbase.producer.KafkaTopicSender +import org.radarbase.producer.io.GzipContentEncoding +import org.radarbase.producer.io.UnsupportedMediaTypeException +import org.radarbase.producer.io.timeout +import org.radarbase.producer.io.unsafeSsl +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.topic.AvroTopic +import org.radarbase.util.RadarProducerDsl +import org.radarbase.util.TimeoutConfig +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Duration +import java.util.* +import kotlin.coroutines.CoroutineContext +import kotlin.reflect.javaType +import kotlin.reflect.typeOf + +/** + * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new + * sender must be constructed with [.sender] per AvroTopic. This implementation is + * blocking and unbuffered, so flush, clear and close do not do anything. + */ +class RestKafkaSender(config: Config) : KafkaSender { + private val allowUnsafe: Boolean = config.allowUnsafe + private val contentType: ContentType = config.contentType + val schemaRetriever: SchemaRetriever = requireNotNull(config.schemaRetriever) { + "Missing schemaRetriever from configuration" + } + /** Get the current REST client. */ + val restClient: HttpClient + + private val _connectionState: ConnectionState = config.connectionState + ?: ConnectionState(TimeoutConfig(DEFAULT_TIMEOUT)) + override val connectionState: Flow + get() = _connectionState.state + + private val baseUrl: String = requireNotNull(config.baseUrl) + private val headers: Headers = config.headers.build() + private val connectionTimeout: Duration = config.connectionTimeout + private val contentEncoding = config.contentEncoding + private val originalHttpClient = config.httpClient + private val ioContext: CoroutineContext = config.ioContext + + /** + * Construct a RestSender. + */ + init { + restClient = config.httpClient?.config { + configure() + } ?: HttpClient(CIO) { + configure() + } + } + + private fun HttpClientConfig<*>.configure() { + timeout(connectionTimeout) + install(ContentNegotiation) { + this.register( + KAFKA_REST_BINARY_ENCODING, + AvroContentConverter(schemaRetriever, binary = true) + ) + this.register( + KAFKA_REST_JSON_ENCODING, + AvroContentConverter(schemaRetriever, binary = false) + ) + } + when(contentEncoding) { + GZIP_CONTENT_ENCODING -> install(GzipContentEncoding) + else -> {} + } + if (allowUnsafe) { + unsafeSsl() + } + defaultRequest { + url(baseUrl) + contentType(contentType) + accept(ContentType.Application.Json) + headers { + appendAll(this@RestKafkaSender.headers) + } + } + } + + inner class RestKafkaTopicSender( + override val topic: AvroTopic, + ) : KafkaTopicSender { + @OptIn(ExperimentalStdlibApi::class) + override suspend fun send(records: RecordData) { + try { + val response: HttpResponse = withContext(Dispatchers.IO) { + restClient.post { + url("topics/${topic.name}") + val kType = typeOf>() + val reifiedType = kType.javaType + setBody(records, TypeInfo(RecordData::class, reifiedType, kType)) + } + } + if (response.status.isSuccess()) { + _connectionState.didConnect() + logger.debug("Added message to topic {}", topic) + } else if (response.status == HttpStatusCode.Unauthorized || response.status == HttpStatusCode.Forbidden) { + _connectionState.wasUnauthorized() + throw AuthenticationException("Request unauthorized") + } else if (response.status == HttpStatusCode.UnsupportedMediaType) { + throw UnsupportedMediaTypeException( + response.request.contentType(), + response.request.headers[HttpHeaders.ContentEncoding], + ) + } else { + _connectionState.didDisconnect() + throw RestException(response.status, response.bodyAsText()) + } + } catch (ex: IOException) { + _connectionState.didDisconnect() + throw ex + } + } + } + + @Throws(SchemaValidationException::class) + override fun sender(topic: AvroTopic): KafkaTopicSender { + return RestKafkaTopicSender(topic) + } + + @Throws(AuthenticationException::class) + override suspend fun resetConnection(): Boolean { + if (connectionState.first() === ConnectionState.State.CONNECTED) { + return true + } + val lastState = try { + val response = withContext(Dispatchers.IO) { + restClient.head { + url("") + } + } + if (response.status.isSuccess()) { + _connectionState.didConnect() + ConnectionState.State.CONNECTED + } else if (response.status == HttpStatusCode.Unauthorized) { + _connectionState.wasUnauthorized() + throw AuthenticationException("HEAD request unauthorized") + } else { + _connectionState.didDisconnect() + val bodyString = response.bodyAsText() + logger.warn( + "Failed to make heartbeat request to {} (HTTP status code {}): {}", + restClient, response.status, bodyString + ) + ConnectionState.State.DISCONNECTED + } + } catch (ex: IOException) { + // no stack trace is needed + _connectionState.didDisconnect() + logger.warn("Failed to make heartbeat request to {}: {}", restClient, ex.toString()) + ConnectionState.State.DISCONNECTED + } + return lastState === ConnectionState.State.CONNECTED + } + + fun config(config: Config.() -> Unit): RestKafkaSender { + val oldConfig = toConfig() + val newConfig = toConfig().apply(config) + return if (oldConfig == newConfig) this else RestKafkaSender(newConfig) + } + + private fun toConfig() = Config().apply { + baseUrl = this@RestKafkaSender.baseUrl + httpClient = this@RestKafkaSender.originalHttpClient + schemaRetriever = this@RestKafkaSender.schemaRetriever + headers = HeadersBuilder().apply { appendAll(this@RestKafkaSender.headers) } + contentType = this@RestKafkaSender.contentType + contentEncoding = this@RestKafkaSender.contentEncoding + connectionTimeout = this@RestKafkaSender.connectionTimeout + allowUnsafe = this@RestKafkaSender.allowUnsafe + ioContext = this@RestKafkaSender.ioContext + } + + @RadarProducerDsl + class Config { + var ioContext: CoroutineContext = Dispatchers.IO + var baseUrl: String? = null + var schemaRetriever: SchemaRetriever? = null + var connectionState: ConnectionState? = null + var httpClient: HttpClient? = null + var headers = HeadersBuilder() + var connectionTimeout: Duration = Duration.ofSeconds(30) + var contentEncoding: String? = null + var allowUnsafe: Boolean = false + var contentType: ContentType = KAFKA_REST_JSON_ENCODING + + fun httpClient(config: HttpClientConfig<*>.() -> Unit = {}) { + httpClient = httpClient?.config(config) + ?: HttpClient(CIO, config) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || javaClass != other.javaClass) return false + other as Config + return schemaRetriever == other.schemaRetriever && + connectionState == other.connectionState && + headers.build() == other.headers.build() && + httpClient == other.httpClient && + contentType == other.contentType && + baseUrl == other.baseUrl && + connectionTimeout == other.connectionTimeout && + contentEncoding == other.contentEncoding && + ioContext == other.ioContext + } + override fun hashCode(): Int = headers.hashCode() + } + + companion object { + private val logger = LoggerFactory.getLogger(RestKafkaSender::class.java) + val DEFAULT_TIMEOUT: Duration = Duration.ofSeconds(20) + val KAFKA_REST_BINARY_ENCODING = ContentType("application", "vnd.radarbase.avro.v1+binary") + val KAFKA_REST_JSON_ENCODING = ContentType("application", "vnd.kafka+json") + const val GZIP_CONTENT_ENCODING = "gzip" + + fun restKafkaSender(builder: Config.() -> Unit): RestKafkaSender = + RestKafkaSender(Config().apply(builder)) + } +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt deleted file mode 100644 index a9ad76a7..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestSender.kt +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.Headers -import okhttp3.MediaType -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.Protocol -import okhttp3.Request -import org.apache.avro.SchemaValidationException -import org.radarbase.config.ServerConfig -import org.radarbase.producer.AuthenticationException -import org.radarbase.producer.KafkaSender -import org.radarbase.producer.KafkaTopicSender -import org.radarbase.producer.rest.RestClient.Companion.bodyString -import org.radarbase.producer.rest.RestClient.Companion.globalRestClient -import org.radarbase.topic.AvroTopic -import org.slf4j.LoggerFactory -import java.io.IOException -import java.net.MalformedURLException -import java.util.* -import java.util.concurrent.TimeUnit - -/** - * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new - * sender must be constructed with [.sender] per AvroTopic. This implementation is - * blocking and unbuffered, so flush, clear and close do not do anything. - */ -class RestSender private constructor(builder: Builder) : KafkaSender { - /** - * Get the current request properties. - */ - @get:Synchronized - var requestProperties: RequestProperties - private set - private var connectionTestRequest: Request.Builder - /** Get the schema retriever. */ - /** Set the schema retriever. */ - @get:Synchronized - @set:Synchronized - var schemaRetriever: SchemaRetriever? - - /** Get the current REST client. */ - @get:Synchronized - var restClient: RestClient = requireNotNull(builder.httpClient).withConfiguration { - protocols(listOf(Protocol.HTTP_1_1)) - } - private set(value) { - try { - connectionTestRequest = value.requestBuilder("").head() - } catch (ex: MalformedURLException) { - throw IllegalArgumentException("Schemaless topics do not have a valid URL", ex) - } - field = value - state.reset() - } - private val state: ConnectionState - - /** - * Construct a RestSender. - */ - init { - schemaRetriever = Objects.requireNonNull(builder.schemaRetriever) - requestProperties = RequestProperties( - KAFKA_REST_ACCEPT_ENCODING, - if (builder.useBinaryContent) KAFKA_REST_BINARY_ENCODING else KAFKA_REST_AVRO_ENCODING, - builder.headers.build(), - builder.useBinaryContent - ) - state = builder.connectionState - ?: ConnectionState(RestClient.DEFAULT_TIMEOUT, TimeUnit.SECONDS) - restClient = builder.httpClient?.withConfiguration { - protocols(listOf(Protocol.HTTP_1_1)) - } ?: globalRestClient { - protocols(listOf(Protocol.HTTP_1_1)) - } - try { - connectionTestRequest = restClient.requestBuilder("").head() - } catch (ex: MalformedURLException) { - throw IllegalArgumentException("Schemaless topics do not have a valid URL", ex) - } - } - - /** - * Set the connection timeout. This affects both the connection state as the HTTP client - * setting. - * @param connectionTimeout timeout - * @param unit time unit - */ - @Synchronized - fun setConnectionTimeout(connectionTimeout: Long, unit: TimeUnit) { - if (connectionTimeout == restClient.timeout) return - restClient = restClient.withConfiguration { - timeout(connectionTimeout, unit) - } - state.setTimeout(connectionTimeout, unit) - } - - /** - * Set the Kafka REST Proxy settings. This affects the REST client. - * @param kafkaConfig server configuration of the Kafka REST proxy. - */ - @Synchronized - fun setKafkaConfig(kafkaConfig: ServerConfig) { - if (kafkaConfig == restClient.server) return - restClient = restClient.withConfiguration { - server = kafkaConfig - } - } - - /** Get a request to check the connection status. */ - @Synchronized - private fun getConnectionTestRequest(): Request { - return connectionTestRequest.headers(requestProperties.headers).build() - } - - /** Set the compression of the REST client. */ - @Synchronized - fun setCompression(useCompression: Boolean) { - restClient = restClient.withConfiguration { - gzipCompression(useCompression) - } - } - /** Get the headers used in requests. */ - /** Set the headers used in requests. */ - @get:Synchronized - @set:Synchronized - var headers: Headers - get() = requestProperties.headers - set(additionalHeaders) { - requestProperties = RequestProperties( - requestProperties.acceptType, - requestProperties.contentType, additionalHeaders, - requestProperties.binary - ) - state.reset() - } - - @Throws(SchemaValidationException::class) - override fun sender(topic: AvroTopic): KafkaTopicSender { - return RestTopicSender(topic, this, state) - } - - /** - * Get the current request context. - */ - @get:Synchronized - val requestContext: RequestContext - get() = RequestContext(restClient, requestProperties) - - @Throws(AuthenticationException::class) - override fun resetConnection(): Boolean { - if (state.state === ConnectionState.State.CONNECTED) { - return true - } - try { - restClient.request(getConnectionTestRequest()).use { response -> - if (response.isSuccessful) { - state.didConnect() - } else if (response.code == 401) { - state.wasUnauthorized() - } else { - state.didDisconnect() - val bodyString = response.bodyString() - logger.warn( - "Failed to make heartbeat request to {} (HTTP status code {}): {}", - restClient, response.code, bodyString - ) - } - } - } catch (ex: IOException) { - // no stack trace is needed - state.didDisconnect() - logger.warn("Failed to make heartbeat request to {}: {}", restClient, ex.toString()) - } - if (state.state === ConnectionState.State.UNAUTHORIZED) { - throw AuthenticationException("HEAD request unauthorized") - } - return state.state === ConnectionState.State.CONNECTED - } - - @get:Throws(AuthenticationException::class) - override val isConnected: Boolean - get() = when (state.state) { - ConnectionState.State.CONNECTED -> true - ConnectionState.State.DISCONNECTED -> false - ConnectionState.State.UNAUTHORIZED -> throw AuthenticationException("Unauthorized") - ConnectionState.State.UNKNOWN -> resetConnection() - } - - override fun close() { - // noop - } - - /** - * Revert to a legacy connection if the server does not support the latest protocols. - * @param acceptEncoding accept encoding to use in the legacy connection. - * @param contentEncoding content encoding to use in the legacy connection. - * @param binary whether to send the data as binary. - */ - @Synchronized - fun useLegacyEncoding( - acceptEncoding: String, - contentEncoding: MediaType, - binary: Boolean, - ) { - logger.debug( - "Reverting to encoding {} -> {} (binary: {})", - contentEncoding, acceptEncoding, binary - ) - requestProperties = RequestProperties( - acceptEncoding, - contentEncoding, - requestProperties.headers, binary - ) - } - - class Builder internal constructor(){ - var schemaRetriever: SchemaRetriever? = null - var connectionState: ConnectionState? = null - var httpClient: RestClient? = null - var headers = Headers.Builder() - /** - * Whether to try to send binary content. This only works if the server supports it. If not, - * there may be an additional round-trip. - */ - var useBinaryContent = false - - /** Build a new RestSender. */ - fun build(): RestSender { - return RestSender(this) - } - } - - class RequestContext( - val client: RestClient, - val properties: RequestProperties, - ) - class RequestProperties( - val acceptType: String, - val contentType: MediaType, - val headers: Headers, - val binary: Boolean, - ) - - companion object { - private val logger = LoggerFactory.getLogger(RestSender::class.java) - const val KAFKA_REST_ACCEPT_ENCODING = - "application/vnd.kafka.v2+json, application/vnd.kafka+json, application/json" - const val KAFKA_REST_ACCEPT_LEGACY_ENCODING = - "application/vnd.kafka.v1+json, application/vnd.kafka+json, application/json" - @JvmField - val KAFKA_REST_BINARY_ENCODING: MediaType = - "application/vnd.radarbase.avro.v1+binary".toMediaType() - @JvmField - val KAFKA_REST_AVRO_ENCODING: MediaType = - "application/vnd.kafka.avro.v2+json; charset=utf-8".toMediaType() - @JvmField - val KAFKA_REST_AVRO_LEGACY_ENCODING: MediaType = - "application/vnd.kafka.avro.v1+json; charset=utf-8".toMediaType() - - fun restSender(builder: Builder.() -> Unit): RestSender = - Builder().apply(builder).build() - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt deleted file mode 100644 index 435b5a3a..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestTopicSender.kt +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.Request -import okhttp3.Response -import org.apache.avro.SchemaValidationException -import org.json.JSONException -import org.radarbase.data.AvroRecordData -import org.radarbase.data.RecordData -import org.radarbase.producer.AuthenticationException -import org.radarbase.producer.KafkaTopicSender -import org.radarbase.topic.AvroTopic -import org.slf4j.LoggerFactory -import java.io.IOException - -internal class RestTopicSender( - topic: AvroTopic, - sender: RestSender, - state: ConnectionState -) : KafkaTopicSender { - private val topic: AvroTopic - private var requestData: RecordRequest - private val sender: RestSender - private val state: ConnectionState - - init { - this.topic = topic - this.sender = sender - this.state = state - requestData = if (sender.requestContext.properties.binary) { - try { - BinaryRecordRequest(topic) - } catch (ex: IllegalArgumentException) { - logger.warn( - "Cannot use Binary encoding for incompatible topic {}: {}", - topic, ex.toString() - ) - JsonRecordRequest(topic) - } - } else { - JsonRecordRequest(topic) - } - } - - @Throws(IOException::class, SchemaValidationException::class) - override fun send(key: K, value: V) { - send(AvroRecordData(topic, key, listOf(value))) - } - - /** - * Actually make a REST request to the Kafka REST server and Schema Registry. - * - * @param records values to send - * @throws IOException if records could not be sent - */ - @Throws(IOException::class, SchemaValidationException::class) - override fun send(records: RecordData) { - val context = sender.requestContext - val request = buildRequest(context, records) - var doResend = false - try { - context.client.request(request).use { response -> - if (response.isSuccessful) { - state.didConnect() - logger.debug("Added message to topic {}", topic) - } else if (response.code == 401 || response.code == 403) { - state.wasUnauthorized() - } else if (response.code == 415) { - downgradeConnection(request, response) - doResend = true - } else { - throw UncheckedRequestException.fail(request, response, null) - } - } - } catch (ex: IOException) { - state.didDisconnect() - UncheckedRequestException.fail(request, null, ex).rethrow() - } catch (ex: UncheckedRequestException) { - state.didDisconnect() - ex.rethrow() - } finally { - requestData.reset() - } - if (state.state === ConnectionState.State.UNAUTHORIZED) { - throw AuthenticationException("Request unauthorized") - } - if (doResend) { - send(records) - } - } - - @Throws(IOException::class, SchemaValidationException::class) - private fun updateRecords(context: RestSender.RequestContext, records: RecordData) { - if (!context.properties.binary && requestData is BinaryRecordRequest<*, *>) { - requestData = JsonRecordRequest(topic) - } - val sendTopic = topic.name - val retriever = sender.schemaRetriever - val keyMetadata: ParsedSchemaMetadata - val valueMetadata: ParsedSchemaMetadata - try { - keyMetadata = retriever!!.getOrSetSchemaMetadata( - sendTopic, false, topic.keySchema, -1 - ) - valueMetadata = retriever.getOrSetSchemaMetadata( - sendTopic, true, topic.valueSchema, -1 - ) - } catch (ex: JSONException) { - throw IOException("Failed to get schemas for topic $topic", ex) - } catch (ex: IOException) { - throw IOException("Failed to get schemas for topic $topic", ex) - } - requestData.prepare(keyMetadata, valueMetadata, records) - } - - @Throws(IOException::class) - private fun downgradeConnection(request: Request, response: Response) { - if (requestData is BinaryRecordRequest<*, *>) { - state.didConnect() - logger.warn( - "Binary Avro encoding is not supported." - + " Switching to JSON encoding." - ) - sender.useLegacyEncoding( - RestSender.KAFKA_REST_ACCEPT_ENCODING, RestSender.KAFKA_REST_AVRO_ENCODING, - false - ) - requestData = JsonRecordRequest(topic) - } else if (request.header("Accept") == RestSender.KAFKA_REST_ACCEPT_ENCODING) { - state.didConnect() - logger.warn( - "Latest Avro encoding is not supported. Switching to legacy " - + "encoding." - ) - sender.useLegacyEncoding( - RestSender.KAFKA_REST_ACCEPT_LEGACY_ENCODING, - RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING, - false - ) - } else { - val body = request.body - val contentType = body?.contentType() - if (contentType == null - || contentType == RestSender.KAFKA_REST_AVRO_LEGACY_ENCODING - ) { - throw UncheckedRequestException.fail( - request, response, - IOException("Content-Type $contentType not accepted by server.") - ) - } else { - // the connection may have been downgraded already - state.didConnect() - logger.warn("Content-Type changed during request") - } - } - } - - @Throws(IOException::class, SchemaValidationException::class) - private fun buildRequest( - context: RestSender.RequestContext, - records: RecordData - ): Request { - updateRecords(context, records) - val sendToUrl = context.client.relativeUrl("topics/" + topic.name) - val requestBody: TopicRequestBody - val requestBuilder = Request.Builder() - .url(sendToUrl) - .headers(context.properties.headers) - .header("Accept", context.properties.acceptType) - var contentType = context.properties.contentType - if (contentType == RestSender.KAFKA_REST_BINARY_ENCODING - && requestData !is BinaryRecordRequest<*, *> - ) { - contentType = RestSender.KAFKA_REST_AVRO_ENCODING - } - requestBody = TopicRequestBody(requestData, contentType) - return requestBuilder.post(requestBody).build() - } - - override fun clear() { - // nothing - } - - override fun flush() { - // nothing - } - - override fun close() { - // noop - } - - companion object { - private val logger = LoggerFactory.getLogger(RestTopicSender::class.java) - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt deleted file mode 100644 index 8a8630a7..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRestClient.kt +++ /dev/null @@ -1,103 +0,0 @@ -package org.radarbase.producer.rest - -import okhttp3.MediaType -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.Request -import okhttp3.RequestBody -import okio.BufferedSink -import org.apache.avro.Schema -import org.json.JSONException -import org.json.JSONObject -import java.io.IOException - -/** REST client for Confluent schema registry. */ -class SchemaRestClient( - private val client: RestClient, -) { - /** Retrieve schema metadata from server. */ - @Throws(JSONException::class, IOException::class) - fun retrieveSchemaMetadata(subject: String, version: Int): ParsedSchemaMetadata { - val isLatest = version <= 0 - val node = requestJson(buildString(60) { - append("/subjects/") - append(subject) - append("/versions/") - if (isLatest) { - append("latest") - } else { - append(version) - } - }) - val newVersion = if (isLatest) node.getInt("version") else version - val schemaId = node.getInt("id") - val schema = parseSchema(node.getString("schema")) - return ParsedSchemaMetadata(schemaId, newVersion, schema) - } - - @Throws(IOException::class) - private fun requestJson(path: String): JSONObject { - val request: Request = client.requestBuilder(path) - .addHeader("Accept", "application/json") - .build() - val response = client.requestString(request) - return JSONObject(response) - } - - /** Parse a schema from string. */ - fun parseSchema(schemaString: String): Schema { - return Schema.Parser().parse(schemaString) - } - - /** Add a schema to a subject. */ - @Throws(IOException::class) - fun addSchema(subject: String, schema: Schema): Int { - val request: Request = client.requestBuilder("/subjects/$subject/versions") - .addHeader("Accept", "application/json") - .post(SchemaRequestBody(schema)) - .build() - val response = client.requestString(request) - val node = JSONObject(response) - return node.getInt("id") - } - - /** Request metadata for a schema on a subject. */ - @Throws(IOException::class) - fun requestMetadata(subject: String, schema: Schema): ParsedSchemaMetadata { - val request: Request = client.requestBuilder("/subjects/$subject") - .addHeader("Accept", "application/json") - .post(SchemaRequestBody(schema)) - .build() - val response = client.requestString(request) - val node = JSONObject(response) - return ParsedSchemaMetadata( - node.getInt("id"), - node.getInt("version"), schema - ) - } - - /** Retrieve schema metadata from server. */ - @Throws(JSONException::class, IOException::class) - fun retrieveSchemaById(id: Int): Schema { - val node = requestJson("/schemas/ids/$id") - return parseSchema(node.getString("schema")) - } - - private class SchemaRequestBody( - private val schema: Schema - ) : RequestBody() { - override fun contentType(): MediaType = CONTENT_TYPE - - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - sink.write(SCHEMA) - sink.writeUtf8(JSONObject.quote(schema.toString())) - sink.writeByte('}'.code) - } - - companion object { - private val SCHEMA = "{\"schema\":".toByteArray() - private val CONTENT_TYPE: MediaType = "application/vnd.schemaregistry.v1+json; charset=utf-8" - .toMediaType() - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt deleted file mode 100644 index d8e7935b..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/TopicRequestBody.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.MediaType -import okhttp3.Request -import okhttp3.RequestBody -import okio.BufferedSink -import java.io.IOException - -/** - * TopicRequestData in a RequestBody. - */ -internal class TopicRequestBody( - protected val data: RecordRequest<*, *>, - private val mediaType: MediaType -) : RequestBody() { - override fun contentType(): MediaType? { - return mediaType - } - - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - data.writeToSink(sink) - } - - companion object { - @JvmStatic - @Throws(IOException::class) - fun topicRequestContent(request: Request?, maxLength: Int): String? { - request ?: return null - val body = request.body as TopicRequestBody? ?: return null - return body.data.content(maxLength) - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt deleted file mode 100644 index a8f597e8..00000000 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/UncheckedRequestException.kt +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.Request -import okhttp3.Response -import org.radarbase.producer.rest.RestClient.Companion.bodyString -import org.radarbase.producer.rest.TopicRequestBody.Companion.topicRequestContent -import java.io.IOException - -/** Unchecked exception for failures during request handling. */ -class UncheckedRequestException -/** - * Unchecked exception. - * @param message exception message. - * @param cause cause of this exception, may be null - */ - (message: String?, cause: IOException?) : RuntimeException(message, cause) { - /** - * Rethrow this exception using either its cause, if that is an IOException, or using - * the current exception. - * @throws IOException if the cause of the exception was an IOException. - * @throws UncheckedRequestException if the cause of the exception was not an IOException. - */ - @Throws(IOException::class) - fun rethrow() { - if (cause is IOException) { - throw (cause as IOException?)!! - } else { - throw IOException(this) - } - } - - companion object { - private const val serialVersionUID: Long = 1 - private const val LOG_CONTENT_LENGTH = 1024 - - /** - * Create a new UncheckedRequestException based on given call. - * - * @param request call request - * @param response call response, may be null - * @param cause exception cause, may be null - * @return new exception - * @throws IOException if the request or response cannot be constructed into a message. - */ - @Throws(IOException::class) - fun fail( - request: Request?, - response: Response?, cause: IOException? - ): UncheckedRequestException { - val message = buildString(128) { - append("FAILED to transmit message") - val content: String? = if (response != null) { - append(" (HTTP status code ") - append(response.code) - append(')') - response.bodyString() - } else { - null - } - val requestContent = topicRequestContent(request, LOG_CONTENT_LENGTH) - if (requestContent != null || content != null) { - append(':') - } - if (requestContent != null) { - append("\n ") - append(requestContent) - } - if (content != null) { - append("\n ") - append(content) - } - } - return UncheckedRequestException(message, cause) - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/ParsedSchemaMetadata.kt similarity index 96% rename from radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt rename to radar-commons/src/main/java/org/radarbase/producer/schema/ParsedSchemaMetadata.kt index 1b5808ba..64ff333c 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/ParsedSchemaMetadata.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/ParsedSchemaMetadata.kt @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.radarbase.producer.rest +package org.radarbase.producer.schema import org.apache.avro.Schema diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt new file mode 100644 index 00000000..762ee004 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.schema + +import kotlinx.serialization.Serializable +import org.apache.avro.Schema + +/** + * Parsed schema metadata from a Schema Registry. + */ +@Serializable +data class SchemaMetadata +/** + * Schema metadata. + * @param id schema ID, may be null. + * @param version schema version, may be null. + * @param schema parsed schema. + */( + val id: Int? = null, + val version: Int? = null, + val schema: String? = null, +) { + fun toParsedSchemaMetadata() = ParsedSchemaMetadata( + id = checkNotNull(id) { "Need id to parse schema metadata" }, + version = version, + schema = Schema.Parser().parse( + checkNotNull(schema) { "Need schema to parse it" } + ), + ) + } diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt new file mode 100644 index 00000000..e27fcf22 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt @@ -0,0 +1,105 @@ +package org.radarbase.producer.schema + +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.plugins.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.serialization.kotlinx.json.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import org.apache.avro.Schema +import org.radarbase.producer.rest.RestException +import java.io.IOException +import kotlin.coroutines.CoroutineContext + +/** REST client for Confluent schema registry. */ +class SchemaRestClient( + httpClient: HttpClient, + baseUrl: String, + private val ioContext: CoroutineContext = Dispatchers.IO, +) { + private val httpClient: HttpClient = httpClient.config { + install(ContentNegotiation) { + json(Json { + ignoreUnknownKeys = true + coerceInputValues = true + }) + } + defaultRequest { + url(baseUrl) + accept(ContentType.Application.Json) + } + } + + /** Retrieve schema metadata from server. */ + @Throws(IOException::class) + suspend fun retrieveSchemaMetadata( + subject: String, + version: Int + ): ParsedSchemaMetadata { + val isLatest = version <= 0 + val versionPath = if (isLatest) "latest" else version + return schemaGet("subjects/$subject/versions/$versionPath") + .toParsedSchemaMetadata() + } + + @Throws(IOException::class) + suspend fun schemaGet(path: String): SchemaMetadata = withContext(ioContext) { + val response = httpClient.get { + url(path) + } + if (response.status.isSuccess()) { + response.body() + } else { + throw RestException(response.status, response.bodyAsText()) + } + } + + @Throws(IOException::class) + suspend fun schemaPost( + path: String, + schema: Schema + ): SchemaMetadata = withContext(ioContext) { + val response = httpClient.post { + url(path) + contentType(ContentType.Application.Json) + setBody(SchemaMetadata(schema = schema.toString())) + } + if (response.status.isSuccess()) { + response.body() + } else { + throw RestException(response.status, response.bodyAsText()) + } + } + + /** Add a schema to a subject. */ + @Throws(IOException::class) + suspend fun addSchema(subject: String, schema: Schema): Int { + val result = schemaPost("subjects/$subject/versions", schema) + return checkNotNull(result.id) { "Missing schema ID in request result" } + } + + /** Request metadata for a schema on a subject. */ + @Throws(IOException::class) + suspend fun requestMetadata( + subject: String, + schema: Schema + ): ParsedSchemaMetadata = withContext(ioContext) { + val result = schemaPost("subjects/$subject", schema) + ParsedSchemaMetadata( + id = checkNotNull(result.id) { "Missing schema ID in request result" }, + version = result.version, + schema = schema + ) + } + + /** Retrieve schema metadata from server. */ + suspend fun retrieveSchemaById(id: Int): Schema = + schemaGet("/schemas/ids/$id") + .toParsedSchemaMetadata() + .schema +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt similarity index 60% rename from radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt rename to radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt index 49f93280..becd355f 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/SchemaRetriever.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt @@ -13,40 +13,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.radarbase.producer.rest +package org.radarbase.producer.schema +import io.ktor.client.* +import io.ktor.client.engine.cio.* +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.http.* +import kotlinx.coroutines.Dispatchers import org.apache.avro.Schema -import org.json.JSONException -import org.radarbase.util.CacheConfig +import org.radarbase.producer.rest.RestException +import org.radarbase.util.RadarProducerDsl import org.radarbase.util.TimedInt import org.radarbase.util.TimedValue import org.radarbase.util.TimedVariable.Companion.prune +import org.radarbase.util.TimeoutConfig import org.slf4j.LoggerFactory import java.io.IOException import java.time.Duration import java.util.* +import java.util.Objects.hash import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentMap +import kotlin.coroutines.CoroutineContext /** * Retriever of an Avro Schema. */ -open class SchemaRetriever @JvmOverloads constructor( - client: RestClient, - private val cacheConfig: CacheConfig = CacheConfig(MAX_VALIDITY), -) { +open class SchemaRetriever(config: Config) { private val idCache: ConcurrentMap> = ConcurrentHashMap() private val schemaCache: ConcurrentMap = ConcurrentHashMap() private val subjectVersionCache: ConcurrentMap> = ConcurrentHashMap() - private val restClient: SchemaRestClient = SchemaRestClient(client) + + private val baseUrl = config.baseUrl + private val ioContext = config.ioContext + private val httpClient = requireNotNull(config.httpClient) { "Missing HTTP client" } + + private val restClient: SchemaRestClient = SchemaRestClient(httpClient, baseUrl, ioContext) + private val schemaTimeout = config.schemaTimeout /** * Add schema metadata to the retriever. This implementation only adds it to the cache. * @return schema ID */ - @Throws(JSONException::class, IOException::class) - fun addSchema(topic: String, ofValue: Boolean, schema: Schema): Int { + @Throws(IOException::class) + suspend fun addSchema(topic: String, ofValue: Boolean, schema: Schema): Int { val subject = subject(topic, ofValue) val id = restClient.addSchema(subject, schema) cache(ParsedSchemaMetadata(id, null, schema), subject, false) @@ -58,18 +70,20 @@ open class SchemaRetriever @JvmOverloads constructor( * * @param version version to get or 0 if the latest version can be used. */ - @Throws(JSONException::class, IOException::class) - open fun getOrSetSchemaMetadata( - topic: String, ofValue: Boolean, schema: Schema, + @Throws(IOException::class) + open suspend fun getOrSet( + topic: String, + ofValue: Boolean, + schema: Schema, version: Int ): ParsedSchemaMetadata { return try { - getBySubjectAndVersion(topic, ofValue, version) + getByVersion(topic, ofValue, version) } catch (ex: RestException) { - if (ex.statusCode == 404) { + if (ex.status == HttpStatusCode.NotFound) { logger.warn("Schema for {} value was not yet added to the schema registry.", topic) addSchema(topic, ofValue, schema) - getMetadata(topic, ofValue, schema, version <= 0) + metadata(topic, ofValue, schema, version <= 0) } else { throw ex } @@ -78,28 +92,28 @@ open class SchemaRetriever @JvmOverloads constructor( /** Get a schema by its ID. */ @Throws(IOException::class) - fun getById(id: Int): Schema { + suspend fun getById(id: Int): Schema { var value = idCache[id] if (value == null || value.isExpired) { - value = TimedValue(restClient.retrieveSchemaById(id), cacheConfig) + value = TimedValue(restClient.retrieveSchemaById(id), schemaTimeout) idCache[id] = value - schemaCache[value.value] = TimedInt(id, cacheConfig) + schemaCache[value.value] = TimedInt(id, schemaTimeout) } return value.value } /** Gets a schema by ID and check that it is present in the given topic. */ @Throws(IOException::class) - fun getBySubjectAndId(topic: String, ofValue: Boolean, id: Int): ParsedSchemaMetadata { + suspend fun getById(topic: String, ofValue: Boolean, id: Int): ParsedSchemaMetadata { val schema = getById(id) val subject = subject(topic, ofValue) - val metadata = getCachedVersion(subject, id, null, schema) - return metadata ?: getMetadata(topic, ofValue, schema) + return cached(subject, id, null, schema) + ?: metadata(topic, ofValue, schema) } /** Get schema metadata. Cached schema metadata will be used if present. */ - @Throws(JSONException::class, IOException::class) - fun getBySubjectAndVersion( + @Throws(IOException::class) + suspend fun getByVersion( topic: String, ofValue: Boolean, version: Int @@ -116,27 +130,31 @@ open class SchemaRetriever @JvmOverloads constructor( metadata } else { val schema = getById(id.value) - val metadata = getCachedVersion(subject, id.value, version, schema) - metadata ?: getMetadata(topic, ofValue, schema, version <= 0) + val metadata = cached(subject, id.value, version, schema) + metadata ?: metadata(topic, ofValue, schema, version <= 0) } } /** Get all schema versions in a subject. */ @Throws(IOException::class) - fun getMetadata(topic: String, ofValue: Boolean, schema: Schema): ParsedSchemaMetadata { - return getMetadata(topic, ofValue, schema, false) - } + open suspend fun metadata( + topic: String, + ofValue: Boolean, + schema: Schema + ): ParsedSchemaMetadata = metadata(topic, ofValue, schema, false) /** Get the metadata of a specific schema in a topic. */ @Throws(IOException::class) - fun getMetadata( - topic: String, ofValue: Boolean, schema: Schema, + suspend fun metadata( + topic: String, + ofValue: Boolean, + schema: Schema, ofLatestVersion: Boolean ): ParsedSchemaMetadata { val id = schemaCache[schema] val subject = subject(topic, ofValue) if (id != null && !id.isExpired) { - val metadata = getCachedVersion(subject, id.value, null, schema) + val metadata = cached(subject, id.value, null, schema) if (metadata != null) { return metadata } @@ -156,13 +174,15 @@ open class SchemaRetriever @JvmOverloads constructor( * @return metadata if present. Returns null if no metadata is cached or if no version is cached * and the reportedVersion is null. */ - protected fun getCachedVersion( - subject: String, id: Int, - reportedVersion: Int?, schema: Schema? + protected fun cached( + subject: String, + id: Int, + reportedVersion: Int?, + schema: Schema? ): ParsedSchemaMetadata? { var version = reportedVersion if (version == null || version <= 0) { - version = findCachedVersion(id, subjectVersionCache[subject]) + version = subjectVersionCache[subject]?.find(id) if (version == null || version <= 0) { return null } @@ -170,14 +190,12 @@ open class SchemaRetriever @JvmOverloads constructor( return ParsedSchemaMetadata(id, version, schema!!) } - private fun findCachedVersion(id: Int, cache: ConcurrentMap?): Int? { - cache ?: return null - return cache.entries.find { (k, v) -> !v.isExpired && k != 0 && v.value == id } - ?.key - } + private fun ConcurrentMap.find(id: Int): Int? = entries + .find { (k, v) -> !v.isExpired && k != 0 && v.value == id } + ?.key protected fun cache(metadata: ParsedSchemaMetadata, subject: String, latest: Boolean) { - val id = TimedInt(metadata.id, cacheConfig) + val id = TimedInt(metadata.id, schemaTimeout) schemaCache[metadata.schema] = id if (metadata.version != null) { val versionCache = subjectVersionCache.computeIfAbsent( @@ -189,7 +207,7 @@ open class SchemaRetriever @JvmOverloads constructor( versionCache[0] = id } } - idCache[metadata.id] = TimedValue(metadata.schema, cacheConfig) + idCache[metadata.id] = TimedValue(metadata.schema, schemaTimeout) } /** @@ -212,9 +230,54 @@ open class SchemaRetriever @JvmOverloads constructor( schemaCache.clear() } + @RadarProducerDsl + class Config( + val baseUrl: String, + ) { + var httpClient: HttpClient? = null + var schemaTimeout: TimeoutConfig = DEFAULT_SCHEMA_TIMEOUT_CONFIG + var ioContext: CoroutineContext = Dispatchers.IO + fun httpClient(config: HttpClientConfig<*>.() -> Unit) { + httpClient = httpClient?.config(config) + ?: HttpClient(CIO) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Config + + return baseUrl == other.baseUrl && + httpClient == other.httpClient && + schemaTimeout == other.schemaTimeout && + ioContext == other.ioContext + } + + override fun hashCode(): Int = hash(baseUrl, httpClient, schemaTimeout, ioContext) + } + + fun config(config: Config.() -> Unit): SchemaRetriever { + val currentConfig = toConfig() + val newConfig = toConfig().apply(config) + return if (currentConfig != newConfig) { + SchemaRetriever(newConfig) + } else this + } + + private fun toConfig(): Config = Config(baseUrl = baseUrl).apply { + httpClient = this@SchemaRetriever.httpClient + schemaTimeout = this@SchemaRetriever.schemaTimeout + ioContext = this@SchemaRetriever.ioContext + } + companion object { private val logger = LoggerFactory.getLogger(SchemaRetriever::class.java) - private val MAX_VALIDITY = Duration.ofDays(1) + private val DEFAULT_SCHEMA_TIMEOUT_CONFIG = TimeoutConfig(Duration.ofDays(1)) + + fun schemaRetriever(baseUrl: String, config: Config.() -> Unit): SchemaRetriever { + return SchemaRetriever(Config(baseUrl).apply(config)) + } /** The subject in the Avro Schema Registry, given a Kafka topic. */ @JvmStatic diff --git a/radar-commons/src/main/java/org/radarbase/util/Annotations.kt b/radar-commons/src/main/java/org/radarbase/util/Annotations.kt new file mode 100644 index 00000000..65f13bd6 --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/Annotations.kt @@ -0,0 +1,8 @@ +package org.radarbase.util + +/** + * A marker annotations for DSLs. + */ +@DslMarker +@Target(AnnotationTarget.CLASS, AnnotationTarget.TYPEALIAS, AnnotationTarget.TYPE, AnnotationTarget.FUNCTION) +annotation class RadarProducerDsl diff --git a/radar-commons/src/main/java/org/radarbase/util/RestUtils.kt b/radar-commons/src/main/java/org/radarbase/util/RestUtils.kt deleted file mode 100644 index 816dbdf1..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/RestUtils.kt +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.util - -import okhttp3.internal.platform.Platform -import okhttp3.internal.tls.OkHostnameVerifier -import org.slf4j.LoggerFactory -import java.security.GeneralSecurityException -import java.security.KeyStore -import java.security.SecureRandom -import java.security.cert.X509Certificate -import javax.net.ssl.* - -/** Utility methods and variables for OkHttp initialization. */ -object RestUtils { - private val logger = LoggerFactory.getLogger(RestUtils::class.java) - - /** OkHttp3 default hostname verifier. */ - val DEFAULT_HOSTNAME_VERIFIER: HostnameVerifier = OkHostnameVerifier - - /** OkHttp3 hostname verifier for unsafe connections. */ - val UNSAFE_HOSTNAME_VERIFIER = HostnameVerifier { _, _ -> true } - - /** Unsafe OkHttp3 trust manager that trusts all certificates. */ - val UNSAFE_TRUST_MANAGER = arrayOf( - object : X509TrustManager { - override fun checkClientTrusted(chain: Array, authType: String) = Unit - - override fun checkServerTrusted(chain: Array, authType: String) = Unit - - override fun getAcceptedIssuers(): Array = arrayOf() - } - ) - - /** Unsafe OkHttp3 SSLSocketFactory that trusts all certificates. */ - val UNSAFE_SSL_FACTORY: SSLSocketFactory? - - init { - val factory: SSLSocketFactory? = try { - SSLContext.getInstance("SSL").apply { - init(null, UNSAFE_TRUST_MANAGER, SecureRandom()) - }.socketFactory - } catch (e: GeneralSecurityException) { - logger.error("Failed to initialize unsafe SSL factory", e) - null - } - UNSAFE_SSL_FACTORY = factory - } - - /** - * Default OkHttp3 trust manager that trusts all certificates. - * Copied from private method in OkHttpClient. - */ - fun systemDefaultTrustManager(): X509TrustManager { - return try { - val trustManagerFactory = TrustManagerFactory.getInstance( - TrustManagerFactory.getDefaultAlgorithm() - ) - trustManagerFactory.init(null as KeyStore?) - val trustManagers = trustManagerFactory.trustManagers - check(trustManagers.size == 1 && trustManagers[0] is X509TrustManager) { - "Unexpected default trust managers:" + trustManagers.contentToString() - } - trustManagers[0] as X509TrustManager - } catch (e: GeneralSecurityException) { - throw IllegalStateException("No System TLS", e) - } - } - - /** - * Default OkHttp3 SSLSocketFactory that trusts all certificates. - * Copied from private method in OkHttpClient. - */ - fun systemDefaultSslSocketFactory(trustManager: X509TrustManager): SSLSocketFactory { - return try { - val sslContext: SSLContext = Platform.get().newSSLContext() - sslContext.init(null, arrayOf(trustManager), null) - sslContext.socketFactory - } catch (e: GeneralSecurityException) { - throw IllegalStateException("No System TLS", e) - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt b/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt index a39989c2..446ec984 100644 --- a/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt +++ b/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt @@ -2,7 +2,7 @@ package org.radarbase.util class TimedInt( val value: Int, - cacheConfig: CacheConfig, + cacheConfig: TimeoutConfig, ) : TimedVariable(cacheConfig) { override fun equals(other: Any?): Boolean { if (this === other) { diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt b/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt index 91806990..b9555c04 100644 --- a/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt +++ b/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt @@ -2,7 +2,7 @@ package org.radarbase.util class TimedValue( val value: T, - private val cacheConfig: CacheConfig, + cacheConfig: TimeoutConfig, ) : TimedVariable(cacheConfig) { override fun equals(other: Any?): Boolean { if (this === other) { diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt b/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt index 99f3fc9b..91f0b44d 100644 --- a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt +++ b/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt @@ -1,7 +1,7 @@ package org.radarbase.util open class TimedVariable( - private val cacheConfig: CacheConfig, + private val cacheConfig: TimeoutConfig, ) { protected val expiry: Long = cacheConfig.currentExpiryTime diff --git a/radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt b/radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt similarity index 92% rename from radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt rename to radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt index 0610e99b..399ea208 100644 --- a/radar-commons/src/main/java/org/radarbase/util/CacheConfig.kt +++ b/radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt @@ -2,7 +2,7 @@ package org.radarbase.util import java.time.Duration -data class CacheConfig( +data class TimeoutConfig( val validity: Duration, val timeSource: () -> Long = System::currentTimeMillis, ) { diff --git a/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java b/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java index 68cd2c87..11beed4b 100644 --- a/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java +++ b/radar-commons/src/test/java/org/radarbase/data/AvroDatumDecoderTest.java @@ -16,11 +16,11 @@ package org.radarbase.data; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import org.apache.avro.specific.SpecificData; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.radarbase.topic.AvroTopic; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt b/radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt similarity index 84% rename from radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt rename to radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt index 0717c5a9..5184cc64 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/AvroDataMapperFactoryTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt @@ -1,4 +1,4 @@ -package org.radarbase.producer.rest +package org.radarbase.producer.avro import org.apache.avro.Schema import org.apache.avro.SchemaValidationException @@ -6,20 +6,14 @@ import org.apache.avro.generic.GenericDatumReader import org.apache.avro.generic.GenericDatumWriter import org.apache.avro.io.DecoderFactory import org.apache.avro.io.EncoderFactory -import org.junit.Assert -import org.junit.Before -import org.junit.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import org.radarcns.kafka.ObservationKey import java.io.ByteArrayOutputStream import java.io.IOException class AvroDataMapperFactoryTest { - private lateinit var factory: AvroDataMapperFactory - @Before - fun setUp() { - factory = AvroDataMapperFactory() - } - @Test @Throws(SchemaValidationException::class, IOException::class) fun mapRecord() { @@ -27,44 +21,48 @@ class AvroDataMapperFactoryTest { MEASUREMENT_KEY_SCHEMA, ObservationKey.getClassSchema(), "{\"userId\":\"u\", \"sourceId\": \"s\"}" ) - Assert.assertEquals("{\"projectId\":null,\"userId\":\"u\",\"sourceId\":\"s\"}", actual) + assertEquals("{\"projectId\":null,\"userId\":\"u\",\"sourceId\":\"s\"}", actual) } - @Test(expected = SchemaValidationException::class) + @Test @Throws(SchemaValidationException::class) fun mapRecordIncomplete() { - factory.createMapper( - INCOMPLETE_MEASUREMENT_KEY_SCHEMA, - ObservationKey.getClassSchema(), - null - ) + assertThrows { + AvroDataMapperFactory.createMapper( + INCOMPLETE_MEASUREMENT_KEY_SCHEMA, + ObservationKey.getClassSchema(), + null + ) + } } @Test @Throws(SchemaValidationException::class, IOException::class) fun mapEnumLarger() { val actual = doMap(SMALL_ENUM_SCHEMA, LARGE_ENUM_SCHEMA, "{\"e\":\"A\"}") - Assert.assertEquals("{\"e\":\"A\"}", actual) + assertEquals("{\"e\":\"A\"}", actual) } - @Test(expected = SchemaValidationException::class) + @Test @Throws(SchemaValidationException::class) fun mapEnumSmaller() { - factory.createMapper(LARGE_ENUM_SCHEMA, SMALL_ENUM_SCHEMA, null) + assertThrows { + AvroDataMapperFactory.createMapper(LARGE_ENUM_SCHEMA, SMALL_ENUM_SCHEMA, null) + } } @Test @Throws(SchemaValidationException::class, IOException::class) fun mapEnumSmallerUnknown() { val actual = doMap(LARGE_ENUM_SCHEMA, UNKNOWN_ENUM_SCHEMA, "{\"e\":\"C\"}") - Assert.assertEquals("{\"e\":\"UNKNOWN\"}", actual) + assertEquals("{\"e\":\"UNKNOWN\"}", actual) } @Test @Throws(SchemaValidationException::class, IOException::class) fun mapEnumSmallerDefault() { val actual = doMap(LARGE_ENUM_SCHEMA, DEFAULT_ENUM_SCHEMA, "{\"e\":\"C\"}") - Assert.assertEquals("{\"e\":\"A\"}", actual) + assertEquals("{\"e\":\"A\"}", actual) } @Test @@ -94,35 +92,35 @@ class AvroDataMapperFactoryTest { "\"bfd\":\"abc\"," + "\"unmapped\":10}" ) - Assert.assertEquals( + assertEquals( "{" + - "\"e\":\"A\"," + - "\"i\":1," + - "\"l\":2.0," + - "\"d\":3.0," + - "\"f\":4.0," + - "\"sI\":5," + - "\"sD\":6.5," + - "\"sU\":\"\"," + - "\"sUi\":{\"int\":7}," + - "\"sUe\":\"A\"," + - "\"uS\":{\"string\":\"s\"}," + - "\"se2\":\"B\"," + - "\"se3\":\"A\"," + - "\"a\":[1.0,2.0]," + - "\"m\":{\"a\":9.0}," + - "\"fS\":\"YWI=\"," + - "\"bS\":\"YWI=\"," + - "\"fb\":\"ab\"," + - "\"bf\":\"ab\"," + - "\"bfd\":\"aa\"" + - "}", actual + "\"e\":\"A\"," + + "\"i\":1," + + "\"l\":2.0," + + "\"d\":3.0," + + "\"f\":4.0," + + "\"sI\":5," + + "\"sD\":6.5," + + "\"sU\":\"\"," + + "\"sUi\":{\"int\":7}," + + "\"sUe\":\"A\"," + + "\"uS\":{\"string\":\"s\"}," + + "\"se2\":\"B\"," + + "\"se3\":\"A\"," + + "\"a\":[1.0,2.0]," + + "\"m\":{\"a\":9.0}," + + "\"fS\":\"YWI=\"," + + "\"bS\":\"YWI=\"," + + "\"fb\":\"ab\"," + + "\"bf\":\"ab\"," + + "\"bfd\":\"aa\"" + + "}", actual ) } @Throws(IOException::class, SchemaValidationException::class) private fun doMap(from: Schema, to: Schema, value: String): String { - val mapper = factory.createMapper(from, to, null) + val mapper = AvroDataMapperFactory.createMapper(from, to, null) val reader = GenericDatumReader(from) val decoder = DecoderFactory.get().jsonDecoder(from, value) val readValue = reader.read(null, decoder) diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt similarity index 84% rename from radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt rename to radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt index 279b8c7c..832410a9 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordRequestTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt @@ -1,13 +1,19 @@ package org.radarbase.producer.rest -import okio.Buffer +import io.ktor.http.content.* +import io.ktor.util.* +import io.ktor.utils.io.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runTest import org.apache.avro.SchemaValidationException import org.apache.avro.io.BinaryEncoder import org.apache.avro.io.EncoderFactory import org.apache.avro.specific.SpecificDatumWriter -import org.junit.Assert -import org.junit.Test +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Test import org.radarbase.data.AvroRecordData +import org.radarbase.producer.schema.ParsedSchemaMetadata import org.radarbase.topic.AvroTopic import org.radarcns.kafka.ObservationKey import org.radarcns.kafka.RecordSet @@ -20,10 +26,11 @@ import java.io.InputStreamReader import java.nio.ByteBuffer import java.util.zip.GZIPOutputStream -class BinaryRecordRequestTest { +@OptIn(ExperimentalCoroutinesApi::class) +class BinaryRecordContentTest { @Test @Throws(SchemaValidationException::class, IOException::class) - fun writeToStream() { + fun writeToStream() = runTest { val k = ObservationKey("test", "a", "b") val v = EmpaticaE4BloodVolumePulse( 0.0, 0.0, @@ -32,15 +39,19 @@ class BinaryRecordRequestTest { val t = AvroTopic( "t", k.schema, v.schema, k.javaClass, v.javaClass ) - val request = BinaryRecordRequest(t) - request.prepare( + val request = BinaryRecordContent( + AvroRecordData(t, k, listOf(v)), ParsedSchemaMetadata(2, 1, k.schema), - ParsedSchemaMetadata(4, 2, v.schema), - AvroRecordData(t, k, listOf(v)) + ParsedSchemaMetadata(4, 2, v.schema) ) - val buffer = Buffer() - request.writeToSink(buffer) - Assert.assertArrayEquals(EXPECTED, buffer.readByteArray()) + + val channel = ByteChannel() + launch { + val content = request.createContent() as OutgoingContent.WriteChannelContent + content.writeTo(channel) + channel.close() + } + assertArrayEquals(EXPECTED, channel.toByteArray()) } @Test @@ -86,7 +97,7 @@ class BinaryRecordRequestTest { val encoder = EncoderFactory.get().binaryEncoder(out, null) writer.write(recordSet, encoder) encoder.flush() - Assert.assertArrayEquals(EXPECTED, out.toByteArray()) + assertArrayEquals(EXPECTED, out.toByteArray()) } @Test @@ -94,7 +105,7 @@ class BinaryRecordRequestTest { fun testSize() { val writer = SpecificDatumWriter(PhoneAcceleration.`SCHEMA$`) val records: MutableList = ArrayList(540) - requireNotNull(BinaryRecordRequestTest::class.java.getResourceAsStream("android_phone_acceleration.csv")) + requireNotNull(BinaryRecordContentTest::class.java.getResourceAsStream("android_phone_acceleration.csv")) .use { stream -> InputStreamReader(stream).use { reader -> BufferedReader(reader).use { br -> diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt index df66fb9b..a8dada55 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt @@ -15,31 +15,51 @@ */ package org.radarbase.producer.rest -import org.junit.Assert -import org.junit.Test +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.single +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Timeout +import org.radarbase.util.TimeoutConfig +import org.slf4j.LoggerFactory +import java.time.Duration import java.util.concurrent.TimeUnit class ConnectionStateTest { @Test - fun testState() { - val state = ConnectionState(10, TimeUnit.MILLISECONDS) - Assert.assertEquals(ConnectionState.State.UNKNOWN, state.state) + @Timeout(1, unit = TimeUnit.SECONDS) + fun testState() = runBlocking { + var state = ConnectionState(TimeoutConfig(Duration.ofMillis(10))) + logger.info("initial state set") + state.assertEqualTo(ConnectionState.State.UNKNOWN) + logger.info("setting to didConnect") state.didConnect() - Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) + state.assertEqualTo(ConnectionState.State.CONNECTED) state.didDisconnect() - Assert.assertEquals(ConnectionState.State.DISCONNECTED, state.state) - Thread.sleep(15) - Assert.assertEquals(ConnectionState.State.DISCONNECTED, state.state) + state.assertEqualTo(ConnectionState.State.DISCONNECTED) + delay(15) + state.assertEqualTo(ConnectionState.State.DISCONNECTED) state.didConnect() - Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) - Thread.sleep(10) - Assert.assertEquals(ConnectionState.State.UNKNOWN, state.state) - state.setTimeout(25, TimeUnit.MILLISECONDS) + delay(15) + state.assertEqualTo(ConnectionState.State.UNKNOWN) + state = ConnectionState(TimeoutConfig(Duration.ofMillis(25))) state.didConnect() - Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) - Thread.sleep(10) - Assert.assertEquals(ConnectionState.State.CONNECTED, state.state) - Thread.sleep(15) - Assert.assertEquals(ConnectionState.State.UNKNOWN, state.state) + state.assertEqualTo(ConnectionState.State.CONNECTED) + delay(10) + state.assertEqualTo(ConnectionState.State.CONNECTED) + delay(15) + state.assertEqualTo(ConnectionState.State.UNKNOWN) + } + + private suspend fun ConnectionState.assertEqualTo(expected: ConnectionState.State) { + assertEquals(expected, state.first()) + } + + companion object { + private val logger = LoggerFactory.getLogger(ConnectionStateTest::class.java) } } diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt deleted file mode 100644 index c91e7e9c..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/RestClientTest.kt +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import okhttp3.Request -import okhttp3.mockwebserver.MockResponse -import okhttp3.mockwebserver.MockWebServer -import org.junit.Assert -import org.junit.Before -import org.junit.Test -import org.radarbase.config.ServerConfig -import java.net.URL -import java.util.concurrent.TimeUnit - -class RestClientTest { - private lateinit var server: MockWebServer - private lateinit var config: ServerConfig - private lateinit var client: RestClient - @Before - fun setUp() { - server = MockWebServer() - config = ServerConfig(server.url("base").toUrl()) - client = RestClient.newRestClient { - server = config - timeout(1, TimeUnit.SECONDS) - } - } - - @Test - @Throws(Exception::class) - fun request() { - server.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) - val request = client.buildRequest("myPath") - client.request(request).use { response -> - Assert.assertTrue(response.isSuccessful) - Assert.assertEquals( - "{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", - response.body!!.string() - ) - } - val recordedRequest = server.takeRequest() - Assert.assertEquals("GET", recordedRequest.method) - Assert.assertEquals("/base/myPath", recordedRequest.path) - } - - @Test - @Throws(Exception::class) - fun requestStringPath() { - server.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) - client.request("myPath").use { response -> - Assert.assertTrue(response.isSuccessful) - Assert.assertEquals( - "{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", - response.body!!.string() - ) - } - val recordedRequest = server.takeRequest() - Assert.assertEquals("GET", recordedRequest.method) - Assert.assertEquals("/base/myPath", recordedRequest.path) - } - - @Test - @Throws(Exception::class) - fun requestString() { - server.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) - val response = client.requestString(client.buildRequest("myPath")) - Assert.assertEquals("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}", response) - val recordedRequest = server.takeRequest() - Assert.assertEquals("GET", recordedRequest.method) - Assert.assertEquals("/base/myPath", recordedRequest.path) - } - - @Test(expected = RestException::class) - @Throws(Exception::class) - fun requestStringEmpty() { - server.enqueue(MockResponse().setResponseCode(500)) - client.requestString(client.buildRequest("myPath")) - } - - @Test - @Throws(Exception::class) - fun requestBuilder() { - val request: Request = client.buildRequest("myPath") - Assert.assertEquals(request.url, URL(config.url, "myPath").toHttpUrlOrNull()) - } - - @Throws(Exception::class) - @Test - fun testRelativeUrl() { - val url = client.relativeUrl("myPath") - Assert.assertEquals(server.hostName, url.host) - Assert.assertEquals(server.port.toLong(), url.port.toLong()) - Assert.assertEquals("http", url.scheme) - Assert.assertEquals("/base/myPath", url.encodedPath) - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt new file mode 100644 index 00000000..cbd4e0ee --- /dev/null +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt @@ -0,0 +1,327 @@ +/* + * Copyright 2017 The Hyve and King's College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.radarbase.producer.rest + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeType +import io.ktor.util.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.apache.avro.SchemaValidationException +import org.apache.avro.io.DecoderFactory +import org.apache.avro.specific.SpecificDatumReader +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.* +import org.radarbase.data.AvroRecordData +import org.radarbase.producer.AuthenticationException +import org.radarbase.producer.rest.RestKafkaSender.Companion.restKafkaSender +import org.radarbase.producer.schema.ParsedSchemaMetadata +import org.radarbase.producer.schema.SchemaRetriever +import org.radarbase.topic.AvroTopic +import org.radarcns.kafka.ObservationKey +import org.radarcns.kafka.RecordSet +import org.radarcns.passive.phone.PhoneLight +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.charset.StandardCharsets +import java.util.* +import java.util.zip.GZIPInputStream + +@OptIn(ExperimentalCoroutinesApi::class) +class RestKafkaSenderTest { + private lateinit var retriever: SchemaRetriever + private lateinit var sender: RestKafkaSender + private lateinit var webServer: MockWebServer + + @BeforeEach + fun setUp() { + webServer = MockWebServer().apply { + start() + } + retriever = mock() + sender = restKafkaSender { + baseUrl = webServer.url("/").toUrl().toExternalForm() + httpClient() + schemaRetriever = retriever + } + } + + @AfterEach + fun tearDown() { + webServer.close() + } + + @Test + @Throws(Exception::class) + fun sender() = runTest { + sender = sender.config { + ioContext = coroutineContext + with(headers) { + append("Cookie", "ab") + append("Cookie", "bc") + } + } + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + retriever.stub { + onBlocking { metadata("test", false, keySchema) }.doReturn(keySchemaMetadata) + onBlocking { metadata("test", true, valueSchema) }.doReturn(valueSchemaMetadata) + } + webServer.enqueueJson("{\"offset\": 100}") + topicSender.send(key, value) + verify(retriever, times(1)) + .metadata("test", false, keySchema) + verify(retriever, times(1)) + .metadata("test", true, valueSchema) + val request = webServer.takeRequest() + assertEquals("/topics/test", request.path) + val body = READER.readTree(request.body.inputStream()) + assertEquals(10, body["key_schema_id"].asInt().toLong()) + assertEquals(10, body["value_schema_id"].asInt().toLong()) + val records = body["records"] + assertEquals(JsonNodeType.ARRAY, records.nodeType) + assertEquals(1, records.size().toLong()) + checkChildren(records) + val receivedHeaders = request.headers + assertEquals(listOf("ab,bc"), receivedHeaders.values("Cookie")) + } + + @Test + @Throws(Exception::class) + fun sendBinary() = runTest { + sender = sender.config { + ioContext = coroutineContext + contentType = RestKafkaSender.KAFKA_REST_BINARY_ENCODING + } + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + retriever.stub { + onBlocking { metadata("test", false, keySchema) }.doReturn(keySchemaMetadata) + onBlocking { metadata("test", true, valueSchema) }.doReturn(valueSchemaMetadata) + } + webServer.enqueueJson("{\"offset\": 100}") + topicSender.send(key, value) + verify(retriever, times(1)) + .metadata("test", false, keySchema) + verify(retriever, times(1)) + .metadata("test", true, valueSchema) + val request = webServer.takeRequest() + assertEquals("/topics/test", request.path) + var decoder = DecoderFactory.get().directBinaryDecoder(request.body.inputStream(), null) + val recordSetDatumReader = SpecificDatumReader(RecordSet.getClassSchema()) + val recordSet = recordSetDatumReader.read(null, decoder) + assertNull(recordSet.userId) + assertEquals("b", recordSet.sourceId) + assertEquals(2, recordSet.keySchemaVersion) + assertEquals(2, recordSet.valueSchemaVersion) + assertEquals(1, recordSet.data.size) + decoder = DecoderFactory.get().directBinaryDecoder(recordSet.data[0].moveToByteArray().inputStream(), decoder) + val phoneLightDatumReader = SpecificDatumReader(PhoneLight.getClassSchema()) + val decodedValue = phoneLightDatumReader.read(null, decoder) + assertEquals(value, decodedValue) + } + + @Test + @Throws(Exception::class) + fun sendTwo() = runTest { + sender = sender.config { + ioContext = coroutineContext + } + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + + retriever.stub { + onBlocking { metadata("test", false, keySchema) }.doReturn(keySchemaMetadata) + onBlocking { metadata("test", true, valueSchema) }.doReturn(valueSchemaMetadata) + } + webServer.enqueueJson("{\"offset\": 100}") + topicSender.send(AvroRecordData(topic, key, listOf(value, value))) + verify(retriever, times(1)) + .metadata("test", false, keySchema) + verify(retriever, times(1)) + .metadata("test", true, valueSchema) + val request = webServer.takeRequest() + assertEquals("/topics/test", request.path) + val bodyString = request.body.readString(StandardCharsets.UTF_8) + logger.info("Reading: {}", bodyString) + val body = READER.readTree(bodyString) + assertEquals(10, body["key_schema_id"].asInt().toLong()) + assertEquals(10, body["value_schema_id"].asInt().toLong()) + val records = body["records"] + assertEquals(JsonNodeType.ARRAY, records.nodeType) + assertEquals(2, records.size().toLong()) + checkChildren(records) + } + + @Test + @Throws(Exception::class) + fun resetConnection() = runTest { + sender = sender.config { + ioContext = coroutineContext + } + var nRequests = 0 + webServer.enqueue(MockResponse().setResponseCode(500)) + assertFalse(sender.resetConnection()) + assertEquals(++nRequests, webServer.requestCount) + var request = webServer.takeRequest() + assertEquals("/", request.path) + assertEquals("HEAD", request.method) + webServer.enqueue(MockResponse()) + assertEquals(nRequests, webServer.requestCount) + assertTrue(sender.resetConnection()) + assertEquals(++nRequests, webServer.requestCount) + request = webServer.takeRequest() + assertEquals("/", request.path) + assertEquals("HEAD", request.method) + } + + @Test + @Throws(Exception::class) + fun resetConnectionUnauthorized() = runTest { + sender = sender.config { + ioContext = coroutineContext + } + webServer.enqueue(MockResponse().setResponseCode(401)) + webServer.enqueue(MockResponse().setResponseCode(401)) + try { + sender.resetConnection() + fail("Authentication exception expected") + } catch (ex: AuthenticationException) { + assertEquals(1, webServer.requestCount.toLong()) + // success + } + try { + sender.resetConnection() + fail("Authentication exception expected") + } catch (ex: AuthenticationException) { + assertEquals(2, webServer.requestCount.toLong()) + // success + } + webServer.enqueue(MockResponse().setResponseCode(200)) + try { + assertTrue(sender.resetConnection()) + assertEquals(3, webServer.requestCount.toLong()) + } catch (ex: AuthenticationException) { + fail("Unexpected authentication failure") + } + } + + @Test + @Throws( + IOException::class, + InterruptedException::class, + SchemaValidationException::class + ) + fun withCompression() = runTest { + sender = sender.config { + contentEncoding = RestKafkaSender.GZIP_CONTENT_ENCODING + } + webServer.enqueueJson("{\"offset\": 100}") + val keySchema = ObservationKey.getClassSchema() + val valueSchema = PhoneLight.getClassSchema() + val topic = AvroTopic( + "test", + keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + ) + val topicSender = sender.sender(topic) + val key = ObservationKey("test", "a", "b") + val value = PhoneLight(0.1, 0.2, 0.3f) + val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) + val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) + retriever.stub { + onBlocking { metadata("test", false, keySchema) }.doReturn(keySchemaMetadata) + onBlocking { metadata("test", true, valueSchema) }.doReturn(valueSchemaMetadata) + } + topicSender.send(key, value) + + val request = webServer.takeRequest() + assertEquals("gzip", request.getHeader("Content-Encoding")) + request.body.inputStream().use { `in` -> + GZIPInputStream(`in`).use { gzipIn -> + val body = READER.readTree(gzipIn) + assertEquals(10, body["key_schema_id"].asInt().toLong()) + assertEquals(10, body["value_schema_id"].asInt().toLong()) + val records = body["records"] + assertEquals(JsonNodeType.ARRAY, records.nodeType) + assertEquals(1, records.size().toLong()) + checkChildren(records) + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(RestKafkaSenderTest::class.java) + private val FACTORY = JsonFactory() + private val READER = ObjectMapper(FACTORY).reader() + private fun checkChildren(records: JsonNode) { + for (child in records) { + val jsonKey = child["key"] + assertEquals(JsonNodeType.OBJECT, jsonKey.nodeType) + assertEquals("a", jsonKey["userId"].asText()) + assertEquals("b", jsonKey["sourceId"].asText()) + val jsonValue = child["value"] + assertEquals(JsonNodeType.OBJECT, jsonValue.nodeType) + assertEquals(0.1, jsonValue["time"].asDouble(), 0.0) + assertEquals(0.2, jsonValue["timeReceived"].asDouble(), 0.0) + assertEquals(0.3f, jsonValue["light"].asDouble().toFloat(), 0f) + } + } + + fun MockWebServer.enqueueJson( + body: String, + builder: MockResponse.() -> Unit = {} + ) = enqueue( + MockResponse() + .setBody(body) + .setHeader("Content-Type", "application/json; charset=utf-8") + .apply(builder) + ) + } +} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt deleted file mode 100644 index 797175db..00000000 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/RestSenderTest.kt +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright 2017 The Hyve and King's College London - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.radarbase.producer.rest - -import com.fasterxml.jackson.core.JsonFactory -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.JsonNodeType -import okhttp3.Headers.Companion.headersOf -import okhttp3.mockwebserver.MockResponse -import okhttp3.mockwebserver.MockWebServer -import org.apache.avro.SchemaValidationException -import org.json.JSONException -import org.junit.Assert -import org.junit.Before -import org.junit.Rule -import org.junit.Test -import org.mockito.Mockito -import org.radarbase.config.ServerConfig -import org.radarbase.data.AvroRecordData -import org.radarbase.producer.AuthenticationException -import org.radarbase.producer.rest.RestSender.Companion.restSender -import org.radarbase.topic.AvroTopic -import org.radarcns.kafka.ObservationKey -import org.radarcns.passive.phone.PhoneLight -import java.io.IOException -import java.util.* -import java.util.zip.GZIPInputStream - -class RestSenderTest { - private lateinit var retriever: SchemaRetriever - private lateinit var sender: RestSender - - @Rule - @JvmField - var webServer = MockWebServer() - - @Before - fun setUp() { - retriever = Mockito.mock(SchemaRetriever::class.java) - sender = restSender { - httpClient = RestClient.newRestClient { - server = ServerConfig(webServer.url("/").toUrl()) - } - schemaRetriever = retriever - } - } - - @Test - @Throws(Exception::class) - fun sender() { - val keySchema = ObservationKey.getClassSchema() - val valueSchema = PhoneLight.getClassSchema() - val topic = AvroTopic( - "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java - ) - sender.headers = headersOf("Cookie", "ab", "Cookie", "bc") - val topicSender = sender.sender(topic) - val key = ObservationKey("test", "a", "b") - val value = PhoneLight(0.1, 0.2, 0.3f) - val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) - val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) - Mockito.`when`( - retriever - .getOrSetSchemaMetadata("test", false, keySchema, -1) - ) - .thenReturn(keySchemaMetadata) - Mockito.`when`( - retriever - .getOrSetSchemaMetadata("test", true, valueSchema, -1) - ) - .thenReturn(valueSchemaMetadata) - webServer.enqueue( - MockResponse() - .setHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"offset\": 100}") - ) - topicSender.send(key, value) - Mockito.verify(retriever, Mockito.times(1)) - .getOrSetSchemaMetadata("test", false, keySchema, -1) - Mockito.verify(retriever, Mockito.times(1)) - .getOrSetSchemaMetadata("test", true, valueSchema, -1) - val request = webServer.takeRequest() - Assert.assertEquals("/topics/test", request.path) - val body = READER.readTree(request.body.inputStream()) - Assert.assertEquals(10, body["key_schema_id"].asInt().toLong()) - Assert.assertEquals(10, body["value_schema_id"].asInt().toLong()) - val records = body["records"] - Assert.assertEquals(JsonNodeType.ARRAY, records.nodeType) - Assert.assertEquals(1, records.size().toLong()) - checkChildren(records) - val receivedHeaders = request.headers - Assert.assertEquals(listOf("ab", "bc"), receivedHeaders.values("Cookie")) - } - - @Test - @Throws(Exception::class) - fun sendTwo() { - val keySchema = ObservationKey.getClassSchema() - val valueSchema = PhoneLight.getClassSchema() - val topic = AvroTopic( - "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java - ) - val topicSender = sender.sender(topic) - val key = ObservationKey("test", "a", "b") - val value = PhoneLight(0.1, 0.2, 0.3f) - val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) - val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) - Mockito.`when`( - retriever - .getOrSetSchemaMetadata("test", false, keySchema, -1) - ) - .thenReturn(keySchemaMetadata) - Mockito.`when`( - retriever - .getOrSetSchemaMetadata("test", true, valueSchema, -1) - ) - .thenReturn(valueSchemaMetadata) - webServer.enqueue( - MockResponse() - .setHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"offset\": 100}") - ) - topicSender.send(AvroRecordData(topic, key, Arrays.asList(value, value))) - Mockito.verify(retriever, Mockito.times(1)) - .getOrSetSchemaMetadata("test", false, keySchema, -1) - Mockito.verify(retriever, Mockito.times(1)) - .getOrSetSchemaMetadata("test", true, valueSchema, -1) - val request = webServer.takeRequest() - Assert.assertEquals("/topics/test", request.path) - val body = READER.readTree(request.body.inputStream()) - Assert.assertEquals(10, body["key_schema_id"].asInt().toLong()) - Assert.assertEquals(10, body["value_schema_id"].asInt().toLong()) - val records = body["records"] - Assert.assertEquals(JsonNodeType.ARRAY, records.nodeType) - Assert.assertEquals(2, records.size().toLong()) - checkChildren(records) - } - - @Test - @Throws(Exception::class) - fun resetConnection() { - var nRequests = 0 - webServer.enqueue(MockResponse().setResponseCode(500)) - Assert.assertFalse(sender.isConnected) - Assert.assertEquals(++nRequests, webServer.requestCount) - var request = webServer.takeRequest() - Assert.assertEquals("/", request.path) - Assert.assertEquals("HEAD", request.method) - webServer.enqueue(MockResponse().setResponseCode(500)) - Assert.assertFalse(sender.resetConnection()) - Assert.assertEquals(++nRequests, webServer.requestCount) - request = webServer.takeRequest() - Assert.assertEquals("/", request.path) - Assert.assertEquals("HEAD", request.method) - webServer.enqueue(MockResponse()) - Assert.assertFalse(sender.isConnected) - Assert.assertEquals(nRequests, webServer.requestCount) - Assert.assertTrue(sender.resetConnection()) - Assert.assertEquals(++nRequests, webServer.requestCount) - request = webServer.takeRequest() - Assert.assertEquals("/", request.path) - Assert.assertEquals("HEAD", request.method) - } - - @Test - @Throws(Exception::class) - fun resetConnectionUnauthorized() { - webServer.enqueue(MockResponse().setResponseCode(401)) - try { - sender.isConnected - Assert.fail("Authentication exception expected") - } catch (ex: AuthenticationException) { - // success - } - try { - sender.isConnected - Assert.fail("Authentication exception expected") - } catch (ex: AuthenticationException) { - // success - } - webServer.enqueue(MockResponse().setResponseCode(401)) - try { - sender.resetConnection() - Assert.fail("Authentication exception expected") - } catch (ex: AuthenticationException) { - Assert.assertEquals(2, webServer.requestCount.toLong()) - // success - } - webServer.enqueue(MockResponse().setResponseCode(200)) - try { - Assert.assertTrue(sender.resetConnection()) - } catch (ex: AuthenticationException) { - Assert.assertEquals(3, webServer.requestCount.toLong()) - Assert.fail("Unexpected authentication failure") - } - } - - @Test - @Throws( - IOException::class, - InterruptedException::class, - SchemaValidationException::class, - JSONException::class - ) - fun withCompression() { - sender.setCompression(true) - webServer.enqueue( - MockResponse() - .setHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"offset\": 100}") - ) - val keySchema = ObservationKey.getClassSchema() - val valueSchema = PhoneLight.getClassSchema() - val topic = AvroTopic( - "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java - ) - val topicSender = sender.sender(topic) - val key = ObservationKey("test", "a", "b") - val value = PhoneLight(0.1, 0.2, 0.3f) - val keySchemaMetadata = ParsedSchemaMetadata(10, 2, keySchema) - val valueSchemaMetadata = ParsedSchemaMetadata(10, 2, valueSchema) - Mockito.`when`( - retriever - .getOrSetSchemaMetadata("test", false, keySchema, -1) - ) - .thenReturn(keySchemaMetadata) - Mockito.`when`( - retriever - .getOrSetSchemaMetadata("test", true, valueSchema, -1) - ) - .thenReturn(valueSchemaMetadata) - topicSender.send(key, value) - val request = webServer.takeRequest() - Assert.assertEquals("gzip", request.getHeader("Content-Encoding")) - request.body.inputStream().use { `in` -> - GZIPInputStream(`in`).use { gzipIn -> - val body = READER.readTree(gzipIn) - Assert.assertEquals(10, body["key_schema_id"].asInt().toLong()) - Assert.assertEquals(10, body["value_schema_id"].asInt().toLong()) - val records = body["records"] - Assert.assertEquals(JsonNodeType.ARRAY, records.nodeType) - Assert.assertEquals(1, records.size().toLong()) - checkChildren(records) - } - } - } - - companion object { - private val FACTORY = JsonFactory() - private val READER = ObjectMapper(FACTORY).reader() - private fun checkChildren(records: JsonNode) { - for (child in records) { - val jsonKey = child["key"] - Assert.assertEquals(JsonNodeType.OBJECT, jsonKey.nodeType) - Assert.assertEquals("a", jsonKey["userId"].asText()) - Assert.assertEquals("b", jsonKey["sourceId"].asText()) - val jsonValue = child["value"] - Assert.assertEquals(JsonNodeType.OBJECT, jsonValue.nodeType) - Assert.assertEquals(0.1, jsonValue["time"].asDouble(), 0.0) - Assert.assertEquals(0.2, jsonValue["timeReceived"].asDouble(), 0.0) - Assert.assertEquals(0.3f, jsonValue["light"].asDouble().toFloat(), 0f) - } - } - } -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt similarity index 62% rename from radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt rename to radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt index 06f3dffb..3d12b944 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRestClientTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt @@ -13,35 +13,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.radarbase.producer.rest +package org.radarbase.producer.schema -import okhttp3.mockwebserver.MockResponse +import io.ktor.client.* +import io.ktor.client.engine.cio.* +import io.ktor.client.plugins.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest import okhttp3.mockwebserver.MockWebServer import org.apache.avro.Schema -import org.junit.After -import org.junit.Assert.assertEquals -import org.junit.Before -import org.junit.Test -import org.radarbase.config.ServerConfig +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.radarbase.producer.io.timeout +import org.radarbase.producer.rest.RestKafkaSenderTest.Companion.enqueueJson import java.io.IOException +import java.time.Duration import java.util.* -import java.util.concurrent.TimeUnit +@OptIn(ExperimentalCoroutinesApi::class) class SchemaRestClientTest { private lateinit var mockServer: MockWebServer private lateinit var retriever: SchemaRestClient - @Before + @BeforeEach fun setUp() { mockServer = MockWebServer() retriever = SchemaRestClient( - RestClient.globalRestClient { - server = ServerConfig("http://${mockServer.hostName}:${mockServer.port}/base") - timeout(1L, TimeUnit.SECONDS) - } + HttpClient(CIO) { + timeout(Duration.ofSeconds(1)) + }, + baseUrl = "http://${mockServer.hostName}:${mockServer.port}/base/" ) } - @After + @AfterEach @Throws(IOException::class) fun tearDown() { mockServer.close() @@ -49,8 +55,8 @@ class SchemaRestClientTest { @Test @Throws(Exception::class) - fun retrieveSchemaMetadata() { - mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + fun retrieveSchemaMetadata() = runTest { + mockServer.enqueueJson("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}") val (id, version, schema) = retriever.retrieveSchemaMetadata("bla-value", -1) assertEquals(10, id) assertEquals(2, version) @@ -60,8 +66,8 @@ class SchemaRestClientTest { @Test @Throws(Exception::class) - fun retrieveSchemaMetadataVersion() { - mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) + fun retrieveSchemaMetadataVersion() = runTest { + mockServer.enqueueJson("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}") val (id, version, schema) = retriever.retrieveSchemaMetadata("bla-value", 2) assertEquals(10, id) assertEquals(2, version) diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt similarity index 69% rename from radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt rename to radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt index 1f3ba57e..a1854358 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/SchemaRetrieverTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt @@ -13,38 +13,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.radarbase.producer.rest +package org.radarbase.producer.schema +import io.ktor.client.plugins.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest import okhttp3.mockwebserver.MockResponse import okhttp3.mockwebserver.MockWebServer import org.apache.avro.Schema -import org.junit.After -import org.junit.Assert -import org.junit.Assert.assertEquals -import org.junit.Before -import org.junit.Test -import org.radarbase.config.ServerConfig -import org.radarbase.producer.rest.RestClient.Companion.globalRestClient -import org.radarbase.producer.rest.SchemaRetriever.Companion.subject +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.radarbase.producer.io.timeout +import org.radarbase.producer.rest.RestKafkaSenderTest.Companion.enqueueJson +import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever +import org.radarbase.producer.schema.SchemaRetriever.Companion.subject import java.io.IOException -import java.util.concurrent.TimeUnit +import java.time.Duration +@OptIn(ExperimentalCoroutinesApi::class) class SchemaRetrieverTest { private lateinit var mockServer: MockWebServer private lateinit var retriever: SchemaRetriever @Suppress("HttpUrlsUsage") - @Before + @BeforeEach fun setUp() { - mockServer = MockWebServer() - val restClient = globalRestClient { - server = ServerConfig("http://${mockServer.hostName}:${mockServer.port}/base") - timeout(1L, TimeUnit.SECONDS) + mockServer = MockWebServer().apply { + start() + } + retriever = schemaRetriever("http://${mockServer.hostName}:${mockServer.port}/base/") { + httpClient { + defaultRequest { + url("http://${mockServer.hostName}:${mockServer.port}/base/") + } + timeout(Duration.ofSeconds(1)) + } } - retriever = SchemaRetriever(restClient) } - @After + @AfterEach @Throws(IOException::class) fun tearDown() { mockServer.close() @@ -58,18 +68,18 @@ class SchemaRetrieverTest { // Already queried schema is cached and does not need another request @Test - fun testSchemaMetadata() { + fun testSchemaMetadata() = runTest { // Not yet queried schema needs a new request, so if the server does not respond, an // IOException is thrown. - mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) - val (id, version, schema) = retriever.getBySubjectAndVersion("bla", true, 2) + mockServer.enqueueJson("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}") + val (id, version, schema) = retriever.getByVersion("bla", true, 2) assertEquals(10, id) assertEquals(2, version) assertEquals(Schema.create(Schema.Type.STRING), schema) assertEquals("/base/subjects/bla-value/versions/2", mockServer.takeRequest().path) // Already queried schema is cached and does not need another request - val (id1, version1, schema1) = retriever.getBySubjectAndVersion("bla", true, 2) + val (id1, version1, schema1) = retriever.getByVersion("bla", true, 2) assertEquals(10, id1) assertEquals(2, version1) assertEquals(Schema.create(Schema.Type.STRING), schema1) @@ -78,8 +88,8 @@ class SchemaRetrieverTest { // Not yet queried schema needs a new request, so if the server does not respond, an // IOException is thrown. mockServer.enqueue(MockResponse().setResponseCode(500)) - Assert.assertThrows(IOException::class.java) { - retriever.getBySubjectAndVersion( + assertThrows { + retriever.getByVersion( "bla", false, 2 @@ -89,8 +99,8 @@ class SchemaRetrieverTest { @Test @Throws(Exception::class) - fun addSchemaMetadata() { - mockServer.enqueue(MockResponse().setBody("{\"id\":10}")) + fun addSchemaMetadata() = runTest { + mockServer.enqueueJson("{\"id\":10}") var id = retriever.addSchema("bla", true, Schema.create(Schema.Type.STRING)) assertEquals(10, id.toLong()) assertEquals(1, mockServer.requestCount.toLong()) @@ -100,7 +110,7 @@ class SchemaRetrieverTest { Schema.Field("a", Schema.create(Schema.Type.INT), "that a", 10) ) val record = Schema.createRecord("C", "that C", "org.radarcns", false, schemaFields) - mockServer.enqueue(MockResponse().setBody("{\"id\":11}")) + mockServer.enqueueJson("{\"id\":11}") id = retriever.addSchema("bla", true, record) assertEquals(11, id.toLong()) request = mockServer.takeRequest() @@ -111,11 +121,11 @@ class SchemaRetrieverTest { } @Test - fun getOrSetSchemaMetadataSet() { + fun getOrSetSchemaMetadataSet() = runTest { mockServer.enqueue(MockResponse().setResponseCode(404)) - mockServer.enqueue(MockResponse().setBody("{\"id\":10}")) - mockServer.enqueue(MockResponse().setBody("{\"id\":10, \"version\": 2}")) - var metadata = retriever.getOrSetSchemaMetadata( + mockServer.enqueueJson("{\"id\":10}") + mockServer.enqueueJson("{\"id\":10, \"version\": 2}") + var metadata = retriever.getOrSet( "bla", true, Schema.create(Schema.Type.STRING), @@ -128,7 +138,7 @@ class SchemaRetrieverTest { val request = mockServer.takeRequest() assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.body.readUtf8()) assertEquals("/base/subjects/bla-value/versions", request.path) - metadata = retriever.getOrSetSchemaMetadata( + metadata = retriever.getOrSet( "bla", true, Schema.create(Schema.Type.STRING), @@ -139,9 +149,9 @@ class SchemaRetrieverTest { } @Test - fun getOrSetSchemaMetadataGet() { - mockServer.enqueue(MockResponse().setBody("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}")) - var metadata = retriever.getOrSetSchemaMetadata( + fun getOrSetSchemaMetadataGet() = runTest { + mockServer.enqueueJson("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}") + var metadata = retriever.getOrSet( "bla", true, Schema.create(Schema.Type.STRING), @@ -153,7 +163,7 @@ class SchemaRetrieverTest { assertEquals(1, mockServer.requestCount.toLong()) val request = mockServer.takeRequest() assertEquals("/base/subjects/bla-value/versions/2", request.path) - metadata = retriever.getOrSetSchemaMetadata( + metadata = retriever.getOrSet( "bla", true, Schema.create(Schema.Type.STRING), diff --git a/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt index 2422d5a2..26cbefde 100644 --- a/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt +++ b/radar-commons/src/test/java/org/radarbase/topic/KafkaTopicTest.kt @@ -1,19 +1,22 @@ package org.radarbase.topic -import org.junit.Assert -import org.junit.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import java.util.* class KafkaTopicTest { - @Test(expected = IllegalArgumentException::class) + @Test fun invalidTopicName() { - KafkaTopic("bla$") + assertThrows { + KafkaTopic("bla$") + } } @Test fun testName() { val topic = KafkaTopic("aba") - Assert.assertEquals("aba", topic.name) + assertEquals("aba", topic.name) } @Test @@ -30,7 +33,7 @@ class KafkaTopicTest { randomString.sort() randomTopic.sort() for (i in 0 until randomSize) { - Assert.assertEquals(randomString[i], randomTopic[i].name) + assertEquals(randomString[i], randomTopic[i].name) } } } diff --git a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt index 55dd4c72..62841f04 100644 --- a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt +++ b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt @@ -18,8 +18,9 @@ package org.radarbase.topic import org.apache.avro.Schema import org.apache.avro.SchemaBuilder import org.apache.avro.generic.GenericRecord -import org.junit.Assert -import org.junit.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import org.radarcns.kafka.ObservationKey import org.radarcns.passive.phone.PhoneAcceleration @@ -52,7 +53,7 @@ class SensorTopicTest { ) } - @Test(expected = IllegalArgumentException::class) + @Test fun missingUserId() { val keySchema = SchemaBuilder.record("key").fields() .name("sourceId").type(Schema.create(Schema.Type.STRING)).noDefault() @@ -62,14 +63,17 @@ class SensorTopicTest { .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() .endRecord() - SensorTopic( - "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java - ) + + assertThrows { + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } } - @Test(expected = IllegalArgumentException::class) + @Test fun missingTime() { val keySchema = SchemaBuilder.record("key").fields() .name("userId").type(Schema.create(Schema.Type.STRING)).noDefault() @@ -79,25 +83,29 @@ class SensorTopicTest { .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() .endRecord() - SensorTopic( - "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java - ) + assertThrows { + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } } - @Test(expected = IllegalArgumentException::class) + @Test fun notARecord() { val keySchema = Schema.create(Schema.Type.STRING) val valueSchema = SchemaBuilder.record("value").fields() .name("timeReceived").type(Schema.create(Schema.Type.DOUBLE)).noDefault() .name("value").type(Schema.create(Schema.Type.DOUBLE)).noDefault() .endRecord() - SensorTopic( - "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java - ) + assertThrows { + SensorTopic( + "test", + keySchema, valueSchema, + GenericRecord::class.java, GenericRecord::class.java + ) + } } @Test @@ -111,24 +119,28 @@ class SensorTopicTest { ObservationKey.getClassSchema(), PhoneAcceleration.getClassSchema(), ObservationKey::class.java, PhoneAcceleration::class.java ) - Assert.assertEquals(expected, topic) + assertEquals(expected, topic) } - @Test(expected = IllegalArgumentException::class) + @Test fun parseUnexistingKey() { - SensorTopic.parse( - "test", - "unexisting." + ObservationKey::class.java.name, - PhoneAcceleration::class.java.name - ) + assertThrows { + SensorTopic.parse( + "test", + "unexisting." + ObservationKey::class.java.name, + PhoneAcceleration::class.java.name + ) + } } - @Test(expected = IllegalArgumentException::class) + @Test fun parseUnexistingValue() { - SensorTopic.parse( - "test", - ObservationKey::class.java.name, - "unexisting." + PhoneAcceleration::class.java.name - ) + assertThrows { + SensorTopic.parse( + "test", + ObservationKey::class.java.name, + "unexisting." + PhoneAcceleration::class.java.name + ) + } } } diff --git a/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt b/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt index 5800541d..66fd0bbc 100644 --- a/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt +++ b/radar-commons/src/test/java/org/radarbase/util/Base64Test.kt @@ -1,7 +1,7 @@ package org.radarbase.util -import org.junit.Assert -import org.junit.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test import java.util.* import java.util.concurrent.ThreadLocalRandom import kotlin.text.Charsets.UTF_8 @@ -15,7 +15,7 @@ class Base64Test { while (i < 2000) { val src = ByteArray(i) random.nextBytes(src) - Assert.assertEquals( + assertEquals( Base64Encoder.encode(src), String(javaEncoder.encode(src), UTF_8), ) From cdfefc45778edc1edd0bc0eadec41dfb41a5375d Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 14 Mar 2023 16:08:32 +0100 Subject: [PATCH 06/25] Bumped versions --- build.gradle.kts | 29 ++++++++--------------- gradle.properties | 26 +++++++++++++------- gradle/wrapper/gradle-wrapper.jar | Bin 60756 -> 61608 bytes gradle/wrapper/gradle-wrapper.properties | 3 ++- gradlew | 12 ++++++---- gradlew.bat | 1 + radar-commons-server/build.gradle.kts | 4 +++- settings.gradle.kts | 18 ++++++++++++++ 8 files changed, 59 insertions(+), 34 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ac8b9e50..7885d79d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -18,13 +18,10 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile * limitations under the License. */ plugins { - kotlin("jvm") version "1.7.21" apply false - kotlin("plugin.serialization") version "1.7.21" apply false - id("com.github.davidmc24.gradle.plugin.avro") version "1.5.0" apply false - id("io.github.gradle-nexus.publish-plugin") version "1.1.0" - id("com.github.ben-manes.versions") version "0.44.0" - id("org.jetbrains.dokka") version "1.7.20" apply false - idea + kotlin("jvm") apply false + id("org.jetbrains.dokka") apply false + id("com.github.ben-manes.versions") + id("io.github.gradle-nexus.publish-plugin") `maven-publish` signing } @@ -46,7 +43,6 @@ subprojects { apply(plugin = "java") apply(plugin = "java-library") apply(plugin = "org.jetbrains.kotlin.jvm") - apply(plugin = "idea") tasks.withType { targetCompatibility = JavaVersion.VERSION_11.toString() @@ -55,8 +51,8 @@ subprojects { tasks.withType { kotlinOptions { jvmTarget = "11" - languageVersion = "1.7" - apiVersion = "1.7" + languageVersion = "1.8" + apiVersion = "1.8" } } @@ -87,12 +83,6 @@ subprojects { } } - idea { - module { - isDownloadSources = true - } - } - val sourcesJar by tasks.registering(Jar::class) { from(myProject.the()["main"].allSource) archiveClassifier.set("sources") @@ -201,8 +191,8 @@ subprojects { stdout.clear() }) - onOutput(KotlinClosure2({ td, toe -> - stdout.addAll(toe.getMessage().split("(?m)$").toList()) + onOutput(KotlinClosure2({ _, toe -> + stdout.addAll(toe.message.split("(?m)$").toList()) while (stdout.size > 100) { stdout.remove() } @@ -258,5 +248,6 @@ nexusPublishing { } tasks.wrapper { - gradleVersion = "7.5.1" + val gradleWrapperVersion: String by project + gradleVersion = gradleWrapperVersion } diff --git a/gradle.properties b/gradle.properties index 68349877..9ef87588 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,15 +1,23 @@ -slf4jVersion=2.0.3 -confluentVersion=7.3.0 -kafkaVersion=7.3.0-ce +gradleWrapperVersion=8.0.2 + +kotlinVersion=1.8.10 +avroPluginVersion=1.6.0 +dependencyUpdatePluginVersion=0.46.0 +nexusPluginVersion=1.3.0 +dokkaVersion=1.8.10 + +slf4jVersion=2.0.6 +confluentVersion=7.3.1 +kafkaVersion=7.3.1-ce avroVersion=1.11.1 -jacksonVersion=2.14.0 +jacksonVersion=2.14.2 okhttpVersion=4.10.0 -junitVersion=5.9.1 -mockitoVersion=4.9.0 -mockitoKotlinVersion=4.0.0 +junitVersion=5.9.2 +mockitoVersion=5.2.0 +mockitoKotlinVersion=4.1.0 hamcrestVersion=2.2 -radarSchemasVersion=0.8.1 +radarSchemasVersion=0.8.2 orgJsonVersion=20220924 opencsvVersion=5.7.1 -ktorVersion=2.1.3 +ktorVersion=2.2.4 coroutinesVersion=1.6.4 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e5832f090a2944b7473328c07c9755baa3196..ccebba7710deaf9f98673a68957ea02138b60d0a 100644 GIT binary patch delta 39304 zcmY(qV{|1@vn?9iwrv|7+qP{xJ5I+=$F`jv+ji1XM;+U~ea?CBp8Ne-wZ>TWb5_k- zRW+A?gIDZj+Jtg0hJQDi3-TohW5u_A^b9Act5-!5t~)TlFb=zVn=`t z9)^XDzg&l+L`qLt4olX*h+!l<%~_&Vw6>AM&UIe^bzcH_^nRaxG56Ee#O9PxC z4a@!??RT zo4;dqbZam)(h|V!|2u;cvr6(c-P?g0}dxtQKZt;3GPM9 zb3C?9mvu{uNjxfbxF&U!oHPX_Mh66L6&ImBPkxp}C+u}czdQFuL*KYy=J!)$3RL`2 zqtm^$!Q|d&5A@eW6F3|jf)k<^7G_57E7(W%Z-g@%EQTXW$uLT1fc=8&rTbN1`NG#* zxS#!!9^zE}^AA5*OxN3QKC)aXWJ&(_c+cmnbAjJ}1%2gSeLqNCa|3mqqRs&md+8Mp zBgsSj5P#dVCsJ#vFU5QX9ALs^$NBl*H+{)+33-JcbyBO5p4^{~3#Q-;D8(`P%_cH> zD}cDevkaj zWb`w02`yhKPM;9tw=AI$|IsMFboCRp-Bi6@6-rq1_?#Cfp|vGDDlCs6d6dZ6dA!1P zUOtbCT&AHlgT$B10zV3zSH%b6clr3Z7^~DJ&cQM1ViJ3*l+?p-byPh-=Xfi#!`MFK zlCw?u)HzAoB^P>2Gnpe2vYf>)9|_WZg5)|X_)`HhgffSe7rX8oWNgz3@e*Oh;fSSl zCIvL>tl%0!;#qdhBR4nDK-C;_BQX0=Xg$ zbMtfdrHf$N8H?ft=h8%>;*={PQS0MC%KL*#`8bBZlChij69=7&$8*k4%Sl{L+p=1b zq1ti@O2{4=IP)E!hK%Uyh(Lm6XN)yFo)~t#_ydGo7Cl_s7okAFk8f-*P^wFPK14B* zWnF9svn&Me_y$dm4-{e58(;+S0rfC1rE(x0A-jDrc!-hh3ufR9 zLzd#Kqaf!XiR}wwVD%p_yubuuYo4fMTb?*pL>B?20bvsGVB>}tB?d&GVF`=bYRWgLuT!!j9c?umYj%eI(omP#Dd(mfF zXsr`)AOp%MTxp#z*J0DSA=~z?@{=YkqdbaDQujr?gNja^H+zXw9?dT9hlWs;a#+55 zkt%8xRaIEo&)2L9EY9eP74cjcnj%AV_+e41HH0Jac6n-mv=N`p7@Fjj@|{sh)QBql zE-YPr6eSr=L$!etl>$G9`TRJ<0WMyu1dl8rTroqF<~#+ZT>d1?f=V=$;OE$5Dypr1 zw(XXBVrtJ=Jv)?x0t4n$3GgUdyD%zkA50>QqY-Yc`EpwSGE19r5_6#-iqn*FNv%dr zyqIbbZJh#;63!5!q*JJB$&P>25-YG~{TiRL%|XOHhD4=ArIXpCwq&CKv|%D|9GqtB zS$1=t>o4M7d$t@hiH<#~zXU|hHAjdUTv zR<71yhm7y}b)n71$uBDfOzts(xyTfYnLQZvY$^s+S~EBF%f)s-mRxde5P|KPVm%C; zZCD9A7>f`v5yd!?1A*pwv!`q-a?GvRJJhR@-@ov~wchVU(`qLhp7EbDY;rHG%vhG% z+{P>zTOzG8d`odv;7*f>x=92!a}R#w9!+}_-tjS7pT>iXI15ZU6Wq#LD4|}>-w52} zfyV=Kpp?{Nn6GDu7-EjCxtsZzn5!RS6;Chg*2_yLu2M4{8zq1~+L@cpC}pyBH`@i{ z;`2uuI?b^QKqh7m&FGiSK{wbo>bcR5q(yqpCFSz(uCgWT?BdX<-zJ?-MJsBP59tr*f9oXDLU$Q{O{A9pxayg$FH&waxRb6%$Y!^6XQ?YZu_`15o z5-x{C#+_j|#jegLc{(o@b6dQZ`AbnKdBlApt77RR4`B-n@osJ-e^wn8*rtl8)t@#$ z@9&?`aaxC1zVosQTeMl`eO*#cobmBmO8M%6M3*{ghT_Z zOl0QDjdxx{oO`ztr4QaPzLsAf_l0(dB)ThiN@u(s?IH%HNy&rfSvQtSCe_ zz}+!R2O*1GNHIeoIddaxY#F7suK};8HrJeqXExUc=bVHnfkb2_;e8=}M>7W*UhSc- z8Ft~|2zxgAoY2_*4x=8i-Z6HTJbxVK^|FP)q=run-O0 z8oaSHO~wi?rJ~?J1zb^_;1on-zg=pw#mRjl*{!pl#EG$-9ZC*{T6$ntv=c_wgD}^B z#x%li0~0}kKl6Tvn61Ns|N4W_wzpwDqOcy7-3Z@q%w>r_3?th#weak;I_|haGk%#F&h| zEAxvb?ZqYZ$D$m+#F|tZG%s-+E5#Y1Et@v5Ch>?)Y9-tNv&p+>OjC%)dHr?U9_(mK zw2q=JjP&MCPIv{fdJI}dsBxL7AIzs8wepikGD4p#-q*QTkxz26{vaNZROLTrIpR3; z*Az3fcjD8lj)vUto~>!}7H53lK3+l(%c*fW#a{R2d$3<3cm~%VcWh+jqR8h0>v;V( zF4y9jCzmgw?-P`2X%&HK;?E*Nn}HAYUn!~uz8}IDzW+(ht{cx9Nzf%QR%Rhw(O2%QE#3rtsx~4V%Xnd> z`7oVbWl%nCDuck_L5CY%^lWGPW+m|o*PF`gv7{SxuIOpIR-0qu{fcqWsN(m8okFaNN=g9DgQ`8c4#Q3akjh=aXJMDnWmCheHhg+#qh$hgz%LMg7X%37AY*j5CJleB!%~_a!8mIK?3h6j_r(= ztV8qvPak21zIC7uLlg12BryEy%e`-{3dSV8n=@u`dyXqC&!d4mmV8hsait2SF z1^~hKzbVcsEr)H+HCzy&2rW0f>Bx?x{)K}$bRn){2Pa8eHtc`pcMt~JF-ekZr10N@>J^3U% zZ?5Lu>mOxi3mX7t_=3Z))A-82rs^6+g8*3w^;w+}^Am!S!c zcjkGeB+sQ5ucZt4aN$8rIH{+-KqWtHU2A&`KCT!%E@)=CqBQf`5^_KNLCk(#6~Hbj z?vTfwWpQsYc39-!g?VV8&;a^tEFN}mp(p7ZVKDejD~rvUs6FwcA9Ug>(jNnODeLnX zB09V$hNck7A3=>09Li^14a%frrt>+5MTVa5}d!8W~$r?{T^~f%YV&2oFFOdHZ+W-461bP_f zr=XH50NN@@gtQ=n>79e3$wtL*NGUKC<|S2(7%o+m>ijJIXaXVnVwfpZWH@fYUkYQJ z*P3%$4*N5xy4ahW`!Y9jH@`j}FQJ2Qw^$0yhJWA{Z&Spb(%?y(4)#+p5UTN&;j&@Y z8y*+wx`xfLXy2L7RLK~6I8^WRt&%h0dwRI60j%;!J(f`80Wl`t96JFu(~0^IRS*g-$IGS$#+8QxY?}x25E^_h!`yuuOJz9c>a3L`vc) z06t3`-)vWQI>tBkAzNtINbOsRmd2G=Ka($9B?iBJCCR$$wF)J>dY4q#l|!uI<()=8%evp ziiTDYFWO5?r_X@tBOcSN@&r|&xTDB!fF}g@NGHTM{{y8olafox=dOCu9O9u!#kenG zJgVQ3-&u}&`fvU|t-fAUzq+Tl75wtC3u3_pf7$qoouVoWN~mIUtXP?!l3ohg;LYHs zT>fB>F-lyg(ilR;OCS;9&o7SY2^ugYlWO}ai<12xzvh+R=5$2kJq@=h*IVVVZ)^$u27tLhOLV# z4nn+w3^prURshPx6UM_kXLNAh1ana69ZeS#TC$no-1Qu{ z#V0rjhzC3fh(L<6AVo^=E6Yq!c`Lre}$T!52UafPazM<+x=PO%{Q`xH9T9w7mJG6XV zscF#ORMKOf5z#a4Y`3WQ>47NKy;Sro_qS={sx3d?5H9Juy}DedhY_QOG}`P6M{855 zZp1owcyiDbOG}k-l@8!dVW?^|T(Z(8MWn+ltFu*8<=i88c`=Wq*Z@(bMC4Mr6`nV@ zkp*FSI;2+D^DD|>Sw21i7izopJO;_3sZ}u3uO_g#jIK&Y5z~H(WokolB9;3AX)|n~ zUe`jzAX4znlT#{R+7)ZyM?Q@uVO83DOXInC*fhbdd1Py~QexaxUbrIeE}rDD7u zK<;xyI9QY7*K5UYnt?e)AlCBB55cu?wSi+2Hz{$5kZ&o(5Av9`$Qb9C=Zc*|X}A*j z@nZl>XzxW`1a%Vum01W=VAu*FCNGaDqs#KLa)Xk6j@YB*57;O~6*KO>6u)-kWL%Zw z@AEm1o=j-$EGhu`41tWMH1j@{vAJot5bF#IpZu!-X=B|6ff22;3K|h-1ms*IS3Hb0 z@IAOeZp8Gf4>Qsbq=QK-uPS{9>7*jGBc;#N*L>&H*M1);i-0evQDR7(R%4rGSTD82 z{s3fpyvZxqH$vR3D5=2tIXF*MP^G!*5D`<$vMul9(GJjX|7om3f^!Wyzy*DaYj5_v z=~&Ypytt&>;CICFz=uY6oSLPPX03A(a=&*gPnddD$mA8?C)_P#_YLp;>-{^Xb6BQ^ zOtfbSrB$B+18pQ*Gw?;65qfB|rAxt2ct)1ti`>7_+Z6fh+U9zQpCb>;%AP2|9#kZK zw2K12j2*BzMzayoT%;?@7J=;CX!FSI{IF1SB}O-jZjT(0-AMe$FZgR%&Y3t+jD$Q+ zy3cGCGye@~FJOFx$03w;Q7iA-tN=%d@iUfP0?>2=Rw#(@)tTVT%1hR>=zHFQo*48- z)B&MKmZ8Nuna(;|M>h(Fu(zVYM-$4f*&)eF6OfW|9i{NSa zjIEBx$ZDstG3eRGP$H<;IAZXgRQ4W7@pg!?zl<~oqgDtap5G0%0BPlnU6eojhkPP( z&Iad8H2M2~dZPcA*lrwd(Bx9|XmkM0pV}3Am5^0MFl4fQ=7r3oEjG(kR0?NOs)O$> zglB)6Hm4n<03+Y?*hVb311}d&WGA`X3W!*>QOLRcZpT}0*Sxu(fwxEWL3p;f8SAsg zBFwY`%Twg&{Cox+DqJe8Di+e*CG??GVny0~=F)B5!N%HW(pud_`43@ye*^)MY_IWa z$Frnbs`&@zY~IuX5ph`05}S|V=TkrOq8$rL`0ahD$?LrT&_Y#Tc8azVT)l_D8M+H_ zwnRoF6PP>`+Mqv$b%Ad`GHUfIZ@ST(BUlOxEa32u%(4m}wGC|-5|W-bXR2n~cB_yG zdKsN(g38z1mDrOc#N*(sn0Em{uloQaQjI5a+dB{O62cX8ma-1$31T<;mG2&x-M1zQ zChtb`2r&k{?mjH5`}lw?O9JV!uOn?UP3M#fHUp=cxBb%PML70LPmiQKcq^FvojvtcZOCYEydgWQNAIrV0%IkxPmv)Qs^S zmLvL{F2@2dL%N^h=e6PRXa2lFh-sVtYlM1Qpp~@J7a19T>r^m-c7jZvDu*fb`U(;T zS-<-##+6Cv75X~D?Qq?ues%u!jBF(Y zIUnJIJJp~diP4wdU?54`;#zd^hZHa?76P3cnLEu#V!{F@Hpqm#X4W1HN8!VX5v&6W zKQ#Ri6w9~%aVjl6Q88)_;gH4||&p%hS9?1k@B725D5=L&$fMhxMi2%8__R)RBc0Hvur>!w7Xa6Uvni@ z-M$OMYiA1HoMqfnHs&K5H%2ezc5dj>A_TuZd4Qr!KJ5ZhljtBjT3*^sPX90A&m8*M z?Xx3`iM%6$mb>}UAvhvUS3*TGaL^sQ(hFc<_CRoL-r&;oX@N0g;K0y5*nQK=w#nvi zLnfCUUy*@0?cxGZMmRuvu}0w(AUq@uC^A4b41vdVsmKSrdL4BxqOJw8sUY)P>r+p) zw%X%tIjoew%BG{L`f^ocMtx~wQ(jAr%ZK}Vy>x7%xo_X;VkZ!ic|WNCH)WW;t4 zE~|&S+p@_f9xIx!=(f#uExcWOs`qDQKPnm;gxYBzj4iO%W+**s-`c#vqk z;hpHcBSV*Wa%DTA(u_u{isR4PgcO1>x?|AccFc^w;-Bxq_O+5jQV3$yUVaQlg4s59 zs@|ZELO22k&s6~h4q4%O)Ew;~wKkI65kC&(Ck>2G9~@ab3!5R=kIvfu>T>l!Mz3}L z*yeB){8laO${1xC@s%#F_E89?YUbqXSgp9mI3c`;=cLihTb=>+nr~i_xFq>r_+ieN zltGcpCFW2R-6j@74ChKK(ZFbs!!s=@nq2$6b z60H$h$(&CfxyO0UwlHEY^S<7wu|@6JK{)c|w_(C4-+FSF?iy8{FY1l65}9X1$Qa#( z)yNhnz5lG480H9oJsRdRHFxddQ{piIFZqGDOc0oyD6^D(CxW~fDWXKtbd3}~z2m4? zxyJ}qey{})xa{GBpPnR7{8@{vL!KF3)1$w>==~^CYQ&`SrlKA}ca_{ywJ&)(vrONU z`MZ=`jXu0zp@nH+24+c`FoWh&+$TLyJZ+(ygHExS!WXObvm6yqOsB;JVbA&ir^I>* zhim~-oI&{L^o24mh6HpUGd1d$GA)u>uQw*=J`5HhW=)yiaEx)dd2uZk$sKGbS`c$5 zI)L$3^TMIB-4r0!(uZ^oejT5P`S&a;UQ8$~+)8D^s5DGypyq4wL<;6PFm|Jy^;mz1 zhi+-pt=w^`v&IBWgK}Lo`fn~pTs3{~&ANBOzaUZz~c zM*cyzx1{QIcv_UUq9oW`FAFf#Fki3iara|&1HtpR2#wu>TutxnMh0Dh_cHiBPUfQo+v>aK09@y3!5u>0;;mKBv_oBXxPU(bBkNlj~o18?(tNrXa4g~o(#m3(ajqPU0qoaH~DjedUbfA0fcbp4M=u_@gF zNNP~e%ENNEkS4%P*L3#BYa5cw{(CeP@sY+Er(eD{Rkh@n0|uCl>|Eio-xm z2uEt#(w0yH2Wxv>6h1^3Th)^%Kctp-{mjFZ1?<#>SVoc8aUeAfG47|~>&=;=JtaOR zaBj&@I7<*`&^j!J>bH@^{Ta&l>)t-I=38&}ik2kJwn1#rw~@>3apDL0fAVFuAn1Mx z7zoG%)c^l)gWkgjH^l>!B(I#l5nTnmj2ZPt7VepToH8YL3@rC3aAUTZ7E{(vtGrn67u#c1>T4151-2olaIYPwPBA_P9^ zT)MH&vb|0#h>+^T3#**}Ven2sZdL3Myq!p+bzU$gK2Kk^jkJwh zepO$%drajHu=2bgO0y}tI#t~}5b`KJY;IQj&#lk(`Vwa z-+Lp^Np?>+Wia|z#`I!SW@sAEvijh>buf;(!)G}jWelyra1x)OM!Wgn_XTvimNQE) ztbtgCMUXPV=MA>P-2G%cFd2IK!5^8tVO!lG(qnQUa**au$Q=?*1vV$Jh7e0SFjUzu zUBRpkDW<$z4_DV9R0guKEc~Bfjx+=_srm=zVW<>Tdg>JCA5baQoWvwRmwg~bDwqCb zX=({}xx?ZQ+8$?GObN_F5=aR;r|jXBa!y7-e-F;SwB3ACQWt9+(E%P6OXa{1&5=|n zOm;d~Jktyf6=j!PQbUg{1;@4MbO*LrEJBsJ707zdY5i7{qdeEWtkxCb49bX~&x@{0 zuS6$E`tJpaCl*s}-TVm1)FFEVcPSQ77Auu1O|Yly)|~WZ-lO!0cL*4{bWW)q4JDTV ze#}fJv9pObE8eF`Bb4bgGUjZ#V5Gr;DKS1co@Qyxe!&FFH0I3`5$lUU{{kh$|uY(m+FQuf)ZS?{Hm zG(9h)3g;SwO-ZNXoU{ZXEQLqTXihvJFlW&PeTeR_$JSs-v;?7?wq*wVwE0oERWzp@ z(6CbDb_gM~XG`^xYv|#Y=lNU$ahYFXLZq1+Fqp?C|0(C7v1NgSoOl0V?-yU3?l*sw zR4`CpcdL6jfUk7J=F~FXC$HI&T_u-`H(RZ-ao9wk5~gsP}#JMbr-9IybPT zKE^{Fr6qspSUwfQ8!X6iBFRieSIT3-z$*e}$sw(l{>f4+L*4~%*-#IItJVbrxSI=^ zRn4&|Xk?{W=ZP5qRfLmU_$V;HBNK<>V%Xm>*Dc*9E)jcyO+$?IN`?VF<#{8H0N-^yEhtR5j>6ZK70+5rd6|5|0IB-&jR{Y;y-sDA@lqXvt*g zJ4lh`cLzraz-=Dj_Xb7&-ysYy1NB8^inO3K;4@#%~2xu?Xj)(s9b}a$R!s2KhpDZ|%6md^c_{(sD=32)hrm>lo=?HLmLJ z`%yhND<$<5$Bk$VQDXyxUXKFEHBES>xY_Wr$w(0DH;PiNT*W+7Ka&=(#3 zffXt$z?CQ&k?~6w3aeq9#TD!MHU41rqQ4)V0T&p>3MDzP#!|LND|RZ{jm!28xYgor zzqECq^uXX;@QZj@y*K^v#knPc6XsdK8dCl>gC(?>ay(OZx$@JoJqSsw%L?z*o0$x! zJl`lfuoEsW#ZpFBGd5!u_<$HfM5lvqK5`0NndUuZo~o-o;lu3x=^Azmo` zN3;zN)wef2A~_IFS|Qa$6+IjSuxNvS$yV4BEO8ILZ2tig<%IJN>2QD|WAc=gzu*G$ z$uF6}^rmERp&BUfDhtCX1Z_C0;}yF-4FBuF?$AfVX3}B zsCI{^qUP?}QrD{*Xpm$tjfm0sSuK(-&1jC_{@{>rfiBu>BltP*njy|0kTOgt@4-^6 zIL9_bYl)7gD`GeaCV3Qyq5CMPAFRkU(6FmMXAN$k_A(wgsvq=l6B0hKtxq zqH^ZaE+Y>&vJmdIP2=dC&S2QNkH%D`QN9!Pk35k@pR`(YxhE~vDE%AcRVa|=UtO2Oj=$*Pk-V!HiuZ1NxMF3TPe~xz;p@8VeEr;$M^aI zUtQM8+o8`!uCob zmsiMx{H41NPFS>1Xisf183g&fQG)hrwes%FEyxmg39MlU)gf|>-omm!gQU4On zJt@Pjytp;5<8Mle9(*8f($*m39Z!ty+{mQCdxc$(V|M$B zr#eh)yv#~2zhGwJ8UZ}F&pJ7t*4$iRgRx06-3!t}3qC6j6#D}m7)kqE%UO8v_?Dz; z38?6qb4N>u!792F7G?!yokb>#^NsYMc&$MgC4l^gS0Drk2-|;8IE=*50R~Qs#u$N$ zv>5Pi{y>G}F%*~3MwRW{0c)~_;V^qSmag?}c#ax5AG;k-$?p{I9qavY;eKKZ0jDV{ zdE)sMaGHstenmqaLckjCOWqRfs2OQwrxm(t>O_z5L0M~If5&qDGgn6Vl zlY4H_5AG1-u$Dk~o$_KC`(D85yqHT!n0)yQTA{&jARG^PEf8>a&YqE;M}-Wp6QThi zN| zGol9%&|!Ii`vDvQBn_pnmw5sDUq<6Wv-5FtOW0g5j?qCjHTumdX-35<+hAp~s}U5o z8A^MHK72zh$;)()ZxtQ zcqxsR(Nk)^i(0;m-eI-C8ngrA1FlVll9w4SP5Es4w#EUnr{DH(_0fWkfJ30G*jbb8=*9)gLqh+vS4@+Lu87{+2-Rc=$2HXTNNQ5 zl_RUQAs)1~Wo@>QoIxsQcIT>g)ontxy_!aw&;D{+wGNm%Z~V`*@|MXlQJ-d4yw5q; z{>OTNV}36~p|1xM5cZ==f|diNvsx?%BGl7YN%7D&M!4);aYe0 z&l%66;NGL-NBX%cy@#QWh{*|>PUTd%Ym(O4$|0Qs6BZ8VUIVTH8r-m{r96wJgp>dd z?AloIfb)6s_}};+94HCmoH~pdEfgs1c7v?!1n{Gwzp_80Abg(A9z5(I00&G+?UCeq zLr;g3KR7HU&kurul@pX(w;?IhoG_An2=$m4%TQ*ljt+C0QhK$tXR6z1+{I7U@+lr6 z3#;S21J(?NyBpFST+o9v<_+uiQQ|X!2U#^rxCOp;B(|0pT_TCutj@ID^6lxy%h74o zwwlWhHPv+nZ7vp%RT@)FfGYHtbSF4{qKcDPXfaHc=9MkYMmCgk^}UV|R8+n75d#?_ z^2G`}aKe&_O60Z(@Y`7$PW^OV{<%Oz$iZ4nuF#Gt@`cstRqFy?b4`x$5KP$Zbm*Zn z#)~b;LtZu%IEl7ZsP@bmSU1>I3n`rg+^_xVib^`ZqSehsV}^Mg0Go~YT(>a~juFW? z6N9NcFkL)Lfl}D3>U?XL*!5;4XN?CAV zBm5ldOm8_qw6%se4w?6m>#;|b5Sj}tV55zS9hVOuvKfAu&gv3J@Lo{iM4inB&jg71J1i;&WM@HS}O ze$SmM#w~dWP=cFB$`S4sX^q~tkqy2Hq4u`9z?xkCq;^7K?v}gkJO~(DX@(N!CRnvu ztdL2eg78}_lTHNXu4jo`NS3BC=h6ZFgRz7}azu4T?^I5{9zCjHUUV~?65=)4(UADPnk|!@Y=pZIpKy5}(F$HFBx`6tDy- zcO4n)uU)tJL$zi9XR7L1V@opZY;(W+M@`(OwJF{rSuNDnXaLx^aRYx4^wMY|7pyDv zMhVd+AY@V`0e|dFu@=duX(O>g9N{#PF+yB|R2FcIi}p(quk+tB%#=lSf&Dz;61-9? zYO@hNy`IvQ!Q1TaH}RUtTcnO( z38tR-%<7MyBeutubg6VDI^r9WPfGb%*;mM_eag!S9A2;4K2?!3e_bg@yi&#b?8eFI zPOH)(2KS`5h^-wJD;(-eO~7RI-m>kpv;|P&-rJ!L9KKF1mZlK5g77(gmJ`Pg0e)Em zb!bj8#@i^ozayNY!wx`w8Bxxx;lnBwIo1!IY>Oka7@!v@x29~l6q&!Lmm7xUQvxC` zv_fK;_4{tB9tpKHBgdc5JSq)0MiECOA_Pd47Ary}8DrihLeUU?Rr1+sVp6s@B9nDy zxqSzw=K#ofa9jC@cKtPlg-<~V0B|vh_^*5zh|>IHGLBR;%KLlKiHTD}RpvfqoSLb` zqh}LbOxh{O@-yzxX|SceOiEicwYNV>)(5b|7acaZkIF^e^my8Bel;Pv^kbM#TAvW?+CPF-8w%jc?1iYrdPR0M+d6Bel#l zH5d9O=N9fJNoqbh?Y#3V6<1pe-gj?W$|uU+bs9!UZSHqGXHtm|5U{pTI44G0MhCpR z%Vi%K#j`EqHCPy{JXljh>OAF@4XYyIfTNI$7f1_lQ+5mUbGgY_(yjIPfSUP`JxjOj z&d#n1)i_tHxMtfH@B>DJPAy$N5Pj%{hWh!{Gg}ha%$(o3*DU<~5W`|~~0Ahu6Kd{Oo6(Lo< z-jZ-n?Es`IPrA0FSw#bfR&7X+tR`)tlVThp<=YocC_di1<_BLyr0>l-sQuWF_d0%73{0&0z7ZH3Dkd3#MoU#^6xv$ zXJU1vZi*v4su^N807`n?Wj0W;k<(dT32}WGwmN*$!t^^oX$c8H@Q0(Nm?#LpyrSw?4}%AO%qG*7mpdDlVs-PO-ZH92;-F<9p9u#vfdMIZQ$zS}x36hydt6K5#nkHECWqmCcZr z1K}IM6v3ggF@qPpO*@~)T?M!iJ0U%ZY&CsX6kX)*gz^mU8i^?eC^P#a2=JB7P(Pk; zk0%5B>!WMOEvbQVj(00{)?fDeJ>xbf;XBG76irB^TFxM&pa|8MBR3KIs=Ps{9+Z)Z zWB6fH$9!Q)A%N|>=(8jEyrBv@ugtma(1orem3;ob0%$W&@_KAD{N+U#k8M}x$N)he z3vNZy(m92FH9wZ#$%Fd`V=&k{vH|g!g017(?A=hAG@|ULAdEnX>Q@fpUHxA=c1j0D zZXMQ5ttT8Yt4E57$+dHrG7Ad76KMUEf1Fj8?1XL^$^(k&6~BdkC00xpFF*MpnfPK| z3QFGIQFykL4B^A>XkeK?`BF|kRy6BzaCD334C zBvGQrlnqc>3-FiJL7t@v*osEMRC-sLJPyZ+jA03nQjXK$A;!M%zyqx@an%oD;xOi4 zWy4%$y;?mGvF}d-Vthx$c_aSX(<<>tj(dU5at51WLnw=th>`zM{jxwMu})!CY;cB} z?6J;}jgo}qKEAR}#!XI#OiGn-^GR!;W;IXA{09K%gSj?--Dn`xkMs(&HdPK3i9aZ- zVJIt${*+=#cJ*-@r@FP^9Mx)(+>N9OdLbMQUb-7|@g6t96$rF+oixyf*{?${!SZD8j3z-I*6c!|=$4o+ru7srWWe_qH&NZg-5jPq6QZ zdF$;6zUQ_BI$cjM2l}spQo!ijnAoPLeni(its-$FhjWOzBBwoU)?BG+kChS!Sr`^g zDMKYUVU9~G(%fZ5A!mNX4**Nw9D;ML5obF_;bm}zz^AHv3zw_aS zyf1JiifW6oiJfS7y93Vn?T-ZX=N0-yVH($bVE3>42>CdAqAwQ9?+?YW5iw7Y zeQ2j2Sm*@jqf8kl5x!Jzg#xsWJi3{j{v6-QeGEoF8sI2?$wjS*3tqjk1om6602hQkROLQ|U)0w&iMA7O>LrwZnEzSp%g$zv;uBN^6jI2LKi9(Z{d#Krqc~gEv)^bw5X@_0Q++t+mm25YE6nGMcHx+&_(^*bzIeehm(6h&srgPimn~AQ ze0pz~wmGI({WV=ct>xfG7kWZPo#h8L;XrD_o=^lBeHL!A+FkdHQ(0Yrs#b$Wyc*SP zV9Bn5iRN$I%hB(O+>RH(EdVK|`OSzU2m8D4V3sW`7l7;2r(}?crNbV?+}8t5N`z47 z2yDvlPyLvIMhygG1ix1Fai2KA>S8cUa=t;vnjl^nc!FCEL>);a(`cSNiY1Rx_d=0?a=FP{AQ?GrJia_&-UIkmb^UDTC0g7yp@m>h_d38@&Iy z(AkpzKdr6qE==pde{115P$?$1OaM8rB}t4gswVOgO>Y?0!Qx6hA{mTCU6ODL4oFdJ z8wKx-FshQ6D0Ut(i;1++lGC#6uc#Mf_n{(p6W8Bro!1Fxr-U02*wZ30nH>ooyI#b_ zfUnO3%Aos~x*&lNu=oRX^n6_&r+raSY*vk+;JJs>2PfJGq1;E|0ZbtJ> zczCsLujO86xDPxx0|SOLx)IVJ`mM#XdPaYWE6xG>6hg^Mo`5 zm+d*3Pyd?OB2OuBaL6K0n$atjx0O~cVnH=WJ=AuPTNITe6#*QVHc4CnLDQm#VDgP& zC^%IZi-Jj&%e7z2L67o^J?TPT`7>M9 zY$Nxrga-8XrtCpK5 zAlXC9dbLh*qr9mn-redGmX*V0bCm4L8ra2kwZ{MsZ@;w$w4aIiMQCZCdfPu*()Rp{ zF`<1QfG_vk_T>w&R;29dGiV@I&4@fpyY2R$^4H(a46>SwC|G}{R!hTqckS$3#SuHJ z?7}5y8EBeuwGbgy3gC9T5d1$}oda{FQG;$fwr$%^I<{^5#5PaIcE`4DcASo_jykqa ztm&Csw`%6A+P~qgUGG|ZJy3%BG8}dj?uA;~8%sGFw-Tz8OVl9`Rn1EWSK0U30(3DX z_~ccQ_K=Kd4(?a(>N`rQ6>ON*Vq1!PT{4_v8)WhVeyE&~0rH2v^B3%>yG7CRw`np* zK7Y6_w}b@mhQ~mW_jAU?3bUBC6qHac9JLQdKLpFgNrZ}8fx_y@L#4}({3?;Ee_))^ z%fF{jveoeoSbRG;RNyBzj7RdLUwg~YNr zS`sY#E+7ZyetVe&Qmg&3nXntMHCu3l)}!TQJL4O zAH-Vuos7{k0OwAyov|aF<1O-C;ZA;Wt&dn##mEXPHoK%!izEOerda$eav&gAB(}Ye z_+a#%vov6iRmuqNa)vTTA9D(07qTs+Dq#DeChp0jJ3=Ws6e!E!08(EuJEFfO>b#q# zBlAom<{{Y@c0`Xu3<+O|hL{LF;?b(4%ndJdiXRMCu+6^y!za69i8_E7aj>ml3{%QCIs(tAptIiV>q=rmgDAe z)q8)x`b6?A&rG2%jp*y3s!sJd3v? z>t3#jY>Sci5&)WoGxj_hL7s&$pvdzCt|bbGE@t#@F>m{jwY6ndtN)jDS~| zxie$yDZfo_lb^CLCTWU5PUGw&en1abNQvM8C_YpP9A{4Ua58 zAxu8AV2(VF*M1c+Ga3ZRhrfwl4P5DNY8aTRr6juNX%fm$^2{Jf%Y?cX8>2* zs0#n z0n6=OM3HVO`RR(;acPNFxe3<<0(oQAw;qveEzl7ndwKdc7iX0h$*M~+eWMW@PlN3F zE_Iu8n32d&ZI>H@{|g)@TxkN}puT-W{8tiT`k#tOpA#WaUmHUk^AlM%gB8(;99}d? zr+^YwX8w;>fkqtdTtONw_rf3Kak5w?z(OXRnA4*p%WS|+t?)n}q@LELezz7-U0eGp zQ% zDvDT1JZ)#7<|tPWMH&^JXo;o47*Zo6jElO=HWE3-ZdxcCUan5kE%CO~n1es*?hvWQ zuC*qkZsP%^GhP6>FRmT>9pXffsWU@mb=$N<_=?T+Tn-+zF=yM4<4|2h6kWT^r}{%?Jttf}|$L zLcA^CW|kT3+Fq(DYgcktv10|CA=h10i@A+d;6#cwU@y7so(?C$_KV3CDGY z5j73sAsg?Hz-6#4+G~vsum7UUqEe=9d| z3-zF%&H@~$*^d9NbDLDGWBJpsPk|BLXQlK)Xt3^7P;0crIOw3KkIC+kR>O!RXI808 zHWmf}1%a!<8pjhA+-r~~7ha6@{LhtdmTd->9FvEiO1P5`?V?%bN;7vKMrkxkV$ZNh zau(Ci*kG#bGr^%G?UMO<=j_fIC018^!PY`54iIf($+(Btl`o~B*DTZ0_9vRq)9z8g zrGXQ~2Pf-5H<0b-1uNRqJ>%x1cDuKY^%ip)jeNff!VIN-#>}7R!#WPCaGonvX@gXLjOcOWnWC!B9t=@2_o>R^xHFiu83^B6c5HRi`>Fyf*;1^e?f+ zy8)}Q?cBNUX3ZU4XIpr-qOpQ5nj`pSl!iMrr^GlwAy&3mYoelhNI^V72#O7pUkmaG zMrEzbSmA66)q8lP(YS(mQmk@XEtwDEMZf~g9ns0u#$WTj2*%V0PhUYIqd3af1s((o z`Q5MpnWePbxKy(Ac_sML*m$4=VFu{>ugRM6Xkmk}dq?b?1t}ryzeg!Eu`KSKhNF$+ zE6xn}0`Uu8tJ4i%JnkH@4S_fpuoij=7{eIW;w&F#Cu5l8GHNq)Jrcq!(AL(-gJg5$ zg?uRPRAjfAM7{UC{K7|YV>e}-x$m?Nr2FcaOZCv-Z5%L z&W^66Z)iDg2w#vFHelFoP{&)Z#-tM>KNl`{7ec=NAEixsci;P83Ki)jW-5EirH3{U zDO*uST&!>oT+bHvXMq;x!b+P6C+AN&+DNTjs!qi=Lr<6HpiiWLn@W~|d75&TKKFkh zLE){8NGe75)yNfqhgJj)%0$ImI4o z->!E^EUrEOP_1kZBI9-7#HVHj6hy+~Tre=w-iJWALp$&E@USJg$>26-Wdb!Q?8KJ_Oxm@5g$1vN1|CUqUT54}Tq*&DHCAgy+cyPTH@1nr7m~28-{9I;@=MfHM=0oP&TC z#l^CkS$)Y)uW_#u)9zJ0gL7%j+uW;DHA5d4ah+n0zIxURQ*x4&CXu}-fXFn%h~!tv zD~%8Q+zZZ-z7zwCSah+MnOI=wAB`MzgWO!T3{4}~dulk1#SNXy!|>yz=zE6W_iOWvVI_kfj?>fvJ8 zN6-cVEv=6V`(8#KFD9_uT)6cm>$pxnA`yGTZ7QRP?kCoL-ASRCC@8VXOm)30o|gl( z;E(}%8x|aTg4^|pUSwm97};0ICiCf-L+Ka&$+XxdX3pLWmxi|~LdwwsMpbN2`Ya>$ zkmwL0_oyBHfyDGo#P%*K14Ji2q1m60SiI{}lrx~V0_PKPI|EKrZ@0tF3JCY=dO5TG19B@c8S$PMW^58$QWA zX6I*d!*#xyGt#bGMsgHhHW7>w$jE!{yNmog@vm2?tUWq+yx}{k6-Y;XvJCNOOIi8A> z6WH;WEFEWA%l1&rgO?~s^u??mW~VcgV9FMLvi#p0n3S#R@1m3+zM?<}H+4zOz(;Bj zbvpsRS*b>iMpQHk6+kF_iU|CH z2ct5E@(CvV9JPDl@JDt*DLU8vDQD|ANAQ@>>Pg7=b8+^YQnAHfTB%~r9PYUYuT)>^ z=%<^$WFgiYvKf5bp$=fY8*~vo>WDO2j`n?+qrq@!ygV8vdB&2ezkO8zwE{^A;{Q+ z@D$5lwN`HMfS)LL^Zdu&6^lGDZHmXBeyPQ(6M1M{qsv>{pUE{IDv(Rg!YYtQ6yAi_}ouv=vLm+DpfTJgXW>k*6sz6 zJ|TBnBm{7WsRqGm@P3$DP@xhe7nBv4@2mxXN`<(3eG3Fg2Mf@9D=`T~(P*pPl@h26Nf*X^%^fN!SyO zp~uO{)YBX>=^g6)Arr2+hdT`~lE-l1uqo270xO{Hvv%wyL`?f&nRKAI_TF!hIAvOd z^qIFMLhlpZn)WpeT&0QfJPy=zu9&|VNn$w&$v3?D8KU|b!|Mh|;XMxi6E1mNrN8=Q zWWxfB9K_Tkj!u#7QX-=kx`ba@cKQX|a?I)hvj6&oNC@F2v}I+Lg(e%(23RB5|MQpI z(ZrF;aRZX|KtuHgVT&FquC_C@_sk%2*zM{YP#iqCw+z>z{)4 zgYMfmvTrGcCltVGJvjgW*01`eT%D+S$nZ#6BU$O?A7RN&z*W)FVJ!v}z@asID0#;F zEvRQUO%QT<7~GMW)@&-c^PM9v3E@JOPQPM%h@Sg0N=p6SIkkeWP=s zF3h~Z1jnOsHNx%@WXuyHf(=LkdSHSBVemL`kq};YoNSmeg%YOq5pq6VI#Z}a3ZexX zhq`-9_Nf8zv$t~sLgPbjFBT|7$3A8mEOYN>yd&Zc{#AqJbUppzF+PP6*tg^;y+bi0 zo|(84n!vi7Iei1VaC$b4m_jMUR$||5<)<5TBl>U-Orx^9Ok%y6Nkhs{EDWq0c%#!o zo)^Z{a{+_d>fyp=@Fu-o=&;#G6$*Y0A!+~B$U@aa>RZEV*XC#JNCJIKBbqfsmT)aL zd(_`oB_R6mXFnmcSTL1pWfRq>A=%|i#` zSE~H_J1BT#T9FOSJ{e2H!gS2--Cdz8?R8WyL|TE0o5TsxRIjQY`NPDCq2RHG0%BDk ziNhGp_$os6bq&6{J4YAigh4;7?Xi;9@FA%dx{@(7saTs&J#&$Sh^f{j!Ce)J>mAHE zM4(ihP7M<-2NEf}57?h>C&f)d_CY{{G7rT!rSsFZwfW9c^S7g;IuSc7n7KcmXWb8f z5{ZdxkTT{?yc_Z=8|cvEGkw=KYa;f-C(>D&bT&4d%F1i~{G{EU(q`)7HoEmUvibeG z+S}XPT3eyBvj5R&=!}kK(Uy*k%7Vu7QebJPonL{69fyeJutrN|wVR=~8)-wYjo`C0 zECWZUc+!CAz>Ta!(uv8XiN-YwUaMcx>+eXkT8ETu6WM_-aT0D+qznh{qDB+SDGdR3 z*_$(iC;yy0XEzsnlB1zDup&InKe+%pDo1GX*2`De#5;(AfdV&9CIUTPltw$z?d{mb4tbs>VX& z;LIH^m_dJS+xj?~*|23;Zv-gtR)Oh9eMD6e7^MD?QfaP_agSr+X?W)3t2c&R?>Lb}~=3zW091MJo~i%bPWA#O9!3^}aV zQsG^CDTG)_t3tZ!hExM>{rwCuEPzO9pNuOT2pGmF4cLPeII*aRl1P_0M$hq4N~_h?9(Z8nNcc z*{nGrSvk_P1@xapg;Sr@*Bb3IVD_o)D%1I=4r(*_E5h^r=5z`+ouHxrI$#trF60E#blj>D9Kv_)jPPmNgjBlWKk=;RlLOgL?w3T67b_ zgTd_p&{}2TlzY*L673**1%PEvqM?5F=8y3@OM21q)0hbN#S>YZy`{~S32c2^X2uOt z56JYQ+#j6VHRl$*tiWm7NuLnuer|%@zIVcNN6hwN1U%+EsJ$4mEqig=gqK)!l5)PtKj1TPFYNQDFY=Mn>5&?J@q&OuNmy z?yJf^|L}#W7KZxT|chAgkJ@>AMZa#QN;K`;BmGf z@zd6qireD%45{k{Km3nyq0l&}q2&b@ zu1|E5x#!7uthitF;bSjwarp=3oS*n48qYRy`MdRY?~FTHoS8Baxs?UxcT{1Z>v{9f z0-2@x=SUmSD(qPVrjoV5Ldi`N-bE>k zC-No2$$qi=EGa}Eo{k#!2}bn&wEjMOCHIrP@gC`5epjdS?`8IH@l3Y5+xF1o0DVLj z1S~>~X6@k{dgz>Iyvr$6Ub!O^<9sD<;BlTtm$EEEBl>&|E*cQPdJ!*yFQ{2lrbLxJ&-?h7A(_L_3HBb zmy&PUFOoiDq^n4T9Q?1c#2|l`_>o|hO5r?m+zQcW1lJ_%8}#n}4kl_&-~7P3+o$I@ z{9iLpq%R1Cb`rF!oD+A2w=RJgfoaU}uo-YK+Q9wxXNL_S$1Jl|k>|;l z9ndlfpFc+Dw3L&eW4w-guoPHy+f80)`BJg&fP*n@v@U6u)k>%&{!^xAw91fps;R$= zk%opTc9}W$WfFVz>=1Z}ryjSnpHI$zDC1jer`~%qu6{U7b+V%30^bY|R-#<5Zwh{n zL&f1LxRAVSXZ4G6CDakQYH|zKlDfqi8t4m9vYvF!y(+Y}NO&O3&1}y7{V4d-75)P@ zM4`+o-Ew8S#;SpyWEl+NLrfMMTjW8vDw)@owX|S?5md4#(fqw+?0al)nLnMqBmz-d z%!McAvQ6i}xfFy@T~=j-I#~0D&sgM1mUfz=(09D#`_DLFlXUut8BvHBLX2xe3NYn) zUENDU-GNz$9Ii~zW{~AhfNiLy8(~;c>O3Qi<~s4JKpLzir;XPp3dAuf*i$Wx8&=&h z6$u)^RJtoAdpExunn@40?6n#;Lfd4_IemAd-pqW6y%Wo0-rwUj3TX?ulK*l&NdZ1- z2Jb%xRPNOAO&++l$!ym=mH(BT14?VXPfw`GJPyhCusbsm_AB&Z>@L-I@Y5To)-^fA znd#0yRD$-w8!I z(SXb~d?TJCOLfU|C2E;3tab%XzfntN2K)mk0ea1fvCgO24_>-oJysJQbWTrMyoH*C0t`s~oFGYHE-M=Q1af`+XfI`A@`}_U`MF)*NzW(fz1vJnN#}If`6=lo5VlS5U=AefvMX%By8Qq$s?rdDLZ0Fp?0CBi)gjsH{2k~cB zreeNzM_i3~lW1-HR#fsY*VJ&;d@!BhSBO`26=FgO04s(uF5+;u$Jq?JsBum!BQd# zlJr$@?TG4=fVt7M5e(4%bHs2LE5z-#tGuyz9N7UyWxUef_ zM8ft}YDNG~%Jco8IQ*7Y49ns!E6YXjrS$u_Y28<^=^=J{#qI~gp3@;#@j-2cfW#t0 z70P@pd_M3Vb-L!J6B$iAR@KJIa+!AeyF@bspbI4l<+s~H4oi`LEK@-ra`QuCK`LMl zdU#e!Pr*S$@v;Sy8(pooy`r>4FDu#BMy{%qt}?BxM9)^93NU!SiFd~|oqT=%?30GP zE^6|(rJt_eJ8jKx0WB*VhJ_)iI_2;TSCOFDrx%DNAa{?FBFv2Z<|Z6C7!J?mqR#gZ}|6#&E?7g z9)FaWPBwqd_}RpV;xWLBI(kx>ltM{YYy%aSg_hYkghi{7V|OBIPq&xhY;QW_lg`|z zPA<;OTWY_H9upQ^eV0TfES5URpuYYC$%O!?-*e8|Y@u`QFd`sI;6Fj@AfU&?^b{7| zF~@UxvN#7sBPvI+j(fiIw|;{Vk_=?>>c9z9awh`?qWLSrXpu}8gIOe#Rf)yv$^rS4 zQa#Ch#c!TW&%#UF=3y@jVs^t+O-8JFGTo_0RP7!Io1e&#SxRY6*}cyXK@P8&C)efq z1?;^E6QK6~S19@g7$u^1$u zH5Vw@ng!80CMoVaz+U#d55A$;=XNK{y3#eXLhC!r-&JqOh1Ix$D&Ng`Jh7q=NL^?8oY1?4Nf+YiomKA+;3_7AkN zot-{7))AI6Nm~}Y&DXeF9p-g^>&#XP%ieTKuT>{|s0Nuw86#=)nOTwXM13ij5#av2 z&v_F2qD!GxHWz|(&YV|-`vCJEAGLzZAsu?tIq^_8P*F9v?^BZ8gCg_KRJ-P)i6|r7 zg>q=rpCAezNSEGFd3{0wg^{nS_S(gBWqzsQ8u)fHrH#<9bcB>B<=P9g7QQ(C;<~?z z!F4>PS826LwoN` zK#CPere|VyK2&{a@$?0FVlS$yC;$rCRgur;f*?0Ec0*Jb*vdD#&=XBqrNa9A!l3p3 zXNFh1O%?I-`5luZNT3BbdjHNqu=rdfR5$$c@%1SQ>$zCb3lv~b+EMoO6}wU!v@1jY zCG!PI92U+%=R|lwv=E0T@(Ysq*a9n7MD)?SG|r!w${)!z{d9S(MYRCPI_Q8R;0c^AMYfr8_IV}NV`D$wiBY)*0P{|%`i-~ z5}B}U5~VEb8;~K(D8k*zB#`jY8$%U@EjPB|4u-DKrQ0>M@|#oUlVxG>K5_F))3yX3 z>SU)xN^24D>b1_;T8#CEGG)+V#rHu2xH3!qjQQN)wrA=iCoh$-3ExETU@e|@nRlYv z6?i4#`(&ZVB!lAH9ej?Em%oMXfM*s)*{KdH9IzwyfIa^Iylgu0`k(66n*&jE`$ z#cSTmsQTBAPKnGu{a-^SOwct(hW|EAlK=fhBmW--!TAg&r8Wm1$Tn#KZbMs0U`;^R zCQqs>)`^ac05@U{%Lyh{AW7Xl1V~=b^zcj*5v*vl)pt5iU3nX%ryl`eM00P$=$!|| ztQ6b!o|8PPkG#H3Ur={vQ&An=kNe$kzis`xzJA)yd%G4#fzSy9&WIu~5~~UHWWZ!c zaH7P=YFSVcCZP=i8$yfOEiAlUVt+Xz?NSN+`srmfIyC9SJ2T|Kp6neK>)4YVv2pwt zxhMLU5z{_bM~duKvI~z9!QgoY=z**!$g)>;H2Vgy?ITZhHK3n)JIl1vP?v1m}RGeQcvnMFfqdoX0<_&};f!z%u^OunVVhByakeJ%gQ2J|(>TR;5 zM3AK1xWLg+`HL02M%prR)nwRStg7>zg;TS(yQv5kNqI0#oFjp!DqyTGDs?*|OwHEY z_X_Wyq;-yQQ)ennM_rv>k(NznFak0o9wbJ!GL=kp%Pnb&;Pm4N^xW69)aj<?q)&xk98Mm~GoMp(9pQByiCm0BA(FWA%u#>7pzn^JdCnHxjN#L}Jk zGjv>uohypMIA@pq#BQKuAwG8_ezZk{dCXOqbj9Qb}Q`^5(-+yW0<|IHdCo3 zF8KG^#2Uqu4jzA*kLbj4S=2Zz=f+fqX(^l>Kc`iHwES~RFbrFj34xa!a42kj|CFlGh%)FeltAr zXlU^4?Tyd&8+c#EU-{>z;QGJS=zV2>&w0!L5c@mcei<(UC39gLc+YI*|q)_2kMjN_=* zw<-_5V!P2AT@k#{QBhGJz##iU!2j;`EqiHGtjj^;1Yb2Yi#kflvol<3iCyO((rOA4gHf*TN$t4 z4bEiA@32nHS1bHNzDZe)p4BXGS>O9T(R!gKqUv{>`g2&v6!Fnk)TPOGVkwbB2Py9aPRlv2We2Vf6#Nc+^ZUi@7Ql=&nfx z2!O)sW{a80QQw%d)t)M8%Sh=RzppdfzUyS1)z6v)w|F9y=f^iZ6q;^BV2Lz5$Q1vy zv2E%54l7G%gco`Yb(kmyhdkO@sKSnusw(VZEbFg*+33*~M=^pD zYFX-3+@oKe&sA{fwrN9!&a4vy?9c5s0f2iw7Y)*4gr{b(J0NAZxjdG696&Vfk_R}_ zn-o4D94}L$F+d~JkV&*EKlE)BrCZACVvD(7HfI|S3Vht6F3=DdJCxiA?4U+T;j1hf z{!u-12wcp)gRU`$z_&8*|Gc~GHt+(y%I^AA{FUV)GCE&R%Vr)(6B{-L%1ur(Serr- zd|q3%Fhmpn5p7z6#L_v`_^170zQo_ufs?qCO@J?w}&alFy+c z$CIzILZ5;a)$}7+BcclfWfl=^YDxu@e<-^S5IUU@Q@7>Di>d(3NV-!5#a=9zuT35Hkmu=EsvN<9Kd3#YL{lVVhx}Tx<^!-| zoXdINIm2X#j1rbW~0#eJJ_Z5 z+_2C%0WMr&mjd_ z#A^r8snFEWk(0CYxcDS@|MI3iC?K$>(u3n6B5GLtiP!%fq`J@{2Dyi)@C9v8F| zONdBw-(dGcZw!behA~cx)q_l3NS4>Z_5_))2BtM~g#@V1oDqqu+NMNTUR zBWpVqqEhvsODr+Tst8&&erl}CX$b`9z@(U26FQ%IAa>oOB0e#~rQCg6nlnP^`Q`ZM zGU)w3q}CujVUXXy`~u#;$P&}Hl=GWziP@L8xMxU!Md zk||E5#6T1|Bu>TIsrB3^zU%eOt$#73cW{*fa|jnq%M4`|+VKX`MM)w{K4v_bf+F+G z0c&snF)SASh+xyEuGt;8NgG{)c!s>WFvF`3B4vB{ons`uBsi^(p7jP>hglnL>r~=8 zGgf1+4{oom2SHPkiWa&akMy^`8@!b}tK~4;NuZrh5ZrmlSVDZLRoKr>(zrA0^I9T$ zc1@40J&$8&eQ&3iwrYb``>U1CTS?4L@W}!t&tVXOCUJ?)Wv+$RmVnT(ws2b`jtlkLgxyJjyvjC)f<&5;J0dxHHR^72%E&9o9*G(WoHaiVNk14 zBT_1EjuH_uAiCkWTkJtQPTWM2Z9P2#{EXKe!cV` z4-b&t#pv{dq&WJYqn@!D0z*D^E1A_}CxQI-*xJ^P|13dGHpKMg?9M`k_o3`?)`R_{dV+_|2i{>Ne6CedHsS__%}6)I20R=`|5>x z%@8@bSMtbFBqm3(8B>VD4fA`10O`nL91P)$OK;i?e=*O@w=ue;(M>l_q@}wfiK0QnfA7!J}8C*%5bO}(Y#cK z(%1=%NWRCwydNA_vU??SiVEjXmCLwQ<(Io8<<}jbE=$uV}qHGuTYU}vWNXO!^5X$NJv?z5o$9r*n@14zwChU-wRFMyCyR#q}D@l;YxO1b) zzuphcPq9CAi*ApRN5`ItTWWE6%)MMD)78ohr)Z4b~aWyDoQ{fsd>k9U) ztaQYY?YK#bQsj)+r=so4XyM1y{H|>QNT(l6aElF7Si2=7Eo-VF)1D|1ZDAYga6|B8 z=9*M(i$lO$xyGoDA}X)E%7YGB(PFIz<3nhbT!|W%b8LZj7&=D|hBF6etlg}{;Z7TvLp`*? z7SW2NHf<&7rq$i9NON%3b+GN(vIs2(4&Wh!jH_KbRukYfi;;3ITwX(O;g+n2Aw5iB zi=kv{Oxnqj#RN>nmK*rR>bna2cATHVQhuDDU4J>2#mpSN3Oe`pXLXnKTyQBxJ2KFqYTn#r0oO4BPm3Pxs5xHLE|Tlp&k~zu zPcXkUT<6@($nX@|sBgo6O>9*-l^b};_#Hzg>)T93ECW50`~vq>dsQl5!mVaMsC=`%_i(wH)7tt3$1p%jyff zO5Wv8wB4JpKofsI)xlIQDOTFIGhJ|Yb>j^;N9^i~mTwcO^==wK{d?G+g{HpXFgPHj zQ$eME<{k9Y`@Yp(BsUb!Bw_vRCyl|6ZmIhIk;*kFQ)~ZZ^PEH9e)G{939l8niT*Xc zC`K&?jx$K9qXzXXWGjosuwljU?LRp{7ujSnE=E#$xeX8GuK9Y95I13>M;8053Y$F7iF_Uxfx7 z4l6af3YYQHmPZC3UkaW+hRr3JOw&1mbVpH`GccY~@Y2ld-x32hg^php|H;=C9U*MxO z8V?g?^=Ix-hLww!?wzB?i#|L+%}z#X6kQ|sM;L?b$PG1y3aiaaw@cb*qTUuxXZI(=-abNt~ia_rZs zd`xm{s9;5;8OeD~sWTihpFkZn^K|=xPqOp^7MN)B(8z_oiU&uQwJdnjfbbJIg>vdn zuvK?I^p25>GElPFC_4cxbB>wV8}QCvTrtT8J8sJ}z+{N#0^5wYE0XeR7+Q0L&Q|2W zZi%%n$3z<-GOacTs44^)QWl9;4>inX4B*A@AaexgM)7j%ZHb=DGxgX;^rW=#vwN1=L8;)8zphVmdR6bXeO(kc;{d zpkk6G8x_Qm&6J~>b^vKeHQzh}mC_VHZpc;kk3Bw#eihuCj_2-29A&h?$=U5y%2Y=a{v3}S zP{<{`xM2Ai;&3ZLOZ##x83(MtCmCkb6X4?rk5=7JcmD)Z{HnRlcMlBvs<}6QuFu#u zT}}V~YXZ3V!WBqFRcnq{x~TsiKI0Tkv9tHficz4%bK!*~x%;c~{@1f*?ibTlp1>NO z9Smj&hU=os)z9Z$;k~qW58w39>U;io!-CtBh;NIr`!x|NfmUTy6{=1%e76a&vDc}S zO-7bkATko*LZ|mQ6N4MA!->qW*IIrO1+*4d#Y4pK__v5djdlg|TsgG|DzZaA7sFhT z60}>Fp(x#^qaH*niKYSjv#;e~fg~Rews{OuJ#Fw<6aOL~jcYC_=?VZ39aY;`-E{U~ zy!712?5F59F3r3Kt#{dx>Q+bDEA=X0|K~PaaTgp|&e$J`B2wK1JqXtUZGgLO34uun z4V4Prh;7Zgv5Q}HfV(Q;SWu&R)9T?9wJA{gT^~u_mKAX=@xDc3Pd3(De-Uk$nK{X> zavq|&v`dNzFrxjiLyd*K%4haF=DNzWTHI9MJNoh<#aGQ`dhTHzq2jLC?zm;sYJ|hp zde;Kz(6TtVTHmgii+F}$3)@&L&U8T=*yfqf)cR%vAVr)p?V(wH zjU{1>DfGvDCp469u_YzrZNCiN+gY3j;)tkOYfw_92oEsG5LnrK zng0Gt!T8<;`~xq^V$X4>`EomH;}yz9ZAOMsOnhXND4jYfn4Q#a2$|~LyBg|IrrzIy zXqMYB#0Zbpsha_o@YD}neg;8VolOXY4_#t)CVQ6o%c}hMwemQ33X2IbmYU(o z0b-K_w06?*(l zG)+oOz<@xaQUUL(4Ft=wlqc>u*KH_Jmw>JK23ZFwCeu*s)uSQ1$wwk&`GR14)6HOB z8#xvajSh)`!qt+8-liopQ*3_wFwU{r=?}z51EFh(R;FDA7>7msceL$0YaFcKBT#t=2iW2d>GOwGzk=%|grV)~i>l`Xk)86vpm<(^Hl@8k zjsHl0mIa#@pUxXCJk8%MGzFBTrd?Aw*^CI`H)^{)3;c;XkmM)eXRu+M4nlOZp`R{R zw-GW7`L|s|9T3zfQ<5h^t))z^ndgh%X@L8IZWx1>1Q33a75~*z6CqV6%28DaBE}Dj zm?!h<-x7x+OK>+AGm1q)@qV)g+N<026Wshv*957VqQv?A7~SEMnUS1xVWZU+U7_NP zQYu!r&P;Yoon~SI-<-0+1sMR`ku;Y+wH){l;=YU zmJfYY7aA;Gwe`$!!alnKh!rT6UOXLcp=vx zaMS>p=Dn`-rn=7fJ_K=!8j(Xk_lV>VwO65!Z|ppCryy|^mykC#U{}gx@V4O^M?wn) z>G`sCxzs*amwvIIPH{k}Q(Ywk*V%1=Nbzl*YT{1uDgJVg{18k>6Ha0a#38J;uf(UC z_uQh2%MSqS4QpG$S^!BZXXgaLllo~OeK!Wj;csmo^D48OpXcAErzk`;=6dq!Nd}5^ zL$Jq(FZfdt_hL?e0uvtfPChKH{fVH!Ce=imqWh3*oeEli>~((I<&ra|GF<^Uz?aqJ zA@{fW6u2^P$pTA3nMhme$v1#2t0r=u%^5!m&U&!1`NDNili6tiA7nGy1NpvDOSXiW2-zD(cR_o z3m!J%U3NbZfETEYpiLZ zl%EV85{)AsLm84#huW^YfW#IqOg>3@4v5ZcGyet+0@BZu44O9D^K?Ev&Oetm1t&bH zJ>WZ@6dx*Xzg3itYc_xzu~dZ3?!KnR^}2WE_G**)QQR+GmZb)?bX6wISSLTd8tE;{ zBavC$w>CU{Uz^(yx8e@-kFfR^VbF1s{x@^KCLWiagf%iufd4pWk$pieu;;&~OmbaF zjkiWl420hcoqM%rpbaegTSNR^-z2gwV&=5a?MW5fqi<#=Y7apA()`@K_mu( z`gZwhQrGvqJarQwZA?&v25T|2)T^ISP%mobt(R`YYVxcXq<;V&$$q(1@gaV1!MrV! zP3U%UB`yYOhZ|1Pf0cFBaZ#*Y7#8Vn7nfYRK|)|bIz^F?5CjS7k}emJ@Ja{{-HkM| zfOLt7C?F!;wUPo?T7_>!zbokcGrQ+~&e`+U{&r^1JP+^qBev9VT)92LkIyg9L%o3{ zIvun1LUiM^{BWbSw*>DP>>A;Ikgr3?r&s0B`f+;;>wLEL9?ReTsPlaO9Z3|K!_6vT zttu9#R&#q_Q^9prk*z#mr^j_T-MH9AU&e|ga74pNLcD_G$y&6A@uOmVjO4ce zm-`4MT%Y81rGB~)5_R06GmRaKI4sd_!0{&Ae85}Z@I-QPhg5u)_v(!pzrflz zRUO0IL?Br@|5#Mkv_)e3; z;YmJ1kJa+YqbRd|{_o-eij$9of@}wD#MSG=ghxH^4?^}wY!!V%xo7!D-4;CpUAZZ4 z=${A`+G+E%3f#yVfMvZwK$l)4sZzRh+(@=_O1qRQk4fXYl<)?`hnSscHTmg++ztcu zRHyRY`~7Y_P7m0;$-GY}-J`<29{#c@6&UrT3?)Xf}7J5@w@D{*B!uIH$3tzk+8$6IR7lm5iS{nk#Ze> zJVv!CD91E`4BXo?rC*iw4I6QCNpNwLrH#n1*N z8q$D#FSExj*VMPe|f1zlShPl0k~ zvu0*Mf#C&VB-_i3H18#X_deYFZ`V3Bm&emCO<@tN9*8 zTNP=h)&KLw*KqnxA_0d=y!_$fQ2RtMp3o!8q(#+;1}mN@oS@he`Xj#@MO9hj5GrB| zU8z?C=qQcmyPTrrU3cX}@D!C)KcHWoe5+wz?R6n7{zvmmT_3ptuPrid7H(18IN zzNkik>~iUHJCy%XTQIMi9IFg?F3q}EL#o<=WjG%kS;@3!9P`ybhMdL{OEfh=_9;U* z7QbpB_M7{135QIWG?uiY^)gIcq>AX^3x+Y${M`yNsxjC;Q7Pg9vR#N6l`Cy`V&*D{ zx@E*@EvXsy1Fv}mbn>f0MdM|dc1RyGG@)&Vf7Ryvr>gnFAkAk29|NP58Uuq9nC+zp z1UE>K@%lbyR7=z+npLS|M1YwXD;Zvcd7PCO^W&RjWJ`EfqWUO&5wq0mVt}P33f0$l z-=^%OzD2??^JWp( z$MZ^v*Y*J!a+e!Grr`)W_)ccc7Tta*~X#Ztu6l+ zZE2Tvwcgc*X>8&$Vsr+SI+>Xm8M`XT$~@_Ts=aOBZ!@;N7v~5a+peKl6-yp#Dzn;f zG%Co_FaU`$TosB3A8K7+pAf7ZQZUw1zJ_$15^VanxFlTX;S>dWQ+1EH5Xbkc6c4#{ zoO)eGU(iPLcX1L@)bg(j7*jPvqOBtBhH!zCst8w<~(hM#%v#q^*Z$)b%$u>4Ebr)IP zJXX1EZ}gSbUGZ&uCdqTRiVXhuLJ&d+VHGu0y@~RzO|EOv%fo?U9SvcP@2V5V3Jh^7 zrZj6O;x&$$rH$KK2o%<|YZIo2%9bYGMAsrq?^YyMR?BN3NBAb#<#H@EGyGR3Q-|Hk zaUpvla?#FGS1D7$Kh_3$15mMrX_t^(g56nGs&z~a@5p0@Hul<&S#epNX-eI>blDzj z>oN%}=ve0D+vh?)z^aAanJnyb12U@dpoTrJCaJvRa>^X2`rr+Riui3?XlCrT;r)|3 zC%+;1luaS;7RCI)35SNIiFe?&G3b;;I?!K~4mFKc2Jm*uI7_xYZW zu#AeDilMWS=Zn~WO8YlZb@e=CozAl_+&xy<5iymENLrfYnuiE+W?L+r7wYR0V zDm%@(Mm*07)D4>9lU=(0P}94%rulvD8TtA1$;P#R?F<3`ts zG0XcCd{aHslmCin(rn+RZhg6__buo$TLd{2sk_bb&9KW<_Zu}*XB#X%>bk1;jnJCa zVOa>C!_DGHY|}(V;?((RUb=*XHkIR?Fr(mG0 zmF%qJ)bh;~cyD3ohiS?J2LXT=Ex^4q3BPljoMs{(`Oh zdgP+CNmcEjMT9`Y2g<6V?u`2slA5@x?(fBv*hF<(2oL+vM(iVs0W)mx$gf#n^X%+! zn;BnB64{=ZH(|TK;m^o{a@bmHe((aZ68|F9!?DFV#W1WVXUp&P6=XWlQa_?Ku5r&X zAT2axxhAQw+#)lV;AvgbF+*?m)i461N@#lZxO@q>`}-qk#;(7S^KL2afaHpysp{G# z#2Q<5%K)k+ZrCKlgX(UJj|~I*txBXj{eZy@W1bMyIU;vFE+(2w1?YVu%V5j07t*}|3+J`@XOv2Q-=dw!F1$m1R_GnH zI6D8jY`XFk_zV$G;!QrwkArxf`0$`xrCZ(Vb@$fEnzVU{-S?&B^I`OO@B4~IMON{> z`7uphV8qWmT|+Fk7i=N{vG6T3ge&BW`Rhl*Q3YHg5w_kkq&i3g!Kef-UiUp33BNLP z=jSM|Y?K$d52tWwM0zhwF%2IepVrwlUW8;m<2o>h9Gqq+Dsd zcPMfz%|FE9@hmfp8K5t8ki!b7U?QAYCf!wk_cDI*h-AJYWxjT=jz1JrQ=BL`LDArI_0xhDsKfBx;%?5CMmjX*c3DUcbK*%41{Aw)Byz96=`# zD#^<-toG7#q>#``$N!)J_IP zn~u3?F5}$`aRzl3bo$Pj?ZwPOCel&%+?UG}b(a<3)T&$y{GNI)m&*s>I&&}5c)rN6 zRIx=^SKfPssH~OBNa~P0GV!~pPWt4nl9s*NEuEX6L*uHl4EaLKGAu*5(o=1WRV{de zPz>bR6XxPye2`s1ck_G%T~OkRnlP5=Cy^$woNJmlr__!H7G9V2$BR^XFmuq4=nwBd zdX2?Af<B<3WS3<32sEVb2gEye?0TUVA$6X`1c0Uprvh^p2#&uE*1&Q zWL8FNu1(FL(YZ8kwB1FYQF--?*y}V<$}WcWd=x=Z^AjYDhEtMB&fWKF15T+MN)#jp z!Q@_4zr=fkChReVf_KJpc_hdXTOArxOrICR_`&H0nKgWS{ZOQYBFU>RNt{Cb%gC$w zGLFh1x^8?2gf^ys32t3Ep0WOga`-H5UE5caSzH@xDkb%NIclGw6j_Z~5j!J@3S}Yr zHJFJ%UJdg36}_IDMY{AFs%m5@@&tWs?*u=yjx>=Gy3ks!Oewa$rp&6%;dfLr88?Re zcygpKV(@iF*a$FH>C5dxL#!sLOWDdBBkr!|9?j~M*5D)=Ph1fr#Z38BLJEh~c&TTBRuk28b3ocvJW{y66kf=(z^z0u~OoAWh0?R={^cD3Cr$0C6V*GD)BS9}(Su(H17*kP{NnEd8J9 z+7ukK5^$N41V6u@3JQk(dE^BtQv)}pVZiznHP|FT0|hQk6F>@RPK}=yDL~_`X*gsZ z1{&Z%1q_$x&tX|Uux-@A1#cQCLAAt1>szvjwTJ& zpCkE0%n-<)qk&9bImI|$A>;iM^FzI`5EBC<7aW)?=ki#WI&}(=BLBwctlKV6L1zFHR0PvUnJGFCyUguI9Q~;gefT_aYshyK{I_JcxeCjl~ zzuWWIq0=Cd3XF7h4k@8%}NnL-r{Qd&?=6<$Y z7M>vg3U~qQf%n%upOUqK`(+f|k#gsvDtnzeZG7TAlf)@06S(jmfN+s>@DaaLxbZUM z8Tfe;nBVxkptnd6&IOKiZ_L?DKRrE-Lc9V4o0?qCsO$`l*p=mHJFF=boqeoeTIi6`64gcR2??U-SAKpBKbccnWV@Wje!p z_p?B8bCnvhS@bI!E-}A_!(BJGUy}_8tid7Jt*4D7%|!q8BD(c#H=nly*=w?3KjuMn z{7di+3&6GhV@Zpx!y!{Wpu6y2v-tZa-;es--$!21Mn5puEAW@Yxw{*`@!(io9|Rp> O;LQ#O9OtPYE&l^NyjRr# delta 38549 zcmZ5{V|XP%yJRM|&53Q>wr$(S2~W%u+qRvFZBH<9H?{@XP4|KruX8q#P-TXc9O^Bo8;bfNnum8g?VxrLff+d*!C>hkeMibP$TG7F# zn@<8^e$AdQog9Fo3vyoDB_kB)^mI##CDA>oS7C5kQ$u!)u2+nyUv0F; z#b~fbbH;VDLG@m*S1G1XANgfs|B3EqlmN4LF_HW2-U6E+zWQrx=)BCFbeT+waP0i# z{sUcqFmv@QdVlR6J)PG<4Y<4amAx%O2g&)h(=YGR5_|1G$@PMQd4@eu;QNQ~w0*4A z3!{@@MRjVwKOJZ^kveB92)5GTR$@aYXF3#Gg7VNmALLejWo(l1BoHkDJ5E;Xj%i zZIL*Ub>Br+Yy=$7$;pAMox@qgF4jGdGPz1`W*-2oObkAav*r@R4j@g#Pjg)nMfu$- zeeh!@=@Ex*;%IyWMv^84rk`l!q_p5Mmf2qV;k}gP-DxbZsu_;f0FI>%yD>Ar01>{P z@%^I0r6Bg8n9w62(i=hdJe31^J9P=2yvXGU*ObZ(8voF%Z>7g0qQy12za^P61F74m z{7*o#%^`*1`}x#GSrd^}Tu{2iY!RvL6?g+pc%I}2W>?HdO-#?OX-&MHHcV^ZzT%KwiZ;J2f&?; zo(7P8C=KHjsaHY#)_Yc*`4OCz<+f7`XElHt-EnNsiye=|rh@?{Dr={G7hsklbU-l> z+JI1tq)4<2uHK@2@H`)2?`o$4IBD`=X{vk7%$oT=8$QnxpY7tjPWRJoRiYcId?E)I_W#E(fWCPK1 zS6m2{weRSW2F9ErPy9PqX)oP4rqpkka?2;}DGCQ>C0#-RiB2dT6zEcMy5_R|zb5xz zz^4T>JDO}L9h4}H6nNH)g!dA`^#ESPiJkIB&MPWm$4323 zdFVx9Wr0m={K~0HS9gcBY7n?&sP#pN2$}eNeBfhvj zRQRJQDxV(R>d{%)-HR=e03&G$?eZ1H(P2!yH4Mfz6*jAp>nermiL`ik7Pi#Z%v^Z{ zJA)$j<_{eHI^XVHpXa$+MYT5` z>zap}M$~?(r*=xLo|rViLyQofqcn+px^4KIt`p5*BEeWe!&#WpL2-+qaFC z>}Wlow5jn{mf2j$bw*+(ZU{Mc_wqf7=2N45tmdn=<3QDxxa3MEXB4f_j~ub(n6`|W z(JUV>ELi zconE7mHSrDzBVisbIyQ)eO;*|H8RIt-aqqrhAR)fV4Rk@=%IP=02+gOD%ML_wouK+biOw%);W0}s1$yX|S? zVrntz@Hu!FlcG=Ntioc57+rk(>|@Eyq=xL&tr*m5dS@eoq6dX?EZn#i^LuKVz;a8v zVj}1k0Y5(|+TU=?>quZ~sN9!~cW7Ga%;Ym@8}_Y$Iv9l$)*J`Y%bg3tGS}O3mUHgK zVwRrgR}N=)fpaN#=d<@X~bC2q|<^LCFc*9$5a&FSW=0?_Hg` zxfQCGrGjk0;=35uS1(tZUBQHuepMR9etcv2F9^>519_TjdiYe&mp5=dZ+LcfngR-K z8MiVD5?pC4Z|Lk8`q5KhQPDzm19+dL!|7xp<5j2j7zn<1-Q^UOJOdSa=X zu-&Lw04nmn7j;0!n-QOaguE-7^ebl=K5=R3Fimm(pQ1zDuIB@acUF;=L^mgsBsA7e zci3T-%9=Fq$zjA}-k%JwpQEfCEQZ&{m^(6&W; ztyMi1=dQC-*He7s@lx}H(u_Z7|2UIq8A+=cg%cRw?|uOMH}OkGaQ-jWC-XntukjZo z$Q%*t8+?l|Vhs4d0TQJV^50(J9$w2FM!>g!<==m$1(wvmvZV%F68hiDJ>doN|0RqR zSXTa^b0QT0tzaXgiVEs&6jl*qD->E|uvl6wCcLQZ-LH1TOR-JLEy)0=f+6i65@B*? z9~JjmajtVtA!hpRxafa#r)P4V`+51<{e!-~eq;Uku0_E?z9b{AA%;MPZif442K&MT z*pc0tkaPp)&eA6<7{yaO)-mc% zQ)w#drpJexjZFF(tZKNYn6|HPemEn@=Q1|MzSOaHV)$v9+MtD&<<+&C34oS;Q(7sX z;&P2aNSar%VjYFkF0L14Rk-Zcsw~>xOOi6szk17QrO23Z;Nf-SdE5{+hKhLlvdpL9 zwbh9MaN?l)y6k4t@^(edbsluYy62x>l5(I%qem+4qjfh5X8W#*YTHUZCL^YEcPk^H zBH@JvIWFP88auvy^ewK3FC-4rzuc#O=GnRu6@xl zq*Oprpa|nBZmDMA_yi4mwM;<*Th2FEZJ}1ceke*%tDC^RDRc{=yU@%bt$4=rDoY$M zx^KOs-Fv+jDG6UqRX}Hqedq=mWphbb1l=9cD&{9ARc4-uhF+PzsJ$#AEpQ)D-K1yM zEU0un1$1L8w~HBU!9hCicgN1lhHOVW&dQyni4Zg4kkjSU5I23?wCpyoeuLFat84^d zfYY7knz22mdc|q@iBD!mTFXlkPOo$*g+I5eHSk8dF*x$IVc&`=1~zE5lFYsMlL3?= zn7BIsda~}f6c)z@oiRRMS_ci-Xt!2}$$Ky_C?|Wlw>Fe);vfcw)8oS5mfQ$l8`@Mf z>7H#cTx=5dIuH|4Tz)6oJVdVtTZ;j_BTwSigSZKvi2A;7b0|X!|9FDCxwBsBxkJh? z`?|H^2Qx-VE+?IMUhMP+v;73yeNxG6@Bb2)T+Wx&TCl+|aX_ncfV*ah0Ml=(gmk2q z)}V4h*pU#eby}X1{1%D!gJ|fS)D~ic8IngJC}dNgB8RN*$G+C%6^~zHuo=4^W&LY1 zeS#3U^%|YcD6Ko>URgTKQc_8#096B{+s`OyZ7|0!uMT z%Ez?1)ta~OChNNB743tmKS_deaRg;Al5Ngt-fbxb!aVzGZf5Sw%F5g6&%=5L37D1Z zR&atUgl(KBh@-y5=6kd(gnFZg!eyrEghsDlUaZ|D8rQJOB#5(y%6GM`F6L54xyxZ= zwJZ=&00mqjlIHm`1SU86yxg;iyG_=h3sfM|Z#^mM3mxV#=_;l6!h25}@X4XX%L1lt zSuAe*9v$C~;%|M#s;0#@+$N9Xfr{0hEN&E5N#TSEHx;!Ho?k)BMo(RjIa0u{rX~8X zbbTfqPW#o2gi6LZXBLXi+Nz_U*m!;dXrh3#Ov^L-soBcbGj${aq6s3iQJlpwt<&3x zH*0~WgJ|JZi?1n@CUwXA`{HB|19KgrA);|}>rkNuukuw9L+*f7M#Qn|4o3QCap|W~ zk(~L|YL-~UX1mUONp3_UZrl^|bNhp~yWMDHZEl~9lo-~ELo4hKmU$N6T4;+*o;koK z`wZ~_=h;KNuqJS^oj~xaAKo`aJ+gEMr8b(&Aa1T+$fk0 z{?VZP9H~rN|gmLW*gEJFjSkK>t4l)s=Pwr<#jU?SPE z@H|042|T2{qyN^p;Nkeqom55kRCYTF)(E0-nsz6V_) z8xZYopR-{rpJEhEfU^yq>juJFt!yewU|8-4PyC^uikFGWw(u3UPvnD%`z+&Rwyt!< zlD9wKb+y^@=ek0$=JTPzI0%9zGLlWn{2mfXo(7fz_jCm#ykZvD!QXX;%rLYEjCiKS9aPhzsdgHz4Imh`008JU3H`5eGwK{72q-yZ9 zIBT@jXRmurFT6E->bH28jJOnryM|ocmyAC>k+geviZgP4S|qL2I_B-CqB?NcS?=^# zq}NbHpD-%cIaRllJ}VYc?6D!%U7#@P514KMJS#S6xEOnt?nJ0X- z#W8Jl_*JgOl&l#LA<}g{G`463yb!04Uc*fn^!@wK7D)B=L2uF*9d_l_*=Y)kO-L;% zm#GSG;N9V9Zux;%C zcj>^ZAcF+5;W>gy`oE}FJKG2Xd1%A6xF@&=ecR?qv0B4xG}aPg$lhlP2sc={qfw!z zx%-{tNkZ5a2fqWMa$Zvp*`X*MSVulyHSWL)*csmZx}9$_*DzY-t?~u0WCnTT;53X` zDNwu{&9w13cRcEaS0Am2bM3R^INd1#hLnAo8Np4__wOSd;z!?g5ssz1Ph0}(L?m^6 zR&bf#fUQ9?BEvf<;f8eBIDhqB&!Cb{-HxN}F0X@;`7`;y>?@>C3hJe)5-LU-chw~? zo>V8R%g>+u0$hQ_>j5aYzEcmYE^&6im=GF7NAPMptQ}LHU+~M*soLK9k}$A#{79@wRqy>`qA`1IdR;JYp5(-@&bOim+w0i?e#Rn;>r9BTTtY`ILO<{v`0W z{jfF|CwyS?H){(1y;8!NI_8|A~=OAX6+NkckuwDBzG1P^NCE zj3J6C=>4Zjr9*8xGKMTl(?%5AhqXMsQc)p$C9yR8rHJTa&nsibDH<^P_eU&q=Br7- zGYQ`P;oJ>$n56xI`03m>@5{EolC?14?;Y9?DlUXZRa%o72HtJXX+Z*csy>GjEA!DY zI?{o%04zrgw&_(vv34{MO4H&pK)_qT!Y`1Y^p$TeZAWOromYg`NiJ2(B0U!R2?1HE zqHt1h#4+oYL}3nUfD0A|;N1EmZgEbL*Mk(zVZ&rGxRS;qp`2 zLeWYZTB>L0PPDSX*pl0&2qD}$M$o)s($l^|+a%|pX(>2h_zVv`*U-5c&|LrX7d7fS z=2r4qZx9u-`r>JK-v)nDUZqfO6_?|7;vjjM(@OlqT8^jKvGGFqefz~589wg4J|No) zm3IE~cw?<`&#|1YH7w+2qGC!W2hXmx ztLX1TE0Qs=u_ZfX3gx#J0BW1(4-xt00+ztR99CwF4@5;BT*-N-jgJUdEP%VN&KR{2 zZ-5&w`nbWhV6As=lgbgNbIK!BhVLx}m_p%9a$JOMla)uhxv%9DWY%?8$HXjAPT+H9 zOIBu;1L`A|FP~&c|6mabt36|tg6^ynKD%oLTmA3313W^5Vb7B+TSY3qCu%8?gmo+5x;H|yAYHiZk^dp{3m(JB>y}R3 zA(27!X~)J5op;u#A_Q~%C**%3Qoxnk+yXQh7!?}mmXjKEcSs1B)j9XZ(8T=P$2D8J z{7pWYPSRE4EX5-vlwZqUvbI&S4x63{aWVHyQbkBIZzrY!m9E#E0EMaz|nA#rF) z&g^Q0-g_3;Eh`3K3;hNMjFa48C;``hO^q>W1gBz@eIEm8tLmRF;o)piEAuu6hhr-I0RC7PYti26d*6u5wADR z>Z^Cml!co|v$%$Y4fM(`ck(Xf7+)j`NoSf?YNv2yOkM)$AJCSR4dWB8%t&OIUU1vC zmu$sSFB#LKK2yRn@F7(xi*I1v6E=cHQYP);F&11ARh%rMzX2)LgeJju; zT2r%aZ*I14;i_#bH6&kMvgBdfNqfPURd-+JVl@&t_AC3Fw=`oRZ{khRqHmiC+@f+R z3<4U_-&6q(60)-HkpjVskm*I&@+kdwz28kE!*QU3{`@^PaJ{lLGD>$aoSJ4c((~1wn})9+wsrX)`J!jb zUvKQ}Vp8+qlFdtmX%L;(dxe>qQ*-0Bwbt$${je$z&opWYUu>hb5LyPGUZgqbW>CB* zRf`6Q%G1kJK(GCYzaCb}gi@cQ=F#wOs8*+cuu=sl@2#=yrm4MET$5d8mC})=r_SWN z?zCMyF58lE4`0(EKUlND*x0l zg)%(TXvDF{>`mZA-$m^#t?cMQ8ZR>V{SX0AK-L@V4Ru-bC2_rOnv+n4`g30%-htU0 z?~U0Ty91BBCkS=xPVuvQnRdMpf4bRg1#M@oV)o;^kjfpY{$Dhq{3Fi+1I&OSP;jC?b;x!{R>NC{PUGxd829pY)c5mr*3oZA zey00TuW?Jf5Op#w+%TpL=?E?YSSS3O_HgOmYf=}uGgpoAKhgJh8(99(%)K*l0jFl3 zazE_(GDgQ#HA6i|Y;H&obS`vN#dm{!a8h_gvIi3NTu~hl#64mX1rVwO&vg;}8sXPL zPBrI*Vv|q^=K&CW5RTkFKOy90@;QJl(yRrPi6D2D7~OG|op~dn(*+9vrB%z@fxAYm zxXLI_zR9RHCalr*K73m6qX%4R+tQp&vdB;B&5dw8rBJsxvTx*rC{P7PHb+~igG!Cg z(yH8|n!l1yu{|ha5dg=!Qfl_y0nJd4`S`=WyQbo2B4bde)}hUfE-GV_muF6d)IG9S z0(=HEcY@eYBiUmvi-Q2b-`|@I#1*+y`*}HoHP@yQux&Sc1AwI_fzuVOX2JuWlNe`I zj9=;+fFFVf(s5A*jq)FHs`m(eDM7=o_xwTVfIV&3p$F~MfaciW?KuYqF^5N8#8Vn` z^B5s0Ta6I*uIfop2&f5pN7U=H)T&JT=>w|n=rP_r1Lzty9nkxLpArhRo8P6);}svo zrAd*M;Wp^=!5xLPxQ;Ehu`%Q0`U`C=Y7t_j#;%PP@2W_=lj6{8UtV{kspgSPNA?Pn zZ?qSJT%-P0yDTXE>YM3j6O+C9j`YPnGT1`;Cj6i1M={c0T=tK4J^n*Lf&a0AL4Vlz z0FKJbDwyMcy$N>Ku<+=j5)~viA)1krNh{&zB5|p+O;`rCan@$CZ?K*Di|^x|AD3Un z&?820g204R@3-4$zR5#Rr`+Ujxd)v#T^9@t1VLnurX)bR@uf^q%hJ(!>Pf?MuWrVr z#JHv&W?C-fGS0h65O&}KjbDGuGa0Ja1N^qx@!~4PGMnw(*EtP=UD@OWqrfB2Ee5Bb zT1^)mIR-2C=`0WfljKR1F>dSV>D~oJDc3Mb`sPuZd3?w0rHI;kvqR=U{#*u4q0uL7prrA!{DRqWyCpg?32r6l zQy`A!Tg{)X8)Y58D{qK?d8YlNkM^dbmccfBFk2i&z*%tv*KR0YnAbKX7!T zwC@Zq{v<>?bsaoiNDt4o#w=j_Ve$o2+=EM_a4YsF2=--i+!B14%ZX)#R+gGp?6+lH ztrru_iWrx6x*zi!|vOYo4n?hdv(Hz2B zqQ?B5_xk_IVH%L~LZEX_weO{_2b~nDTn1B=rL>AW@;^mr`&?+TB(aLVLrKy(6O4oO zxflT>eLUzBRSPV1-s>8Eaa9xM1`!ExP#`)?1$vswD03}Q)j>~S_&!I@c6}Pdmc>Lm zp(^tVR0HQs=wc(ha+k~O&kK?nbnz8`+pexG;xlA}KZQwWmecLM{D>$IgLfux_weEt zt5vGavgaE%oHgY>V>L*>5mO1nxakaZmYxB5Xjx3+@00D;yw6j}IQ@E?hs2|8o?Bd_ zc40mBvYin>7~K1^&J5KRzuN6mD0>4DhgJF?V+KwlpgS%jnyDFj`Z>OGNoOPtskX67 z(|MkO<|L>T2^9VVIEF4`(#uyB@l8*e&VR7frj_JzPqHcFJ=V`{t13yOQ-RBw%L{-+ zb$ll?oxxt9zK%*`r77GrqI*bIZSS2zlNH=LeMfarrfFk_e)W!3CLi%>P+w(;UIi_$ z&GU)!hB|N(P*oS&gJ?eJo}c45?>gg#(wz&3A8>)+uu9x}57}@hHT^Mdq1j#4y;8Nm z&7!bAJ3G6;NGv$kmx|HzWPEe$Y7c1HE%S1#cVJ;kDVi^nB3VL(J`RAWO3n589gbE+ ziVrr7*DMzfyPUm5?KSA}j71vghO@8yrMsXT)54&^6-qH}8Wmt0vxuiR4{@Eh0*iJE zh4^PC)7rDcnj7V>kXk$t#m`Zw7S3;|KM3tkO8X#gR7* z{QvPsj;zEpL0|kH5b%M7EuI2C-%$RqcTzm|B3T3a5R?HNPr0V*K}x8i#kNXMtBw?W z$G2CAgQcQ@{;OY~;pWq4e}i0-c!2TBOaUHEB@}#H>guJB>Hrc0&E3q*1w72o+{jD>UafOTUPX+hdaNP zpGGN!%=UQ_cZ)gWt@=#I|O#K7jC%YJMPM4_Mii_0vJ!8Bei=;&uu#AlVZY7nt(8f z%<~%F-a(d|1joy@sFtKBxNg?b=4XfP*AlAr0>bk9X&<~ji%l+pj2m!{r(N_wmTe#l zxfr5>$e0Lrn3wm5A!YmA!J)RZbt_(7bQd3kAhm3Me4%-~G`xulBKwJKcalCMx57Zz6E&IwmpTr>#D5|(2vuS)GNyXM`A=$LE$Z5w`j1-pK>d4I zGlH0epg~Nc1b_%j1gs)DJ(1c4H4EDB;i;%H7%5bm3U)G&T&aq>240gl>8}kxCUY{3 zdPRz(7i$0@*8a_U8tl6J1z+KloR|I=Ppg3d|G#KI$h15j&7~rlaFBfe^{~dwnL^IusO-5T`^W={%mkMlD;V9IIm~L0vDoS~ia$(0j-Sb*Q3-v5 zWO$x`O9MnUH3Tu^wUtL)1+LS^`28cb$Qit?;Wd$B(K=5X%bHVj2aHT6J`u8u2AsGJ z(b`LqF9BxD@Q&jjw7Y&UR|Fbz4gQP+rjA=~tqR&bzP=N|A*msh4E8=Vjhi5INl2|# zBnIwA@joLM{b(41sLh9^A*vR*O9Ky9I-m0h9)L0(X$D~O<%(J6#i#NDr7J@R9j#?+1bp*e3L_h@QWD?E*`aKdU zEM6qgs3{Ox+#gcjA5rZv^NC+qxv73ua)F25nbD7|1o5vsnN;)sWey z9HR56;76dsVt*(Mnowk9a^9F?vw7;RH0`(HBSjgX?!2yBKqJzKdqJo=OiqaoI$Ef_Az|#P<$DfWsR|<* znqC|_4^3foH0s#fQV-2_9MiMk(_YLf=GL_%6W)16x4b(hY-nz!{1l!~8orS#*-^|8 z8RS?*fpNVe=xYRh?Dw)f{YmB(B(%y2{IeKhy9mtR@ruUuju8_Y(I+r-BB+XTU$s37 zW^CCit`7jNR-L-yq)Cw>y{Lcub_L{bX_IIt2zZ+tsRfA)&YtPT>0PbwTPjldXz+iClZxn*3c0Q~>?D~nnp2!o9d2IfHf zPhI>UNgEneC<_a)H8B`X{*uei;`Z}vx7=(NG;!F6xJ+klZ#-5P0>hK%N^eR=nbGk} zks_Xt%0g@B5$ha6OF&I9!2l02iG&R8vOoygRO}o=pSVHam~A%Q3=<4SB6R>89}oK6 z%~_l|!;Ah<@mBWM^wjiRU0+phAo71k4c7gwLCkmGWcLNo<{VgW!Y;6R4MS!YTD+(I zs&s#6NBUc_ul)9kB(z?1h(P4~sy1v&M1Fr7KsdM4j57j=Vp^?gqX4KFOqm`uw?`7usg?6@|oIr!0o=<*xK6?1+XLU~4ZBjnHdvl){HrSzC6tHT$SC3$>Ep zLRD4uH_HTN@k*8|2|^7^C+*?eq(h@`a2XAfySjgT;UP=?ta1GIpe<>By?WhbV!u)A z1Q%INw51h6R$h1YA(_g{hl?!j%@yvIRhQ5MIrYxE7BJ@PYB|G!MZ|i!{#!+$lhFaN z6x|P1Btz3AlccXU<^ISJ+ny~4xLMigGta@(uYiS$s3~R~yn8ocC=KEVxoXvLyjz-f$d@MJP=DZ?PUA{w0F^kzTo37imnqVaZ8@+OV*gp}GVibDbLILPQ| z^N_hz?4m$+HfwA|7&VToMT?pkM7>lVVPc0`=B-b?~EU{astL#Kj|&gD%|MPua;53ZF)%FOpeFytCO_28%c0 zF_PLySuVZby>xi;Xb)1gJ*=jX6#O)Y;#O+`Lr0rU%UxU{Yl_89CaAW+P@l=e!))?D8|4wBU0n97p zl)+ZkQpKNLVIGyh4gbSjJuDGr>|q)5=ukDAct5E& zU9mI1TbhBQPwU!hv)-sya3k-d4x!4o`DmseqbGZ9z3pZdij zQw=8;${&-BodQ4?gVY-aV6o^hU=AkrKQa>UXMVA9D+u8}h5>FDjOK&eixaEtpJWji z5Kl(&5qv{>D>87H2>XrtlI29Sn20NRe_%FhGBHdyI@V>LN9B+8DWnByONrW4D~M|a zx7iowZC2fDX=|QiBw+3rAmvJ9M2I*(V`8uDN`Lz^Y7?MW*81Ay_COB&kSG(oZZ@Gn zSHy(T*N-i+?G^nkmBP-*r6_Bob)C2I}PAS3Ng(*Rfakvzc5EuOGRJKhsN6 zr8^h}ya7(?r!R(5{YT4JD%~#Ao7Keglc>*ENX?@uUs6b;fdNAE1ZpGEMoYud-Y?$m zCTr_}JVcmqwU<-p+T&b-SI&^H{gkMrPCS8DHfo*^9L=exuR{Re8c%Ys7K5?laTw%?j!`7D@CNu_1=Ld&fD@vC-xOupQL* zi8M4`L`3wg9Uc_;b=7ZSYS7J7*X+bQ{dMAJR7g2 zes3IOt*)8HNm@;BHUC~z07{?rQkDO^4W4`-)ldbVEh~F~MGNB>u|8V@e z?{bKD)(jkkd}=xa?OZjLw;0_EgD zDFHEu*n8jUS^K^#5QR3iHsT7)(Mb1A)TZz5RvGSISAm}4+o1!QbTIxkV2rr_*K||l z=gZqTw*Xej!Zx>!nb4@ud-D1=N?+HN?a?Etu?yJPw>X=#@rjdWc-~;u_~SsDJ%k~Q zyGPSjlaAhcsff^lp-eqz%O6{@O+Cj=!5xj2IEP*(g(QbLSeM~rfo>!f5Pdwh+)8AY zKjaTU{;W-FICoD6nCJ#dfKXX%QOzIvjg2@U`v5LWG};)fI_w4TkK?Y11Hm)}YmjZs zs7JNrrD@XD!xE(qK0hepR{l3Ff1(ayxzq5G-uVq&cW)*Im7+ z57JJpFwpw^ezHXOrRIgTevw7yDP5Y6uDj~vn199pbs)r0&XiJL-_nn=9JwTqcD|N8 z>#*(ZU}edy)Ue?!85L7OtvdKDpkRWoAc=9&h)nM)wDkmQ2GM>9{#b~oZAQLqf?@8+ zPCSM!5RY)bz7b;#I3(+ppL%ITDky;(U!)^9neu6!q_xQ~^N^wCJ63SWpbEFd@RFe} zUW#Dq0>_zFjG8nWIad7&g62QweVmx?)Yu*2J`y*e1^nRrjaLs6KJ?=S61cl%rqex> zjM`ir52LsUZ}qU?{rl=kC>W=@mMb-j=%3cT;eqd+6AE(=K=p_=XAP1_5U^{T)q8UP zC19-)qfL8G>K1Tz_inpcu`$g{a&e@4bvRMeOq&%*cs-Vw<%)4cE)ZJ4!d)@-=arUo zcGP^*qoXotZ1FHxH#pRsW<9B5-wx?dqDcafb+pAgPB;`jA)z?G4rnM{xoFD3H)HdG zUU5PQOdM_nFm!}mz$Fxns3Q~3?#GG2*Uda2x=lY@7cJCje6@36E)lYa)7O((Bt`Ml z94O(SX|rbCXASSaNuhY8Kh z4Fc`^;{N>7d^AlcY+8aQjcf2=sgxpBmt>yX_WlS1G>7Qq52=HH?o*#A9#qX*7!P;y zB%bz$oc1SMmp|qy&QcdG@XkKGuFE4n(g>>SbmPG=GCZ+kyx%b}D&-dmF3;n60@e3< z=TshT&sOeofv21BJNHoKM5--+P|V4)koTdsD7)re_=g%ls!7~)rcq3YT|3f!1O~G1 zy7s^T*j!rW!Cfe^XR$#H<+tf8esql9E9f7Dk^k;hf^q9v`q5BtEgY_H+)T(jOUkA|PGh9s^TC#O`T(o5YGZ_01pud1BDEKw)z+%7@RMP-^?&b-;CHG#~(z?f1kY# zTtBAlC2crR^v(0}u1MP;r}gK(rkrvemyTMRXM^!p>nS+IiQ7eCL1K2}O!UtS&0y%} zEZv+YymKWP7L$qZF^I~D4$FpLvoLZ2Jj(Zusmy_J?zw&PbWxKk?&*J@nx${Ya_jK< zR^C7OfOJpWnx&3&>Lwf1vp*PFXLt8+d90_;lqX;P{nRbjk#Kzp+myWe%1rCiOa@H# zLVS9`-}j)9IM-esW6xl*(Zx{?Eyzw(6E-;$a^8?3r$Ac_|v+>vn2n1rMj6)h+j;ou69ON2uSazy%?D{IMM2(<^Z6AQ} zo;c2`CEJpJ5(b7}yIjDAt1)>Nf`oSu@}1QeyC4n9%X4>wu$UqrYeU7 zIXE))PRI&6FN;E=m=MIWK`}E=dHCW4dvoBO$1PULx z)f>hOvrY?HCjdl`-co%1c|76G7fpBQjOr5_f6MI9MXGg7k)CRGrCC?~X7r7;8Q<$* zN+7Sa3RV(|t1|xKk144I;9#eyz<#~IeMLF7s<&!DR_fw;S*-0C%x&f0oOU>!v-nc} z$UtycdLACidmAwHPB*=pnfOz%f?*ro1`Y|&@86V%6Jkvw(4}l@XZrfxO90CKPQNNx z`gbQK)3^>}jVy)>G#X_d~nwE<3n3rX`H=~<9C zuQ#tVZeYA+AommNYCv?7eA9UHE9a`=_t9M|zbn#tgC&6ITnmHN=>GWgzrA}<;VB+S z!x|MNVu9KsZmzF{lCLFD7H)gJD&3|>LdV zkXc@L$}D@ug8m>!7~(6KS*Q|a;;i!ai)Or~NNja3Gg7eU^f~Z!>Ff3FjUbpkbG8s| z81HH@>f8E=D+DzgPqYIxtZ5)Q~`$<6NAfN*2K~v?KImq26G^J z7Ym)=XcY7NPz}wwKbr~}eCO@#wA*gRu`4hklNruW35HHDXP**Ym@8L*pSkPwq44Zj zlOpWkbz*>S2o4?4lVA5Q#OJgB7HCxOc9DKxvS(Z?_|$lUmu>kW0Uf(h?-KK18Rzk_D$e#wKQ*lV0yqy@H3Z+*p_V zF#~$pUd$S$Paq}EqT~v4UeXu-v@_mgq5Y`v{cvDN8^*ELsnZHXts^(D)abFxz3)}XM6=eP0X zK{$`a0tJ%hk&$7VbFZ$rRTo>a3lF;#=!T9?IqUGVvxeoH4Y5<{jhwUQ9yl$CrtIgu zvIm`|r6OaM|Haigc83+U>pG3u*tTspMq}HyjTPH$Y#WVj+qUfn&FR_WJ!kB_zpWoI z$5_w2@9VjyH1a<`!v80dQ^*%e+LCbXC4omV&tyS;DNE}mI zacqeLm=82u4x;*9uve5K`ZaS8HC#N>4Gk}038mt7uQ0C1ba zJWgQVK!r9i;%N7-xHHbCJV!|@L26ov>3I!1va4dX;5yG^+LG%+B}fs0!yQnq=0p5r zT=Ha2I=g(DY`o}9Lf8EFnfv+;73-3k!I?f)3Kn;j%3lc-==MLW6cwVdHaZlCWhV6( zs7WTCLd)e&L3~elOoOC0A-DBZb%2278B>yR5_~d0-#Q>)@>Gv+1l0k0Ma#*c@KyL| zdm~Jq)w{kUn}RThLSN@T)NZpE#9_&Y{;I{&$j!R^e0h4_NQ!zJHeV0o-nQOugJ98r z0PNqtwjFzL5sI&ziZung`F-IIrk!)|b}5h}I%KaBoweObS=qd>zKtRY~ zlY~mKlLQ^{fDL6-b*wLZ=e0VzaAul_z}o{9Z_ND*=)Y9(g;Yq>MUyC)XNxo?di0zXU!%nf(i2rNlu{ zS@Z@VU@%2~B{Pv557<5HXl`!kzk1}Ja2o&KHEGeF?#i~Y_sk`dM^^(67IAS-e-)d`!PfX|ny1+g zV}w7_u9)!@lF{fLILHl}xu02q07=$bTvgKF58YAdbwPlhV$%IHyyx za!q-lRH{45DW!+Mu)5U<#l?xEKI4Qo)-K%?rlpKWx?NM@cVxQ$aWk$yP(bclJ*E&% z?+9!8{$vjEIP}miJ4^2!qhnSbnSBx{6=9`5k<65501^Z%cYr_Mug2w!zAz{K0j$EK z0@#&CX!|8cg9zhc0|(PAar!Q_Xtl;s^10bj7iQTyvty;*8ps&?B0)#_Xd*MINzzd* z(AB{ku<*&?DEEyg+~ma|xZJRGRg$k70SM#eg?WLzu-B2sAeAV~Xg-1R=$iw+;|#yu z$1h4$@U6$es8I@bS$*WHX|w{BHk4E$0R3HQ2>m_l^CGW#)=<^sf^OLEJ_CH6Y8A_2qOqNrbn2L5D6`7Gp}q9PB7zI&o}{5gtl=S?Aph=3aFj&-5h;aEr-TX5^%6{vPBZtm)H|F_o?*MaOrS{j>Q}0xu8I z17v>&iXxRDyLLf4BPTj1U=N{A|EvK)#065<2n~h@T^>XVej&*aypT+6{1Tk(4eB~h zpuc%dz(Rg#!?FN1B1lry4Ib+OF@yeGe)HwypK zcoqD;jiu=~4zM8VOe!Mf*7s0vj@&ZtvxxV^kT~_7St|Qte5?_PeFH?2h5Vm8@{~`D zoEe3-!oEM|=lXF)lg7e})hc~(IwFxrb_vhzkRl*&7GVm~b2;*gy8ZH&K45~t>7|LC zoswvto?9L+yrgWm>iPTENuj zAr*El@m)y&OZwMq4m*3!QJg>N&K(V)1b|QIUfS1DQBZrf0`!6TXvrk@u`JtOZq$=I zGt|UZB6Wt0*5EmcXv0mx>0WJ$0uNp%LxOW-k~kPk2Han44nw_YB7=7{=zFX#7<@g6 z<*%KW;gc0JX=x$3)KuoF`T2BsihBVDT)$U_neCTc`SiNaz0vhmDj_;>pw)p80=?&< z$g8D_4ewxm6uaKu`(R+%?P`~A;Art1cn(~HeJU~Ec}j$}bD!H#%KCiZt@&%92rWHC z?O?X%^~OEm%Zx|2t{QsH>=?9?WzaJTueM$6xVX1ek>~FWb;t9UaP8D0@uo!jf zU-!^XEE!u%IV963#9Rm2qy~^ZX+%X;O6r?1P4_2$ZptLqy4U%MgBGj}gK=g;i8Wb$ z$YPv~^s|NHkCU#Wl9Ox8&pz6M(<3gJMdeHl+v1Fyq?5Ibv0Yh@jfun3Vf(Z}Cj)PW zdW+H|`X#*cMDugq*54)=T{uIBHe)R9Ddq~GTBkt2Dx58s&A&(# zBQ|fLpBf&eQV8ru#yBt1FpV*Sm6FyfM#E4JJU zu2jCF_aCu4N7+{LgezduDy(l%RC;$^%9Z>VW!;@=f!}t| z_0;5MTO=7ngg&9xU{dO(C43@3Hw$qNDZr$dT5ZH2{xgK(T_5IxQ|X15_%q= zfBDXUlo5v9dG21>Vb&t20m{{DM3@DvAw%}!8QM*ur|1{t+@J5h`1K=*Xs<}fP3J6n zf?#U^5~&1c;jt+(d_8oiCYEN2aTfN^acmMy(tB)_3Q|D&=J$e!COSn6J!7dTGka12 z8+paI^;vQ-HPo{L+=3eG43)7{(ax%;?X&I!@>!pYBm}&5!3oTb;iwn!g*#tKeGT>+|i;fH@y^?x6#a{{Y3^1(nr{GdQU*#5(tn>!hr*d+b+rU$m1 zmBrA$u4GST?Ks&6f0k>MqcHz-Hi>=YiRBgL8N3TgGZd?^5+qFRe#+@9a!6FN-D}m<2}3P?&xuT&f4Mbc$s_1^@DW4AqSIS#wp%w z3J~b5Tx3=340}m=3fIL<&$mFH*Q6XNxC+RI`&p;sA5oWvyL?WdWQC? zNSJs<5bHQdC+3%0a67d>A7wmZ3}(pEMif}XdP{kv&f`WIqJv&dd0lr+MF1H+4EQ@N zAva#|9~B3ZwFXgEswfmYXQzjHP-yOe=3Apl_nudA3IBvEmR!mFP{+P?f^$*s2B9c{ z5&Dt4xi&fS>S{mr$+7Q@(>Qn}(x|)aidi`1>rh3}tMNlOQ_nAy6e4x}To#?vN&OLc z2{5nU-k$8yELmJ2QwEbA?7&R2I^B?qjX7;4%dQ8)2zPA0zLZ!j_2lWVqgQxmya$ch z`qBE}3m!WMx&sOkeedHmt5n@Yf)QA?v${*WbG%&I0d2e%$1vh;yHN+OjbU1)HFX;!!&J)@OHngw)N`-lU4x? zGa9sHV~@*)8lgH-H?FO_O;1k!$}q)=@tjx_*S#ONEpVz!uXAp$*;K2Bs8wSUN%k}F zr>nM7N_O_^>P7Kh0Xsuo57Zn=jx)ob#pUX_}BHFn5S#1`jD zij+Na>)7*b88MTyh_fu((7w_cq2F*ipuzZtaoO$#IUGRk=kV0Bw{CA4Ee$iQ(|P)L z_GUTjB+n~E7|puFoQ3 zv<==LI9p>Zgt%1anN))y=Aj#e(47KI3G9VE5fzVyN976~&KL>uZ{L`F>%acj;%=OS z{3P{1%BhS31cdmX5s(02Ft#ytb{^7%@z7pM5g5_hZhXYs__;4C1r6H3r6&aqvuY5I z4@G;IsNoifD(q38V@uvZR#ZxtOrBigtpVFaSL~7>Ts%9A!rdpBM-StDX5;dF)|5@n zI@#@Jaq;)1n^LnOMCv5-Ce!E6_a(>sy6q(AA=ml(xBl0ZGb0KxNAp*adT9>uIQ?948y57%$ILNr1lPPZW7%_wIKZ@|9ehto&FvK zfmS~pzsonq`&n(kC-#>fU52yjcaKv90r|a$p%>6OI^-#(Il710%+Ae$rA}cscG#5) zos;|}og0$7+Q2*jjMMAXwOipRg+OlzGeWEq!t{4PCT-`ii26JfP3=$`Bl1)+4QE8H zh@_R;D@*>_QGq4$6na6M65EC70!;=-$O`Rd%{?Td?VcHs|E@~o?m^Wrl)_ojDRm?# zbcJGe^*rmkS$J=T_?g^Nwpr;Q8ULnot?pSVOo_gIyjSTdcyuK^{5_;r(W7*HrJ_^% z=t5;#b(`J=53M6il|zL<$y4J9IfazwM$xlY154FIWe+O}BYG&>L|a9^I2vuC?IMPl zAD?|?3S;mMfmf<(ETPn1)z%ajWezsqo-R_`8+uWWW z6oOJ@XP#Q$+;CR4_oiy9tOjeq?>C;UsV?p4=&A+~c`wi5+a7{ z?B72^m-)N>?0ON!!qirHw`b@W8$D*NW$JPyOJb@ z-Ti)GZK4F%ji(rbWiw682)Qw&{I^$VVNOgFx^{Y&?Oh$QO3YyN_2a1>>00ScEKdL2 zoe+P!s=WB%Dh1C}0`zycX_@AL$Op)Sdfz%>iwvn$^^_!biU-69s4%c zs;?;2b}K&6=Eo3xV|@>&#YD^?E~jWgXmZ)6s7=umGq~v5Of29LG(YhaC zFe@1@MOQO=jUAmX&Qc;#Pn6A)coB-g3xHO4EQpAZz@%JS3=P*mTGSFJKV~>8$GPFu z8#DqU^M&dJv=O3i;l;B>r#NlVd3Dncj7@K+_e7Xo1jRV z!||_$miJYZtOZ z`Ax-7YU&N)P{36-WTzOI33aqmuGLT$BKNU##?kHwCpy{^6lxd1W_x#FUdmhGbwFFX{E3noB%fFyQX2zyD8Y6f;-}F z)q}VPTO1$|@n3eWl*{&)jBxo?`7viW7o%(D)|~wf&sVRI)J3vz;|xHe*?@=Ax<`Hy zE*s2UIQ`zPTv&Q)X<$0YhKc}_@bAjQ_Lq-PXc~EOkqp}{%W~mNUABJa3U(*|F54$< zSbw*Jy&FoR6dr%!H0&{U_~jlmVY#ubSk+9DG%GhCe*d1;{%>;p7x~;~>D}jtzj%*4 zkT=J8%Ks`yrNekvat8!`nCcLl&*~n8z0%_Rpv$PeUt#;p1Be_*yk^4wsJK(~lQ|gq z(_GaeigGy?f@4>w$sF+MMT3NV#+@$rOT1O+^f|a+-s*$i@8?13pA8w04E%*xY(L?H z8|aPPcVrlxJ05m5t%ZcL=)>{LX(Gtb#Jf5F;hiIMF=xC8Dkh+4z-X_;-*OD?+$7%N zK1lO`IiL}>fSX$GGwU=a>e!P_;||n@Q-np_EpxFJa|p)!NOpRg$QAn6ouIIMNwoiJ zlArjG5pson=>yC^XbXF`7hWAfTj~&R%KJ?CzP_1YEWe>(oxO=-c`XFv`lhLkkvIc- zP2MmvO(x7iqCf$4DR-#;USF05UV0B4(9A+eln#y5$lk~R7rOxkuzejHOnGs;I@*X0 zCE-H%vk{!0K}PEj{=WjzwBNUgKwI)vmtkUn-dYfkq%}fhHu58du#vxTB{G7p6~BZFScbpq6eI>Q=r|K^J{<@ESR#O0wNn8Rt(2w>|j5_ zg{v~Bqp@A1-3y8u3^Wt{l9nSF3g=Vy9|c;Y6%_+u5HG#YK0$>DgA=UWg#>woV-Lgv zD!~8@x5cgRT7Z@f_j0!BURIUZu~AnIynAQ<)fV}*L5}URu`<*w?$S!Z4ncyF`X}F# z0Xj9J7X)CUyBrfDtsEn*9Pm%iX7&dV(^Eenyyulv7h{of@V%b*oR*PtBCj!}qBn)G zBrMIvgW3bV$QCGF#U;hC_I+Bx%$^)0Tz?m3*)1s&B9JP%L zTTe+C#zoXmq<{8j>5o|RE_&%Wr{QStP+o&SToG^#sw_pop2(`8`ptXUVPB1>ptL;( zti%V!W<-~p0xIMsb~9xhL6;M|x7F&nUk+lbyM-5J-^)kp>9Kf$TI|UF?T5Ec#6^X% zhK8XgvTLNB-_WFbZaPI;RWhy|iRJiB0w482lRZv&W+$)Fx7=jny*x^xCPD3lr@=$- zaeknk6Hf}1hJlrV`Padi05!NkNzd*_Qd3}9)UQm4UqknOJqD4JfiH=OCui(6@&{|? zV2`_pHyi?QX$&bEb`y=(T>k3#$zGCUUR)Bn|A@iCold?WwC=h=XHcVWAgu31;AKJa z*~v2!>QAw1%vDs-n%t_PZ&Wrp_?Y`U1(5)BR8e438b+{ZecE?9#dlsobftzAuHd&s zx!*B@8Sw(%g z$;l|a#e^v+|6pe|CQhR+{{3^WWp+25*eWK_PlC@>t81zZaFfTpMr$*ZUPn@0j=Bay ziv;*+cBCR2`?p&fcZ0^NjMZ{^J!3A30I zLBi?n&Llh-I|7(&p6h)~6WDo6s>jk;uKw_U4ICRpOWNrBFn+jOA{$@+!scxQr-NVi znoaH*rE?R$o5&MevSr*@Ew+FpCY}r zpeVxlW?{_QK1OW5G7aZW;sUS-@+UDrg6_=Wh6V0a#C9n4D(}5JK8J#o{qEc#zqS&; z2|rp;4W z71&v&YC+Y#D`|=A=hqfM(Vqg=kFGwd=Xv&$4}2u#$*Vd$;A!mch{ps&I=I|`tUyRC z&EqO~HBqT>oHl7lrwU0&0t_8ZmV*ZB>zDMTrhtdA*RIqA6ITqJ08vFHc41`3`hkk3 zGLYrN?swvtp?lztPg#Rq$_@70)tK#tOEthY$01IH;LS&p+$sR3CJ#_*N3qkAa4tiq zvMfAm%CRcf#mO65Cp~Fy&)PUAlly6M6Yi3E3IoMsDxWt(K2^B(;oe8Z@J_eWKcoEE z6hi@K4L%c@VIJZ8AfMO+UQ?M|2;tK7bQ2#odlIm&Uu|D)|60Du1sTV z+uE=8rg(OiD5j^-BMXe!JUk_d)X>#V%nuGJwPqGay&3a~VU{N_S}FNa*QE`PTKu~m9?{EL75CHh{8hD2YAIv(nyPDfTD)3b zGa^NXUF zf!czxMW-Vxkg$R4r#Ge96;L&p;g!ktnoA98!V0jTc>_&^?>mw=fd@0EW^XV^f1OR{ zUe1U*3|ipvBR;N4&n&=&e-T@}ka(GLjbQVH93BtaVa`s>N+3&)8zJ%I2AyhR(e1&V zy+49E2?9{fEA6d0dO~Pz@z804`;~%4(9!Orya7|=Xcfw3BKa$5Ub^|5XkNtU{ukJ>%IaYrog}dG4wtZ z%cJpgw>1BiX<(jEc|KBZ3_?yeYQeE@j_M~Wdj|B&zhFJ#UEr0{gLQAOGs9*l=Hm-u zZ|lU{+Cd$CFPh~o4ibC*L0IaS?nn0L;_PJ?iT0*7!WE)YdhmwtYVrXsi%7{t8sYi$ zqUJ|X!`Ve`h#dC%8;B(fQ8O{oxsSSep*aY%vhok{jp|h)o?nyxQ4mB5SesPS1ed!Z zY7YQN9EhMh_xY*GlkFIJO{&hmRsIif!Jl<+C~u_c!y(&D%eA9$Gt*;h&g{RoiwU)# z52-lNQ}&=In@L4hT$cX0nVo9wFpR*t=!QOC^X%9$6Sx@h?cRon5OHu{U_Xe5hGyva zmF|Q{8TTq);7-p%V}|u#b#2)2o?CY)KOe9R#lPh^oxcsJe@ZjucT2#MS^)d4Y%Xa z1F*Y%#xGMKS76$MLxBFfmjA7no^AKJLl`V_2OmelS_BOJnuqPD?FvGf(y=0V&#z-B#QtaZV`}{yu!seHrRuKXBldomMgrx@UXHX}a z>l|d!tq4=UoR-K}a88GCF;D{3<8Or5hD&-DNQG=BwzAzA9TWg5xM{OJW6wK^*@H3D zQiP~~17^9)d^o?|!`*dZ3aFPtLzucs=ADxi`Eb5H;?^K=;^1c-LQjYXqO zZy5UI;DOL!BQ_YeZ^FXT>6hO#rOeEi*EB(&^47KDyjEzR1nMJy)~^K@#JmJ7d+iid zYu!}-HT)i-}QBbq^W;{Ae#M& zAxZeV$2&gDc7*#FmKp872Pfi9!tFNEHs;`a(5oO4Ve%Xhjd<4=rn&A2Lzqzi?PcO{ zPlDV>rXL1|5VMS@3db6rwg5-OYoB6k797Jpt|Dxy&Mw5WODZqWvcPNpY|%ELcrB$G zu@rBMbCfa05l8=SJbR3tQgmnpseEX-^@kjYcy%=+LKcmSkKBr`&=?zmED_R zH&uBF4GocgRyTC(H7Pq+*KE-4-qaPKJ&|v>xI1e-S2RywOqS$! zp((V>Bn{$Pv6Ro6@M3)wL!Z&m*M;W)yGFtrOu?AvQ1{xk|T06zDc1valS+QGwNbd{CS; z79$)G`2Q4NV3vs~wLkmN++eDxLQk8M?f!9D+I?(tv>wprRJBvfzXIhSyr2XMcMT`0 zUg;2X54vU!;9$GM8L3}cx=HpbVY@>cVY_4PB|Sv@IPb~=?G45IThM)=cF?Kp<;t21 zcfDT)uu~vF&T0%pe#GC3K>RSOAv~Z&@vGQ1e{BnNehmrK-)Dx1J5Y!9n|cF+und6` zWmdMZH5dTRaYEo{U{0?+`G;KJ%^eg3Fqn(>fejGvqx6#fTZ*A3)iTzSlO6BWm0wi& zw#0=YTcAm_T3RkOVMAIDn1+3Y_RxBuu!7Q>7p|nS;PclU1v^!ZhGgR%ErS~3nt z_Z~e2itnyR(aqV+vsOo~yBTsTECA_Sr%r5EI;q()iPnmG$!dBU)cG7n))fcKHG)&4n;mpa03&4`rrq(>GVD(1nUh2kVyi3}CLT>#Y~3?B&e z_Im&6EX9p}E8G)h?a{Gq6VDZ9`!k)?WBO@Rf`<1v3jCNFr(Cm*KbV6I_mjk5Z0tGa zPp(y-6M^iQ!bX-b_`yZswebB94N8*v;7|pd3RLNpKg)8vYRS4QpI3RdhJS}32Dk6G zC@xoDa}y0^bPvSsd+AdQMmg^u(C2N#Eu9=+d>cp+;y8*)UF*o_ zwtfrQ4Un6?kZkmW{`vD)9V+gRZ&H7~scxh=G4*iQQZpI*Q+)>YWq^qZ8Vgg1%)dA0 zO|+4C=fs*;(XdrU%~JGikvTh$QYMoC&-O zjicFTTcSP4zK=a%GvwC{Z#cr(WEr*P_P>J5?6X8QeHX}lo`}E5KA!ULrIJ^|K$D;s z<%PWbsU~juaKHu;=YdBboU{c3DM3!JZ!b~ob3uW*;4b1`J}voKPswBENO)BMlBp#f z516L|Ec*6Oslo;?W&}&R^a6LrtGD@96Hr{-`LY~AI9urL$M30f2lF|@mUNkd@g+x; z@`eyoX~oDSZz*6ov*+(bf8qviHiWIe*wmhCa(Y)gDXON^XMtnHKdc3VYz#B;YWhOp zvX(khqLzyuVe0j-@n38?MLz!7#6gMDY?V!ps1_;`YW(rdXO8S zVn3~VFaJl~Oq(>j#vz;$k82CQQhsC4^vB=vlIO5sRGNRy9B;kf20$$WBK(cZL?XS|f+u7E$c9VSaA~Z}|1k3kY8@we~)r=InkPetr9&b@$wn z;<@)fyc+wTUXA|$)!j)lrR;zW+_L=#NbyhVVr|$Aq#>+KBw0a5tBl>PI(Sn<%Q3sk zzoho9v!VragVKy2io>jp8}e2b3y+goTb{WOIoWHU4=*E(Amn@;ND^|P#o!^G@DnWb zr&QyP|9Wb2{7QK7sRQpCk2Nj~`0{Fzzd71+1M4n2cfkyo&Lg&-M%uuuK4<)Z_7(4UHH&bEtG#9-f|`S#m!h8N#GRvVLr56$x6-=d#hoRAtOs?U9at?+JI^qY6XkmT`WG<2|v@R$HwX?Pgh+0k7ts0mq7w zTpribKhcJMAS^}YH0gjX0hfwn7HsH&ddSHouTdOvhOW;@d=*=pZ_|`~e+hgI&sY^& z6#SpdQHQZeA3C>hv^g$>sYvpKp@42ZFx6OI*X+W4*d*9gUyRSI@#bL zyAEeUKRGHzA_crmMr#Z&&oUNS&rA1$@Md1zF2l@lQwLu&y9uwhS7C(JFlHEx zhbuh#j10<&yk;P|nosxh04*hVls;Q%;%ElxbH1;r9DEgpEmb0ro^%KnmK$@FDM;Ht zLyAk8b4Y85V4nY82>78JQFcCxeJENFumJ{EpEg7MK&UHU=E zn$GFzxiw#MHXHISgTs2E%S9>DGGjiOjb0XWVf;R^lMJkJFrCvDltv*zR}neE7rB~* z1|p*goGQHG9}G#g8;A?KADTDh^X0rVX_DAEzr3@e?{(wt&iz97)!3QI_pk#+NL&!| zQ6quYEa9%XwjTkxvvEdeTi=5gdR@3`!(~)YkZCBiJ`~YTWs#)rE zOI15XG7!%mQF6=gG;wn2<4#Upcrtma4>)2rT-S*fR~*A~={?VqDT*A^D7|rJCWmhIqw_bp5VVy5+HW^bg=%&M~Up z9wcDT^gk3W1xoHhc*OpYWHTOb-MfTV{cRmiv-p6?PHZ6VOB=755Z#|}^^&leqo3mS z2^m(m@>%%;M-5JWFVVDv!&NUmIZ7s2xUK<N4TuA$^@hJ5kz z?q{*JcIC2UrFTy;$Xpo6%igO|>2Dgi)39wbeslmj#a&2BEM~IJX?|EK#g~DNQ1;tW zd+sELGsU=%j?i_OO_Ye!QBUj6&)YKSG>n`WRP ltSba#rbH)&uY59oK&k!`i zQCd6QpF5CDEY?ki^7weSN^Iv#?+%_P*hf#@>-ifX2IX8DwyTR;os#GP^|CHs`i%Un+7fyyC?CsGcK; z`7yxeTABjw{(NNRpv?E(BwOI;dA)GQK6wnVu+~-&LzjFQX!twDMn2dZ57(QwA4ZaQ zEIYdI-?NiF38Tc0AXdbEkRY4va}J_hSmcVu-Dmb=uNMqexy z7oT<%k9ZLBq#LiPIGPG<;+;ytmeO}ci>GIetLCMAvkzpbBqa9J*ixOj2MBr%9>Wn} zv>1m!MntP$mw7>s+~M_ubQY%&0fgLg4WX+yhaPs*g1lhQM2QbXfGYzBd$q^p_38u(qv97>8>PCy0lyN_`}Nj}|KEJpWz!P7-j&g+%l{Z_E#YNN*! z!3nC$X}G^aqRp}4fbf98R~t-p$aI)P#IPDm{>iwDV*mHqE2%65sH8}Xd&D0pQj0Va`oOB#XhKJNcH2Iiy%;$@P|tAvhdZVgY;og&-2HT9Vc@UK1U;BojzM3fT6V!#+gf(Il~n>HG)(A z;Dw2h+n^&?&TmF`*lui?u^9MggpR@Of}TdC$d|p#E{Bfwl-p}N@5h`qB>&Gi>__Xj z7$N!DBuL*t2KLhfmk1%Srk%XX*9WfGUln?5E?q+Evni0e;%U|&5JC39E-pfMg#Gd> zhG*N-?#8QI(9Q0KVo*2YIwo{IFT!7v9SCG6a?yATO>Om<{;^%gyEJ5KCv)d4EHon8 zo4s1B57q9C-P*eogzm7OSpOrvVT%uhpq{Z8oX)fx)>l72!3mxn1x#93OIldO_g zgyU44zP94Aw!YZb6!>9a-wg@9);82wh;=#46sG8;b+Fg7FVv;x`}&;$C5zDPxtLbm zBLt?&%F~oc3d`hXXtOi3&8?q+!EF$q#jS`B;X-wBIG=kdn@SMIljUA09P5`(k=#UU|TAw2%_EFZ4ulu znCwPiitFb!XnU{PDXg9$I;OJK>ZTfugf^m|C6SRg(VII?Qic~-#7JtDq0ewJ;dT0ZNS@E_0j)aZOw80q?lS8g0Z6&iepWY>WkPn`fFaEOzo!^jB*vA+y-dP}j*N|(T8dC*=;HQ{6<@H6PaG;O% zA?-J|n?~-I8Xd!IiLSCZqMY#kh?^>DFDRXddzDp(3X1n2LP24Fh8E{*d{;lpu*t(o z4<5a6xQ7{dZTYAe?qPj>`G5-g8U^|v8A`j^UfKUP_SHV%Qd#OwPz7!8b0YkW!n&vb zYb`!tY(*LLMN$8L1NjxC&;FWgbd6mYQ*S8B%tgpBYCn<-cmK0-_*v7ymRCu9!sdX zl(+mZP$7j+Ro-Heb)=PAN()ZDF^!3t@1UN%a)T&#NHdK~_A}D_b9#|tS%`6@qaj1> z+*JTgAax^SPB`H@|K-A|%ob_;q?>|n|5_kE(tgQ4MgHa7$Dw2`L7(MKe#W|>@8w#v zV*?9djy8ah3V>W?crj7#;y4-}CLsNfhcW67t_Ib&YMcgE}uryl&7+x!q931N-^SHGuQE(LoLa}mpkci3t*< zThQu7S!a#s?S{{u#Ydp&B7l6vg3j8Uvqc|0Zo}bQqd7lp1IC8Ts!;%p(ldK{IaXxQ zAP*{OY3nqbWmsG92;=!C-`(tL>NvnW^^vO<^-| z_!hNvA^HK@_Mvs4V<`&?J`#>BS2MlXI$OH-O*Gr(@Ld9r8F_Lsv)%)q-D+^p%qo3i zDC$16$UeOCNqQ5xtI(_}#@!eSY1C#25q=f^}= zLdWnEJa9}ZOQJO$_-4mC2Z~saRF%T%L0Tb6H67?lmMy4RGbgTBA$!V^ba4Q{R1zDB z8RFxqVl{eQJziL_njVBhR&#SEwt4EzsYCuN-l!3@Nt{eMnJwM4(uL>Nj-ql-Hk+VA z4Y`jYt0~kKNg2++hkbqMj>=+W3t}p8BOfXIGZaOIpBZ%?&Dqf;M#1r6j?ssnGZs#$ z0uZ_pek{v+V(NvTWZMj&5RAav8akMY0<)x-wc;L`mz96CFI!E46QL!#>iej=VprVo zjVh%N?3Lr8NCyb7wFN9aIAW9q27O7A1&nS`I&2t)Z-#(KQBE+WntZ=%ju%QdAJ!+G zZWQvK*^jfLe0|L|dDP{?^i`cZkeHEyqIk~TtI`66ZkHqxf#^86S4hC}r?prw=4fq2 z6+zydlR^zdEv*GlwlL*AU^ zLFhf}S_19zGKtq*Mm^!SB2_8p;oA}91={gDx>h}*o_9016T)srei5>+pv!K^2RsG2 z=vW_t$l5>CJ;`NABK`qH{tRt*Zdi!B67}So(LG^!+v9sx(!}3ThsLwMz;|hF$u~a7 zBj1mS^t#to$^Go{0M>dqocVA z{uyQQ{U1~o=meVg$8;S`^?COXtwNLd#5x(TJ<&se!6CzbU-!IxB35N7-1TeR?;=VF>buyi}CEh0(= zD{L&Ej0^xvX=^`MY1%p(qtJUOOHFGc;Gegkl{1P*VxG5ePFDr`UH%&jXx%xUh70mr zQr`*<)(y^4nLy{oBRzUAtR*&oEF}!jljhmQ*#-^McoLUoj@MhP)A75?>EV~ZK}}n3 z9_0ec_k7G5T~5h~J!@hZ@!y=Yur9P2#7_lQ%zFT-G(INXbWf5T(u?+K>ozrKydwJS z_t|kiC!On~iaClc#5btCV<_TNBH18jj9#<}Lw{8dJ-7fC3SGe@v{&{j9U1w#1tgFx zQwZHOjz&V>p;k}LBp%W@15xlAvKb`dv{c8iARPW6!q7-Fm`?sR7&<>Sh`F2nFXzY3 z128}#Prm*ySK?4PXT19jQA(2$3oWR#M>S7-oV+KnMLc1)9S~jn;P1YlF5=dsU*IX=O$D;CVc;M1 zpNkC$ii#?oZJvlz4x@HMr1t}UZw5bpkM{y@^$JMX!nj5Fd9V2etnX)Z0mSWoJRP~y zYjBp4$TbY5^c6iA`2zuaZW)o!QWo#jr#IM#6Xx~%+=92BuyZeYb2r9Uh`V$@3LgHc zle&Kl{Y?`*gE(Bt9iU+hSdW6%=<)adi_+?aZQuu?@cFyJ0&%xJ<~U&fC1oNda2XMB z)Z&~1ABu7~CRqn>|M5*r*oCL;3%lHZ8PiwA5yppYu@1V}^Ozh7os5h3$snmUvBh7c)q+aK9$6r`5 zp6f7c&2>G)mY^5b*cGsUCX2Pl$VPZ0eRfsVm|}cn-&cpJY1KR~LU36L^4PZ6%G?-7Zq%+iMiFGfh;4?_EuvSO~p&Mk=w{`OF zxsI^mkdJff-5;&yr(RjBl%{}a03SFkl1o>wC*@GDI(&F?H^`VT=i8R#VMk-V6{^wq zxgKV3==$>>=ur`s$ng5;hf|ej1u$<*dZ%+YhCDfjU{T1S+0xmUd#-kW%GnUU1h>K- zyiZj=sd6t=2fd&4OrsaGGc3u3_GLU)yyv5wh8PH?VyEZX!SwOn9sq*Xw z_5`Ag{!8d9IGsgtX6+A$TcqW=q<0)eZx7qsinmLHxZ*C3aI^zNx{Dc`R~V7}7q7~# z{kXE2Fo&2;pW}@%wp`@@DeKDPq57gYV~H`0FqjBq5VA!WAzK*PE7`JT&z2=yn1m?1 zQCYK%rVz43T9E8S%Wn!XwhXc*OQHOpDWc||nRnj#o_o%__r1@&cklatYiaC3Z{O%x zr}}i??46Ax6b@y&gqT!|4!!>SsUJ>1$V( zfOQ5yBlnZa&}rF^E+biBj!mEEK;qMlJEFL`(-owHDd(Mu2+;|mChaYnmC+6;%`WnkBIWzga^UH{%04o;0OsvJtjGc@CLZen2-Y#T zI_E9*kDb)bp$-ATT%jX4FqoenZK4O$pkW~4Eppn zc-3?MD}ia1+alIM4CNO>`^jR!%|btsx61u|5vLe^2yV-`Q%KYnuS_+B)jK6Rn&f}fx1Vw|ud|FXZ;w-{n2 zu3&ns&v!CxmA~jjlcuKF^lUmCFxT2H}N=Kl%k_tpl zZ+Uhk6c3i=?f1wb-glW=Xy~ zr=iPz6HuLU-(ahfGKlnEye!PCX&Fgb6%=EK8%u&X#Y$aMQO(6%WIh+oH3`?Mc`3vm zT*R2BqPppSTug$RV>I0Niw7siC~Rh~z|(h)ZHa+RxyvAT!T^=WdFeVm+v{h&Evz5B zVFE%CdZKD`FO*?uJz9~1mKjQQs0NpzsM=AugBHDy4P$;gpYOf3T!?fT zwa9%nTE`^KcVf#%6z%j$6)XaX2qn|2X%~_rAr&_=s(JW=D5W1z>vZs$2)0qlqEFHG zsdi`b@X=68vtDred)`WW;~o@7PeH*AOb`f@CkHTZIWVgjRuX*kj`;WSF*9jecgX&h zD@pDUYP!R?4{4(WjcWH8!~+Fg=XF(r2%dGAE#vhYKipxDjHEfg-t;tf_XF~Ito?rT z&(@vC->w7=DdMmw;U8-A!FN{IXGqG=7Uwx^Xv=9fQ@+H+X=?=PslkRs$2`DSQFzrC z1K-EJ;h1^4k!7fE!#H;^UNSA{Gd7<;9Sl;0gox5RDk3)!iKRBCeD->I>M@iNr^U$THx-;P-y2+*g^Mc=-Ki2sDn7S-v0)&S zSyNpSitc3dqmOr_IJll-#S`gs{ycG54eA{PKN572*mlcfKs!a_8OyDMQwDJY`_}i> ziSEZJr-D&}&glw+6^zP~cJ~W{8|9>Dua7F+ug`sXsi7_DCj+-ZlRHV-`yH-pChdrX zokpT-d`+UI#j%2#oQf*$K{u6|uI#ILTh~W+PE)4_(_E9Y#)3PByv&Yf5K)q*;`nXgi_%{tf62 zH>N_`4`F{{D)NxHy4Ecg^Y=or{zsL=Dvh?7Su_dynKunPfYZb?E$q)Kt4 z>Gf+^h3(q!L4S&OyfB4#I3iOxNrLTzdTssrOMa*llstCg5 zCRNAlJTtq%VquldFSyXgDZ*-AFIRs&Mb0%-b~Yv~qsSR~Q=h&Vld5?0#{B4IvDpK% zIf^Abu(GDlCfp5fCfp}#!10%quNH`wj;#v_%!>TSBwkkhjndjUxm5?I;-KTmP2YXT z=nHb<+Y#2!&t8s4YDpSbNZlVj#MI&n!>BmQEhCV(al(@ckvT_}NsiR}zH#V%6J(tZ zsS^w;e=Fwj=7sLZe13cg_l0!qi#)Er)Agsk<2-)IQf<=Kn|x8U{=uVt-?*?d!{r)G zh&LxH6lwl3L8lBaAD|SPsJWrm%=_gK3^Sp}*=h{O7=JO{&~G7{Y*1~cmY4O!#b8b( zW=YRnZ=(MkB>W|VW=HfQYIqr6Om#UWtumBhCnd5ym5;37TXzuLfNI`vsQ~= z4$P~WoiQ(AA$)u8w)B|pofKZrFkU3Q{N>&1_|pXmQM^}bPG?%O4;dW`d} zrH+9Yb?n5%8(@ux89xzS1sL^emj>&P+E3p1mVz@~>U%0}wAPt=!f0j{mDf$Gi1U5& zZ=ZKo+QdrLxLHI$wvSB@MsUKZ z#Lh|-ShSg`$}l72P#L`w$^T7fB(ps|#j3w#drZA>eMl*f$Kid=Foj}tp*{YauM8{AH8ZenMqaeL zvZCuv7agUGpK6ifJ3E8cDM4uzVIp;J=TECppVDu{8VkOyKbukD*HNYv5BwpLDt7Ti zkeV;I6+C#a;3e!Vk>^S4;a8FD<4imI_RA@B6TftJWG(Lo{cF8N)`Sx+KFi=y38gFY zDb^+WbZ?4XZne+Jzm`zNBVUG=+-s*Ah4534EZl{=KW7qTA37Q<9!Rc#!S{q=LqpzR`@UtM7L9|F11)Xmq&ytN~%tcJ=UkL zy5;a6$EVW#cjIRs6dH-~>DZ;182Tc+^~cWNbaj)s_E?_L}9Ydd^9m{ZB^o-d(4w0McMR>I5z}c{L#@lcBiv(^wSt> zWKC3qmPTRaLBi7D6OhLj+KxD7j##vTszFksjG5l!t{BT~%jR>8tRAP(^hk!&4tksl zO6UOCsR90Inj1_pEZN7OpQBo-?K^OzxsU?pUdX6m$bRhxi-vHFLL&bgyOiP*CpRY3U#a`F=_N{yg@luJW8T`~M_Q)YUXUb#; zaFLCsSxdUDxOcXAu#B3RYgbLi3FH&`Nx6yI|H_X$N?BW5Rss74A>-y z3|4A*Ykjo*FcK8O1Q_7hOT(goAr~+lWyOvNb7Lo2IRF8@U-@nV<_%ITqK07pe0x+k`x&Y%xJy4e7 zHVF8}g8}|aw`jC{JS`k$kLiv z#5fEjAWhzFIgSwgQ|OjdVG&U1OJHf(-e^c{-niDTL$W0x_XO}P0zBpSv@47v2bTK8 zy({pRq8y%*f}#L8AgDd=D6IDQ}Z95F`jSKI;v;)Be z2>mkv{^tDI#^S;~f_H%BpLut+wTDz>+n?N)ZP;%J(@xgBGe}Td(oQ5kEc?!`Fkn}0 zGI>|+j1bU=M8Hs=Mm9h*{}g^~7Pv@elFfdzuw9Ald5L6CA|6>}_pNtII%jtur@z;H zZliGFRE0Ybt~vOwR`#sL+x91y+m8PYVce-HVh#x!d*6I(fn=qi;OgJn`X8G3)Bykh diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661e..bdc9a83b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index a69d9cb6..79a61d42 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,10 +80,10 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' @@ -143,12 +143,16 @@ fi if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac diff --git a/gradlew.bat b/gradlew.bat index 53a6b238..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% diff --git a/radar-commons-server/build.gradle.kts b/radar-commons-server/build.gradle.kts index 79c59a62..24515ad1 100644 --- a/radar-commons-server/build.gradle.kts +++ b/radar-commons-server/build.gradle.kts @@ -15,7 +15,9 @@ */ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile -apply(plugin = "com.github.davidmc24.gradle.plugin.avro") +plugins { + id("com.github.davidmc24.gradle.plugin.avro") +} description = "RADAR Common server library utilities." diff --git a/settings.gradle.kts b/settings.gradle.kts index 2fdfd50d..0fca0917 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,3 +17,21 @@ include(":radar-commons") include(":radar-commons-testing") include(":radar-commons-server") + +pluginManagement { + plugins { + val kotlinVersion: String by settings + kotlin("jvm") version kotlinVersion + kotlin("plugin.serialization") version kotlinVersion + + val avroPluginVersion: String by settings + id("com.github.davidmc24.gradle.plugin.avro") version avroPluginVersion + val nexusPluginVersion: String by settings + id("io.github.gradle-nexus.publish-plugin") version nexusPluginVersion + + val dependencyUpdatePluginVersion: String by settings + id("com.github.ben-manes.versions") version dependencyUpdatePluginVersion + val dokkaVersion: String by settings + id("org.jetbrains.dokka") version dokkaVersion + } +} From 49a81519e9538da57809c27d11746e8d056320b6 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 14 Mar 2023 16:16:13 +0100 Subject: [PATCH 07/25] Added radar-commons-kotlin package --- radar-commons-kotlin/.gitignore | 2 + radar-commons-kotlin/build.gradle.kts | 12 + .../kotlin/coroutines/CacheConfig.kt | 26 ++ .../radarbase/kotlin/coroutines/CachedMap.kt | 67 +++++ .../radarbase/kotlin/coroutines/CachedSet.kt | 46 ++++ .../kotlin/coroutines/CachedValue.kt | 231 ++++++++++++++++++ .../radarbase/kotlin/coroutines/Extensions.kt | 136 +++++++++++ .../kotlin/coroutines/CachedValueTest.kt | 179 ++++++++++++++ .../kotlin/coroutines/ExtensionsKtTest.kt | 88 +++++++ settings.gradle.kts | 1 + 10 files changed, 788 insertions(+) create mode 100644 radar-commons-kotlin/.gitignore create mode 100644 radar-commons-kotlin/build.gradle.kts create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CacheConfig.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt create mode 100644 radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt create mode 100644 radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt diff --git a/radar-commons-kotlin/.gitignore b/radar-commons-kotlin/.gitignore new file mode 100644 index 00000000..3c0160d0 --- /dev/null +++ b/radar-commons-kotlin/.gitignore @@ -0,0 +1,2 @@ +build/ +out/ diff --git a/radar-commons-kotlin/build.gradle.kts b/radar-commons-kotlin/build.gradle.kts new file mode 100644 index 00000000..f965f6ad --- /dev/null +++ b/radar-commons-kotlin/build.gradle.kts @@ -0,0 +1,12 @@ +description = "Library for Kotlin utility classes and functions" + +dependencies { + val slf4jVersion: String by project + implementation("org.slf4j:slf4j-api:$slf4jVersion") + + val coroutinesVersion: String by project + api(platform("org.jetbrains.kotlinx:kotlinx-coroutines-bom:$coroutinesVersion")) + api("org.jetbrains.kotlinx:kotlinx-coroutines-core") + + testImplementation("org.hamcrest:hamcrest:2.2") +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CacheConfig.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CacheConfig.kt new file mode 100644 index 00000000..59a6aa3b --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CacheConfig.kt @@ -0,0 +1,26 @@ +package org.radarbase.kotlin.coroutines + +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds + +data class CacheConfig( + /** Duration after which the cache is considered stale and should be refreshed. */ + val refreshDuration: Duration = 30.minutes, + /** Duration after which the cache may be refreshed if the cache does not fulfill a certain + * requirement. This should be shorter than [refreshDuration] to have effect. */ + val retryDuration: Duration = 1.minutes, + /** Time until the result may be recomputed when an exception is set for the cache. */ + val exceptionCacheDuration: Duration = 10.seconds, + /** + * Number of simultaneous computations that may occur. Increase if the time to computation + * is very variable. + */ + val maxSimultaneousCompute: Int = 1, +) { + init { + require(retryDuration > Duration.ZERO) { "Cache fetch duration $retryDuration must be positive" } + require(refreshDuration >= retryDuration) { "Cache maximum age $refreshDuration must be at least fetch timeout $retryDuration" } + require(maxSimultaneousCompute > 0) { "At least one context must be able to compute the result" } + } +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt new file mode 100644 index 00000000..52402515 --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2020 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.radarbase.kotlin.coroutines + +/** Set of data that is cached for a duration of time. */ +class CachedMap( + cacheConfig: CacheConfig = CacheConfig(), + supplier: suspend () -> Map, +): CachedValue>(cacheConfig, supplier) { + /** Whether the cache contains [key]. If it does not contain the value and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. */ + suspend fun contains(key: K): Boolean = test { key in it } + + /** + * Find a pair matching [predicate]. + * If it does not contain the value and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. + * @return value if found and null otherwise + */ + suspend fun find(predicate: (K, V) -> Boolean): Pair? = query( + { map -> + map.entries + .find { (k, v) -> predicate(k, v) } + ?.toPair() + }, + { it != null }, + ).value + + /** + * Find a pair matching [predicate]. + * If it does not contain the value and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. + * @return value if found and null otherwise + */ + suspend fun findValue(predicate: (V) -> Boolean): V? = query( + { map -> map.values.find { predicate(it) } }, + { it != null }, + ).value + + /** + * Get the value. + * If the cache is empty and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. + */ + override suspend fun get(): Map = get { it.isNotEmpty() }.value + + /** + * Get the value. + * If the cache is empty and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. + */ + suspend fun get(key: K): V? = query({ it[key] }, { it != null }).value +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt new file mode 100644 index 00000000..361bd89d --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2020 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.radarbase.kotlin.coroutines + +/** + * Set of data that is cached for a duration of time. + * + * @param supplier How to update the cache. + */ +class CachedSet( + cacheConfig: CacheConfig = CacheConfig(), + supplier: suspend () -> Set, +): CachedValue>(cacheConfig, supplier) { + /** Whether the cache contains [value]. If it does not contain the value and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. */ + suspend fun contains(value: T): Boolean = test { value in it } + + /** + * Find a value matching [predicate]. + * If it does not contain the value and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. + * @return value if found and null otherwise + */ + suspend fun find(predicate: (T) -> Boolean): T? = query({ it.find(predicate) }, { it != null }).value + + /** + * Get the value. + * If the cache is empty and [CacheConfig.retryDuration] + * has passed since the last try, it will update the cache and try once more. + */ + override suspend fun get(): Set = get { it.isNotEmpty() }.value +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt new file mode 100644 index 00000000..cc507fac --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt @@ -0,0 +1,231 @@ +package org.radarbase.kotlin.coroutines + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.sync.Semaphore +import java.util.concurrent.atomic.AtomicReference +import kotlin.time.Duration +import kotlin.time.ExperimentalTime +import kotlin.time.TimeMark +import kotlin.time.TimeSource + +internal typealias DeferredCache = CompletableDeferred> + +/** + * Caches a value with full support for coroutines. The value that will be cached is computed by + * [supplier]. + * Only one coroutine context will compute the value at a time, other coroutine contexts will wait + * for it to finish. + */ +open class CachedValue( + private val config: CacheConfig, + private val supplier: suspend () -> T, +) { + private val cache = AtomicReference>>() + private val semaphore: Semaphore? = if (config.maxSimultaneousCompute > 1) { + Semaphore(config.maxSimultaneousCompute - 1) + } else { + null + } + + /** + * Query the cached value by running [transform] and return its result if valid. If + * [evaluateValid] returns false on the result, the cache computation is reevaluated if + * [CacheConfig.retryDuration] has been reached. + */ + suspend fun query( + transform: suspend (T) -> R, + evaluateValid: (R) -> Boolean = { true }, + ): CacheResult { + val deferredResult = raceForDeferred() + val deferred = deferredResult.value + + return if (deferredResult is CacheMiss) { + val result = deferred.computeAndCache() + CacheMiss(transform(result)) + } else { + val concurrentResult = deferred.concurrentComputeAndCache() + if (concurrentResult != null) { + CacheMiss(transform(concurrentResult)) + } else { + deferred.awaitCache(transform, evaluateValid) + } + } + } + + /** + * Whether the contained value is stale. + * If [duration] is provided, it is considered stale only if the value is older than [duration]. + * If no value is cache, it is not considered stale. + */ + suspend fun isStale(duration: Duration? = null): Boolean { + val currentDeferred = cache.get() + if (currentDeferred == null || !currentDeferred.isCompleted) { + return false + } + val result = currentDeferred.await() + return if (duration == null) { + result.isExpired() + } else { + result.isExpired(duration) + } + } + + /** + * Get cached value. If the cache is expired, fetch it again. The first coroutine context + * that reaches this method will call [computeAndCache], others coroutine contexts will use the + * value computed by the first. The result is not computed more + * often than [CacheConfig.retryDuration]. If the result was an exception, the exception is + * rethrown from cache. It is recomputed if the [CacheConfig.exceptionCacheDuration] has passed. + */ + open suspend fun get(): T = query({ it }) { false }.value + + /** + * Get cached value. If the cache is expired, fetch it again. The first coroutine context + * that reaches this method will call [computeAndCache], others coroutine contexts will use the + * value computed by the first. If the value was retrieved from cache and [evaluateValid] + * returns false for that value, the result is recomputed. The result is not computed more + * often than [CacheConfig.retryDuration]. If the result was an exception, the exception is + * rethrown from cache. It is recomputed if the [CacheConfig.exceptionCacheDuration] has passed. + */ + suspend inline fun get(noinline evaluateValid: (T) -> Boolean): CacheResult = query({ it }, evaluateValid) + + /** + * Test the cached value by running [predicate] and return its result if true. If + * [predicate] returns false on the result, the cache computation is reevaluated if + * [CacheConfig.retryDuration] has been reached. + */ + suspend inline fun test(noinline predicate: (T) -> Boolean): Boolean { + return query(predicate) { it }.value + } + + private suspend fun DeferredCache.computeAndCache(): T { + val result = try { + val value = supplier() + complete(CacheValue(value)) + value + } catch (ex: Throwable) { + complete(CacheError(ex)) + throw ex + } + return result + } + + private suspend fun DeferredCache.concurrentComputeAndCache(): T? { + if (isCompleted) return null + + return semaphore?.tryWithPermitOrNull { + if (isCompleted) { + null + } else { + computeAndCache() + } + } + } + + private suspend fun DeferredCache.awaitCache( + transform: suspend (T) -> R, + evaluateValid: (R) -> Boolean, + ): CacheResult { + val result = await().map(transform) + return if (result.isExpired(evaluateValid)) { + // Either no new coroutine context had updated the cache value, then update it to + // null. Otherwise, another suspend context is active and get() will await the + // result from that context + cache.compareAndSet(this, null) + query(transform) { false } + } else { + val value = result.getOrThrow() + CacheHit(value) + } + } + + /** + * Race for the first suspend context to create a CompletableDeferred object. All other contexts + * will use that context to read their values. + * + * @return a pair of a CompletableDeferred value and a boolean, if true this context is the + * winner, if false this should use the deferred to read its value. + */ + private fun raceForDeferred(): CacheResult> { + var result: CacheResult> + + do { + val previousDeferred = cache.get() + result = if (previousDeferred == null) { + CacheMiss(CompletableDeferred()) + } else { + CacheHit(previousDeferred) + } + } while (!cache.compareAndSet(previousDeferred, result.value)) + + return result + } + + private inline fun CacheContents.isExpired( + evaluateValid: (R) -> Boolean = { true } + ): Boolean = if (this is CacheError) { + isExpired(config.exceptionCacheDuration) + } else { + this as CacheValue + isExpired(config.refreshDuration) || + (!evaluateValid(value) && isExpired(config.retryDuration)) + } + + /** + * Remove value from cache. Note that this does not cancel existing computations for the + * value, but the computed value will then not be stored. + */ + fun clear() { + cache.set(null) + } + + @OptIn(ExperimentalTime::class) + internal sealed class CacheContents( + time: TimeMark? = null, + ) { + protected val time: TimeMark = time ?: TimeSource.Monotonic.markNow() + + open fun isExpired(age: Duration): Boolean = (time + age).hasPassedNow() + + abstract fun getOrThrow(): T + + @Suppress("UNCHECKED_CAST") + abstract suspend fun map(transform: suspend (T) -> R): CacheContents + } + + @OptIn(ExperimentalTime::class) + internal class CacheError( + val exception: Throwable, + ) : CacheContents() { + override fun isExpired(age: Duration): Boolean = exception is CancellationException || super.isExpired(age) + override fun getOrThrow(): T = throw exception + @Suppress("UNCHECKED_CAST") + override suspend fun map(transform: suspend (T) -> R): CacheContents = this as CacheError + } + + @OptIn(ExperimentalTime::class) + internal class CacheValue( + val value: T, + time: TimeMark? = null, + ) : CacheContents(time) { + override fun getOrThrow(): T = value + + override suspend fun map(transform: suspend (T) -> R): CacheContents = try { + CacheValue(transform(value), time = time) + } catch (ex: Throwable) { + CacheError(ex) + } + } + + /** Result from cache of type [T]. */ + sealed interface CacheResult { + val value: T + } + + /** Cache hit, meaning the value was computed by another coroutine. */ + data class CacheHit(override val value: T) : CacheResult + + /** Cache miss, meaning the value was computed by the current coroutine. */ + data class CacheMiss(override val value: T) : CacheResult +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt new file mode 100644 index 00000000..87cbf27f --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt @@ -0,0 +1,136 @@ +@file:Suppress("unused") + +package org.radarbase.kotlin.coroutines + +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.consume +import kotlinx.coroutines.sync.Semaphore +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.EmptyCoroutineContext +import kotlin.time.Duration + +/** + * Try to acquire a semaphore permit, and run [block] if successful. + * If this cannot be achieved without blocking, return null. + * @return result of [block] or null if no permit could be acquired. + */ +suspend fun Semaphore.tryWithPermitOrNull(block: suspend () -> T): T? { + if (!tryAcquire()) return null + return try { + block() + } finally { + release() + } +} + +/** + * Get a future value via coroutine suspension. + * The future is evaluated in context [Dispatchers.IO]. + */ +suspend fun Future.suspendGet( + duration: Duration? = null, +): T = coroutineScope { + val channel = Channel() + launch { + try { + channel.receive() + } catch (ex: CancellationException) { + cancel(true) + } + } + try { + withContext(Dispatchers.IO) { + if (duration != null) { + get(duration.inWholeMilliseconds, TimeUnit.MILLISECONDS) + } else { + get() + } + } + } catch (ex: InterruptedException) { + throw CancellationException("Future was interrupted", ex) + } finally { + channel.send(Unit) + } +} + +/** + * Transform each value in the iterable in a separate coroutine and await termination. + */ +suspend inline fun Iterable.forkJoin( + coroutineContext: CoroutineContext = Dispatchers.Default, + crossinline transform: suspend CoroutineScope.(T) -> R +): List = coroutineScope { + map { t -> async(coroutineContext) { transform(t) } } + .awaitAll() +} + +/** + * Consume the first value produced by the producer on its provided channel. Once a value is sent + * by the producer, its coroutine is cancelled. + * @throws kotlinx.coroutines.channels.ClosedReceiveChannelException if the producer does not + * produce any values. + */ +suspend inline fun consumeFirst( + coroutineContext: CoroutineContext = Dispatchers.Default, + crossinline producer: suspend CoroutineScope.(emit: suspend (T) -> Unit) -> Unit +): T = coroutineScope { + val channel = Channel() + + val producerJob = launch(coroutineContext) { + try { + producer(channel::send) + } finally { + channel.close() + } + } + + val result = channel.consume { receive() } + producerJob.cancel() + result +} + +/** + * Transforms each value with [transform] and returns the first value where [predicate] returns + * true. Each value is transformed and evaluated in its own async context. If no transformed value + * satisfies predicate, null is returned. + */ +suspend fun Iterable.forkFirstOfOrNull( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + transform: suspend CoroutineScope.(T) -> R, + predicate: suspend CoroutineScope.(R) -> Boolean, +): R? = consumeFirst(coroutineContext) { emit -> + forkJoin(coroutineContext) { t -> + val result = transform(t) + if (predicate(result)) { + emit(result) + } + } + emit(null) +} + +suspend fun Iterable.forkFirstOfNotNullOrNull( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + transform: suspend CoroutineScope.(T) -> R? +): R? = forkFirstOfOrNull(coroutineContext, transform) { it != null } + +/** + * Returns true as soon as [predicate] returns true on a value, or false if [predicate] does + * not return true on any of the values. All values are evaluated in a separate async context using + * [forkJoin]. + */ +suspend fun Iterable.forkAny( + coroutineContext: CoroutineContext = EmptyCoroutineContext, + predicate: suspend CoroutineScope.(T) -> Boolean +): Boolean = forkFirstOfOrNull(coroutineContext, predicate) { it } ?: false + +operator fun Set.plus(elements: Set): Set = when { + isEmpty() -> elements + elements.isEmpty() -> this + else -> buildSet(size + elements.size) { + addAll(this) + addAll(elements) + } +} diff --git a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt new file mode 100644 index 00000000..f9d1400e --- /dev/null +++ b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt @@ -0,0 +1,179 @@ +package org.radarbase.kotlin.coroutines + +import kotlinx.coroutines.* +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.slf4j.LoggerFactory +import java.time.Duration +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.ExperimentalTime +import kotlin.time.TimeMark +import kotlin.time.TimeSource + +@OptIn(ExperimentalTime::class, DelicateCoroutinesApi::class) +internal class CachedValueTest { + private lateinit var config: CacheConfig + + private val calls: AtomicInteger = AtomicInteger(0) + + @BeforeEach + fun setUp() { + calls.set(0) + config = CacheConfig( + refreshDuration = 20.milliseconds, + retryDuration = 10.milliseconds, + exceptionCacheDuration = 10.milliseconds + ) + } + + @Test + fun get() { + val cache = CachedValue(config) { calls.incrementAndGet() } + runBlocking(GlobalScope.coroutineContext) { + assertThat("Initial value should refresh", cache.get(), `is`(1)) + assertThat("No refresh within threshold", cache.get(), `is`(1)) + delay(10) + assertThat("Refresh after threshold", cache.get(), `is`(2)) + assertThat("No refresh after threshold", cache.get(), `is`(2)) + } + } + + @Test + fun getInvalid() { + val cache = CachedValue(config) { calls.incrementAndGet() } + runBlocking { + assertThat("Initial value should refresh", cache.get { it < 0 }, equalTo(CachedValue.CacheMiss(1))) + assertThat("No refresh within threshold", cache.get { it < 0 }, equalTo(CachedValue.CacheHit(1))) + delay(10) + assertThat("Refresh after threshold", cache.get { it < 0 }, equalTo(CachedValue.CacheMiss(2))) + assertThat("No refresh after threshold", cache.get { it < 0 }, equalTo(CachedValue.CacheHit(2))) + } + } + + @Test + fun getValid() { + val cache = CachedValue(config) { calls.incrementAndGet() } + runBlocking { + assertThat("Initial value should refresh", cache.get { it >= 0 }, equalTo(CachedValue.CacheMiss(1))) + assertThat("No refresh within threshold", cache.get { it >= 0 }, equalTo(CachedValue.CacheHit(1))) + delay(10) + assertThat("No refresh after valid value", cache.get { it >= 0 }, equalTo(CachedValue.CacheHit(1))) + } + } + + @Test + fun refresh() { + val cache = CachedValue(config) { calls.incrementAndGet() } + + runBlocking { + assertThat("Initial get calls supplier", cache.get(), `is`(1)) + assertThat("Next get uses cache", cache.get(), `is`(1)) + cache.clear() + assertThat("Next get uses cache", cache.get(), `is`(2)) + } + } + + @Test + fun query() { + val cache = CachedValue(config) { calls.incrementAndGet() } + + runBlocking { + assertThat("Initial value should refresh", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheMiss(2))) + assertThat("No refresh within threshold", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheHit(2))) + delay(10) + assertThat( + "Retry because predicate does not match", + cache.query({ it + 1 }, { it > 2 }), + equalTo(CachedValue.CacheMiss(3)) + ) + assertThat("No refresh within threshold", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheHit(3))) + delay(10) + assertThat( + "No retry because predicate matches", + cache.query({ it + 1 }, { it > 2 }), + equalTo(CachedValue.CacheHit(3)) + ) + delay(10) + assertThat( + "Refresh after refresh threshold since last retry", + cache.query({ it + 1 }, { it > 2 }), + equalTo(CachedValue.CacheMiss(4)) + ) + } + } + + + @Test + fun getMultithreaded() { + val cache = CachedValue(config) { + calls.incrementAndGet() + delay(50.milliseconds) + calls.get() + } + + runBlocking { + (0 .. 5) + .forkJoin { + cache.get() + } + .forEach { + assertThat("Get the same value in all contexts", it, `is`(1)) + } + } + + assertThat("No more calls are made", calls.get(), `is`(1)) + } + + @Test + fun getMulti2threaded() { + val cache = CachedValue(config.copy( + maxSimultaneousCompute = 2 + )) { + calls.incrementAndGet() + delay(50.milliseconds) + calls.get() + } + + runBlocking { + val values = (0 .. 5) + .forkJoin { + cache.get() + } + + assertThat(values[0], lessThan(3)) + values.forEach { + assertThat("Get the same value in all contexts", it, `is`(values[0])) + } + } + + assertThat("Two threads should be computing the value", calls.get(), `is`(2)) + } + + + @Test + fun throwTest() { + val cache = CachedValue(config.copy(refreshDuration = 20.milliseconds)) { + val newValue = calls.incrementAndGet() + if (newValue % 2 == 0) throw IllegalStateException() else newValue + } + + runBlocking { + assertThat(cache.get(), `is`(1)) + assertThat(cache.get(), `is`(1)) + delay(21.milliseconds) + assertThrows { cache.get() } + assertThrows { cache.get() } + delay(11.milliseconds) + assertThat(cache.get(), `is`(3)) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(CachedValueTest::class.java) + } +} diff --git a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt new file mode 100644 index 00000000..7cb9a18c --- /dev/null +++ b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt @@ -0,0 +1,88 @@ +package org.radarbase.kotlin.coroutines + +import kotlinx.coroutines.* +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.greaterThan +import org.hamcrest.Matchers.lessThan +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.fail +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.ExperimentalTime +import kotlin.time.measureTime + +@OptIn(ExperimentalTime::class) +class ExtensionsKtTest { + companion object { + @BeforeAll + @JvmStatic + fun setUpClass() { + runBlocking { + println("warmed up coroutines") + } + } + } + + @Test + fun testConsumeFirst() = runBlocking { + val inBlockingTime = measureTime { + val first = consumeFirst { emit -> + listOf( + async(Dispatchers.Default) { + delay(200.milliseconds) + emit("a") + fail("Should be cancelled") + }, + async(Dispatchers.Default) { + delay(50.milliseconds) + emit("b") + }, + ).awaitAll() + } + assertEquals("b", first) + } + assertThat(inBlockingTime, greaterThan(50.milliseconds)) + assertThat(inBlockingTime, lessThan(200.milliseconds)) + } + + @Test + fun testForkJoin() = runBlocking { + val inBlockingTime = measureTime { + val result = listOf(100.milliseconds, 50.milliseconds) + .forkJoin { + delay(it) + it + } + assertEquals(listOf(100.milliseconds, 50.milliseconds), result) + } + assertThat(inBlockingTime, greaterThan(100.milliseconds)) + } + + + @Test + fun testForkJoinFirst() = runBlocking { + val inBlockingTime = measureTime { + val result: Duration? = consumeFirst { emit -> + listOf(200.milliseconds, 50.milliseconds) + .forkJoin { + delay(it) + emit(it) + } + emit(null) + } + assertEquals(50.milliseconds, result) + } + assertThat(inBlockingTime, lessThan(200.milliseconds)) + assertThat(inBlockingTime, greaterThan(50.milliseconds)) + } + + @Test + fun testConcurrentAny() { + runBlocking { + assertTrue(listOf(1, 2, 3, 4).forkAny { it > 3 }) + assertFalse(listOf(1, 2, 3, 4).forkAny { it < 1 }) + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 0fca0917..b5556838 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,6 +15,7 @@ */ include(":radar-commons") +include(":radar-commons-kotlin") include(":radar-commons-testing") include(":radar-commons-server") From 848d4a97bd74800526ad02a4cb7f25bdd6cab595 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 16 Mar 2023 12:04:26 +0100 Subject: [PATCH 08/25] Removed dependency on MP and TimedValue --- radar-commons-kotlin/build.gradle.kts | 10 + .../kotlin/coroutines/CachedValue.kt | 49 +++- .../radarbase/ktor/auth/AuthTokenHolder.kt | 51 ++++ .../ktor/auth/ClientCredentialsConfig.kt | 24 ++ .../radarbase/ktor/auth/OAuth2AccessToken.kt | 23 ++ .../ktor/auth/OAuthClientProvider.kt | 177 ++++++++++++++ radar-commons-testing/build.gradle.kts | 1 + .../java/org/radarbase/mock/MockProducer.kt | 103 ++++---- .../java/org/radarbase/util/Oscilloscope.kt | 14 +- radar-commons/build.gradle.kts | 5 +- .../producer/io/HttpClientExtensions.kt | 4 +- .../producer/rest/ConnectionState.kt | 53 +++-- .../producer/rest/RestKafkaSender.kt | 39 ++- .../producer/schema/SchemaRestClient.kt | 8 +- .../producer/schema/SchemaRetriever.kt | 224 +++++++----------- .../main/java/org/radarbase/util/TimedInt.kt | 19 -- .../java/org/radarbase/util/TimedValue.kt | 19 -- .../java/org/radarbase/util/TimedVariable.kt | 21 -- .../java/org/radarbase/util/TimeoutConfig.kt | 13 - .../producer/rest/ConnectionStateTest.kt | 34 +-- .../producer/rest/RestKafkaSenderTest.kt | 10 +- .../producer/schema/SchemaRestClientTest.kt | 4 +- .../producer/schema/SchemaRetrieverTest.kt | 57 +---- 23 files changed, 556 insertions(+), 406 deletions(-) create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/ClientCredentialsConfig.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuth2AccessToken.kt create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedInt.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedValue.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt diff --git a/radar-commons-kotlin/build.gradle.kts b/radar-commons-kotlin/build.gradle.kts index f965f6ad..4b84bf4b 100644 --- a/radar-commons-kotlin/build.gradle.kts +++ b/radar-commons-kotlin/build.gradle.kts @@ -1,3 +1,7 @@ +plugins { + kotlin("plugin.serialization") +} + description = "Library for Kotlin utility classes and functions" dependencies { @@ -8,5 +12,11 @@ dependencies { api(platform("org.jetbrains.kotlinx:kotlinx-coroutines-bom:$coroutinesVersion")) api("org.jetbrains.kotlinx:kotlinx-coroutines-core") + val ktorVersion: String by project + api(platform("io.ktor:ktor-bom:$ktorVersion")) + api("io.ktor:ktor-client-auth") + implementation("io.ktor:ktor-client-content-negotiation") + implementation("io.ktor:ktor-serialization-kotlinx-json") + testImplementation("org.hamcrest:hamcrest:2.2") } diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt index cc507fac..7e8a0dbb 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt @@ -2,8 +2,10 @@ package org.radarbase.kotlin.coroutines import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.isActive import kotlinx.coroutines.sync.Semaphore import java.util.concurrent.atomic.AtomicReference +import kotlin.coroutines.coroutineContext import kotlin.time.Duration import kotlin.time.ExperimentalTime import kotlin.time.TimeMark @@ -18,7 +20,7 @@ internal typealias DeferredCache = CompletableDeferred( - private val config: CacheConfig, + val config: CacheConfig, private val supplier: suspend () -> T, ) { private val cache = AtomicReference>>() @@ -59,15 +61,40 @@ open class CachedValue( * If no value is cache, it is not considered stale. */ suspend fun isStale(duration: Duration? = null): Boolean { + val result = getFromCache() + return when { + result == null -> false + duration != null -> result.isExpired(duration) + else -> result.isExpired() + } + } + + /** + * Get the current contents from cache. This will not cause a computation. + * @return contents if it is present in cache, null otherwise. + */ + suspend fun getFromCache(): CacheContents? { val currentDeferred = cache.get() if (currentDeferred == null || !currentDeferred.isCompleted) { - return false + return null } - val result = currentDeferred.await() - return if (duration == null) { - result.isExpired() - } else { - result.isExpired(duration) + return currentDeferred.await() + } + + /** + * Set cached value. + */ + suspend fun set(value: T) { + while (coroutineContext.isActive) { + val deferred = raceForDeferred().value + + val newValue = CacheValue(value) + deferred.complete(newValue) + if (deferred.await() == newValue) { + return + } else { + cache.compareAndSet(deferred, null) + } } } @@ -162,7 +189,7 @@ open class CachedValue( return result } - private inline fun CacheContents.isExpired( + inline fun CacheContents.isExpired( evaluateValid: (R) -> Boolean = { true } ): Boolean = if (this is CacheError) { isExpired(config.exceptionCacheDuration) @@ -181,7 +208,7 @@ open class CachedValue( } @OptIn(ExperimentalTime::class) - internal sealed class CacheContents( + sealed class CacheContents( time: TimeMark? = null, ) { protected val time: TimeMark = time ?: TimeSource.Monotonic.markNow() @@ -195,7 +222,7 @@ open class CachedValue( } @OptIn(ExperimentalTime::class) - internal class CacheError( + class CacheError internal constructor( val exception: Throwable, ) : CacheContents() { override fun isExpired(age: Duration): Boolean = exception is CancellationException || super.isExpired(age) @@ -205,7 +232,7 @@ open class CachedValue( } @OptIn(ExperimentalTime::class) - internal class CacheValue( + class CacheValue internal constructor( val value: T, time: TimeMark? = null, ) : CacheContents(time) { diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt new file mode 100644 index 00000000..9c1dd8c7 --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt @@ -0,0 +1,51 @@ +package org.radarbase.ktor.auth + +import kotlinx.coroutines.CompletableDeferred +import java.util.concurrent.atomic.AtomicReference + +internal class AuthTokenHolder( + private val loadTokens: suspend () -> T? +) { + private val refreshTokensDeferred = AtomicReference?>(null) + private val loadTokensDeferred = AtomicReference?>(null) + + internal fun clearToken() { + loadTokensDeferred.set(null) + refreshTokensDeferred.set(null) + } + + internal suspend fun loadToken(): T? { + var deferred: CompletableDeferred? + do { + deferred = loadTokensDeferred.get() + val newValue = deferred ?: CompletableDeferred() + } while (!loadTokensDeferred.compareAndSet(deferred, newValue)) + + return if (deferred != null) { + deferred.await() + } else { + val newTokens = loadTokens() + loadTokensDeferred.get()!!.complete(newTokens) + newTokens + } + } + + internal suspend fun setToken(block: suspend () -> T?): T? { + var deferred: CompletableDeferred? + do { + deferred = refreshTokensDeferred.get() + val newValue = deferred ?: CompletableDeferred() + } while (!refreshTokensDeferred.compareAndSet(deferred, newValue)) + + val newToken = if (deferred == null) { + val newTokens = block() + refreshTokensDeferred.get()!!.complete(newTokens) + refreshTokensDeferred.set(null) + newTokens + } else { + deferred.await() + } + loadTokensDeferred.set(CompletableDeferred(newToken)) + return newToken + } +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/ClientCredentialsConfig.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/ClientCredentialsConfig.kt new file mode 100644 index 00000000..3584c895 --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/ClientCredentialsConfig.kt @@ -0,0 +1,24 @@ +package org.radarbase.ktor.auth + +data class ClientCredentialsConfig( + val tokenUrl: String, + val clientId: String? = null, + val clientSecret: String? = null, +) { + /** + * Fill in the client ID and client secret from environment variables. The variables are + * `<prefix>_CLIENT_ID` and `<prefix>_CLIENT_SECRET`. + */ + fun copyWithEnv(prefix: String = "MANAGEMENT_PORTAL"): ClientCredentialsConfig { + var result = this + val envClientId = System.getenv("${prefix}_CLIENT_ID") + if (envClientId != null) { + result = result.copy(clientId = envClientId) + } + val envClientSecret = System.getenv("${prefix}_CLIENT_SECRET") + if (envClientSecret != null) { + result = result.copy(clientSecret = envClientSecret) + } + return result + } +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuth2AccessToken.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuth2AccessToken.kt new file mode 100644 index 00000000..c8137818 --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuth2AccessToken.kt @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2021. The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * See the file LICENSE in the root of this repository. + */ + +package org.radarbase.ktor.auth + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +data class OAuth2AccessToken( + @SerialName("access_token") val accessToken: String? = null, + @SerialName("refresh_token") val refreshToken: String? = null, + @SerialName("expires_in") val expiresIn: Long = 0, + @SerialName("token_type") val tokenType: String? = null, + @SerialName("user_id") val externalUserId: String? = null, + @SerialName("scope") val scope: String? = null, +) diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt new file mode 100644 index 00000000..0e179db7 --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt @@ -0,0 +1,177 @@ +package org.radarbase.ktor.auth + +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.plugins.auth.* +import io.ktor.client.plugins.auth.providers.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.request.* +import io.ktor.client.request.forms.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.http.auth.* +import io.ktor.serialization.kotlinx.json.* +import io.ktor.util.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import org.slf4j.LoggerFactory + +private val logger = LoggerFactory.getLogger(Auth::class.java) + +/** + * Installs the client's [BearerAuthProvider]. + */ +fun Auth.clientCredentials(block: ClientCredentialsAuthConfig.() -> Unit) { + with(ClientCredentialsAuthConfig().apply(block)) { + this@clientCredentials.providers.add(ClientCredentialsAuthProvider(_requestToken, _loadTokens, _sendWithoutRequest, realm)) + } +} + +fun Auth.clientCredentials( + authConfig: ClientCredentialsConfig, + targetHost: String? = null, +): Flow { + requireNotNull(authConfig.clientId) { "Missing client ID" } + requireNotNull(authConfig.clientSecret) { "Missing client secret"} + val flow = MutableStateFlow(null) + + clientCredentials { + if (targetHost != null) { + sendWithoutRequest { request -> + request.url.host == targetHost + } + } + requestToken { + val response = client.submitForm( + url = authConfig.tokenUrl, + formParameters = Parameters.build { + append("grant_type", "client_credentials") + append("client_id", authConfig.clientId) + append("client_secret", authConfig.clientSecret) + } + ) { + accept(ContentType.Application.Json) + markAsRequestTokenRequest() + } + val refreshTokenInfo: OAuth2AccessToken? = if (!response.status.isSuccess()) { + logger.error("Failed to fetch new token: {}", response.bodyAsText()) + null + } else { + response.body() + } + flow.value = refreshTokenInfo + refreshTokenInfo + } + } + + return flow +} + +/** + * Parameters to be passed to [BearerAuthConfig.refreshTokens] lambda. + */ +class RequestTokenParams( + val client: HttpClient, +) { + /** + * Marks that this request is for requesting auth tokens, resulting in a special handling of it. + */ + fun HttpRequestBuilder.markAsRequestTokenRequest() { + attributes.put(Auth.AuthCircuitBreaker, Unit) + } +} + +/** + * A configuration for [BearerAuthProvider]. + */ +@KtorDsl +class ClientCredentialsAuthConfig { + internal var _requestToken: suspend RequestTokenParams.() -> OAuth2AccessToken? = { null } + internal var _loadTokens: suspend () -> OAuth2AccessToken? = { null } + internal var _sendWithoutRequest: (HttpRequestBuilder) -> Boolean = { true } + + var realm: String? = null + + /** + * Configures a callback that refreshes a token when the 401 status code is received. + */ + fun requestToken(block: suspend RequestTokenParams.() -> OAuth2AccessToken?) { + _requestToken = block + } + + /** + * Configures a callback that loads a cached token from a local storage. + * Note: Using the same client instance here to make a request will result in a deadlock. + */ + fun loadTokens(block: suspend () -> OAuth2AccessToken?) { + _loadTokens = block + } + + /** + * Sends credentials without waiting for [HttpStatusCode.Unauthorized]. + */ + fun sendWithoutRequest(block: (HttpRequestBuilder) -> Boolean) { + _sendWithoutRequest = block + } +} + +/** + * An authentication provider for the Bearer HTTP authentication scheme. + * Bearer authentication involves security tokens called bearer tokens. + * As an example, these tokens can be used as a part of OAuth flow to authorize users of your application + * by using external providers, such as Google, Facebook, Twitter, and so on. + * + * You can learn more from [Bearer authentication](https://ktor.io/docs/bearer-client.html). + */ +class ClientCredentialsAuthProvider( + private val requestToken: suspend RequestTokenParams.() -> OAuth2AccessToken?, + loadTokens: suspend () -> OAuth2AccessToken?, + private val sendWithoutRequestCallback: (HttpRequestBuilder) -> Boolean = { true }, + private val realm: String?, +) : AuthProvider { + + @Suppress("OverridingDeprecatedMember") + @Deprecated("Please use sendWithoutRequest function instead", replaceWith = ReplaceWith("sendWithoutRequest(request)")) + override val sendWithoutRequest: Boolean + get() = error("Deprecated") + + private val tokensHolder = AuthTokenHolder(loadTokens) + + override fun sendWithoutRequest(request: HttpRequestBuilder): Boolean = sendWithoutRequestCallback(request) + + /** + * Checks if current provider is applicable to the request. + */ + override fun isApplicable(auth: HttpAuthHeader): Boolean { + if (auth.authScheme != AuthScheme.Bearer) return false + if (realm == null) return true + if (auth !is HttpAuthHeader.Parameterized) return false + + return auth.parameter("realm") == realm + } + + /** + * Adds an authentication method headers and credentials. + */ + override suspend fun addRequestHeaders(request: HttpRequestBuilder, authHeader: HttpAuthHeader?) { + val token = tokensHolder.loadToken() ?: return + + request.headers { + if (contains(HttpHeaders.Authorization)) { + remove(HttpHeaders.Authorization) + } + append(HttpHeaders.Authorization, "Bearer ${token.accessToken}") + } + } + + override suspend fun refreshToken(response: HttpResponse): Boolean { + val newToken = tokensHolder.setToken { + requestToken(RequestTokenParams(response.call.client)) + } + return newToken != null + } + + fun clearToken() { + tokensHolder.clearToken() + } +} diff --git a/radar-commons-testing/build.gradle.kts b/radar-commons-testing/build.gradle.kts index 14515706..a4a46584 100644 --- a/radar-commons-testing/build.gradle.kts +++ b/radar-commons-testing/build.gradle.kts @@ -38,6 +38,7 @@ description = "RADAR Common testing library mocking code and utilities." dependencies { api(project(":radar-commons")) api(project(":radar-commons-server")) + api(project(":radar-commons-kotlin")) val avroVersion: String by project api("org.apache.avro:avro:$avroVersion") val radarSchemasVersion: String by project diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt index ff24faa1..3bd9e3d2 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt @@ -26,8 +26,8 @@ import kotlinx.coroutines.* import org.apache.avro.SchemaValidationException import org.radarbase.config.ServerConfig import org.radarbase.config.YamlConfigLoader -import org.radarbase.management.auth.ClientCredentialsConfig -import org.radarbase.management.auth.clientCredentials +import org.radarbase.ktor.auth.ClientCredentialsConfig +import org.radarbase.ktor.auth.clientCredentials import org.radarbase.mock.config.AuthConfig import org.radarbase.mock.config.BasicMockConfig import org.radarbase.mock.config.MockDataConfig @@ -40,18 +40,17 @@ import org.radarbase.producer.rest.ConnectionState import org.radarbase.producer.rest.RestKafkaSender.Companion.restKafkaSender import org.radarbase.producer.schema.SchemaRetriever import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever -import org.radarbase.util.TimeoutConfig import org.radarcns.kafka.ObservationKey import org.radarcns.passive.empatica.* import org.slf4j.LoggerFactory import java.io.IOException import java.nio.file.Path import java.nio.file.Paths -import java.time.Duration import java.time.Instant import java.util.* import java.util.concurrent.atomic.AtomicBoolean import kotlin.system.exitProcess +import kotlin.time.Duration.Companion.seconds /** * A Mock Producer class that can be used to stream data. It can use MockFileSender and MockDevice @@ -68,13 +67,13 @@ class MockProducer @JvmOverloads constructor( private var files: MutableList private val senders: List private val retriever: SchemaRetriever - private var job: Job? = null + private val job: Job = SupervisorJob() init { val numDevices = mockConfig.numberOfDevices retriever = schemaRetriever(mockConfig.schemaRegistry.urlString) { httpClient { - timeout(Duration.ofSeconds(10)) + timeout(10.seconds) } } val dataConfigs = mockConfig.data @@ -128,10 +127,12 @@ class MockProducer @JvmOverloads constructor( useCompression: Boolean, authConfig: AuthConfig? ): List { - val sharedState = ConnectionState(TimeoutConfig(Duration.ofSeconds(10))) + val scope = CoroutineScope(job) + val sharedState = ConnectionState(10.seconds, scope) return (0 until numDevices) .map { restKafkaSender { + this.scope = scope schemaRetriever = retriever connectionState = sharedState @@ -154,7 +155,7 @@ class MockProducer @JvmOverloads constructor( if (useCompression) { install(GzipContentEncoding) } - timeout(Duration.ofSeconds(10)) + timeout(10.seconds) } } } @@ -163,8 +164,7 @@ class MockProducer @JvmOverloads constructor( /** Start sending data. */ @Throws(IOException::class) suspend fun start() { - job = SupervisorJob() - withContext(Dispatchers.Default + job!!) { + withContext(Dispatchers.Default + job) { for (device in devices) { launch { with(device) { @@ -179,14 +179,13 @@ class MockProducer @JvmOverloads constructor( } } } - job = null } /** Stop sending data and clean up all resources. */ @Throws(IOException::class, InterruptedException::class, SchemaValidationException::class) suspend fun shutdown() { - job?.run { - logger.info("Shutting down mock devices") + logger.info("Shutting down mock devices") + job.run { cancel() join() } @@ -196,42 +195,48 @@ class MockProducer @JvmOverloads constructor( } private fun defaultDataConfig(): List { - val acceleration = MockDataConfig() - acceleration.topic = "android_empatica_e4_acceleration" - acceleration.frequency = 32 - acceleration.valueSchema = EmpaticaE4Acceleration::class.java.name - acceleration.setInterval(-2.0, 2.0) - acceleration.valueFields = listOf("x", "y", "z") - val battery = MockDataConfig() - battery.topic = "android_empatica_e4_battery_level" - battery.valueSchema = EmpaticaE4BatteryLevel::class.java.name - battery.frequency = 1 - battery.setInterval(0.0, 1.0) - battery.setValueField("batteryLevel") - val bvp = MockDataConfig() - bvp.topic = "android_empatica_e4_blood_volume_pulse" - bvp.valueSchema = EmpaticaE4BloodVolumePulse::class.java.name - bvp.frequency = 64 - bvp.setInterval(60.0, 90.0) - bvp.setValueField("bloodVolumePulse") - val eda = MockDataConfig() - eda.topic = "android_empatica_e4_electrodermal_activity" - eda.valueSchema = EmpaticaE4ElectroDermalActivity::class.java.name - eda.setValueField("electroDermalActivity") - eda.frequency = 4 - eda.setInterval(0.01, 0.05) - val ibi = MockDataConfig() - ibi.topic = "android_empatica_e4_inter_beat_interval" - ibi.valueSchema = EmpaticaE4InterBeatInterval::class.java.name - ibi.setValueField("interBeatInterval") - ibi.frequency = 1 - ibi.setInterval(40.0, 150.0) - val temperature = MockDataConfig() - temperature.topic = "android_empatica_e4_temperature" - temperature.valueSchema = EmpaticaE4Temperature::class.java.name - temperature.frequency = 4 - temperature.setInterval(20.0, 60.0) - temperature.setValueField("temperature") + val acceleration = MockDataConfig().apply { + topic = "android_empatica_e4_acceleration" + frequency = 32 + valueSchema = EmpaticaE4Acceleration::class.java.name + setInterval(-2.0, 2.0) + valueFields = listOf("x", "y", "z") + } + val battery = MockDataConfig().apply { + topic = "android_empatica_e4_battery_level" + valueSchema = EmpaticaE4BatteryLevel::class.java.name + frequency = 1 + setInterval(0.0, 1.0) + setValueField("batteryLevel") + } + val bvp = MockDataConfig().apply { + topic = "android_empatica_e4_blood_volume_pulse" + valueSchema = EmpaticaE4BloodVolumePulse::class.java.name + frequency = 64 + setInterval(60.0, 90.0) + setValueField("bloodVolumePulse") + } + val eda = MockDataConfig().apply { + topic = "android_empatica_e4_electrodermal_activity" + valueSchema = EmpaticaE4ElectroDermalActivity::class.java.name + setValueField("electroDermalActivity") + frequency = 4 + setInterval(0.01, 0.05) + } + val ibi = MockDataConfig().apply { + topic = "android_empatica_e4_inter_beat_interval" + valueSchema = EmpaticaE4InterBeatInterval::class.java.name + setValueField("interBeatInterval") + frequency = 1 + setInterval(40.0, 150.0) + } + val temperature = MockDataConfig().apply { + topic = "android_empatica_e4_temperature" + valueSchema = EmpaticaE4Temperature::class.java.name + frequency = 4 + setInterval(20.0, 60.0) + setValueField("temperature") + } return listOf(acceleration, battery, bvp, eda, ibi, temperature) } diff --git a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt index 0ce21e25..dc44a06a 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt @@ -18,6 +18,9 @@ package org.radarbase.util import kotlinx.coroutines.delay import org.slf4j.LoggerFactory import java.util.concurrent.atomic.AtomicInteger +import kotlin.time.Duration +import kotlin.time.Duration.Companion.nanoseconds +import kotlin.time.Duration.Companion.seconds /** * Oscilloscope gives out a regular beat, at a given frequency per second. The intended way to use @@ -27,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger class Oscilloscope( private val frequency: Int ) { - private val timeStep: Long = 1_000_000_000L / frequency + private val timeStep: Duration = 1.seconds / frequency private val baseTime: Long = System.nanoTime() private var iteration: AtomicInteger = AtomicInteger(0) @@ -46,11 +49,10 @@ class Oscilloscope( suspend fun beat(): Int { val currentTime = System.nanoTime() val currentIteration = iteration.getAndIncrement() - val timeToSleep = baseTime + currentIteration * timeStep - currentTime - if (timeToSleep > 0) { - val millis = (timeToSleep / 1_000_000L).coerceAtLeast(1L) - logger.info("delaying {} millis", millis) - delay(millis) + val timeToSleep = (baseTime - currentTime).nanoseconds + timeStep * currentIteration + if (timeToSleep > Duration.ZERO) { + logger.info("delaying {} millis", timeToSleep.inWholeMilliseconds) + delay(timeToSleep) } return currentIteration % frequency + 1 } diff --git a/radar-commons/build.gradle.kts b/radar-commons/build.gradle.kts index 177be4b3..319e1407 100644 --- a/radar-commons/build.gradle.kts +++ b/radar-commons/build.gradle.kts @@ -19,8 +19,9 @@ dependencies { api("org.apache.avro:avro:$avroVersion") api(kotlin("reflect")) - val ktorVersion: String by project + implementation(project(":radar-commons-kotlin")) + val ktorVersion: String by project api(platform("io.ktor:ktor-bom:$ktorVersion")) api("io.ktor:ktor-client-core") api("io.ktor:ktor-client-cio") @@ -28,8 +29,6 @@ dependencies { implementation("io.ktor:ktor-client-content-negotiation") implementation("io.ktor:ktor-serialization-kotlinx-json") - api("org.radarbase:managementportal-client:0.9.0-SNAPSHOT") - val coroutinesVersion: String by project api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt b/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt index 64950963..71e15baa 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt @@ -4,12 +4,12 @@ import io.ktor.client.* import io.ktor.client.engine.cio.* import io.ktor.client.plugins.* import java.security.cert.X509Certificate -import java.time.Duration +import kotlin.time.Duration import javax.net.ssl.X509TrustManager fun HttpClientConfig<*>.timeout(duration: Duration) { install(HttpTimeout) { - val millis = duration.toMillis() + val millis = duration.inWholeMilliseconds connectTimeoutMillis = millis socketTimeoutMillis = millis requestTimeoutMillis = millis diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt index 35eb66dc..df3dfb00 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt @@ -15,11 +15,10 @@ */ package org.radarbase.producer.rest -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.map -import org.radarbase.util.TimedValue -import org.radarbase.util.TimeoutConfig +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import kotlin.coroutines.EmptyCoroutineContext +import kotlin.time.Duration /** * Current connection status of a KafkaSender. After a timeout occurs this will turn to @@ -34,46 +33,52 @@ import org.radarbase.util.TimeoutConfig * A connection state could be shared with multiple HTTP clients if they are talking to the same * server. * - * @param timeoutConfig timeout config + * @param timeout timeout after which the connected state will be reset to unknown. * @throws IllegalArgumentException if the timeout is not strictly positive. */ class ConnectionState( - private val timeoutConfig: TimeoutConfig, + private val timeout: Duration, + scope: CoroutineScope = CoroutineScope(EmptyCoroutineContext) ) { /** State symbols of the connection. */ enum class State { CONNECTED, DISCONNECTED, UNKNOWN, UNAUTHORIZED } - val state: Flow - get() = mutableState.map { - if (it.value === State.CONNECTED && it.isExpired) - State.UNKNOWN - else - it.value - } + val scope = scope + Job() + + private val mutableState = MutableStateFlow(State.UNKNOWN) - private val mutableState = MutableStateFlow(TimedValue(State.UNKNOWN, timeoutConfig)) + @OptIn(ExperimentalCoroutinesApi::class) + val state: Flow = mutableState + .transformLatest { state -> + emit(state) + if (state == State.CONNECTED) { + delay(timeout) + emit(State.UNKNOWN) + } + } + .shareIn(this.scope + Dispatchers.Unconfined, SharingStarted.Eagerly, replay = 1) init { - mutableState.tryEmit(TimedValue(State.UNKNOWN, timeoutConfig)) + mutableState.value = State.UNKNOWN } /** For a sender to indicate that a connection attempt succeeded. */ - suspend fun didConnect() { - mutableState.emit(TimedValue(State.CONNECTED, timeoutConfig)) + fun didConnect() { + mutableState.value = State.CONNECTED } /** For a sender to indicate that a connection attempt failed. */ - suspend fun didDisconnect() { - mutableState.emit(TimedValue(State.DISCONNECTED, timeoutConfig)) + fun didDisconnect() { + mutableState.value = State.DISCONNECTED } - suspend fun wasUnauthorized() { - mutableState.emit(TimedValue(State.UNAUTHORIZED, timeoutConfig)) + fun wasUnauthorized() { + mutableState.value = State.UNAUTHORIZED } - suspend fun reset() { - mutableState.emit(TimedValue(State.UNKNOWN, timeoutConfig)) + fun reset() { + mutableState.value = State.UNKNOWN } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt index 419effb6..ca658d96 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt @@ -24,10 +24,9 @@ import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* import io.ktor.util.reflect.* -import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.* import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.first -import kotlinx.coroutines.withContext import org.apache.avro.SchemaValidationException import org.radarbase.data.RecordData import org.radarbase.producer.AuthenticationException @@ -40,14 +39,13 @@ import org.radarbase.producer.io.unsafeSsl import org.radarbase.producer.schema.SchemaRetriever import org.radarbase.topic.AvroTopic import org.radarbase.util.RadarProducerDsl -import org.radarbase.util.TimeoutConfig import org.slf4j.LoggerFactory import java.io.IOException -import java.time.Duration import java.util.* -import kotlin.coroutines.CoroutineContext import kotlin.reflect.javaType import kotlin.reflect.typeOf +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds /** * RestSender sends records to the Kafka REST Proxy. It does so using an Avro JSON encoding. A new @@ -55,6 +53,7 @@ import kotlin.reflect.typeOf * blocking and unbuffered, so flush, clear and close do not do anything. */ class RestKafkaSender(config: Config) : KafkaSender { + val scope = config.scope private val allowUnsafe: Boolean = config.allowUnsafe private val contentType: ContentType = config.contentType val schemaRetriever: SchemaRetriever = requireNotNull(config.schemaRetriever) { @@ -64,7 +63,8 @@ class RestKafkaSender(config: Config) : KafkaSender { val restClient: HttpClient private val _connectionState: ConnectionState = config.connectionState - ?: ConnectionState(TimeoutConfig(DEFAULT_TIMEOUT)) + ?: ConnectionState(DEFAULT_TIMEOUT, scope) + override val connectionState: Flow get() = _connectionState.state @@ -73,7 +73,6 @@ class RestKafkaSender(config: Config) : KafkaSender { private val connectionTimeout: Duration = config.connectionTimeout private val contentEncoding = config.contentEncoding private val originalHttpClient = config.httpClient - private val ioContext: CoroutineContext = config.ioContext /** * Construct a RestSender. @@ -119,15 +118,13 @@ class RestKafkaSender(config: Config) : KafkaSender { override val topic: AvroTopic, ) : KafkaTopicSender { @OptIn(ExperimentalStdlibApi::class) - override suspend fun send(records: RecordData) { + override suspend fun send(records: RecordData) = scope.async { try { - val response: HttpResponse = withContext(Dispatchers.IO) { - restClient.post { - url("topics/${topic.name}") - val kType = typeOf>() - val reifiedType = kType.javaType - setBody(records, TypeInfo(RecordData::class, reifiedType, kType)) - } + val response: HttpResponse = restClient.post { + url("topics/${topic.name}") + val kType = typeOf>() + val reifiedType = kType.javaType + setBody(records, TypeInfo(RecordData::class, reifiedType, kType)) } if (response.status.isSuccess()) { _connectionState.didConnect() @@ -148,7 +145,7 @@ class RestKafkaSender(config: Config) : KafkaSender { _connectionState.didDisconnect() throw ex } - } + }.await() } @Throws(SchemaValidationException::class) @@ -198,6 +195,7 @@ class RestKafkaSender(config: Config) : KafkaSender { } private fun toConfig() = Config().apply { + scope = this@RestKafkaSender.scope baseUrl = this@RestKafkaSender.baseUrl httpClient = this@RestKafkaSender.originalHttpClient schemaRetriever = this@RestKafkaSender.schemaRetriever @@ -206,18 +204,17 @@ class RestKafkaSender(config: Config) : KafkaSender { contentEncoding = this@RestKafkaSender.contentEncoding connectionTimeout = this@RestKafkaSender.connectionTimeout allowUnsafe = this@RestKafkaSender.allowUnsafe - ioContext = this@RestKafkaSender.ioContext } @RadarProducerDsl class Config { - var ioContext: CoroutineContext = Dispatchers.IO + var scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) var baseUrl: String? = null var schemaRetriever: SchemaRetriever? = null var connectionState: ConnectionState? = null var httpClient: HttpClient? = null var headers = HeadersBuilder() - var connectionTimeout: Duration = Duration.ofSeconds(30) + var connectionTimeout: Duration = 30.seconds var contentEncoding: String? = null var allowUnsafe: Boolean = false var contentType: ContentType = KAFKA_REST_JSON_ENCODING @@ -239,14 +236,14 @@ class RestKafkaSender(config: Config) : KafkaSender { baseUrl == other.baseUrl && connectionTimeout == other.connectionTimeout && contentEncoding == other.contentEncoding && - ioContext == other.ioContext + scope == other.scope } override fun hashCode(): Int = headers.hashCode() } companion object { private val logger = LoggerFactory.getLogger(RestKafkaSender::class.java) - val DEFAULT_TIMEOUT: Duration = Duration.ofSeconds(20) + val DEFAULT_TIMEOUT: Duration = 20.seconds val KAFKA_REST_BINARY_ENCODING = ContentType("application", "vnd.radarbase.avro.v1+binary") val KAFKA_REST_JSON_ENCODING = ContentType("application", "vnd.kafka+json") const val GZIP_CONTENT_ENCODING = "gzip" diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt index e27fcf22..17de167b 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt @@ -78,9 +78,13 @@ class SchemaRestClient( /** Add a schema to a subject. */ @Throws(IOException::class) - suspend fun addSchema(subject: String, schema: Schema): Int { + suspend fun addSchema(subject: String, schema: Schema): ParsedSchemaMetadata { val result = schemaPost("subjects/$subject/versions", schema) - return checkNotNull(result.id) { "Missing schema ID in request result" } + return ParsedSchemaMetadata( + id = checkNotNull(result.id) { "Missing schema ID in request result" }, + version = result.version, + schema = schema, + ) } /** Request metadata for a schema on a subject. */ diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt index becd355f..cd37a16f 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt @@ -21,30 +21,29 @@ import io.ktor.client.plugins.* import io.ktor.client.request.* import io.ktor.http.* import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch import org.apache.avro.Schema -import org.radarbase.producer.rest.RestException +import org.radarbase.kotlin.coroutines.CacheConfig +import org.radarbase.kotlin.coroutines.CachedValue import org.radarbase.util.RadarProducerDsl -import org.radarbase.util.TimedInt -import org.radarbase.util.TimedValue -import org.radarbase.util.TimedVariable.Companion.prune -import org.radarbase.util.TimeoutConfig -import org.slf4j.LoggerFactory import java.io.IOException -import java.time.Duration import java.util.* import java.util.Objects.hash import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentMap import kotlin.coroutines.CoroutineContext +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.minutes + +typealias VersionCache = ConcurrentMap> /** * Retriever of an Avro Schema. */ open class SchemaRetriever(config: Config) { - private val idCache: ConcurrentMap> = ConcurrentHashMap() - private val schemaCache: ConcurrentMap = ConcurrentHashMap() - private val subjectVersionCache: ConcurrentMap> = - ConcurrentHashMap() + private val schemaCache: ConcurrentMap> = ConcurrentHashMap() + private val subjectVersionCache: ConcurrentMap = ConcurrentHashMap() private val baseUrl = config.baseUrl private val ioContext = config.ioContext @@ -58,57 +57,32 @@ open class SchemaRetriever(config: Config) { * @return schema ID */ @Throws(IOException::class) - suspend fun addSchema(topic: String, ofValue: Boolean, schema: Schema): Int { + suspend fun addSchema(topic: String, ofValue: Boolean, schema: Schema): Int = coroutineScope { val subject = subject(topic, ofValue) - val id = restClient.addSchema(subject, schema) - cache(ParsedSchemaMetadata(id, null, schema), subject, false) - return id - } + val metadata = restClient.addSchema(subject, schema) - /** - * Get schema metadata, and if none is found, add a new schema. - * - * @param version version to get or 0 if the latest version can be used. - */ - @Throws(IOException::class) - open suspend fun getOrSet( - topic: String, - ofValue: Boolean, - schema: Schema, - version: Int - ): ParsedSchemaMetadata { - return try { - getByVersion(topic, ofValue, version) - } catch (ex: RestException) { - if (ex.status == HttpStatusCode.NotFound) { - logger.warn("Schema for {} value was not yet added to the schema registry.", topic) - addSchema(topic, ofValue, schema) - metadata(topic, ofValue, schema, version <= 0) - } else { - throw ex + launch { + cachedMetadata(subject, metadata.schema).set(metadata) + } + if (metadata.version != null) { + launch { + cachedVersion(subject, metadata.version).set(metadata) } } + metadata.id } - /** Get a schema by its ID. */ - @Throws(IOException::class) - suspend fun getById(id: Int): Schema { - var value = idCache[id] - if (value == null || value.isExpired) { - value = TimedValue(restClient.retrieveSchemaById(id), schemaTimeout) - idCache[id] = value - schemaCache[value.value] = TimedInt(id, schemaTimeout) + private fun cachedMetadata( + subject: String, + schema: Schema + ): CachedValue = schemaCache.computeIfAbsent(schema) { + CachedValue(schemaTimeout) { + val metadata = restClient.requestMetadata(subject, schema) + if (metadata.version != null) { + cachedVersion(subject, metadata.version).set(metadata) + } + metadata } - return value.value - } - - /** Gets a schema by ID and check that it is present in the given topic. */ - @Throws(IOException::class) - suspend fun getById(topic: String, ofValue: Boolean, id: Int): ParsedSchemaMetadata { - val schema = getById(id) - val subject = subject(topic, ofValue) - return cached(subject, id, null, schema) - ?: metadata(topic, ofValue, schema) } /** Get schema metadata. Cached schema metadata will be used if present. */ @@ -123,25 +97,37 @@ open class SchemaRetriever(config: Config) { subject, ::ConcurrentHashMap ) - val id = versionMap[version.coerceAtLeast(0)] - return if (id == null || id.isExpired) { - val metadata = restClient.retrieveSchemaMetadata(subject, version) - cache(metadata, subject, version <= 0) - metadata - } else { - val schema = getById(id.value) - val metadata = cached(subject, id.value, version, schema) - metadata ?: metadata(topic, ofValue, schema, version <= 0) + val metadata = versionMap.cachedVersion(subject, version).get() + if (version <= 0 && metadata.version != null) { + versionMap.cachedVersion(subject, metadata.version).set(metadata) } + return metadata } - /** Get all schema versions in a subject. */ - @Throws(IOException::class) - open suspend fun metadata( - topic: String, - ofValue: Boolean, - schema: Schema - ): ParsedSchemaMetadata = metadata(topic, ofValue, schema, false) + private suspend fun cachedVersion( + subject: String, + version: Int + ): CachedValue = subjectVersionCache + .computeIfAbsent( + subject, + ::ConcurrentHashMap + ) + .cachedVersion(subject, version) + + private suspend fun VersionCache.cachedVersion( + subject: String, + version: Int + ): CachedValue { + val useVersion = version.coerceAtLeast(0) + val versionId = computeIfAbsent(useVersion) { + CachedValue(schemaTimeout) { + val metadata = restClient.retrieveSchemaMetadata(subject, version) + cachedMetadata(subject, metadata.schema).set(metadata) + metadata + } + } + return versionId + } /** Get the metadata of a specific schema in a topic. */ @Throws(IOException::class) @@ -149,75 +135,44 @@ open class SchemaRetriever(config: Config) { topic: String, ofValue: Boolean, schema: Schema, - ofLatestVersion: Boolean ): ParsedSchemaMetadata { - val id = schemaCache[schema] val subject = subject(topic, ofValue) - if (id != null && !id.isExpired) { - val metadata = cached(subject, id.value, null, schema) - if (metadata != null) { - return metadata - } - } - val metadata = restClient.requestMetadata(subject, schema) - cache(metadata, subject, ofLatestVersion) - return metadata - } - - /** - * Get cached metadata. - * @param subject schema registry subject - * @param id schema ID. - * @param reportedVersion version requested by the client. Null if no version was requested. - * This version will be used if the actual version was not cached. - * @param schema schema to use. - * @return metadata if present. Returns null if no metadata is cached or if no version is cached - * and the reportedVersion is null. - */ - protected fun cached( - subject: String, - id: Int, - reportedVersion: Int?, - schema: Schema? - ): ParsedSchemaMetadata? { - var version = reportedVersion - if (version == null || version <= 0) { - version = subjectVersionCache[subject]?.find(id) - if (version == null || version <= 0) { - return null - } - } - return ParsedSchemaMetadata(id, version, schema!!) + return cachedMetadata(subject, schema).get() } - private fun ConcurrentMap.find(id: Int): Int? = entries - .find { (k, v) -> !v.isExpired && k != 0 && v.value == id } - ?.key + private suspend fun MutableCollection>.prune() { + val iter = iterator() + while (iter.hasNext()) { + val staleValue = iter.next().getFromCache() + ?: continue - protected fun cache(metadata: ParsedSchemaMetadata, subject: String, latest: Boolean) { - val id = TimedInt(metadata.id, schemaTimeout) - schemaCache[metadata.schema] = id - if (metadata.version != null) { - val versionCache = subjectVersionCache.computeIfAbsent( - subject, - ::ConcurrentHashMap - ) - versionCache[metadata.version] = id - if (latest) { - versionCache[0] = id + if ( + staleValue is CachedValue.CacheError || + (staleValue is CachedValue.CacheValue && + staleValue.isExpired(schemaTimeout.refreshDuration)) + ) { + iter.remove() } } - idCache[metadata.id] = TimedValue(metadata.schema, schemaTimeout) } /** * Remove expired entries from cache. */ - fun pruneCache() { - schemaCache.prune() - idCache.prune() - for (versionMap in subjectVersionCache.values) { - versionMap.prune() + suspend fun pruneCache() = coroutineScope { + launch { + schemaCache.values.prune() + } + + launch { + val subjectsIter = subjectVersionCache.values.iterator() + while (subjectsIter.hasNext()) { + val versionMap = subjectsIter.next() + versionMap.values.prune() + if (versionMap.isEmpty()) { + subjectsIter.remove() + } + } } } @@ -226,7 +181,6 @@ open class SchemaRetriever(config: Config) { */ fun clearCache() { subjectVersionCache.clear() - idCache.clear() schemaCache.clear() } @@ -235,7 +189,7 @@ open class SchemaRetriever(config: Config) { val baseUrl: String, ) { var httpClient: HttpClient? = null - var schemaTimeout: TimeoutConfig = DEFAULT_SCHEMA_TIMEOUT_CONFIG + var schemaTimeout: CacheConfig = DEFAULT_SCHEMA_TIMEOUT_CONFIG var ioContext: CoroutineContext = Dispatchers.IO fun httpClient(config: HttpClientConfig<*>.() -> Unit) { httpClient = httpClient?.config(config) @@ -262,7 +216,9 @@ open class SchemaRetriever(config: Config) { val newConfig = toConfig().apply(config) return if (currentConfig != newConfig) { SchemaRetriever(newConfig) - } else this + } else { + this + } } private fun toConfig(): Config = Config(baseUrl = baseUrl).apply { @@ -272,8 +228,10 @@ open class SchemaRetriever(config: Config) { } companion object { - private val logger = LoggerFactory.getLogger(SchemaRetriever::class.java) - private val DEFAULT_SCHEMA_TIMEOUT_CONFIG = TimeoutConfig(Duration.ofDays(1)) + private val DEFAULT_SCHEMA_TIMEOUT_CONFIG = CacheConfig( + refreshDuration = 1.days, + retryDuration = 1.minutes, + ) fun schemaRetriever(baseUrl: String, config: Config.() -> Unit): SchemaRetriever { return SchemaRetriever(Config(baseUrl).apply(config)) diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt b/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt deleted file mode 100644 index 446ec984..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimedInt.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.radarbase.util - -class TimedInt( - val value: Int, - cacheConfig: TimeoutConfig, -) : TimedVariable(cacheConfig) { - override fun equals(other: Any?): Boolean { - if (this === other) { - return true - } - if (other == null || javaClass != other.javaClass) { - return false - } - other as TimedInt - return value == other.value && expiry == other.expiry - } - - override fun hashCode(): Int = value -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt b/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt deleted file mode 100644 index b9555c04..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimedValue.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.radarbase.util - -class TimedValue( - val value: T, - cacheConfig: TimeoutConfig, -) : TimedVariable(cacheConfig) { - override fun equals(other: Any?): Boolean { - if (this === other) { - return true - } - if (other == null || javaClass != other.javaClass) { - return false - } - other as TimedValue<*> - return value == other.value && expiry == other.expiry - } - - override fun hashCode(): Int = value.hashCode() -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt b/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt deleted file mode 100644 index 91f0b44d..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimedVariable.kt +++ /dev/null @@ -1,21 +0,0 @@ -package org.radarbase.util - -open class TimedVariable( - private val cacheConfig: TimeoutConfig, -) { - protected val expiry: Long = cacheConfig.currentExpiryTime - - val isExpired: Boolean - get() = cacheConfig.isPassed(expiry) - - companion object { - internal fun MutableMap<*, out TimedVariable>.prune() { - val iterator = values.iterator() - for (value in iterator) { - if (value.isExpired) { - iterator.remove() - } - } - } - } -} diff --git a/radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt b/radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt deleted file mode 100644 index 399ea208..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/TimeoutConfig.kt +++ /dev/null @@ -1,13 +0,0 @@ -package org.radarbase.util - -import java.time.Duration - -data class TimeoutConfig( - val validity: Duration, - val timeSource: () -> Long = System::currentTimeMillis, -) { - val currentExpiryTime: Long - get() = timeSource() + validity.toMillis() - - fun isPassed(timeMillis: Long): Boolean = timeMillis < timeSource() -} diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt index a8dada55..e5b75f4e 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/ConnectionStateTest.kt @@ -15,51 +15,43 @@ */ package org.radarbase.producer.rest -import kotlinx.coroutines.* +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.flowOn -import kotlinx.coroutines.flow.last -import kotlinx.coroutines.flow.single -import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.Timeout -import org.radarbase.util.TimeoutConfig -import org.slf4j.LoggerFactory -import java.time.Duration import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.milliseconds class ConnectionStateTest { @Test @Timeout(1, unit = TimeUnit.SECONDS) fun testState() = runBlocking { - var state = ConnectionState(TimeoutConfig(Duration.ofMillis(10))) - logger.info("initial state set") + var state = ConnectionState(10.milliseconds) state.assertEqualTo(ConnectionState.State.UNKNOWN) - logger.info("setting to didConnect") state.didConnect() state.assertEqualTo(ConnectionState.State.CONNECTED) state.didDisconnect() state.assertEqualTo(ConnectionState.State.DISCONNECTED) - delay(15) + delay(15.milliseconds) state.assertEqualTo(ConnectionState.State.DISCONNECTED) state.didConnect() - delay(15) + delay(15.milliseconds) state.assertEqualTo(ConnectionState.State.UNKNOWN) - state = ConnectionState(TimeoutConfig(Duration.ofMillis(25))) + state.scope.cancel() + state = ConnectionState(25.milliseconds) state.didConnect() state.assertEqualTo(ConnectionState.State.CONNECTED) - delay(10) + delay(10.milliseconds) state.assertEqualTo(ConnectionState.State.CONNECTED) - delay(15) + delay(20.milliseconds) state.assertEqualTo(ConnectionState.State.UNKNOWN) + state.scope.cancel() } - private suspend fun ConnectionState.assertEqualTo(expected: ConnectionState.State) { + private suspend inline fun ConnectionState.assertEqualTo(expected: ConnectionState.State) { assertEquals(expected, state.first()) } - - companion object { - private val logger = LoggerFactory.getLogger(ConnectionStateTest::class.java) - } } diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt index cbd4e0ee..8a7224e4 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt @@ -75,7 +75,7 @@ class RestKafkaSenderTest { @Throws(Exception::class) fun sender() = runTest { sender = sender.config { - ioContext = coroutineContext + scope = this@runTest with(headers) { append("Cookie", "ab") append("Cookie", "bc") @@ -119,7 +119,7 @@ class RestKafkaSenderTest { @Throws(Exception::class) fun sendBinary() = runTest { sender = sender.config { - ioContext = coroutineContext + scope = this@runTest contentType = RestKafkaSender.KAFKA_REST_BINARY_ENCODING } val keySchema = ObservationKey.getClassSchema() @@ -163,7 +163,7 @@ class RestKafkaSenderTest { @Throws(Exception::class) fun sendTwo() = runTest { sender = sender.config { - ioContext = coroutineContext + scope = this@runTest } val keySchema = ObservationKey.getClassSchema() val valueSchema = PhoneLight.getClassSchema() @@ -204,7 +204,7 @@ class RestKafkaSenderTest { @Throws(Exception::class) fun resetConnection() = runTest { sender = sender.config { - ioContext = coroutineContext + scope = this@runTest } var nRequests = 0 webServer.enqueue(MockResponse().setResponseCode(500)) @@ -226,7 +226,7 @@ class RestKafkaSenderTest { @Throws(Exception::class) fun resetConnectionUnauthorized() = runTest { sender = sender.config { - ioContext = coroutineContext + scope = this@runTest } webServer.enqueue(MockResponse().setResponseCode(401)) webServer.enqueue(MockResponse().setResponseCode(401)) diff --git a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt index 3d12b944..425e743c 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt @@ -29,8 +29,8 @@ import org.junit.jupiter.api.Test import org.radarbase.producer.io.timeout import org.radarbase.producer.rest.RestKafkaSenderTest.Companion.enqueueJson import java.io.IOException -import java.time.Duration import java.util.* +import kotlin.time.Duration.Companion.seconds @OptIn(ExperimentalCoroutinesApi::class) class SchemaRestClientTest { @@ -41,7 +41,7 @@ class SchemaRestClientTest { mockServer = MockWebServer() retriever = SchemaRestClient( HttpClient(CIO) { - timeout(Duration.ofSeconds(1)) + timeout(1.seconds) }, baseUrl = "http://${mockServer.hostName}:${mockServer.port}/base/" ) diff --git a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt index a1854358..a8821351 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt @@ -31,7 +31,7 @@ import org.radarbase.producer.rest.RestKafkaSenderTest.Companion.enqueueJson import org.radarbase.producer.schema.SchemaRetriever.Companion.schemaRetriever import org.radarbase.producer.schema.SchemaRetriever.Companion.subject import java.io.IOException -import java.time.Duration +import kotlin.time.Duration.Companion.seconds @OptIn(ExperimentalCoroutinesApi::class) class SchemaRetrieverTest { @@ -49,7 +49,7 @@ class SchemaRetrieverTest { defaultRequest { url("http://${mockServer.hostName}:${mockServer.port}/base/") } - timeout(Duration.ofSeconds(1)) + timeout(1.seconds) } } } @@ -119,57 +119,4 @@ class SchemaRetrieverTest { request.body.readUtf8() ) } - - @Test - fun getOrSetSchemaMetadataSet() = runTest { - mockServer.enqueue(MockResponse().setResponseCode(404)) - mockServer.enqueueJson("{\"id\":10}") - mockServer.enqueueJson("{\"id\":10, \"version\": 2}") - var metadata = retriever.getOrSet( - "bla", - true, - Schema.create(Schema.Type.STRING), - -1 - ) - assertEquals(10, metadata.id) - assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) - assertEquals(3, mockServer.requestCount.toLong()) - mockServer.takeRequest() - val request = mockServer.takeRequest() - assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.body.readUtf8()) - assertEquals("/base/subjects/bla-value/versions", request.path) - metadata = retriever.getOrSet( - "bla", - true, - Schema.create(Schema.Type.STRING), - -1 - ) - assertEquals(10, metadata.id) - assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) - } - - @Test - fun getOrSetSchemaMetadataGet() = runTest { - mockServer.enqueueJson("{\"id\":10,\"version\":2,\"schema\":\"\\\"string\\\"\"}") - var metadata = retriever.getOrSet( - "bla", - true, - Schema.create(Schema.Type.STRING), - 2 - ) - assertEquals(10, metadata.id) - assertEquals(2, metadata.version) - assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) - assertEquals(1, mockServer.requestCount.toLong()) - val request = mockServer.takeRequest() - assertEquals("/base/subjects/bla-value/versions/2", request.path) - metadata = retriever.getOrSet( - "bla", - true, - Schema.create(Schema.Type.STRING), - 2 - ) - assertEquals(10, metadata.id) - assertEquals(Schema.create(Schema.Type.STRING), metadata.schema) - } } From 8f3291d204fe5478ddb9c20ba7012b0078dab3c8 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 16 Mar 2023 12:27:24 +0100 Subject: [PATCH 09/25] Fix publish build --- build.gradle.kts | 13 +++++++------ radar-commons-server/build.gradle.kts | 11 +++++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7885d79d..3c96e360 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,4 @@ import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask -import java.util.LinkedList import org.jetbrains.kotlin.gradle.tasks.KotlinCompile /* @@ -63,7 +62,6 @@ subprojects { mavenCentral() mavenLocal() maven(url = "https://packages.confluent.io/maven/") - maven(url = "https://oss.sonatype.org/content/repositories/snapshots") } dependencies { @@ -186,15 +184,18 @@ subprojects { tasks.withType { useJUnitPlatform() - val stdout = LinkedList() + val numberOfLines = 100 + val stdout = ArrayDeque(numberOfLines) beforeTest(closureOf { stdout.clear() }) onOutput(KotlinClosure2({ _, toe -> - stdout.addAll(toe.message.split("(?m)$").toList()) - while (stdout.size > 100) { - stdout.remove() + toe.message.split("(?m)$").forEach { line -> + if (stdout.size == numberOfLines) { + stdout.removeFirst() + } + stdout.addLast(line) } })) diff --git a/radar-commons-server/build.gradle.kts b/radar-commons-server/build.gradle.kts index 24515ad1..63e9aa7d 100644 --- a/radar-commons-server/build.gradle.kts +++ b/radar-commons-server/build.gradle.kts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import org.jetbrains.dokka.gradle.DokkaTask import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { @@ -47,10 +48,16 @@ dependencies { testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") } +val generateAvroJava by tasks + tasks.withType { - dependsOn(tasks.named("generateAvroJava")) + dependsOn(generateAvroJava) } tasks.withType { - dependsOn(tasks.named("generateAvroJava")) + dependsOn(generateAvroJava) +} + +tasks.withType { + dependsOn(generateAvroJava) } From deb0dd9a32100e28992846f6d60a5a3a644346b9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 20 Mar 2023 12:17:57 +0100 Subject: [PATCH 10/25] Use buildSrc for build --- build.gradle.kts | 196 ++---------------- buildSrc/build.gradle.kts | 15 ++ .../main/kotlin/DependencyUpdatesPlugin.kt | 46 ++++ .../main/kotlin/KotlinConventionsPlugin.kt | 61 ++++++ .../src/main/kotlin/RadarPublishingPlugin.kt | 128 ++++++++++++ .../src/main/kotlin/RootConventionsPlugin.kt | 43 ++++ buildSrc/src/main/kotlin/Versions.kt | 26 +++ gradle.properties | 24 +-- radar-commons-kotlin/build.gradle.kts | 9 +- radar-commons-server/build.gradle.kts | 21 +- radar-commons-testing/build.gradle.kts | 34 ++- radar-commons/build.gradle.kts | 29 +-- settings.gradle.kts | 21 +- 13 files changed, 381 insertions(+), 272 deletions(-) create mode 100644 buildSrc/build.gradle.kts create mode 100644 buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt create mode 100644 buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt create mode 100644 buildSrc/src/main/kotlin/RadarPublishingPlugin.kt create mode 100644 buildSrc/src/main/kotlin/RootConventionsPlugin.kt create mode 100644 buildSrc/src/main/kotlin/Versions.kt diff --git a/build.gradle.kts b/build.gradle.kts index 3c96e360..93342449 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,3 @@ -import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile - /* * Copyright 2017 The Hyve and King's College London * @@ -17,166 +14,43 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile * limitations under the License. */ plugins { - kotlin("jvm") apply false - id("org.jetbrains.dokka") apply false - id("com.github.ben-manes.versions") - id("io.github.gradle-nexus.publish-plugin") - `maven-publish` - signing + kotlin("plugin.serialization") version Versions.Plugins.kotlinSerialization apply false + id("com.github.davidmc24.gradle.plugin.avro") version Versions.Plugins.avro apply false } val githubRepoName = "RADAR-base/radar-commons" val githubUrl = "https://github.com/$githubRepoName" -val githubIssueUrl = "https://github.com/$githubRepoName/issues" -val website = "https://radar-base.org" -allprojects { - version = "0.16.0-SNAPSHOT" - group = "org.radarbase" -} +apply() +apply() subprojects { - val myProject = this - // Apply the plugins - apply(plugin = "java") - apply(plugin = "java-library") - apply(plugin = "org.jetbrains.kotlin.jvm") - - tasks.withType { - targetCompatibility = JavaVersion.VERSION_11.toString() - sourceCompatibility = JavaVersion.VERSION_11.toString() - } - tasks.withType { - kotlinOptions { - jvmTarget = "11" - languageVersion = "1.8" - apiVersion = "1.8" - } - } - - //---------------------------------------------------------------------------// - // Dependencies // - //---------------------------------------------------------------------------// - repositories { - mavenCentral() - mavenLocal() - maven(url = "https://packages.confluent.io/maven/") - } + apply() + apply() dependencies { - val coroutinesVersion: String by project - configurations["testImplementation"]("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion") - - val junitVersion: String by project - configurations["testImplementation"]("org.junit.jupiter:junit-jupiter-api:$junitVersion") - configurations["testRuntimeOnly"]("org.junit.jupiter:junit-jupiter-engine:$junitVersion") + configurations["testImplementation"]("org.jetbrains.kotlinx:kotlinx-coroutines-test:${Versions.coroutines}") } - afterEvaluate { - configurations { - named("implementation") { - resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") + radarPublishing { + githubUrl.set("https://github.com/$githubRepoName") + developers { + developer { + id.set("blootsvoets") + name.set("Joris Borgdorff") + email.set("joris@thehyve.nl") + organization.set("The Hyve") } - } - } - - val sourcesJar by tasks.registering(Jar::class) { - from(myProject.the()["main"].allSource) - archiveClassifier.set("sources") - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - val classes by tasks - dependsOn(classes) - } - - apply(plugin = "org.jetbrains.dokka") - val dokkaJar by tasks.registering(Jar::class) { - from("$buildDir/dokka/javadoc") - archiveClassifier.set("javadoc") - val dokkaJavadoc by tasks - dependsOn(dokkaJavadoc) - } - - tasks.withType { - compression = Compression.GZIP - archiveExtension.set("tar.gz") - } - - tasks.withType { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version - ) - } - } - - apply(plugin = "maven-publish") - apply(plugin = "signing") - - val assemble by tasks - assemble.dependsOn(sourcesJar) - assemble.dependsOn(dokkaJar) - - val mavenJar by publishing.publications.creating(MavenPublication::class) { - from(components["java"]) - - artifact(sourcesJar) - artifact(dokkaJar) - - afterEvaluate { - pom { - name.set(myProject.name) - description.set(myProject.description) - url.set(githubUrl) - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - developers { - developer { - id.set("blootsvoets") - name.set("Joris Borgdorff") - email.set("joris@thehyve.nl") - organization.set("The Hyve") - } - developer { - id.set("nivemaham") - name.set("Nivethika Mahasivam") - email.set("nivethika@thehyve.nl") - organization.set("The Hyve") - } - } - issueManagement { - system.set("GitHub") - url.set(githubIssueUrl) - } - organization { - name.set("RADAR-base") - url.set("https://radar-base.org") - } - scm { - connection.set("scm:git:$githubUrl") - url.set(githubUrl) - } + developer { + id.set("nivemaham") + name.set("Nivethika Mahasivam") + email.set("nivethika@thehyve.nl") + organization.set("The Hyve") } } } - signing { - useGpgCmd() - isRequired = true - sign(tasks["sourcesJar"], tasks["dokkaJar"]) - sign(mavenJar) - } - - tasks.withType { - onlyIf { gradle.taskGraph.hasTask(myProject.tasks["publish"]) } - } - //---------------------------------------------------------------------------// // Style checking // //---------------------------------------------------------------------------// @@ -222,33 +96,3 @@ subprojects { } } } - -tasks.withType { - val acceptedVersion = "(RELEASE|FINAL|GA|-ce|^[0-9,.v-]+(-r)?)$" - .toRegex(RegexOption.IGNORE_CASE) - rejectVersionIf { - !acceptedVersion.containsMatchIn(candidate.version) - } -} - -fun Project.propertyOrEnv(propertyName: String, envName: String): String? { - return if (hasProperty(propertyName)) { - property(propertyName)?.toString() - } else { - System.getenv(envName) - } -} - -nexusPublishing { - repositories { - sonatype { - username.set(propertyOrEnv("ossrh.user", "OSSRH_USER")) - password.set(propertyOrEnv("ossrh.password", "OSSRH_PASSWORD")) - } - } -} - -tasks.wrapper { - val gradleWrapperVersion: String by project - gradleVersion = gradleWrapperVersion -} diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 00000000..0192ce2f --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,15 @@ +plugins { + `kotlin-dsl` +} + +repositories { + mavenCentral() + gradlePluginPortal() +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.10") + implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.8.10") + implementation("com.github.ben-manes:gradle-versions-plugin:0.46.0") + implementation("io.github.gradle-nexus:publish-plugin:1.3.0") +} diff --git a/buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt b/buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt new file mode 100644 index 00000000..6bbe25a1 --- /dev/null +++ b/buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt @@ -0,0 +1,46 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.artifacts.repositories.MavenArtifactRepository +import org.gradle.api.provider.Property +import org.gradle.kotlin.dsl.create +import com.github.benmanes.gradle.versions.VersionsPlugin +import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask +import org.gradle.kotlin.dsl.apply +import org.gradle.kotlin.dsl.configure + +fun Project.dependencyUpdates(configure: DependencyUpdatesPluginExtension.() -> Unit) { + configure(configure) +} + +interface DependencyUpdatesPluginExtension { + val regex: Property + val minorUpdatesOnly: Property +} + +class DependencyUpdatesPlugin : Plugin { + override fun apply(project: Project): Unit = with(project) { + val extension = extensions.create("radarDependencies").apply { + regex.convention("(^[0-9,.v-]+(-r)?|RELEASE|FINAL|GA|-CE)$") + minorUpdatesOnly.convention(false) + } + + apply() + + tasks.withType(DependencyUpdatesTask::class.java) { + doFirst { + allprojects { + repositories.removeAll { + it is MavenArtifactRepository && + it.url.toString().contains("snapshot", ignoreCase = true) + } + } + } + val isStable = extension.regex.get().toRegex(RegexOption.IGNORE_CASE) + val checkMinorOnly = extension.minorUpdatesOnly.get() + rejectVersionIf { + (!checkMinorOnly || candidate.version.split('.', limit = 2)[0] != currentVersion.split('.', limit = 2)[0]) + && !isStable.containsMatchIn(candidate.version) + } + } + } +} diff --git a/buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt b/buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt new file mode 100644 index 00000000..76d3fe5d --- /dev/null +++ b/buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt @@ -0,0 +1,61 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.compile.JavaCompile +import org.gradle.kotlin.dsl.* +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +class KotlinConventionsPlugin : Plugin { + override fun apply(project: Project) = with(project) { + apply(plugin = "kotlin") + + repositories { + mavenCentral() { + mavenContent { + releasesOnly() + } + } + mavenLocal() + maven(url = "https://packages.confluent.io/maven/") { + mavenContent { + releasesOnly() + } + } + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") { + mavenContent { + snapshotsOnly() + } + } + } + + dependencies { + configurations["testImplementation"]("org.junit.jupiter:junit-jupiter-api:${Versions.junit}") + configurations["testRuntimeOnly"]("org.junit.jupiter:junit-jupiter-engine:${Versions.junit}") + } + + tasks.withType { + options.release.set(Versions.java.toInt()) + } + + tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(Versions.java)) + val kotlinVersion = KotlinVersion.fromVersion( + Versions.Plugins.kotlinSerialization + .splitToSequence('.') + .take(2) + .joinToString(separator = "."), + ) + apiVersion.set(kotlinVersion) + languageVersion.set(kotlinVersion) + } + } + + afterEvaluate { + configurations.named("implementation") { + resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") + } + } + } +} diff --git a/buildSrc/src/main/kotlin/RadarPublishingPlugin.kt b/buildSrc/src/main/kotlin/RadarPublishingPlugin.kt new file mode 100644 index 00000000..0b084a8c --- /dev/null +++ b/buildSrc/src/main/kotlin/RadarPublishingPlugin.kt @@ -0,0 +1,128 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.DuplicatesStrategy +import org.gradle.api.provider.Property +import org.gradle.api.publish.PublishingExtension +import org.gradle.api.publish.maven.MavenPomDeveloperSpec +import org.gradle.api.publish.maven.MavenPublication +import org.gradle.api.publish.maven.plugins.MavenPublishPlugin +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.api.tasks.bundling.Compression +import org.gradle.api.tasks.bundling.Jar +import org.gradle.api.tasks.bundling.Tar +import org.gradle.kotlin.dsl.* +import org.gradle.plugins.signing.Sign +import org.gradle.plugins.signing.SigningExtension +import org.gradle.plugins.signing.SigningPlugin +import org.jetbrains.dokka.gradle.DokkaPlugin + +fun Project.radarPublishing(configure: RadarPublishingExtension.() -> Unit) { + configure(configure) +} + +interface RadarPublishingExtension { + val githubUrl: Property + val developers: Property Unit> + + fun developers(configure: MavenPomDeveloperSpec.() -> Unit) { + developers.set(configure) + } +} + +class RadarPublishingPlugin : Plugin { + override fun apply(project: Project): Unit = with(project) { + val extension = extensions.create("radarPublishing") + + val sourcesJar by tasks.registering(Jar::class) { + from(project.the()["main"].allSource) + archiveClassifier.set("sources") + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + val classes by tasks + dependsOn(classes) + } + + apply() + + val dokkaJar by tasks.registering(Jar::class) { + from("$buildDir/dokka/javadoc") + archiveClassifier.set("javadoc") + val dokkaJavadoc by tasks + dependsOn(dokkaJavadoc) + } + + tasks.withType { + compression = Compression.GZIP + archiveExtension.set("tar.gz") + } + + tasks.withType { + manifest { + attributes( + "Implementation-Title" to project.name, + "Implementation-Version" to project.version + ) + } + } + + apply() + + val assemble by tasks + assemble.dependsOn(sourcesJar) + assemble.dependsOn(dokkaJar) + + val publisingExtension = extensions.getByName("publishing") + val mavenJar by publisingExtension.publications.creating(MavenPublication::class) { + from(components["java"]) + + artifact(sourcesJar) + artifact(dokkaJar) + + afterEvaluate { + val githubUrl = requireNotNull(extension.githubUrl.orNull) { "Missing githubUrl value in radarPublishing" } + pom { + name.set(project.name) + description.set(project.description) + url.set(githubUrl) + licenses { + license { + name.set("The Apache Software License, Version 2.0") + url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") + distribution.set("repo") + } + } + if (extension.developers.isPresent) { + developers { + val developerBlock = extension.developers.get() + developerBlock() + } + } + issueManagement { + system.set("GitHub") + url.set("$githubUrl/issues") + } + organization { + name.set("RADAR-base") + url.set("https://radar-base.org") + } + scm { + connection.set("scm:git:$githubUrl") + url.set(githubUrl) + } + } + } + } + + apply() + + extensions.configure("signing") { + useGpgCmd() + isRequired = true + sign(tasks["sourcesJar"], tasks["dokkaJar"]) + sign(mavenJar) + } + + tasks.withType { + onlyIf { gradle.taskGraph.hasTask(tasks["publish"]) } + } + } +} diff --git a/buildSrc/src/main/kotlin/RootConventionsPlugin.kt b/buildSrc/src/main/kotlin/RootConventionsPlugin.kt new file mode 100644 index 00000000..25620c51 --- /dev/null +++ b/buildSrc/src/main/kotlin/RootConventionsPlugin.kt @@ -0,0 +1,43 @@ +import io.github.gradlenexus.publishplugin.NexusPublishExtension +import io.github.gradlenexus.publishplugin.NexusPublishPlugin +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.wrapper.Wrapper +import org.gradle.kotlin.dsl.apply +import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.named +import org.gradle.kotlin.dsl.repositories + +class RootConventionsPlugin : Plugin { + override fun apply(project: Project) = with(project) { + allprojects { + version = Versions.project + group = "org.radarbase" + } + + tasks.named("wrapper") { + gradleVersion = Versions.wrapper + } + + apply() + + project.extensions.configure { + repositories { + sonatype { + username.set(propertyOrEnv("ossrh.user", "OSSRH_USER")) + password.set(propertyOrEnv("ossrh.password", "OSSRH_PASSWORD")) + } + } + } + } + + companion object { + private fun Project.propertyOrEnv(propertyName: String, envName: String): String? { + return if (hasProperty(propertyName)) { + property(propertyName)?.toString() + } else { + System.getenv(envName) + } + } + } +} diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt new file mode 100644 index 00000000..acdf9770 --- /dev/null +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -0,0 +1,26 @@ +object Versions { + const val project = "0.16.0-SNAPSHOT" + const val wrapper = "8.0.2" + + object Plugins { + const val kotlinSerialization = "1.8.10" + const val avro = "1.6.0" + } + + const val java = "11" + const val slf4j = "2.0.7" + const val confluent = "7.3.2" + const val kafka = "7.3.2-ce" + const val avro = "1.11.1" + const val jackson = "2.14.2" + const val okhttp = "4.10.0" + const val junit = "5.9.2" + const val mockito = "5.2.0" + const val mockitoKotlin = "4.1.0" + const val hamcrest = "2.2" + const val radarSchemas = "0.8.2" + const val orgJson = "20220924" + const val opencsv = "5.7.1" + const val ktor = "2.2.4" + const val coroutines = "1.6.4" +} diff --git a/gradle.properties b/gradle.properties index 9ef87588..821e1274 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,23 +1 @@ -gradleWrapperVersion=8.0.2 - -kotlinVersion=1.8.10 -avroPluginVersion=1.6.0 -dependencyUpdatePluginVersion=0.46.0 -nexusPluginVersion=1.3.0 -dokkaVersion=1.8.10 - -slf4jVersion=2.0.6 -confluentVersion=7.3.1 -kafkaVersion=7.3.1-ce -avroVersion=1.11.1 -jacksonVersion=2.14.2 -okhttpVersion=4.10.0 -junitVersion=5.9.2 -mockitoVersion=5.2.0 -mockitoKotlinVersion=4.1.0 -hamcrestVersion=2.2 -radarSchemasVersion=0.8.2 -orgJsonVersion=20220924 -opencsvVersion=5.7.1 -ktorVersion=2.2.4 -coroutinesVersion=1.6.4 +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 diff --git a/radar-commons-kotlin/build.gradle.kts b/radar-commons-kotlin/build.gradle.kts index 4b84bf4b..65416862 100644 --- a/radar-commons-kotlin/build.gradle.kts +++ b/radar-commons-kotlin/build.gradle.kts @@ -5,15 +5,12 @@ plugins { description = "Library for Kotlin utility classes and functions" dependencies { - val slf4jVersion: String by project - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation("org.slf4j:slf4j-api:${Versions.slf4j}") - val coroutinesVersion: String by project - api(platform("org.jetbrains.kotlinx:kotlinx-coroutines-bom:$coroutinesVersion")) + api(platform("org.jetbrains.kotlinx:kotlinx-coroutines-bom:${Versions.coroutines}")) api("org.jetbrains.kotlinx:kotlinx-coroutines-core") - val ktorVersion: String by project - api(platform("io.ktor:ktor-bom:$ktorVersion")) + api(platform("io.ktor:ktor-bom:${Versions.ktor}")) api("io.ktor:ktor-client-auth") implementation("io.ktor:ktor-client-content-negotiation") implementation("io.ktor:ktor-serialization-kotlinx-json") diff --git a/radar-commons-server/build.gradle.kts b/radar-commons-server/build.gradle.kts index 63e9aa7d..c20f826d 100644 --- a/radar-commons-server/build.gradle.kts +++ b/radar-commons-server/build.gradle.kts @@ -26,26 +26,19 @@ dependencies { api(project(":radar-commons")) // For POJO classes and ConfigLoader - val jacksonVersion: String by project - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") implementation("com.fasterxml.jackson.core:jackson-databind") - val avroVersion: String by project - api("org.apache.avro:avro:$avroVersion") + api("org.apache.avro:avro:${Versions.avro}") - val kafkaVersion: String by project - implementation("org.apache.kafka:kafka-clients:$kafkaVersion") + implementation("org.apache.kafka:kafka-clients:${Versions.kafka}") - val mockitoVersion: String by project - testImplementation("org.mockito:mockito-core:$mockitoVersion") + testImplementation("org.mockito:mockito-core:${Versions.mockito}") // Direct producer uses KafkaAvroSerializer if initialized - val confluentVersion: String by project - testImplementation("io.confluent:kafka-avro-serializer:$confluentVersion") - val radarSchemasVersion: String by project - testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion") - val slf4jVersion: String by project - testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") + testImplementation("io.confluent:kafka-avro-serializer:${Versions.confluent}") + testImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") + testRuntimeOnly("org.slf4j:slf4j-simple:${Versions.slf4j}") } val generateAvroJava by tasks diff --git a/radar-commons-testing/build.gradle.kts b/radar-commons-testing/build.gradle.kts index a4a46584..e9c5694d 100644 --- a/radar-commons-testing/build.gradle.kts +++ b/radar-commons-testing/build.gradle.kts @@ -39,31 +39,23 @@ dependencies { api(project(":radar-commons")) api(project(":radar-commons-server")) api(project(":radar-commons-kotlin")) - val avroVersion: String by project - api("org.apache.avro:avro:$avroVersion") - val radarSchemasVersion: String by project - api("org.radarbase:radar-schemas-commons:$radarSchemasVersion") - val opencsvVersion: String by project - implementation("com.opencsv:opencsv:$opencsvVersion") - val jacksonVersion: String by project - implementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + api("org.apache.avro:avro:${Versions.avro}") + api("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") + + implementation("com.opencsv:opencsv:${Versions.opencsv}") + implementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) implementation("com.fasterxml.jackson.core:jackson-databind") - val kafkaVersion: String by project - implementation("org.apache.kafka:kafka-clients:$kafkaVersion") - val confluentVersion: String by project - implementation("io.confluent:kafka-avro-serializer:$confluentVersion") - val ktorVersion: String by project - implementation(platform("io.ktor:ktor-bom:$ktorVersion")) + implementation("org.apache.kafka:kafka-clients:${Versions.kafka}") + implementation("io.confluent:kafka-avro-serializer:${Versions.confluent}") + + implementation(platform("io.ktor:ktor-bom:${Versions.ktor}")) implementation("io.ktor:ktor-serialization-kotlinx-json") - val slf4jVersion: String by project - applicationRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") + applicationRuntimeOnly("org.slf4j:slf4j-simple:${Versions.slf4j}") - val hamcrestVersion: String by project - testImplementation("org.hamcrest:hamcrest:$hamcrestVersion") - testImplementation("org.slf4j:slf4j-simple:$slf4jVersion") - val mockitoVersion: String by project - testImplementation("org.mockito:mockito-core:$mockitoVersion") + testImplementation("org.hamcrest:hamcrest:${Versions.hamcrest}") + testImplementation("org.slf4j:slf4j-simple:${Versions.slf4j}") + testImplementation("org.mockito:mockito-core:${Versions.mockito}") } diff --git a/radar-commons/build.gradle.kts b/radar-commons/build.gradle.kts index 319e1407..ccca3830 100644 --- a/radar-commons/build.gradle.kts +++ b/radar-commons/build.gradle.kts @@ -15,37 +15,28 @@ repositories { // In this section you declare the dependencies for your production and test code dependencies { - val avroVersion: String by project - api("org.apache.avro:avro:$avroVersion") + api("org.apache.avro:avro:${Versions.avro}") api(kotlin("reflect")) implementation(project(":radar-commons-kotlin")) - val ktorVersion: String by project - api(platform("io.ktor:ktor-bom:$ktorVersion")) + api(platform("io.ktor:ktor-bom:${Versions.ktor}")) api("io.ktor:ktor-client-core") api("io.ktor:ktor-client-cio") api("io.ktor:ktor-client-auth") implementation("io.ktor:ktor-client-content-negotiation") implementation("io.ktor:ktor-serialization-kotlinx-json") - val coroutinesVersion: String by project - api("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.coroutines}") // The production code uses the SLF4J logging API at compile time - val slf4jVersion: String by project - implementation("org.slf4j:slf4j-api:$slf4jVersion") + implementation("org.slf4j:slf4j-api:${Versions.slf4j}") - val jacksonVersion: String by project - testImplementation(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")) + testImplementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) testImplementation("com.fasterxml.jackson.core:jackson-databind") - val radarSchemasVersion: String by project - testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion") - val mockitoVersion: String by project - testImplementation("org.mockito:mockito-core:$mockitoVersion") - val mockitoKotlinVersion: String by project - testImplementation("org.mockito.kotlin:mockito-kotlin:$mockitoKotlinVersion") - val okhttpVersion: String by project - testImplementation("com.squareup.okhttp3:mockwebserver:$okhttpVersion") - testRuntimeOnly("org.slf4j:slf4j-simple:$slf4jVersion") + testImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") + testImplementation("org.mockito:mockito-core:${Versions.mockito}") + testImplementation("org.mockito.kotlin:mockito-kotlin:${Versions.mockitoKotlin}") + testImplementation("com.squareup.okhttp3:mockwebserver:${Versions.okhttp}") + testRuntimeOnly("org.slf4j:slf4j-simple:${Versions.slf4j}") } diff --git a/settings.gradle.kts b/settings.gradle.kts index b5556838..19f1e0e9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,19 +20,14 @@ include(":radar-commons-testing") include(":radar-commons-server") pluginManagement { - plugins { - val kotlinVersion: String by settings - kotlin("jvm") version kotlinVersion - kotlin("plugin.serialization") version kotlinVersion - - val avroPluginVersion: String by settings - id("com.github.davidmc24.gradle.plugin.avro") version avroPluginVersion - val nexusPluginVersion: String by settings - id("io.github.gradle-nexus.publish-plugin") version nexusPluginVersion + val kotlin = "1.8.10" + val avro = "1.6.0" + val dependencyUpdate = "0.46.0" + val nexus = "1.3.0" + val dokka = "1.8.10" - val dependencyUpdatePluginVersion: String by settings - id("com.github.ben-manes.versions") version dependencyUpdatePluginVersion - val dokkaVersion: String by settings - id("org.jetbrains.dokka") version dokkaVersion + plugins { + kotlin("plugin.serialization") version kotlin + id("com.github.davidmc24.gradle.plugin.avro") version avro } } From a5b2d1961c99721f009b45acc74301d0c471b0c1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 22 Mar 2023 11:55:38 +0100 Subject: [PATCH 11/25] Make separate plugin for configuring RADAR projects --- .editorconfig | 26 ++ .github/workflows/main.yml | 9 + .github/workflows/publish_snapshots.yml | 15 +- .github/workflows/release.yml | 14 + build.gradle.kts | 21 +- buildSrc/build.gradle.kts | 1 + .../main/kotlin/KotlinConventionsPlugin.kt | 61 ----- buildSrc/src/main/kotlin/Versions.kt | 7 +- gradle.properties | 2 + radar-commons-gradle/build.gradle.kts | 57 ++++ .../plugin/RadarDependencyManagementPlugin.kt | 18 +- .../gradle/plugin/RadarKotlinPlugin.kt | 89 +++++++ .../gradle/plugin}/RadarPublishingPlugin.kt | 8 +- .../gradle/plugin/RadarRootProjectPlugin.kt | 34 ++- .../org/radarbase/gradle/plugin/Versions.kt | 9 + .../radarbase/kotlin/coroutines/CachedMap.kt | 6 +- .../radarbase/kotlin/coroutines/CachedSet.kt | 2 +- .../kotlin/coroutines/CachedValue.kt | 5 +- .../radarbase/kotlin/coroutines/Extensions.kt | 12 +- .../radarbase/ktor/auth/AuthTokenHolder.kt | 2 +- .../ktor/auth/OAuthClientProvider.kt | 4 +- .../kotlin/coroutines/CachedValueTest.kt | 34 ++- .../kotlin/coroutines/ExtensionsKtTest.kt | 1 - .../org/radarbase/util/RollingTimeAverage.kt | 4 +- .../java/org/radarbase/mock/MockDevice.kt | 2 +- .../java/org/radarbase/mock/MockFileSender.kt | 8 +- .../java/org/radarbase/mock/MockProducer.kt | 43 +-- .../radarbase/mock/data/HeaderHierarchy.kt | 2 +- .../org/radarbase/mock/data/MockCsvParser.kt | 41 +-- .../mock/data/MockRecordValidator.kt | 16 +- .../java/org/radarbase/util/Oscilloscope.kt | 2 +- radar-commons/build.gradle.kts | 4 +- .../java/org/radarbase/config/ServerConfig.kt | 15 +- .../org/radarbase/data/AvroDatumDecoder.kt | 11 +- .../org/radarbase/data/AvroDatumEncoder.kt | 2 +- .../java/org/radarbase/data/AvroEncoder.kt | 7 +- .../java/org/radarbase/data/AvroRecordData.kt | 2 +- .../org/radarbase/data/AvroRecordWriter.kt | 2 +- .../java/org/radarbase/data/RecordData.kt | 2 +- .../org/radarbase/data/RemoteSchemaEncoder.kt | 6 +- .../org/radarbase/producer/KafkaSender.kt | 5 +- .../radarbase/producer/KafkaTopicSender.kt | 2 +- .../producer/avro/AvroDataMapperFactory.kt | 54 ++-- .../producer/io/DirectBinaryEncoder.kt | 2 +- .../java/org/radarbase/producer/io/Encoder.kt | 8 +- .../producer/io/GzipContentEncoding.kt | 2 - .../producer/io/HttpClientExtensions.kt | 2 +- .../io/UnsupportedMediaTypeException.kt | 2 +- .../producer/rest/AvroContentConverter.kt | 8 +- .../producer/rest/BinaryRecordContent.kt | 8 +- .../producer/rest/ConnectionState.kt | 2 +- .../producer/rest/JsonRecordContent.kt | 6 +- .../producer/rest/RestKafkaSender.kt | 29 ++- .../producer/schema/SchemaMetadata.kt | 16 +- .../producer/schema/SchemaRestClient.kt | 18 +- .../producer/schema/SchemaRetriever.kt | 18 +- .../java/org/radarbase/topic/AvroTopic.kt | 12 +- .../java/org/radarbase/topic/KafkaTopic.kt | 8 +- .../java/org/radarbase/topic/SensorTopic.kt | 8 +- .../java/org/radarbase/util/Annotations.kt | 8 - .../util/{Base64.kt => Base64Encoder.kt} | 14 +- .../org/radarbase/util/RadarProducerDsl.kt | 13 + .../main/java/org/radarbase/util/Strings.kt | 10 +- .../avro/AvroDataMapperFactoryTest.kt | 244 +++++++++--------- .../producer/rest/BinaryRecordContentTest.kt | 41 +-- .../producer/rest/RestKafkaSenderTest.kt | 26 +- .../producer/schema/SchemaRestClientTest.kt | 3 +- .../producer/schema/SchemaRetrieverTest.kt | 6 +- .../org/radarbase/topic/SensorTopicTest.kt | 46 ++-- settings.gradle.kts | 14 +- 70 files changed, 753 insertions(+), 488 deletions(-) create mode 100644 .editorconfig delete mode 100644 buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt create mode 100644 radar-commons-gradle/build.gradle.kts rename buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt => radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt (79%) create mode 100644 radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt rename {buildSrc/src/main/kotlin => radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin}/RadarPublishingPlugin.kt (94%) rename buildSrc/src/main/kotlin/RootConventionsPlugin.kt => radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarRootProjectPlugin.kt (54%) create mode 100644 radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/Versions.kt delete mode 100644 radar-commons/src/main/java/org/radarbase/util/Annotations.kt rename radar-commons/src/main/java/org/radarbase/util/{Base64.kt => Base64Encoder.kt} (95%) create mode 100644 radar-commons/src/main/java/org/radarbase/util/RadarProducerDsl.kt diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..ce7dcfe8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig helps developers define and maintain consistent +# coding styles between different editors and IDEs +# editorconfig.org + +root = true + +[*] +# Change these settings to your own preference +indent_style = space +indent_size = 4 + +# We recommend you to keep these unchanged +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{kt,kts}] +ktlint_standard_no-wildcard-imports = disabled + +[*.md] +trim_trailing_whitespace = false + +[*.{json,yaml,yml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e9e0d2f..fdf9e8ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,3 +33,12 @@ jobs: # Gradle check - name: Check run: ./gradlew check + + - name: Check version + run: + projectVersion=$(./gradlew properties | grep '^version:.*$') + pluginVersion=$(./gradlew :radar-commons-plugin:properties | grep '^version:.*$') + if [ "$projectVersion" != "$pluginVersion" ]; then + echo "Project version $projectVersion does not match plugin version $pluginVersion" + exit 1 + fi diff --git a/.github/workflows/publish_snapshots.yml b/.github/workflows/publish_snapshots.yml index 0467415f..dd68ccd6 100644 --- a/.github/workflows/publish_snapshots.yml +++ b/.github/workflows/publish_snapshots.yml @@ -17,10 +17,6 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - - name: Has SNAPSHOT version - id: is-snapshot - run: grep "version = '.*-SNAPSHOT'" build.gradle - - uses: actions/setup-java@v3 with: distribution: temurin @@ -29,6 +25,9 @@ jobs: - name: Setup Gradle uses: gradle/gradle-build-action@v2 + - name: Has SNAPSHOT version + run: ./gradlew properties | grep '^version:.*-SNAPSHOT$' + - name: Install gpg secret key run: | cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import @@ -39,3 +38,11 @@ jobs: OSSRH_USER: ${{ secrets.OSSRH_USER }} OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} run: ./gradlew -Psigning.gnupg.keyName=CBEF2CF0 -Psigning.gnupg.executable=gpg -Psigning.gnupg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} publish + + - name: Plugin has SNAPSHOT version + run: ./gradlew :radar-commons-plugin:properties | grep '^version:.*-SNAPSHOT$' + + - name: Publish gradle plugin + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./gradlew :radar-commons-plugin:publish diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0686b60..69d204de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,15 @@ jobs: - name: Setup Gradle uses: gradle/gradle-build-action@v2 + - name: Check version + run: + projectVersion=$(./gradlew properties | grep '^version:.*$') + pluginVersion=$(./gradlew :radar-commons-plugin:properties | grep '^version:.*$') + if [ "$projectVersion" != "$pluginVersion" ]; then + echo "Project version $projectVersion does not match plugin version $pluginVersion" + exit 1 + fi + # Compile code - name: Compile code run: ./gradlew assemble @@ -44,3 +53,8 @@ jobs: OSSRH_USER: ${{ secrets.OSSRH_USER }} OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} run: ./gradlew -Psigning.gnupg.keyName=CBEF2CF0 -Psigning.gnupg.executable=gpg -Psigning.gnupg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} publish closeAndReleaseSonatypeStagingRepository + + - name: Publish gradle plugin + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./gradlew :radar-commons-plugin:publish diff --git a/build.gradle.kts b/build.gradle.kts index 93342449..46491766 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -13,21 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import org.radarbase.gradle.plugin.radarKotlin +import org.radarbase.gradle.plugin.radarPublishing +import org.radarbase.gradle.plugin.radarRootProject + plugins { kotlin("plugin.serialization") version Versions.Plugins.kotlinSerialization apply false id("com.github.davidmc24.gradle.plugin.avro") version Versions.Plugins.avro apply false + id("org.radarbase.radar-root-project") + id("org.radarbase.radar-dependency-management") } val githubRepoName = "RADAR-base/radar-commons" val githubUrl = "https://github.com/$githubRepoName" -apply() -apply() +radarRootProject { + projectVersion.set(Versions.project) +} subprojects { // Apply the plugins - apply() - apply() + apply(plugin = "org.radarbase.radar-kotlin") + apply(plugin = "org.radarbase.radar-publishing") dependencies { configurations["testImplementation"]("org.jetbrains.kotlinx:kotlinx-coroutines-test:${Versions.coroutines}") @@ -51,6 +58,12 @@ subprojects { } } + radarKotlin { + javaVersion.set(Versions.java) + kotlinVersion.set(Versions.Plugins.kotlin) + junitVersion.set(Versions.junit) + } + //---------------------------------------------------------------------------// // Style checking // //---------------------------------------------------------------------------// diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 0192ce2f..0b7c1c73 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -12,4 +12,5 @@ dependencies { implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.8.10") implementation("com.github.ben-manes:gradle-versions-plugin:0.46.0") implementation("io.github.gradle-nexus:publish-plugin:1.3.0") + implementation("org.jlleitschuh.gradle:ktlint-gradle:11.3.1") } diff --git a/buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt b/buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt deleted file mode 100644 index 76d3fe5d..00000000 --- a/buildSrc/src/main/kotlin/KotlinConventionsPlugin.kt +++ /dev/null @@ -1,61 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.compile.JavaCompile -import org.gradle.kotlin.dsl.* -import org.jetbrains.kotlin.gradle.dsl.JvmTarget -import org.jetbrains.kotlin.gradle.dsl.KotlinVersion -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile - -class KotlinConventionsPlugin : Plugin { - override fun apply(project: Project) = with(project) { - apply(plugin = "kotlin") - - repositories { - mavenCentral() { - mavenContent { - releasesOnly() - } - } - mavenLocal() - maven(url = "https://packages.confluent.io/maven/") { - mavenContent { - releasesOnly() - } - } - maven(url = "https://oss.sonatype.org/content/repositories/snapshots") { - mavenContent { - snapshotsOnly() - } - } - } - - dependencies { - configurations["testImplementation"]("org.junit.jupiter:junit-jupiter-api:${Versions.junit}") - configurations["testRuntimeOnly"]("org.junit.jupiter:junit-jupiter-engine:${Versions.junit}") - } - - tasks.withType { - options.release.set(Versions.java.toInt()) - } - - tasks.withType { - compilerOptions { - jvmTarget.set(JvmTarget.fromTarget(Versions.java)) - val kotlinVersion = KotlinVersion.fromVersion( - Versions.Plugins.kotlinSerialization - .splitToSequence('.') - .take(2) - .joinToString(separator = "."), - ) - apiVersion.set(kotlinVersion) - languageVersion.set(kotlinVersion) - } - } - - afterEvaluate { - configurations.named("implementation") { - resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") - } - } - } -} diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index acdf9770..f3e07b51 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,13 +1,13 @@ object Versions { const val project = "0.16.0-SNAPSHOT" - const val wrapper = "8.0.2" object Plugins { - const val kotlinSerialization = "1.8.10" + const val kotlin = "1.8.10" + const val kotlinSerialization = kotlin const val avro = "1.6.0" } - const val java = "11" + const val java = 11 const val slf4j = "2.0.7" const val confluent = "7.3.2" const val kafka = "7.3.2-ce" @@ -19,7 +19,6 @@ object Versions { const val mockitoKotlin = "4.1.0" const val hamcrest = "2.2" const val radarSchemas = "0.8.2" - const val orgJson = "20220924" const val opencsv = "5.7.1" const val ktor = "2.2.4" const val coroutines = "1.6.4" diff --git a/gradle.properties b/gradle.properties index 821e1274..5d0dd4f5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1,3 @@ org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 + +projectVersion=0.16.0-SNAPSHOT diff --git a/radar-commons-gradle/build.gradle.kts b/radar-commons-gradle/build.gradle.kts new file mode 100644 index 00000000..84753926 --- /dev/null +++ b/radar-commons-gradle/build.gradle.kts @@ -0,0 +1,57 @@ +plugins { + `kotlin-dsl` + `java-gradle-plugin` + kotlin("jvm") version "1.8.10" + `maven-publish` +} + +version = "0.16.0-SNAPSHOT" +group = "org.radarbase" +description = "RADAR common Gradle plugins" + +repositories { + mavenCentral() + gradlePluginPortal() +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.10") + implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.8.10") + implementation("com.github.ben-manes:gradle-versions-plugin:0.46.0") + implementation("io.github.gradle-nexus:publish-plugin:1.3.0") + implementation("org.jlleitschuh.gradle:ktlint-gradle:11.3.1") +} + +gradlePlugin { + plugins { + create("radarRootProject") { + id = "org.radarbase.radar-root-project" + implementationClass = "org.radarbase.gradle.plugin.RadarRootProjectPlugin" + } + create("radarPublishing") { + id = "org.radarbase.radar-publishing" + implementationClass = "org.radarbase.gradle.plugin.RadarPublishingPlugin" + } + create("radarDependencyManagement") { + id = "org.radarbase.radar-dependency-management" + implementationClass = "org.radarbase.gradle.plugin.RadarDependencyManagementPlugin" + } + create("radarKotlin") { + id = "org.radarbase.radar-kotlin" + implementationClass = "org.radarbase.gradle.plugin.RadarKotlinPlugin" + } + } +} + +publishing { + repositories { + maven { + name = "GitHubPackages" + setUrl("https://maven.pkg.github.com/radar-base/radar-commons") + credentials { + username = System.getenv("GITHUB_ACTOR") + password = System.getenv("GITHUB_TOKEN") + } + } + } +} diff --git a/buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt similarity index 79% rename from buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt rename to radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt index 6bbe25a1..2b370813 100644 --- a/buildSrc/src/main/kotlin/DependencyUpdatesPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt @@ -1,25 +1,27 @@ +package org.radarbase.gradle.plugin + +import com.github.benmanes.gradle.versions.VersionsPlugin +import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.artifacts.repositories.MavenArtifactRepository import org.gradle.api.provider.Property -import org.gradle.kotlin.dsl.create -import com.github.benmanes.gradle.versions.VersionsPlugin -import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask import org.gradle.kotlin.dsl.apply import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.create -fun Project.dependencyUpdates(configure: DependencyUpdatesPluginExtension.() -> Unit) { - configure(configure) +fun Project.radarDependencyManagement(configure: RadarDependencyManagementExtension.() -> Unit) { + configure(configure) } -interface DependencyUpdatesPluginExtension { +interface RadarDependencyManagementExtension { val regex: Property val minorUpdatesOnly: Property } -class DependencyUpdatesPlugin : Plugin { +class RadarDependencyManagementPlugin : Plugin { override fun apply(project: Project): Unit = with(project) { - val extension = extensions.create("radarDependencies").apply { + val extension = extensions.create("radarDependencies").apply { regex.convention("(^[0-9,.v-]+(-r)?|RELEASE|FINAL|GA|-CE)$") minorUpdatesOnly.convention(false) } diff --git a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt new file mode 100644 index 00000000..e42f301a --- /dev/null +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt @@ -0,0 +1,89 @@ +package org.radarbase.gradle.plugin + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.provider.Property +import org.gradle.api.tasks.compile.JavaCompile +import org.gradle.kotlin.dsl.* +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jlleitschuh.gradle.ktlint.KtlintExtension +import org.jlleitschuh.gradle.ktlint.KtlintPlugin + +fun Project.radarKotlin(configure: RadarKotlinExtension.() -> Unit) { + configure(configure) +} + +interface RadarKotlinExtension { + val javaVersion: Property + val kotlinVersion: Property + val junitVersion: Property + val ktlintVersion: Property +} + +class RadarKotlinPlugin : Plugin { + override fun apply(project: Project): Unit = with(project) { + val extension = extensions.create("radarKotlin").apply { + javaVersion.convention(Versions.java) + kotlinVersion.convention(Versions.kotlin) + junitVersion.convention(Versions.junit) + ktlintVersion.convention(Versions.ktlint) + } + + apply(plugin = "kotlin") + apply() + + repositories { + mavenCentral() { + mavenContent { + releasesOnly() + } + } + mavenLocal() + maven(url = "https://packages.confluent.io/maven/") { + mavenContent { + releasesOnly() + } + } + maven(url = "https://oss.sonatype.org/content/repositories/snapshots") { + mavenContent { + snapshotsOnly() + } + } + } + + tasks.withType { + options.release.set(extension.javaVersion) + } + + tasks.withType { + compilerOptions { + jvmTarget.set(extension.javaVersion.map { JvmTarget.fromTarget(it.toString()) }) + val kotlinVersion = extension.kotlinVersion.map { version -> + KotlinVersion.fromVersion( + version + .splitToSequence('.') + .take(2) + .joinToString(separator = "."), + ) + } + apiVersion.set(kotlinVersion) + languageVersion.set(kotlinVersion) + } + } + + extensions.configure { + version.set(extension.ktlintVersion) + } + + dependencies { + configurations["testImplementation"](extension.junitVersion.map { "org.junit.jupiter:junit-jupiter-api:$it" }) + configurations["testRuntimeOnly"](extension.junitVersion.map { "org.junit.jupiter:junit-jupiter-engine:$it" }) + } + + configurations.named("implementation") { + resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") + } + } +} diff --git a/buildSrc/src/main/kotlin/RadarPublishingPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarPublishingPlugin.kt similarity index 94% rename from buildSrc/src/main/kotlin/RadarPublishingPlugin.kt rename to radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarPublishingPlugin.kt index 0b084a8c..1093d576 100644 --- a/buildSrc/src/main/kotlin/RadarPublishingPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarPublishingPlugin.kt @@ -1,3 +1,5 @@ +package org.radarbase.gradle.plugin + import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.file.DuplicatesStrategy @@ -17,7 +19,7 @@ import org.gradle.plugins.signing.SigningPlugin import org.jetbrains.dokka.gradle.DokkaPlugin fun Project.radarPublishing(configure: RadarPublishingExtension.() -> Unit) { - configure(configure) + configure(configure) } interface RadarPublishingExtension { @@ -70,8 +72,8 @@ class RadarPublishingPlugin : Plugin { assemble.dependsOn(sourcesJar) assemble.dependsOn(dokkaJar) - val publisingExtension = extensions.getByName("publishing") - val mavenJar by publisingExtension.publications.creating(MavenPublication::class) { + val publishingExtension = extensions.getByName("publishing") + val mavenJar by publishingExtension.publications.creating(MavenPublication::class) { from(components["java"]) artifact(sourcesJar) diff --git a/buildSrc/src/main/kotlin/RootConventionsPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarRootProjectPlugin.kt similarity index 54% rename from buildSrc/src/main/kotlin/RootConventionsPlugin.kt rename to radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarRootProjectPlugin.kt index 25620c51..e83edc8c 100644 --- a/buildSrc/src/main/kotlin/RootConventionsPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarRootProjectPlugin.kt @@ -1,22 +1,44 @@ +package org.radarbase.gradle.plugin + import io.github.gradlenexus.publishplugin.NexusPublishExtension import io.github.gradlenexus.publishplugin.NexusPublishPlugin import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.provider.Property import org.gradle.api.tasks.wrapper.Wrapper import org.gradle.kotlin.dsl.apply import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.create import org.gradle.kotlin.dsl.named -import org.gradle.kotlin.dsl.repositories -class RootConventionsPlugin : Plugin { +fun Project.radarRootProject(configure: RadarRootProjectExtension.() -> Unit) { + configure(configure) +} + +interface RadarRootProjectExtension { + val group: Property + val projectVersion: Property + val gradleVersion: Property +} + +class RadarRootProjectPlugin : Plugin { override fun apply(project: Project) = with(project) { + val extension = extensions.create("radarRootProject").apply { + group.convention("org.radarbase") + gradleVersion.convention(Versions.wrapper) + } + allprojects { - version = Versions.project - group = "org.radarbase" + afterEvaluate { + version = extension.projectVersion.get() + group = extension.group.get() + } } - tasks.named("wrapper") { - gradleVersion = Versions.wrapper + afterEvaluate { + tasks.named("wrapper") { + gradleVersion = extension.gradleVersion.get() + } } apply() diff --git a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/Versions.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/Versions.kt new file mode 100644 index 00000000..5cbfa17b --- /dev/null +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/Versions.kt @@ -0,0 +1,9 @@ +package org.radarbase.gradle.plugin + +object Versions { + const val wrapper = "8.0.2" + const val kotlin = "1.8.10" + const val ktlint = "0.48.2" + const val java = 11 + const val junit = "5.9.2" +} diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt index 52402515..54f1f209 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedMap.kt @@ -17,10 +17,10 @@ package org.radarbase.kotlin.coroutines /** Set of data that is cached for a duration of time. */ -class CachedMap( +class CachedMap( cacheConfig: CacheConfig = CacheConfig(), - supplier: suspend () -> Map, -): CachedValue>(cacheConfig, supplier) { + supplier: suspend () -> Map, +) : CachedValue>(cacheConfig, supplier) { /** Whether the cache contains [key]. If it does not contain the value and [CacheConfig.retryDuration] * has passed since the last try, it will update the cache and try once more. */ suspend fun contains(key: K): Boolean = test { key in it } diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt index 361bd89d..4b79c495 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedSet.kt @@ -24,7 +24,7 @@ package org.radarbase.kotlin.coroutines class CachedSet( cacheConfig: CacheConfig = CacheConfig(), supplier: suspend () -> Set, -): CachedValue>(cacheConfig, supplier) { +) : CachedValue>(cacheConfig, supplier) { /** Whether the cache contains [value]. If it does not contain the value and [CacheConfig.retryDuration] * has passed since the last try, it will update the cache and try once more. */ suspend fun contains(value: T): Boolean = test { value in it } diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt index 7e8a0dbb..914ea074 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/CachedValue.kt @@ -190,13 +190,13 @@ open class CachedValue( } inline fun CacheContents.isExpired( - evaluateValid: (R) -> Boolean = { true } + evaluateValid: (R) -> Boolean = { true }, ): Boolean = if (this is CacheError) { isExpired(config.exceptionCacheDuration) } else { this as CacheValue isExpired(config.refreshDuration) || - (!evaluateValid(value) && isExpired(config.retryDuration)) + (!evaluateValid(value) && isExpired(config.retryDuration)) } /** @@ -227,6 +227,7 @@ open class CachedValue( ) : CacheContents() { override fun isExpired(age: Duration): Boolean = exception is CancellationException || super.isExpired(age) override fun getOrThrow(): T = throw exception + @Suppress("UNCHECKED_CAST") override suspend fun map(transform: suspend (T) -> R): CacheContents = this as CacheError } diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt index 87cbf27f..33d6fc7e 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/kotlin/coroutines/Extensions.kt @@ -61,7 +61,7 @@ suspend fun Future.suspendGet( */ suspend inline fun Iterable.forkJoin( coroutineContext: CoroutineContext = Dispatchers.Default, - crossinline transform: suspend CoroutineScope.(T) -> R + crossinline transform: suspend CoroutineScope.(T) -> R, ): List = coroutineScope { map { t -> async(coroutineContext) { transform(t) } } .awaitAll() @@ -75,7 +75,7 @@ suspend inline fun Iterable.forkJoin( */ suspend inline fun consumeFirst( coroutineContext: CoroutineContext = Dispatchers.Default, - crossinline producer: suspend CoroutineScope.(emit: suspend (T) -> Unit) -> Unit + crossinline producer: suspend CoroutineScope.(emit: suspend (T) -> Unit) -> Unit, ): T = coroutineScope { val channel = Channel() @@ -97,7 +97,7 @@ suspend inline fun consumeFirst( * true. Each value is transformed and evaluated in its own async context. If no transformed value * satisfies predicate, null is returned. */ -suspend fun Iterable.forkFirstOfOrNull( +suspend fun Iterable.forkFirstOfOrNull( coroutineContext: CoroutineContext = EmptyCoroutineContext, transform: suspend CoroutineScope.(T) -> R, predicate: suspend CoroutineScope.(R) -> Boolean, @@ -111,9 +111,9 @@ suspend fun Iterable.forkFirstOfOrNull( emit(null) } -suspend fun Iterable.forkFirstOfNotNullOrNull( +suspend fun Iterable.forkFirstOfNotNullOrNull( coroutineContext: CoroutineContext = EmptyCoroutineContext, - transform: suspend CoroutineScope.(T) -> R? + transform: suspend CoroutineScope.(T) -> R?, ): R? = forkFirstOfOrNull(coroutineContext, transform) { it != null } /** @@ -123,7 +123,7 @@ suspend fun Iterable.forkFirstOfNotNullOrNull( */ suspend fun Iterable.forkAny( coroutineContext: CoroutineContext = EmptyCoroutineContext, - predicate: suspend CoroutineScope.(T) -> Boolean + predicate: suspend CoroutineScope.(T) -> Boolean, ): Boolean = forkFirstOfOrNull(coroutineContext, predicate) { it } ?: false operator fun Set.plus(elements: Set): Set = when { diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt index 9c1dd8c7..ac1abfa7 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/AuthTokenHolder.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.CompletableDeferred import java.util.concurrent.atomic.AtomicReference internal class AuthTokenHolder( - private val loadTokens: suspend () -> T? + private val loadTokens: suspend () -> T?, ) { private val refreshTokensDeferred = AtomicReference?>(null) private val loadTokensDeferred = AtomicReference?>(null) diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt index 0e179db7..26df4678 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/OAuthClientProvider.kt @@ -32,7 +32,7 @@ fun Auth.clientCredentials( targetHost: String? = null, ): Flow { requireNotNull(authConfig.clientId) { "Missing client ID" } - requireNotNull(authConfig.clientSecret) { "Missing client secret"} + requireNotNull(authConfig.clientSecret) { "Missing client secret" } val flow = MutableStateFlow(null) clientCredentials { @@ -48,7 +48,7 @@ fun Auth.clientCredentials( append("grant_type", "client_credentials") append("client_id", authConfig.clientId) append("client_secret", authConfig.clientSecret) - } + }, ) { accept(ContentType.Application.Json) markAsRequestTokenRequest() diff --git a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt index f9d1400e..c329b384 100644 --- a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt +++ b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt @@ -1,21 +1,19 @@ package org.radarbase.kotlin.coroutines -import kotlinx.coroutines.* +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.slf4j.LoggerFactory -import java.time.Duration import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.ExperimentalTime -import kotlin.time.TimeMark -import kotlin.time.TimeSource -@OptIn(ExperimentalTime::class, DelicateCoroutinesApi::class) +@OptIn(DelicateCoroutinesApi::class) internal class CachedValueTest { private lateinit var config: CacheConfig @@ -27,7 +25,7 @@ internal class CachedValueTest { config = CacheConfig( refreshDuration = 20.milliseconds, retryDuration = 10.milliseconds, - exceptionCacheDuration = 10.milliseconds + exceptionCacheDuration = 10.milliseconds, ) } @@ -89,25 +87,24 @@ internal class CachedValueTest { assertThat( "Retry because predicate does not match", cache.query({ it + 1 }, { it > 2 }), - equalTo(CachedValue.CacheMiss(3)) + equalTo(CachedValue.CacheMiss(3)), ) assertThat("No refresh within threshold", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheHit(3))) delay(10) assertThat( "No retry because predicate matches", cache.query({ it + 1 }, { it > 2 }), - equalTo(CachedValue.CacheHit(3)) + equalTo(CachedValue.CacheHit(3)), ) delay(10) assertThat( "Refresh after refresh threshold since last retry", cache.query({ it + 1 }, { it > 2 }), - equalTo(CachedValue.CacheMiss(4)) + equalTo(CachedValue.CacheMiss(4)), ) } } - @Test fun getMultithreaded() { val cache = CachedValue(config) { @@ -117,7 +114,7 @@ internal class CachedValueTest { } runBlocking { - (0 .. 5) + (0..5) .forkJoin { cache.get() } @@ -131,16 +128,18 @@ internal class CachedValueTest { @Test fun getMulti2threaded() { - val cache = CachedValue(config.copy( - maxSimultaneousCompute = 2 - )) { + val cache = CachedValue( + config.copy( + maxSimultaneousCompute = 2, + ), + ) { calls.incrementAndGet() delay(50.milliseconds) calls.get() } runBlocking { - val values = (0 .. 5) + val values = (0..5) .forkJoin { cache.get() } @@ -154,7 +153,6 @@ internal class CachedValueTest { assertThat("Two threads should be computing the value", calls.get(), `is`(2)) } - @Test fun throwTest() { val cache = CachedValue(config.copy(refreshDuration = 20.milliseconds)) { diff --git a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt index 7cb9a18c..04eb05d9 100644 --- a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt +++ b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/ExtensionsKtTest.kt @@ -60,7 +60,6 @@ class ExtensionsKtTest { assertThat(inBlockingTime, greaterThan(100.milliseconds)) } - @Test fun testForkJoinFirst() = runBlocking { val inBlockingTime = measureTime { diff --git a/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt index 6b298097..8dd5491a 100644 --- a/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt +++ b/radar-commons-server/src/main/java/org/radarbase/util/RollingTimeAverage.kt @@ -72,7 +72,7 @@ class RollingTimeAverage( } else { val time = Duration.between(windowStart, deque.last.time) val removedRate = Duration.between(localFirstTime.time, windowStart).toMillis() / - Duration.between(localFirstTime.time, deque.first.time).toMillis().toDouble() + Duration.between(localFirstTime.time, deque.first.time).toMillis().toDouble() val removedValue = localFirstTime.value + deque.first.value * removedRate 1000.0 * (total - removedValue) / time.toMillis() } @@ -85,7 +85,7 @@ class RollingTimeAverage( get() = average.roundToInt() private class TimeCount( - val value: Double + val value: Double, ) { val time: Instant = Instant.now() } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt index bede832c..69d06595 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockDevice.kt @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicBoolean class MockDevice( sender: KafkaSender, private val key: K, - private val generators: List> + private val generators: List>, ) { private val baseFrequency: Int private val sender: KafkaSender diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt index e79b0b0d..47ba5255 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockFileSender.kt @@ -54,8 +54,10 @@ class MockFileSender( } override fun toString(): String { - return ("MockFileSender{" - + "parser=" + parser - + '}') + return ( + "MockFileSender{" + + "parser=" + parser + + '}' + ) } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt index 3bd9e3d2..a6fbae13 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/MockProducer.kt @@ -82,8 +82,9 @@ class MockProducer @JvmOverloads constructor( val generators: List> = createGenerators(dataConfigs) val mockFiles: List = createMockFiles(dataConfigs, root) senders = createSenders( - mockConfig, numDevices + mockFiles.size, - mockConfig.authConfig + mockConfig, + numDevices + mockFiles.size, + mockConfig.authConfig, ) devices = ArrayList(numDevices) @@ -109,13 +110,15 @@ class MockProducer @JvmOverloads constructor( @Throws(IOException::class) private fun createSenders( - mockConfig: BasicMockConfig, numDevices: Int, authConfig: AuthConfig + mockConfig: BasicMockConfig, + numDevices: Int, + authConfig: AuthConfig, ): List = createRestSenders( numDevices, retriever, mockConfig.restProxy, mockConfig.hasCompression(), - authConfig + authConfig, ) /** Create senders that produce data to Kafka via the REST proxy. */ @@ -125,7 +128,7 @@ class MockProducer @JvmOverloads constructor( retriever: SchemaRetriever, restProxy: ServerConfig, useCompression: Boolean, - authConfig: AuthConfig? + authConfig: AuthConfig?, ): List { val scope = CoroutineScope(job) val sharedState = ConnectionState(10.seconds, scope) @@ -146,7 +149,7 @@ class MockProducer @JvmOverloads constructor( ClientCredentialsConfig( authConfig.tokenUrl, authConfig.clientId, - authConfig.clientSecret + authConfig.clientSecret, ).copyWithEnv(), restProxy.host, ) @@ -241,7 +244,7 @@ class MockProducer @JvmOverloads constructor( } private fun createGenerators( - configs: List + configs: List, ): List> = configs.mapNotNull { config -> if (config.dataFile == null) { RecordGenerator(config, ObservationKey::class.java) @@ -253,7 +256,7 @@ class MockProducer @JvmOverloads constructor( @Throws(IOException::class, CsvValidationException::class) private fun createMockFiles( configs: List, - dataRoot: Path? + dataRoot: Path?, ): List { val now = Instant.now() var parent = dataRoot @@ -314,19 +317,21 @@ class MockProducer @JvmOverloads constructor( @Throws(IOException::class, InterruptedException::class, SchemaValidationException::class) private fun waitForProducer(producer: MockProducer, duration: Long) { val isShutdown = AtomicBoolean(false) - Runtime.getRuntime().addShutdownHook(Thread { - try { - if (!isShutdown.get()) { - runBlocking { - producer.shutdown() + Runtime.getRuntime().addShutdownHook( + Thread { + try { + if (!isShutdown.get()) { + runBlocking { + producer.shutdown() + } } + } catch (ex: InterruptedException) { + logger.warn("Shutdown interrupted", ex) + } catch (ex: Exception) { + logger.warn("Failed to shutdown producer", ex) } - } catch (ex: InterruptedException) { - logger.warn("Shutdown interrupted", ex) - } catch (ex: Exception) { - logger.warn("Failed to shutdown producer", ex) - } - }) + }, + ) if (duration <= 0L) { try { logger.info("Producing data until interrupted") diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt index deb18f3d..f5815af4 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/HeaderHierarchy.kt @@ -12,7 +12,7 @@ import java.util.* class HeaderHierarchy( val name: String? = null, index: Int = -1, - private val parent: HeaderHierarchy? = null + private val parent: HeaderHierarchy? = null, ) { /** The index of current element. */ val index: Int = index diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt index 3c135962..76db8e72 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockCsvParser.kt @@ -50,7 +50,7 @@ class MockCsvParser constructor( private val config: MockDataConfig, root: Path?, private val startTime: Instant, - private val retriever: SchemaRetriever + private val retriever: SchemaRetriever, ) : Closeable { lateinit var topic: AvroTopic private val csvReader: CSVReader @@ -69,7 +69,7 @@ class MockCsvParser constructor( for (i in header.indices) { headers.add( i, - header[i].split("\\.".toRegex()).dropLastWhile { it.isEmpty() } + header[i].split("\\.".toRegex()).dropLastWhile { it.isEmpty() }, ) } currentLine = csvReader.readNext() @@ -88,8 +88,10 @@ class MockCsvParser constructor( topic = AvroTopic( config.topic, - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java + keySchema, + valueSchema, + GenericRecord::class.java, + GenericRecord::class.java, ) } @@ -106,12 +108,12 @@ class MockCsvParser constructor( val key = parseRecord( currentLine, topic.keySchema, - checkNotNull(headers.children["key"]) { "Missing key fields" } + checkNotNull(headers.children["key"]) { "Missing key fields" }, ) val value = parseRecord( currentLine, topic.valueSchema, - checkNotNull(headers.children["value"]) { "Missing value fields" } + checkNotNull(headers.children["value"]) { "Missing value fields" }, ) incrementRow() return Record(key, value) @@ -136,7 +138,7 @@ class MockCsvParser constructor( private fun parseRecord( rawValues: Array?, schema: Schema, - headers: HeaderHierarchy + headers: HeaderHierarchy, ): GenericRecord { val record = GenericRecordBuilder(schema) val children = headers.children @@ -155,7 +157,7 @@ class MockCsvParser constructor( Schema.Type.NULL, Schema.Type.INT, Schema.Type.LONG, Schema.Type.FLOAT, Schema.Type.DOUBLE, Schema.Type.BOOLEAN, Schema.Type.STRING, Schema.Type.ENUM, Schema.Type.BYTES -> parseScalar( rawValues, schema, - headers + headers, ) Schema.Type.UNION -> parseUnion(rawValues, schema, headers) @@ -163,8 +165,8 @@ class MockCsvParser constructor( Schema.Type.ARRAY -> parseArray(rawValues, schema, headers) Schema.Type.MAP -> parseMap(rawValues, schema, headers) else -> throw IllegalArgumentException( - "Cannot handle schemas of type " - + schema.type + " in " + headers + "Cannot handle schemas of type " + + schema.type + " in " + headers, ) } } @@ -172,7 +174,7 @@ class MockCsvParser constructor( private fun parseScalar( rawValues: Array?, schema: Schema, - headers: HeaderHierarchy + headers: HeaderHierarchy, ): Any? { val fieldHeader = headers.index require(fieldHeader < rawValues!!.size) { "Row is missing value for " + headers.name } @@ -183,8 +185,9 @@ class MockCsvParser constructor( } private fun parseMap( - rawValues: Array?, schema: Schema, - headers: HeaderHierarchy + rawValues: Array?, + schema: Schema, + headers: HeaderHierarchy, ): Map = buildMap { for (child in headers.children.values) { put(child.name!!, parseValue(rawValues, schema.valueType, child)) @@ -194,7 +197,7 @@ class MockCsvParser constructor( private fun parseUnion( rawValues: Array?, schema: Schema, - headers: HeaderHierarchy + headers: HeaderHierarchy, ): Any = requireNotNull( schema.types.firstNotNullOfOrNull { subSchema -> try { @@ -203,13 +206,13 @@ class MockCsvParser constructor( // skip bad union member null } - } + }, ) { "Cannot handle union types ${schema.types} in $headers" } private fun parseArray( rawValues: Array?, schema: Schema, - headers: HeaderHierarchy + headers: HeaderHierarchy, ): List { val children = headers.children val arrayLength = children.keys.stream() @@ -242,7 +245,7 @@ class MockCsvParser constructor( private fun parseScalar( fieldString: String?, schema: Schema, - headers: HeaderHierarchy + headers: HeaderHierarchy, ): Any? { return when (schema.type) { Schema.Type.NULL -> if (fieldString.isNullOrEmpty() || fieldString == "null") { @@ -259,8 +262,8 @@ class MockCsvParser constructor( Schema.Type.ENUM -> parseEnum(schema, fieldString) Schema.Type.BYTES -> parseBytes(fieldString) else -> throw IllegalArgumentException( - "Cannot handle scalar schema of type " - + schema.type + " in " + headers + "Cannot handle scalar schema of type " + + schema.type + " in " + headers, ) } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt index 0e7b4b87..de15f9a9 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/mock/data/MockRecordValidator.kt @@ -31,8 +31,10 @@ import java.time.Instant * unique User_ID and Source_ID and having increasing timestamp at each raw. */ class MockRecordValidator( - private val config: MockDataConfig, private val duration: Long, private val root: Path, - private val retriever: SchemaRetriever + private val config: MockDataConfig, + private val duration: Long, + private val root: Path, + private val retriever: SchemaRetriever, ) { private var timePos = 0 private var time: Double @@ -88,7 +90,7 @@ class MockRecordValidator( private fun checkRecord( record: Record, last: Record?, - line: Long + line: Long, ) { val previousTime = time time = record.value[timePos] as Double @@ -129,15 +131,17 @@ class MockRecordValidator( val margin = 50L if (duration <= interval - margin || duration > interval + 1000L + margin) { error( - "Data does not cover " + duration + " milliseconds but " - + interval + " instead.", -1L, null + "Data does not cover " + duration + " milliseconds but " + + interval + " instead.", + -1L, + null, ) } } companion object { private val logger = LoggerFactory.getLogger( - MockRecordValidator::class.java + MockRecordValidator::class.java, ) } } diff --git a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt index dc44a06a..0e4473fb 100644 --- a/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt +++ b/radar-commons-testing/src/main/java/org/radarbase/util/Oscilloscope.kt @@ -28,7 +28,7 @@ import kotlin.time.Duration.Companion.seconds * [.willRestart] in the condition of the loop. */ class Oscilloscope( - private val frequency: Int + private val frequency: Int, ) { private val timeStep: Duration = 1.seconds / frequency private val baseTime: Long = System.nanoTime() diff --git a/radar-commons/build.gradle.kts b/radar-commons/build.gradle.kts index ccca3830..96c98614 100644 --- a/radar-commons/build.gradle.kts +++ b/radar-commons/build.gradle.kts @@ -4,9 +4,9 @@ plugins { description = "RADAR Common utilities library." -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // Sources and classpath configurations // -//---------------------------------------------------------------------------// +// ---------------------------------------------------------------------------// // In this section you declare where to find the dependencies of your project repositories { diff --git a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt index db720151..43421590 100644 --- a/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt +++ b/radar-commons/src/main/java/org/radarbase/config/ServerConfig.kt @@ -34,6 +34,7 @@ class ServerConfig { /** Server protocol. */ /** Set server protocol. */ var protocol: String? = null + /** * Set the absolute path. If the path is empty, it will be set to the root. The path * will be ended with a single slash. The path will be prepended with a single slash if needed. @@ -125,8 +126,8 @@ class ServerConfig { check( protocol == null || - protocol.equals("http", ignoreCase = true) || - protocol.equals("https", ignoreCase = true) + protocol.equals("http", ignoreCase = true) || + protocol.equals("https", ignoreCase = true), ) { "Server is not an HTTP(S) server, so it cannot use a HTTP proxyHost." } return Proxy(Proxy.Type.HTTP, InetSocketAddress(proxyHost, proxyPort)) } @@ -142,11 +143,11 @@ class ServerConfig { } other as ServerConfig return host == other.host && - port == other.port && - isUnsafe == other.isUnsafe && - protocol == other.protocol && - proxyHost == other.proxyHost && - proxyPort == other.proxyPort + port == other.port && + isUnsafe == other.isUnsafe && + protocol == other.protocol && + proxyHost == other.proxyHost && + proxyPort == other.proxyPort } override fun hashCode(): Int { diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt index d52185cc..b6ffbbfa 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumDecoder.kt @@ -47,9 +47,10 @@ class AvroDatumDecoder( private inner class AvroRecordReader( private val schema: Schema, - private val reader: DatumReader + private val reader: DatumReader, ) : AvroReader { private var decoder: Decoder? = null + @Throws(IOException::class) override fun decode(`object`: ByteArray): T { return decode(`object`, 0) @@ -59,13 +60,15 @@ class AvroDatumDecoder( override fun decode(`object`: ByteArray, offset: Int): T { decoder = if (binary) { decoderFactory.binaryDecoder( - `object`, offset, `object`.size - offset, - decoder as? BinaryDecoder + `object`, + offset, + `object`.size - offset, + decoder as? BinaryDecoder, ) } else { decoderFactory.jsonDecoder( schema, - ByteArrayInputStream(`object`, offset, `object`.size - offset) + ByteArrayInputStream(`object`, offset, `object`.size - offset), ) } return reader.read(null, decoder) diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt index 00abe2b3..23d5704e 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroDatumEncoder.kt @@ -34,7 +34,7 @@ class AvroDatumEncoder( @Suppress("UNCHECKED_CAST") @Throws(IOException::class) - override fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter { + override fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter { val writer = genericData.createDatumWriter(schema) as DatumWriter return AvroRecordWriter(encoderFactory, schema, writer, binary) } diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt index 62e18200..09ad87c0 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroEncoder.kt @@ -23,12 +23,13 @@ import java.io.IOException interface AvroEncoder { /** Create a new writer. This method is thread-safe, but the class it returns is not. */ @Throws(IOException::class) - fun writer(schema: Schema, clazz: Class): AvroWriter = + fun writer(schema: Schema, clazz: Class): AvroWriter = writer(schema, clazz, schema) + @Throws(IOException::class) - fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter + fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter - interface AvroWriter { + interface AvroWriter { /** * Encode an object. This method is not thread-safe. Call * [.setReaderSchema] before calling encode. diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt index 77d0047f..7289c988 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordData.kt @@ -16,7 +16,7 @@ import org.radarbase.topic.AvroTopic * @throws IllegalArgumentException if the values are empty. * @throws NullPointerException if any of the parameters are null. */ -class AvroRecordData( +class AvroRecordData( override val topic: AvroTopic, override val key: K, private val records: List, diff --git a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt index 84d30966..868983c6 100644 --- a/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt +++ b/radar-commons/src/main/java/org/radarbase/data/AvroRecordWriter.kt @@ -33,7 +33,7 @@ import java.io.IOException * should be with JSON encoding. * @throws IOException if an encoder cannot be constructed. */ -class AvroRecordWriter( +class AvroRecordWriter( encoderFactory: EncoderFactory, schema: Schema, private val writer: DatumWriter, diff --git a/radar-commons/src/main/java/org/radarbase/data/RecordData.kt b/radar-commons/src/main/java/org/radarbase/data/RecordData.kt index 7acd8100..8b1d513b 100644 --- a/radar-commons/src/main/java/org/radarbase/data/RecordData.kt +++ b/radar-commons/src/main/java/org/radarbase/data/RecordData.kt @@ -7,7 +7,7 @@ import org.radarbase.topic.AvroTopic * @param key type * @param value type */ -interface RecordData : Iterable { +interface RecordData : Iterable { /** * Topic that the data belongs to. * @return Avro topic. diff --git a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt index f6f11e2a..c65793e0 100644 --- a/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/data/RemoteSchemaEncoder.kt @@ -18,11 +18,11 @@ import java.io.IOException class RemoteSchemaEncoder( private val binary: Boolean, ) : AvroEncoder { - override fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter { + override fun writer(schema: Schema, clazz: Class, readerSchema: Schema): AvroWriter { return SchemaEncoderWriter(binary, schema, clazz, readerSchema) } - class SchemaEncoderWriter( + class SchemaEncoderWriter( binary: Boolean, schema: Schema, clazz: Class, @@ -56,7 +56,7 @@ class RemoteSchemaEncoder( override fun encode(`object`: T): ByteArray = encoder.encode( requireNotNull(mapper.convertAvro(`object`)) { "Cannot map $`object` to Avro" - } + }, ) } } diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt index eb2965c4..8b816c11 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaSender.kt @@ -19,10 +19,7 @@ import io.ktor.client.* import io.ktor.client.engine.* import io.ktor.client.engine.cio.* import io.ktor.client.plugins.* -import kotlinx.coroutines.channels.BroadcastChannel -import kotlinx.coroutines.channels.SendChannel import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.SharedFlow import org.apache.avro.SchemaValidationException import org.radarbase.producer.rest.ConnectionState import org.radarbase.topic.AvroTopic @@ -35,7 +32,7 @@ import java.io.IOException interface KafkaSender { /** Get a non thread-safe sender instance. */ @Throws(IOException::class, SchemaValidationException::class) - fun sender(topic: AvroTopic): KafkaTopicSender + fun sender(topic: AvroTopic): KafkaTopicSender /** * If the sender is no longer connected, try to reconnect. diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt index 2a98fda0..6a79fdb2 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt @@ -9,7 +9,7 @@ import java.io.IOException /** * Sender for a single topic. Should be created through a [KafkaSender]. */ -interface KafkaTopicSender { +interface KafkaTopicSender { val topic: AvroTopic /** diff --git a/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt b/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt index b460b4dd..3b839860 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/avro/AvroDataMapperFactory.kt @@ -79,7 +79,9 @@ object AvroDataMapperFactory { } val resolvedFrom = if (from.type == Schema.Type.UNION) { nonNullUnionSchema(from) - } else from + } else { + from + } return if (from.type == Schema.Type.UNION && to.type != Schema.Type.UNION) { defaultVal ?: throw validationException(to, from, "Cannot map union to non-union without a default value") @@ -134,8 +136,8 @@ object AvroDataMapperFactory { private fun mapBytes(from: Schema, to: Schema, defaultVal: Any?): AvroDataMapper { return if (from.type == Schema.Type.BYTES && to.type == Schema.Type.BYTES) { IDENTITY_MAPPER - } else if (from.type == Schema.Type.FIXED && to.type == Schema.Type.FIXED - && from.fixedSize == to.fixedSize + } else if (from.type == Schema.Type.FIXED && to.type == Schema.Type.FIXED && + from.fixedSize == to.fixedSize ) { IDENTITY_MAPPER } else if (from.type == Schema.Type.FIXED && to.type == Schema.Type.BYTES) { @@ -177,10 +179,10 @@ object AvroDataMapperFactory { } val fromFields = from.fields val toFields = arrayOfNulls( - fromFields.size + fromFields.size, ) val fieldMappers = arrayOfNulls( - fromFields.size + fromFields.size, ) val filledPositions = BooleanArray(to.fields.size) for (i in fromFields.indices) { @@ -191,13 +193,15 @@ object AvroDataMapperFactory { fieldMappers[i] = createMapper( fromField.schema(), toField.schema(), - toField.defaultVal() + toField.defaultVal(), ) } filledPositions.forEachIndexed { i, isFilled -> if (!isFilled && to.fields[i].defaultVal() == null) { - throw validationException(to, from, - "Cannot map to record without default value for new field ${to.fields[i].name()}" + throw validationException( + to, + from, + "Cannot map to record without default value for new field ${to.fields[i].name()}", ) } } @@ -208,7 +212,7 @@ object AvroDataMapperFactory { private class RecordMapper constructor( private val toSchema: Schema, private val toFields: Array, - private val fieldMappers: Array + private val fieldMappers: Array, ) : AvroDataMapper { override fun convertAvro(`object`: Any?): GenericRecord { val builder = GenericRecordBuilder(toSchema) @@ -222,9 +226,11 @@ object AvroDataMapperFactory { } override fun toString(): String { - return ("RecordMapper{" - + "fieldMappers=" + fieldMappers.contentToString() - + ", toFields=" + toFields.contentToString() + '}') + return ( + "RecordMapper{" + + "fieldMappers=" + fieldMappers.contentToString() + + ", toFields=" + toFields.contentToString() + '}' + ) } } @@ -253,7 +259,9 @@ object AvroDataMapperFactory { private inline fun Any?.asAvroType(from: Schema, to: Schema): T { if (this !is T) { throw validationException( - to, from, "${to.type} type cannot be mapped from ${this?.javaClass?.name} Java type." + to, + from, + "${to.type} type cannot be mapped from ${this?.javaClass?.name} Java type.", ) } return this @@ -295,8 +303,10 @@ object AvroDataMapperFactory { if (to.hasEnumSymbol("UNKNOWN")) { defaultString = "UNKNOWN" } else { - throw validationException(to, from, - "Cannot map enum symbols without default value" + throw validationException( + to, + from, + "Cannot map enum symbols without default value", ) } } @@ -309,7 +319,7 @@ object AvroDataMapperFactory { symbol } } - } + } } else if (from.type == Schema.Type.ENUM && to.type == Schema.Type.STRING) { AvroDataMapper { it.toString() } } else { @@ -342,7 +352,9 @@ object AvroDataMapperFactory { Schema.Type.DOUBLE -> StringToNumberMapper(defaultVal, String::toDouble) Schema.Type.FLOAT -> StringToNumberMapper(defaultVal, String::toFloat) else -> throw validationException( - to, from, "Cannot map numeric type with non-numeric type" + to, + from, + "Cannot map numeric type with non-numeric type", ) } } else { @@ -353,7 +365,9 @@ object AvroDataMapperFactory { Schema.Type.FLOAT -> AvroDataMapper { it.asAvroType(from, to).toFloat() } Schema.Type.STRING -> AvroDataMapper { it.toString() } else -> throw validationException( - to, from, "Cannot map numeric type with non-numeric type" + to, + from, + "Cannot map numeric type with non-numeric type", ) } } @@ -386,6 +400,8 @@ object AvroDataMapperFactory { to: Schema, message: String, ): SchemaValidationException = SchemaValidationException( - to, from, IllegalArgumentException(message) + to, + from, + IllegalArgumentException(message), ) } diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt b/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt index 62ea0241..c82df2b4 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/io/DirectBinaryEncoder.kt @@ -47,7 +47,7 @@ import java.util.* * @see Decoder */ class DirectBinaryEncoder( - var out: ByteWriteChannel + var out: ByteWriteChannel, ) : BinaryEncoder() { // the buffer is used for writing floats, doubles, and large longs. private val buf = ByteArray(12) diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt b/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt index a1de5621..0411c5f1 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/io/Encoder.kt @@ -46,7 +46,7 @@ import java.nio.ByteBuffer * * @see Decoder */ -interface Encoder: Closeable { +interface Encoder : Closeable { /** * "Writes" a null value. (Doesn't actually write anything, but advances the * state of the parser if this class is stateful.) @@ -241,7 +241,7 @@ interface Encoder: Closeable { * out.writeBoolean(r.boolField); * } * out.writeArrayEnd(); - * + * * * @throws AvroTypeException If this is a stateful writer and an array is not * expected @@ -300,7 +300,7 @@ interface Encoder: Closeable { * out.writeBoolean(entry.getValue().boolField); * } * out.writeMapEnd(); - * + * * * @throws AvroTypeException If this is a stateful writer and a map is not * expected @@ -330,7 +330,7 @@ interface Encoder: Closeable { * out.writeIndex(1); * out.writeLong(record.longField); * out.writeBoolean(record.boolField); - * + * * * @throws AvroTypeException If this is a stateful writer and a map is not * expected diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt b/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt index 4aaecf5b..3df18ccb 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/io/GzipContentEncoding.kt @@ -10,7 +10,6 @@ import io.ktor.util.cio.* import io.ktor.utils.io.* import kotlinx.coroutines.coroutineScope - /** * A plugin that allows you to enable specified compression algorithms (such as `gzip` and `deflate`) and configure their settings. * This plugin serves two primary purposes: @@ -73,7 +72,6 @@ class GzipContentEncoding private constructor() { } } - private class GzipReadChannel( private val original: ByteReadChannel, ) : OutgoingContent.ReadChannelContent() { diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt b/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt index 71e15baa..99e1ece4 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/io/HttpClientExtensions.kt @@ -4,8 +4,8 @@ import io.ktor.client.* import io.ktor.client.engine.cio.* import io.ktor.client.plugins.* import java.security.cert.X509Certificate -import kotlin.time.Duration import javax.net.ssl.X509TrustManager +import kotlin.time.Duration fun HttpClientConfig<*>.timeout(duration: Duration) { install(HttpTimeout) { diff --git a/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt b/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt index 0efeddc3..34de28a1 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/io/UnsupportedMediaTypeException.kt @@ -7,5 +7,5 @@ class UnsupportedMediaTypeException( contentType: ContentType?, contentEncoding: String?, ) : IOException( - "Unsupported media type ${contentType ?: "unknown"} with ${contentEncoding ?: "no"} encoding" + "Unsupported media type ${contentType ?: "unknown"} with ${contentEncoding ?: "no"} encoding", ) diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt index 68e4bc3e..b8634e33 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/AvroContentConverter.kt @@ -19,7 +19,7 @@ class AvroContentConverter( contentType: ContentType, charset: Charset, typeInfo: TypeInfo, - value: Any? + value: Any?, ): OutgoingContent? { if (value !is RecordData<*, *>) return null @@ -28,14 +28,14 @@ class AvroContentConverter( schemaRetriever.metadata( topic = value.topic.name, ofValue = false, - schema = value.topic.keySchema + schema = value.topic.keySchema, ) } val valueSchema = async { schemaRetriever.metadata( topic = value.topic.name, ofValue = true, - schema = value.topic.valueSchema + schema = value.topic.valueSchema, ) } val maker = if (binary) { @@ -48,7 +48,7 @@ class AvroContentConverter( JsonRecordContent( records = value, keySchemaMetadata = keySchema.await(), - valueSchemaMetadata = valueSchema.await() + valueSchemaMetadata = valueSchema.await(), ) } maker.createContent() diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt index e17b4e9e..42785c50 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/BinaryRecordContent.kt @@ -10,22 +10,22 @@ import org.radarbase.producer.io.FunctionalWriteChannelContent import org.radarbase.producer.schema.ParsedSchemaMetadata import org.slf4j.LoggerFactory -class BinaryRecordContent( +class BinaryRecordContent( private val records: RecordData<*, V>, keySchemaMetadata: ParsedSchemaMetadata, valueSchemaMetadata: ParsedSchemaMetadata, -): AvroRecordContent { +) : AvroRecordContent { private val valueEncoder = RemoteSchemaEncoder.SchemaEncoderWriter( binary = true, schema = records.topic.valueSchema, clazz = records.topic.valueClass, - readerSchema = valueSchemaMetadata.schema + readerSchema = valueSchemaMetadata.schema, ) private val sourceId = records.sourceId ?: throw AvroDataMapperFactory.validationException( records.topic.keySchema, keySchemaMetadata.schema, - "Cannot map record without source ID" + "Cannot map record without source ID", ) private val keySchemaVersion = requireNotNull(keySchemaMetadata.version) { diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt index df3dfb00..78440880 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/ConnectionState.kt @@ -38,7 +38,7 @@ import kotlin.time.Duration */ class ConnectionState( private val timeout: Duration, - scope: CoroutineScope = CoroutineScope(EmptyCoroutineContext) + scope: CoroutineScope = CoroutineScope(EmptyCoroutineContext), ) { /** State symbols of the connection. */ enum class State { diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt index 23ffa513..7fa481b9 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/JsonRecordContent.kt @@ -8,7 +8,7 @@ import org.radarbase.producer.io.FunctionalWriteChannelContent import org.radarbase.producer.schema.ParsedSchemaMetadata import org.slf4j.LoggerFactory -class JsonRecordContent( +class JsonRecordContent( private val records: RecordData, private val keySchemaMetadata: ParsedSchemaMetadata, private val valueSchemaMetadata: ParsedSchemaMetadata, @@ -17,13 +17,13 @@ class JsonRecordContent( binary = false, schema = records.topic.keySchema, clazz = records.topic.keyClass, - readerSchema = keySchemaMetadata.schema + readerSchema = keySchemaMetadata.schema, ) private val valueEncoder = RemoteSchemaEncoder.SchemaEncoderWriter( binary = false, schema = records.topic.valueSchema, clazz = records.topic.valueClass, - readerSchema = valueSchemaMetadata.schema + readerSchema = valueSchemaMetadata.schema, ) override fun createContent(): OutgoingContent = diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt index ca658d96..92a49381 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt @@ -59,6 +59,7 @@ class RestKafkaSender(config: Config) : KafkaSender { val schemaRetriever: SchemaRetriever = requireNotNull(config.schemaRetriever) { "Missing schemaRetriever from configuration" } + /** Get the current REST client. */ val restClient: HttpClient @@ -90,14 +91,14 @@ class RestKafkaSender(config: Config) : KafkaSender { install(ContentNegotiation) { this.register( KAFKA_REST_BINARY_ENCODING, - AvroContentConverter(schemaRetriever, binary = true) + AvroContentConverter(schemaRetriever, binary = true), ) this.register( KAFKA_REST_JSON_ENCODING, - AvroContentConverter(schemaRetriever, binary = false) + AvroContentConverter(schemaRetriever, binary = false), ) } - when(contentEncoding) { + when (contentEncoding) { GZIP_CONTENT_ENCODING -> install(GzipContentEncoding) else -> {} } @@ -114,7 +115,7 @@ class RestKafkaSender(config: Config) : KafkaSender { } } - inner class RestKafkaTopicSender( + inner class RestKafkaTopicSender( override val topic: AvroTopic, ) : KafkaTopicSender { @OptIn(ExperimentalStdlibApi::class) @@ -175,7 +176,9 @@ class RestKafkaSender(config: Config) : KafkaSender { val bodyString = response.bodyAsText() logger.warn( "Failed to make heartbeat request to {} (HTTP status code {}): {}", - restClient, response.status, bodyString + restClient, + response.status, + bodyString, ) ConnectionState.State.DISCONNECTED } @@ -229,14 +232,14 @@ class RestKafkaSender(config: Config) : KafkaSender { if (other == null || javaClass != other.javaClass) return false other as Config return schemaRetriever == other.schemaRetriever && - connectionState == other.connectionState && - headers.build() == other.headers.build() && - httpClient == other.httpClient && - contentType == other.contentType && - baseUrl == other.baseUrl && - connectionTimeout == other.connectionTimeout && - contentEncoding == other.contentEncoding && - scope == other.scope + connectionState == other.connectionState && + headers.build() == other.headers.build() && + httpClient == other.httpClient && + contentType == other.contentType && + baseUrl == other.baseUrl && + connectionTimeout == other.connectionTimeout && + contentEncoding == other.contentEncoding && + scope == other.scope } override fun hashCode(): Int = headers.hashCode() } diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt index 762ee004..677df431 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt @@ -33,11 +33,11 @@ data class SchemaMetadata val version: Int? = null, val schema: String? = null, ) { - fun toParsedSchemaMetadata() = ParsedSchemaMetadata( - id = checkNotNull(id) { "Need id to parse schema metadata" }, - version = version, - schema = Schema.Parser().parse( - checkNotNull(schema) { "Need schema to parse it" } - ), - ) - } + fun toParsedSchemaMetadata() = ParsedSchemaMetadata( + id = checkNotNull(id) { "Need id to parse schema metadata" }, + version = version, + schema = Schema.Parser().parse( + checkNotNull(schema) { "Need schema to parse it" }, + ), + ) +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt index 17de167b..5d30f212 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt @@ -24,10 +24,12 @@ class SchemaRestClient( ) { private val httpClient: HttpClient = httpClient.config { install(ContentNegotiation) { - json(Json { - ignoreUnknownKeys = true - coerceInputValues = true - }) + json( + Json { + ignoreUnknownKeys = true + coerceInputValues = true + }, + ) } defaultRequest { url(baseUrl) @@ -39,7 +41,7 @@ class SchemaRestClient( @Throws(IOException::class) suspend fun retrieveSchemaMetadata( subject: String, - version: Int + version: Int, ): ParsedSchemaMetadata { val isLatest = version <= 0 val versionPath = if (isLatest) "latest" else version @@ -62,7 +64,7 @@ class SchemaRestClient( @Throws(IOException::class) suspend fun schemaPost( path: String, - schema: Schema + schema: Schema, ): SchemaMetadata = withContext(ioContext) { val response = httpClient.post { url(path) @@ -91,13 +93,13 @@ class SchemaRestClient( @Throws(IOException::class) suspend fun requestMetadata( subject: String, - schema: Schema + schema: Schema, ): ParsedSchemaMetadata = withContext(ioContext) { val result = schemaPost("subjects/$subject", schema) ParsedSchemaMetadata( id = checkNotNull(result.id) { "Missing schema ID in request result" }, version = result.version, - schema = schema + schema = schema, ) } diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt index cd37a16f..a128a19a 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt @@ -74,7 +74,7 @@ open class SchemaRetriever(config: Config) { private fun cachedMetadata( subject: String, - schema: Schema + schema: Schema, ): CachedValue = schemaCache.computeIfAbsent(schema) { CachedValue(schemaTimeout) { val metadata = restClient.requestMetadata(subject, schema) @@ -90,12 +90,12 @@ open class SchemaRetriever(config: Config) { suspend fun getByVersion( topic: String, ofValue: Boolean, - version: Int + version: Int, ): ParsedSchemaMetadata { val subject = subject(topic, ofValue) val versionMap = subjectVersionCache.computeIfAbsent( subject, - ::ConcurrentHashMap + ::ConcurrentHashMap, ) val metadata = versionMap.cachedVersion(subject, version).get() if (version <= 0 && metadata.version != null) { @@ -106,17 +106,17 @@ open class SchemaRetriever(config: Config) { private suspend fun cachedVersion( subject: String, - version: Int + version: Int, ): CachedValue = subjectVersionCache .computeIfAbsent( subject, - ::ConcurrentHashMap + ::ConcurrentHashMap, ) .cachedVersion(subject, version) private suspend fun VersionCache.cachedVersion( subject: String, - version: Int + version: Int, ): CachedValue { val useVersion = version.coerceAtLeast(0) val versionId = computeIfAbsent(useVersion) { @@ -148,8 +148,10 @@ open class SchemaRetriever(config: Config) { if ( staleValue is CachedValue.CacheError || - (staleValue is CachedValue.CacheValue && - staleValue.isExpired(schemaTimeout.refreshDuration)) + ( + staleValue is CachedValue.CacheValue && + staleValue.isExpired(schemaTimeout.refreshDuration) + ) ) { iter.remove() } diff --git a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt index 443c8243..b678fc8e 100644 --- a/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt +++ b/radar-commons/src/main/java/org/radarbase/topic/AvroTopic.kt @@ -28,7 +28,7 @@ import java.util.* * @param keyClass Java class for keys * @param valueClass Java class for values */ -open class AvroTopic( +open class AvroTopic( name: String, val keySchema: Schema, val valueSchema: Schema, @@ -40,7 +40,9 @@ open class AvroTopic( Array(fields.size) { i -> fields[i].schema().type } - } else null + } else { + null + } get() = field?.copyOf() /** @@ -96,7 +98,7 @@ open class AvroTopic( * @param schemaClass class name of the SpecificRecord to use * @param class type to return * @return Instantiated class of given specific record class - */ + */ @Suppress("UNCHECKED_CAST") fun parseSpecificRecord(schemaClass: String): K { return try { @@ -108,12 +110,12 @@ open class AvroTopic( } catch (ex: ClassCastException) { throw IllegalArgumentException( "Schema $schemaClass cannot be instantiated", - ex + ex, ) } catch (ex: ReflectiveOperationException) { throw IllegalArgumentException( "Schema $schemaClass cannot be instantiated", - ex + ex, ) } } diff --git a/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt index 6258261e..3c1dc449 100644 --- a/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt +++ b/radar-commons/src/main/java/org/radarbase/topic/KafkaTopic.kt @@ -23,12 +23,14 @@ package org.radarbase.topic * possible underscores. */ open class KafkaTopic( - val name: String + val name: String, ) : Comparable { init { require(name.matches(TOPIC_NAME_PATTERN)) { - ("Kafka topic " + name + " is not ASCII-alphanumeric " - + "with possible underscores.") + ( + "Kafka topic " + name + " is not ASCII-alphanumeric " + + "with possible underscores." + ) } } diff --git a/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt index 01b130b8..0fcdd0f7 100644 --- a/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt +++ b/radar-commons/src/main/java/org/radarbase/topic/SensorTopic.kt @@ -28,7 +28,7 @@ import org.apache.avro.specific.SpecificRecord * @param keyClass actual key class * @param valueClass actual value class */ -class SensorTopic( +class SensorTopic( name: String, keySchema: Schema, valueSchema: Schema, @@ -62,8 +62,10 @@ class SensorTopic( val parseAvro = AvroTopic.parse(topic, keySchema, valueSchema) return SensorTopic( parseAvro.name, - parseAvro.keySchema, parseAvro.valueSchema, - parseAvro.keyClass, parseAvro.valueClass + parseAvro.keySchema, + parseAvro.valueSchema, + parseAvro.keyClass, + parseAvro.valueClass, ) } } diff --git a/radar-commons/src/main/java/org/radarbase/util/Annotations.kt b/radar-commons/src/main/java/org/radarbase/util/Annotations.kt deleted file mode 100644 index 65f13bd6..00000000 --- a/radar-commons/src/main/java/org/radarbase/util/Annotations.kt +++ /dev/null @@ -1,8 +0,0 @@ -package org.radarbase.util - -/** - * A marker annotations for DSLs. - */ -@DslMarker -@Target(AnnotationTarget.CLASS, AnnotationTarget.TYPEALIAS, AnnotationTarget.TYPE, AnnotationTarget.FUNCTION) -annotation class RadarProducerDsl diff --git a/radar-commons/src/main/java/org/radarbase/util/Base64.kt b/radar-commons/src/main/java/org/radarbase/util/Base64Encoder.kt similarity index 95% rename from radar-commons/src/main/java/org/radarbase/util/Base64.kt rename to radar-commons/src/main/java/org/radarbase/util/Base64Encoder.kt index 9f78000a..44292d6b 100644 --- a/radar-commons/src/main/java/org/radarbase/util/Base64.kt +++ b/radar-commons/src/main/java/org/radarbase/util/Base64Encoder.kt @@ -46,8 +46,8 @@ package org.radarbase.util * * Note: needed because it is only included in Android API level 26. * - * @author Xueming Shen - * @since 1.8 + * @author Xueming Shen + * @since 1.8 */ object Base64Encoder { /** @@ -68,9 +68,9 @@ object Base64Encoder { * byte array using the [Base64] encoding scheme. The returned byte * array is of the length of the resulting bytes. * - * @param src + * @param src * the byte array to encode - * @return A newly-allocated byte array containing the resulting + * @return A newly-allocated byte array containing the resulting * encoded bytes. */ fun encode(src: ByteArray): String { @@ -81,15 +81,15 @@ object Base64Encoder { var srcP = 0 while (srcP < fullDataLen) { val bits = (src[srcP].toInt() and 0xff).shl(16) or - (src[srcP + 1].toInt() and 0xff).shl(8) or - (src[srcP + 2].toInt() and 0xff) + (src[srcP + 1].toInt() and 0xff).shl(8) or + (src[srcP + 2].toInt() and 0xff) dst[dstP++] = BASE_64_CHAR[bits.ushr(18) and 0x3f] dst[dstP++] = BASE_64_CHAR[bits.ushr(12) and 0x3f] dst[dstP++] = BASE_64_CHAR[bits.ushr(6) and 0x3f] dst[dstP++] = BASE_64_CHAR[bits and 0x3f] srcP += 3 } - if (srcP < srcLen) { // 1 or 2 leftover bytes + if (srcP < srcLen) { // 1 or 2 leftover bytes val b0 = src[srcP++].toInt() and 0xff dst[dstP++] = BASE_64_CHAR[b0 shr 2] if (srcP == srcLen) { diff --git a/radar-commons/src/main/java/org/radarbase/util/RadarProducerDsl.kt b/radar-commons/src/main/java/org/radarbase/util/RadarProducerDsl.kt new file mode 100644 index 00000000..6c88aa2b --- /dev/null +++ b/radar-commons/src/main/java/org/radarbase/util/RadarProducerDsl.kt @@ -0,0 +1,13 @@ +package org.radarbase.util + +/** + * A marker annotations for DSLs. + */ +@DslMarker +@Target( + AnnotationTarget.CLASS, + AnnotationTarget.TYPEALIAS, + AnnotationTarget.TYPE, + AnnotationTarget.FUNCTION, +) +annotation class RadarProducerDsl diff --git a/radar-commons/src/main/java/org/radarbase/util/Strings.kt b/radar-commons/src/main/java/org/radarbase/util/Strings.kt index 7cee213a..a03d60b6 100644 --- a/radar-commons/src/main/java/org/radarbase/util/Strings.kt +++ b/radar-commons/src/main/java/org/radarbase/util/Strings.kt @@ -25,10 +25,12 @@ object Strings { * Compiles a pattern that checks if it is contained in another string in a case-insensitive * way. */ - fun String.toIgnoreCaseRegex(): Regex = toRegex(setOf( - RegexOption.IGNORE_CASE, - RegexOption.LITERAL - )) + fun String.toIgnoreCaseRegex(): Regex = toRegex( + setOf( + RegexOption.IGNORE_CASE, + RegexOption.LITERAL, + ), + ) /** * Converts given bytes to a hex string. diff --git a/radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt b/radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt index 5184cc64..2ad26622 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/avro/AvroDataMapperFactoryTest.kt @@ -18,8 +18,9 @@ class AvroDataMapperFactoryTest { @Throws(SchemaValidationException::class, IOException::class) fun mapRecord() { val actual = doMap( - MEASUREMENT_KEY_SCHEMA, ObservationKey.getClassSchema(), - "{\"userId\":\"u\", \"sourceId\": \"s\"}" + MEASUREMENT_KEY_SCHEMA, + ObservationKey.getClassSchema(), + "{\"userId\":\"u\", \"sourceId\": \"s\"}", ) assertEquals("{\"projectId\":null,\"userId\":\"u\",\"sourceId\":\"s\"}", actual) } @@ -31,7 +32,7 @@ class AvroDataMapperFactoryTest { AvroDataMapperFactory.createMapper( INCOMPLETE_MEASUREMENT_KEY_SCHEMA, ObservationKey.getClassSchema(), - null + null, ) } } @@ -69,52 +70,55 @@ class AvroDataMapperFactoryTest { @Throws(SchemaValidationException::class, IOException::class) fun mapAll() { val actual = doMap( - ALL_TYPES_SCHEMA, ALL_TYPES_ALT_SCHEMA, "{" + - "\"e\":\"A\"," + - "\"i\":1," + - "\"l\":2," + - "\"d\":3.0," + - "\"f\":4.0," + - "\"sI\":\"5\"," + - "\"sD\":\"6.5\"," + - "\"sU\":null," + - "\"sUi\":{\"string\":\"7\"}," + - "\"sUe\":null," + - "\"uS\":\"s\"," + - "\"se2\":\"B\"," + - "\"se3\":\"g\"," + - "\"a\":[1,2]," + - "\"m\":{\"a\":9}," + - "\"fS\":\"ab\"," + - "\"bS\":\"ab\"," + - "\"fb\":\"ab\"," + - "\"bf\":\"ab\"," + - "\"bfd\":\"abc\"," + - "\"unmapped\":10}" + ALL_TYPES_SCHEMA, + ALL_TYPES_ALT_SCHEMA, + "{" + + "\"e\":\"A\"," + + "\"i\":1," + + "\"l\":2," + + "\"d\":3.0," + + "\"f\":4.0," + + "\"sI\":\"5\"," + + "\"sD\":\"6.5\"," + + "\"sU\":null," + + "\"sUi\":{\"string\":\"7\"}," + + "\"sUe\":null," + + "\"uS\":\"s\"," + + "\"se2\":\"B\"," + + "\"se3\":\"g\"," + + "\"a\":[1,2]," + + "\"m\":{\"a\":9}," + + "\"fS\":\"ab\"," + + "\"bS\":\"ab\"," + + "\"fb\":\"ab\"," + + "\"bf\":\"ab\"," + + "\"bfd\":\"abc\"," + + "\"unmapped\":10}", ) assertEquals( "{" + - "\"e\":\"A\"," + - "\"i\":1," + - "\"l\":2.0," + - "\"d\":3.0," + - "\"f\":4.0," + - "\"sI\":5," + - "\"sD\":6.5," + - "\"sU\":\"\"," + - "\"sUi\":{\"int\":7}," + - "\"sUe\":\"A\"," + - "\"uS\":{\"string\":\"s\"}," + - "\"se2\":\"B\"," + - "\"se3\":\"A\"," + - "\"a\":[1.0,2.0]," + - "\"m\":{\"a\":9.0}," + - "\"fS\":\"YWI=\"," + - "\"bS\":\"YWI=\"," + - "\"fb\":\"ab\"," + - "\"bf\":\"ab\"," + - "\"bfd\":\"aa\"" + - "}", actual + "\"e\":\"A\"," + + "\"i\":1," + + "\"l\":2.0," + + "\"d\":3.0," + + "\"f\":4.0," + + "\"sI\":5," + + "\"sD\":6.5," + + "\"sU\":\"\"," + + "\"sUi\":{\"int\":7}," + + "\"sUe\":\"A\"," + + "\"uS\":{\"string\":\"s\"}," + + "\"se2\":\"B\"," + + "\"se3\":\"A\"," + + "\"a\":[1.0,2.0]," + + "\"m\":{\"a\":9.0}," + + "\"fS\":\"YWI=\"," + + "\"bS\":\"YWI=\"," + + "\"fb\":\"ab\"," + + "\"bf\":\"ab\"," + + "\"bfd\":\"aa\"" + + "}", + actual, ) } @@ -135,95 +139,95 @@ class AvroDataMapperFactoryTest { companion object { private val MEASUREMENT_KEY_SCHEMA = Schema.Parser().parse( "{" + - " \"namespace\": \"org.radarcns.key\"," + - " \"type\": \"record\"," + - " \"name\": \"MeasurementKey\"," + - " \"doc\": \"Measurement key in the RADAR-base project\"," + - " \"fields\": [" + - " {\"name\": \"userId\", \"type\": \"string\", \"doc\": \"user ID\"}," + - " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}" + - " ]" + - "}" + " \"namespace\": \"org.radarcns.key\"," + + " \"type\": \"record\"," + + " \"name\": \"MeasurementKey\"," + + " \"doc\": \"Measurement key in the RADAR-base project\"," + + " \"fields\": [" + + " {\"name\": \"userId\", \"type\": \"string\", \"doc\": \"user ID\"}," + + " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}" + + " ]" + + "}", ) private val INCOMPLETE_MEASUREMENT_KEY_SCHEMA = Schema.Parser().parse( "{" + - " \"namespace\": \"org.radarcns.key\"," + - " \"type\": \"record\"," + - " \"name\": \"MeasurementKey\"," + - " \"doc\": \"Measurement key in the RADAR-base project\"," + - " \"fields\": [" + - " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}" + - " ]" + - "}" + " \"namespace\": \"org.radarcns.key\"," + + " \"type\": \"record\"," + + " \"name\": \"MeasurementKey\"," + + " \"doc\": \"Measurement key in the RADAR-base project\"," + + " \"fields\": [" + + " {\"name\": \"sourceId\", \"type\": \"string\", \"doc\": \"device source ID\"}" + + " ]" + + "}", ) private val SMALL_ENUM_SCHEMA = Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}}" - + "]}" + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}}" + + "]}", ) private val LARGE_ENUM_SCHEMA = Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"C\"]}}" - + "]}" + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"C\"]}}" + + "]}", ) private val UNKNOWN_ENUM_SCHEMA = Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"UNKNOWN\"]}}" - + "]}" + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\", \"UNKNOWN\"]}}" + + "]}", ) private val DEFAULT_ENUM_SCHEMA = Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}" - + "]}" + "{\"type\":\"record\",\"name\":\"E\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}" + + "]}", ) private val ALL_TYPES_SCHEMA = Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," - + "{\"name\": \"i\", \"type\": \"int\"}," - + "{\"name\": \"l\", \"type\": \"long\"}," - + "{\"name\": \"d\", \"type\": \"double\"}," - + "{\"name\": \"f\", \"type\": \"float\"}," - + "{\"name\": \"sI\", \"type\": \"string\"}," - + "{\"name\": \"sD\", \"type\": \"string\"}," - + "{\"name\": \"sU\", \"type\": [\"null\", \"string\"]}," - + "{\"name\": \"sUi\", \"type\": [\"null\", \"string\"]}," - + "{\"name\": \"sUe\", \"type\": [\"null\", {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}]}," - + "{\"name\": \"uS\", \"type\": \"string\"}," - + "{\"name\": \"se2\", \"type\": \"string\"}," - + "{\"name\": \"se3\", \"type\": \"string\"}," - + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"int\"}}}," - + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"int\"}}}," - + "{\"name\": \"fS\", \"type\": {\"name\": \"f1\", \"type\":\"fixed\", \"size\": 2}}," - + "{\"name\": \"bS\", \"type\": \"bytes\"}," - + "{\"name\": \"fb\", \"type\": {\"name\": \"f2\",\"type\": \"fixed\", \"size\": 2}}," - + "{\"name\": \"bf\", \"type\": \"bytes\"}," - + "{\"name\": \"bfd\", \"type\": \"bytes\"}," - + "{\"name\": \"unmapped\", \"type\": \"int\"}" - + "]}" + "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," + + "{\"name\": \"i\", \"type\": \"int\"}," + + "{\"name\": \"l\", \"type\": \"long\"}," + + "{\"name\": \"d\", \"type\": \"double\"}," + + "{\"name\": \"f\", \"type\": \"float\"}," + + "{\"name\": \"sI\", \"type\": \"string\"}," + + "{\"name\": \"sD\", \"type\": \"string\"}," + + "{\"name\": \"sU\", \"type\": [\"null\", \"string\"]}," + + "{\"name\": \"sUi\", \"type\": [\"null\", \"string\"]}," + + "{\"name\": \"sUe\", \"type\": [\"null\", {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}]}," + + "{\"name\": \"uS\", \"type\": \"string\"}," + + "{\"name\": \"se2\", \"type\": \"string\"}," + + "{\"name\": \"se3\", \"type\": \"string\"}," + + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"int\"}}}," + + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"int\"}}}," + + "{\"name\": \"fS\", \"type\": {\"name\": \"f1\", \"type\":\"fixed\", \"size\": 2}}," + + "{\"name\": \"bS\", \"type\": \"bytes\"}," + + "{\"name\": \"fb\", \"type\": {\"name\": \"f2\",\"type\": \"fixed\", \"size\": 2}}," + + "{\"name\": \"bf\", \"type\": \"bytes\"}," + + "{\"name\": \"bfd\", \"type\": \"bytes\"}," + + "{\"name\": \"unmapped\", \"type\": \"int\"}" + + "]}", ) private val ALL_TYPES_ALT_SCHEMA = Schema.Parser().parse( - "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" - + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," - + "{\"name\": \"i\", \"type\": \"long\"}," - + "{\"name\": \"l\", \"type\": \"double\"}," - + "{\"name\": \"d\", \"type\": \"float\"}," - + "{\"name\": \"f\", \"type\": \"double\"}," - + "{\"name\": \"sI\", \"type\": \"int\", \"default\": 0}," - + "{\"name\": \"sD\", \"type\": \"double\", \"default\": 0.0}," - + "{\"name\": \"sU\", \"type\": \"string\", \"default\": \"\"}," - + "{\"name\": \"sUi\", \"type\": [\"null\", \"int\"], \"default\":null}," - + "{\"name\": \"sUe\", \"type\": {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," - + "{\"name\": \"uS\", \"type\": [\"null\", \"string\"]}," - + "{\"name\": \"se2\", \"type\": {\"name\": \"SE2\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," - + "{\"name\": \"se3\", \"type\": {\"name\": \"SE3\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," - + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"float\"}}}," - + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"float\"}}}," - + "{\"name\": \"fS\", \"type\": \"string\"}," - + "{\"name\": \"bS\", \"type\": \"string\"}," - + "{\"name\": \"fb\", \"type\": \"bytes\"}," - + "{\"name\": \"bf\", \"type\": {\"name\": \"f3\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}," - + "{\"name\": \"bfd\", \"type\": {\"name\": \"f4\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}" - + "]}" + "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" + + "{\"name\": \"e\", \"type\": {\"type\": \"enum\", \"name\": \"Enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," + + "{\"name\": \"i\", \"type\": \"long\"}," + + "{\"name\": \"l\", \"type\": \"double\"}," + + "{\"name\": \"d\", \"type\": \"float\"}," + + "{\"name\": \"f\", \"type\": \"double\"}," + + "{\"name\": \"sI\", \"type\": \"int\", \"default\": 0}," + + "{\"name\": \"sD\", \"type\": \"double\", \"default\": 0.0}," + + "{\"name\": \"sU\", \"type\": \"string\", \"default\": \"\"}," + + "{\"name\": \"sUi\", \"type\": [\"null\", \"int\"], \"default\":null}," + + "{\"name\": \"sUe\", \"type\": {\"name\": \"SE\", \"type\": \"enum\", \"symbols\": [\"A\"]}, \"default\": \"A\"}," + + "{\"name\": \"uS\", \"type\": [\"null\", \"string\"]}," + + "{\"name\": \"se2\", \"type\": {\"name\": \"SE2\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," + + "{\"name\": \"se3\", \"type\": {\"name\": \"SE3\", \"type\": \"enum\", \"symbols\": [\"A\", \"B\"]}, \"default\": \"A\"}," + + "{\"name\": \"a\", \"type\": {\"type\":\"array\", \"items\": {\"type\": \"float\"}}}," + + "{\"name\": \"m\", \"type\": {\"type\":\"map\", \"values\": {\"type\": \"float\"}}}," + + "{\"name\": \"fS\", \"type\": \"string\"}," + + "{\"name\": \"bS\", \"type\": \"string\"}," + + "{\"name\": \"fb\", \"type\": \"bytes\"}," + + "{\"name\": \"bf\", \"type\": {\"name\": \"f3\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}," + + "{\"name\": \"bfd\", \"type\": {\"name\": \"f4\",\"type\":\"fixed\", \"size\": 2}, \"default\": \"aa\"}" + + "]}", ) } } diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt index 832410a9..f932a3cc 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/BinaryRecordContentTest.kt @@ -33,16 +33,21 @@ class BinaryRecordContentTest { fun writeToStream() = runTest { val k = ObservationKey("test", "a", "b") val v = EmpaticaE4BloodVolumePulse( - 0.0, 0.0, - 0.0f + 0.0, + 0.0, + 0.0f, ) val t = AvroTopic( - "t", k.schema, v.schema, k.javaClass, v.javaClass + "t", + k.schema, + v.schema, + k.javaClass, + v.javaClass, ) val request = BinaryRecordContent( AvroRecordData(t, k, listOf(v)), ParsedSchemaMetadata(2, 1, k.schema), - ParsedSchemaMetadata(4, 2, v.schema) + ParsedSchemaMetadata(4, 2, v.schema), ) val channel = ByteChannel() @@ -83,10 +88,10 @@ class BinaryRecordContentTest { 0, 0, 0, - 0 - ) - ) - ) + 0, + ), + ), + ), ) .setProjectId(null) .setUserId(null) @@ -119,7 +124,7 @@ class BinaryRecordContentTest { values[1].toDouble(), values[2].toFloat(), values[3].toFloat(), - values[4].toFloat() + values[4].toFloat(), ) val out = ByteArrayOutputStream() encoder = EncoderFactory.get().binaryEncoder(out, encoder) @@ -157,15 +162,15 @@ class BinaryRecordContentTest { // zig-zag encoding schema used. // See http://avro.apache.org/docs/1.8.1/spec.html#binary_encoding private val EXPECTED = byteArrayOf( - 2, // key version x2 - 4, // value version x2 - 0, // null project ID - 0, // null user ID - 2, 'b'.code.toByte(), // string length x2, sourceId - 2, // number of records x2 - 40, // number of bytes in the first value x2 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // value - 0 // end of array + 2, // key version x2 + 4, // value version x2 + 0, // null project ID + 0, // null user ID + 2, 'b'.code.toByte(), // string length x2, sourceId + 2, // number of records x2 + 40, // number of bytes in the first value x2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // value + 0, // end of array ) } } diff --git a/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt b/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt index 8a7224e4..85418c75 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/rest/RestKafkaSenderTest.kt @@ -85,7 +85,10 @@ class RestKafkaSenderTest { val valueSchema = PhoneLight.getClassSchema() val topic = AvroTopic( "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + keySchema, + valueSchema, + ObservationKey::class.java, + PhoneLight::class.java, ) val topicSender = sender.sender(topic) val key = ObservationKey("test", "a", "b") @@ -126,7 +129,10 @@ class RestKafkaSenderTest { val valueSchema = PhoneLight.getClassSchema() val topic = AvroTopic( "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + keySchema, + valueSchema, + ObservationKey::class.java, + PhoneLight::class.java, ) val topicSender = sender.sender(topic) val key = ObservationKey("test", "a", "b") @@ -169,7 +175,10 @@ class RestKafkaSenderTest { val valueSchema = PhoneLight.getClassSchema() val topic = AvroTopic( "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + keySchema, + valueSchema, + ObservationKey::class.java, + PhoneLight::class.java, ) val topicSender = sender.sender(topic) val key = ObservationKey("test", "a", "b") @@ -257,7 +266,7 @@ class RestKafkaSenderTest { @Throws( IOException::class, InterruptedException::class, - SchemaValidationException::class + SchemaValidationException::class, ) fun withCompression() = runTest { sender = sender.config { @@ -268,7 +277,10 @@ class RestKafkaSenderTest { val valueSchema = PhoneLight.getClassSchema() val topic = AvroTopic( "test", - keySchema, valueSchema, ObservationKey::class.java, PhoneLight::class.java + keySchema, + valueSchema, + ObservationKey::class.java, + PhoneLight::class.java, ) val topicSender = sender.sender(topic) val key = ObservationKey("test", "a", "b") @@ -316,12 +328,12 @@ class RestKafkaSenderTest { fun MockWebServer.enqueueJson( body: String, - builder: MockResponse.() -> Unit = {} + builder: MockResponse.() -> Unit = {}, ) = enqueue( MockResponse() .setBody(body) .setHeader("Content-Type", "application/json; charset=utf-8") - .apply(builder) + .apply(builder), ) } } diff --git a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt index 425e743c..c58ed1fe 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRestClientTest.kt @@ -36,6 +36,7 @@ import kotlin.time.Duration.Companion.seconds class SchemaRestClientTest { private lateinit var mockServer: MockWebServer private lateinit var retriever: SchemaRestClient + @BeforeEach fun setUp() { mockServer = MockWebServer() @@ -43,7 +44,7 @@ class SchemaRestClientTest { HttpClient(CIO) { timeout(1.seconds) }, - baseUrl = "http://${mockServer.hostName}:${mockServer.port}/base/" + baseUrl = "http://${mockServer.hostName}:${mockServer.port}/base/", ) } diff --git a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt index a8821351..962e9556 100644 --- a/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt +++ b/radar-commons/src/test/java/org/radarbase/producer/schema/SchemaRetrieverTest.kt @@ -92,7 +92,7 @@ class SchemaRetrieverTest { retriever.getByVersion( "bla", false, - 2 + 2, ) } } @@ -107,7 +107,7 @@ class SchemaRetrieverTest { var request = mockServer.takeRequest() assertEquals("{\"schema\":\"\\\"string\\\"\"}", request.body.readUtf8()) val schemaFields = listOf( - Schema.Field("a", Schema.create(Schema.Type.INT), "that a", 10) + Schema.Field("a", Schema.create(Schema.Type.INT), "that a", 10), ) val record = Schema.createRecord("C", "that C", "org.radarcns", false, schemaFields) mockServer.enqueueJson("{\"id\":11}") @@ -116,7 +116,7 @@ class SchemaRetrieverTest { request = mockServer.takeRequest() assertEquals( "{\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"C\\\",\\\"namespace\\\":\\\"org.radarcns\\\",\\\"doc\\\":\\\"that C\\\",\\\"fields\\\":[{\\\"name\\\":\\\"a\\\",\\\"type\\\":\\\"int\\\",\\\"doc\\\":\\\"that a\\\",\\\"default\\\":10}]}\"}", - request.body.readUtf8() + request.body.readUtf8(), ) } } diff --git a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt index 62841f04..f9744a57 100644 --- a/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt +++ b/radar-commons/src/test/java/org/radarbase/topic/SensorTopicTest.kt @@ -33,10 +33,11 @@ class SensorTopicTest { val keySchema = SchemaBuilder.record("key").fields() .name("projectId").type( Schema.createUnion( - Schema.create(Schema.Type.NULL), Schema.create( - Schema.Type.STRING - ) - ) + Schema.create(Schema.Type.NULL), + Schema.create( + Schema.Type.STRING, + ), + ), ).withDefault(null) .name("userId").type(Schema.create(Schema.Type.STRING)).noDefault() .name("sourceId").type(Schema.create(Schema.Type.STRING)).noDefault() @@ -48,8 +49,10 @@ class SensorTopicTest { .endRecord() SensorTopic( "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java + keySchema, + valueSchema, + GenericRecord::class.java, + GenericRecord::class.java, ) } @@ -67,8 +70,10 @@ class SensorTopicTest { assertThrows { SensorTopic( "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java + keySchema, + valueSchema, + GenericRecord::class.java, + GenericRecord::class.java, ) } } @@ -86,8 +91,10 @@ class SensorTopicTest { assertThrows { SensorTopic( "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java + keySchema, + valueSchema, + GenericRecord::class.java, + GenericRecord::class.java, ) } } @@ -102,8 +109,10 @@ class SensorTopicTest { assertThrows { SensorTopic( "test", - keySchema, valueSchema, - GenericRecord::class.java, GenericRecord::class.java + keySchema, + valueSchema, + GenericRecord::class.java, + GenericRecord::class.java, ) } } @@ -112,12 +121,15 @@ class SensorTopicTest { fun parseTopic() { val topic: SensorTopic = SensorTopic.parse( "test", - ObservationKey::class.java.name, PhoneAcceleration::class.java.name + ObservationKey::class.java.name, + PhoneAcceleration::class.java.name, ) val expected = SensorTopic( "test", - ObservationKey.getClassSchema(), PhoneAcceleration.getClassSchema(), - ObservationKey::class.java, PhoneAcceleration::class.java + ObservationKey.getClassSchema(), + PhoneAcceleration.getClassSchema(), + ObservationKey::class.java, + PhoneAcceleration::class.java, ) assertEquals(expected, topic) } @@ -128,7 +140,7 @@ class SensorTopicTest { SensorTopic.parse( "test", "unexisting." + ObservationKey::class.java.name, - PhoneAcceleration::class.java.name + PhoneAcceleration::class.java.name, ) } } @@ -139,7 +151,7 @@ class SensorTopicTest { SensorTopic.parse( "test", ObservationKey::class.java.name, - "unexisting." + PhoneAcceleration::class.java.name + "unexisting." + PhoneAcceleration::class.java.name, ) } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 19f1e0e9..5ca62c8a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,21 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +rootProject.name = "radar-commons" include(":radar-commons") +include(":radar-commons-server") include(":radar-commons-kotlin") include(":radar-commons-testing") -include(":radar-commons-server") pluginManagement { - val kotlin = "1.8.10" - val avro = "1.6.0" - val dependencyUpdate = "0.46.0" - val nexus = "1.3.0" - val dokka = "1.8.10" - - plugins { - kotlin("plugin.serialization") version kotlin - id("com.github.davidmc24.gradle.plugin.avro") version avro - } + includeBuild("radar-commons-gradle") } From aa476526c0a1a42838c2e142892bcd778869c084 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 22 Mar 2023 13:15:37 +0100 Subject: [PATCH 12/25] Fix GA --- .github/workflows/main.yml | 2 +- .github/workflows/release.yml | 2 +- gradle.properties | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fdf9e8ef..6aa25a9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: run: ./gradlew check - name: Check version - run: + run: | projectVersion=$(./gradlew properties | grep '^version:.*$') pluginVersion=$(./gradlew :radar-commons-plugin:properties | grep '^version:.*$') if [ "$projectVersion" != "$pluginVersion" ]; then diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69d204de..3d91372a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: uses: gradle/gradle-build-action@v2 - name: Check version - run: + run: | projectVersion=$(./gradlew properties | grep '^version:.*$') pluginVersion=$(./gradlew :radar-commons-plugin:properties | grep '^version:.*$') if [ "$projectVersion" != "$pluginVersion" ]; then diff --git a/gradle.properties b/gradle.properties index 5d0dd4f5..821e1274 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1 @@ org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 - -projectVersion=0.16.0-SNAPSHOT From 021dab4036af61666bf83146b51482dfbd74d137 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 22 Mar 2023 13:18:01 +0100 Subject: [PATCH 13/25] Fix radar-commons-gradle references --- .github/workflows/main.yml | 2 +- .github/workflows/publish_snapshots.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6aa25a9f..0bb5810c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: - name: Check version run: | projectVersion=$(./gradlew properties | grep '^version:.*$') - pluginVersion=$(./gradlew :radar-commons-plugin:properties | grep '^version:.*$') + pluginVersion=$(./gradlew :radar-commons-gradle:properties | grep '^version:.*$') if [ "$projectVersion" != "$pluginVersion" ]; then echo "Project version $projectVersion does not match plugin version $pluginVersion" exit 1 diff --git a/.github/workflows/publish_snapshots.yml b/.github/workflows/publish_snapshots.yml index dd68ccd6..0262d401 100644 --- a/.github/workflows/publish_snapshots.yml +++ b/.github/workflows/publish_snapshots.yml @@ -40,9 +40,9 @@ jobs: run: ./gradlew -Psigning.gnupg.keyName=CBEF2CF0 -Psigning.gnupg.executable=gpg -Psigning.gnupg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} publish - name: Plugin has SNAPSHOT version - run: ./gradlew :radar-commons-plugin:properties | grep '^version:.*-SNAPSHOT$' + run: ./gradlew :radar-commons-gradle:properties | grep '^version:.*-SNAPSHOT$' - name: Publish gradle plugin env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./gradlew :radar-commons-plugin:publish + run: ./gradlew :radar-commons-gradle:publish diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d91372a..591f1c8b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: - name: Check version run: | projectVersion=$(./gradlew properties | grep '^version:.*$') - pluginVersion=$(./gradlew :radar-commons-plugin:properties | grep '^version:.*$') + pluginVersion=$(./gradlew :radar-commons-gradle:properties | grep '^version:.*$') if [ "$projectVersion" != "$pluginVersion" ]; then echo "Project version $projectVersion does not match plugin version $pluginVersion" exit 1 @@ -57,4 +57,4 @@ jobs: - name: Publish gradle plugin env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./gradlew :radar-commons-plugin:publish + run: ./gradlew :radar-commons-gradle:publish From 49921a8938127207429eb06e33e58f5ebd40e3b2 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 22 Mar 2023 13:20:59 +0100 Subject: [PATCH 14/25] Remove unused dependencies --- buildSrc/build.gradle.kts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 0b7c1c73..876c922b 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -4,13 +4,4 @@ plugins { repositories { mavenCentral() - gradlePluginPortal() -} - -dependencies { - implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.10") - implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.8.10") - implementation("com.github.ben-manes:gradle-versions-plugin:0.46.0") - implementation("io.github.gradle-nexus:publish-plugin:1.3.0") - implementation("org.jlleitschuh.gradle:ktlint-gradle:11.3.1") } From cd99906e5ef744adcbd907bac04ec38727788f4a Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Mar 2023 11:09:49 +0100 Subject: [PATCH 15/25] Documentation updates --- README.md | 4 +- build.gradle.kts | 9 +- radar-commons-gradle/README.md | 89 +++++++++++++++++++ radar-commons-gradle/build.gradle.kts | 11 +++ .../plugin/RadarDependencyManagementPlugin.kt | 12 +-- .../gradle/plugin/RadarKotlinPlugin.kt | 61 ++++++++++++- 6 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 radar-commons-gradle/README.md diff --git a/README.md b/README.md index e9be27fe..f9efa295 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ sender.sender(topic).use { topicSender -> } } ``` -Note that this code above does not include any flows for registering a source with the managmentportal. +Note that this code above does not include any flows for registering a source with the ManagementPortal. For server utilities, include `radar-commons-server`: ```gradle @@ -90,6 +90,8 @@ To test your backend with a MockProducer, copy `testing/mock.yml.template` to `t ``` to send data to your backend. +To use the RADAR Gradle plugins, see the README of the `radar-commons-gradle` directory. + ## Contributing For latest code use `dev` branch. This is released on JFrog's OSS Artifactory. To use that release, add the following fragment to your `build.gradle` file. diff --git a/build.gradle.kts b/build.gradle.kts index 46491766..ffbf6ca1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,6 @@ +import org.radarbase.gradle.plugin.radarKotlin +import org.radarbase.gradle.plugin.radarPublishing + /* * Copyright 2017 The Hyve and King's College London * @@ -13,15 +16,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import org.radarbase.gradle.plugin.radarKotlin -import org.radarbase.gradle.plugin.radarPublishing -import org.radarbase.gradle.plugin.radarRootProject - plugins { kotlin("plugin.serialization") version Versions.Plugins.kotlinSerialization apply false id("com.github.davidmc24.gradle.plugin.avro") version Versions.Plugins.avro apply false id("org.radarbase.radar-root-project") id("org.radarbase.radar-dependency-management") + id("org.radarbase.radar-kotlin") apply false + id("org.radarbase.radar-publishing") apply false } val githubRepoName = "RADAR-base/radar-commons" diff --git a/radar-commons-gradle/README.md b/radar-commons-gradle/README.md new file mode 100644 index 00000000..47869a7c --- /dev/null +++ b/radar-commons-gradle/README.md @@ -0,0 +1,89 @@ +# radar-commons-gradle + +A Gradle plugin to do some common RADAR-base tasks. + +## Usage + +Add the following block to `settings.gradle.kts` to get access to the RADAR-base plugins. + +```gradle +pluginManagement { + repositories { + gradlePluginPortal() + mavenCentral() + maven(url = "https://maven.pkg.github.com/radar-base/radar-commons") { + credentials { + username = System.getenv("GITHUB_ACTOR") + ?: extra.properties["gpr.user"] as? String + ?: extra.properties["public.gpr.user"] as? String + password = System.getenv("GITHUB_TOKEN") + ?: extra.properties["gpr.token"] as? String + ?: (extra.properties["public.gpr.token"] as? String)?.let { + Base64.getDecoder().decode(it).decodeToString() + } + } + } + } +} +``` + +We recommend to store a Base64 encoded PAT in your projects' `gradle.properties` with only `read:packages` access, created in your [GitHub Developer settings](https://github.com/settings/tokens/new?scopes=read:packages&description=GPR%20for%20Gradle). The Base64 encoded token should be stored as `public.gpr.token` and the associated username as `public.gpr.user`. To use your personal PAT, store the PAT in `~/.gradle/gradle.properties` with keys `gpr.user` and `gpr.token`. Use the following PAT if needed +```properties +public.gpr.user=radar-public +public.gpr.token=Z2hwX0h0d0FHSmJzeEpjenBlUVIycVhWb0RpNGdZdHZnZzJTMFVJZA== +``` +Note that the above credentials may be changed or revoked at any time. + +Then use the plugins with the following root project configurations: + +```gradle +import org.radarbase.gradle.plugin.radarKotlin +import org.radarbase.gradle.plugin.radarPublishing + +plugins { + val radarCommonsVersion = "..." + id("org.radarbase.radar-root-project") version radarCommonsVersion + id("org.radarbase.radar-dependency-management") version radarCommonsVersion + id("org.radarbase.radar-kotlin") version radarCommonsVersion apply false + id("org.radarbase.radar-publishing") version radarCommonsVersion apply false +} + +radarRootProject { + projectVersion.set(Versions.project) + group.set("org.radarbase") // is already default value + gradleVersion.set(Versions.gradle) // already has a default value +} + +radarDependencies { + regex.set("(^[0-9,.v-]+(-r)?|RELEASE|FINAL|GA|-CE)$") // default value + // default value, if set to true then disregard major version + // updates, e.g. 5.0.0 -> 6.0.0 is not allowed but 1.6.0 -> 1.7.0 is allowed. + rejectMajorVersionUpdates.set(false) +} + +subprojects { + apply(plugin = "org.radarbase.radar-kotlin") + apply(plugin = "org.radarbase.radar-publishing") + + radarKotlin { + javaVersion.set(Versions.java) // already has a default value + kotlinVersion.set(Versions.Plugins.kotlin) // already has a default value + junitVersion.set(Versions.junit) // already has a default value + ktlintVersion.set(Versions.ktlint) // already has a default value + } + + // Both values are required to be set to use radar-publishing. + // This will force the use of GPG signing maven publications. + radarPublishing { + githubUrl.set("https://github.com/RADAR-base/my-project") + developers { + developer { + id.set("myhandle") + name.set("My Name") + email.set("my@email.com") + organization.set("My company") + } + } + } +} +``` diff --git a/radar-commons-gradle/build.gradle.kts b/radar-commons-gradle/build.gradle.kts index 84753926..fbe5aa42 100644 --- a/radar-commons-gradle/build.gradle.kts +++ b/radar-commons-gradle/build.gradle.kts @@ -1,3 +1,5 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + plugins { `kotlin-dsl` `java-gradle-plugin` @@ -43,6 +45,15 @@ gradlePlugin { } } +tasks.withType { + options.release.set(11) +} +tasks.withType { + kotlinOptions { + jvmTarget = "11" + } +} + publishing { repositories { maven { diff --git a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt index 2b370813..8baf32b6 100644 --- a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarDependencyManagementPlugin.kt @@ -10,20 +10,20 @@ import org.gradle.kotlin.dsl.apply import org.gradle.kotlin.dsl.configure import org.gradle.kotlin.dsl.create -fun Project.radarDependencyManagement(configure: RadarDependencyManagementExtension.() -> Unit) { - configure(configure) +fun Project.radarDependencyManagement(block: RadarDependencyManagementExtension.() -> Unit) { + configure(block) } interface RadarDependencyManagementExtension { val regex: Property - val minorUpdatesOnly: Property + val rejectMajorVersionUpdates: Property } class RadarDependencyManagementPlugin : Plugin { override fun apply(project: Project): Unit = with(project) { val extension = extensions.create("radarDependencies").apply { regex.convention("(^[0-9,.v-]+(-r)?|RELEASE|FINAL|GA|-CE)$") - minorUpdatesOnly.convention(false) + rejectMajorVersionUpdates.convention(false) } apply() @@ -38,9 +38,9 @@ class RadarDependencyManagementPlugin : Plugin { } } val isStable = extension.regex.get().toRegex(RegexOption.IGNORE_CASE) - val checkMinorOnly = extension.minorUpdatesOnly.get() + val rejectMajorVersionUpdates = extension.rejectMajorVersionUpdates.get() rejectVersionIf { - (!checkMinorOnly || candidate.version.split('.', limit = 2)[0] != currentVersion.split('.', limit = 2)[0]) + (!rejectMajorVersionUpdates || candidate.version.split('.', limit = 2)[0] != currentVersion.split('.', limit = 2)[0]) && !isStable.containsMatchIn(candidate.version) } } diff --git a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt index e42f301a..22cd776c 100644 --- a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt @@ -2,8 +2,12 @@ package org.radarbase.gradle.plugin import org.gradle.api.Plugin import org.gradle.api.Project +import org.gradle.api.plugins.ApplicationPlugin +import org.gradle.api.plugins.JavaApplication import org.gradle.api.provider.Property import org.gradle.api.tasks.compile.JavaCompile +import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.testing.logging.TestExceptionFormat import org.gradle.kotlin.dsl.* import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion @@ -12,13 +16,15 @@ import org.jlleitschuh.gradle.ktlint.KtlintExtension import org.jlleitschuh.gradle.ktlint.KtlintPlugin fun Project.radarKotlin(configure: RadarKotlinExtension.() -> Unit) { - configure(configure) + configure(configure) } interface RadarKotlinExtension { val javaVersion: Property val kotlinVersion: Property val junitVersion: Property + val log4j2Version: Property + val slf4jVersion: Property val ktlintVersion: Property } @@ -29,6 +35,7 @@ class RadarKotlinPlugin : Plugin { kotlinVersion.convention(Versions.kotlin) junitVersion.convention(Versions.junit) ktlintVersion.convention(Versions.ktlint) + slf4jVersion.convention(Versions.ktlint) } apply(plugin = "kotlin") @@ -82,6 +89,58 @@ class RadarKotlinPlugin : Plugin { configurations["testRuntimeOnly"](extension.junitVersion.map { "org.junit.jupiter:junit-jupiter-engine:$it" }) } + tasks.withType { + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + exceptionFormat = TestExceptionFormat.FULL + } + useJUnitPlatform() + } + + afterEvaluate { + if (extension.slf4jVersion.isPresent) { + dependencies { + val implementation by configurations + implementation("org.slf4j:slf4j-api:${extension.slf4jVersion.get()}") + } + } + if (extension.log4j2Version.isPresent) { + dependencies { + val log4j2Version = extension.log4j2Version.get() + + if (plugins.hasPlugin("application")) { + val runtimeOnly by configurations + runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") + runtimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") + runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") + } else { + val testRuntimeOnly by configurations + testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2Version") + testRuntimeOnly("org.apache.logging.log4j:log4j-core:$log4j2Version") + testRuntimeOnly("org.apache.logging.log4j:log4j-jul:$log4j2Version") + } + } + + tasks.withType { + if ("java.util.logging.manager" !in systemProperties) { + systemProperty( + "java.util.logging.manager", + "org.apache.logging.log4j.jul.LogManager" + ) + } + } + + if (plugins.hasPlugin(ApplicationPlugin::class)) { + extensions.configure { + if (applicationDefaultJvmArgs.none { "-Djava.util.logging.manager=" in it }) { + applicationDefaultJvmArgs += "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" + } + } + } + } + } + configurations.named("implementation") { resolutionStrategy.cacheChangingModulesFor(0, "SECONDS") } From 81406e3b8ada6c274f6c75c80722277e97132dad Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Mar 2023 11:51:43 +0100 Subject: [PATCH 16/25] Build simplifications --- build.gradle.kts | 43 ++------------------------ radar-commons-kotlin/build.gradle.kts | 2 -- radar-commons-server/build.gradle.kts | 1 - radar-commons-testing/build.gradle.kts | 1 - radar-commons/build.gradle.kts | 4 --- 5 files changed, 2 insertions(+), 49 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ffbf6ca1..edaed552 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -39,6 +39,7 @@ subprojects { dependencies { configurations["testImplementation"]("org.jetbrains.kotlinx:kotlinx-coroutines-test:${Versions.coroutines}") + configurations["testRuntimeOnly"]("org.slf4j:slf4j-simple:${Versions.slf4j}") } radarPublishing { @@ -63,50 +64,10 @@ subprojects { javaVersion.set(Versions.java) kotlinVersion.set(Versions.Plugins.kotlin) junitVersion.set(Versions.junit) + slf4jVersion.set(Versions.slf4j) } //---------------------------------------------------------------------------// // Style checking // //---------------------------------------------------------------------------// - - tasks.withType { - useJUnitPlatform() - - val numberOfLines = 100 - val stdout = ArrayDeque(numberOfLines) - beforeTest(closureOf { - stdout.clear() - }) - - onOutput(KotlinClosure2({ _, toe -> - toe.message.split("(?m)$").forEach { line -> - if (stdout.size == numberOfLines) { - stdout.removeFirst() - } - stdout.addLast(line) - } - })) - - afterTest(KotlinClosure2({ td, tr -> - if (tr.resultType == TestResult.ResultType.FAILURE) { - println() - print("${td.className}.${td.name} FAILED") - if (stdout.isEmpty()) { - println(" without any output") - } else { - println(" with last 100 lines of output:") - println("=".repeat(100)) - stdout.forEach { print(it) } - println("=".repeat(100)) - } - } - })) - - testLogging { - showExceptions = true - showCauses = true - showStackTraces = true - setExceptionFormat("full") - } - } } diff --git a/radar-commons-kotlin/build.gradle.kts b/radar-commons-kotlin/build.gradle.kts index 65416862..33cced0f 100644 --- a/radar-commons-kotlin/build.gradle.kts +++ b/radar-commons-kotlin/build.gradle.kts @@ -5,8 +5,6 @@ plugins { description = "Library for Kotlin utility classes and functions" dependencies { - implementation("org.slf4j:slf4j-api:${Versions.slf4j}") - api(platform("org.jetbrains.kotlinx:kotlinx-coroutines-bom:${Versions.coroutines}")) api("org.jetbrains.kotlinx:kotlinx-coroutines-core") diff --git a/radar-commons-server/build.gradle.kts b/radar-commons-server/build.gradle.kts index c20f826d..f3522cda 100644 --- a/radar-commons-server/build.gradle.kts +++ b/radar-commons-server/build.gradle.kts @@ -38,7 +38,6 @@ dependencies { // Direct producer uses KafkaAvroSerializer if initialized testImplementation("io.confluent:kafka-avro-serializer:${Versions.confluent}") testImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") - testRuntimeOnly("org.slf4j:slf4j-simple:${Versions.slf4j}") } val generateAvroJava by tasks diff --git a/radar-commons-testing/build.gradle.kts b/radar-commons-testing/build.gradle.kts index e9c5694d..a1513bcc 100644 --- a/radar-commons-testing/build.gradle.kts +++ b/radar-commons-testing/build.gradle.kts @@ -56,6 +56,5 @@ dependencies { applicationRuntimeOnly("org.slf4j:slf4j-simple:${Versions.slf4j}") testImplementation("org.hamcrest:hamcrest:${Versions.hamcrest}") - testImplementation("org.slf4j:slf4j-simple:${Versions.slf4j}") testImplementation("org.mockito:mockito-core:${Versions.mockito}") } diff --git a/radar-commons/build.gradle.kts b/radar-commons/build.gradle.kts index 96c98614..02d3db94 100644 --- a/radar-commons/build.gradle.kts +++ b/radar-commons/build.gradle.kts @@ -29,14 +29,10 @@ dependencies { api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.coroutines}") - // The production code uses the SLF4J logging API at compile time - implementation("org.slf4j:slf4j-api:${Versions.slf4j}") - testImplementation(platform("com.fasterxml.jackson:jackson-bom:${Versions.jackson}")) testImplementation("com.fasterxml.jackson.core:jackson-databind") testImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}") testImplementation("org.mockito:mockito-core:${Versions.mockito}") testImplementation("org.mockito.kotlin:mockito-kotlin:${Versions.mockitoKotlin}") testImplementation("com.squareup.okhttp3:mockwebserver:${Versions.okhttp}") - testRuntimeOnly("org.slf4j:slf4j-simple:${Versions.slf4j}") } From 4bb17e8025c6a57fa142c353fdec120545a6b6ea Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Mar 2023 14:09:54 +0100 Subject: [PATCH 17/25] Add back SchemaRetriever.getById --- .../gradle/plugin/RadarKotlinPlugin.kt | 19 ++++++ .../producer/schema/SchemaRetriever.kt | 67 ++++++++++++------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt index 22cd776c..886a1fd5 100644 --- a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt @@ -5,6 +5,7 @@ import org.gradle.api.Project import org.gradle.api.plugins.ApplicationPlugin import org.gradle.api.plugins.JavaApplication import org.gradle.api.provider.Property +import org.gradle.api.tasks.Copy import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.testing.Test import org.gradle.api.tasks.testing.logging.TestExceptionFormat @@ -98,6 +99,24 @@ class RadarKotlinPlugin : Plugin { useJUnitPlatform() } + + tasks.register("downloadDependencies") { + doFirst { + configurations["compileClasspath"].files + configurations["runtimeClasspath"].files + println("Downloaded all dependencies") + } + outputs.upToDateWhen { false } + } + + tasks.register("copyDependencies") { + from(configurations.named("runtimeClasspath").map { it.files }) + into("$buildDir/third-party/") + doLast { + println("Copied third-party runtime dependencies") + } + } + afterEvaluate { if (extension.slf4jVersion.isPresent) { dependencies { diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt index a128a19a..044b6022 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt @@ -28,6 +28,7 @@ import org.radarbase.kotlin.coroutines.CacheConfig import org.radarbase.kotlin.coroutines.CachedValue import org.radarbase.util.RadarProducerDsl import java.io.IOException +import java.lang.ref.SoftReference import java.util.* import java.util.Objects.hash import java.util.concurrent.ConcurrentHashMap @@ -42,6 +43,7 @@ typealias VersionCache = ConcurrentMap> * Retriever of an Avro Schema. */ open class SchemaRetriever(config: Config) { + private val idCache: ConcurrentMap> = ConcurrentHashMap() private val schemaCache: ConcurrentMap> = ConcurrentHashMap() private val subjectVersionCache: ConcurrentMap = ConcurrentHashMap() @@ -72,19 +74,6 @@ open class SchemaRetriever(config: Config) { metadata.id } - private fun cachedMetadata( - subject: String, - schema: Schema, - ): CachedValue = schemaCache.computeIfAbsent(schema) { - CachedValue(schemaTimeout) { - val metadata = restClient.requestMetadata(subject, schema) - if (metadata.version != null) { - cachedVersion(subject, metadata.version).set(metadata) - } - metadata - } - } - /** Get schema metadata. Cached schema metadata will be used if present. */ @Throws(IOException::class) suspend fun getByVersion( @@ -104,6 +93,46 @@ open class SchemaRetriever(config: Config) { return metadata } + + /** Get schema metadata. Cached schema metadata will be used if present. */ + @Throws(IOException::class) + suspend fun getById( + topic: String, + ofValue: Boolean, + id: Int, + ): ParsedSchemaMetadata { + val subject = subject(topic, ofValue) + val schema = idCache[id]?.get() + ?: restClient.retrieveSchemaById(id) + + return cachedMetadata(subject, schema).get() + } + + /** Get the metadata of a specific schema in a topic. */ + @Throws(IOException::class) + suspend fun metadata( + topic: String, + ofValue: Boolean, + schema: Schema, + ): ParsedSchemaMetadata { + val subject = subject(topic, ofValue) + return cachedMetadata(subject, schema).get() + } + + private fun cachedMetadata( + subject: String, + schema: Schema, + ): CachedValue = schemaCache.computeIfAbsent(schema) { + CachedValue(schemaTimeout) { + val metadata = restClient.requestMetadata(subject, schema) + if (metadata.version != null) { + cachedVersion(subject, metadata.version).set(metadata) + } + idCache[metadata.id] = SoftReference(metadata.schema) + metadata + } + } + private suspend fun cachedVersion( subject: String, version: Int, @@ -123,23 +152,13 @@ open class SchemaRetriever(config: Config) { CachedValue(schemaTimeout) { val metadata = restClient.retrieveSchemaMetadata(subject, version) cachedMetadata(subject, metadata.schema).set(metadata) + idCache[metadata.id] = SoftReference(metadata.schema) metadata } } return versionId } - /** Get the metadata of a specific schema in a topic. */ - @Throws(IOException::class) - suspend fun metadata( - topic: String, - ofValue: Boolean, - schema: Schema, - ): ParsedSchemaMetadata { - val subject = subject(topic, ofValue) - return cachedMetadata(subject, schema).get() - } - private suspend fun MutableCollection>.prune() { val iter = iterator() while (iter.hasNext()) { From f709a66f1ff108cad56e4d0f19bab0e61bb70ffd Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Mar 2023 16:19:32 +0100 Subject: [PATCH 18/25] Added ktor auth extension functions and opened schemaRetriever for mocking --- .../kotlin/org/radarbase/ktor/auth/Extensions.kt | 15 +++++++++++++++ .../radarbase/producer/schema/SchemaRetriever.kt | 11 +++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt new file mode 100644 index 00000000..dbb70090 --- /dev/null +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt @@ -0,0 +1,15 @@ + +import io.ktor.client.request.* +import io.ktor.http.* +import io.ktor.util.* + +fun HttpRequestBuilder.basicAuth(username: String, password: String) { + val credentials = "$username:$password" + .toByteArray(Charsets.UTF_8) + .encodeBase64() + headers[HttpHeaders.Authorization] = "Basic $credentials" +} + +fun HttpRequestBuilder.bearer(token: String) { + headers[HttpHeaders.Authorization] = "Bearer $token" +} diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt index 044b6022..6ec144fa 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt @@ -76,7 +76,7 @@ open class SchemaRetriever(config: Config) { /** Get schema metadata. Cached schema metadata will be used if present. */ @Throws(IOException::class) - suspend fun getByVersion( + open suspend fun getByVersion( topic: String, ofValue: Boolean, version: Int, @@ -93,10 +93,9 @@ open class SchemaRetriever(config: Config) { return metadata } - /** Get schema metadata. Cached schema metadata will be used if present. */ @Throws(IOException::class) - suspend fun getById( + open suspend fun getById( topic: String, ofValue: Boolean, id: Int, @@ -110,7 +109,7 @@ open class SchemaRetriever(config: Config) { /** Get the metadata of a specific schema in a topic. */ @Throws(IOException::class) - suspend fun metadata( + open suspend fun metadata( topic: String, ofValue: Boolean, schema: Schema, @@ -180,7 +179,7 @@ open class SchemaRetriever(config: Config) { /** * Remove expired entries from cache. */ - suspend fun pruneCache() = coroutineScope { + open suspend fun pruneCache() = coroutineScope { launch { schemaCache.values.prune() } @@ -200,7 +199,7 @@ open class SchemaRetriever(config: Config) { /** * Remove all entries from cache. */ - fun clearCache() { + open fun clearCache() { subjectVersionCache.clear() schemaCache.clear() } From eb84e54e0875cc3da525609f0830dc7dae890f68 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Mar 2023 16:21:57 +0100 Subject: [PATCH 19/25] Remove unused function --- .../src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt index dbb70090..bafa0b5f 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt @@ -1,14 +1,6 @@ import io.ktor.client.request.* import io.ktor.http.* -import io.ktor.util.* - -fun HttpRequestBuilder.basicAuth(username: String, password: String) { - val credentials = "$username:$password" - .toByteArray(Charsets.UTF_8) - .encodeBase64() - headers[HttpHeaders.Authorization] = "Basic $credentials" -} fun HttpRequestBuilder.bearer(token: String) { headers[HttpHeaders.Authorization] = "Bearer $token" From e71dc2f52e7c1b2d40ddc557fdf5c2be8cdc2dcc Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 28 Mar 2023 14:27:55 +0200 Subject: [PATCH 20/25] Misc fixes --- .../kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt | 2 +- .../src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt index 886a1fd5..eff4de65 100644 --- a/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt +++ b/radar-commons-gradle/src/main/kotlin/org/radarbase/gradle/plugin/RadarKotlinPlugin.kt @@ -43,7 +43,7 @@ class RadarKotlinPlugin : Plugin { apply() repositories { - mavenCentral() { + mavenCentral { mavenContent { releasesOnly() } diff --git a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt index bafa0b5f..c48bab65 100644 --- a/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt +++ b/radar-commons-kotlin/src/main/kotlin/org/radarbase/ktor/auth/Extensions.kt @@ -1,3 +1,4 @@ +package org.radarbase.ktor.auth import io.ktor.client.request.* import io.ktor.http.* From 402f05aa39a3de5c0fcdc11f3f6d04af659b96f7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 28 Mar 2023 14:36:13 +0200 Subject: [PATCH 21/25] Allow publishing the gradle plugins via gradle properties --- radar-commons-gradle/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radar-commons-gradle/build.gradle.kts b/radar-commons-gradle/build.gradle.kts index fbe5aa42..ea2313ec 100644 --- a/radar-commons-gradle/build.gradle.kts +++ b/radar-commons-gradle/build.gradle.kts @@ -61,7 +61,9 @@ publishing { setUrl("https://maven.pkg.github.com/radar-base/radar-commons") credentials { username = System.getenv("GITHUB_ACTOR") + ?: extra.properties["gpr.user"] as? String password = System.getenv("GITHUB_TOKEN") + ?: extra.properties["gpr.key"] as? String } } } From 24104d2a50d8c77b366adae09429fbcd685bcf62 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 28 Mar 2023 15:45:13 +0200 Subject: [PATCH 22/25] More consistent test results --- .../kotlin/coroutines/CachedValueTest.kt | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt index c329b384..bbb90c14 100644 --- a/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt +++ b/radar-commons-kotlin/src/test/kotlin/org/radarbase/kotlin/coroutines/CachedValueTest.kt @@ -9,7 +9,6 @@ import org.hamcrest.Matchers.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows -import org.slf4j.LoggerFactory import java.util.concurrent.atomic.AtomicInteger import kotlin.time.Duration.Companion.milliseconds @@ -23,9 +22,9 @@ internal class CachedValueTest { fun setUp() { calls.set(0) config = CacheConfig( - refreshDuration = 20.milliseconds, - retryDuration = 10.milliseconds, - exceptionCacheDuration = 10.milliseconds, + refreshDuration = 40.milliseconds, + retryDuration = 20.milliseconds, + exceptionCacheDuration = 20.milliseconds, ) } @@ -35,7 +34,7 @@ internal class CachedValueTest { runBlocking(GlobalScope.coroutineContext) { assertThat("Initial value should refresh", cache.get(), `is`(1)) assertThat("No refresh within threshold", cache.get(), `is`(1)) - delay(10) + delay(20.milliseconds) assertThat("Refresh after threshold", cache.get(), `is`(2)) assertThat("No refresh after threshold", cache.get(), `is`(2)) } @@ -47,7 +46,7 @@ internal class CachedValueTest { runBlocking { assertThat("Initial value should refresh", cache.get { it < 0 }, equalTo(CachedValue.CacheMiss(1))) assertThat("No refresh within threshold", cache.get { it < 0 }, equalTo(CachedValue.CacheHit(1))) - delay(10) + delay(20.milliseconds) assertThat("Refresh after threshold", cache.get { it < 0 }, equalTo(CachedValue.CacheMiss(2))) assertThat("No refresh after threshold", cache.get { it < 0 }, equalTo(CachedValue.CacheHit(2))) } @@ -59,7 +58,7 @@ internal class CachedValueTest { runBlocking { assertThat("Initial value should refresh", cache.get { it >= 0 }, equalTo(CachedValue.CacheMiss(1))) assertThat("No refresh within threshold", cache.get { it >= 0 }, equalTo(CachedValue.CacheHit(1))) - delay(10) + delay(20.milliseconds) assertThat("No refresh after valid value", cache.get { it >= 0 }, equalTo(CachedValue.CacheHit(1))) } } @@ -83,20 +82,20 @@ internal class CachedValueTest { runBlocking { assertThat("Initial value should refresh", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheMiss(2))) assertThat("No refresh within threshold", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheHit(2))) - delay(10) + delay(20.milliseconds) assertThat( "Retry because predicate does not match", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheMiss(3)), ) assertThat("No refresh within threshold", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheHit(3))) - delay(10) + delay(20.milliseconds) assertThat( "No retry because predicate matches", cache.query({ it + 1 }, { it > 2 }), equalTo(CachedValue.CacheHit(3)), ) - delay(10) + delay(20.milliseconds) assertThat( "Refresh after refresh threshold since last retry", cache.query({ it + 1 }, { it > 2 }), @@ -109,7 +108,7 @@ internal class CachedValueTest { fun getMultithreaded() { val cache = CachedValue(config) { calls.incrementAndGet() - delay(50.milliseconds) + delay(100.milliseconds) calls.get() } @@ -134,7 +133,7 @@ internal class CachedValueTest { ), ) { calls.incrementAndGet() - delay(50.milliseconds) + delay(100.milliseconds) calls.get() } @@ -155,7 +154,7 @@ internal class CachedValueTest { @Test fun throwTest() { - val cache = CachedValue(config.copy(refreshDuration = 20.milliseconds)) { + val cache = CachedValue(config.copy(refreshDuration = 40.milliseconds)) { val newValue = calls.incrementAndGet() if (newValue % 2 == 0) throw IllegalStateException() else newValue } @@ -163,15 +162,11 @@ internal class CachedValueTest { runBlocking { assertThat(cache.get(), `is`(1)) assertThat(cache.get(), `is`(1)) - delay(21.milliseconds) + delay(42.milliseconds) assertThrows { cache.get() } assertThrows { cache.get() } - delay(11.milliseconds) + delay(22.milliseconds) assertThat(cache.get(), `is`(3)) } } - - companion object { - private val logger = LoggerFactory.getLogger(CachedValueTest::class.java) - } } From 05e750784007b49e68ecd66c659fdcc061e6f70c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 30 Mar 2023 15:21:24 +0200 Subject: [PATCH 23/25] Access direct requests to SchemaClient --- .../producer/schema/SchemaRestClient.kt | 61 ++++++++++++------- .../producer/schema/SchemaRetriever.kt | 3 +- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt index 5d30f212..e6f0fc89 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt @@ -8,6 +8,7 @@ import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* import io.ktor.serialization.kotlinx.json.* +import io.ktor.util.reflect.* import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json @@ -37,6 +38,34 @@ class SchemaRestClient( } } + suspend inline fun request( + noinline requestBuilder: HttpRequestBuilder.() -> Unit, + ): T = request(typeInfo(), requestBuilder) + + suspend fun request( + typeInfo: TypeInfo, + requestBuilder: HttpRequestBuilder.() -> Unit, + ): T = withContext(ioContext) { + val response = httpClient.request { + requestBuilder() + } + if (!response.status.isSuccess()) { + throw RestException(response.status, response.bodyAsText()) + } + response.body(typeInfo) + } + + suspend fun requestEmpty( + requestBuilder: HttpRequestBuilder.() -> Unit, + ) = withContext(ioContext) { + val response = httpClient.request { + requestBuilder() + } + if (!response.status.isSuccess()) { + throw RestException(response.status, response.bodyAsText()) + } + } + /** Retrieve schema metadata from server. */ @Throws(IOException::class) suspend fun retrieveSchemaMetadata( @@ -50,32 +79,20 @@ class SchemaRestClient( } @Throws(IOException::class) - suspend fun schemaGet(path: String): SchemaMetadata = withContext(ioContext) { - val response = httpClient.get { - url(path) - } - if (response.status.isSuccess()) { - response.body() - } else { - throw RestException(response.status, response.bodyAsText()) - } + suspend fun schemaGet(path: String): SchemaMetadata = request { + method = HttpMethod.Get + url(path) } @Throws(IOException::class) suspend fun schemaPost( path: String, schema: Schema, - ): SchemaMetadata = withContext(ioContext) { - val response = httpClient.post { - url(path) - contentType(ContentType.Application.Json) - setBody(SchemaMetadata(schema = schema.toString())) - } - if (response.status.isSuccess()) { - response.body() - } else { - throw RestException(response.status, response.bodyAsText()) - } + ): SchemaMetadata = request { + method = HttpMethod.Post + url(path) + contentType(ContentType.Application.Json) + setBody(SchemaMetadata(schema = schema.toString())) } /** Add a schema to a subject. */ @@ -94,9 +111,9 @@ class SchemaRestClient( suspend fun requestMetadata( subject: String, schema: Schema, - ): ParsedSchemaMetadata = withContext(ioContext) { + ): ParsedSchemaMetadata { val result = schemaPost("subjects/$subject", schema) - ParsedSchemaMetadata( + return ParsedSchemaMetadata( id = checkNotNull(result.id) { "Missing schema ID in request result" }, version = result.version, schema = schema, diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt index 6ec144fa..b3c0de8d 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRetriever.kt @@ -51,7 +51,8 @@ open class SchemaRetriever(config: Config) { private val ioContext = config.ioContext private val httpClient = requireNotNull(config.httpClient) { "Missing HTTP client" } - private val restClient: SchemaRestClient = SchemaRestClient(httpClient, baseUrl, ioContext) + val restClient: SchemaRestClient = SchemaRestClient(httpClient, baseUrl, ioContext) + private val schemaTimeout = config.schemaTimeout /** From e6be6c6e97ea61dd43243d4d61808194cbc25f51 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 4 Apr 2023 14:30:06 +0200 Subject: [PATCH 24/25] Fix schema get ID null --- .../main/java/org/radarbase/producer/schema/SchemaMetadata.kt | 4 ++-- .../java/org/radarbase/producer/schema/SchemaRestClient.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt index 677df431..2eb8096a 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaMetadata.kt @@ -33,8 +33,8 @@ data class SchemaMetadata val version: Int? = null, val schema: String? = null, ) { - fun toParsedSchemaMetadata() = ParsedSchemaMetadata( - id = checkNotNull(id) { "Need id to parse schema metadata" }, + fun toParsedSchemaMetadata(defaultId: Int? = null) = ParsedSchemaMetadata( + id = checkNotNull(id ?: defaultId) { "Need id to parse schema metadata" }, version = version, schema = Schema.Parser().parse( checkNotNull(schema) { "Need schema to parse it" }, diff --git a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt index e6f0fc89..50746c2b 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/schema/SchemaRestClient.kt @@ -123,6 +123,6 @@ class SchemaRestClient( /** Retrieve schema metadata from server. */ suspend fun retrieveSchemaById(id: Int): Schema = schemaGet("/schemas/ids/$id") - .toParsedSchemaMetadata() + .toParsedSchemaMetadata(id) .schema } From 860e1fb359491bdf30bbfa1b107d28fa478061fd Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 2 May 2023 15:10:55 +0200 Subject: [PATCH 25/25] Docs and make API more consistent --- README.md | 69 ++++++++---------- buildSrc/src/main/kotlin/Versions.kt | 16 ++-- gradle/wrapper/gradle-wrapper.jar | Bin 61608 -> 62076 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 7 +- radar-commons-gradle/build.gradle.kts | 4 +- .../org/radarbase/gradle/plugin/Versions.kt | 6 +- .../kotlin/coroutines/CachedValue.kt | 29 +++++--- .../radarbase/producer/KafkaTopicSender.kt | 13 +++- .../producer/rest/RestKafkaSender.kt | 7 ++ 10 files changed, 89 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index f9efa295..6ab1c972 100644 --- a/README.md +++ b/README.md @@ -12,51 +12,46 @@ repositories { } dependencies { - implementation("org.radarbase:radar-commons:0.15.0") + implementation("org.radarbase:radar-commons:1.0.0") } ``` Example use, after adding [`radar-schemas`](https://github.com/radar-base/radar-schemas) to classpath: + ```kotlin // Set URLs for RADAR-base installation -val baseUrl = "..." -val kafkaUrl = "$baseUrl/kafka/" -val schemaUrl = "$baseUrl/schema/" -val oauthHeaders = ... -val key = ObservationKey("myProject", "myUser", "mySource") - -// Configure RADAR-base clients -val client = RestClient.global().apply { - server(ServerConfig(kafkaUrl)) - gzipCompression(true) -}.build() +val baseUrl = "https://..." +val oauthToken = ... + +val kafkaSender = restKafkaSender { + baseUrl = "$baseUrl/kafka/" + headers.append("Authorization", "Bearer $oauthToken") + httpClient { + timeout(10.seconds) + } + schemaRetriever ("$baseUrl/schema/") +} -val schemaRetriever = SchemaRetriever(ServerConfig(schemaUrl), 30) +// Configure topic to send data over +val topic = AvroTopic( + "linux_raspberry_temperature", + ObservationKey.getClassSchema(), + RaspberryTemperature.getClassSchema(), + ObservationKey::class.java, + RaspberryTemperature::class.java +) -val restSender = RestSender.Builder().apply { - httpClient(client) - schemaRetriever(schemaRetriever) - useBinaryContent(true) - headers(oauthHeaders) -}.build() +val topicSender = kafkaSender.sender(topic) -val sender = BatchedKafkaSender(restSender, 60_000L, 1000L) +val key = ObservationKey("myProject", "myUser", "mySource") -// Configure topic to send data over -val topic = AvroTopic("linux_raspberry_temperature", - ObservationKey.getClassSchema(), RaspberryTemperature.getClassSchema(), - ObservationKey::class.java, RaspberryTemperature::class.java) - -// Send data to topic. Be sure to close -// the sender after use. Preferably, a sender is reused -// for many observations so that requests are efficiently -// batched. -sender.sender(topic).use { topicSender -> - readValuesFromSystem() { value -> - topicSender.send(key, value) - } +// Send data to topic. +runBlocking { + val values: List = readValuesFromSystem() + topicSender.send(key, values) } ``` + Note that this code above does not include any flows for registering a source with the ManagementPortal. For server utilities, include `radar-commons-server`: @@ -67,7 +62,7 @@ repositories { } dependencies { - implementation("org.radarbase:radar-commons-server:0.15.0") + implementation("org.radarbase:radar-commons-server:1.0.0") } ``` @@ -80,7 +75,7 @@ repositories { } dependencies { - testImplementation("org.radarbase:radar-commons-testing:0.15.0") + testImplementation("org.radarbase:radar-commons-testing:1.0.0") } ``` @@ -107,9 +102,9 @@ configurations.all { } dependencies { - implementation("org.radarbase:radar-commons:0.15.1-SNAPSHOT") + implementation("org.radarbase:radar-commons:1.0.1-SNAPSHOT") } ``` -Code should be formatted using the [Google Java Code Style Guide](https://google.github.io/styleguide/javaguide.html). +Code should be formatted using the Kotlin official style guide, in addition to ktlint rules. If you want to contribute a feature or fix browse our [issues](https://github.com/RADAR-base/radar-commons/issues), and please make a pull request. diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index f3e07b51..44592243 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,10 +1,10 @@ object Versions { - const val project = "0.16.0-SNAPSHOT" + const val project = "1.0.0" object Plugins { - const val kotlin = "1.8.10" + const val kotlin = "1.8.21" const val kotlinSerialization = kotlin - const val avro = "1.6.0" + const val avro = "1.7.0" } const val java = 11 @@ -12,14 +12,14 @@ object Versions { const val confluent = "7.3.2" const val kafka = "7.3.2-ce" const val avro = "1.11.1" - const val jackson = "2.14.2" - const val okhttp = "4.10.0" - const val junit = "5.9.2" - const val mockito = "5.2.0" + const val jackson = "2.15.0" + const val okhttp = "4.11.0" + const val junit = "5.9.3" + const val mockito = "5.3.1" const val mockitoKotlin = "4.1.0" const val hamcrest = "2.2" const val radarSchemas = "0.8.2" const val opencsv = "5.7.1" - const val ktor = "2.2.4" + const val ktor = "2.3.0" const val coroutines = "1.6.4" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index ccebba7710deaf9f98673a68957ea02138b60d0a..c1962a79e29d3e0ab67b14947c167a862655af9b 100644 GIT binary patch delta 8979 zcmY*fV{{$d(moANW81db*tXT!Nn`UgX2ZtD$%&n`v2C-lt;YD?@2-14?EPcUv!0n* z`^Ws4HP4i8L%;4p*JkD-J9ja2aKi!sX@~#-MY5?EPBK~fXAl)Ti}^QGH@6h+V+|}F zv=1RqQxhWW9!hTvYE!)+*m%jEL^9caK;am9X8QP~a9X0N6(=WSX8KF#WpU-6TjyR3 zpKhscivP97d$DGc{KI(f#g07u{Jr0wn#+qNr}yW}2N3{Kx0lCq%p4LBKil*QDTEyR zg{{&=GAy_O0VJ(8ZbtS4tPeeeILKK(M?HtQY!6K^wt zxsPH>E%g%V@=!B;kWF54$xjC&4hO!ZEG0QFMHLqe!tgH;%vO62BQj||nokbX&2kxF zzg#N!2M|NxFL#YdwOL8}>iDLr%2=!LZvk_&`AMrm7Zm%#_{Ot_qw=HkdVg{f9hYHF zlRF*9kxo~FPfyBD!^d6MbD?BRZj(4u9j!5}HFUt+$#Jd48Fd~ahe@)R9Z2M1t%LHa z_IP|tDb0CDl(fsEbvIYawJLJ7hXfpVw)D-)R-mHdyn5uZYefN0rZ-#KDzb`gsow;v zGX>k|g5?D%Vn_}IJIgf%nAz{@j0FCIEVWffc1Z+lliA}L+WJY=MAf$GeI7xw5YD1) z;BJn$T;JI5vTbZ&4aYfmd-XPQd)YQ~d({>(^5u>Y^5rfxEUDci9I5?dXp6{zHG=Tc z6$rLd^C~60=K4ptlZ%Fl-%QLc-x{y=zU$%&4ZU}4&Yu?jF4eqB#kTHhty`Aq=kJE% zzq(5OS9o1t-)}S}`chh1Uu-Sl?ljxMDVIy5j`97Eqg7L~Ak9NSZ?!5M>5TRMXfD#} zFlMmFnr%?ra>vkvJQjmWa8oB{63qPo1L#LAht%FG|6CEe9KP2&VNe_HNb7M}pd*!t zpGL0vzCU02%iK@AKWxP^64fz-U#%u~D+FV?*KdPY9C_9{Ggn;Y;;iKE0b|}KmC&f(WIDcFtvRPDju z?Dc&_dP4*hh!%!6(nYB*TEJs<4zn*V0Nw1O4VzYaNZul>anE2Feb@T$XkI?)u6VK$bg* z22AY7|Ju!_jwc2@JX(;SUE>VDWRD|d56WYUGLAAwPYXU9K&NgY{t{dyMskUBgV%@p zMVcFn>W|hJA?3S?$k!M|1S2e1A&_~W2p$;O2Wpn`$|8W(@~w>RR4kxHdEr`+q|>m@ zTYp%Ut+g`T#HkyE5zw<5uhFvt2=k5fM3!8OxvGgMRS|t7RaJn7!2$r_-~a%C7@*Dq zGUp2g0N^HzLU=%bROVFi2J;#`7#WGTUI$r!(wmbJlbS`E#ZpNp7vOR#TwPQWNf$IW zoX>v@6S8n6+HhUZB7V^A`Y9t4ngdfUFZrDOayMVvg&=RY4@0Z~L|vW)DZTIvqA)%D zi!pa)8L7BipsVh5-LMH4bmwt2?t88YUfIRf!@8^gX$xpKTE^WpM!-=3?UVw^Cs`Y7 z2b<*~Q=1uqs79{h&H_8+X%><4qSbz_cSEa;Hkdmtq5uwGTY+|APD{i_zYhLXqT7HO zT^Am_tW?Cmn%N~MC0!9mYt-~WK;hj-SnayMwqAAHo#^ALwkg0>72&W}5^4%|Z|@T; zwwBQTg*&eXC}j8 zra77(XC^p&&o;KrZ$`_)C$@SDWT+p$3!;ZB#yhnK{CxQc&?R}ZQMcp`!!eXLLhiP8W zM=McHAMnUMlar8XLXk&jx#HBH3U0jbhJuqa~#l`aB)N6;WI(Im322o#{K&92l6(K z)(;=;-m!%9@j#WSA1uniU(^x(UTi+%idMd)x*!*Hub0Rg7DblI!cqo9QUZf29Y#?XN!K!|ovJ7~!^H}!zsaMl(57lpztQ7V zyo#`qJ4jv1zGAW2uIkU3o&7_=lYWz3=SR!sgfuYp{Um<*H%uW8MdUT2&o*QKjD3PEH zHz;H}qCN~`GFsJ_xz$9xga*@VzJTH7-3lggkBM&7xlz5#qWfkgi=#j%{&f-NMsaSv zeIZ60Jpw}QV+t`ovOJxVhYCXe8E7r*eLCJ{lP6sqc}BYrhjXlt(6e9nw=2Le1gOT0 zZX!q9r#DZ&8_cAhWPeq~CJkGvpRU&q8>rR@RBW4~@3j1X>RBum#U z1wjcEdB`|@sXAWxk2*TOj> zr(j{nr1;Mk3x^gvAtZsahY=ou{eAJi-d(XISF-?+Q6{Um4+lu?aA=S33@k=6^OT?F z8TE`ha;q@=ZQ-dlt!q49;Wjjl<&Yee^!h5MFkd)Oj=fsvxytK%!B z-P#YJ)8^dMi=wpKmt43|apX6v2dNXzZ-WHlLEh`JoKFNjCK7LhO^P5XW?Y~rjGcIpv$2v41rE}~0{aj9NVpDXGdD6W8{fyzioQdu&xkn8 zhT*^NY0zv>Om?h3XAku3p-4SHkK@fXrpi{T=@#bwY76TsD4$tAHAhXAStdb$odc z02~lZyb!fG_7qrU_F5 zoOG|pEwdyDhLXDwlU>T|;LF@ACJk(qZ*2h6GB@33mKk};HO^CQM(N7@Ml5|8IeHzt zdG4f$q}SNYA4P=?jV!mJ%3hRKwi&!wFptWZRq4bpV9^b7&L>nW%~Y|junw!jHj%85 z3Ck6%`Y=Abvrujnm{`OtE0uQkeX@3JPzj#iO#eNoAX6cDhM+cc2mLk8;^bG62mtjQ zj|kxI2W|4n{VqMqB?@YnA0y}@Mju)&j3UQ4tSdH=Eu?>i7A50b%i$pc{YJki7ubq7 zVTDqdkGjeAuZdF)KBwR6LZob}7`2935iKIU2-I;88&?t16c-~TNWIcQ8C_cE_F1tv z*>4<_kimwX^CQtFrlk)i!3-+2zD|=!D43Qqk-LtpPnX#QQt%eullxHat97k=00qR|b2|M}`q??yf+h~};_PJ2bLeEeteO3rh+H{9otNQDki^lu)(`a~_x(8NWLE*rb%T=Z~s?JC|G zXNnO~2SzW)H}p6Zn%WqAyadG=?$BXuS(x-2(T!E&sBcIz6`w=MdtxR<7M`s6-#!s+ znhpkcNMw{c#!F%#O!K*?(Hl(;Tgl9~WYBB(P@9KHb8ZkLN>|}+pQ)K#>ANpV1IM{Q z8qL^PiNEOrY*%!7Hj!CwRT2CN4r(ipJA%kCc&s;wOfrweu)H!YlFM z247pwv!nFWbTKq&zm4UVH^d?H2M276ny~@v5jR2>@ihAmcdZI-ah(&)7uLQM5COqg?hjX2<75QU4o5Q7 zZ5gG;6RMhxLa5NFTXgegSXb0a%aPdmLL4=`ox2smE)lDn^!;^PNftzTf~n{NH7uh_ zc9sKmx@q1InUh_BgI3C!f>`HnO~X`9#XTI^Yzaj1928gz8ClI!WIB&2!&;M18pf0T zsZ81LY3$-_O`@4$vrO`Cb&{apkvUwrA0Z49YfZYD)V4;c2&`JPJuwN_o~2vnyW_b! z%yUSS5K{a*t>;WJr&$A_&}bLTTXK23<;*EiNHHF-F<#hy8v2eegrqnE=^gt+|8R5o z_80IY4&-!2`uISX6lb0kCVmkQ{D}HMGUAkCe`I~t2~99(<#}{E;{+Y0!FU>leSP(M zuMoSOEfw3OC5kQ~Y2)EMlJceJlh}p?uw}!cq?h44=b2k@T1;6KviZGc_zbeTtTE$@EDwUcjxd#fpK=W*U@S#U|YKz{#qbb*|BpcaU!>6&Ir zhsA+ywgvk54%Nj>!!oH>MQ+L~36v1pV%^pOmvo7sT|N}$U!T6l^<3W2 z6}mT7Cl=IQo%Y~d%l=+;vdK)yW!C>Es-~b^E?IjUU4h6<86tun6rO#?!37B)M8>ph zJ@`~09W^@5=}sWg8`~ew=0>0*V^b9eG=rBIGbe3Ko$pj!0CBUTmF^Q}l7|kCeB(pX zi6UvbUJWfKcA&PDq?2HrMnJBTW#nm$(vPZE;%FRM#ge$S)i4!y$ShDwduz@EPp3H? z`+%=~-g6`Ibtrb=QsH3w-bKCX1_aGKo4Q7n-zYp->k~KE!(K@VZder&^^hIF6AhiG z;_ig2NDd_hpo!W1Un{GcB@e{O@P3zHnj;@SzYCxsImCHJS5I&^s-J6?cw92qeK8}W zk<_SvajS&d_tDP~>nhkJSoN>UZUHs?)bDY`{`;D^@wMW0@!H1I_BYphly0iqq^Jp; z_aD>eHbu@e6&PUQ4*q*ik0i*$Ru^_@`Mbyrscb&`8|c=RWZ>Ybs16Q?Cj1r6RQA5! zOeuxfzWm(fX!geO(anpBCOV|a&mu|$4cZ<*{pb1F{`-cm1)yB6AGm7b=GV@r*DataJ^I!>^lCvS_@AftZiwtpszHmq{UVl zKL9164tmF5g>uOZ({Jg~fH~QyHd#h#E;WzSYO~zt)_ZMhefdm5*H1K-#=_kw#o%ch zgX|C$K4l4IY8=PV6Q{T8dd`*6MG-TlsTEaA&W{EuwaoN+-BDdSL2>|lwiZ++4eR8h zNS1yJdbhAWjW4k`i1KL)l#G*Y=a0ouTbg8R1aUU`8X7p*AnO+uaNF9mwa+ooA)hlj zR26XBpQ-{6E9;PQAvq2<%!M1;@Q%r@xZ16YRyL&v}9F`Nnx#RLUc<78w$S zZElh==Rnr2u<*qKY|aUR9(A|{cURqP81O-1a@X)khheokEhC}BS-g~|zRbn-igmID z$Ww!O0-j!t(lx>-JH+0KW3*Bgafpm>%n=`(ZLa^TWd*-je!Xi7H*bZ8pz`HPFYeC? zk>`W)4Cj6*A3A8g$MEhp*<@qO&&>3<4YI%0YAMmQvD3 z${78Fa2mqiI>P7|gE)xs$cg3~^?UBb4y6B4Z#0Fzy zN8Gf!c+$uPS`VRB=wRV1f)>+PEHBYco<1?ceXET}Q-tKI=E`21<15xTe@%Bhk$v09 zVpoL_wNuw)@^O+C@VCeuWM}(%C(%lTJ}7n)JVV!^0H!3@)ydq#vEt;_*+xos$9i?{ zCw5^ZcNS&GzaeBmPg6IKrbT`OSuKg$wai+5K}$mTO-Z$s3Y+vb3G}x%WqlnQS1;|Z zlZ$L{onq1Ag#5JrM)%6~ToQ}NmM2A(7X5gy$nVI=tQFOm;7|Oeij{xb_KU{d@%)2z zsVqzTl@XPf(a95;P;oBm9Hlpo`9)D9>G>!Bj=ZmX{ces=aC~E^$rTO5hO$#X65jEA zMj1(p+HXdOh7FAV;(_)_RR#P>&NW?&4C7K1Y$C$i**g;KOdu|JI_Ep zV-N$wuDRkn6=k|tCDXU%d=YvT!M1nU?JY;Pl`dxQX5+660TX7~q@ukEKc!Iqy2y)KuG^Q-Y%$;SR&Mv{%=CjphG1_^dkUM=qI*3Ih^Bk621n`6;q(D;nB_y|~ zW*1ps&h|wcET!#~+Ptsiex~YVhDiIREiw1=uwlNpPyqDZ`qqv9GtKwvxnFE}ME93fD9(Iq zz=f&4ZpD~+qROW6Y2AjPj9pH*r_pS_f@tLl88dbkO9LG0+|4*Xq(Eo7fr5MVg{n<+p>H{LGr}UzToqfk_x6(2YB~-^7>%X z+331Ob|NyMST64u|1dK*#J>qEW@dKNj-u}3MG)ZQi~#GzJ_S4n5lb7vu&>;I-M49a z0Uc#GD-KjO`tQ5ftuSz<+`rT)cLio$OJDLtC`t)bE+Nu@Rok2;`#zv1=n z7_CZr&EhVy{jq(eJPS)XA>!7t<&ormWI~w0@Y#VKjK)`KAO~3|%+{ z$HKIF?86~jH*1p=`j#}8ON0{mvoiN7fS^N+TzF~;9G0_lQ?(OT8!b1F8a~epAH#uA zSN+goE<-psRqPXdG7}w=ddH=QAL|g}x5%l-`Kh69D4{M?jv!l))<@jxLL$Eg2vt@E zc6w`$?_z%awCE~ca)9nMvj($VH%2!?w3c(5Y4&ZC2q#yQ=r{H2O839eoBJ{rfMTs8 zn2aL6e6?;LY#&(BvX_gC6uFK`0yt zJbUATdyz5d3lRyV!rwbj0hVg#KHdK0^A7_3KA%gKi#F#-^K%1XQbeF49arI2LA|Bj z?=;VxKbZo(iQmHB5eAg=8IPRqyskQNR!&KEPrGv&kMr(8`4oe?vd?sIZJK+JY04kc zXWk)4N|~*|0$4sUV3U6W6g+Z3;nN<~n4H17QT*%MCLt_huVl@QkV`A`jyq<|q=&F_ zPEOotTu9?zGKaPJ#9P&ljgW!|Vxhe+l85%G5zpD5kAtn*ZC})qEy!v`_R}EcOn)&# z-+B52@Zle@$!^-N@<_=LKF}fqQkwf1rE(OQP&8!En}jqr-l0A0K>77K8{zT%wVpT~ zMgDx}RUG$jgaeqv*E~<#RT?Q)(RGi8bUm(1X?2OAG2!LbBR+u1r7$}s=lKqu&VjXP zUw3L9DH({yj)M%OqP%GC+$}o0iG|*hN-Ecv3bxS|Mxpmz*%x`w7~=o9BKfEVzr~K- zo&Fh`wZ{#1Jd5QFM4&!PabL!tf%TfJ4wi;45AqWe$x}8*c2cgqua`(6@ErE&P{K5M zQfwGQ4Qg&M3r4^^$B?_AdLzqtxn5nb#kItDY?BTW z#hShspeIDJ1FDmfq@dz1TT`OV;SS0ImUp`P6GzOqB3dPfzf?+w^40!Wn*4s!E;iHW zNzpDG+Vmtnh%CyfAX>X z{Y=vt;yb z;TBRZpw##Kh$l<8qq5|3LkrwX%MoxqWwclBS6|7LDM(I31>$_w=;{=HcyWlak3xM1 z_oaOa)a;AtV{*xSj6v|x%a42{h@X-cr%#HO5hWbuKRGTZS)o=^Id^>H5}0p_(BEXX zx3VnRUj6&1JjDI);c=#EYcsg;D5TFlhe)=nAycR1N)YSHQvO+P5hKe9T0ggZT{oF@ z#i3V4TpQlO1A8*TWn|e}UWZ(OU;Isd^ zb<#Vj`~W_-S_=lDR#223!xq8sRjAAVSY2MhRyUyHa-{ql=zyMz?~i_c&dS>eb>s>#q#$UI+!&6MftpQvxHA@f|k2(G9z zAQCx-lJ-AT;PnX%dY5}N$m6tFt5h6;Mf78TmFUN9#4*qBNg4it3-s22P+|Rw zG@X%R0sm*X07ZZEOJRbDkcjr}tvaVWlrwJ#7KYEw&X`2lDa@qb!0*SHa%+-FU!83q zY{R15$vfL56^Nj42#vGQlQ%coT4bLr2s5Y0zBFp8u&F(+*%k4xE1{s75Q?P(SL7kf zhG?3rfM9V*b?>dOpwr%uGH7Xfk1HZ!*k`@CNM77g_mGN=ucMG&QX19B!%y77w?g#b z%k3x6q_w_%ghL;9Zk_J#V{hxK%6j`?-`UN?^e%(L6R#t#97kZaOr1{&<8VGVs1O>} z6~!myW`ja01v%qy%WI=8WI!cf#YA8KNRoU>`_muCqpt_;F@rkVeDY}F7puI_wBPH9 zgRGre(X_z4PUO5!VDSyg)bea1x_a7M z4AJ?dd9rf{*P`AY+w?g_TyJlB5Nks~1$@PxdtpUGGG##7j<$g&BhKq0mXTva{;h5E ztcN!O17bquKEDC#;Yw2yE>*=|WdZT9+ycgUR^f?~+TY-E552AZlzYn{-2CLRV9mn8 z+zNoWLae^P{co`F?)r;f!C=nnl*1+DI)mZY!frp~f%6tX2g=?zQL^d-j^t1~+xYgK zv;np&js@X=_e7F&&ZUX|N6Q2P0L=fWoBuh*L7$3~$-A)sdy6EQ@Pd-)|7lDA@%ra2 z4jL@^w92&KC>H(=v2j!tVE_3w0KogtrNjgPBsTvW F{TFmrHLU;u delta 8469 zcmY*q~ZGqoW{=01$bgB@1Nex`%9%S2I04)5Jw9+UyLS&r+9O2bq{gY;dCa zHW3WY0%Dem?S7n5JZO%*yiT9fb!XGk9^Q`o-EO{a^j%&)ZsxsSN@2k2eFx1*psqn0e*crIbAO}Rd~_BifMu*q7SUn{>WD$=7n_$uiQ0wGc$?u1hM%gf??nL?m22h!8{ zYmFMLvx6fjz*nwF^tAqx1uv0yEW9-tcIV5Q{HNh`9PMsuqD8VE%oAs5FsWa0mLV$L zPAF5e^$tJ8_Kwp!$N1M<#Z154n!X6hFpk8)eMLu; zaXS71&`24 zV`x~}yAxBw##Oj@qo_@DcBqc+2TB&=bJyZWTeR55zG<{Z@T^hSbMdm~Ikkr?4{7WT zcjPyu>0sDjl7&?TL@ z)cW?lW@Pfwu#nm7E1%6*nBIzQrKhHl`t54$-m>j8f%0vVr?N0PTz`}VrYAl+8h^O~ zuWQj@aZSZmGPtcVjGq-EQ1V`)%x{HZ6pT-tZttJOQm?q-#KzchbH>>5-jEX*K~KDa z#oO&Qf4$@}ZGQ7gxn<;D$ziphThbi6zL^YC;J#t0GCbjY)NHdqF=M4e(@|DUPY_=F zLcX1HAJ+O-3VkU#LW`4;=6szwwo%^R4#UK}HdAXK` z{m!VZj5q9tVYL=^TqPH*6?>*yr>VxyYF4tY{~?qJ*eIoIU0}-TLepzga4g}}D7#Qu zn;6I;l!`xaL^8r*Tz*h`^(xJCnuVR_O@Gl*Q}y$lp%!kxD`%zN19WTIf`VX*M=cDp z*s4<9wP|ev;PARRV`g$R*QV@rr%Ku~z(2-s>nt{JI$357vnFAz9!ZsiiH#4wOt+!1 zM;h;EN__zBn)*-A^l!`b?b*VI-?)Sj6&Ov3!j9k$5+#w)M>`AExCm0!#XL+E{Bp)s;Hochs+-@@)7_XDMPby#p<9mLu+S{8e2Jn`1`1nrffBfy4u)p7FFQWzgYt zXC}GypRdkTUS+mP!jSH$K71PYI%QI-{m;DvlRb*|4GMPmvURv0uD2bvS%FOSe_$4zc--*>gfRMKN|D ztP^WFfGEkcm?sqXoyRmuCgb?bSG17#QSv4~XsbPH>BE%;bZQ_HQb?q%CjykL7CWDf z!rtrPk~46_!{V`V<;AjAza;w-F%t1^+b|r_um$#1cHZ1|WpVUS&1aq?Mnss|HVDRY z*sVYNB+4#TJAh4#rGbr}oSnxjD6_LIkanNvZ9_#bm?$HKKdDdg4%vxbm-t@ZcKr#x z6<$$VPNBpWM2S+bf5IBjY3-IY2-BwRfW_DonEaXa=h{xOH%oa~gPW6LTF26Y*M)$N z=9i`Y8};Qgr#zvU)_^yU5yB;9@yJjrMvc4T%}a|jCze826soW-d`V~eo%RTh)&#XR zRe<8$42S2oz|NVcB%rG(FP2U&X>3 z4M^}|K{v64>~rob;$GO55t;Nb&T+A3u(>P6;wtp6DBGWbX|3EZBDAM2DCo&4w|WGpi;~qUY?Ofg$pX&`zR~)lr)8}z^U3U38Nrtnmf~e7$i=l>+*R%hQgDrj%P7F zIjyBCj2$Td=Fp=0Dk{=8d6cIcW6zhK!$>k*uC^f}c6-NR$ zd<)oa+_fQDyY-}9DsPBvh@6EvLZ}c)C&O-+wY|}RYHbc2cdGuNcJ7#yE}9=!Vt-Q~ z4tOePK!0IJ0cW*jOkCO? zS-T!bE{5LD&u!I4tqy;dI*)#e^i)uIDxU?8wK1COP3Qk{$vM3Sm8(F2VwM?1A+dle z6`M6bbZye|kew%w9l`GS74yhLluJU5R=#!&zGwB7lmTt}&eCt0g(-a;Mom-{lL6u~ zFgjyUs1$K*0R51qQTW_165~#WRrMxiUx{0F#+tvgtcjV$U|Z}G*JWo6)8f!+(4o>O zuaAxLfUl;GHI}A}Kc>A8h^v6C-9bb}lw@rtA*4Q8)z>0oa6V1>N4GFyi&v69#x&CwK*^!w&$`dv zQKRMKcN$^=$?4to7X4I`?PKGi(=R}d8cv{74o|9FwS zvvTg0D~O%bQpbp@{r49;r~5`mcE^P<9;Zi$?4LP-^P^kuY#uBz$F!u1d{Ens6~$Od zf)dV+8-4!eURXZZ;lM4rJw{R3f1Ng<9nn2_RQUZDrOw5+DtdAIv*v@3ZBU9G)sC&y!vM28daSH7(SKNGcV z&5x#e#W2eY?XN@jyOQiSj$BlXkTG3uAL{D|PwoMp$}f3h5o7b4Y+X#P)0jlolgLn9xC%zr3jr$gl$8?II`DO6gIGm;O`R`bN{;DlXaY4b`>x6xH=Kl@ z!>mh~TLOo)#dTb~F;O z8hpjW9Ga?AX&&J+T#RM6u*9x{&%I8m?vk4eDWz^l2N_k(TbeBpIwcV4FhL(S$4l5p z@{n7|sax){t!3t4O!`o(dYCNh90+hl|p%V_q&cwBzT*?Nu*D0wZ)fPXv z@*;`TO7T0WKtFh8~mQx;49VG_`l`g|&VK}LysK%eU4})Cvvg3YN)%;zI?;_Nr z)5zuU1^r3h;Y+mJov*->dOOj>RV^u2*|RraaQWsY5N?Uu)fKJOCSL2^G=RB%(4K{* zx!^cB@I|kJR`b+5IK}(6)m=O{49P5E^)!XvD5zVuzJH{01^#$@Cn514w41BB;FAoS2SYl3SRrOBDLfl5MvgA3 zU6{T?BW}l~8vU;q@p9IOM(=;WdioeQmt?X|=L9kyM&ZsNc*-Knv8@U*O96T@4ZiJ$ zeFL2}pw_~Tm3d4#q!zZS0km@vYgym33C0h(6D)6|Y)*UXI^T`(QPQh$WF?&h(3QYh zqGw@?BTk@VA_VxK@z?a@UrMhY zUD16oqx4$$6J_k0HnXgARm}N#(^yA1MLdbwmEqHnX*JdHN>$5k2E|^_bL< zGf5Z+D!9dXR>^(5F&5gIew1%kJtFUwI5P1~I$4LL_6)3RPzw|@2vV;Q^MeQUKzc=KxSTTX`}u%z?h~;qI#%dE@OZwehZyDBsWTc&tOC1c%HS#AyTJ= zQixj=BNVaRS*G!;B$}cJljeiVQabC25O+xr4A+32HVb;@+%r}$^u4-R?^3yij)0xb z86i@aoVxa%?bfOE;Bgvm&8_8K(M-ZEj*u9ms_Hk#2eL`PSnD#At!0l{f!v`&Kg}M$n(&R)?AigC5Z?T7Jv^lrDL!yYS{4 zq_H}oezX-Svu>dp)wE@khE@aR5vY=;{C-8Hws++5LDpArYd)U47jc-;f~07_TPa^1 zO`0+uIq)@?^!%JXCDid+nt|c@NG1+ce@ijUX&@rV9UiT|m+t-nqVB7?&UX*|{yDBFw9x52&dTh@;CL)Q?6s1gL=CUQTX7#TJPs9cpw<4>GFMUKo|f{! z&(%2hP6ghr%UFVO-N^v9l|tKy>&e%8us}wT0N*l(tezoctVtLmNdGPOF6oaAGJI5R zZ*|k@z3H!~Mm9fXw{bbP6?lV-j#Rfgnjf++O7*|5vz2#XK;kk ztJbi%r0{U5@QwHYfwdjtqJ6?;X{Ul3?W0O0bZ$k*y z4jWsNedRoCb7_|>nazmq{T3Y_{<5IO&zQ?9&uS@iL+|K|eXy^F>-60HDoVvovHelY zy6p(}H^7b+$gu@7xLn_^oQryjVu#pRE5&-w5ZLCK&)WJ5jJF{B>y;-=)C;xbF#wig zNxN^>TwzZbV+{+M?}UfbFSe#(x$c)|d_9fRLLHH?Xbn!PoM{(+S5IEFRe4$aHg~hP zJYt`h&?WuNs4mVAmk$yeM;8?R6;YBMp8VilyM!RXWj<95=yp=4@y?`Ua8 znR^R?u&g%`$Wa~usp|pO$aMF-en!DrolPjD_g#{8X1f=#_7hH8i|WF+wMqmxUm*!G z*4p980g{sgR9?{}B+a0yiOdR()tWE8u)vMPxAdK)?$M+O_S+;nB34@o<%lGJbXbP` z5)<({mNpHp&45UvN`b&K5SD#W){}6Y_d4v~amZPGg|3GdlWDB;;?a=Z{dd zELTfXnjCqq{Dgbh9c%LjK!Epi1TGI{A7AP|eg2@TFQiUd4Bo!JsCqsS-8ml`j{gM& zEd7yU`djX!EX2I{WZq=qasFzdDWD`Z?ULFVIP!(KQP=fJh5QC9D|$JGV95jv)!sYWY?irpvh06rw&O?iIvMMj=X zr%`aa(|{Ad=Vr9%Q(61{PB-V_(3A%p&V#0zGKI1O(^;tkS{>Y<`Ql@_-b7IOT&@?l zavh?#FW?5otMIjq+Bp?Lq)w7S(0Vp0o!J*~O1>av;)Cdok@h&JKaoHDV6IVtJ?N#XY=lknPN+SN8@3Gb+D-X*y5pQ)wnIpQlRR!Rd)@0LdA85}1 zu7W6tJ*p26ovz+`YCPePT>-+p@T_QsW$uE`McLlXb;k}!wwWuh$YC4qHRd=RS!s>2 zo39VCB-#Ew?PAYOx`x!@0qa5lZKrE?PJEwVfkww#aB_$CLKlkzHSIi4p3#IeyA@u@ z`x^!`0HJxe>#V7+Grku^in>Ppz|TD*`Ca4X%R3Yo|J=!)l$vYks|KhG{1CEfyuzK( zLjCz{5l}9>$J=FC?59^85awK0$;^9t9UxwOU8kP7ReVCc*rPOr(9uMY*aCZi2=JBu z(D0svsJRB&a9nY;6|4kMr1Er5kUVOh1TuBwa3B2C<+rS|xJo&Lnx3K-*P83eXQCJ= z(htQSA3hgOMcs`#NdYB17#zP_1N_P0peHrNo1%NsYn=;PgLXTic6b#{Y0Z~x9Ffav z^3eO+diquPfo1AXW*>G(JcGn{yN?segqKL$Wc9po(Kex z#tw_};zd++we+MPhOOgaXSmguul67JOvBysmg?wRf=OUeh(XyRcyY@8RTV@xck_c~ zLFMWAWb4^7xwR)3iO1PIs1<}L3CMJ1L-}s=>_y!`!FvYf^pJO|&nII{!Dz+b?=bUd zPJUUn))z)-TcpqKF(1tr-x1;lS?SB@mT#O7skl0sER{a|d?&>EKKaw* zQ>D^m*pNgV`54BKv?knU-T5bcvBKnI@KZo^UYjKp{2hpCo?_6v(Sg77@nQa{tSKbn zUgMtF>A3hndGocRY+Snm#)Q4%`|Qq3YTOU^uG}BGlz!B=zb?vB16sN&6J`L(k1r+$ z5G6E9tJ~Iwd!d!NH7Q%Z@BR@0e{p6#XF2))?FLAVG`npIjih*I+0!f6;+DM zLOP-qDsm9=ZrI!lfSDn%XuF17$j~gZE@I}S(Ctw&Te75P5?Fj%FLT;p-tm33FaUQc z5cR;$SwV|N0xmjox3V~XL3sV?YN}U0kkfmygW@a5JOCGgce6JyzGmgN$?NM%4;wEhUMg0uTTB~L==1Fvc(6)KMLmU z(12l^#g&9OpF7+Ll30F6(q=~>NIY=-YUJJ}@&;!RYnq*xA9h!iMi`t;B2SUqbyNGn zye@*0#Uu`OQy%utS%IA%$M1f4B|bOH={!3K1=Tc7Ra|%qZgZ{mjAGKXb)}jUu1mQ_ zRW7<;tkHv(m7E0m>**8D;+2ddTL>EcH_1YqCaTTu_#6Djm z*64!w#=Hz<>Fi1n+P}l#-)0e0P4o+D8^^Mk& zhHeJoh2paKlO+8r?$tx`qEcm|PSt6|1$1q?r@VvvMd1!*zAy3<`X9j?ZI|;jE-F(H zIn1+sm(zAnoJArtytHC|0&F0`i*dy-PiwbD-+j`ezvd4C`%F1y^7t}2aww}ZlPk)t z=Y`tm#jNM$d`pG%F42Xmg_pZnEnvC%avz=xNs!=6b%%JSuc(WObezkCeZ#C|3PpXj zkR8hDPyTIUv~?<%*)6=8`WfPPyB9goi+p$1N2N<%!tS2wopT2x`2IZi?|_P{GA|I5 z?7DP*?Gi#2SJZ!x#W9Npm)T;=;~Swyeb*!P{I^s@o5m_3GS2Lg?VUeBdOeae7&s5$ zSL_VuTJih_fq7g8O8b0g+GbmE+xG}^Wx`g~{mWTyr@=h zKlAymoHeZa`DgR?Pj8Yc+I|MrSB>X*ts#wNFOJxs!3aGE)xeTHlF`fC5^g(DTacl$ zx!ezQJdwIyc$8RyNS~Wh{0pp>8NcW)*J=7AQYdT?(QhJuq4u`QniZ!%6l{KWp-0Xp z4ZC6(E(_&c$$U_cmGFslsyX6(62~m*z8Yx2p+F5xmD%6A7eOnx`1lJA-Mrc#&xZWJ zzXV{{OIgzYaq|D4k^j%z|8JB8GnRu3hw#8Z@({sSmsF(x>!w0Meg5y(zg!Z0S^0k# z5x^g1@L;toCK$NB|Fn( cache.set(null) } - @OptIn(ExperimentalTime::class) - sealed class CacheContents( - time: TimeMark? = null, - ) { + sealed class CacheContents + @ExperimentalTime + constructor(time: TimeMark?) { + + @OptIn(ExperimentalTime::class) + constructor() : this(null) + + @ExperimentalTime protected val time: TimeMark = time ?: TimeSource.Monotonic.markNow() + @OptIn(ExperimentalTime::class) open fun isExpired(age: Duration): Boolean = (time + age).hasPassedNow() abstract fun getOrThrow(): T - @Suppress("UNCHECKED_CAST") abstract suspend fun map(transform: suspend (T) -> R): CacheContents } - @OptIn(ExperimentalTime::class) - class CacheError internal constructor( + class CacheError + internal constructor( val exception: Throwable, ) : CacheContents() { override fun isExpired(age: Duration): Boolean = exception is CancellationException || super.isExpired(age) @@ -233,12 +237,19 @@ open class CachedValue( } @OptIn(ExperimentalTime::class) - class CacheValue internal constructor( + class CacheValue + @ExperimentalTime + internal constructor( val value: T, - time: TimeMark? = null, + time: TimeMark?, ) : CacheContents(time) { + + @OptIn(ExperimentalTime::class) + constructor(value: T) : this(value, null) + override fun getOrThrow(): T = value + @OptIn(ExperimentalTime::class) override suspend fun map(transform: suspend (T) -> R): CacheContents = try { CacheValue(transform(value), time = time) } catch (ex: Throwable) { diff --git a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt index 6a79fdb2..e9e73c8d 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/KafkaTopicSender.kt @@ -21,7 +21,18 @@ interface KafkaTopicSender { * @throws IOException if the client could not send a message */ @Throws(IOException::class, SchemaValidationException::class) - suspend fun send(key: K, value: V) = send(AvroRecordData(topic, key, listOf(value))) + suspend fun send(key: K, value: V) = send(key, listOf(value)) + + /** + * Send a message to Kafka eventually. + * + * @param key key of a kafka record to send + * @param values values for kafka records to send + * @throws AuthenticationException if the client failed to authenticate itself + * @throws IOException if the client could not send a message + */ + @Throws(IOException::class, SchemaValidationException::class) + suspend fun send(key: K, values: List) = send(AvroRecordData(topic, key, values)) /** * Send a message to Kafka eventually. Contained offsets must be strictly monotonically diff --git a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt index 92a49381..13fd50b4 100644 --- a/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt +++ b/radar-commons/src/main/java/org/radarbase/producer/rest/RestKafkaSender.kt @@ -227,6 +227,13 @@ class RestKafkaSender(config: Config) : KafkaSender { ?: HttpClient(CIO, config) } + fun schemaRetriever(schemaBaseUrl: String, builder: SchemaRetriever.Config.() -> Unit = {}) { + schemaRetriever = SchemaRetriever.schemaRetriever(schemaBaseUrl) { + httpClient = this@Config.httpClient + builder() + } + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other == null || javaClass != other.javaClass) return false