From 9ecc0deccbb94508aa1167b34a3fd65c89fa4c68 Mon Sep 17 00:00:00 2001 From: Likhitha Priya Date: Tue, 16 Jan 2024 14:32:14 +0000 Subject: [PATCH 1/4] Mirroring nested debian repos --- lib/rmt/mirror/base.rb | 6 +- lib/rmt/mirror/debian.rb | 16 +- lib/rmt/mirror/file_reference.rb | 4 + spec/fixtures/files/debian/Nested_Packages.gz | Bin 0 -> 6419 bytes spec/fixtures/files/debian/Nested_Release | 247 ++++++++++++++++++ spec/lib/rmt/file_validator_spec.rb | 24 -- spec/lib/rmt/mirror/base_spec.rb | 3 +- spec/lib/rmt/mirror/debian_spec.rb | 34 ++- spec/lib/rmt/mirror/repomd_spec.rb | 10 - spec/support/matchers.rb | 43 +++ 10 files changed, 346 insertions(+), 41 deletions(-) create mode 100644 spec/fixtures/files/debian/Nested_Packages.gz create mode 100644 spec/fixtures/files/debian/Nested_Release create mode 100644 spec/support/matchers.rb diff --git a/lib/rmt/mirror/base.rb b/lib/rmt/mirror/base.rb index 27412e4fb..d4c74893d 100644 --- a/lib/rmt/mirror/base.rb +++ b/lib/rmt/mirror/base.rb @@ -111,8 +111,8 @@ def cleanup_temp_dirs @temp_dirs = {} end - def enqueue(ref) - @enqueued << ref + def enqueue(refs) + @enqueued.concat(Array(refs)) end def download_enqueued(continue_on_error: false) @@ -155,7 +155,7 @@ def replace_directory(source:, destination:, with_backup: true, &block) def copy_directory_content(source:, destination:) replace_directory(source: source, destination: destination, with_backup: false) do - FileUtils.mv(Dir.glob(source), destination) + FileUtils.mv(Dir.glob(source), destination, force: true) end end end diff --git a/lib/rmt/mirror/debian.rb b/lib/rmt/mirror/debian.rb index 351958169..a98df88c5 100644 --- a/lib/rmt/mirror/debian.rb +++ b/lib/rmt/mirror/debian.rb @@ -28,9 +28,17 @@ def mirror_metadata # We need to make sure downloading the InRelease file which is not referenced # anywhere metadata_refs << inrelease - metadata_refs.each { |ref| enqueue(ref) } + # The nested debian structure only contains the zipped version of packages sometimes + # However, the release file still contains a reference to the unzipped versions + # So, we don't error if they don't exist + packages, remaining = metadata_refs.partition { |ref| ref.relative_path.match(/(Packages|Sources|Translation)(-\w+)?$/) } + enqueue(packages) + download_enqueued(continue_on_error: true) + + enqueue(remaining) download_enqueued + metadata_refs end @@ -39,6 +47,12 @@ def mirror_packages(metadata_refs) packagelists.each do |packagelist| parse_package_list(packagelist).each do |ref| + # In a nested debian repository stucture, the metadata and packages are stored in different locations + # so we need to update the base_url if we encounter the nested structure + # We assume that if the base_url contains '/dists/', it's a nested debian structure + if ref.base_url.match?(%r{/dists/\w*/$}) + ref.base_url.sub!(%r{(/dists/\w*/$)}, '/') + end enqueue(ref) if need_to_download?(ref) end end diff --git a/lib/rmt/mirror/file_reference.rb b/lib/rmt/mirror/file_reference.rb index d958d9d4e..c48bb3119 100644 --- a/lib/rmt/mirror/file_reference.rb +++ b/lib/rmt/mirror/file_reference.rb @@ -49,4 +49,8 @@ def remote_path def cache_timestamp File.mtime(cache_path).utc if cache_path && File.exist?(cache_path) end + + def to_s + relative_path + end end diff --git a/spec/fixtures/files/debian/Nested_Packages.gz b/spec/fixtures/files/debian/Nested_Packages.gz new file mode 100644 index 0000000000000000000000000000000000000000..d3744b96b863a6b3891eef5a58966f6eb13c2221 GIT binary patch literal 6419 zcmV+u8SLgCiwFP!00002|LuKib6d%g<#+xHeC&=E+4abJDLjs`r^7oF#t=9j-c``ujLOqXc(ZjJ`{szo~c zFP$$|cd-}dqMW;L?&{t3^3ItHLl^cNTz;{*o4*?kd}GhP(_6i6iXlKKsLTrq_IhW^Zq9X5U`nLw#GnU3YHtu$OfgLh}&e-{>a zS1%_;S_{H8a`S4E#>h#ew4S5O#mm|2fBl_(IWf#NH;#tYr&cufQcFuljq@c%6Qppd zrjFBHh%YBiOXU(P<)Ru*&W*@%XwqskO}yBw>N2&UIjDElhweSM5SmK#jFDN$e0sZn zVzZmYVy275L~gEb=Ximm6P{g;NBrh$!q;umZ{}CLoP32duEy^(JjK8{AZ3(FDlzpm6DRS@`S6Tlal_~oiJ;1HGm)6v^-k3@*Y_q5~vTVs|r|6uiE@T4I zS$d*IIK4eU<+RRu{%$h8x}E8jUN5KDmm6G6^jdD3&R5U`qF1lKTA|_QcAPrt7}3Qe z!1UF}W~jovC{QGmBD(KOuq@=cpM8HfTTagh9$roscqES@)jkuZk9fnJektB?*Xv_F zB4wiY*h@CFTAZWg9lS%!%!OTv-8pL=)eZLGCACHYE)7?_Z8M4C4zvN0p+%MMP4ewM z_DTy$vvDN=wwbz$SL=kRst;@cv&^w7!41RFy6mKiRk-42eN1Fd#NOf!&$Vr?n?4G? zVv_TradRxq!62wx&Xv@%N6fIFfZp}x^!n}j)lEC!fNqPy)f-Pj@Xd00+djZx&i5er z-3n`y{}`=Jc5f^uzq$R(<@x6ByTkv?v+2$9&G~G)SS~I$K)u%GCxPnd!kpU!k-EO^ z2s(&JD|dRp&LkbjSD zOX~v|MRU2afKo2fg?$F-?>&rTvF|DEuumQ=B#89Vfka;sbyaG?Nj-DsvcWz=LQqTI zgY~NE;3}q22{k6$orFL=waCWou-|j8V7AU=a1RQsAQ`O&Z^@A10)z22yIxApflN1^ z1m+QPOc~ZF_p$%#ci&G&w};mk9>*paj=PyYh4>y2e_HKEVi@k7T(xZQ$7 zTubSoCf&LkyLrH?Uyc64qhBvqM0tL5wSmWKI2Mm^=90ORZ{7~gpjm=J{HU!%2ZR{i zlh|(j*+VtZ$yfkvJ!lL6NsO7vC#21-e6NU>(p;rfSNP z4JTu0gL5!lUu;WI0LfKp31|CQ4fwc&dyl3Oc#qomI7c+FAWJe*9C5wG&*=47~+@WqWE3hgoA$Ksmup8s~5yf>|>v4(ogkdCoHfd(*H zqO|2Fwsyb*UJyPV^hChigDoJ_0n{5jJAqpJSnH!#T%W?CX3}eu$0T2n+63sS>AjM z-qN__h&ZLNKI5WC=X^=%KycbbpM-bkYk4=lZu<6gpYDFPAb)*#d8ugkX9DrvpdU+A zfZ|OlgNjbJ|V`-Y01eot)A_tt1)bM+t z*V@6TZEYUZ1F-;r3R-a%lq%)m3TXfzFUheRZ;43AntHd;z@6QFp#NcXda~*@;JnAd z!OcGg?dZag#UkqAvE4{T7L$T$tWj%!nyXyu&Ghzk22Q_rrm`Bxb>_uh#0%@53-?t6 z$0D6V*|~IYZLh_tiv*ScF*bCXM$u#{FtBFE4U^w!wt^O#c?oq zM@vt;H>Csq%g!O1dt2=A!2#45Zda9Cip_3tUaeZ6qr`@zz8B*h6;-kf%g7egwsRGA z?{OX^QiJ+z;3BOt%qZHNM>MDW|>u9SU?}Hqp`u z7BC|)e`}>0rrunM;^UJ*e}j$#IOl^No%iW%sq_EIeC20H${!j3V}YKt-BA}B8Nfz_ zpbn&QrpDxn$#y%K32$fADVtopl1~JI2wcMV0z@U2UbrG4K+C~ zK3Vtu0KXr6-~@7$k_`+s5^Zac;2>f^zNM$->FkbAMm*(TWD@uu1iQw0@{J<6opI#d!U+$ zYp+pCC2QlZZYx`0J2IR*^_pYPfPG2aHFHYJ6n$=R_DOB)DdbW!#OFwAdSU15gS`71 z0^jWwWBvPdxk2XHAYPp_J|99)6X=kbOL!8;w>qDl3}oxfF9^wh(%Tt} zzK62DUCwzi`RX^nL6ZgU_uPJcI^4&=uK(xTDV(^+i61F3z&Ydn)#3Gt&oz(H#IgDN z81t^SEIS~-b1^JXwnL+)IOam37##Pu_3yBTyGcbTF%L4Ma+m6D%PjrnH(`8M9qca9`UIOnTN{8yK5?SX=fA-95m;YXI44 z-4>r5K5$3~QNF??a6?;4e$p@&cXjc(`2Vju*Yiv$?~VJ>)nerLD`sK=%Weh`?lI?4 zgd6e7C4qbHTaTaHfExU$Cy3W99%kW{LDwT-gZF?;lXX{7kZOs}c*op;`G5e?T`!{W z!CIhoVM9P44n^%Tv1=9<*aKLx2lynnC3Tct0ly#I7*LOMj(Lh z@CJQNp(c6~;CHvP8*aDro6Av__NBVuC&c*up+CBy01Ra3xVz)($TU}ZIL}ZkfXBn9 zrX;j2NQb_xP79!4E99$!-M0d7?UMvZUv;I4gkJWtm(Z%Q^f=a4a0mbw!0m^i1fnA~ zR`43YoV8?E45&2B2nHe4a!z_5=Ks0s_+;Dj$>QB&saH>CFu|kiiM83)eB`ax7` z1gGaLRbfggI{P$LdApb`KU=EOe<>2XS0uJRh7qmNsyH!VHqDI^lxvUa_#nG#(+og8MN#dvVIV%$6z0c;hm)k7z`QeI#z4IO`9%bQpy|JNlu3Qq~Hs8FotZxSW@TI zB_~pH={0l7CIM2Q>>VwRk{ko2o1qPJbPte>Cx3|K44grZDnU4On{8F+BE&>1Ox%0> zXl22*yy5H9owxnf);xY}5#FQNU5)mU;B|p$Z!)#A^=ukcpNndxj5&5?8td{R)BDL( z2WQzjbZ7{Uv`aE$>c`g+_XA?SMBf=4Wyebjw~=9tm7Vl#fFNracLooRP^nVM8{w96 z^4aFb1Z?wIG||PAq=$82i{*<+pXu9a1%Pkw>bKC@lZDQIgwT9SVMc|unCx3tsn)_q z1&$dXqm7c{^f5fomsh8kf%)|7g>!xeoX21vJH0a|`FH7YcvKU+A1mHMA@5SjAO~>E zAPe2rZCSb)Dh*l?G$4oqqd2#WUtRZAym1aT!Dgo-iwBTE7;1++C1$bsS-{+N+4&CT%H6jhH50|E=A#ol zi^c3*KErwo^FMlZ=uzr$J+w#HCE7jD3!IksTn8|y~;1ysDpuhz>Z(C{(qRJ(* zYBhkS)ufRSo(sJAn3AcD!DPklHin2}b$$`;H5G_5)3rqj5N5SnGwNAVGrk6EQl{<$ z;e`$A8UENK=7;-O{TH1tr+ut~(=`#k<++Zc+yn3%IwxN(72Kug0D{jJ#?gC#JC5Ca z6u&F=c`|;T|B})>2-BBOAR3CyK>bUsag@56R4DfX3`mOGrOny(TAKy} zsc5VDYqgweqAW=S-Vu~xJ3%>!a)clE3H}>#zk(A-A4bAD zHH|Thy)UYSU7ovvkbpX8@anP84y?pw?_dmtD?$ZGiUR{`p~KHtZvsQBBB+4z2f{m# zRihtlC%z!4aDZftCQ(eKI5sgrha#X=4Tsptmx}cNPZjC^5L-FAq#Yq)ms;sz_<1Sl zX-(F_OqvDG6#>HM)V4`NNzAdw5SX<{;}pkaBKcGTjiRiv)lckgms;u2x*D7^WMEKS zp(RxwX%s^uV@(nlSm>xTt~lz0jOj_u*%cRqGmn>N7`;(Z$Y<4nnpX;V@<$$X%?BBnh{NK%`{cbip^3vJ#^3C$dYws1?9C-1I zd%gYbp_ccctIxk~)6eH$xb5U`KZ}!C{eP%Gn9c5OT2|tYToQv7j`^{aAR!W>CqsZo zflVeTjcp}_wPJF&!6rmYtto3$$&uYyfYlW=l19Dqj$Fnv7S42E9JIWE^M$Tx-$kV7$+UIe z>aqQy9Fd3rR<+4du_F%=vnJsrvE_%JVG}k^S4tuOO#Q`f5pI{u>0jr3yEUeEBN*Eu z$DvU7Dxeqjyz*jgXC60*Fh4XZH`jf$4Ne$9-yAx37v|I8-Hh{57!KL&?VA8eg2&q- zikk~5IPV2GcHWF~%i#KU+ADO3D676EDmhUN3D8S2Y9Rmy%GOmA+m8N0x+4Z=+fCvO z+Esb%771`BU>2>#h&Qv9tl70vU>rjv5dsmc#D;V%h9~Ef4Ru*Esng5rM_@kJ&rswUe$E8(eog0IbhCL}{`JH4=)(Y>!Qx)c zZ+@J%af8+9yjcG@hf%$FauLC1$Ghhd)5jJ$p4POPnrSRa_byIpRS{Yx5at2QOYGFQKPg-^O)xG>w? z!fg@w<>u{J<6UiuK~+o*+sQZ<3vr!996_&;E5#UREP=@6lye_D5Raw6yS;*Iwh{WJ zplz4cs2wZCQUNRSu8PNPv!k)NWvsA88w}pzee>)?Z1zEs(0cejhy-83-Cj4Ix5!^u6v@sG1V$O^=tDYf}Du+W{OCnZ7oWN%%^pZgZEuD* zxe_X#p&Cg(LP|TuiLE8j<=mRhY-AKGB!ymYxdh--1i%fpXgPs$U|{aXvXWIn&LcbV zfo)iJHF))f0ecl&U{`7#pl!lIoMY`kPG=q7#9zu5H@_fCvE7vNv{v?Z|2wUr&DLtC zHL}{z>7*8Rdj$Q%3EMggcv>5eY#{J+VUe>tKCOYV;_0*&_7<9)*3kWG{!`o9E1dp? z6ZKV<)i0ao?`Pz{h=q@eg~8wtl?n&&wl#G)RuW{hKLwI4Rh7ue*g?0Nwzk^*wfXU- z4YJ+i!MXx2PmPS0mYfx9n&7oY6LA#;g3+;c$Sz@&K=wq8kG|Fra!We0L_s2$QW=Zk zo5WVdj7=T7q?)Bj635QOGz!}ek^~qD5CDBj5;%w*R|(Rj7A}sYv9Eq)dCkb!@CU?f zXtdM!cd$SEgy&C5$?kEgCkW2RXv@bLz$YTtCu7J{9_`3^l@~`(KAq+k25^s?1}U_+ zbE8_Q<>o})D4ng@SzBvvIY5FrNVM&wXl}NY2D@VNYVFuJAXUw#b{}i6#Hz$~S2s42 zv$iMWSBY8|ty;1s`{2+z6l1QbFlB+J%DA^ll1qwABcT8xDaR!VCV!nS-%LCX-12<+ z_}6CmY>h3ZXm^BO+L+KOX*%{D3);&EE@JJ&PnK>FjbEhO={n_?%my}DfRpTgg~p?I zC$?qJ_B$yX!?{*6$rkT@BN@|>rqL)+*nVx3RD@5an5!wJJMa3K|7?v)iKBSP)@+R5 zv4IL3??F11`C&`j8-mz9vFE*T={Xb~e=7&&m0013PyV(E$ literal 0 HcmV?d00001 diff --git a/spec/fixtures/files/debian/Nested_Release b/spec/fixtures/files/debian/Nested_Release new file mode 100644 index 000000000..994d6a636 --- /dev/null +++ b/spec/fixtures/files/debian/Nested_Release @@ -0,0 +1,247 @@ +Origin: LP-PPA-ondrej-nginx +Label: PPA for NGINX Stable with HTTP/2 +Suite: focal +Version: 20.04 +Codename: focal +Date: Fri, 26 May 2023 6:27:05 UTC +Architectures: amd64 arm64 armhf i386 ppc64el riscv64 s390x +Components: main +Description: Ubuntu Focal 20.04 +MD5Sum: + 981e4c2e020d5475bd5777f729031621 29008 main/binary-amd64/Packages + 6ada021f2fe814338373c5a673c1c6b2 6419 main/binary-amd64/Packages.gz + 9bca98c20b2144ee04c50e4e264db472 5888 main/binary-amd64/Packages.xz + 84af87c4c8bf68f1fcbf02582b5ce913 134 main/binary-amd64/Release + c007d99ef2dde4bf29d93f21d0f185e0 29012 main/binary-arm64/Packages + e3fee760a2f822199332f8958936afc8 6419 main/binary-arm64/Packages.gz + 675add161dec2c4631b12c9b29f3667f 5888 main/binary-arm64/Packages.xz + 939331613d2fb545cda2f5676de1c1f1 134 main/binary-arm64/Release + e98e5de578d289c3f3ee8f966809c213 28987 main/binary-armhf/Packages + 0a03f421a376d4aceb756bbe272053ac 6416 main/binary-armhf/Packages.gz + b6d4270c8e60192983c3f0cfbd1db77b 5892 main/binary-armhf/Packages.xz + f84736f614b57271e3d22a3951a19d2d 134 main/binary-armhf/Release + fd7f16dc03cd8a2d268fe79eea2704c5 6104 main/binary-i386/Packages + 53ca7c17d1ead2cc686731b4a1409396 1948 main/binary-i386/Packages.gz + 3271ae3116b9534ceb8474930936bfb3 1956 main/binary-i386/Packages.xz + 2e1a91fb5c7eb14df241b1e166669076 133 main/binary-i386/Release + 8d3678e87822ab86d31822c2b8d52799 29155 main/binary-ppc64el/Packages + 1afd795683b1730c1a5d4c79717294fe 6412 main/binary-ppc64el/Packages.gz + 9d0303aff7cfa5a69f2e2b6edf3a6d33 5888 main/binary-ppc64el/Packages.xz + 21159d15afb4176c1213c114818e4a7c 136 main/binary-ppc64el/Release + 6a142d4c73c3de4bd7a158224b54ae75 3486 main/binary-riscv64/Packages + 10cc0dbdcf2876118f69a9e6d6e6fe0e 1136 main/binary-riscv64/Packages.gz + fec587d40efdc9f9453caf4b076a967f 1196 main/binary-riscv64/Packages.xz + 57bd194347cd7044aae39becf8c8d8e0 136 main/binary-riscv64/Release + 6a142d4c73c3de4bd7a158224b54ae75 3486 main/binary-s390x/Packages + 10cc0dbdcf2876118f69a9e6d6e6fe0e 1136 main/binary-s390x/Packages.gz + fec587d40efdc9f9453caf4b076a967f 1196 main/binary-s390x/Packages.xz + 714ddb4d49a18cc6303ac88f5d333071 134 main/binary-s390x/Release + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-amd64/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-amd64/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-amd64/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-arm64/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-arm64/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-arm64/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-armhf/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-armhf/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-armhf/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-i386/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-i386/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-i386/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-ppc64el/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-ppc64el/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-ppc64el/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-riscv64/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-riscv64/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-riscv64/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debian-installer/binary-s390x/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debian-installer/binary-s390x/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debian-installer/binary-s390x/Packages.xz + 29d21cda14d75364d4fbb3355bd5ba40 25018 main/debug/binary-amd64/Packages + 4ac2959903ff3d6b528a21d6e5b4c011 5749 main/debug/binary-amd64/Packages.gz + 8bbfe12fd0c138b30ee2af0b87c7b288 5196 main/debug/binary-amd64/Packages.xz + 86f234a20296a1d01cb3fe8034667b00 25018 main/debug/binary-arm64/Packages + 0e4ba42f4baaa2ee12202477060e8ddc 5769 main/debug/binary-arm64/Packages.gz + 7b07d584e20aa5b36ace351607735bac 5220 main/debug/binary-arm64/Packages.xz + 1ec9c36b0ea93e7dea98f09adc1f8707 25018 main/debug/binary-armhf/Packages + d74367f4d3c5677a8fc7f92639058d76 5747 main/debug/binary-armhf/Packages.gz + 5e5cbb7f5e5d4222a01ab46e19d63f80 5220 main/debug/binary-armhf/Packages.xz + f08b9fb6c7399c68f8f030f5d6b09c7c 2310 main/debug/binary-i386/Packages + 6d7018fc7e0555fb38e7353ba3a4fe09 898 main/debug/binary-i386/Packages.gz + 2618090a38f4d10415227b29e5570b99 936 main/debug/binary-i386/Packages.xz + b25ad9e4cb17837d2de8116b4b5ba907 25138 main/debug/binary-ppc64el/Packages + 7e7af3a9a7b79c0efa8bfa947c6ed00e 5763 main/debug/binary-ppc64el/Packages.gz + a960a106b6786d9d704ba820faa2c5f4 5204 main/debug/binary-ppc64el/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debug/binary-riscv64/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debug/binary-riscv64/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debug/binary-riscv64/Packages.xz + d41d8cd98f00b204e9800998ecf8427e 0 main/debug/binary-s390x/Packages + fa9ac5a217b5547bc7dd4e6e894fe135 20 main/debug/binary-s390x/Packages.gz + 8dc5aea5b03dff8595f096f9e368e888 32 main/debug/binary-s390x/Packages.xz + 9d2bb1e8a4dad6c79b8591b8efe047b4 201 main/i18n/Index + 0508faa634d3fa9644d31ac334fd1ef7 22237 main/i18n/Translation-en + 7596563fa51ed6a2a0c22b46b36e3c8f 6367 main/i18n/Translation-en.gz + a96f01ca2262e16a8665176c0a0ed9fb 6104 main/i18n/Translation-en.xz + d0f5ef2d6ab566c9b2896e3090bbfbff 135 main/source/Release + fb3bbd7ba98d272ecfe6de3150601a81 6376 main/source/Sources + a52a826380a399d6a46a0e868190c6d7 1988 main/source/Sources.gz + 483f7cc86349f109d7852cfad31244e0 1980 main/source/Sources.xz +SHA1: + 7dbf83e8cdeffa8d285f02bc6a3b7c3401c731fe 29008 main/binary-amd64/Packages + 42bbbea0e89d5617336ed77932d98e0125b313e0 6419 main/binary-amd64/Packages.gz + 3b0f33025b5c23173ecd964579e44786fb59e044 5888 main/binary-amd64/Packages.xz + 073c3a508f1a7c0d0eaabe90c8b92f71a282eefb 134 main/binary-amd64/Release + 6f5bc2660c5a28db5243453a20e7bd83969f551e 29012 main/binary-arm64/Packages + f3ddb8ffcc7ad8d0d31145ac23d1235b06189dda 6419 main/binary-arm64/Packages.gz + 0487913ecb36df1d87bc42c2937073a771c067ee 5888 main/binary-arm64/Packages.xz + 7b51e81aa38daac027bb404899f552176933fa29 134 main/binary-arm64/Release + 80470be8395380648fb6cc399a7fc3e030a03100 28987 main/binary-armhf/Packages + 80de1b12afb36973528e7f57be8958b24d9f4988 6416 main/binary-armhf/Packages.gz + fb2600cb259cf8dae132f60d0437f5a5343d30dd 5892 main/binary-armhf/Packages.xz + fec206b6af21360582dfad81e74aa1d7c282c1f2 134 main/binary-armhf/Release + 6e476c133eeae8235301896a09384a5eabfebe52 6104 main/binary-i386/Packages + 89e9309543b217defbc4f9128c83bd79e966c53e 1948 main/binary-i386/Packages.gz + c4d7ef2f558262f95374b0e2f65bb5ed40e1334b 1956 main/binary-i386/Packages.xz + 519def120d2562d22dd5653e0ab017cb12a62fab 133 main/binary-i386/Release + 74685ba30eb67550b925434cba04419547798be8 29155 main/binary-ppc64el/Packages + be8983257a1dbebf575f0653b2c9b2cad25094ca 6412 main/binary-ppc64el/Packages.gz + 0b69336a50fb3c5f0642a1077ade32b6a11de291 5888 main/binary-ppc64el/Packages.xz + c078d934eb9fcd89ad9ec6c60bd14a67c41e78ec 136 main/binary-ppc64el/Release + 1dd4ec2925bc96d4280c3860494339359067a0b8 3486 main/binary-riscv64/Packages + c6697615bcea3f76bd483c06167b7aa651aaf8aa 1136 main/binary-riscv64/Packages.gz + 066056e9cbd492e4316d170f8d6f64474c60161a 1196 main/binary-riscv64/Packages.xz + 63e3b27ae80db472d30eb55a7cceb8da49f96377 136 main/binary-riscv64/Release + 1dd4ec2925bc96d4280c3860494339359067a0b8 3486 main/binary-s390x/Packages + c6697615bcea3f76bd483c06167b7aa651aaf8aa 1136 main/binary-s390x/Packages.gz + 066056e9cbd492e4316d170f8d6f64474c60161a 1196 main/binary-s390x/Packages.xz + f94dac699569fb3ed00607b09075397537600cf9 134 main/binary-s390x/Release + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-amd64/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-amd64/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-amd64/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-arm64/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-arm64/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-arm64/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-armhf/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-armhf/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-armhf/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-i386/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-i386/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-i386/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-ppc64el/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-ppc64el/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-ppc64el/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-riscv64/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-riscv64/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-riscv64/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debian-installer/binary-s390x/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debian-installer/binary-s390x/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debian-installer/binary-s390x/Packages.xz + 8228cfdd7d0c848b0782d6890bcb6315b8a9a198 25018 main/debug/binary-amd64/Packages + 4d03307b0f3054591c7af1bcc03d9e66e6a2f979 5749 main/debug/binary-amd64/Packages.gz + 2356d87c0a1daadb8925c5aac2fc13d454785444 5196 main/debug/binary-amd64/Packages.xz + 5230b4e3e00f4cb06c41cbc4e9fa8225063aa033 25018 main/debug/binary-arm64/Packages + 30515603e2f5a19e5bc6f8d94f76b956fa9acf7b 5769 main/debug/binary-arm64/Packages.gz + f72d63d70a492f3a68fd7dd1f96a8b73d34a62c6 5220 main/debug/binary-arm64/Packages.xz + bea33a15fb9f61c872c502abb39209b21b3172b9 25018 main/debug/binary-armhf/Packages + d9210cb8f555d4cb6f28e7a17545ff68c27a2970 5747 main/debug/binary-armhf/Packages.gz + e564780fcc0a42b1deb27657d652c9d1f3a5600e 5220 main/debug/binary-armhf/Packages.xz + 2d8f585e0b6a6cee3d9cc0f4e51ec0c7fa2a8a57 2310 main/debug/binary-i386/Packages + 06b07ca65b4c65a07a60bfcd032c8489c342765c 898 main/debug/binary-i386/Packages.gz + 4122a5a6193cdd3ecffeb6e1f820341c5b1bca33 936 main/debug/binary-i386/Packages.xz + 3be6bce33adc89fc127eade07d6277de31f1b60e 25138 main/debug/binary-ppc64el/Packages + 93809d71b5cc2c4f8e4319eb098ce9f1933701c0 5763 main/debug/binary-ppc64el/Packages.gz + dcbd113bde6073ccfcd05f6297762d99f6287c91 5204 main/debug/binary-ppc64el/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debug/binary-riscv64/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debug/binary-riscv64/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debug/binary-riscv64/Packages.xz + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 main/debug/binary-s390x/Packages + 89892054d65b8b0dd6a081b33a97b6f2bd1fa267 20 main/debug/binary-s390x/Packages.gz + 9746882f4236fa1c3a8f86be2f1d9c46680c0b10 32 main/debug/binary-s390x/Packages.xz + 0deb7607a910cd17a05d8f0868d7e42ef1fa63bf 201 main/i18n/Index + ea7790fa38613fc67895a31632db8040aabecaf8 22237 main/i18n/Translation-en + 3a93ea38456709e95161485dd0a9367101222b33 6367 main/i18n/Translation-en.gz + 2499ac6b0369a515853c8247bfdd8d79054d4841 6104 main/i18n/Translation-en.xz + 64e3312676d3bf4cf8ddf89b4a11acbd492df8ec 135 main/source/Release + 30d636af01d70eb5ba9261cb931045095c9ba4bf 6376 main/source/Sources + a274a2f4451b2e3f8d9bf87b339c994faa895ff6 1988 main/source/Sources.gz + 455299697da9ca492a6b66f33115a732b1919923 1980 main/source/Sources.xz +SHA256: + 46d7bcf8e069c84a49123e9b70d317f99a189eacacc5e413a4c77856e54f788f 29008 main/binary-amd64/Packages + 9f8c4ece8742794c12a676158d8731b69b66b0766626270590ed09a01f65bfd3 6419 main/binary-amd64/Packages.gz + 207586b58affe7f867c471ffd9b63e80165053f6e54ee4a6d011b149fbfcf866 5888 main/binary-amd64/Packages.xz + b68e91c4b89dbe245fdef76c4efa09f5168948970f9b712ce93547222ca25bb3 134 main/binary-amd64/Release + 3afe05d9b12c2fc3bd8af4f06f2dce85fd32b9e1a4096d7cb7151fe36ef40365 29012 main/binary-arm64/Packages + 91c66887eefa4ae758e34ca9aa2c2b8cf1c48ceaadb999a3b7a67bb37b65b2ab 6419 main/binary-arm64/Packages.gz + fd26475212f801c28738efff64bc00977e5d30a09ec6d1574549a0431aff121f 5888 main/binary-arm64/Packages.xz + c39f2350a6c893090fa1162aa3281dbf8e591c154122a007c458b537298dcc0f 134 main/binary-arm64/Release + d2c02d68c59a7cbed3008d52f92b452a79949f2ca83e3fce59606091c3bfc267 28987 main/binary-armhf/Packages + 70b5e42def597e161558e9b67ee49d9378fe4ac80aa0280034e9c412e544616b 6416 main/binary-armhf/Packages.gz + ea9babc2bbd4a3e629ca22d0326e132b5535502c6863a74879bc4eee0a405698 5892 main/binary-armhf/Packages.xz + 0c7c3934800ac686bbd1324734f75645a524648c7779456291bd1d67f4720035 134 main/binary-armhf/Release + 206edb9a58c28b1c9a46fbf3e3451f81a8d10cc17374d8e201e6d8791d70a487 6104 main/binary-i386/Packages + 79391443843abee883a484ed4c07936cba3959ab4dade3852028428159b9c411 1948 main/binary-i386/Packages.gz + 323439e17b04db5fbdc44dea2895277596f2f5a3c36998410e3613c8450f8fa7 1956 main/binary-i386/Packages.xz + 4451595be754481c35f60469afb31ae169aa08b4be0134886e6b0615330051af 133 main/binary-i386/Release + c0ae699ba49cd84459ed9a08b7ee423f802617e889f26bcd955902ec5c6ef695 29155 main/binary-ppc64el/Packages + 2ee11bc55599681abda0b6480b8d59bf3009c13d0585b268652487bc4a72d486 6412 main/binary-ppc64el/Packages.gz + 2dbe2921df2d3cf3aed903a07c4f218a10d5d35a9e55b5bada8a0ef6f5566687 5888 main/binary-ppc64el/Packages.xz + b7a4a21d465320c4236bb9a2925b23a6df7b6cb92f638c93396aca965072eb2e 136 main/binary-ppc64el/Release + cde25cab3406f13557a82d944249bda1ee47b4664957b6eeb86121217433b2ed 3486 main/binary-riscv64/Packages + 1e86ab07bd8c98caf3c05f0a1091296be7e3faaa1cda1f764de8f4f26eea2d50 1136 main/binary-riscv64/Packages.gz + 8804e1312631b6603674df1ac27fdfaa8ccb00cd682cd8c937a30d7b0f32ff9c 1196 main/binary-riscv64/Packages.xz + 4d097996f75a8548bafddec01e04eae871e13f34fe1dd66e98b38e68149c08ed 136 main/binary-riscv64/Release + cde25cab3406f13557a82d944249bda1ee47b4664957b6eeb86121217433b2ed 3486 main/binary-s390x/Packages + 1e86ab07bd8c98caf3c05f0a1091296be7e3faaa1cda1f764de8f4f26eea2d50 1136 main/binary-s390x/Packages.gz + 8804e1312631b6603674df1ac27fdfaa8ccb00cd682cd8c937a30d7b0f32ff9c 1196 main/binary-s390x/Packages.xz + 46e21a856aff2287d16e31009478a90a70ff0d8516554ce21a57550940168ae1 134 main/binary-s390x/Release + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-amd64/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-amd64/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-amd64/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-arm64/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-arm64/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-arm64/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-armhf/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-armhf/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-armhf/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-i386/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-i386/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-i386/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-ppc64el/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-ppc64el/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-ppc64el/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-riscv64/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-riscv64/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-riscv64/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debian-installer/binary-s390x/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debian-installer/binary-s390x/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debian-installer/binary-s390x/Packages.xz + 9dd99bc9b479ae0e96f2f464ec3f61081c8a359968642ea90b09e516695764c6 25018 main/debug/binary-amd64/Packages + 7290c8d07e7439776c8bcf69e40fdb11467a4168cc4ee931138c9f056681679e 5749 main/debug/binary-amd64/Packages.gz + f925fc9cdf353e52c5ff0bb2a01a54030667d0102fa3fc58629e8bb601b89b0c 5196 main/debug/binary-amd64/Packages.xz + bcba848fbb00447ca549629c9855ee3f3afdc2d3ad08f6df6ce71732022573e4 25018 main/debug/binary-arm64/Packages + 170d70b521b415448b114d8b1d4a9d04abdb852a5a05e3ea56e505c1c81997e6 5769 main/debug/binary-arm64/Packages.gz + b1604294ff3204a716d0ca38ffae8d858383f6d458f8d537a485836a98b72d6d 5220 main/debug/binary-arm64/Packages.xz + ac78b7c2222c54b356e49b9876cffde2d574055376b6a357bb8c09b616a51ceb 25018 main/debug/binary-armhf/Packages + 4108ca6d29b26372f7408bd80307eb51dd868d414bd8cdb81a49d8c3534b9434 5747 main/debug/binary-armhf/Packages.gz + eeb527823a3421bcc9e4a4e81ac19c516ed97eb2b8a589be2d94652fd6159c90 5220 main/debug/binary-armhf/Packages.xz + d70b2b304e1ad8fbdcc9862e3168fbaa66c67ac84efc1285b9e0a2446417bc57 2310 main/debug/binary-i386/Packages + e89277865c7203cf3135783401c5edb04f1168c7ed91bd904bc693b397f862bf 898 main/debug/binary-i386/Packages.gz + 7fe9a00b9a5a7039d0a58bbb4a4ce77796d6b51125121c3100b69ec1eb2b73c8 936 main/debug/binary-i386/Packages.xz + be4145d43170960e2a63fe37ce49056ea767d877f7c99266b09a95c539d0520c 25138 main/debug/binary-ppc64el/Packages + f24c97af7e759d451b22697cbc37830aae632df31f85513a0652159b3d2973e7 5763 main/debug/binary-ppc64el/Packages.gz + e221f45db4abbb95efa4b52501545e1829bc50664c887069ee6915419a8e81dd 5204 main/debug/binary-ppc64el/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debug/binary-riscv64/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debug/binary-riscv64/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debug/binary-riscv64/Packages.xz + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 main/debug/binary-s390x/Packages + 9ceffb7310338057cfe71a4ae1e2c98d2c485d81cdef906532a801f457a38d64 20 main/debug/binary-s390x/Packages.gz + 0040f94d11d0039505328a90b2ff48968db873e9e7967307631bf40ef5679275 32 main/debug/binary-s390x/Packages.xz + 54a93f60720cf899f7a11e4029f08cbc9579ae0b67d7bb2858c819f290a20506 201 main/i18n/Index + 94da79e29963e1fb6ef793e21c987d2e0a29d4bd26929c9459ce2e30002a437c 22237 main/i18n/Translation-en + e2fef447d114c13919243a38b108375f16bd24f91676e95878d67e7f32637cf9 6367 main/i18n/Translation-en.gz + 7fb69dc91fc9edbb8bbf5f1086f49f325c6a75e5431cdcf9cdd09aacff48d5a3 6104 main/i18n/Translation-en.xz + d7758bdad693afe4d0644eef0ca7bd357204e2d464422e769e4dd1ec804dff92 135 main/source/Release + aa1da047cfd128322cc35434ceeb1ff7c1061eacb510292470f8a88c99a7d958 6376 main/source/Sources + abc69bee0a5e5a731f835d6cfb1157ff4cace91082e00da4a8ea1e4cfe2a8889 1988 main/source/Sources.gz + a1cc34c569d0d64c1d3147872fed6d9496f85299bcc3c1b72a9414c7c1b9db9f 1980 main/source/Sources.xz +Acquire-By-Hash: yes diff --git a/spec/lib/rmt/file_validator_spec.rb b/spec/lib/rmt/file_validator_spec.rb index f50f9c54a..488259a0e 100644 --- a/spec/lib/rmt/file_validator_spec.rb +++ b/spec/lib/rmt/file_validator_spec.rb @@ -238,30 +238,6 @@ def initialize(deep_verify) end end - RSpec::Matchers.define :contain_records_like do |expected| - match do |actual| - record_struct = Struct.new(:local_path, :checksum, :checksum_type, :size) - - @actual = actual.map { |r| record_struct.new(r.local_path, r.checksum, r.checksum_type, r.size) } - @expected = expected.map { |r| record_struct.new(r.local_path, r.checksum, r.checksum_type, r.size) } - - actual.all? do |record| - expected.any? do |object| - record.local_path == object.local_path && - record.checksum == object.checksum && - record.checksum_type == object.checksum_type && - record.size == object.size - end - end - end - - failure_message do |actual| - "expected that collection #{actual} would contain #{expected}" - end - - diffable - end - describe '#find_valid_files_by_checksum' do let(:valid_file) do fixture_path = file_fixture('dummy_product/product/apples-0.1-0.x86_64.rpm').to_s diff --git a/spec/lib/rmt/mirror/base_spec.rb b/spec/lib/rmt/mirror/base_spec.rb index 767595cfe..6270ba9db 100644 --- a/spec/lib/rmt/mirror/base_spec.rb +++ b/spec/lib/rmt/mirror/base_spec.rb @@ -177,6 +177,7 @@ it 'downloads enqueued contents and clear queue' do expect(downloader).to receive(:download_multi) + expect(base.enqueued.count).to eq(1) base.download_enqueued expect(base.enqueued).to be_empty end @@ -236,7 +237,7 @@ it 'copies content from source to destination without backup' do expect(base).to receive(:replace_directory).with(source: src, destination: dest, with_backup: false).and_yield - expect(FileUtils).to receive(:mv).with(Dir.glob(src), dest) + expect(FileUtils).to receive(:mv).with(Dir.glob(src), dest, force: true) base.copy_directory_content(source: src, destination: dest) end end diff --git a/spec/lib/rmt/mirror/debian_spec.rb b/spec/lib/rmt/mirror/debian_spec.rb index 44cda251a..ac139bde4 100644 --- a/spec/lib/rmt/mirror/debian_spec.rb +++ b/spec/lib/rmt/mirror/debian_spec.rb @@ -83,9 +83,10 @@ end describe '#mirror_metadata' do + let(:release_fixture) { 'Release' } let(:config) do { - relative_path: 'Release', + relative_path: release_fixture, base_dir: file_fixture('debian/'), base_url: 'https://updates.suse.de/Debian/' } @@ -100,7 +101,7 @@ it 'succeeds' do allow(debian).to receive(:check_signature) allow(debian).to receive(:parse_release_file).and_return([]) - expect(debian).to receive(:download_enqueued) + expect(debian).to receive(:download_enqueued).twice debian.mirror_metadata end @@ -110,7 +111,36 @@ it 'downloads and parses the file' do expect(debian).to receive(:download_cached!).with(release_path, to: 'bar') expect(debian).to receive(:check_signature) + expect(debian).to receive(:download_enqueued).twice + debian.mirror_metadata + end + end + + context 'nested debian repository' do + let(:release_fixture) { 'Nested_Release' } + + before do + allow(debian).to receive(:download_cached!).and_return(release_ref) + allow(debian).to receive(:check_signature) + described_class.send(:public, :enqueued) + end + + it 'ignores non-existent references from release file' do + allow(debian).to receive(:download_enqueued) + allow(debian).to receive(:enqueue) + + expect(debian).to receive(:download_enqueued).with(continue_on_error: true) + expect(debian).to receive(:enqueue).with(all(be_like_relative_path(/(Packages|Sources|Translation)(-\w+)?$/))) + + debian.mirror_metadata + end + + it 'calls download_enqueued for the remaining valid paths' do + expect(debian).to receive(:download_enqueued).with(continue_on_error: true) expect(debian).to receive(:download_enqueued) + expect(debian).to receive(:enqueue).with(all(be_like_relative_path(/(Packages|Sources|Translation)(-\w+)?$/))).once + expect(debian).to receive(:enqueue).once + debian.mirror_metadata end end diff --git a/spec/lib/rmt/mirror/repomd_spec.rb b/spec/lib/rmt/mirror/repomd_spec.rb index 63a514779..9a015c6dc 100644 --- a/spec/lib/rmt/mirror/repomd_spec.rb +++ b/spec/lib/rmt/mirror/repomd_spec.rb @@ -3,16 +3,6 @@ RSpec.describe RMT::Mirror::Repomd do subject(:repomd) { described_class.new(**repomd_configuration) } - RSpec::Matchers.define :file_reference_containing_path do |expected| - match do |actual| - actual.local_path.include?(expected) - end - - failure_message do |actual| - "expected that file path #{actual.local_path} would contain #{expected}" - end - end - let(:logger) { RMT::Logger.new('/dev/null') } diff --git a/spec/support/matchers.rb b/spec/support/matchers.rb new file mode 100644 index 000000000..ae9e73c79 --- /dev/null +++ b/spec/support/matchers.rb @@ -0,0 +1,43 @@ +RSpec::Matchers.define :file_reference_containing_path do |expected| + match do |actual| + actual.local_path.include?(expected) + end + + failure_message do |actual| + "expected that file path #{actual.local_path} would contain #{expected}" + end +end + +RSpec::Matchers.define :be_like_relative_path do |regex| + match do |actual| + actual.relative_path =~ regex + end + + failure_message do |actual| + "expected that file path #{actual.relative_path} would contain #{regex}" + end +end + +RSpec::Matchers.define :contain_records_like do |expected| + match do |actual| + record_struct = Struct.new(:local_path, :checksum, :checksum_type, :size) + + @actual = actual.map { |r| record_struct.new(r.local_path, r.checksum, r.checksum_type, r.size) } + @expected = expected.map { |r| record_struct.new(r.local_path, r.checksum, r.checksum_type, r.size) } + + actual.all? do |record| + expected.any? do |object| + record.local_path == object.local_path && + record.checksum == object.checksum && + record.checksum_type == object.checksum_type && + record.size == object.size + end + end + end + + failure_message do |actual| + "expected that collection #{actual} would contain #{expected}" + end + + diffable +end From 4f9562cb4bafc75c2c27b1986ca2fcd559f2625c Mon Sep 17 00:00:00 2001 From: Natnael Getahun Date: Wed, 17 Jan 2024 12:21:43 +0100 Subject: [PATCH 2/4] Improve log messages Increase test coverage Fix variable naming --- lib/rmt/cli/export.rb | 2 +- lib/rmt/cli/import.rb | 2 +- lib/rmt/mirror.rb | 3 + lib/rmt/mirror/base.rb | 1 - lib/rmt/mirror/debian.rb | 8 ++- lib/rmt/mirror/file_reference.rb | 4 -- lib/rmt/mirror/license.rb | 1 + spec/lib/rmt/cli/export_spec.rb | 2 +- spec/lib/rmt/mirror/base_spec.rb | 27 ++++++-- spec/lib/rmt/mirror/debian_spec.rb | 12 ++-- spec/lib/rmt/mirror/license_spec.rb | 9 ++- spec/lib/rmt/mirror/repomd_spec.rb | 18 ++--- spec/lib/rmt/mirror/suma_product_tree_spec.rb | 4 +- spec/lib/rmt/mirror_spec.rb | 66 ++++++++++++++++--- spec/support/matchers.rb | 2 + 15 files changed, 116 insertions(+), 45 deletions(-) diff --git a/lib/rmt/cli/export.rb b/lib/rmt/cli/export.rb index edc666edc..d8265e1f5 100644 --- a/lib/rmt/cli/export.rb +++ b/lib/rmt/cli/export.rb @@ -34,7 +34,7 @@ def repos(path) begin suma_product_tree.mirror rescue RMT::Mirror::Exception => e - logger.warn(e.message) + logger.warn(_('Exporting SUSE Manager product tree failed: %{error_message}') % { error_message: e.message }) end repos_file = File.join(path, 'repos.json') diff --git a/lib/rmt/cli/import.rb b/lib/rmt/cli/import.rb index a51d9cd58..1ca7dba2b 100644 --- a/lib/rmt/cli/import.rb +++ b/lib/rmt/cli/import.rb @@ -25,7 +25,7 @@ def repos(path) url: suma_repo_url ).mirror rescue RMT::Mirror::Exception => e - logger.warn(e.message) + logger.warn(_('Importing suma product tree failed: %{error_message}') % { error_message: e.message }) end repos = JSON.parse(File.read(repos_file)) diff --git a/lib/rmt/mirror.rb b/lib/rmt/mirror.rb index 8c3436c16..1c71d1f5d 100644 --- a/lib/rmt/mirror.rb +++ b/lib/rmt/mirror.rb @@ -37,6 +37,9 @@ def repository_mirror_class end def repository_type + # We search repomd structure first since it is more common + # Debian is less common + search = { repomd: File.join(repository.external_url, RPM_FILE_NEEDLE), debian: File.join(repository.external_url, DEB_FILE_NEEDLE) diff --git a/lib/rmt/mirror/base.rb b/lib/rmt/mirror/base.rb index d4c74893d..7a20b2c56 100644 --- a/lib/rmt/mirror/base.rb +++ b/lib/rmt/mirror/base.rb @@ -80,7 +80,6 @@ def repository_path(*args) File.join(mirroring_base_dir, repository.local_path, *args) end - # FIXME: Write some specs for me! def create_repository_path FileUtils.mkpath(repository_path) unless Dir.exist?(repository_path) rescue StandardError => e diff --git a/lib/rmt/mirror/debian.rb b/lib/rmt/mirror/debian.rb index a98df88c5..50be91727 100644 --- a/lib/rmt/mirror/debian.rb +++ b/lib/rmt/mirror/debian.rb @@ -3,6 +3,8 @@ class RMT::Mirror::Debian < RMT::Mirror::Base SIGNATURE_FILE_NAME = 'Release.gpg'.freeze KEY_FILE_NAME = 'Release.key'.freeze INRELEASE_FILE_NAME = 'InRelease'.freeze + NESTED_REPOSITORY_REGEX = %r{/dists/\w*/$}.freeze + DETECT_NONMANDATORY_FILES = %r{/(Packages|Sources|Translation)(-\w+)?$/}.freeze def mirror_implementation create_repository_path @@ -32,7 +34,7 @@ def mirror_metadata # The nested debian structure only contains the zipped version of packages sometimes # However, the release file still contains a reference to the unzipped versions # So, we don't error if they don't exist - packages, remaining = metadata_refs.partition { |ref| ref.relative_path.match(/(Packages|Sources|Translation)(-\w+)?$/) } + packages, remaining = metadata_refs.partition { |ref| ref.relative_path.match(DETECT_NONMANDATORY_FILES) } enqueue(packages) download_enqueued(continue_on_error: true) @@ -50,8 +52,8 @@ def mirror_packages(metadata_refs) # In a nested debian repository stucture, the metadata and packages are stored in different locations # so we need to update the base_url if we encounter the nested structure # We assume that if the base_url contains '/dists/', it's a nested debian structure - if ref.base_url.match?(%r{/dists/\w*/$}) - ref.base_url.sub!(%r{(/dists/\w*/$)}, '/') + if ref.base_url.match?(NESTED_REPOSITORY_REGEX) + ref.base_url.sub!(NESTED_REPOSITORY_REGEX, '/') end enqueue(ref) if need_to_download?(ref) end diff --git a/lib/rmt/mirror/file_reference.rb b/lib/rmt/mirror/file_reference.rb index c48bb3119..d958d9d4e 100644 --- a/lib/rmt/mirror/file_reference.rb +++ b/lib/rmt/mirror/file_reference.rb @@ -49,8 +49,4 @@ def remote_path def cache_timestamp File.mtime(cache_path).utc if cache_path && File.exist?(cache_path) end - - def to_s - relative_path - end end diff --git a/lib/rmt/mirror/license.rb b/lib/rmt/mirror/license.rb index 4bcc7a05f..be4dc546b 100644 --- a/lib/rmt/mirror/license.rb +++ b/lib/rmt/mirror/license.rb @@ -18,6 +18,7 @@ def licenses_available? end request.run + logger.debug("No license directory found for repository '#{uri}'") false end diff --git a/spec/lib/rmt/cli/export_spec.rb b/spec/lib/rmt/cli/export_spec.rb index 41c6cf481..140c1423d 100644 --- a/spec/lib/rmt/cli/export_spec.rb +++ b/spec/lib/rmt/cli/export_spec.rb @@ -95,7 +95,7 @@ allow_any_instance_of(RMT::Mirror).to receive(:mirror_now) expect(suma_product_tree_double).to receive(:mirror).and_raise(RMT::Mirror::Exception, 'black mirror') - expect_any_instance_of(RMT::Logger).to receive(:warn).with('black mirror') + expect_any_instance_of(RMT::Logger).to receive(:warn).with(/black mirror/) command end end diff --git a/spec/lib/rmt/mirror/base_spec.rb b/spec/lib/rmt/mirror/base_spec.rb index 6270ba9db..8c83f5e5a 100644 --- a/spec/lib/rmt/mirror/base_spec.rb +++ b/spec/lib/rmt/mirror/base_spec.rb @@ -1,9 +1,9 @@ require 'rails_helper' describe RMT::Mirror::Base do - subject(:base) { described_class.new(**configuration) } + subject(:base) { described_class.new(**mirror_configuration) } - let(:configuration) do + let(:mirror_configuration) do { repository: repository, logger: logger, @@ -86,6 +86,21 @@ end end + describe '#create_repository_path' do + it 'creates the repository path' do + allow(Dir).to receive(:exist?).and_return(false) + expect(FileUtils).to receive(:mkpath).with('/rspec/repository/update/hype/15.3/product/') + base.create_repository_path + end + + it 'fails when it could not create the repository path' do + allow(Dir).to receive(:exist?).and_return(false) + allow(FileUtils).to receive(:mkpath).and_raise(StandardError) + + expect { base.create_repository_path }.to raise_error(RMT::Mirror::Exception, /Could not create local directory/) + end + end + describe '#cleanup_temp_dirs' do let(:temp_metadata) { '/tmp/metadata' } let(:temp_licenses) { '/tmp/licenses' } @@ -124,15 +139,15 @@ end describe '#check_signature' do - let(:config) do + let(:ref_configuration) do { base_dir: '/tmp', base_url: 'https://updates.suse.de/' } end - let(:signature_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.gpg', **config) } - let(:key_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.key', **config) } - let(:metadata) { RMT::Mirror::FileReference.new(relative_path: 'metadata', **config) } + let(:signature_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.gpg', **ref_configuration) } + let(:key_file) { RMT::Mirror::FileReference.new(relative_path: 'repo.key', **ref_configuration) } + let(:metadata) { RMT::Mirror::FileReference.new(relative_path: 'metadata', **ref_configuration) } let(:gpg_checker) do RMT::GPG.new( metadata_file: metadata.local_path, diff --git a/spec/lib/rmt/mirror/debian_spec.rb b/spec/lib/rmt/mirror/debian_spec.rb index ac139bde4..398274202 100644 --- a/spec/lib/rmt/mirror/debian_spec.rb +++ b/spec/lib/rmt/mirror/debian_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' describe RMT::Mirror::Debian do - subject(:debian) { described_class.new(**configuration) } + subject(:debian) { described_class.new(**debian_mirror_configuration) } let(:repository) do create :repository, @@ -11,7 +11,7 @@ # Configuration for Debian mirroring instance let(:mirroring_base_dir) { '/test/repository/base/path/' } - let(:configuration) do + let(:debian_mirror_configuration) do { repository: repository, logger: RMT::Logger.new('/dev/null'), @@ -84,14 +84,14 @@ describe '#mirror_metadata' do let(:release_fixture) { 'Release' } - let(:config) do + let(:release_configuration) do { relative_path: release_fixture, base_dir: file_fixture('debian/'), base_url: 'https://updates.suse.de/Debian/' } end - let(:release_ref) { RMT::Mirror::FileReference.new(**config) } + let(:release_ref) { RMT::Mirror::FileReference.new(**release_configuration) } before do allow(debian).to receive(:temp).with(:metadata).and_return('bar') @@ -194,14 +194,14 @@ end describe '#parse_release_file' do - let(:config) do + let(:release_configuration) do { relative_path: rel_path, base_dir: file_fixture('debian/'), base_url: 'https://updates.suse.de/Debian/' } end - let(:release_ref) { RMT::Mirror::FileReference.new(**config) } + let(:release_ref) { RMT::Mirror::FileReference.new(**release_configuration) } context 'Release file is valid' do let(:rel_path) { 'Release' } diff --git a/spec/lib/rmt/mirror/license_spec.rb b/spec/lib/rmt/mirror/license_spec.rb index f4bea4446..1cec189dc 100644 --- a/spec/lib/rmt/mirror/license_spec.rb +++ b/spec/lib/rmt/mirror/license_spec.rb @@ -14,20 +14,22 @@ let(:license_configuration) do { repository: repository, - logger: RMT::Logger.new('/dev/null'), + logger: logger, mirroring_base_dir: base_dir } end + let(:logger) { RMT::Logger.new('/dev/null') } + let(:fixture) { 'directory.yast' } - let(:config) do + let(:license_listing_configuration) do { relative_path: fixture, base_dir: file_fixture(''), base_url: 'https://updates.suse.de/sles/' } end - let(:licenses_ref) { RMT::Mirror::FileReference.new(**config) } + let(:licenses_ref) { RMT::Mirror::FileReference.new(**license_listing_configuration) } before do allow(FileUtils).to receive(:mkpath).with(license.repository_path).and_return(nil) @@ -42,6 +44,7 @@ it 'returns false if directory.yast is not available' do stub_request(:head, license.repository_url('directory.yast')).to_return(status: 404, body: '', headers: {}) + expect(logger).to receive(:debug) expect(license.licenses_available?).to eq(false) end diff --git a/spec/lib/rmt/mirror/repomd_spec.rb b/spec/lib/rmt/mirror/repomd_spec.rb index 9a015c6dc..16954b535 100644 --- a/spec/lib/rmt/mirror/repomd_spec.rb +++ b/spec/lib/rmt/mirror/repomd_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' RSpec.describe RMT::Mirror::Repomd do - subject(:repomd) { described_class.new(**repomd_configuration) } + subject(:repomd) { described_class.new(**repomd_mirror_configuration) } let(:logger) { RMT::Logger.new('/dev/null') } @@ -15,7 +15,7 @@ # Configuration for Debian mirroring instance let(:base_dir) { '/test/repository/base/path/' } - let(:repomd_configuration) do + let(:repomd_mirror_configuration) do { repository: repository, logger: logger, @@ -25,14 +25,14 @@ # Configuration for file reference to an arbitrary fixture let(:fixture) { 'repodata/repomd.xml' } - let(:config) do + let(:file_ref_configuration) do { relative_path: fixture, base_dir: file_fixture('dummy_repo/'), base_url: 'https://updates.suse.com/sample/repository/15.4/' } end - let(:repomd_ref) { RMT::Mirror::FileReference.new(**config) } + let(:repomd_ref) { RMT::Mirror::FileReference.new(**file_ref_configuration) } before do described_class.send(:public, *described_class.protected_instance_methods) @@ -75,15 +75,15 @@ end describe '#mirror_metadata' do - let(:x_config) do + let(:ref_configuration) do { base_dir: base_dir, base_url: 'https://updates.suse.com/sample/repository/15.4/', cache_dir: repomd.repository_path } end - let(:signature_file) { RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.asc', **x_config) } - let(:key_file) { RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.key', **x_config) } + let(:signature_file) { RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.asc', **ref_configuration) } + let(:key_file) { RMT::Mirror::FileReference.new(relative_path: 'repodata/repomd.xml.key', **ref_configuration) } let(:repomd_parser) { RepomdParser::RepomdXmlParser.new(repomd_ref.local_path) } before do @@ -134,7 +134,7 @@ let(:fixture) { 'repodata/repomd.xml' } let(:fixture_base_dir) { file_fixture('dummy_repo') } let(:repomd_package_ref) do - RMT::Mirror::FileReference.new(**config).tap do |ref| + RMT::Mirror::FileReference.new(**file_ref_configuration).tap do |ref| ref.type = type end end @@ -171,7 +171,7 @@ describe '#mirror_packages' do let(:fixture) { 'repodata/abf421e45af5cd686f050bab3d2a98e0a60d1b5ca3b07c86cb948fc1abfa675e-primary.xml.gz' } - let(:primary_ref) { RMT::Mirror::FileReference.new(**config).tap { |ref| ref.type = :primary } } + let(:primary_ref) { RMT::Mirror::FileReference.new(**file_ref_configuration).tap { |ref| ref.type = :primary } } let(:package_ref) { instance_double(RMT::Mirror::FileReference) } it 'downloads the reference packages' do diff --git a/spec/lib/rmt/mirror/suma_product_tree_spec.rb b/spec/lib/rmt/mirror/suma_product_tree_spec.rb index 96627603c..495141d34 100644 --- a/spec/lib/rmt/mirror/suma_product_tree_spec.rb +++ b/spec/lib/rmt/mirror/suma_product_tree_spec.rb @@ -1,9 +1,9 @@ require 'rails_helper' RSpec.describe RMT::Mirror::SumaProductTree do - subject(:suma) { described_class.new(**configuration) } + subject(:suma) { described_class.new(**suma_mirror_configuration) } - let(:configuration) do + let(:suma_mirror_configuration) do { logger: RMT::Logger.new('/dev/null'), mirroring_base_dir: base_dir diff --git a/spec/lib/rmt/mirror_spec.rb b/spec/lib/rmt/mirror_spec.rb index 2f23e4479..5829ec02d 100644 --- a/spec/lib/rmt/mirror_spec.rb +++ b/spec/lib/rmt/mirror_spec.rb @@ -27,26 +27,28 @@ it 'uses the credentials from the repository if the repository has credentials' do stub_request(:head, authenticated_repo_url) - .to_return(status: 200, body: '', headers: {}) + .to_return(status: 200, body: '', headers: {}) allow(repository).to receive(:auth_token).and_return('token') expect(RMT::HttpRequest).to receive(:new) - .with(authenticated_repo_url, method: :head, followlocation: true) - .and_call_original + .with(authenticated_repo_url, method: :head, followlocation: true) + .and_call_original mirror.repository_type end it 'uses no credentials from the repository if the repository has no credentials' do stub_request(:head, repomd_url) - .to_return(status: 200, body: '', headers: {}) + .to_return(status: 200, body: '', headers: {}) expect(RMT::HttpRequest).to receive(:new) - .with(URI.parse(repomd_url), method: :head, followlocation: true) - .and_call_original + .with(URI.parse(repomd_url), method: :head, followlocation: true) + .and_call_original mirror.repository_type end + + # it 'continues checking repo if fi' end context 'repomd repository' do @@ -68,8 +70,8 @@ end end - context 'debian flat repository' do - it 'detects a flat debian repository' do + context 'debian repository' do + it 'detects a debian repository' do stub_request(:head, repomd_url).to_return(status: 404, body: '', headers: {}) stub_request(:head, debian_url).to_return(status: 200, body: '', headers: {}) @@ -77,6 +79,19 @@ end end + context 'local file as URI' do + let(:url) { 'file:///test/export/SUSE/Products/SLE-Product-SLES/15-SP4/x86_64/product/' } + let(:path_repomd) { File.join(URI.join(url).path, 'repodata/repomd.xml') } + let(:path_debian) { File.join(URI.join(url).path, 'Release') } + + it 'continues checking for repository type' do + allow(File).to receive(:exist?).with(path_repomd).and_return(false) + allow(File).to receive(:exist?).with(path_debian).and_return(true) + + expect(mirror.repository_type).to eq(:debian) + end + end + context 'unknown repository type' do it 'raises if a unknown repository type is detected' do stub_request(:head, repomd_url).to_return(status: 404, body: '', headers: {}) @@ -86,4 +101,39 @@ end end end + + describe '#mirror_now' do + context 'repomd repository' do + let(:repomd_mirror) { instance_double('RMT::Mirror::Repomd') } + + before do + allow(mirror).to receive(:repository_type).and_return(:repomd) + allow(RMT::Mirror::Repomd).to receive(:new).and_return(repomd_mirror) + end + + it 'creates a repomd mirror instance and calls mirror' do + expect(repomd_mirror).to receive(:mirror) + mirror.mirror_now + end + end + + context 'debian repository' do + let(:debian_mirror) { instance_double('RMT::Mirror::Debian') } + + before do + allow(mirror).to receive(:repository_type).and_return(:debian) + allow(RMT::Mirror::Debian).to receive(:new).and_return(debian_mirror) + end + + it 'creates a debian mirror instance and calls mirror' do + expect(debian_mirror).to receive(:mirror) + mirror.mirror_now + end + end + + it 'raises an exception if the repository type is unknown' do + allow(mirror).to receive(:repository_type).and_return(nil) + expect { mirror.mirror_now }.to raise_error(RMT::Mirror::Exception) + end + end end diff --git a/spec/support/matchers.rb b/spec/support/matchers.rb index ae9e73c79..999d07d9b 100644 --- a/spec/support/matchers.rb +++ b/spec/support/matchers.rb @@ -18,6 +18,7 @@ end end +# rubocop:disable Lint/StructNewOverride RSpec::Matchers.define :contain_records_like do |expected| match do |actual| record_struct = Struct.new(:local_path, :checksum, :checksum_type, :size) @@ -41,3 +42,4 @@ diffable end +# rubocop:enable Lint/StructNewOverride From d9cf5c78ef8bd60a0bd63d047382b4239b0674ef Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Thu, 18 Jan 2024 10:47:08 +0100 Subject: [PATCH 3/4] I'm a not working commit! --- lib/rmt/mirror/debian.rb | 18 ++++++---- .../Packages.gz} | Bin .../debian/{Nested_Release => nested/Release} | 0 spec/lib/rmt/mirror/debian_spec.rb | 32 ++++++++++++------ 4 files changed, 33 insertions(+), 17 deletions(-) rename spec/fixtures/files/debian/{Nested_Packages.gz => nested/Packages.gz} (100%) rename spec/fixtures/files/debian/{Nested_Release => nested/Release} (100%) diff --git a/lib/rmt/mirror/debian.rb b/lib/rmt/mirror/debian.rb index 50be91727..49da648e6 100644 --- a/lib/rmt/mirror/debian.rb +++ b/lib/rmt/mirror/debian.rb @@ -3,7 +3,7 @@ class RMT::Mirror::Debian < RMT::Mirror::Base SIGNATURE_FILE_NAME = 'Release.gpg'.freeze KEY_FILE_NAME = 'Release.key'.freeze INRELEASE_FILE_NAME = 'InRelease'.freeze - NESTED_REPOSITORY_REGEX = %r{/dists/\w*/$}.freeze + NESTED_REPOSITORY_REGEX = %r{/dists/.*/$}.freeze DETECT_NONMANDATORY_FILES = %r{/(Packages|Sources|Translation)(-\w+)?$/}.freeze def mirror_implementation @@ -49,12 +49,6 @@ def mirror_packages(metadata_refs) packagelists.each do |packagelist| parse_package_list(packagelist).each do |ref| - # In a nested debian repository stucture, the metadata and packages are stored in different locations - # so we need to update the base_url if we encounter the nested structure - # We assume that if the base_url contains '/dists/', it's a nested debian structure - if ref.base_url.match?(NESTED_REPOSITORY_REGEX) - ref.base_url.sub!(NESTED_REPOSITORY_REGEX, '/') - end enqueue(ref) if need_to_download?(ref) end end @@ -76,6 +70,16 @@ def parse_package_list(packagelist) ref.size = current[:size].to_i ref.type = :deb + # In a nested debian repository stucture, the metadata and packages are stored in different locations + # so we need to update the base_url if we encounter the nested structure + # We assume that if the base_url contains '/dists/', it's a nested debian structure + # + # FIXME: Is there a better way to detect a nested structure? + if ref.base_url.match?(NESTED_REPOSITORY_REGEX) + ref.base_url.sub!(NESTED_REPOSITORY_REGEX, '/') + ref.base_dir.sub!(NESTED_REPOSITORY_REGEX, '/') + end + packages << ref current = {} end diff --git a/spec/fixtures/files/debian/Nested_Packages.gz b/spec/fixtures/files/debian/nested/Packages.gz similarity index 100% rename from spec/fixtures/files/debian/Nested_Packages.gz rename to spec/fixtures/files/debian/nested/Packages.gz diff --git a/spec/fixtures/files/debian/Nested_Release b/spec/fixtures/files/debian/nested/Release similarity index 100% rename from spec/fixtures/files/debian/Nested_Release rename to spec/fixtures/files/debian/nested/Release diff --git a/spec/lib/rmt/mirror/debian_spec.rb b/spec/lib/rmt/mirror/debian_spec.rb index 398274202..704e2b912 100644 --- a/spec/lib/rmt/mirror/debian_spec.rb +++ b/spec/lib/rmt/mirror/debian_spec.rb @@ -2,12 +2,7 @@ describe RMT::Mirror::Debian do subject(:debian) { described_class.new(**debian_mirror_configuration) } - - let(:repository) do - create :repository, - name: 'HYPE product repository debian 15.3', - external_url: 'https://updates.suse.com/sample/repository/15.3/' - end + let(:repository) { create :repository, :debian_flat } # Configuration for Debian mirroring instance let(:mirroring_base_dir) { '/test/repository/base/path/' } @@ -155,20 +150,37 @@ end it 'download packages to disk' do + allow(debian).to receive(:parse_package_list).with(packages_ref).and_call_original + expect(debian).to receive(:enqueue).exactly(4).times - expect(debian).to receive(:parse_package_list).with(packages_ref).and_call_original expect(debian).to receive(:download_enqueued) debian.mirror_packages([packages_ref, non_package_ref]) end it 'does not download the file if not needed' do - expect(debian).to receive(:need_to_download?).exactly(3).times.and_return(true) - expect(debian).to receive(:need_to_download?).with(any_args) { |ref| ref.size == 206796 }.and_return(false) + allow(debian).to receive(:need_to_download?).exactly(3).times.and_return(true) + allow(debian).to receive(:need_to_download?).with(any_args) { |ref| ref.size == 206796 }.and_return(false) + allow(debian).to receive(:parse_package_list).with(packages_ref).and_call_original + expect(debian).to receive(:enqueue).exactly(3).times - expect(debian).to receive(:parse_package_list).with(packages_ref).and_call_original expect(debian).to receive(:download_enqueued) + debian.mirror_packages([packages_ref]) end + + context 'nested debian repository' do + let(:fixture) { 'nested/Packages.gz' } + let(:repository) { create :repository, :debian } + + it 'downloads packages but saves them to altered base directory', focus: true do + allow(debian).to receive(:need_to_download?).and_return(true) + + expect(debian).to receive(:enqueue).with(base_dir_matches(/foo/)).exactly(35).times + expect(debian).to receive(:download_enqueued) + + debian.mirror_packages([packages_ref]) + end + end end describe '#parse_package_list' do From 546dd3fe51f589a18ffb26cc3378b27013dae8c4 Mon Sep 17 00:00:00 2001 From: Luis Caparroz Date: Thu, 18 Jan 2024 15:32:11 +0100 Subject: [PATCH 4/4] Fix cache_dir for nested Debian repos --- lib/rmt/mirror/debian.rb | 9 +-- spec/lib/rmt/mirror/debian_spec.rb | 55 +++++++++++-------- spec/lib/rmt/mirror/suma_product_tree_spec.rb | 2 +- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/lib/rmt/mirror/debian.rb b/lib/rmt/mirror/debian.rb index 49da648e6..009623e94 100644 --- a/lib/rmt/mirror/debian.rb +++ b/lib/rmt/mirror/debian.rb @@ -73,11 +73,12 @@ def parse_package_list(packagelist) # In a nested debian repository stucture, the metadata and packages are stored in different locations # so we need to update the base_url if we encounter the nested structure # We assume that if the base_url contains '/dists/', it's a nested debian structure - # - # FIXME: Is there a better way to detect a nested structure? if ref.base_url.match?(NESTED_REPOSITORY_REGEX) - ref.base_url.sub!(NESTED_REPOSITORY_REGEX, '/') - ref.base_dir.sub!(NESTED_REPOSITORY_REGEX, '/') + ref.tap do |r| + r.base_url.sub!(NESTED_REPOSITORY_REGEX, '/') + r.base_dir.sub!(NESTED_REPOSITORY_REGEX, '/') + r.cache_dir.sub!(NESTED_REPOSITORY_REGEX, '/') + end end packages << ref diff --git a/spec/lib/rmt/mirror/debian_spec.rb b/spec/lib/rmt/mirror/debian_spec.rb index 704e2b912..0796c5ee5 100644 --- a/spec/lib/rmt/mirror/debian_spec.rb +++ b/spec/lib/rmt/mirror/debian_spec.rb @@ -2,7 +2,12 @@ describe RMT::Mirror::Debian do subject(:debian) { described_class.new(**debian_mirror_configuration) } - let(:repository) { create :repository, :debian_flat } + + let(:repository) do + create :repository, + name: 'HYPE product repository debian 15.3', + external_url: 'https://updates.suse.com/sample/repository/15.3/' + end # Configuration for Debian mirroring instance let(:mirroring_base_dir) { '/test/repository/base/path/' } @@ -112,7 +117,7 @@ end context 'nested debian repository' do - let(:release_fixture) { 'Nested_Release' } + let(:release_fixture) { 'nested/Release' } before do allow(debian).to receive(:download_cached!).and_return(release_ref) @@ -150,37 +155,20 @@ end it 'download packages to disk' do - allow(debian).to receive(:parse_package_list).with(packages_ref).and_call_original - expect(debian).to receive(:enqueue).exactly(4).times + expect(debian).to receive(:parse_package_list).with(packages_ref).and_call_original expect(debian).to receive(:download_enqueued) debian.mirror_packages([packages_ref, non_package_ref]) end it 'does not download the file if not needed' do - allow(debian).to receive(:need_to_download?).exactly(3).times.and_return(true) - allow(debian).to receive(:need_to_download?).with(any_args) { |ref| ref.size == 206796 }.and_return(false) - allow(debian).to receive(:parse_package_list).with(packages_ref).and_call_original - + expect(debian).to receive(:need_to_download?).exactly(3).times.and_return(true) + expect(debian).to receive(:need_to_download?).with(any_args) { |ref| ref.size == 206796 }.and_return(false) expect(debian).to receive(:enqueue).exactly(3).times + expect(debian).to receive(:parse_package_list).with(packages_ref).and_call_original expect(debian).to receive(:download_enqueued) - debian.mirror_packages([packages_ref]) end - - context 'nested debian repository' do - let(:fixture) { 'nested/Packages.gz' } - let(:repository) { create :repository, :debian } - - it 'downloads packages but saves them to altered base directory', focus: true do - allow(debian).to receive(:need_to_download?).and_return(true) - - expect(debian).to receive(:enqueue).with(base_dir_matches(/foo/)).exactly(35).times - expect(debian).to receive(:download_enqueued) - - debian.mirror_packages([packages_ref]) - end - end end describe '#parse_package_list' do @@ -203,6 +191,27 @@ expect { debian.parse_package_list(packages_ref) }.to raise_error(RMT::Mirror::Exception, /unexpected end of file/) end end + + context 'nested repository structure' do + let(:fixture) { 'nested/Packages.gz' } + let(:repository) do + create :repository, + name: 'HYPE product repository debian 15.3', + external_url: 'https://ppa.launchpadcontent.net/ondrej/nginx/ubuntu/dists/focal/' + end + + it 'removes dists/ from mirroring path and external URL' do + packages = debian.parse_package_list(packages_ref) + + expect(packages).to all( + have_attributes( + base_url: 'https://ppa.launchpadcontent.net/ondrej/nginx/ubuntu/', + base_dir: mirroring_base_dir + 'ondrej/nginx/ubuntu/', + cache_dir: mirroring_base_dir + 'ondrej/nginx/ubuntu/' + ) + ) + end + end end describe '#parse_release_file' do diff --git a/spec/lib/rmt/mirror/suma_product_tree_spec.rb b/spec/lib/rmt/mirror/suma_product_tree_spec.rb index 495141d34..e010c441d 100644 --- a/spec/lib/rmt/mirror/suma_product_tree_spec.rb +++ b/spec/lib/rmt/mirror/suma_product_tree_spec.rb @@ -23,7 +23,7 @@ describe '#mirror' do before do - allow(suma).to receive(:downloader).and_return downloader + allow(RMT::Downloader).to receive(:new).and_return downloader end it 'mirrors the product_tree file' do