From 6556d0b48ad32d40032f33d9271ee1b009350972 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Fri, 25 Jul 2025 20:27:20 +0200 Subject: [PATCH 1/2] chore(cubestore): ClusterSend chunking by partition count --- rust/cubestore/cubestore/src/config/mod.rs | 12 ++++ .../src/queryplanner/query_executor.rs | 68 +++++++++++++++++-- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/rust/cubestore/cubestore/src/config/mod.rs b/rust/cubestore/cubestore/src/config/mod.rs index fb433a3fd452c..a6b39d9ca4a0c 100644 --- a/rust/cubestore/cubestore/src/config/mod.rs +++ b/rust/cubestore/cubestore/src/config/mod.rs @@ -364,6 +364,8 @@ pub struct Config { pub trait ConfigObj: DIService { fn partition_split_threshold(&self) -> u64; + fn cluster_send_split_threshold(&self) -> u64; + fn partition_size_split_threshold_bytes(&self) -> u64; fn max_partition_split_threshold(&self) -> u64; @@ -555,6 +557,7 @@ pub trait ConfigObj: DIService { #[derive(Debug, Clone)] pub struct ConfigObjImpl { pub partition_split_threshold: u64, + pub cluster_send_split_threshold: u64, pub partition_size_split_threshold_bytes: u64, pub max_partition_split_threshold: u64, pub compaction_chunks_total_size_threshold: u64, @@ -662,6 +665,10 @@ impl ConfigObj for ConfigObjImpl { self.partition_split_threshold } + fn cluster_send_split_threshold(&self) -> u64 { + self.cluster_send_split_threshold + } + fn partition_size_split_threshold_bytes(&self) -> u64 { self.partition_size_split_threshold_bytes } @@ -1242,6 +1249,10 @@ impl Config { "CUBESTORE_PARTITION_SPLIT_THRESHOLD", 1048576 * 2, ), + cluster_send_split_threshold: env_parse( + "CUBESTORE_CLUSTER_SEND_SPLIT_THRESHOLD", + 4, + ), partition_size_split_threshold_bytes: env_parse_size( "CUBESTORE_PARTITION_SIZE_SPLIT_THRESHOLD", 100 * 1024 * 1024, @@ -1597,6 +1608,7 @@ impl Config { .join(format!("{}-local-store", name)), dump_dir: None, partition_split_threshold: 20, + cluster_send_split_threshold: 4, partition_size_split_threshold_bytes: 2 * 1024, max_partition_split_threshold: 20, compaction_chunks_count_threshold: 1, diff --git a/rust/cubestore/cubestore/src/queryplanner/query_executor.rs b/rust/cubestore/cubestore/src/queryplanner/query_executor.rs index 4bf2755c49add..eae412340458a 100644 --- a/rust/cubestore/cubestore/src/queryplanner/query_executor.rs +++ b/rust/cubestore/cubestore/src/queryplanner/query_executor.rs @@ -1104,7 +1104,7 @@ impl ClusterSendExec { ps } - fn issue_filters(ps: &[IdRow]) -> Vec<(u64, RowRange)> { + fn issue_filters(ps: &[IdRow]) -> Vec<(IdRow, RowRange)> { if ps.is_empty() { return Vec::new(); } @@ -1114,7 +1114,7 @@ impl ClusterSendExec { if multi_id.is_none() { return ps .iter() - .map(|p| (p.get_id(), RowRange::default())) + .map(|p| (p.clone(), RowRange::default())) .collect(); } let filter = RowRange { @@ -1129,7 +1129,7 @@ impl ClusterSendExec { } else { filter.clone() }; - r.push((p.get_id(), pf)) + r.push((p.clone(), pf)) } r } @@ -1138,7 +1138,8 @@ impl ClusterSendExec { c: &dyn ConfigObj, logical: Vec>, ) -> Vec<(String, (Vec<(u64, RowRange)>, Vec))> { - let mut m: HashMap<_, (Vec<(u64, RowRange)>, Vec)> = HashMap::new(); + let mut m: HashMap<_, (Vec<(IdRow, RowRange)>, Vec)> = + HashMap::new(); for ps in &logical { let inline_table_ids = ps .iter() @@ -1178,7 +1179,64 @@ impl ClusterSendExec { let mut r = m.into_iter().collect_vec(); r.sort_unstable_by(|l, r| l.0.cmp(&r.0)); - r + r.into_iter() + .map(|(worker, data)| { + let splitted = Self::split_worker_parititons(c, data); + splitted.into_iter().map(move |data| (worker.clone(), data)) + }) + .flatten() + .collect_vec() + } + + fn split_worker_parititons( + c: &dyn ConfigObj, + partitions: (Vec<(IdRow, RowRange)>, Vec), + ) -> Vec<(Vec<(u64, RowRange)>, Vec)> { + if !partitions.1.is_empty() + || partitions + .0 + .iter() + .any(|(p, _)| p.get_row().multi_partition_id().is_some()) + { + return vec![( + partitions + .0 + .into_iter() + .map(|(p, range)| (p.id, range)) + .collect_vec(), + partitions.1, + )]; + } + let rows_split_threshold = c.partition_split_threshold() * c.cluster_send_split_threshold(); + let file_size_split_threshold = + c.partition_size_split_threshold_bytes() * c.cluster_send_split_threshold(); + let mut result = vec![]; + let mut current_rows = 0; + let mut current_files_size = 0; + let mut current_chunk = vec![]; + let (partitions, _) = partitions; + for (partition, range) in partitions { + let rows = partition.get_row().main_table_row_count(); + let file_size = partition.get_row().file_size().unwrap_or_default(); + if current_rows + rows > rows_split_threshold + || current_files_size + file_size > file_size_split_threshold + { + if !current_chunk.is_empty() { + result.push((std::mem::take(&mut current_chunk), vec![])); + current_rows = 0; + current_files_size = 0; + } + } + + current_rows += rows; + current_files_size += file_size; + current_chunk.push((partition.id, range)); + } + if !current_chunk.is_empty() { + result.push((current_chunk, vec![])); + } + + result } pub fn with_changed_schema( From 77494ad6c896a06f971a4a9a4167051da00f5f12 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Mon, 28 Jul 2025 18:22:53 +0200 Subject: [PATCH 2/2] in work --- .../metastore-1753719230483/000010.sst | Bin 0 -> 1157 bytes .../metastore-1753719230483/000012.sst | Bin 0 -> 1161 bytes .../metastore-1753719230483/CURRENT | 1 + .../metastore-1753719230483/MANIFEST-000005 | Bin 0 -> 484 bytes .../metastore-1753719230483/OPTIONS-000007 | 198 ++++++++++ .../metastore-1753719231346/000010.sst | Bin 0 -> 1157 bytes .../metastore-1753719231346/000012.sst | Bin 0 -> 1161 bytes .../metastore-1753719231346/000014.sst | Bin 0 -> 1161 bytes .../metastore-1753719231346/CURRENT | 1 + .../metastore-1753719231346/MANIFEST-000005 | Bin 0 -> 628 bytes .../metastore-1753719231346/OPTIONS-000007 | 198 ++++++++++ .../metastore-current | 1 + .../metastore/000010.sst | Bin 0 -> 1157 bytes .../metastore/000012.sst | Bin 0 -> 1161 bytes .../metastore/000013.log | 0 .../metastore/000014.sst | Bin 0 -> 1161 bytes .../metastore/CURRENT | 1 + .../metastore/IDENTITY | 1 + .../delete_old_snapshots-local/metastore/LOCK | 0 .../delete_old_snapshots-local/metastore/LOG | 344 ++++++++++++++++++ .../metastore/MANIFEST-000005 | Bin 0 -> 628 bytes .../metastore/OPTIONS-000007 | 198 ++++++++++ .../metastore/archive/000004.log | Bin 0 -> 127 bytes .../metastore/archive/000008.log | Bin 0 -> 127 bytes .../metastore/archive/000011.log | Bin 0 -> 127 bytes .../metastore-1753719230483/000010.sst | Bin 0 -> 1157 bytes .../metastore-1753719230483/000012.sst | Bin 0 -> 1161 bytes .../metastore-1753719230483/CURRENT | 1 + .../metastore-1753719230483/MANIFEST-000005 | Bin 0 -> 484 bytes .../metastore-1753719230483/OPTIONS-000007 | 198 ++++++++++ .../metastore-1753719231346/000010.sst | Bin 0 -> 1157 bytes .../metastore-1753719231346/000012.sst | Bin 0 -> 1161 bytes .../metastore-1753719231346/000014.sst | Bin 0 -> 1161 bytes .../metastore-1753719231346/CURRENT | 1 + .../metastore-1753719231346/MANIFEST-000005 | Bin 0 -> 628 bytes .../metastore-1753719231346/OPTIONS-000007 | 198 ++++++++++ .../metastore-current | 1 + .../src/queryplanner/query_executor.rs | 27 +- 38 files changed, 1355 insertions(+), 14 deletions(-) create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/000010.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/000012.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/CURRENT create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/MANIFEST-000005 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/OPTIONS-000007 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/000010.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/000012.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/000014.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/CURRENT create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/MANIFEST-000005 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/OPTIONS-000007 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore-current create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000010.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000012.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000013.log create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000014.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/CURRENT create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/IDENTITY create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/LOCK create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/LOG create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/MANIFEST-000005 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/OPTIONS-000007 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000004.log create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000008.log create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000011.log create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/000010.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/000012.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/CURRENT create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/MANIFEST-000005 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/OPTIONS-000007 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/000010.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/000012.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/000014.sst create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/CURRENT create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/MANIFEST-000005 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/OPTIONS-000007 create mode 100644 rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-current diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/000010.sst b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/000010.sst new file mode 100644 index 0000000000000000000000000000000000000000..dcbbd6019113dfcfca0887d3bce87f01d55e330c GIT binary patch literal 1157 zcmaJ=&ubGw6rO2Qvq{tRN7@R7LJCE^WH4=-6iN@a*jl8vH;W=+cK0P6y1Nr*CQZ^q zp$9?1izmT@Ac%rTFCIiM>cNvI{{g{+cfqS~Hb1l=4tv=7=KJ1z^WHaPeuff)-}y!R zNW?=RiJFu}LNk=o^z_goC1cKz74=CyEhY1bVZ`Zv{mI$ui*H}{@jfpOgJ8VByjgod zX8y@19jlC4-yeS}lEs9Q-jViQ<_7q~T&}@qhPweU5&6(&rqckcnVX;u1vSxTH4zw4 zBB~eXasaGDybu|5tN2LQER zT+63&iwd~G<}kdI@F6n7bhuYq3{zlcGL1oM~|ar>Rj20T`Sx4onkgY6NEq%!^Czsoq=Fs6%nYv52IzwB1#(0HV4% zB&jhfRmcrH24%-XoEh3Ri}iuf4BMk@3Ytw&oA^?8mLBtO=}ZD7@kQ}vq{tRBW(|bLJGxO2Ggb=P;Zj*51&mY_& zlmE0Q9c!OheciwNgv_Ot^bd95F)zdo^SFk98SaI^L>xeynN9;NXL^)26x2kU)kJ7O ziMU>wF4ES4MAJj)`~^Fs)R$p1idl_EBJ3~`kg0UW792p0H$#(irlST|+(;F|5ri0j zydJ;eT(>Ys*>XK$^jf)6T`eWv))wZMmzGyc z+F%5zch)YgmLlGE1K}Ij!#LU^$Jzw0h0#jHe{hIx^}%^kvsMH3$EK;lP56n+!~v+a z%4#WFoKwIJHiuzNB7oQk)8T$~g;#v994z`v<*K*fm%YX1N_mN|EOX#XzPB(xHavD9 zwdr%VxxX&uVR=_chNI!(*yx1rwxe%zb8D|UzjtkGXY=ay?ds0T#;wh*LS|6L%!mli z-aLJK!O7Bq8_u*ihLh|Ng%Au*7zd__Gd_s31m-6x_gJqjYsjHE;)FzU1={X1SO{_5 z9FXiVl`7(f9f7jrA(4;spin55na9867O3A%5LgQw{l3Eg}YMRebY_Afa2l55lRi)cQNPdK( zyE>FH93)yH&d8t)1Hc~JH2WRO7O4_-5%bVW7#N1}K;@mush-}UjRW>E6=oIu`@L)k z?8_1a)hv;CL&>g@9eGTwiQ4VAdU lx}MR~Oy6qd)AslD3upEuAzJu9%l>12zWZ_d^76;~zW@WCSGWKG literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/CURRENT b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/CURRENT new file mode 100644 index 0000000000000..aa5bb8ea50905 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/CURRENT @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/MANIFEST-000005 b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719230483/MANIFEST-000005 new file mode 100644 index 0000000000000000000000000000000000000000..fb545617fe84779c7bb59ca41c5f7eceb52753dc GIT binary patch literal 484 zcmZS8)^KKEU<~`Yer`cxQDRAc(HCZ(C>91r zCI%LUKRRkZ*%%l(8JO8v7-l$c-CG3|=KylKSeVlpI6FCI8G&XppaDjpUNDel2Ffx9 zq|a*HB))q&R23s5P!t3hnORx)9-eoEjg565h~i~rWM|-D56#QY%P-I45pe3PZ2fV0 z^_ts>WeKffThB3m+PjFG1MC_uE)Evv`({NB)nG0U3o{!7XF3B{D=r5?Y-GUUAgC+@ z191+z0dvseKmMhl5a9-f2rmm;Is*@m5Mg3uh1dacz{NMc?eWR|FQN7_F|t8eENrZh epkQO&4+#+_tRZp_<^VcNvI{{g{+cfqS~Hb1l=4tv=7=KJ1z^WHaPeuff)-}y!R zNW?=RiJFu}LNk=o^z_goC1cKz74=CyEhY1bVZ`Zv{mI$ui*H}{@jfpOgJ8VByjgod zX8y@19jlC4-yeS}lEs9Q-jViQ<_7q~T&}@qhPweU5&6(&rqckcnVX;u1vSxTH4zw4 zBB~eXasaGDybu|5tN2LQER zT+63&iwd~G<}kdI@F6n7bhuYq3{zlcGL1oM~|ar>Rj20T`Sx4onkgY6NEq%!^Czsoq=Fs6%nYv52IzwB1#(0HV4% zB&jhfRmcrH24%-XoEh3Ri}iuf4BMk@3Ytw&oA^?8mLBtO=}ZD7@kQ}vq{tRBW(|bLJGxO2Ggb=P;Zj*51&mY_& zlmE0Q9c!OheciwNgv_Ot^bd95F)zdo^SFk98SaI^L>xeynN9;NXL^)26x2kU)kJ7O ziMU>wF4ES4MAJj)`~^Fs)R$p1idl_EBJ3~`kg0UW792p0H$#(irlST|+(;F|5ri0j zydJ;eT(>Ys*>XK$^jf)6T`eWv))wZMmzGyc z+F%5zch)YgmLlGE1K}Ij!#LU^$Jzw0h0#jHe{hIx^}%^kvsMH3$EK;lP56n+!~v+a z%4#WFoKwIJHiuzNB7oQk)8T$~g;#v994z`v<*K*fm%YX1N_mN|EOX#XzPB(xHavD9 zwdr%VxxX&uVR=_chNI!(*yx1rwxe%zb8D|UzjtkGXY=ay?ds0T#;wh*LS|6L%!mli z-aLJK!O7Bq8_u*ihLh|Ng%Au*7zd__Gd_s31m-6x_gJqjYsjHE;)FzU1={X1SO{_5 z9FXiVl`7(f9f7jrA(4;spin55na9867O3A%5LgQw{l3Eg}YMRebY_Afa2l55lRi)cQNPdK( zyE>FH93)yH&d8t)1Hc~JH2WRO7O4_-5%bVW7#N1}K;@mush-}UjRW>E6=oIu`@L)k z?8_1a)hv;CL&>g@9eGTwiQ4VAdU lx}MR~Oy6qd)AslD3upEuAzJu9%l>12zWZ_d^76;~zW@WCSGWKG literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/000014.sst b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/000014.sst new file mode 100644 index 0000000000000000000000000000000000000000..380ef9fbeaf8b33a10308a67b2d18f304f93b570 GIT binary patch literal 1161 zcmaJ>&1(}u6rX8Qvx!OCnzo06kV5g6!8GXyl!71FTBX*TqR2A4`;rdb-3c?3Ch^dl z;MG6Co&;}t^x(mR;#Cm;0>Psv6}g=KbF9ef-{%vvZUXd`~_Z>J1Gk zPk>}wQZ@)JQc4T66HAnoya@}Uxf7@iIh)O(ogVw84=N`?nI3Rn>1fdyPc#0(E&U?Pp6&&;3$7PBx-I|^E&&srij zphViPFH~r6U!v;)4E{2E(=?QEH%VEGCn6p&5s~?9&K4X&i+5vF@D`E|SKLSy!y&}T ze`#Q#Cp27^zi*1Uk^-%TOm9eQ9eq^1N7R=cgLd7+9A(cBgwdO|dUK=dqHQ*o*H+gy zs@fm}vwHm`;iEZA0UsmmiyX7zBQwlNdMR&wbe%_LETmh)E}R&2H)XbNEZOq zW__cYuPiCx2AjjMCJ{kugcWW=)mk%XgtcI0tzKK@>uVhNY8W(@XG$~s zQkyYmmxq_6JgD7NlHq80JTpD3hwT{q+}gR_T)usEXK(w;_1)&)`qrK8opNqcrp$;0 zUcEkkbIQxph#StdxDQA9(-dMbIAI)^Cf=DzoF%Z(rTjPQv`2Bo4nzuN+8-iV3~Af! zlYEIvm2krjLD}&TZ;lSdVs#)i!}cgkLCaZciO-ei=^>B1s42}Nh%p~XLm5Fty)sQe zwSmbFD4s@|R5H^jiv$e!H6Ebq?!76Rb`$1maZ=T^n4{QUB|vA2Wja))KeCYG6vc3D zs39FVtq^Z&QpOQrk8PUc4&^IUiMB|2Y$Xf~X#!AXZ*G31cj)7QLrjHP1^?ccErDIR zIoybPqcAgJ3orwQ%8$W%>$~^7A|}yubhZuK#lol!_G@vxi9en{eoxMfH?gB!D6H!V kec=8BZeQ0w?S9Ta^A?XB(1QQ-Oyb@5cV8d8xcKqmFIer`cxQDRAc(HCZ(C>91r zCI%LUKRRkZ*%%l(8JO8v7-l$c-CG3|=KylKSeVlpI6FCI8G&XppaDjpUNDel2Ffx9 zq|a*HB))q&R23s5P!t3hnORx)9-eoEjg565h~i~rWM|-D56#QY%P-I45pe3PZ2fV0 z^_ts>WeKffThB3m+PjFG1MC_uE)Evv`({NB)nG0U3o{!7XF3B{D=r5?Y-GUUAgC+@ z191+z0dvseKmMhl5a9-f2rmm;Is*@m5Mg3uh1dacz{NMc?eWR|FQN7_F|t8eENrZh zpkQO&4+#+_tRZp_<^VwEWiNa!{HugMh;~6%!}u} pvU~Y4E~tB$fe{V{?3}C+hrr##jMY7N;O<$cI1Az)elA`XP5|FWf`I@4 literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/OPTIONS-000007 b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/OPTIONS-000007 new file mode 100644 index 0000000000000..d2b2e9bfc5695 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-1753719231346/OPTIONS-000007 @@ -0,0 +1,198 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=7.10.2 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=600 + avoid_flush_during_shutdown=false + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + wal_compression=kNoCompression + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + allow_ingest_behind=false + fail_if_options_file_error=false + persist_stats_to_disk=false + WAL_ttl_seconds=330 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=1000 + table_cache_numshardbits=6 + max_file_opening_threads=16 + use_fsync=false + unordered_write=false + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + track_and_verify_wals_in_manifest=false + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=0 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + memtable_protection_bytes_per_key=0 + bottommost_compression=kNoCompression + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kNoCompression + level0_file_num_compaction_trigger=4 + prefix_extractor=rocksdb.FixedPrefix.13 + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + compression_opts={use_zstd_dict_trainer=true;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=0 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + preserve_internal_time_seconds=0 + force_consistency_checks=true + optimize_filters_for_hits=false + merge_operator=meta_store merge + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=false + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-current b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-current new file mode 100644 index 0000000000000..ea28456422544 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore-current @@ -0,0 +1 @@ +metastore-1753719231346 \ No newline at end of file diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000010.sst b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000010.sst new file mode 100644 index 0000000000000000000000000000000000000000..dcbbd6019113dfcfca0887d3bce87f01d55e330c GIT binary patch literal 1157 zcmaJ=&ubGw6rO2Qvq{tRN7@R7LJCE^WH4=-6iN@a*jl8vH;W=+cK0P6y1Nr*CQZ^q zp$9?1izmT@Ac%rTFCIiM>cNvI{{g{+cfqS~Hb1l=4tv=7=KJ1z^WHaPeuff)-}y!R zNW?=RiJFu}LNk=o^z_goC1cKz74=CyEhY1bVZ`Zv{mI$ui*H}{@jfpOgJ8VByjgod zX8y@19jlC4-yeS}lEs9Q-jViQ<_7q~T&}@qhPweU5&6(&rqckcnVX;u1vSxTH4zw4 zBB~eXasaGDybu|5tN2LQER zT+63&iwd~G<}kdI@F6n7bhuYq3{zlcGL1oM~|ar>Rj20T`Sx4onkgY6NEq%!^Czsoq=Fs6%nYv52IzwB1#(0HV4% zB&jhfRmcrH24%-XoEh3Ri}iuf4BMk@3Ytw&oA^?8mLBtO=}ZD7@kQ}vq{tRBW(|bLJGxO2Ggb=P;Zj*51&mY_& zlmE0Q9c!OheciwNgv_Ot^bd95F)zdo^SFk98SaI^L>xeynN9;NXL^)26x2kU)kJ7O ziMU>wF4ES4MAJj)`~^Fs)R$p1idl_EBJ3~`kg0UW792p0H$#(irlST|+(;F|5ri0j zydJ;eT(>Ys*>XK$^jf)6T`eWv))wZMmzGyc z+F%5zch)YgmLlGE1K}Ij!#LU^$Jzw0h0#jHe{hIx^}%^kvsMH3$EK;lP56n+!~v+a z%4#WFoKwIJHiuzNB7oQk)8T$~g;#v994z`v<*K*fm%YX1N_mN|EOX#XzPB(xHavD9 zwdr%VxxX&uVR=_chNI!(*yx1rwxe%zb8D|UzjtkGXY=ay?ds0T#;wh*LS|6L%!mli z-aLJK!O7Bq8_u*ihLh|Ng%Au*7zd__Gd_s31m-6x_gJqjYsjHE;)FzU1={X1SO{_5 z9FXiVl`7(f9f7jrA(4;spin55na9867O3A%5LgQw{l3Eg}YMRebY_Afa2l55lRi)cQNPdK( zyE>FH93)yH&d8t)1Hc~JH2WRO7O4_-5%bVW7#N1}K;@mush-}UjRW>E6=oIu`@L)k z?8_1a)hv;CL&>g@9eGTwiQ4VAdU lx}MR~Oy6qd)AslD3upEuAzJu9%l>12zWZ_d^76;~zW@WCSGWKG literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000013.log b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000013.log new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000014.sst b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000014.sst new file mode 100644 index 0000000000000000000000000000000000000000..380ef9fbeaf8b33a10308a67b2d18f304f93b570 GIT binary patch literal 1161 zcmaJ>&1(}u6rX8Qvx!OCnzo06kV5g6!8GXyl!71FTBX*TqR2A4`;rdb-3c?3Ch^dl z;MG6Co&;}t^x(mR;#Cm;0>Psv6}g=KbF9ef-{%vvZUXd`~_Z>J1Gk zPk>}wQZ@)JQc4T66HAnoya@}Uxf7@iIh)O(ogVw84=N`?nI3Rn>1fdyPc#0(E&U?Pp6&&;3$7PBx-I|^E&&srij zphViPFH~r6U!v;)4E{2E(=?QEH%VEGCn6p&5s~?9&K4X&i+5vF@D`E|SKLSy!y&}T ze`#Q#Cp27^zi*1Uk^-%TOm9eQ9eq^1N7R=cgLd7+9A(cBgwdO|dUK=dqHQ*o*H+gy zs@fm}vwHm`;iEZA0UsmmiyX7zBQwlNdMR&wbe%_LETmh)E}R&2H)XbNEZOq zW__cYuPiCx2AjjMCJ{kugcWW=)mk%XgtcI0tzKK@>uVhNY8W(@XG$~s zQkyYmmxq_6JgD7NlHq80JTpD3hwT{q+}gR_T)usEXK(w;_1)&)`qrK8opNqcrp$;0 zUcEkkbIQxph#StdxDQA9(-dMbIAI)^Cf=DzoF%Z(rTjPQv`2Bo4nzuN+8-iV3~Af! zlYEIvm2krjLD}&TZ;lSdVs#)i!}cgkLCaZciO-ei=^>B1s42}Nh%p~XLm5Fty)sQe zwSmbFD4s@|R5H^jiv$e!H6Ebq?!76Rb`$1maZ=T^n4{QUB|vA2Wja))KeCYG6vc3D zs39FVtq^Z&QpOQrk8PUc4&^IUiMB|2Y$Xf~X#!AXZ*G31cj)7QLrjHP1^?ccErDIR zIoybPqcAgJ3orwQ%8$W%>$~^7A|}yubhZuK#lol!_G@vxi9en{eoxMfH?gB!D6H!V kec=8BZeQ0w?S9Ta^A?XB(1QQ-Oyb@5cV8d8xcKqmFI 1157 bytes +2025/07/28-18:13:50.488327 6151155712 (Original Log Time 2025/07/28-18:13:50.488261) [db/compaction/compaction_job.cc:885] [default] compacted to: files[1 0 0 0 0 0 0] max score 0.25, MB/sec: 0.8 rd, 0.8 wr, level 0, files in(0, 1) out(1 +0 blob) MB in(0.0, 0.0 +0.0 blob) out(0.0 +0.0 blob), read-write-amplify(0.0) write-amplify(0.0) OK, records in: 3, records dropped: 0 output_compression: NoCompression +2025/07/28-18:13:50.488329 6151155712 (Original Log Time 2025/07/28-18:13:50.488283) EVENT_LOG_v1 {"time_micros": 1753719230488268, "job": 3, "event": "compaction_finished", "compaction_time_micros": 1421, "compaction_time_cpu_micros": 870, "output_level": 0, "num_output_files": 1, "total_output_size": 1157, "num_input_records": 3, "num_output_records": 3, "num_subcompactions": 1, "output_compression": "NoCompression", "num_single_delete_mismatches": 0, "num_single_delete_fallthrough": 0, "lsm_state": [1, 0, 0, 0, 0, 0, 0]} +2025/07/28-18:13:51.315775 6152302592 (Original Log Time 2025/07/28-18:13:51.315764) [db/db_impl/db_impl_compaction_flush.cc:2965] Calling FlushMemTableToOutputFile with column family [default], flush slots available 1, compaction slots available 1, flush slots scheduled 1, compaction slots scheduled 0 +2025/07/28-18:13:51.315777 6152302592 [db/flush_job.cc:861] [default] [JOB 4] Flushing memtable with next log file: 11 +2025/07/28-18:13:51.315785 6152302592 EVENT_LOG_v1 {"time_micros": 1753719231315781, "job": 4, "event": "flush_started", "num_memtables": 1, "num_entries": 3, "num_deletes": 0, "total_data_size": 110, "memory_usage": 840, "flush_reason": "Get Live Files"} +2025/07/28-18:13:51.315786 6152302592 [db/flush_job.cc:890] [default] [JOB 4] Level-0 flush table #12: started +2025/07/28-18:13:51.317126 6152302592 EVENT_LOG_v1 {"time_micros": 1753719231317076, "cf_name": "default", "job": 4, "event": "table_file_creation", "file_number": 12, "file_size": 1161, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 4, "largest_seqno": 6, "table_properties": {"data_size": 126, "index_size": 38, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 0, "raw_key_size": 71, "raw_average_key_size": 23, "raw_value_size": 33, "raw_average_value_size": 11, "num_data_blocks": 1, "num_entries": 3, "num_filter_entries": 0, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "meta_store merge", "prefix_extractor_name": "rocksdb.FixedPrefix.13", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1753719230, "oldest_key_time": 1753719230, "file_creation_time": 1753719231, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ac48a1cb-d5c6-4b3c-b571-6a87aea6cb32", "db_session_id": "WIX42XHIOGEKM4O8FZGI", "orig_file_number": 12, "seqno_to_time_mapping": "N/A"}} +2025/07/28-18:13:51.317274 6152302592 [db/flush_job.cc:1025] [default] [JOB 4] Flush lasted 1500 microseconds, and 1056 cpu microseconds. +2025/07/28-18:13:51.318041 6152302592 (Original Log Time 2025/07/28-18:13:51.317246) [db/flush_job.cc:976] [default] [JOB 4] Level-0 flush table #12: 1161 bytes OK +2025/07/28-18:13:51.318046 6152302592 (Original Log Time 2025/07/28-18:13:51.317284) [db/memtable_list.cc:521] [default] Level-0 commit table #12 started +2025/07/28-18:13:51.318048 6152302592 (Original Log Time 2025/07/28-18:13:51.317641) [db/memtable_list.cc:725] [default] Level-0 commit table #12: memtable #1 done +2025/07/28-18:13:51.318051 6152302592 (Original Log Time 2025/07/28-18:13:51.317946) EVENT_LOG_v1 {"time_micros": 1753719231317930, "job": 4, "event": "flush_finished", "output_compression": "NoCompression", "lsm_state": [2, 0, 0, 0, 0, 0, 0], "immutable_memtables": 0} +2025/07/28-18:13:51.318053 6152302592 (Original Log Time 2025/07/28-18:13:51.317986) [db/db_impl/db_impl_compaction_flush.cc:303] [default] Level summary: files[2 0 0 0 0 0 0] max score 0.50 +2025/07/28-18:13:51.318111 6204387328 [WARN] [db/db_impl/db_impl_files.cc:51] File Deletions Disabled, but already disabled. Counter: 2 +2025/07/28-18:13:51.319077 6204387328 [db/wal_manager.cc:84] Latest Archived log: 4 +2025/07/28-18:13:51.319093 6204387328 [WARN] [db/db_impl/db_impl_files.cc:91] File Deletions Enable, but not really enabled. Counter: 1 +2025/07/28-18:13:51.319102 6204387328 [db/db_filesnapshot.cc:407] Number of log files 2 +2025/07/28-18:13:51.319109 6204387328 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking 000012.sst +2025/07/28-18:13:51.319429 6204387328 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking 000010.sst +2025/07/28-18:13:51.319977 6204387328 [utilities/checkpoint/checkpoint_impl.cc:137] Copying MANIFEST-000005 +2025/07/28-18:13:51.320591 6204387328 [utilities/checkpoint/checkpoint_impl.cc:143] Creating CURRENT +2025/07/28-18:13:51.321124 6204387328 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking OPTIONS-000007 +2025/07/28-18:13:51.321967 6204387328 [db/db_impl/db_impl_files.cc:84] File Deletions Enabled +2025/07/28-18:13:51.321986 6204387328 [db/db_impl/db_impl_files.cc:471] [JOB 0] Try to delete WAL files size 113, prev total WAL file size 113, number of live WAL files 2. +2025/07/28-18:13:51.323019 6204387328 [file/delete_scheduler.cc:78] Deleted file /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000009.sst immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 +2025/07/28-18:13:51.323029 6204387328 EVENT_LOG_v1 {"time_micros": 1753719231323025, "job": 0, "event": "table_file_deletion", "file_number": 9} +2025/07/28-18:13:51.323637 6204387328 [db/wal_manager.cc:287] Move log file /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000008.log to /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000008.log -- OK +2025/07/28-18:13:51.323654 6204387328 [db/wal_manager.cc:287] Move log file /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000008.log to /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000008.log -- IO error: No such file or directory: While renaming a file to /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000008.log: /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000008.log: No such file or directory +2025/07/28-18:13:51.324102 6204387328 [utilities/checkpoint/checkpoint_impl.cc:179] Snapshot DONE. All is good +2025/07/28-18:13:51.324104 6204387328 [utilities/checkpoint/checkpoint_impl.cc:181] Snapshot sequence number: 6 +2025/07/28-18:13:51.346387 13289680896 [utilities/checkpoint/checkpoint_impl.cc:97] Started the snapshot process -- creating snapshot in directory /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/../metastore-1753719231346 +2025/07/28-18:13:51.346393 13289680896 [utilities/checkpoint/checkpoint_impl.cc:112] Snapshot process -- using temporary directory /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/../metastore-1753719231346.tmp +2025/07/28-18:13:51.346473 13289680896 [db/db_impl/db_impl_files.cc:47] File Deletions Disabled +2025/07/28-18:13:51.346710 13289680896 [db/db_impl/db_impl_write.cc:2121] [default] New memtable created with log file: #13. Immutable memtables: 0. +2025/07/28-18:13:51.559974 6152302592 (Original Log Time 2025/07/28-18:13:51.559965) [db/db_impl/db_impl_compaction_flush.cc:2965] Calling FlushMemTableToOutputFile with column family [default], flush slots available 1, compaction slots available 1, flush slots scheduled 1, compaction slots scheduled 0 +2025/07/28-18:13:51.559976 6152302592 [db/flush_job.cc:861] [default] [JOB 5] Flushing memtable with next log file: 13 +2025/07/28-18:13:51.559984 6152302592 EVENT_LOG_v1 {"time_micros": 1753719231559980, "job": 5, "event": "flush_started", "num_memtables": 1, "num_entries": 3, "num_deletes": 0, "total_data_size": 110, "memory_usage": 848, "flush_reason": "Get Live Files"} +2025/07/28-18:13:51.559985 6152302592 [db/flush_job.cc:890] [default] [JOB 5] Level-0 flush table #14: started +2025/07/28-18:13:51.561254 6152302592 EVENT_LOG_v1 {"time_micros": 1753719231561163, "cf_name": "default", "job": 5, "event": "table_file_creation", "file_number": 14, "file_size": 1161, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 7, "largest_seqno": 9, "table_properties": {"data_size": 126, "index_size": 38, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 0, "raw_key_size": 71, "raw_average_key_size": 23, "raw_value_size": 33, "raw_average_value_size": 11, "num_data_blocks": 1, "num_entries": 3, "num_filter_entries": 0, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "meta_store merge", "prefix_extractor_name": "rocksdb.FixedPrefix.13", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1753719231, "oldest_key_time": 1753719231, "file_creation_time": 1753719231, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ac48a1cb-d5c6-4b3c-b571-6a87aea6cb32", "db_session_id": "WIX42XHIOGEKM4O8FZGI", "orig_file_number": 14, "seqno_to_time_mapping": "N/A"}} +2025/07/28-18:13:51.561280 6152302592 [db/flush_job.cc:1025] [default] [JOB 5] Flush lasted 1308 microseconds, and 612 cpu microseconds. +2025/07/28-18:13:51.561425 6152302592 (Original Log Time 2025/07/28-18:13:51.561271) [db/flush_job.cc:976] [default] [JOB 5] Level-0 flush table #14: 1161 bytes OK +2025/07/28-18:13:51.561426 6152302592 (Original Log Time 2025/07/28-18:13:51.561283) [db/memtable_list.cc:521] [default] Level-0 commit table #14 started +2025/07/28-18:13:51.561427 6152302592 (Original Log Time 2025/07/28-18:13:51.561387) [db/memtable_list.cc:725] [default] Level-0 commit table #14: memtable #1 done +2025/07/28-18:13:51.561428 6152302592 (Original Log Time 2025/07/28-18:13:51.561399) EVENT_LOG_v1 {"time_micros": 1753719231561395, "job": 5, "event": "flush_finished", "output_compression": "NoCompression", "lsm_state": [3, 0, 0, 0, 0, 0, 0], "immutable_memtables": 0} +2025/07/28-18:13:51.561429 6152302592 (Original Log Time 2025/07/28-18:13:51.561412) [db/db_impl/db_impl_compaction_flush.cc:303] [default] Level summary: files[3 0 0 0 0 0 0] max score 0.75 +2025/07/28-18:13:51.561489 13289680896 [WARN] [db/db_impl/db_impl_files.cc:51] File Deletions Disabled, but already disabled. Counter: 2 +2025/07/28-18:13:51.562582 13289680896 [db/wal_manager.cc:84] Latest Archived log: 8 +2025/07/28-18:13:51.562597 13289680896 [WARN] [db/db_impl/db_impl_files.cc:91] File Deletions Enable, but not really enabled. Counter: 1 +2025/07/28-18:13:51.562628 13289680896 [db/db_filesnapshot.cc:407] Number of log files 3 +2025/07/28-18:13:51.562644 13289680896 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking 000014.sst +2025/07/28-18:13:51.563366 13289680896 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking 000012.sst +2025/07/28-18:13:51.563745 13289680896 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking 000010.sst +2025/07/28-18:13:51.564175 13289680896 [utilities/checkpoint/checkpoint_impl.cc:137] Copying MANIFEST-000005 +2025/07/28-18:13:51.565085 13289680896 [utilities/checkpoint/checkpoint_impl.cc:143] Creating CURRENT +2025/07/28-18:13:51.565207 13289680896 [utilities/checkpoint/checkpoint_impl.cc:127] Hard Linking OPTIONS-000007 +2025/07/28-18:13:51.565761 13289680896 [db/db_impl/db_impl_files.cc:84] File Deletions Enabled +2025/07/28-18:13:51.565818 13289680896 [db/db_impl/db_impl_files.cc:471] [JOB 0] Try to delete WAL files size 113, prev total WAL file size 113, number of live WAL files 2. +2025/07/28-18:13:51.566701 13289680896 [db/wal_manager.cc:287] Move log file /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000011.log to /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000011.log -- OK +2025/07/28-18:13:51.566751 13289680896 [db/wal_manager.cc:287] Move log file /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000011.log to /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000011.log -- IO error: No such file or directory: While renaming a file to /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000011.log: /Users/war/cube_projects/cube.js/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/000011.log: No such file or directory +2025/07/28-18:13:51.567351 13289680896 [utilities/checkpoint/checkpoint_impl.cc:179] Snapshot DONE. All is good +2025/07/28-18:13:51.567358 13289680896 [utilities/checkpoint/checkpoint_impl.cc:181] Snapshot sequence number: 9 +2025/07/28-18:13:51.627654 6144143360 [db/db_impl/db_impl.cc:504] Shutdown: canceling all background work +2025/07/28-18:13:51.628794 6144143360 [db/db_impl/db_impl.cc:711] Shutdown complete diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/MANIFEST-000005 b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/MANIFEST-000005 new file mode 100644 index 0000000000000000000000000000000000000000..dd37b32b97907c97c73d3b9c49a3f095b1492ded GIT binary patch literal 628 zcmZS8)^KKEU<~`Yer`cxQDRAc(HCZ(C>91r zCI%LUKRRkZ*%%l(8JO8v7-l$c-CG3|=KylKSeVlpI6FCI8G&XppaDjpUNDel2Ffx9 zq|a*HB))q&R23s5P!t3hnORx)9-eoEjg565h~i~rWM|-D56#QY%P-I45pe3PZ2fV0 z^_ts>WeKffThB3m+PjFG1MC_uE)Evv`({NB)nG0U3o{!7XF3B{D=r5?Y-GUUAgC+@ z191+z0dvseKmMhl5a9-f2rmm;Is*@m5Mg3uh1dacz{NMc?eWR|FQN7_F|t8eENrZh zpkQO&4+#+_tRZp_<^VwEWiNa!{HugMh;~6%!}u} pvU~Y4E~tB$fe{V{?3}C+hrr##jMY7N;O<$cI1Az)elA`XP5|FWf`I@4 literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/OPTIONS-000007 b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/OPTIONS-000007 new file mode 100644 index 0000000000000..d2b2e9bfc5695 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/OPTIONS-000007 @@ -0,0 +1,198 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=7.10.2 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=600 + avoid_flush_during_shutdown=false + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + wal_compression=kNoCompression + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + allow_ingest_behind=false + fail_if_options_file_error=false + persist_stats_to_disk=false + WAL_ttl_seconds=330 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=1000 + table_cache_numshardbits=6 + max_file_opening_threads=16 + use_fsync=false + unordered_write=false + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + track_and_verify_wals_in_manifest=false + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=0 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + memtable_protection_bytes_per_key=0 + bottommost_compression=kNoCompression + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kNoCompression + level0_file_num_compaction_trigger=4 + prefix_extractor=rocksdb.FixedPrefix.13 + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + compression_opts={use_zstd_dict_trainer=true;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=0 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + preserve_internal_time_seconds=0 + force_consistency_checks=true + optimize_filters_for_hits=false + merge_operator=meta_store merge + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=false + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000004.log b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000004.log new file mode 100644 index 0000000000000000000000000000000000000000..7181c1593be4892414d144d661d9acdb69e5a279 GIT binary patch literal 127 zcma#*7nGD?U}R)~01(Z{$^;}BIAB5_1sPgG85n_rU;v^SMS%)odZ6MU(Y(amR0fu` m{CqDo4P2#(kLzOdvl>q=VV+>RP literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000008.log b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000008.log new file mode 100644 index 0000000000000000000000000000000000000000..e4123ee2f42ede1800962fc6bafb281b83d70272 GIT binary patch literal 127 zcmaF6`t5QV21XVJ2w((KjI2yRf`J1n#B|}_V&_nxD2xwMA_`OpQi}qZMDr4JQyEy& m^7D-t7`Rt5mE>=2Xy literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000011.log b/rust/cubestore/cubestore/delete_old_snapshots-local/metastore/archive/000011.log new file mode 100644 index 0000000000000000000000000000000000000000..508b3ce2b33b8f43a6eeea8f3af3eb9f46a1417a GIT binary patch literal 127 zcmcCNyR%G&fsvg70vLf5BP$b-VBmlXG5^YRV+>_rgzwQIr{|m~mb_@0H!lk8we@FoTr=0NO+i$p8QV literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/000010.sst b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/000010.sst new file mode 100644 index 0000000000000000000000000000000000000000..dcbbd6019113dfcfca0887d3bce87f01d55e330c GIT binary patch literal 1157 zcmaJ=&ubGw6rO2Qvq{tRN7@R7LJCE^WH4=-6iN@a*jl8vH;W=+cK0P6y1Nr*CQZ^q zp$9?1izmT@Ac%rTFCIiM>cNvI{{g{+cfqS~Hb1l=4tv=7=KJ1z^WHaPeuff)-}y!R zNW?=RiJFu}LNk=o^z_goC1cKz74=CyEhY1bVZ`Zv{mI$ui*H}{@jfpOgJ8VByjgod zX8y@19jlC4-yeS}lEs9Q-jViQ<_7q~T&}@qhPweU5&6(&rqckcnVX;u1vSxTH4zw4 zBB~eXasaGDybu|5tN2LQER zT+63&iwd~G<}kdI@F6n7bhuYq3{zlcGL1oM~|ar>Rj20T`Sx4onkgY6NEq%!^Czsoq=Fs6%nYv52IzwB1#(0HV4% zB&jhfRmcrH24%-XoEh3Ri}iuf4BMk@3Ytw&oA^?8mLBtO=}ZD7@kQ}vq{tRBW(|bLJGxO2Ggb=P;Zj*51&mY_& zlmE0Q9c!OheciwNgv_Ot^bd95F)zdo^SFk98SaI^L>xeynN9;NXL^)26x2kU)kJ7O ziMU>wF4ES4MAJj)`~^Fs)R$p1idl_EBJ3~`kg0UW792p0H$#(irlST|+(;F|5ri0j zydJ;eT(>Ys*>XK$^jf)6T`eWv))wZMmzGyc z+F%5zch)YgmLlGE1K}Ij!#LU^$Jzw0h0#jHe{hIx^}%^kvsMH3$EK;lP56n+!~v+a z%4#WFoKwIJHiuzNB7oQk)8T$~g;#v994z`v<*K*fm%YX1N_mN|EOX#XzPB(xHavD9 zwdr%VxxX&uVR=_chNI!(*yx1rwxe%zb8D|UzjtkGXY=ay?ds0T#;wh*LS|6L%!mli z-aLJK!O7Bq8_u*ihLh|Ng%Au*7zd__Gd_s31m-6x_gJqjYsjHE;)FzU1={X1SO{_5 z9FXiVl`7(f9f7jrA(4;spin55na9867O3A%5LgQw{l3Eg}YMRebY_Afa2l55lRi)cQNPdK( zyE>FH93)yH&d8t)1Hc~JH2WRO7O4_-5%bVW7#N1}K;@mush-}UjRW>E6=oIu`@L)k z?8_1a)hv;CL&>g@9eGTwiQ4VAdU lx}MR~Oy6qd)AslD3upEuAzJu9%l>12zWZ_d^76;~zW@WCSGWKG literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/CURRENT b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/CURRENT new file mode 100644 index 0000000000000..aa5bb8ea50905 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/CURRENT @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/MANIFEST-000005 b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719230483/MANIFEST-000005 new file mode 100644 index 0000000000000000000000000000000000000000..fb545617fe84779c7bb59ca41c5f7eceb52753dc GIT binary patch literal 484 zcmZS8)^KKEU<~`Yer`cxQDRAc(HCZ(C>91r zCI%LUKRRkZ*%%l(8JO8v7-l$c-CG3|=KylKSeVlpI6FCI8G&XppaDjpUNDel2Ffx9 zq|a*HB))q&R23s5P!t3hnORx)9-eoEjg565h~i~rWM|-D56#QY%P-I45pe3PZ2fV0 z^_ts>WeKffThB3m+PjFG1MC_uE)Evv`({NB)nG0U3o{!7XF3B{D=r5?Y-GUUAgC+@ z191+z0dvseKmMhl5a9-f2rmm;Is*@m5Mg3uh1dacz{NMc?eWR|FQN7_F|t8eENrZh epkQO&4+#+_tRZp_<^VcNvI{{g{+cfqS~Hb1l=4tv=7=KJ1z^WHaPeuff)-}y!R zNW?=RiJFu}LNk=o^z_goC1cKz74=CyEhY1bVZ`Zv{mI$ui*H}{@jfpOgJ8VByjgod zX8y@19jlC4-yeS}lEs9Q-jViQ<_7q~T&}@qhPweU5&6(&rqckcnVX;u1vSxTH4zw4 zBB~eXasaGDybu|5tN2LQER zT+63&iwd~G<}kdI@F6n7bhuYq3{zlcGL1oM~|ar>Rj20T`Sx4onkgY6NEq%!^Czsoq=Fs6%nYv52IzwB1#(0HV4% zB&jhfRmcrH24%-XoEh3Ri}iuf4BMk@3Ytw&oA^?8mLBtO=}ZD7@kQ}vq{tRBW(|bLJGxO2Ggb=P;Zj*51&mY_& zlmE0Q9c!OheciwNgv_Ot^bd95F)zdo^SFk98SaI^L>xeynN9;NXL^)26x2kU)kJ7O ziMU>wF4ES4MAJj)`~^Fs)R$p1idl_EBJ3~`kg0UW792p0H$#(irlST|+(;F|5ri0j zydJ;eT(>Ys*>XK$^jf)6T`eWv))wZMmzGyc z+F%5zch)YgmLlGE1K}Ij!#LU^$Jzw0h0#jHe{hIx^}%^kvsMH3$EK;lP56n+!~v+a z%4#WFoKwIJHiuzNB7oQk)8T$~g;#v994z`v<*K*fm%YX1N_mN|EOX#XzPB(xHavD9 zwdr%VxxX&uVR=_chNI!(*yx1rwxe%zb8D|UzjtkGXY=ay?ds0T#;wh*LS|6L%!mli z-aLJK!O7Bq8_u*ihLh|Ng%Au*7zd__Gd_s31m-6x_gJqjYsjHE;)FzU1={X1SO{_5 z9FXiVl`7(f9f7jrA(4;spin55na9867O3A%5LgQw{l3Eg}YMRebY_Afa2l55lRi)cQNPdK( zyE>FH93)yH&d8t)1Hc~JH2WRO7O4_-5%bVW7#N1}K;@mush-}UjRW>E6=oIu`@L)k z?8_1a)hv;CL&>g@9eGTwiQ4VAdU lx}MR~Oy6qd)AslD3upEuAzJu9%l>12zWZ_d^76;~zW@WCSGWKG literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/000014.sst b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/000014.sst new file mode 100644 index 0000000000000000000000000000000000000000..380ef9fbeaf8b33a10308a67b2d18f304f93b570 GIT binary patch literal 1161 zcmaJ>&1(}u6rX8Qvx!OCnzo06kV5g6!8GXyl!71FTBX*TqR2A4`;rdb-3c?3Ch^dl z;MG6Co&;}t^x(mR;#Cm;0>Psv6}g=KbF9ef-{%vvZUXd`~_Z>J1Gk zPk>}wQZ@)JQc4T66HAnoya@}Uxf7@iIh)O(ogVw84=N`?nI3Rn>1fdyPc#0(E&U?Pp6&&;3$7PBx-I|^E&&srij zphViPFH~r6U!v;)4E{2E(=?QEH%VEGCn6p&5s~?9&K4X&i+5vF@D`E|SKLSy!y&}T ze`#Q#Cp27^zi*1Uk^-%TOm9eQ9eq^1N7R=cgLd7+9A(cBgwdO|dUK=dqHQ*o*H+gy zs@fm}vwHm`;iEZA0UsmmiyX7zBQwlNdMR&wbe%_LETmh)E}R&2H)XbNEZOq zW__cYuPiCx2AjjMCJ{kugcWW=)mk%XgtcI0tzKK@>uVhNY8W(@XG$~s zQkyYmmxq_6JgD7NlHq80JTpD3hwT{q+}gR_T)usEXK(w;_1)&)`qrK8opNqcrp$;0 zUcEkkbIQxph#StdxDQA9(-dMbIAI)^Cf=DzoF%Z(rTjPQv`2Bo4nzuN+8-iV3~Af! zlYEIvm2krjLD}&TZ;lSdVs#)i!}cgkLCaZciO-ei=^>B1s42}Nh%p~XLm5Fty)sQe zwSmbFD4s@|R5H^jiv$e!H6Ebq?!76Rb`$1maZ=T^n4{QUB|vA2Wja))KeCYG6vc3D zs39FVtq^Z&QpOQrk8PUc4&^IUiMB|2Y$Xf~X#!AXZ*G31cj)7QLrjHP1^?ccErDIR zIoybPqcAgJ3orwQ%8$W%>$~^7A|}yubhZuK#lol!_G@vxi9en{eoxMfH?gB!D6H!V kec=8BZeQ0w?S9Ta^A?XB(1QQ-Oyb@5cV8d8xcKqmFIer`cxQDRAc(HCZ(C>91r zCI%LUKRRkZ*%%l(8JO8v7-l$c-CG3|=KylKSeVlpI6FCI8G&XppaDjpUNDel2Ffx9 zq|a*HB))q&R23s5P!t3hnORx)9-eoEjg565h~i~rWM|-D56#QY%P-I45pe3PZ2fV0 z^_ts>WeKffThB3m+PjFG1MC_uE)Evv`({NB)nG0U3o{!7XF3B{D=r5?Y-GUUAgC+@ z191+z0dvseKmMhl5a9-f2rmm;Is*@m5Mg3uh1dacz{NMc?eWR|FQN7_F|t8eENrZh zpkQO&4+#+_tRZp_<^VwEWiNa!{HugMh;~6%!}u} pvU~Y4E~tB$fe{V{?3}C+hrr##jMY7N;O<$cI1Az)elA`XP5|FWf`I@4 literal 0 HcmV?d00001 diff --git a/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/OPTIONS-000007 b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/OPTIONS-000007 new file mode 100644 index 0000000000000..d2b2e9bfc5695 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-1753719231346/OPTIONS-000007 @@ -0,0 +1,198 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=7.10.2 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=0 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=600 + avoid_flush_during_shutdown=false + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + wal_compression=kNoCompression + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + allow_ingest_behind=false + fail_if_options_file_error=false + persist_stats_to_disk=false + WAL_ttl_seconds=330 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=1000 + table_cache_numshardbits=6 + max_file_opening_threads=16 + use_fsync=false + unordered_write=false + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + track_and_verify_wals_in_manifest=false + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=0 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + memtable_protection_bytes_per_key=0 + bottommost_compression=kNoCompression + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kNoCompression + level0_file_num_compaction_trigger=4 + prefix_extractor=rocksdb.FixedPrefix.13 + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + compression_opts={use_zstd_dict_trainer=true;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=0 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + preserve_internal_time_seconds=0 + force_consistency_checks=true + optimize_filters_for_hits=false + merge_operator=meta_store merge + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=false + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-current b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-current new file mode 100644 index 0000000000000..ea28456422544 --- /dev/null +++ b/rust/cubestore/cubestore/delete_old_snapshots-remote/metastore-current @@ -0,0 +1 @@ +metastore-1753719231346 \ No newline at end of file diff --git a/rust/cubestore/cubestore/src/queryplanner/query_executor.rs b/rust/cubestore/cubestore/src/queryplanner/query_executor.rs index eae412340458a..7ccb3dfdbf944 100644 --- a/rust/cubestore/cubestore/src/queryplanner/query_executor.rs +++ b/rust/cubestore/cubestore/src/queryplanner/query_executor.rs @@ -979,14 +979,14 @@ impl ClusterSendExec { snapshots: &[Snapshots], tree: &HashMap, ) -> Result, Vec))>, CubeError> { - let partitions = Self::logical_partitions(snapshots, tree)?; - Ok(Self::assign_nodes(config, partitions)) + let (partitions, can_be_splitted) = Self::logical_partitions(snapshots, tree)?; + Ok(Self::assign_nodes(config, partitions, can_be_splitted)) } fn logical_partitions( snapshots: &[Snapshots], tree: &HashMap, - ) -> Result>, CubeError> { + ) -> Result<(Vec>, bool), CubeError> { let mut to_multiply = Vec::new(); let mut multi_partitions = HashMap::>::new(); let mut has_inline_tables = false; @@ -1048,21 +1048,23 @@ impl ClusterSendExec { "Partitioned index queries aren't supported with inline tables".to_string(), )); } - return Ok(Self::distribute_multi_partitions(multi_partitions, tree) + let res = Self::distribute_multi_partitions(multi_partitions, tree) .into_iter() .map(|i| { i.into_iter() .map(|p| InlineCompoundPartition::Partition(p)) .collect() }) - .collect()); + .collect(); + return Ok((res, false)); } - // Ordinary partitions need to be duplicated on multiple machines. + let can_be_splitted = to_multiply.len() == 1; //We can only split partitions if there’s no multiplication for join. + // Ordinary partitions need to be duplicated on multiple machines. let partitions = to_multiply .into_iter() .multi_cartesian_product() .collect::>>(); - Ok(partitions) + Ok((partitions, can_be_splitted)) } fn distribute_multi_partitions( @@ -1137,6 +1139,7 @@ impl ClusterSendExec { fn assign_nodes( c: &dyn ConfigObj, logical: Vec>, + can_be_splitted: bool, ) -> Vec<(String, (Vec<(u64, RowRange)>, Vec))> { let mut m: HashMap<_, (Vec<(IdRow, RowRange)>, Vec)> = HashMap::new(); @@ -1181,7 +1184,7 @@ impl ClusterSendExec { r.sort_unstable_by(|l, r| l.0.cmp(&r.0)); r.into_iter() .map(|(worker, data)| { - let splitted = Self::split_worker_parititons(c, data); + let splitted = Self::split_worker_parititons(c, data, can_be_splitted); splitted.into_iter().map(move |data| (worker.clone(), data)) }) .flatten() @@ -1191,13 +1194,9 @@ impl ClusterSendExec { fn split_worker_parititons( c: &dyn ConfigObj, partitions: (Vec<(IdRow, RowRange)>, Vec), + can_be_splitted: bool, ) -> Vec<(Vec<(u64, RowRange)>, Vec)> { - if !partitions.1.is_empty() - || partitions - .0 - .iter() - .any(|(p, _)| p.get_row().multi_partition_id().is_some()) - { + if !can_be_splitted { return vec![( partitions .0