Add limited support for older databases

Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "sketchlib"
-version = "0.3.0"
+version = "0.4.0"
 authors = [
     "John Lees <jlees@ebi.ac.uk>",
     "Nicholas Croucher <n.croucher@imperial.ac.uk>",
     "Johanna von Wachsmann <wachsmannj@ebi.ac.uk>",
     "Victor Rodriguez Bouza <vrbouza@ebi.ac.uk>",
-    "Joel Hellewell <joel@ebi.ac.uk",
+    "Matthew Russell <marussel@ebi.ac.uk>",
+    "Tommi Maklin",
 ]
 edition = "2021"
 description = "Genome and amino-acid sketching"
@@ -31,8 +32,10 @@ crate-type = ["cdylib", "lib"]
 [features]
 3di = ["dep:pyo3", "dep:pyo3-build-config"]
 
+
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 # data structures
+aligned-vec = "0.6.4"
 hashbrown = { version = "0.14", features = ["rayon", "serde"] }
 ahash = ">=0.8.7"
 roaring = { version = "0.10", features = ["serde"] }
@@ -63,6 +66,7 @@ num_cpus = "1.0"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 # data structures
+aligned-vec = "0.6.4"
 hashbrown = { version = "0.14", features = ["rayon", "serde"] }
 # ahash = ">=0.8.7"
 roaring = { version = "0.10", features = ["serde"] }

README.md

@@ -15,8 +15,11 @@ This is a reimplementation and extension of [pp-sketchlib](https://github.com/ba
 in the rust language. This version is optimised for larger sample numbers, particularly
 allowing subsets of samples to be compared.
 
-v0.2.0 is the first stable release. We intend to keep the file format unchanged after this
-point so sketch libraries will not need to be rebuilt.
+### News
+
+- **Data format changed in v0.4.0**. If you have an older database (see `sketchlib info <db_prefix> | grep "sketch_version"`)
+you should resketch samples with >v0.4 if possible. Otherwise you will need to use an older release available on crates.io.
+- v0.2.0 was the first stable release.
 
 ## Documentation
 

src/cli.rs

@@ -85,6 +85,18 @@ pub fn check_and_set_threads(threads: usize) {
         .unwrap();
 }
 
+/// Sets the global rayon thread pool size for operations that spawn an internal
+/// writer thread (sketching, and appending to an existing sketch database).
+///
+/// Reserves one extra thread for the writer so that `threads` workers remain
+/// available for compute; equivalent to `check_and_set_threads(threads + 1)`.
+/// Callers embedding this crate as a library (rather than via the CLI) should call
+/// this before sketching/appending, and plain [`check_and_set_threads`] before
+/// distance/query/delete operations that don't spawn a writer.
+pub fn set_threads_with_writer(threads: usize) {
+    check_and_set_threads(threads + 1);
+}
+
 /// Options that apply to all subcommands
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]

src/distances/distance_matrix.rs

@@ -164,6 +164,27 @@ impl<'a> DistanceMatrix<'a> {
     pub fn dists_mut(&mut self) -> &mut Vec<f32> {
         &mut self.distances
     }
+
+    /// Iterates over the distances as `(primary, accessory)` pairs.
+    ///
+    /// The second element is `Some` only for [`DistType::CoreAcc`] matrices
+    /// (interleaved core/accessory pairs written by `self_dists_all`/`cross_dists_all`);
+    /// `None` for Jaccard/ANI matrices, where each entry is a single value. Iteration
+    /// order matches the row-major order used by `Display` (ref-outer, query- or
+    /// ref-inner — see `self_dists_all`/`cross_dists_all` docs).
+    pub fn dists_iter(&self) -> Box<dyn Iterator<Item = (f32, Option<f32>)> + '_> {
+        if self.jaccard == DistType::CoreAcc {
+            Box::new(
+                self.distances
+                    .as_chunks::<2>()
+                    .0
+                    .iter()
+                    .map(|pair| (pair[0], Some(pair[1]))),
+            )
+        } else {
+            Box::new(self.distances.iter().copied().map(|d| (d, None)))
+        }
+    }
 }
 
 impl<'a> Distances<'a> for DistanceMatrix<'a> {