add support for SQLite output

6 files changed, 180 insertions, 11 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 03fb13b..c3d5c36 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,6 +24,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.8",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
 name = "alloc-no-stdlib"
 version = "2.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -471,6 +482,18 @@ dependencies = [
 ]
 
 [[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
+[[package]]
 name = "flate2"
 version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -571,6 +594,18 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
+dependencies = [
+ "hashbrown",
+]
 
 [[package]]
 name = "heck"
@@ -616,6 +651,7 @@ dependencies = [
  "num-traits",
  "rayon",
  "roxmltree",
+ "rusqlite",
 ]
 
 [[package]]
@@ -757,6 +793,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
 [[package]]
+name = "libsqlite3-sys"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
+dependencies = [
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
 name = "linux-raw-sys"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1074,6 +1120,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
 [[package]]
+name = "pkg-config"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+
+[[package]]
 name = "plotters"
 version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1285,6 +1337,20 @@ dependencies = [
 ]
 
 [[package]]
+name = "rusqlite"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
+dependencies = [
+ "bitflags",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "smallvec",
+]
+
+[[package]]
 name = "rustc-demangle"
 version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1568,6 +1634,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
 [[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
 name = "version_check"
 version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 3c9b45d..bbbab82 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,6 +26,7 @@ nalgebra = "0.31.4"
 num-traits = "0.2.15"
 rayon = "1.6.1"
 roxmltree = "0.17.0"
+rusqlite = "0.28.0"
 
 [dev-dependencies]
 criterion = "0.4.0"
diff --git a/README.adoc b/README.adoc
index 8636e74..9c673d9 100644
--- a/README.adoc
+++ b/README.adoc
@@ -8,7 +8,7 @@ hittekaart - A GPX track heatmap generator.
 == SYNOPSIS
 
 ----
-hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] FILES...
+hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] [--sqlite] FILES...
 ----
 
 == INSTALLATION
@@ -38,6 +38,29 @@ By default, the directory `tiles/` will be used as the root directory, so a
 tile would be saved as `tiles/{zoom}/{x}/{y}.png`. You can change this by using
 the `--output/-o` option.
 
+=== SQLITE OUTPUT
+
+In order to overcome storage overhead when saving many small files (see the tip
+and table further below), `hittekaart` can instead output a SQLite database
+with the heatmap tile data. To do so, use the `--sqlite` command line option,
+and control where the SQLite file should be placed with `--output`/`-o`.
+
+While this does not allow you to immediately serve the tiles with a HTTP
+server, it does cut down on the wasted space on non-optimal file systems.
+
+The generated SQLite file will have one table with the following schema:
+
+[source,sql]
+----
+CREATE TABLE tiles (
+    zoom INTEGER,
+    x INTEGER,
+    y INTEGER,
+    data BLOB,
+    PRIMARY KEY (zoom, x, y)
+);
+----
+
 === INPUT FILES
 
 `hittekaart` expects GPX track files with the `.gpx` extension. It will parse
@@ -140,8 +163,8 @@ system that is optimized for a large amount of small files, for example by
 setting a smaller block size. Many of the PNG images are smaller than 2 KiB
 (half a standard block); for those 50% of storage is wasted already.
 
-Currently, `hittekaart` does not provide a built-in way to store or serve the
-tiles more efficiently.
+If you don't need the tiles in separate files, you can use the SQLite output
+mode. For the same data as above, the SQLite database would be 73 MiB in size.
 ====
 
 == OPTIONS
@@ -161,7 +184,14 @@ The following options are supported:
     will automatically pick a default.
 
 `-o DIRECTORY`, `--output=DIRECTORY`::
-    Generate the output tiles into the given directory. Defaults to `tiles/`.
+    Generate the output tiles into the given directory. Defaults to `tiles/`
+    when generating single files, and `tiles.sqlite` when storing the tiles in
+    a SQLite database.
+
+`--sqlite`::
+    Output a single SQLite file with all tiles instead of saving each tile as a
+    separate PNG file. In this case, `-o` can be used to set the location of
+    the SQLite database. The schema is described above.
 
 == EXAMPLE
 
diff --git a/src/main.rs b/src/main.rs
index 7230dce..e5b835e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -8,7 +8,7 @@ use color_eyre::{
 use hittekaart::{
     gpx::{self, Compression},
     renderer,
-    storage::{Folder, Storage},
+    storage::{Folder, Sqlite, Storage},
 };
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
 use is_terminal::IsTerminal;
@@ -36,9 +36,15 @@ struct Args {
     #[arg(long, short, default_value_t = 0)]
     threads: usize,
 
-    /// The output directory. Will be created if it does not exist.
-    #[arg(long, short, default_value = "tiles")]
-    output_directory: PathBuf,
+    /// The output directory. Will be created if it does not exist. Defaults to "tiles" for the
+    /// folder-based storage, and "tiles.sqlite" for the SQLite-based storage.
+    #[arg(long, short)]
+    output: Option<PathBuf>,
+
+    /// Store the tiles in a SQLite database. If given, `--output` will determine the SQLite
+    /// filename.
+    #[arg(long)]
+    sqlite: bool,
 }
 
 fn main() -> Result<()> {
@@ -85,7 +91,13 @@ fn main() -> Result<()> {
     let tracks = tracks.into_iter().collect::<Result<Vec<_>>>()?;
     bar.finish();
 
-    let mut storage = Folder::new(args.output_directory.clone());
+    let mut storage: Box<dyn Storage + Send> = if args.sqlite {
+        let output = args.output.unwrap_or_else(|| "tiles.sqlite".into());
+        Box::new(Sqlite::connect(output)?)
+    } else {
+        let output = args.output.unwrap_or_else(|| "tiles".into());
+        Box::new(Folder::new(output))
+    };
     storage.prepare()?;
 
     let multibar = MultiProgress::new();
diff --git a/src/renderer.rs b/src/renderer.rs
index 4840e73..74a321c 100644
--- a/src/renderer.rs
+++ b/src/renderer.rs
@@ -139,7 +139,7 @@ fn colorize_tile(tile: &ImageBuffer<Luma<u8>, Vec<u8>>, max: u32) -> RgbaImage {
 /// rendering the next one.
 ///
 /// This has a way lower memory usage than [`colorize_heatcounter`].
-pub fn lazy_colorization<F: FnMut(RenderedTile) -> Result<()> + Send + Sync>(
+pub fn lazy_colorization<F: FnMut(RenderedTile) -> Result<()> + Send>(
     layer: HeatCounter,
     mut save_callback: F,
 ) -> Result<()> {
diff --git a/src/storage.rs b/src/storage.rs
index fef8194..51a418e 100644
--- a/src/storage.rs
+++ b/src/storage.rs
@@ -1,4 +1,8 @@
-use color_eyre::{eyre::{bail, WrapErr}, Result};
+use color_eyre::{
+    eyre::{bail, WrapErr},
+    Result,
+};
+use rusqlite::{params, Connection};
 use std::{
     fs,
     io::ErrorKind,
@@ -65,3 +69,53 @@ impl Storage for Folder {
         Ok(())
     }
 }
+
+#[derive(Debug)]
+pub struct Sqlite {
+    connection: Connection,
+}
+
+impl Sqlite {
+    pub fn connect<P: AsRef<Path>>(file: P) -> Result<Self> {
+        let path = file.as_ref();
+        if fs::metadata(path).is_ok() {
+            bail!("Path {path:?} already exists, refusing to open")
+        }
+        let connection = Connection::open(path)?;
+        Ok(Sqlite { connection })
+    }
+}
+
+impl Storage for Sqlite {
+    fn prepare(&mut self) -> Result<()> {
+        self.connection.execute(
+            "CREATE TABLE tiles (
+            zoom INTEGER,
+            x INTEGER,
+            y INTEGER,
+            data BLOB,
+            PRIMARY KEY (zoom, x, y)
+        );",
+            (),
+        )?;
+        self.connection.execute("BEGIN;", ())?;
+        Ok(())
+    }
+
+    fn prepare_zoom(&mut self, _zoom: u32) -> Result<()> {
+        Ok(())
+    }
+
+    fn store(&mut self, zoom: u32, x: u64, y: u64, data: &[u8]) -> Result<()> {
+        self.connection.execute(
+            "INSERT INTO tiles (zoom, x, y, data) VALUES (?, ?, ?, ?)",
+            params![zoom, x, y, data],
+        )?;
+        Ok(())
+    }
+
+    fn finish(&mut self) -> Result<()> {
+        self.connection.execute("COMMIT;", ())?;
+        Ok(())
+    }
+}