Skip to content

Commit

Permalink
[new] [#164] Update benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@ptaoussanis
ptaoussanis committed Jan 15, 2024
1 parent 0518ee6 commit 27dcec7
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 80 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ See [here][GitHub releases] for earlier releases.

Since its earliest versions, Nippy has consistently been the **fastest serialization library for Clojure** that I'm aware of. It offers:

- Roundtrip times **>12x faster** than `tools.reader` with **60% smaller** data size.
- Roundtrip times **>2x faster** than `data.fressian` with **30% smaller** data size.
- **0.3x** the roundtrip time and **0.6x** the output size of `tools.reader`
- **0.7x** the roundtrip time and **0.7x** the output size of `data.fressian`

![benchmarks-png](../../raw/master/benchmarks.png)

Expand Down
Binary file modified benchmarks.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
184 changes: 107 additions & 77 deletions test/taoensso/nippy_benchmarks.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@

(comment (-> data freeze-fress thaw-fress))

;;;; Benchable data
;;;; Bench data

(def bench-data
(def default-bench-data
"Subset of stress data suitable for benching."
(let [sd (nippy/stress-data {:comparable? true})]
(reduce-kv
Expand All @@ -42,12 +42,18 @@
(comment
(clojure.set/difference
(set (keys (nippy/stress-data {:comparable? true})))
(set (keys bench-data))))
(set (keys default-bench-data))))

;;;;
;;;; Serialization

(defn- bench1-serialization
[freezer thawer sizer
{:keys [laps warmup bench-data]
:or
{laps 1e4
warmup 25e3
bench-data default-bench-data}}]

(defn- bench1
[{:keys [laps warmup] :or {laps 1e4, warmup 25e3}} freezer thawer sizer]
(let [data-frozen (freezer bench-data)
time-freeze (enc/bench laps {:warmup-laps warmup} (freezer bench-data))
time-thaw (enc/bench laps {:warmup-laps warmup} (thawer data-frozen))
Expand All @@ -58,119 +64,143 @@
:thaw time-thaw
:size data-size}))

(comment (bench1 {} nippy/freeze nippy/thaw count))
(comment (bench1-serialization nippy/freeze nippy/thaw count {}))

(defn- printed-results [results]
(println "\nBenchmark results:")
(doseq [[k v] results] (println k " " v))
(do results))

(defn bench
[{:keys [all? reader? fressian? fressian? lzma2? laps warmup]
:or {laps 1e4
warmup 25e3}}]
(defn bench-serialization
[{:keys [all? reader? fressian? fressian? lzma2? laps warmup bench-data]
:as opts
:or
{laps 1e4
warmup 25e3}}]

(println "\nStarting benchmarks")

(let [results_ (atom {})]
(when (or all? reader?)
(println "Benching Reader...")
(println "- Benching Reader...")
(swap! results_ assoc :reader
(bench1 {:laps laps, :warmup warmup}
freeze-reader thaw-reader
(fn [^String s] (count (.getBytes s "UTF-8"))))))
(bench1-serialization freeze-reader thaw-reader
(fn [^String s] (count (.getBytes s "UTF-8")))
(assoc opts :laps laps, :warmup warmup))))

(when (or all? fressian?)
(println "- Benching Fressian...")
(swap! results_ assoc :fressian (bench1 {:laps laps, :warmup warmup} freeze-fress thaw-fress count)))
(swap! results_ assoc :fressian
(bench1-serialization freeze-fress thaw-fress count
(assoc opts :laps laps, :warmup warmup))))

(when (or all? lzma2?)
(println "- Benching Nippy/LZMA2...")
(swap! results_ assoc :nippy/lzma2
(bench1 {:laps laps, :warmup warmup}
(bench1-serialization
#(nippy/freeze % {:compressor nippy/lzma2-compressor})
#(nippy/thaw % {:compressor nippy/lzma2-compressor})
count)))
count
(assoc opts :laps laps, :warmup warmup))))

(println "- Benching Nippy/encrypted...")
(swap! results_ assoc :nippy/encrypted
(bench1 {:laps laps, :warmup warmup}
(bench1-serialization
#(nippy/freeze % {:password [:cached "p"]})
#(nippy/thaw % {:password [:cached "p"]})
count))
count
(assoc opts :laps laps, :warmup warmup)))

(println "- Benching Nippy/default...")
(swap! results_ assoc :nippy/default (bench1 {:laps laps, :warmup warmup} nippy/freeze nippy/thaw count))
(swap! results_ assoc :nippy/default
(bench1-serialization nippy/freeze nippy/thaw count
(assoc opts :laps laps, :warmup warmup)))

(println "- Benching Nippy/fast...")
(swap! results_ assoc :nippy/fast (bench1 {:laps laps, :warmup warmup} nippy/fast-freeze nippy/fast-thaw count))
(swap! results_ assoc :nippy/fast
(bench1-serialization nippy/fast-freeze nippy/fast-thaw count
(assoc opts :laps laps, :warmup warmup)))

(println "- Benchmarks complete! (Time for cake?)")

(let [results @results_]
(println "\nBenchmark results:")
(doseq [[k v] results] (println k " " v))
(do results))))

(comment
(do
(set! *unchecked-math* false)
(bench {:all? true}))

;; 2023 Sep 25, 2020 Apple MBP M1
;; [com.taoensso/nippy "3.2.0"]
;; [org.clojure/tools.reader "1.3.6"]
;; [org.clojure/data.fressian "1.0.0"]

{:reader {:round 36756, :freeze 6567, :thaw 30189, :size 39374}
:fressian {:round 6579, :freeze 3720, :thaw 2859, :size 22850}
:nippy/lzma2 {:round 33645, :freeze 20411, :thaw 13234, :size 11420}
:nippy/encrypted {:round 3894, :freeze 2260, :thaw 1634, :size 16717}
:nippy/default {:round 2796, :freeze 1467, :thaw 1329, :size 16689}
:nippy/fast {:round 2516, :freeze 1271, :thaw 1245, :size 28454}})

;;;; Compressors

(let [bench-data (nippy/freeze (nippy/stress-data {:comparable? true}) {:compressor nil})]

(defn bench1-compressor
[{:keys [laps warmup] :or {laps 1e4, warmup 2e4}} compressor]
(let [data-compressed (compr/compress compressor bench-data)
time-compress (enc/bench laps {:warmup-laps warmup} (compr/compress compressor bench-data))
time-decompress (enc/bench laps {:warmup-laps warmup} (compr/decompress compressor data-compressed))]

{:round (+ time-compress time-decompress)
:compress time-compress
:decompress time-decompress
:ratio (enc/round2 (/ (count data-compressed) (count bench-data)))}))

(defn bench-compressors [bench1-opts lzma-opts]
(printed-results @results_)))

;;;; Compression

(defn- bench1-compressor
[compressor
{:keys [laps warmup bench-data]
:or
{laps 1e4
warmup 2e4
bench-data default-bench-data}}]

(let [data-frozen (nippy/freeze bench-data {:compressor nil})
data-compressed (compr/compress compressor data-frozen)
time-compress (enc/bench laps {:warmup-laps warmup} (compr/compress compressor data-frozen))
time-decompress (enc/bench laps {:warmup-laps warmup} (compr/decompress compressor data-compressed))]

{:round (+ time-compress time-decompress)
:compress time-compress
:decompress time-decompress
:ratio (enc/round2 (/ (count data-compressed) (count data-frozen)))}))

(defn bench-compressors [opts lzma-opts]
(printed-results
(merge
(let [bench1 #(bench1-compressor bench1-opts %)]
(let [bench1 #(bench1-compressor % opts)]
{:zstd/prepended (bench1 (compr/->ZstdCompressor true))
:zstd/unprepended (bench1 (compr/->ZstdCompressor false))
:lz4 (bench1 (compr/->LZ4Compressor))
:lzo (bench1 (compr/->LZOCompressor))
:snappy/prepended (bench1 (compr/->SnappyCompressor true))
:snappy/unprepended (bench1 (compr/->SnappyCompressor false))})

(let [bench1 #(bench1-compressor (merge bench1-opts lzma-opts) %)]
(let [bench1 #(bench1-compressor % (merge opts lzma-opts))]
{:lzma2/level0 (bench1 (compr/->LZMA2Compressor 0))
:lzma2/level3 (bench1 (compr/->LZMA2Compressor 3))
:lzma2/level6 (bench1 (compr/->LZMA2Compressor 6))
:lzma2/level9 (bench1 (compr/->LZMA2Compressor 9))}))))

;;;; Results

(comment
{:last-updated "2024-01-15"
:clojure-version "1.11.1"
:java-version "OpenJDK 21"
:deps
'[[com.taoensso/nippy "3.4.0-beta1"]
[org.clojure/tools.reader "1.3.6"]
[org.clojure/data.fressian "1.0.0"]
[org.tukaani/xz "1.9"]
[io.airlift/aircompressor "0.25"]]}

(bench-serialization {:all? true})

{:reader {:round 12519, :freeze 3881, :thaw 8638, :size 15880}
:fressian {:round 4143, :freeze 2549, :thaw 1594, :size 12222}
:nippy/lzma2 {:round 15376, :freeze 10050, :thaw 5326, :size 3924}
:nippy/encrypted {:round 3132, :freeze 1508, :thaw 1624, :size 8553}
:nippy/default {:round 2865, :freeze 1279, :thaw 1586, :size 8525}
:nippy/fast {:round 2522, :freeze 1026, :thaw 1496, :size 17088}}

(enc/round2 (/ 2865 12519.0)) ; 0.23 of reader roundtrip time
(enc/round2 (/ 2865 4143.0)) ; 0.69 of fressian roundtrip time

(enc/round2 (/ 8525 15880.0)) ; 0.54 of reader output size
(enc/round2 (/ 8525 12222.0)) ; 0.70 of fressian output size

(bench-compressors
{:laps 1e4 :warmup 2e4}
{:laps 1e2 :warmup 2e2})

;; 2023 Aug 1, 2020 Apple MBP M1
;; [org.tukaani/xz "1.9"]
;; [io.airlift/aircompressor "0.25"]

{:zstd/prepended {:round 1672, :compress 1279, :decompress 393, :ratio 0.53}
:zstd/unprepended {:round 1668, :compress 1271, :decompress 397, :ratio 0.53}
:lz4 {:round 269, :compress 238, :decompress 31, :ratio 0.58}
:lzo {:round 259, :compress 216, :decompress 43, :ratio 0.58}
:snappy/prepended {:round 339, :compress 205, :decompress 134, :ratio 0.58}
:snappy/unprepended {:round 340, :compress 206, :decompress 134, :ratio 0.58}
:lzma2/level0 {:round 30300, :compress 18500, :decompress 11800, :ratio 0.4}
:lzma2/level3 {:round 49200, :compress 35700, :decompress 13500, :ratio 0.4}
:lzma2/level6 {:round 102600, :compress 86700, :decompress 15900, :ratio 0.41}
:lzma2/level9 {:round 434800, :compress 394700, :decompress 40100, :ratio 0.41}})
;; Note that ratio depends on compressibility of stress data
{:lz4 {:round 293, :compress 234, :decompress 59, :ratio 0.5}
:lzo {:round 483, :compress 349, :decompress 134, :ratio 0.46}
:snappy/prepended {:round 472, :compress 296, :decompress 176, :ratio 0.43}
:snappy/unprepended {:round 420, :compress 260, :decompress 160, :ratio 0.43}
:zstd/prepended {:round 2105, :compress 1419, :decompress 686, :ratio 0.3}
:zstd/unprepended {:round 1261, :compress 921, :decompress 340, :ratio 0.3}
:lzma2/level0 {:round 158, :compress 121, :decompress 37, :ratio 0.24}
:lzma2/level3 {:round 536, :compress 436, :decompress 100, :ratio 0.22}
:lzma2/level6 {:round 1136, :compress 1075, :decompress 61, :ratio 0.21}
:lzma2/level9 {:round 2391, :compress 2096, :decompress 295, :ratio 0.21}})
2 changes: 1 addition & 1 deletion test/taoensso/nippy_tests.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,4 +428,4 @@

;;;; Benchmarks

(deftest _benchmarks (is (benchmarks/bench {})))
(deftest _benchmarks (is (benchmarks/bench-serialization {})))

0 comments on commit 27dcec7

Please sign in to comment.