Merge pull request #1753 from basho/mas-i306-compactrecalc

Mas i306 compactrecalc
basho · Mar 30, 2020 · 982edb1 · 982edb1
2 parents 6e8d145 + df5220e
commit 982edb1
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 2 deletions.
diff --git a/priv/riak_kv.schema b/priv/riak_kv.schema
@@ -1117,3 +1117,22 @@
     {commented, 24}
 ]}.
 
+%% @doc Enable the `recalc` compaction strategy within the leveled backend in
+%% riak.  The default (when disabled) is `retain`, but this will leave
+%% uncollected garbage within the, journal. 
+%% It is now recommended from Riak KV 2.9.2 to consider the `recalc` strategy.
+%% This strategy has a side effect of slower startups, and slower recovery
+%% from a wiped ledger - but it will not keep an overhead of garbage within
+%% the Journal.
+%% It should be possible to move from `retain` to `recalc` via configuration
+%% change.  However, it is not possible to switch from `recalc` back to
+%% `retain`.  This switch can only be made for new nodes receiving data
+%% through riak transfers (not inheriting data on disk).
+%% The default `retain` strategy retains a history of key changes in the
+%% journal, whereas the `recalc` strategy discards that history, but will redo
+%% a diff_index_specs calculation when reloading each object.
+{mapping, "leveled_reload_recalc", "riak_kv.leveled_reload_recalc", [
+    {datatype, {flag, enabled, disabled}},
+    {default, disabled},
+    {commented, enabled}
+]}.
diff --git a/rebar.config b/rebar.config
@@ -36,7 +36,7 @@
         {riak_pipe, ".*", {git, "git://github.com/basho/riak_pipe.git", {tag, "riak_kv-2.9.1"}}},
         {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {tag, "riak_kv-2.9.1"}}},
         {eunit_formatters, ".*", {git, "git://github.com/seancribbs/eunit_formatters", {tag, "0.1.2"}}},
-        {leveled, ".*", {git, "https://github.com/martinsumner/leveled.git", {tag, "riak_kv-2.9.1"}}},
+        {leveled, ".*", {git, "https://github.com/martinsumner/leveled.git", {branch, "develop-2.9"}}},
         {kv_index_tictactree, ".*", {git, "https://github.com/martinsumner/kv_index_tictactree.git", {tag, "riak_kv-2.9.1"}}},
         {riak_core, ".*", {git, "https://github.com/basho/riak_core.git", {branch, "develop-2.9"}}},
         {riak_api, ".*", {git, "git://github.com/basho/riak_api.git", {branch, "develop-2.9"}}},

diff --git a/src/riak_kv_leveled_backend.erl b/src/riak_kv_leveled_backend.erl
@@ -122,13 +122,35 @@ start(Partition, Config) ->
     PCL = app_helper:get_prop_or_env(ledger_pagecachelevel, Config, leveled),
 
     BackendPause = app_helper:get_env(riak_kv, backend_pause_ms, ?PAUSE_TIME),
+    LCR = app_helper:get_env(riak_kv, leveled_reload_recalc, false),
 
     case get_data_dir(DataRoot, integer_to_list(Partition)) of
         {ok, DataDir} ->
             {ok, CHBin} = riak_core_ring_manager:get_chash_bin(),
             PartitionCount = chashbin:num_partitions(CHBin),
             RingIndexInc = chash:ring_increment(PartitionCount),
             DBid = Partition div RingIndexInc,
+            FN = filename:join(DataDir, "recalc.lock"),
+            {ok, ReloadStrategy} = 
+                case {LCR, filelib:is_file(FN)} of
+                    {true, true} ->
+                        {ok, recalc};
+                    {true, false} ->
+                        ok = filelib:ensure_dir(FN),
+                        ok = file:write_file(FN, term_to_binary(os:timestamp())),
+                        {ok, recalc};
+                    {false, true} ->
+                        {ok, TS} = file:read_file(FN),
+                        LockTS = calendar:now_to_datetime(binary_to_term(TS)),
+                        lager:error("Cannot start in retain mode " ++
+                                        "due to recalc being set on ~w " ++
+                                        "see FN ~s",
+                                        [LockTS, FN]),
+                        {error, invalid_compaction_change};
+                    {false, false} ->
+                        {ok, retain}
+                end,
+
             StartOpts = [{root_path, DataDir},
                             {max_journalsize, MJS},
                             {max_journalobjectcount, MJC},
@@ -144,7 +166,8 @@ start(Partition, Config) ->
                             {maxrunlength_compactionpercentage, MCP},
                             {singlefile_compactionpercentage, SCP},
                             {snapshot_timeout_short, TOS},
-                            {snapshot_timeout_long, TOL}],
+                            {snapshot_timeout_long, TOL},
+                            {reload_strategy, [{?RIAK_TAG, ReloadStrategy}]}],
             {ok, Bookie} = leveled_bookie:book_start(StartOpts),
             Ref = make_ref(),
             ValidHours = valid_hours(CLH, CTH),