mirror of
https://git.freebsd.org/ports.git
synced 2025-06-30 17:10:33 -04:00
140 lines
4.8 KiB
Erlang
140 lines
4.8 KiB
Erlang
diff -ruN temp/riak-cs-2.1.0/riak_test/src/downgrade_bitcask.erl work/riak-cs-2.1.0/riak_test/src/downgrade_bitcask.erl
|
|
--- temp/riak-cs-2.1.0/riak_test/src/downgrade_bitcask.erl 1970-01-01 10:00:00.000000000 +1000
|
|
+++ riak_test/src/downgrade_bitcask.erl 2015-10-22 08:44:18.415188879 +1100
|
|
@@ -0,0 +1,136 @@
|
|
+% Run this script to downgrade Bitcask files from the format
|
|
+% introduced in Riak 2.0 to the format used in Riak 1.4
|
|
+% Run it by calling escript on it and pointing it to a data
|
|
+% directory after stopping the Riak node.
|
|
+% The script will recursively find all Bitcask files under that
|
|
+% directory and reformat them.
|
|
+% $ escript downgrade_bitcask.erl /my/riak/data/bitcask
|
|
+-module(downgrade_bitcask).
|
|
+-mode(compile).
|
|
+-export([main/1]).
|
|
+
|
|
+-define(HEADER_SIZE, 14).
|
|
+-record(entry, { crc, tstamp, keysz, valsz, key, val}).
|
|
+
|
|
+main([DataDir]) ->
|
|
+ downgrade_if_dir(DataDir).
|
|
+
|
|
+maybe_downgrade_file(F) ->
|
|
+ is_bitcask_file(F) andalso downgrade_file(F).
|
|
+
|
|
+downgrade_if_dir(Dir) ->
|
|
+ case filelib:is_dir(Dir) of
|
|
+ true ->
|
|
+ downgrade_dir(Dir);
|
|
+ false ->
|
|
+ ok
|
|
+ end.
|
|
+
|
|
+downgrade_dir(Dir) ->
|
|
+ {ok, Children0} = file:list_dir(Dir),
|
|
+ Children = [filename:join(Dir, Child) || Child <- Children0],
|
|
+ case is_bitcask_dir(Dir) of
|
|
+ false ->
|
|
+ [downgrade_if_dir(Child) || Child <- Children];
|
|
+ true ->
|
|
+ [maybe_downgrade_file(Child) || Child <- Children]
|
|
+ end.
|
|
+
|
|
+is_bitcask_file(Filename0) ->
|
|
+ Filename = filename:basename(Filename0),
|
|
+ Match = re:run(Filename, "^\\d+\\.bitcask\\.data$"),
|
|
+ nomatch =/= Match.
|
|
+
|
|
+is_bitcask_dir(Dir) ->
|
|
+ case filelib:is_dir(Dir) of
|
|
+ false ->
|
|
+ false;
|
|
+ true ->
|
|
+ {ok, Files} = file:list_dir(Dir),
|
|
+ lists:any(fun is_bitcask_file/1, Files)
|
|
+ end.
|
|
+
|
|
+read_entry(F) ->
|
|
+ case file:read(F, ?HEADER_SIZE) of
|
|
+ {ok, <<CRC:32,Tstamp:32,KeySz:16,ValueSz:32>>} ->
|
|
+ case file:read(F, KeySz+ValueSz) of
|
|
+ {ok, <<Key:KeySz/bytes, Value:ValueSz/bytes>>} ->
|
|
+ % io:format("K: ~p, V: ~p\n", [Key, Value]),
|
|
+ {ok, #entry{crc=CRC, tstamp=Tstamp, keysz=KeySz, valsz=ValueSz,
|
|
+ key=Key, val=Value}};
|
|
+ _ ->
|
|
+ error
|
|
+ end;
|
|
+ eof ->
|
|
+ eof;
|
|
+ _ ->
|
|
+ io:format("Error reading entry\n"),
|
|
+ error
|
|
+ end.
|
|
+
|
|
+downgrade_file(F) ->
|
|
+ Dir = filename:dirname(F),
|
|
+ NewF = F ++ ".new",
|
|
+ HintFile = filename:join(Dir, filename:basename(F, ".data")++".hint"),
|
|
+ NewHF = HintFile ++ ".new",
|
|
+ io:format("Downgrading file ~s\n", [F]),
|
|
+ {ok, Fi} = file:open(F, [read, raw, binary]),
|
|
+ {ok, Fo} = file:open(NewF, [write, raw, binary]),
|
|
+ {ok, Fh} = file:open(NewHF, [write, raw, binary]),
|
|
+ ok = convert_file(Fi, Fo, Fh, 0, 0, fun tx_pre_20/1),
|
|
+ ok = file:close(Fi),
|
|
+ ok = file:close(Fo),
|
|
+ ok = file:close(Fh),
|
|
+ HintBak = HintFile ++ ".bak",
|
|
+ FBak = F ++ ".bak",
|
|
+ ok = file:rename(HintFile, HintBak),
|
|
+ ok = file:rename(F, FBak),
|
|
+ ok = file:rename(NewF, F),
|
|
+ ok = file:rename(NewHF, HintFile),
|
|
+ ok = file:delete(HintBak),
|
|
+ ok = file:delete(FBak),
|
|
+ ok.
|
|
+
|
|
+convert_file(Fi, Fo, Fh, Ofs, Crc, Tx) ->
|
|
+ case read_entry(Fi) of
|
|
+ {ok, Entry} ->
|
|
+ NewEntry = Tx(Entry),
|
|
+ Sz = write_entry(Fo, NewEntry),
|
|
+ NewCrc = write_hint_entry(Fh, Ofs, Sz, Crc, NewEntry),
|
|
+ convert_file(Fi, Fo, Fh, Ofs+Sz, NewCrc, Tx);
|
|
+ eof ->
|
|
+ write_hint_entry(Fh, 16#ffffFFFFffffFFFF, Crc, 0,
|
|
+ #entry{key= <<>>, tstamp=0}),
|
|
+ % io:format("Finished reading file\n", []),
|
|
+ ok;
|
|
+ _ ->
|
|
+ io:format(standard_error, "Error reading file\n", []),
|
|
+ error
|
|
+ end.
|
|
+
|
|
+write_hint_entry(F, Ofs, Sz, Crc, #entry{key=Key, tstamp=Tstamp}) ->
|
|
+ KeySz = size(Key),
|
|
+ Hint = [<<Tstamp:32, KeySz:16, Sz:32, Ofs:64>>, Key],
|
|
+ ok = file:write(F, Hint),
|
|
+ erlang:crc32(Crc, Hint).
|
|
+
|
|
+write_entry(F, #entry {key=Key, val=Value, tstamp=Tstamp}) ->
|
|
+ KeySz = size(Key),
|
|
+ ValueSz = size(Value),
|
|
+ Bytes0 = [<<Tstamp:32>>, <<KeySz:16>>, <<ValueSz:32>>, Key, Value],
|
|
+ Bytes = [<<(erlang:crc32(Bytes0)):32>> | Bytes0],
|
|
+ ok = file:write(F, Bytes),
|
|
+ iolist_size(Bytes).
|
|
+
|
|
+tx_pre_20(Entry =
|
|
+ #entry{key= <<2, BucketSz:16, Bucket:BucketSz/binary,
|
|
+ Key/binary>>}) ->
|
|
+ OldKey=term_to_binary({Bucket, Key}),
|
|
+ % io:format("Converted B/K ~s/~s\n", [Bucket, Key]),
|
|
+ tx_pre_20(Entry#entry{key=OldKey, keysz=size(OldKey)});
|
|
+tx_pre_20(Entry=
|
|
+ #entry{val= <<"bitcask_tombstone2", _/binary>>}) ->
|
|
+ NewVal = <<"bitcask_tombstone">>,
|
|
+ Entry#entry{val=NewVal, valsz=size(NewVal)};
|
|
+tx_pre_20(Entry) ->
|
|
+ Entry.
|