1% Licensed under the Apache License, Version 2.0 (the "License"); you may not 2% use this file except in compliance with the License. You may obtain a copy of 3% the License at 4% 5% http://www.apache.org/licenses/LICENSE-2.0 6% 7% Unless required by applicable law or agreed to in writing, software 8% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10% License for the specific language governing permissions and limitations under 11% the License. 12 13-module(couch_replicator_many_leaves_tests). 14 15-include_lib("couch/include/couch_eunit.hrl"). 16-include_lib("couch/include/couch_db.hrl"). 17 18-import(couch_replicator_test_helper, [ 19 db_url/1, 20 replicate/2 21]). 22 23-define(DOCS_CONFLICTS, [ 24 {<<"doc1">>, 10}, 25 % use some _design docs as well to test the special handling for them 26 {<<"_design/doc2">>, 100}, 27 % a number > MaxURLlength (7000) / length(DocRevisionString) 28 {<<"doc3">>, 210} 29]). 30-define(NUM_ATTS, 2). 31-define(TIMEOUT_EUNIT, 60). 32-define(i2l(I), integer_to_list(I)). 33-define(io2b(Io), iolist_to_binary(Io)). 34 35setup() -> 36 DbName = ?tempdb(), 37 {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), 38 ok = couch_db:close(Db), 39 DbName. 40 41 42setup(remote) -> 43 {remote, setup()}; 44setup({A, B}) -> 45 Ctx = test_util:start_couch([couch_replicator]), 46 Source = setup(A), 47 Target = setup(B), 48 {Ctx, {Source, Target}}. 49 50teardown({remote, DbName}) -> 51 teardown(DbName); 52teardown(DbName) -> 53 ok = couch_server:delete(DbName, [?ADMIN_CTX]), 54 ok. 55 56teardown(_, {Ctx, {Source, Target}}) -> 57 teardown(Source), 58 teardown(Target), 59 ok = application:stop(couch_replicator), 60 ok = test_util:stop_couch(Ctx). 61 62docs_with_many_leaves_test_() -> 63 Pairs = [{remote, remote}], 64 { 65 "Replicate documents with many leaves", 66 { 67 foreachx, 68 fun setup/1, fun teardown/2, 69 [{Pair, fun should_populate_replicate_compact/2} 70 || Pair <- Pairs] 71 } 72 }. 73 74 75should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) -> 76 {lists:flatten(io_lib:format("~p -> ~p", [From, To])), 77 {inorder, [ 78 should_populate_source(Source), 79 should_replicate(Source, Target), 80 should_verify_target(Source, Target), 81 should_add_attachments_to_source(Source), 82 should_replicate(Source, Target), 83 should_verify_target(Source, Target) 84 ]}}. 85 86should_populate_source({remote, Source}) -> 87 should_populate_source(Source); 88should_populate_source(Source) -> 89 {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}. 90 91should_replicate({remote, Source}, Target) -> 92 should_replicate(db_url(Source), Target); 93should_replicate(Source, {remote, Target}) -> 94 should_replicate(Source, db_url(Target)); 95should_replicate(Source, Target) -> 96 {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}. 97 98should_verify_target({remote, Source}, Target) -> 99 should_verify_target(Source, Target); 100should_verify_target(Source, {remote, Target}) -> 101 should_verify_target(Source, Target); 102should_verify_target(Source, Target) -> 103 {timeout, ?TIMEOUT_EUNIT, ?_test(begin 104 {ok, SourceDb} = couch_db:open_int(Source, []), 105 {ok, TargetDb} = couch_db:open_int(Target, []), 106 verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS), 107 ok = couch_db:close(SourceDb), 108 ok = couch_db:close(TargetDb) 109 end)}. 110 111should_add_attachments_to_source({remote, Source}) -> 112 should_add_attachments_to_source(Source); 113should_add_attachments_to_source(Source) -> 114 {timeout, ?TIMEOUT_EUNIT, ?_test(begin 115 {ok, SourceDb} = couch_db:open_int(Source, [?ADMIN_CTX]), 116 add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS), 117 ok = couch_db:close(SourceDb) 118 end)}. 119 120populate_db(DbName) -> 121 {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]), 122 lists:foreach( 123 fun({DocId, NumConflicts}) -> 124 Value = <<"0">>, 125 Doc = #doc{ 126 id = DocId, 127 body = {[ {<<"value">>, Value} ]} 128 }, 129 {ok, _} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]), 130 {ok, _} = add_doc_siblings(Db, DocId, NumConflicts) 131 end, ?DOCS_CONFLICTS), 132 couch_db:close(Db). 133 134add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 -> 135 add_doc_siblings(Db, DocId, NumLeaves, [], []). 136 137add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) -> 138 {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes), 139 {ok, AccRevs}; 140 141add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) -> 142 Value = ?l2b(?i2l(NumLeaves)), 143 Rev = couch_hash:md5_hash(Value), 144 Doc = #doc{ 145 id = DocId, 146 revs = {1, [Rev]}, 147 body = {[ {<<"value">>, Value} ]} 148 }, 149 add_doc_siblings(Db, DocId, NumLeaves - 1, 150 [Doc | AccDocs], [{1, Rev} | AccRevs]). 151 152verify_target(_SourceDb, _TargetDb, []) -> 153 ok; 154verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) -> 155 {ok, SourceLookups} = couch_db:open_doc_revs( 156 SourceDb, 157 DocId, 158 all, 159 [conflicts, deleted_conflicts]), 160 {ok, TargetLookups} = couch_db:open_doc_revs( 161 TargetDb, 162 DocId, 163 all, 164 [conflicts, deleted_conflicts]), 165 SourceDocs = [Doc || {ok, Doc} <- SourceLookups], 166 TargetDocs = [Doc || {ok, Doc} <- TargetLookups], 167 Total = NumConflicts + 1, 168 ?assertEqual(Total, length(TargetDocs)), 169 lists:foreach( 170 fun({SourceDoc, TargetDoc}) -> 171 SourceJson = couch_doc:to_json_obj(SourceDoc, [attachments]), 172 TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]), 173 ?assertEqual(SourceJson, TargetJson) 174 end, 175 lists:zip(SourceDocs, TargetDocs)), 176 verify_target(SourceDb, TargetDb, Rest). 177 178add_attachments(_SourceDb, _NumAtts, []) -> 179 ok; 180add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) -> 181 {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []), 182 SourceDocs = [Doc || {ok, Doc} <- SourceLookups], 183 Total = NumConflicts + 1, 184 ?assertEqual(Total, length(SourceDocs)), 185 NewDocs = lists:foldl( 186 fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) -> 187 NewAtts = lists:foldl(fun(I, AttAcc) -> 188 AttData = crypto:strong_rand_bytes(100), 189 NewAtt = couch_att:new([ 190 {name, ?io2b(["att_", ?i2l(I), "_", 191 couch_doc:rev_to_str({Pos, Rev})])}, 192 {type, <<"application/foobar">>}, 193 {att_len, byte_size(AttData)}, 194 {data, AttData} 195 ]), 196 [NewAtt | AttAcc] 197 end, [], lists:seq(1, NumAtts)), 198 [Doc#doc{atts = Atts ++ NewAtts} | Acc] 199 end, 200 [], SourceDocs), 201 {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []), 202 NewRevs = [R || {ok, R} <- UpdateResults], 203 ?assertEqual(length(NewDocs), length(NewRevs)), 204 add_attachments(SourceDb, NumAtts, Rest). 205 206