1#!/usr/bin/env python 2# 3# svnfsfs_tests.py: testing the 'svnfsfs' tool. 4# 5# Subversion is a tool for revision control. 6# See http://subversion.apache.org for more information. 7# 8# ==================================================================== 9# Licensed to the Apache Software Foundation (ASF) under one 10# or more contributor license agreements. See the NOTICE file 11# distributed with this work for additional information 12# regarding copyright ownership. The ASF licenses this file 13# to you under the Apache License, Version 2.0 (the 14# "License"); you may not use this file except in compliance 15# with the License. You may obtain a copy of the License at 16# 17# http://www.apache.org/licenses/LICENSE-2.0 18# 19# Unless required by applicable law or agreed to in writing, 20# software distributed under the License is distributed on an 21# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 22# KIND, either express or implied. See the License for the 23# specific language governing permissions and limitations 24# under the License. 25###################################################################### 26 27# General modules 28import os 29import logging 30import re 31import shutil 32import sys 33import threading 34import time 35import gzip 36 37logger = logging.getLogger() 38 39# Our testing module 40import svntest 41from svntest.verify import SVNExpectedStdout, SVNExpectedStderr 42from svntest.verify import SVNUnexpectedStderr 43from svntest.verify import UnorderedOutput 44from svntest.main import SVN_PROP_MERGEINFO 45 46# (abbreviation) 47Skip = svntest.testcase.Skip_deco 48SkipUnless = svntest.testcase.SkipUnless_deco 49XFail = svntest.testcase.XFail_deco 50Issues = svntest.testcase.Issues_deco 51Issue = svntest.testcase.Issue_deco 52Wimp = svntest.testcase.Wimp_deco 53SkipDumpLoadCrossCheck = svntest.testcase.SkipDumpLoadCrossCheck_deco 54Item = svntest.wc.StateItem 55 56#---------------------------------------------------------------------- 57 58# How we currently test 'svnfsfs' -- 59# 60# 'svnfsfs stats': Run this on a greek repo, then verify that the 61# various sections are present. The section contents 62# is matched against section-specific patterns. 63# 64# 'svnfsfs dump-index': Tested implicitly by the load-index test 65# 66# 'svnfsfs load-index': Create a greek repo but set shard to 2 and pack 67# it so we can load into a packed shard with more 68# than one revision to test ordering issues etc. 69# r1 also contains a non-trival number of items such 70# that parser issues etc. have a chance to surface. 71# 72# The idea is dump the index of the pack, mess with 73# it to cover lots of UI guarantees but keep the 74# semantics of the relevant bits. Then feed it back 75# to load-index and verify that the result is still 76# a complete, consistent etc. repo. 77# 78###################################################################### 79# Helper routines 80 81def patch_format(repo_dir, shard_size): 82 """Rewrite the format of the FSFS repository REPO_DIR so 83 that it would use sharding with SHARDS revisions per shard.""" 84 85 format_path = os.path.join(repo_dir, "db", "format") 86 contents = open(format_path, 'rb').read() 87 processed_lines = [] 88 89 for line in contents.split(b"\n"): 90 if line.startswith(b"layout "): 91 processed_lines.append(b"layout sharded %d" % shard_size) 92 else: 93 processed_lines.append(line) 94 95 new_contents = b"\n".join(processed_lines) 96 os.chmod(format_path, svntest.main.S_ALL_RW) 97 with open(format_path, 'wb') as f: 98 f.write(new_contents) 99 100###################################################################### 101# Tests 102 103#---------------------------------------------------------------------- 104 105@SkipUnless(svntest.main.is_fs_type_fsfs) 106def test_stats(sbox): 107 "stats output" 108 109 sbox.build(create_wc=False) 110 111 exit_code, output, errput = \ 112 svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir) 113 114 # split output into sections 115 sections = { } 116 117 last_line = '' 118 section_name = '' 119 section_contents = [] 120 for line in output: 121 line = line.rstrip() 122 if line != '': 123 124 # If the first character is not a space, then LINE is a section header 125 if line[0] == ' ': 126 section_contents.append(line) 127 else: 128 129 # Store previous section 130 if section_name != '': 131 sections[section_name] = section_contents 132 133 # Is the output formatted nicely? 134 if last_line != '': 135 logger.warn("Error: no empty line before section '" + line + "'") 136 raise svntest.Failure 137 138 # start new section 139 section_name = line 140 section_contents = [] 141 142 last_line = line 143 144 sections[section_name] = section_contents 145 146 # verify that these sections exist 147 sections_to_find = ['Reading revisions', 148 'Global statistics:', 149 'Noderev statistics:', 150 'Representation statistics:', 151 'Directory representation statistics:', 152 'File representation statistics:', 153 'Directory property representation statistics:', 154 'File property representation statistics:', 155 'Largest representations:', 156 'Extensions by number of representations:', 157 'Extensions by size of changed files:', 158 'Extensions by size of representations:', 159 'Histogram of expanded node sizes:', 160 'Histogram of representation sizes:', 161 'Histogram of file sizes:', 162 'Histogram of file representation sizes:', 163 'Histogram of file property sizes:', 164 'Histogram of file property representation sizes:', 165 'Histogram of directory sizes:', 166 'Histogram of directory representation sizes:', 167 'Histogram of directory property sizes:', 168 'Histogram of directory property representation sizes:'] 169 patterns_to_find = { 170 'Reading revisions' : ['\s+ 0[ 0-9]*'], 171 'Global .*' : ['.*\d+ bytes in .*\d+ revisions', 172 '.*\d+ bytes in .*\d+ changes', 173 '.*\d+ bytes in .*\d+ node revision records', 174 '.*\d+ bytes in .*\d+ representations', 175 '.*\d+ bytes expanded representation size', 176 '.*\d+ bytes with rep-sharing off' ], 177 'Noderev .*' : ['.*\d+ bytes in .*\d+ nodes total', 178 '.*\d+ bytes in .*\d+ directory noderevs', 179 '.*\d+ bytes in .*\d+ file noderevs' ], 180 'Representation .*' : ['.*\d+ bytes in .*\d+ representations total', 181 '.*\d+ bytes in .*\d+ directory representations', 182 '.*\d+ bytes in .*\d+ file representations', 183 '.*\d+ bytes in .*\d+ representations of added file nodes', 184 '.*\d+ bytes in .*\d+ directory property representations', 185 '.*\d+ bytes in .*\d+ file property representations', 186 '.*\d+ average delta chain length', 187 '.*\d+ bytes in header & footer overhead' ], 188 '.* representation statistics:' : 189 ['.*\d+ bytes in .*\d+ reps', 190 '.*\d+ bytes in .*\d+ shared reps', 191 '.*\d+ bytes expanded size', 192 '.*\d+ bytes expanded shared size', 193 '.*\d+ bytes with rep-sharing off', 194 '.*\d+ shared references', 195 '.*\d+ average delta chain length'], 196 'Largest.*:' : ['.*\d+ r\d+ */\S*'], 197 'Extensions by number .*:' : 198 ['.*\d+ \( ?\d+%\) representations'], 199 'Extensions by size .*:' : 200 ['.*\d+ \( ?\d+%\) bytes'], 201 'Histogram of .*:' : ['.*\d+ \.\. < \d+.*\d+ \( ?\d+%\) bytes in *\d+ \( ?\d+%\) items'] 202 } 203 204 # check that the output contains all sections 205 for section_name in sections_to_find: 206 if not section_name in sections.keys(): 207 logger.warn("Error: section '" + section_name + "' not found") 208 raise svntest.Failure 209 210 # check section contents 211 for section_name in sections.keys(): 212 patterns = [] 213 214 # find the suitable patterns for the current section 215 for pattern in patterns_to_find.keys(): 216 if re.match(pattern, section_name): 217 patterns = patterns_to_find[pattern] 218 break; 219 220 if len(patterns) == 0: 221 logger.warn("Error: unexpected section '" + section_name + "' found'") 222 logger.warn(sections[section_name]) 223 raise svntest.Failure 224 225 # each line in the section must match one of the patterns 226 for line in sections[section_name]: 227 found = False 228 229 for pattern in patterns: 230 if re.match(pattern, line): 231 found = True 232 break 233 234 if not found: 235 logger.warn("Error: unexpected line '" + line + "' in section '" 236 + section_name + "'") 237 logger.warn(sections[section_name]) 238 raise svntest.Failure 239 240#---------------------------------------------------------------------- 241 242@SkipUnless(svntest.main.is_fs_type_fsfs) 243@SkipUnless(svntest.main.fs_has_pack) 244@SkipUnless(svntest.main.is_fs_log_addressing) 245def load_index_sharded(sbox): 246 "load-index in a packed repo" 247 248 # Configure two files per shard to trigger packing. 249 sbox.build(create_wc=False) 250 patch_format(sbox.repo_dir, shard_size=2) 251 252 expected_output = ["Packing revisions in shard 0...done.\n"] 253 svntest.actions.run_and_verify_svnadmin(expected_output, [], 254 "pack", sbox.repo_dir) 255 256 # Read P2L index using svnfsfs. 257 exit_code, items, errput = \ 258 svntest.actions.run_and_verify_svnfsfs(None, [], "dump-index", "-r0", 259 sbox.repo_dir) 260 261 # load-index promises to deal with input that 262 # 263 # * uses the same encoding as the dump-index output 264 # * is not in ascending item offset order 265 # * contains lines with the full table header 266 # * invalid or incorrect data in the checksum column and beyond 267 # * starts with an item which does not belong to the first revision 268 # in the pack file 269 # 270 # So, let's mess with the ITEMS list to call in on these promises. 271 272 # not in ascending order 273 items.reverse() 274 275 # multiple headers (there is already one now at the bottom) 276 items.insert(0, " Start Length Type Revision Item Checksum\n") 277 278 # make columns have a variable size 279 # mess with the checksums 280 # add a junk column 281 # keep header lines as are 282 for i in range(0, len(items)): 283 if items[i].find("Start") == -1: 284 columns = items[i].split() 285 columns[5] = columns[5].replace('f','-').replace('0','9') 286 columns.append("junk") 287 items[i] = ' '.join(columns) + "\n" 288 289 # first entry shall be for rev 1, pack starts at rev 0, though 290 for i in range(0, len(items)): 291 if items[i].split()[3] == "1": 292 if i != 1: 293 items[i],items[1] = items[1],items[i] 294 break 295 296 assert(items[1].split()[3] == "1") 297 298 # The STDIN data must be binary. 299 items = svntest.main.ensure_list(map(str.encode, items)) 300 301 # Reload the index 302 exit_code, output, errput = svntest.main.run_command_stdin( 303 svntest.main.svnfsfs_binary, [], 0, False, items, 304 "load-index", sbox.repo_dir) 305 306 # Run verify to see whether we broke anything. 307 expected_output = ["* Verifying metadata at revision 0 ...\n", 308 "* Verifying repository metadata ...\n", 309 "* Verified revision 0.\n", 310 "* Verified revision 1.\n"] 311 svntest.actions.run_and_verify_svnadmin(expected_output, [], 312 "verify", sbox.repo_dir) 313 314@SkipUnless(svntest.main.is_fs_type_fsfs) 315def test_stats_on_empty_repo(sbox): 316 "stats on empty repo shall not crash" 317 318 sbox.build(create_wc=False, empty=True) 319 320 exit_code, output, errput = \ 321 svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir) 322 323######################################################################## 324# Run the tests 325 326 327# list all tests here, starting with None: 328test_list = [ None, 329 test_stats, 330 load_index_sharded, 331 test_stats_on_empty_repo, 332 ] 333 334if __name__ == '__main__': 335 svntest.main.run_tests(test_list) 336 # NOTREACHED 337 338 339### End of file. 340