1#!/usr/bin/env python
2#
3#  svnfsfs_tests.py:  testing the 'svnfsfs' tool.
4#
5#  Subversion is a tool for revision control.
6#  See http://subversion.apache.org for more information.
7#
8# ====================================================================
9#    Licensed to the Apache Software Foundation (ASF) under one
10#    or more contributor license agreements.  See the NOTICE file
11#    distributed with this work for additional information
12#    regarding copyright ownership.  The ASF licenses this file
13#    to you under the Apache License, Version 2.0 (the
14#    "License"); you may not use this file except in compliance
15#    with the License.  You may obtain a copy of the License at
16#
17#      http://www.apache.org/licenses/LICENSE-2.0
18#
19#    Unless required by applicable law or agreed to in writing,
20#    software distributed under the License is distributed on an
21#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
22#    KIND, either express or implied.  See the License for the
23#    specific language governing permissions and limitations
24#    under the License.
25######################################################################
26
27# General modules
28import os
29import logging
30import re
31import shutil
32import sys
33import threading
34import time
35import gzip
36
37logger = logging.getLogger()
38
39# Our testing module
40import svntest
41from svntest.verify import SVNExpectedStdout, SVNExpectedStderr
42from svntest.verify import SVNUnexpectedStderr
43from svntest.verify import UnorderedOutput
44from svntest.main import SVN_PROP_MERGEINFO
45
46# (abbreviation)
47Skip = svntest.testcase.Skip_deco
48SkipUnless = svntest.testcase.SkipUnless_deco
49XFail = svntest.testcase.XFail_deco
50Issues = svntest.testcase.Issues_deco
51Issue = svntest.testcase.Issue_deco
52Wimp = svntest.testcase.Wimp_deco
53SkipDumpLoadCrossCheck = svntest.testcase.SkipDumpLoadCrossCheck_deco
54Item = svntest.wc.StateItem
55
56#----------------------------------------------------------------------
57
58# How we currently test 'svnfsfs' --
59#
60#   'svnfsfs stats':      Run this on a greek repo, then verify that the
61#                         various sections are present. The section contents
62#                         is matched against section-specific patterns.
63#
64#   'svnfsfs dump-index': Tested implicitly by the load-index test
65#
66#   'svnfsfs load-index': Create a greek repo but set shard to 2 and pack
67#                         it so we can load into a packed shard with more
68#                         than one revision to test ordering issues etc.
69#                         r1 also contains a non-trival number of items such
70#                         that parser issues etc. have a chance to surface.
71#
72#                         The idea is dump the index of the pack, mess with
73#                         it to cover lots of UI guarantees but keep the
74#                         semantics of the relevant bits. Then feed it back
75#                         to load-index and verify that the result is still
76#                         a complete, consistent etc. repo.
77#
78######################################################################
79# Helper routines
80
81def patch_format(repo_dir, shard_size):
82  """Rewrite the format of the FSFS repository REPO_DIR so
83  that it would use sharding with SHARDS revisions per shard."""
84
85  format_path = os.path.join(repo_dir, "db", "format")
86  contents = open(format_path, 'rb').read()
87  processed_lines = []
88
89  for line in contents.split(b"\n"):
90    if line.startswith(b"layout "):
91      processed_lines.append(b"layout sharded %d" % shard_size)
92    else:
93      processed_lines.append(line)
94
95  new_contents = b"\n".join(processed_lines)
96  os.chmod(format_path, svntest.main.S_ALL_RW)
97  with open(format_path, 'wb') as f:
98    f.write(new_contents)
99
100######################################################################
101# Tests
102
103#----------------------------------------------------------------------
104
105@SkipUnless(svntest.main.is_fs_type_fsfs)
106def test_stats(sbox):
107  "stats output"
108
109  sbox.build(create_wc=False)
110
111  exit_code, output, errput = \
112    svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir)
113
114  # split output into sections
115  sections = { }
116
117  last_line = ''
118  section_name = ''
119  section_contents = []
120  for line in output:
121    line = line.rstrip()
122    if line != '':
123
124      # If the first character is not a space, then LINE is a section header
125      if line[0] == ' ':
126        section_contents.append(line)
127      else:
128
129        # Store previous section
130        if section_name != '':
131          sections[section_name] = section_contents
132
133          # Is the output formatted nicely?
134          if last_line != '':
135            logger.warn("Error: no empty line before section '" + line + "'")
136            raise svntest.Failure
137
138        # start new section
139        section_name = line
140        section_contents = []
141
142    last_line = line
143
144  sections[section_name] = section_contents
145
146  # verify that these sections exist
147  sections_to_find = ['Reading revisions',
148                      'Global statistics:',
149                      'Noderev statistics:',
150                      'Representation statistics:',
151                      'Directory representation statistics:',
152                      'File representation statistics:',
153                      'Directory property representation statistics:',
154                      'File property representation statistics:',
155                      'Largest representations:',
156                      'Extensions by number of representations:',
157                      'Extensions by size of changed files:',
158                      'Extensions by size of representations:',
159                      'Histogram of expanded node sizes:',
160                      'Histogram of representation sizes:',
161                      'Histogram of file sizes:',
162                      'Histogram of file representation sizes:',
163                      'Histogram of file property sizes:',
164                      'Histogram of file property representation sizes:',
165                      'Histogram of directory sizes:',
166                      'Histogram of directory representation sizes:',
167                      'Histogram of directory property sizes:',
168                      'Histogram of directory property representation sizes:']
169  patterns_to_find = {
170    'Reading revisions' : ['\s+ 0[ 0-9]*'],
171    'Global .*'         : ['.*\d+ bytes in .*\d+ revisions',
172                           '.*\d+ bytes in .*\d+ changes',
173                           '.*\d+ bytes in .*\d+ node revision records',
174                           '.*\d+ bytes in .*\d+ representations',
175                           '.*\d+ bytes expanded representation size',
176                           '.*\d+ bytes with rep-sharing off' ],
177    'Noderev .*'        : ['.*\d+ bytes in .*\d+ nodes total',
178                           '.*\d+ bytes in .*\d+ directory noderevs',
179                           '.*\d+ bytes in .*\d+ file noderevs' ],
180    'Representation .*' : ['.*\d+ bytes in .*\d+ representations total',
181                           '.*\d+ bytes in .*\d+ directory representations',
182                           '.*\d+ bytes in .*\d+ file representations',
183                           '.*\d+ bytes in .*\d+ representations of added file nodes',
184                           '.*\d+ bytes in .*\d+ directory property representations',
185                           '.*\d+ bytes in .*\d+ file property representations',
186                           '.*\d+ average delta chain length',
187                           '.*\d+ bytes in header & footer overhead' ],
188    '.* representation statistics:' :
189                          ['.*\d+ bytes in .*\d+ reps',
190                           '.*\d+ bytes in .*\d+ shared reps',
191                           '.*\d+ bytes expanded size',
192                           '.*\d+ bytes expanded shared size',
193                           '.*\d+ bytes with rep-sharing off',
194                           '.*\d+ shared references',
195                           '.*\d+ average delta chain length'],
196    'Largest.*:'        : ['.*\d+ r\d+ */\S*'],
197    'Extensions by number .*:' :
198                          ['.*\d+ \( ?\d+%\) representations'],
199    'Extensions by size .*:' :
200                          ['.*\d+ \( ?\d+%\) bytes'],
201    'Histogram of .*:'  : ['.*\d+ \.\. < \d+.*\d+ \( ?\d+%\) bytes in *\d+ \( ?\d+%\) items']
202  }
203
204  # check that the output contains all sections
205  for section_name in sections_to_find:
206    if not section_name in sections.keys():
207      logger.warn("Error: section '" + section_name + "' not found")
208      raise svntest.Failure
209
210  # check section contents
211  for section_name in sections.keys():
212    patterns = []
213
214    # find the suitable patterns for the current section
215    for pattern in patterns_to_find.keys():
216      if re.match(pattern, section_name):
217        patterns = patterns_to_find[pattern]
218        break;
219
220    if len(patterns) == 0:
221      logger.warn("Error: unexpected section '" + section_name + "' found'")
222      logger.warn(sections[section_name])
223      raise svntest.Failure
224
225    # each line in the section must match one of the patterns
226    for line in sections[section_name]:
227      found = False
228
229      for pattern in patterns:
230        if re.match(pattern, line):
231          found = True
232          break
233
234      if not found:
235        logger.warn("Error: unexpected line '" + line + "' in section '"
236                    + section_name + "'")
237        logger.warn(sections[section_name])
238        raise svntest.Failure
239
240#----------------------------------------------------------------------
241
242@SkipUnless(svntest.main.is_fs_type_fsfs)
243@SkipUnless(svntest.main.fs_has_pack)
244@SkipUnless(svntest.main.is_fs_log_addressing)
245def load_index_sharded(sbox):
246  "load-index in a packed repo"
247
248  # Configure two files per shard to trigger packing.
249  sbox.build(create_wc=False)
250  patch_format(sbox.repo_dir, shard_size=2)
251
252  expected_output = ["Packing revisions in shard 0...done.\n"]
253  svntest.actions.run_and_verify_svnadmin(expected_output, [],
254                                          "pack", sbox.repo_dir)
255
256  # Read P2L index using svnfsfs.
257  exit_code, items, errput = \
258    svntest.actions.run_and_verify_svnfsfs(None, [], "dump-index", "-r0",
259                                           sbox.repo_dir)
260
261  # load-index promises to deal with input that
262  #
263  # * uses the same encoding as the dump-index output
264  # * is not in ascending item offset order
265  # * contains lines with the full table header
266  # * invalid or incorrect data in the checksum column and beyond
267  # * starts with an item which does not belong to the first revision
268  #   in the pack file
269  #
270  # So, let's mess with the ITEMS list to call in on these promises.
271
272  # not in ascending order
273  items.reverse()
274
275  # multiple headers (there is already one now at the bottom)
276  items.insert(0, "       Start       Length Type   Revision     Item Checksum\n")
277
278  # make columns have a variable size
279  # mess with the checksums
280  # add a junk column
281  # keep header lines as are
282  for i in range(0, len(items)):
283    if items[i].find("Start") == -1:
284      columns = items[i].split()
285      columns[5] = columns[5].replace('f','-').replace('0','9')
286      columns.append("junk")
287      items[i] = ' '.join(columns) + "\n"
288
289  # first entry shall be for rev 1, pack starts at rev 0, though
290  for i in range(0, len(items)):
291    if items[i].split()[3] == "1":
292      if i != 1:
293        items[i],items[1] = items[1],items[i]
294      break
295
296  assert(items[1].split()[3] == "1")
297
298  # The STDIN data must be binary.
299  items = svntest.main.ensure_list(map(str.encode, items))
300
301  # Reload the index
302  exit_code, output, errput = svntest.main.run_command_stdin(
303    svntest.main.svnfsfs_binary, [], 0, False, items,
304    "load-index", sbox.repo_dir)
305
306  # Run verify to see whether we broke anything.
307  expected_output = ["* Verifying metadata at revision 0 ...\n",
308                     "* Verifying repository metadata ...\n",
309                     "* Verified revision 0.\n",
310                     "* Verified revision 1.\n"]
311  svntest.actions.run_and_verify_svnadmin(expected_output, [],
312                                          "verify", sbox.repo_dir)
313
314@SkipUnless(svntest.main.is_fs_type_fsfs)
315def test_stats_on_empty_repo(sbox):
316  "stats on empty repo shall not crash"
317
318  sbox.build(create_wc=False, empty=True)
319
320  exit_code, output, errput = \
321    svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir)
322
323########################################################################
324# Run the tests
325
326
327# list all tests here, starting with None:
328test_list = [ None,
329              test_stats,
330              load_index_sharded,
331              test_stats_on_empty_repo,
332             ]
333
334if __name__ == '__main__':
335  svntest.main.run_tests(test_list)
336  # NOTREACHED
337
338
339### End of file.
340