1# Licensed under a 3-clause BSD style license - see LICENSE.rst
2import argparse
3import glob
4import logging
5import os
6import sys
7
8from astropy.io import fits
9from astropy.io.fits.util import fill
10from astropy import __version__
11
12
13log = logging.getLogger('fitsdiff')
14
15
16DESCRIPTION = """
17Compare two FITS image files and report the differences in header keywords and
18data.
19
20    fitsdiff [options] filename1 filename2
21
22where filename1 filename2 are the two files to be compared.  They may also be
23wild cards, in such cases, they must be enclosed by double or single quotes, or
24they may be directory names.  If both are directory names, all files in each of
25the directories will be included; if only one is a directory name, then the
26directory name will be prefixed to the file name(s) specified by the other
27argument.  for example::
28
29    fitsdiff "*.fits" "/machine/data1"
30
31will compare all FITS files in the current directory to the corresponding files
32in the directory /machine/data1.
33
34This script is part of the Astropy package. See
35https://docs.astropy.org/en/latest/io/fits/usage/scripts.html#fitsdiff
36for further documentation.
37""".strip()
38
39
40EPILOG = fill("""
41If the two files are identical within the specified conditions, it will report
42"No difference is found." If the value(s) of -c and -k takes the form
43'@filename', list is in the text file 'filename', and each line in that text
44file contains one keyword.
45
46Example
47-------
48
49    fitsdiff -k filename,filtnam1 -n 5 -r 1.e-6 test1.fits test2
50
51This command will compare files test1.fits and test2.fits, report maximum of 5
52different pixels values per extension, only report data values larger than
531.e-6 relative to each other, and will neglect the different values of keywords
54FILENAME and FILTNAM1 (or their very existence).
55
56fitsdiff command-line arguments can also be set using the environment variable
57FITSDIFF_SETTINGS.  If the FITSDIFF_SETTINGS environment variable is present,
58each argument present will override the corresponding argument on the
59command-line unless the --exact option is specified.  The FITSDIFF_SETTINGS
60environment variable exists to make it easier to change the
61behavior of fitsdiff on a global level, such as in a set of regression tests.
62""".strip(), width=80)
63
64
65class StoreListAction(argparse.Action):
66    def __init__(self, option_strings, dest, nargs=None, **kwargs):
67        if nargs is not None:
68            raise ValueError("nargs not allowed")
69        super().__init__(option_strings, dest, nargs, **kwargs)
70
71    def __call__(self, parser, namespace, values, option_string=None):
72        setattr(namespace, self.dest, [])
73        # Accept either a comma-separated list or a filename (starting with @)
74        # containing a value on each line
75        if values and values[0] == '@':
76            value = values[1:]
77            if not os.path.exists(value):
78                log.warning(f'{self.dest} argument {value} does not exist')
79                return
80            try:
81                values = [v.strip() for v in open(value, 'r').readlines()]
82                setattr(namespace, self.dest, values)
83            except OSError as exc:
84                log.warning('reading {} for {} failed: {}; ignoring this '
85                            'argument'.format(value, self.dest, exc))
86                del exc
87        else:
88            setattr(namespace, self.dest,
89                    [v.strip() for v in values.split(',')])
90
91
92def handle_options(argv=None):
93    parser = argparse.ArgumentParser(
94        description=DESCRIPTION, epilog=EPILOG,
95        formatter_class=argparse.RawDescriptionHelpFormatter)
96
97    parser.add_argument(
98        '--version', action='version',
99        version=f'%(prog)s {__version__}')
100
101    parser.add_argument(
102        'fits_files', metavar='file', nargs='+',
103        help='.fits files to process.')
104
105    parser.add_argument(
106        '-q', '--quiet', action='store_true',
107        help='Produce no output and just return a status code.')
108
109    parser.add_argument(
110        '-n', '--num-diffs', type=int, default=10, dest='numdiffs',
111        metavar='INTEGER',
112        help='Max number of data differences (image pixel or table element) '
113             'to report per extension (default %(default)s).')
114
115    parser.add_argument(
116        '-r', '--rtol', '--relative-tolerance', type=float, default=None,
117        dest='rtol', metavar='NUMBER',
118        help='The relative tolerance for comparison of two numbers, '
119             'specifically two floating point numbers.  This applies to data '
120             'in both images and tables, and to floating point keyword values '
121             'in headers (default %(default)s).')
122
123    parser.add_argument(
124        '-a', '--atol', '--absolute-tolerance', type=float, default=None,
125        dest='atol', metavar='NUMBER',
126        help='The absolute tolerance for comparison of two numbers, '
127             'specifically two floating point numbers.  This applies to data '
128             'in both images and tables, and to floating point keyword values '
129             'in headers (default %(default)s).')
130
131    parser.add_argument(
132        '-b', '--no-ignore-blanks', action='store_false',
133        dest='ignore_blanks', default=True,
134        help="Don't ignore trailing blanks (whitespace) in string values.  "
135             "Otherwise trailing blanks both in header keywords/values and in "
136             "table column values) are not treated as significant i.e., "
137             "without this option 'ABCDEF   ' and 'ABCDEF' are considered "
138             "equivalent. ")
139
140    parser.add_argument(
141        '--no-ignore-blank-cards', action='store_false',
142        dest='ignore_blank_cards', default=True,
143        help="Don't ignore entirely blank cards in headers.  Normally fitsdiff "
144             "does not consider blank cards when comparing headers, but this "
145             "will ensure that even blank cards match up. ")
146
147    parser.add_argument(
148        '--exact', action='store_true',
149        dest='exact_comparisons', default=False,
150        help="Report ALL differences, "
151             "overriding command-line options and FITSDIFF_SETTINGS. ")
152
153    parser.add_argument(
154        '-o', '--output-file', metavar='FILE',
155        help='Output results to this file; otherwise results are printed to '
156             'stdout.')
157
158    parser.add_argument(
159        '-u', '--ignore-hdus', action=StoreListAction,
160        default=[], dest='ignore_hdus',
161        metavar='HDU_NAMES',
162        help='Comma-separated list of HDU names not to be compared.  HDU '
163             'names may contain wildcard patterns.')
164
165    group = parser.add_argument_group('Header Comparison Options')
166
167    group.add_argument(
168        '-k', '--ignore-keywords', action=StoreListAction,
169        default=[], dest='ignore_keywords',
170        metavar='KEYWORDS',
171        help='Comma-separated list of keywords not to be compared.  Keywords '
172             'may contain wildcard patterns.  To exclude all keywords, use '
173             '"*"; make sure to have double or single quotes around the '
174             'asterisk on the command-line.')
175
176    group.add_argument(
177        '-c', '--ignore-comments', action=StoreListAction,
178        default=[], dest='ignore_comments',
179        metavar='COMMENTS',
180        help='Comma-separated list of keywords whose comments will not be '
181             'compared.  Wildcards may be used as with --ignore-keywords.')
182
183    group = parser.add_argument_group('Table Comparison Options')
184
185    group.add_argument(
186        '-f', '--ignore-fields', action=StoreListAction,
187        default=[], dest='ignore_fields',
188        metavar='COLUMNS',
189        help='Comma-separated list of fields (i.e. columns) not to be '
190             'compared.  All columns may be excluded using "*" as with '
191             '--ignore-keywords.')
192
193    options = parser.parse_args(argv)
194
195    # Determine which filenames to compare
196    if len(options.fits_files) != 2:
197        parser.error('\nfitsdiff requires two arguments; '
198                     'see `fitsdiff --help` for more details.')
199
200    return options
201
202
203def setup_logging(outfile=None):
204    log.setLevel(logging.INFO)
205    error_handler = logging.StreamHandler(sys.stderr)
206    error_handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
207    error_handler.setLevel(logging.WARNING)
208    log.addHandler(error_handler)
209
210    if outfile is not None:
211        output_handler = logging.FileHandler(outfile)
212    else:
213        output_handler = logging.StreamHandler()
214
215        class LevelFilter(logging.Filter):
216            """Log only messages matching the specified level."""
217
218            def __init__(self, name='', level=logging.NOTSET):
219                logging.Filter.__init__(self, name)
220                self.level = level
221
222            def filter(self, rec):
223                return rec.levelno == self.level
224
225        # File output logs all messages, but stdout logs only INFO messages
226        # (since errors are already logged to stderr)
227        output_handler.addFilter(LevelFilter(level=logging.INFO))
228
229    output_handler.setFormatter(logging.Formatter('%(message)s'))
230    log.addHandler(output_handler)
231
232
233def match_files(paths):
234    if os.path.isfile(paths[0]) and os.path.isfile(paths[1]):
235        # shortcut if both paths are files
236        return [paths]
237
238    dirnames = [None, None]
239    filelists = [None, None]
240
241    for i, path in enumerate(paths):
242        if glob.has_magic(path):
243            files = [os.path.split(f) for f in glob.glob(path)]
244            if not files:
245                log.error('Wildcard pattern %r did not match any files.', path)
246                sys.exit(2)
247
248            dirs, files = list(zip(*files))
249            if len(set(dirs)) > 1:
250                log.error('Wildcard pattern %r should match only one '
251                          'directory.', path)
252                sys.exit(2)
253
254            dirnames[i] = set(dirs).pop()
255            filelists[i] = sorted(files)
256        elif os.path.isdir(path):
257            dirnames[i] = path
258            filelists[i] = [f for f in sorted(os.listdir(path))
259                            if os.path.isfile(os.path.join(path, f))]
260        elif os.path.isfile(path):
261            dirnames[i] = os.path.dirname(path)
262            filelists[i] = [os.path.basename(path)]
263        else:
264            log.error(
265                '%r is not an existing file, directory, or wildcard '
266                'pattern; see `fitsdiff --help` for more usage help.', path)
267            sys.exit(2)
268
269        dirnames[i] = os.path.abspath(dirnames[i])
270
271    filematch = set(filelists[0]) & set(filelists[1])
272
273    for a, b in [(0, 1), (1, 0)]:
274        if len(filelists[a]) > len(filematch) and not os.path.isdir(paths[a]):
275            for extra in sorted(set(filelists[a]) - filematch):
276                log.warning('%r has no match in %r', extra, dirnames[b])
277
278    return [(os.path.join(dirnames[0], f),
279             os.path.join(dirnames[1], f)) for f in filematch]
280
281
282def main(args=None):
283    args = args or sys.argv[1:]
284
285    if 'FITSDIFF_SETTINGS' in os.environ:
286        args = os.environ['FITSDIFF_SETTINGS'].split() + args
287
288    opts = handle_options(args)
289
290    if opts.rtol is None:
291        opts.rtol = 0.0
292    if opts.atol is None:
293        opts.atol = 0.0
294
295    if opts.exact_comparisons:
296        # override the options so that each is the most restrictive
297        opts.ignore_keywords = []
298        opts.ignore_comments = []
299        opts.ignore_fields = []
300        opts.rtol = 0.0
301        opts.atol = 0.0
302        opts.ignore_blanks = False
303        opts.ignore_blank_cards = False
304
305    if not opts.quiet:
306        setup_logging(opts.output_file)
307    files = match_files(opts.fits_files)
308
309    close_file = False
310    if opts.quiet:
311        out_file = None
312    elif opts.output_file:
313        out_file = open(opts.output_file, 'w')
314        close_file = True
315    else:
316        out_file = sys.stdout
317
318    identical = []
319    try:
320        for a, b in files:
321            # TODO: pass in any additional arguments here too
322            diff = fits.diff.FITSDiff(
323                a, b,
324                ignore_hdus=opts.ignore_hdus,
325                ignore_keywords=opts.ignore_keywords,
326                ignore_comments=opts.ignore_comments,
327                ignore_fields=opts.ignore_fields,
328                numdiffs=opts.numdiffs,
329                rtol=opts.rtol,
330                atol=opts.atol,
331                ignore_blanks=opts.ignore_blanks,
332                ignore_blank_cards=opts.ignore_blank_cards)
333
334            diff.report(fileobj=out_file)
335            identical.append(diff.identical)
336
337        return int(not all(identical))
338    finally:
339        if close_file:
340            out_file.close()
341        # Close the file if used for the logging output, and remove handlers to
342        # avoid having them multiple times for unit tests.
343        for handler in log.handlers:
344            if isinstance(handler, logging.FileHandler):
345                handler.close()
346            log.removeHandler(handler)
347