1from __future__ import absolute_import
2from __future__ import unicode_literals
3
4from urllib.parse import unquote, quote_plus
5from datetime import datetime, timedelta
6from itertools import tee
7from flask import (
8    render_template, abort, url_for,
9    Response, stream_with_context, request, session, jsonify
10)
11
12import logging
13import json
14from json import dumps
15
16from pypuppetdb.QueryBuilder import (ExtractOperator, AndOperator,
17                                     EqualsOperator, FunctionOperator,
18                                     NullOperator, OrOperator,
19                                     LessEqualOperator, RegexOperator,
20                                     GreaterEqualOperator)
21
22from puppetboard.forms import ENABLED_QUERY_ENDPOINTS, QueryForm
23from puppetboard.utils import (get_or_abort, yield_or_stop,
24                               get_db_version, parse_python)
25from puppetboard.dailychart import get_daily_reports_chart
26
27import commonmark
28
29from puppetboard.core import get_app, get_puppetdb, environments
30
31from puppetboard.version import __version__
32
33REPORTS_COLUMNS = [
34    {'attr': 'end', 'filter': 'end_time',
35     'name': 'End time', 'type': 'datetime'},
36    {'attr': 'status', 'name': 'Status', 'type': 'status'},
37    {'attr': 'certname', 'name': 'Certname', 'type': 'node'},
38    {'attr': 'version', 'filter': 'configuration_version',
39     'name': 'Configuration version'},
40    {'attr': 'agent_version', 'filter': 'puppet_version',
41     'name': 'Agent version'},
42]
43
44CATALOGS_COLUMNS = [
45    {'attr': 'certname', 'name': 'Certname', 'type': 'node'},
46    {'attr': 'catalog_timestamp', 'name': 'Compile Time'},
47    {'attr': 'form', 'name': 'Compare'},
48]
49
50app = get_app()
51graph_facts = app.config['GRAPH_FACTS']
52numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None)
53
54logging.basicConfig(level=numeric_level)
55log = logging.getLogger(__name__)
56
57puppetdb = get_puppetdb()
58
59
60menu_entries = [
61    ('index', 'Overview'),
62    ('nodes', 'Nodes'),
63    ('facts', 'Facts'),
64    ('reports', 'Reports'),
65    ('metrics', 'Metrics'),
66    ('inventory', 'Inventory'),
67    ('catalogs', 'Catalogs'),
68    ('radiator', 'Radiator'),
69    ('query', 'Query')
70]
71
72if not app.config.get('ENABLE_QUERY'):
73    menu_entries.remove(('query', 'Query'))
74
75if not app.config.get('ENABLE_CATALOG'):
76    menu_entries.remove(('catalogs', 'Catalogs'))
77
78
79app.jinja_env.globals.update(menu_entries=menu_entries)
80
81
82@app.template_global()
83def version():
84    return __version__
85
86
87def stream_template(template_name, **context):
88    app.update_template_context(context)
89    t = app.jinja_env.get_template(template_name)
90    rv = t.stream(context)
91    rv.enable_buffering(5)
92    return rv
93
94
95def check_env(env, envs):
96    if env != '*' and env not in envs:
97        abort(404)
98
99
100def metric_params(db_version):
101    query_type = ''
102
103    # Puppet Server is enforcing new metrics API (v2)
104    # starting with versions 6.9.1, 5.3.12, and 5.2.13
105    if (db_version > (6, 9, 0) or
106            (db_version > (5, 3, 11) and db_version < (6, 0, 0)) or
107            (db_version > (5, 2, 12) and db_version < (5, 3, 10))):
108        metric_version = 'v2'
109    else:
110        metric_version = 'v1'
111
112    # Puppet DB version changed the query format from 3.2.0
113    # to 4.0 when querying mbeans
114    if db_version < (4, 0, 0):
115        query_type = 'type=default,'
116
117    return query_type, metric_version
118
119
120@app.context_processor
121def utility_processor():
122    def now(format='%m/%d/%Y %H:%M:%S'):
123        """returns the formated datetime"""
124        return datetime.now().strftime(format)
125
126    return dict(now=now)
127
128
129@app.route('/', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
130@app.route('/<env>/')
131def index(env):
132    """This view generates the index page and displays a set of metrics and
133    latest reports on nodes fetched from PuppetDB.
134
135    :param env: Search for nodes in this (Catalog and Fact) environment
136    :type env: :obj:`string`
137    """
138    envs = environments()
139    metrics = {
140        'num_nodes': 0,
141        'num_resources': 0,
142        'avg_resources_node': 0}
143    check_env(env, envs)
144
145    if env == '*':
146        query = app.config['OVERVIEW_FILTER']
147
148        prefix = 'puppetlabs.puppetdb.population'
149        db_version = get_db_version(puppetdb)
150        query_type, metric_version = metric_params(db_version)
151
152        num_nodes = get_or_abort(
153            puppetdb.metric,
154            "{0}{1}".format(prefix, ':%sname=num-nodes' % query_type),
155            version=metric_version)
156        num_resources = get_or_abort(
157            puppetdb.metric,
158            "{0}{1}".format(prefix, ':%sname=num-resources' % query_type),
159            version=metric_version)
160
161        metrics['num_nodes'] = num_nodes['Value']
162        metrics['num_resources'] = num_resources['Value']
163        try:
164            # Compute our own average because avg_resources_node['Value']
165            # returns a string of the format "num_resources/num_nodes"
166            # example: "1234/9" instead of doing the division itself.
167            metrics['avg_resources_node'] = "{0:10.0f}".format(
168                (num_resources['Value'] / num_nodes['Value']))
169        except ZeroDivisionError:
170            metrics['avg_resources_node'] = 0
171    else:
172        query = AndOperator()
173        query.add(EqualsOperator('catalog_environment', env))
174
175        num_nodes_query = ExtractOperator()
176        num_nodes_query.add_field(FunctionOperator('count'))
177        num_nodes_query.add_query(query)
178
179        if app.config['OVERVIEW_FILTER'] is not None:
180            query.add(app.config['OVERVIEW_FILTER'])
181
182        num_resources_query = ExtractOperator()
183        num_resources_query.add_field(FunctionOperator('count'))
184        num_resources_query.add_query(EqualsOperator("environment", env))
185
186        num_nodes = get_or_abort(
187            puppetdb._query,
188            'nodes',
189            query=num_nodes_query)
190        num_resources = get_or_abort(
191            puppetdb._query,
192            'resources',
193            query=num_resources_query)
194        metrics['num_nodes'] = num_nodes[0]['count']
195        metrics['num_resources'] = num_resources[0]['count']
196        try:
197            metrics['avg_resources_node'] = "{0:10.0f}".format(
198                (num_resources[0]['count'] / num_nodes[0]['count']))
199        except ZeroDivisionError:
200            metrics['avg_resources_node'] = 0
201
202    nodes = get_or_abort(puppetdb.nodes,
203                         query=query,
204                         unreported=app.config['UNRESPONSIVE_HOURS'],
205                         with_status=True,
206                         with_event_numbers=app.config['WITH_EVENT_NUMBERS'])
207
208    nodes_overview = []
209    stats = {
210        'changed': 0,
211        'unchanged': 0,
212        'failed': 0,
213        'unreported': 0,
214        'noop': 0
215    }
216
217    for node in nodes:
218        if node.status == 'unreported':
219            stats['unreported'] += 1
220        elif node.status == 'changed':
221            stats['changed'] += 1
222        elif node.status == 'failed':
223            stats['failed'] += 1
224        elif node.status == 'noop':
225            stats['noop'] += 1
226        else:
227            stats['unchanged'] += 1
228
229        if node.status != 'unchanged':
230            nodes_overview.append(node)
231
232    return render_template(
233        'index.html',
234        metrics=metrics,
235        nodes=nodes_overview,
236        stats=stats,
237        envs=envs,
238        current_env=env
239    )
240
241
242@app.route('/nodes', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
243@app.route('/<env>/nodes')
244def nodes(env):
245    """Fetch all (active) nodes from PuppetDB and stream a table displaying
246    those nodes.
247
248    Downside of the streaming aproach is that since we've already sent our
249    headers we can't abort the request if we detect an error. Because of this
250    we'll end up with an empty table instead because of how yield_or_stop
251    works. Once pagination is in place we can change this but we'll need to
252    provide a search feature instead.
253
254    :param env: Search for nodes in this (Catalog and Fact) environment
255    :type env: :obj:`string`
256    """
257    envs = environments()
258    status_arg = request.args.get('status', '')
259    check_env(env, envs)
260
261    query = AndOperator()
262
263    if env != '*':
264        query.add(EqualsOperator("catalog_environment", env))
265
266    if status_arg in ['failed', 'changed', 'unchanged']:
267        query.add(EqualsOperator('latest_report_status', status_arg))
268    elif status_arg == 'unreported':
269        unreported = datetime.utcnow()
270        unreported = (unreported -
271                      timedelta(hours=app.config['UNRESPONSIVE_HOURS']))
272        unreported = unreported.replace(microsecond=0).isoformat()
273
274        unrep_query = OrOperator()
275        unrep_query.add(NullOperator('report_timestamp', True))
276        unrep_query.add(LessEqualOperator('report_timestamp', unreported))
277
278        query.add(unrep_query)
279
280    if len(query.operations) == 0:
281        query = None
282
283    nodelist = puppetdb.nodes(
284        query=query,
285        unreported=app.config['UNRESPONSIVE_HOURS'],
286        with_status=True,
287        with_event_numbers=app.config['WITH_EVENT_NUMBERS'])
288    nodes = []
289    for node in yield_or_stop(nodelist):
290        if status_arg:
291            if node.status == status_arg:
292                nodes.append(node)
293        else:
294            nodes.append(node)
295    return Response(stream_with_context(
296        stream_template('nodes.html',
297                        nodes=nodes,
298                        envs=envs,
299                        current_env=env)))
300
301
302def inventory_facts():
303    # a list of facts descriptions to go in table header
304    headers = []
305    # a list of inventory fact names
306    fact_names = []
307
308    # load the list of items/facts we want in our inventory
309    try:
310        inv_facts = app.config['INVENTORY_FACTS']
311    except KeyError:
312        inv_facts = [('Hostname', 'fqdn'),
313                     ('IP Address', 'ipaddress'),
314                     ('OS', 'lsbdistdescription'),
315                     ('Architecture', 'hardwaremodel'),
316                     ('Kernel Version', 'kernelrelease')]
317
318    # generate a list of descriptions and a list of fact names
319    # from the list of tuples inv_facts.
320    for desc, name in inv_facts:
321        headers.append(desc)
322        fact_names.append(name)
323
324    return headers, fact_names
325
326
327@app.route('/inventory', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
328@app.route('/<env>/inventory')
329def inventory(env):
330    """Fetch all (active) nodes from PuppetDB and stream a table displaying
331    those nodes along with a set of facts about them.
332
333    :param env: Search for facts in this environment
334    :type env: :obj:`string`
335    """
336    envs = environments()
337    check_env(env, envs)
338    headers, fact_names = inventory_facts()
339
340    return render_template(
341        'inventory.html',
342        envs=envs,
343        current_env=env,
344        fact_headers=headers)
345
346
347@app.route('/inventory/json',
348           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
349@app.route('/<env>/inventory/json')
350def inventory_ajax(env):
351    """Backend endpoint for inventory table"""
352    draw = int(request.args.get('draw', 0))
353
354    envs = environments()
355    check_env(env, envs)
356    headers, fact_names = inventory_facts()
357
358    query = AndOperator()
359    fact_query = OrOperator()
360    fact_query.add([EqualsOperator("name", name) for name in fact_names])
361    query.add(fact_query)
362
363    if env != '*':
364        query.add(EqualsOperator("environment", env))
365
366    facts = puppetdb.facts(query=query)
367
368    fact_data = {}
369    for fact in facts:
370        if fact.node not in fact_data:
371            fact_data[fact.node] = {}
372        fact_data[fact.node][fact.name] = fact.value
373
374    total = len(fact_data)
375
376    return render_template(
377        'inventory.json.tpl',
378        draw=draw,
379        total=total,
380        total_filtered=total,
381        fact_data=fact_data,
382        columns=fact_names)
383
384
385@app.route('/node/<node_name>',
386           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
387@app.route('/<env>/node/<node_name>')
388def node(env, node_name):
389    """Display a dashboard for a node showing as much data as we have on that
390    node. This includes facts and reports but not Resources as that is too
391    heavy to do within a single request.
392
393    :param env: Ensure that the node, facts and reports are in this environment
394    :type env: :obj:`string`
395    """
396    envs = environments()
397    check_env(env, envs)
398    query = AndOperator()
399
400    if env != '*':
401        query.add(EqualsOperator("environment", env))
402
403    query.add(EqualsOperator("certname", node_name))
404
405    node = get_or_abort(puppetdb.node, node_name)
406
407    return render_template(
408        'node.html',
409        node=node,
410        envs=envs,
411        current_env=env,
412        columns=REPORTS_COLUMNS[:2])
413
414
415@app.route('/reports',
416           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
417                     'node_name': None})
418@app.route('/<env>/reports', defaults={'node_name': None})
419@app.route('/reports/<node_name>',
420           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
421@app.route('/<env>/reports/<node_name>')
422def reports(env, node_name):
423    """Query and Return JSON data to reports Jquery datatable
424
425    :param env: Search for all reports in this environment
426    :type env: :obj:`string`
427    """
428    envs = environments()
429    check_env(env, envs)
430    return render_template(
431        'reports.html',
432        envs=envs,
433        current_env=env,
434        node_name=node_name,
435        columns=REPORTS_COLUMNS)
436
437
438@app.route('/reports/json',
439           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
440                     'node_name': None})
441@app.route('/<env>/reports/json', defaults={'node_name': None})
442@app.route('/reports/<node_name>/json',
443           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
444@app.route('/<env>/reports/<node_name>/json')
445def reports_ajax(env, node_name):
446    """Query and Return JSON data to reports Jquery datatable
447
448    :param env: Search for all reports in this environment
449    :type env: :obj:`string`
450    """
451    draw = int(request.args.get('draw', 0))
452    start = int(request.args.get('start', 0))
453    length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
454    paging_args = {'limit': length, 'offset': start}
455    search_arg = request.args.get('search[value]')
456    order_column = int(request.args.get('order[0][column]', 0))
457    order_filter = REPORTS_COLUMNS[order_column].get(
458        'filter', REPORTS_COLUMNS[order_column]['attr'])
459    order_dir = request.args.get('order[0][dir]', 'desc')
460    order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)
461    status_args = request.args.get('columns[1][search][value]', '').split('|')
462    date_args = request.args.get('columns[0][search][value]', '')
463    max_col = len(REPORTS_COLUMNS)
464    for i in range(len(REPORTS_COLUMNS)):
465        if request.args.get("columns[%s][data]" % i, None):
466            max_col = i + 1
467
468    envs = environments()
469    check_env(env, envs)
470    reports_query = AndOperator()
471
472    if env != '*':
473        reports_query.add(EqualsOperator("environment", env))
474
475    if node_name:
476        reports_query.add(EqualsOperator("certname", node_name))
477
478    if search_arg:
479        search_query = OrOperator()
480        search_query.add(RegexOperator("certname", r"%s" % search_arg))
481        search_query.add(RegexOperator("puppet_version", r"%s" % search_arg))
482        search_query.add(RegexOperator(
483            "configuration_version", r"%s" % search_arg))
484        reports_query.add(search_query)
485
486    if date_args:
487        dates = json.loads(date_args)
488
489        if len(dates) > 0:
490            date_query = AndOperator()
491
492            if 'min' in dates:
493                date_query.add(GreaterEqualOperator('end_time', dates['min']))
494
495            if 'max' in dates:
496                date_query.add(LessEqualOperator('end_time', dates['max']))
497
498            reports_query.add(date_query)
499
500    status_query = OrOperator()
501    for status_arg in status_args:
502        if status_arg in ['failed', 'changed', 'unchanged']:
503            arg_query = AndOperator()
504            arg_query.add(EqualsOperator('status', status_arg))
505            arg_query.add(EqualsOperator('noop', False))
506            status_query.add(arg_query)
507            if status_arg == 'unchanged':
508                arg_query = AndOperator()
509                arg_query.add(EqualsOperator('noop', True))
510                arg_query.add(EqualsOperator('noop_pending', False))
511                status_query.add(arg_query)
512        elif status_arg == 'noop':
513            arg_query = AndOperator()
514            arg_query.add(EqualsOperator('noop', True))
515            arg_query.add(EqualsOperator('noop_pending', True))
516            status_query.add(arg_query)
517
518    if len(status_query.operations) == 0:
519        if len(reports_query.operations) == 0:
520            reports_query = None
521    else:
522        reports_query.add(status_query)
523
524    if status_args[0] != 'none':
525        reports = get_or_abort(
526            puppetdb.reports,
527            query=reports_query,
528            order_by=order_args,
529            include_total=True,
530            **paging_args)
531        reports, reports_events = tee(reports)
532        total = None
533    else:
534        reports = []
535        reports_events = []
536        total = 0
537
538    # Convert metrics to relational dict
539    metrics = {}
540    for report in reports_events:
541        if total is None:
542            total = puppetdb.total
543
544        metrics[report.hash_] = {}
545        for m in report.metrics:
546            if m['category'] not in metrics[report.hash_]:
547                metrics[report.hash_][m['category']] = {}
548            metrics[report.hash_][m['category']][m['name']] = m['value']
549
550    if total is None:
551        total = 0
552
553    return render_template(
554        'reports.json.tpl',
555        draw=draw,
556        total=total,
557        total_filtered=total,
558        reports=reports,
559        metrics=metrics,
560        envs=envs,
561        current_env=env,
562        columns=REPORTS_COLUMNS[:max_col])
563
564
565@app.route('/report/<node_name>/<report_id>',
566           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
567@app.route('/<env>/report/<node_name>/<report_id>')
568def report(env, node_name, report_id):
569    """Displays a single report including all the events associated with that
570    report and their status.
571
572    The report_id may be the puppetdb's report hash or the
573    configuration_version. This allows for better integration
574    into puppet-hipchat.
575
576    :param env: Search for reports in this environment
577    :type env: :obj:`string`
578    :param node_name: Find the reports whose certname match this value
579    :type node_name: :obj:`string`
580    :param report_id: The hash or the configuration_version of the desired
581        report
582    :type report_id: :obj:`string`
583    """
584    envs = environments()
585    check_env(env, envs)
586    query = AndOperator()
587    report_id_query = OrOperator()
588
589    report_id_query.add(EqualsOperator("hash", report_id))
590    report_id_query.add(EqualsOperator("configuration_version", report_id))
591
592    if env != '*':
593        query.add(EqualsOperator("environment", env))
594
595    query.add(EqualsOperator("certname", node_name))
596    query.add(report_id_query)
597
598    reports = puppetdb.reports(query=query)
599
600    try:
601        report = next(reports)
602    except StopIteration:
603        abort(404)
604
605    report.version = commonmark.commonmark(report.version)
606
607    return render_template(
608        'report.html',
609        report=report,
610        events=yield_or_stop(report.events()),
611        logs=report.logs,
612        metrics=report.metrics,
613        envs=envs,
614        current_env=env)
615
616
617@app.route('/facts', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
618@app.route('/<env>/facts')
619def facts(env):
620    """Displays an alphabetical list of all facts currently known to
621    PuppetDB.
622
623    :param env: Serves no purpose for this function, only for consistency's
624        sake
625    :type env: :obj:`string`
626    """
627    envs = environments()
628    check_env(env, envs)
629    facts = get_or_abort(puppetdb.fact_names)
630
631    # we consider a column label to count for ~5 lines
632    column_label_height = 5
633
634    # 1 label per different letter and up to 3 more labels for letters spanning
635    # multiple columns.
636    column_label_count = 3 + len(set(map(lambda fact: fact[0].upper(), facts)))
637
638    break_size = (len(facts) + column_label_count * column_label_height) / 4.0
639    next_break = break_size
640
641    facts_columns = []
642    facts_current_column = []
643    facts_current_letter = []
644    letter = None
645    count = 0
646
647    for fact in facts:
648        count += 1
649
650        if count > next_break:
651            next_break += break_size
652            if facts_current_letter:
653                facts_current_column.append(facts_current_letter)
654            if facts_current_column:
655                facts_columns.append(facts_current_column)
656            facts_current_column = []
657            facts_current_letter = []
658            letter = None
659
660        if letter != fact[0].upper():
661            if facts_current_letter:
662                facts_current_column.append(facts_current_letter)
663                facts_current_letter = []
664            letter = fact[0].upper()
665            count += column_label_height
666
667        facts_current_letter.append(fact)
668
669    if facts_current_letter:
670        facts_current_column.append(facts_current_letter)
671    if facts_current_column:
672        facts_columns.append(facts_current_column)
673
674    return render_template('facts.html',
675                           facts_columns=facts_columns,
676                           envs=envs,
677                           current_env=env)
678
679
680@app.route('/fact/<fact>',
681           defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'value': None})
682@app.route('/<env>/fact/<fact>', defaults={'value': None})
683@app.route('/fact/<fact>/<value>',
684           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
685@app.route('/<env>/fact/<fact>/<value>')
686def fact(env, fact, value):
687    """Fetches the specific fact(/value) from PuppetDB and displays per
688    node for which this fact is known.
689
690    :param env: Searches for facts in this environment
691    :type env: :obj:`string`
692    :param fact: Find all facts with this name
693    :type fact: :obj:`string`
694    :param value: Find all facts with this value
695    :type value: :obj:`string`
696    """
697    envs = environments()
698    check_env(env, envs)
699
700    render_graph = False
701    if fact in graph_facts and not value:
702        render_graph = True
703
704    value_json = value
705    if value is not None:
706        value_object = parse_python(value)
707        if type(value_object) is str:
708            value_json = value_object
709        else:
710            value_json = dumps(value_object)
711
712    return render_template(
713        'fact.html',
714        fact=fact,
715        value=value,
716        value_json=value_json,
717        render_graph=render_graph,
718        envs=envs,
719        current_env=env)
720
721
722@app.route('/fact/<fact>/json',
723           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
724                     'node': None, 'value': None})
725@app.route('/<env>/fact/<fact>/json', defaults={'node': None, 'value': None})
726@app.route('/fact/<fact>/<value>/json',
727           defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None})
728@app.route('/fact/<fact>/<path:value>/json',
729           defaults={'env': app.config['DEFAULT_ENVIRONMENT'], 'node': None})
730@app.route('/<env>/fact/<fact>/<value>/json', defaults={'node': None})
731@app.route('/node/<node>/facts/json',
732           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
733                     'fact': None, 'value': None})
734@app.route('/<env>/node/<node>/facts/json',
735           defaults={'fact': None, 'value': None})
736def fact_ajax(env, node, fact, value):
737    """Fetches the specific facts matching (node/fact/value) from PuppetDB and
738    return a JSON table
739
740    :param env: Searches for facts in this environment
741    :type env: :obj:`string`
742    :param node: Find all facts for this node
743    :type node: :obj:`string`
744    :param fact: Find all facts with this name
745    :type fact: :obj:`string`
746    :param value: Filter facts whose value is equal to this
747    :type value: :obj:`string`
748    """
749    draw = int(request.args.get('draw', 0))
750
751    envs = environments()
752    check_env(env, envs)
753
754    render_graph = False
755    if fact in graph_facts and value is None and node is None:
756        render_graph = True
757
758    query = AndOperator()
759    if node is not None:
760        query.add(EqualsOperator("certname", node))
761
762    if env != '*':
763        query.add(EqualsOperator("environment", env))
764
765    if value is not None:
766        # interpret the value as a proper type...
767        value = parse_python(value)
768        # ...to know if it should be quoted or not in the query to PuppetDB
769        # (f.e. a string should, while a number should not)
770        query.add(EqualsOperator('value', value))
771
772    # if we have not added any operations to the query,
773    # then make it explicitly empty
774    if len(query.operations) == 0:
775        query = None
776
777    facts = [f for f in get_or_abort(
778        puppetdb.facts,
779        name=fact,
780        query=query)]
781
782    total = len(facts)
783
784    counts = {}
785    json = {
786        'draw': draw,
787        'recordsTotal': total,
788        'recordsFiltered': total,
789        'data': []}
790
791    for fact_h in facts:
792        line = []
793        if fact is None:
794            line.append(fact_h.name)
795        if node is None:
796            line.append('<a href="{0}">{1}</a>'.format(
797                url_for('node', env=env, node_name=fact_h.node),
798                fact_h.node))
799        if value is None:
800            if isinstance(fact_h.value, str):
801                value_for_url = quote_plus(fact_h.value)
802            else:
803                value_for_url = fact_h.value
804
805            line.append('["{0}", {1}]'.format(
806                url_for(
807                    'fact', env=env, fact=fact_h.name, value=value_for_url),
808                dumps(fact_h.value)))
809
810        json['data'].append(line)
811
812        if render_graph:
813            if fact_h.value not in counts:
814                counts[fact_h.value] = 0
815            counts[fact_h.value] += 1
816
817    if render_graph:
818        json['chart'] = [
819            {"label": "{0}".format(k).replace('\n', ' '),
820             "value": counts[k]}
821            for k in sorted(counts, key=lambda k: counts[k], reverse=True)]
822
823    return jsonify(json)
824
825
826@app.route('/query', methods=('GET', 'POST'),
827           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
828@app.route('/<env>/query', methods=('GET', 'POST'))
829def query(env):
830    """Allows to execute raw, user created querries against PuppetDB. This is
831    currently highly experimental and explodes in interesting ways since none
832    of the possible exceptions are being handled just yet. This will return
833    the JSON of the response or a message telling you what whent wrong /
834    why nothing was returned.
835
836    :param env: Serves no purpose for the query data but is required for the
837        select field in the environment block
838    :type env: :obj:`string`
839    """
840    if not app.config['ENABLE_QUERY']:
841        log.warn('Access to query interface disabled by administrator.')
842        abort(403)
843
844    envs = environments()
845    check_env(env, envs)
846
847    form = QueryForm(meta={
848        'csrf_secret': app.config['SECRET_KEY'],
849        'csrf_context': session})
850    if form.validate_on_submit():
851        if form.endpoints.data not in ENABLED_QUERY_ENDPOINTS:
852            log.warn('Access to query endpoint %s disabled by administrator.',
853                     form.endpoints.data)
854            abort(403)
855
856        if form.endpoints.data == 'pql':
857            query = form.query.data
858        elif form.query.data[0] == '[':
859            query = form.query.data
860        else:
861            query = '[{0}]'.format(form.query.data)
862
863        result = get_or_abort(
864            puppetdb._query,
865            form.endpoints.data,
866            query=query)
867        return render_template('query.html',
868                               form=form,
869                               result=result,
870                               envs=envs,
871                               current_env=env)
872    return render_template('query.html',
873                           form=form,
874                           envs=envs,
875                           current_env=env)
876
877
878@app.route('/metrics', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
879@app.route('/<env>/metrics')
880def metrics(env):
881    """Lists all available metrics that PuppetDB is aware of.
882
883    :param env: While this parameter serves no function purpose it is required
884        for the environments template block
885    :type env: :obj:`string`
886    """
887    envs = environments()
888    check_env(env, envs)
889
890    db_version = get_db_version(puppetdb)
891    query_type, metric_version = metric_params(db_version)
892    if metric_version == 'v1':
893        mbeans = get_or_abort(puppetdb._query, 'mbean')
894        metrics = list(mbeans.keys())
895    elif metric_version == 'v2':
896        # the list response is a dict in the format:
897        # {
898        #   "domain1": {
899        #     "property1": {
900        #      ...
901        #     }
902        #   },
903        #   "domain2": {
904        #     "property2": {
905        #      ...
906        #     }
907        #   }
908        # }
909        # The MBean names are the combination of the domain and the properties
910        # with a ":" in between, example:
911        #   domain1:property1
912        #   domain2:property2
913        # reference: https://jolokia.org/reference/html/protocol.html#list
914        metrics_domains = get_or_abort(puppetdb.metric)
915        metrics = []
916        # get all of the domains
917        for domain in list(metrics_domains.keys()):
918            # iterate over all of the properties in this domain
919            properties = list(metrics_domains[domain].keys())
920            for prop in properties:
921                # combine the current domain and each property with
922                # a ":" in between
923                metrics.append(domain + ':' + prop)
924    else:
925        raise ValueError("Unknown metric version {} for database version {}"
926                         .format(metric_version, db_version))
927
928    return render_template('metrics.html',
929                           metrics=sorted(metrics),
930                           envs=envs,
931                           current_env=env)
932
933
934@app.route('/metric/<path:metric>',
935           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
936@app.route('/<env>/metric/<path:metric>')
937def metric(env, metric):
938    """Lists all information about the metric of the given name.
939
940    :param env: While this parameter serves no function purpose it is required
941        for the environments template block
942    :type env: :obj:`string`
943    """
944    envs = environments()
945    check_env(env, envs)
946
947    db_version = get_db_version(puppetdb)
948    query_type, metric_version = metric_params(db_version)
949
950    name = unquote(metric)
951    metric = get_or_abort(puppetdb.metric, metric, version=metric_version)
952    return render_template(
953        'metric.html',
954        name=name,
955        metric=sorted(metric.items()),
956        envs=envs,
957        current_env=env)
958
959
960@app.route('/catalogs',
961           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
962                     'compare': None})
963@app.route('/<env>/catalogs', defaults={'compare': None})
964@app.route('/catalogs/compare/<compare>',
965           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
966@app.route('/<env>/catalogs/compare/<compare>')
967def catalogs(env, compare):
968    """Lists all nodes with a compiled catalog.
969
970    :param env: Find the nodes with this catalog_environment value
971    :type env: :obj:`string`
972    """
973    envs = environments()
974    check_env(env, envs)
975
976    if not app.config['ENABLE_CATALOG']:
977        log.warning('Access to catalog interface disabled by administrator')
978        abort(403)
979
980    return render_template(
981        'catalogs.html',
982        compare=compare,
983        columns=CATALOGS_COLUMNS,
984        envs=envs,
985        current_env=env)
986
987
988@app.route('/catalogs/json',
989           defaults={'env': app.config['DEFAULT_ENVIRONMENT'],
990                     'compare': None})
991@app.route('/<env>/catalogs/json', defaults={'compare': None})
992@app.route('/catalogs/compare/<compare>/json',
993           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
994@app.route('/<env>/catalogs/compare/<compare>/json')
995def catalogs_ajax(env, compare):
996    """Server data to catalogs as JSON to Jquery datatables
997    """
998    draw = int(request.args.get('draw', 0))
999    start = int(request.args.get('start', 0))
1000    length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
1001    paging_args = {'limit': length, 'offset': start}
1002    search_arg = request.args.get('search[value]')
1003    order_column = int(request.args.get('order[0][column]', 0))
1004    order_filter = CATALOGS_COLUMNS[order_column].get(
1005        'filter', CATALOGS_COLUMNS[order_column]['attr'])
1006    order_dir = request.args.get('order[0][dir]', 'asc')
1007    order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)
1008
1009    envs = environments()
1010    check_env(env, envs)
1011
1012    query = AndOperator()
1013    if env != '*':
1014        query.add(EqualsOperator("catalog_environment", env))
1015    if search_arg:
1016        query.add(RegexOperator("certname", r"%s" % search_arg))
1017    query.add(NullOperator("catalog_timestamp", False))
1018
1019    nodes = get_or_abort(puppetdb.nodes,
1020                         query=query,
1021                         include_total=True,
1022                         order_by=order_args,
1023                         **paging_args)
1024
1025    catalog_list = []
1026    total = None
1027    for node in nodes:
1028        if total is None:
1029            total = puppetdb.total
1030
1031        catalog_list.append({
1032            'certname': node.name,
1033            'catalog_timestamp': node.catalog_timestamp,
1034            'form': compare,
1035        })
1036
1037    if total is None:
1038        total = 0
1039
1040    return render_template(
1041        'catalogs.json.tpl',
1042        total=total,
1043        total_filtered=total,
1044        draw=draw,
1045        columns=CATALOGS_COLUMNS,
1046        catalogs=catalog_list,
1047        envs=envs,
1048        current_env=env)
1049
1050
1051@app.route('/catalog/<node_name>',
1052           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
1053@app.route('/<env>/catalog/<node_name>')
1054def catalog_node(env, node_name):
1055    """Fetches from PuppetDB the compiled catalog of a given node.
1056
1057    :param env: Find the catalog with this environment value
1058    :type env: :obj:`string`
1059    """
1060    envs = environments()
1061    check_env(env, envs)
1062
1063    if app.config['ENABLE_CATALOG']:
1064        catalog = get_or_abort(puppetdb.catalog,
1065                               node=node_name)
1066        return render_template('catalog.html',
1067                               catalog=catalog,
1068                               envs=envs,
1069                               current_env=env)
1070    else:
1071        log.warn('Access to catalog interface disabled by administrator')
1072        abort(403)
1073
1074
1075@app.route('/catalogs/compare/<compare>...<against>',
1076           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
1077@app.route('/<env>/catalogs/compare/<compare>...<against>')
1078def catalog_compare(env, compare, against):
1079    """Compares the catalog of one node, parameter compare, with that of
1080       with that of another node, parameter against.
1081
1082    :param env: Ensure that the 2 catalogs are in the same environment
1083    :type env: :obj:`string`
1084    """
1085    envs = environments()
1086    check_env(env, envs)
1087
1088    if app.config['ENABLE_CATALOG']:
1089        compare_cat = get_or_abort(puppetdb.catalog,
1090                                   node=compare)
1091        against_cat = get_or_abort(puppetdb.catalog,
1092                                   node=against)
1093
1094        return render_template('catalog_compare.html',
1095                               compare=compare_cat,
1096                               against=against_cat,
1097                               envs=envs,
1098                               current_env=env)
1099    else:
1100        log.warn('Access to catalog interface disabled by administrator')
1101        abort(403)
1102
1103
1104@app.route('/radiator', defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
1105@app.route('/<env>/radiator')
1106def radiator(env):
1107    """This view generates a simplified monitoring page
1108    akin to the radiator view in puppet dashboard
1109    """
1110    envs = environments()
1111    check_env(env, envs)
1112
1113    if env == '*':
1114        db_version = get_db_version(puppetdb)
1115        query_type, metric_version = metric_params(db_version)
1116
1117        query = None
1118        metrics = get_or_abort(
1119            puppetdb.metric,
1120            'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type,
1121            version=metric_version)
1122        num_nodes = metrics['Value']
1123    else:
1124        query = AndOperator()
1125        metric_query = ExtractOperator()
1126
1127        query.add(EqualsOperator("catalog_environment", env))
1128        metric_query.add_field(FunctionOperator('count'))
1129        metric_query.add_query(query)
1130
1131        metrics = get_or_abort(
1132            puppetdb._query,
1133            'nodes',
1134            query=metric_query)
1135        num_nodes = metrics[0]['count']
1136
1137    nodes = puppetdb.nodes(
1138        query=query,
1139        unreported=app.config['UNRESPONSIVE_HOURS'],
1140        with_status=True
1141    )
1142
1143    stats = {
1144        'changed_percent': 0,
1145        'changed': 0,
1146        'failed_percent': 0,
1147        'failed': 0,
1148        'noop_percent': 0,
1149        'noop': 0,
1150        'skipped_percent': 0,
1151        'skipped': 0,
1152        'unchanged_percent': 0,
1153        'unchanged': 0,
1154        'unreported_percent': 0,
1155        'unreported': 0,
1156    }
1157
1158    for node in nodes:
1159        if node.status == 'unreported':
1160            stats['unreported'] += 1
1161        elif node.status == 'changed':
1162            stats['changed'] += 1
1163        elif node.status == 'failed':
1164            stats['failed'] += 1
1165        elif node.status == 'noop':
1166            stats['noop'] += 1
1167        elif node.status == 'skipped':
1168            stats['skipped'] += 1
1169        else:
1170            stats['unchanged'] += 1
1171
1172    try:
1173        stats['changed_percent'] = int(100 * (stats['changed'] /
1174                                              float(num_nodes)))
1175        stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes))
1176        stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes))
1177        stats['skipped_percent'] = int(100 * (stats['skipped'] /
1178                                              float(num_nodes)))
1179        stats['unchanged_percent'] = int(100 * (stats['unchanged'] /
1180                                                float(num_nodes)))
1181        stats['unreported_percent'] = int(100 * (stats['unreported'] /
1182                                                 float(num_nodes)))
1183    except ZeroDivisionError:
1184        stats['changed_percent'] = 0
1185        stats['failed_percent'] = 0
1186        stats['noop_percent'] = 0
1187        stats['skipped_percent'] = 0
1188        stats['unchanged_percent'] = 0
1189        stats['unreported_percent'] = 0
1190
1191    if ('Accept' in request.headers and
1192            request.headers["Accept"] == 'application/json'):
1193        return jsonify(**stats)
1194
1195    return render_template(
1196        'radiator.html',
1197        stats=stats,
1198        total=num_nodes
1199    )
1200
1201
1202@app.route('/daily_reports_chart.json',
1203           defaults={'env': app.config['DEFAULT_ENVIRONMENT']})
1204@app.route('/<env>/daily_reports_chart.json')
1205def daily_reports_chart(env):
1206    """Return JSON data to generate a bar chart of daily runs.
1207
1208    If certname is passed as GET argument, the data will target that
1209    node only.
1210    """
1211    certname = request.args.get('certname')
1212    result = get_or_abort(
1213        get_daily_reports_chart,
1214        db=puppetdb,
1215        env=env,
1216        days_number=app.config['DAILY_REPORTS_CHART_DAYS'],
1217        certname=certname,
1218    )
1219    return jsonify(result=result)
1220
1221
1222@app.route('/offline/<path:filename>')
1223def offline_static(filename):
1224    mimetype = 'text/html'
1225    if filename.endswith('.css'):
1226        mimetype = 'text/css'
1227    elif filename.endswith('.js'):
1228        mimetype = 'text/javascript'
1229
1230    return Response(response=render_template('static/%s' % filename),
1231                    status=200, mimetype=mimetype)
1232
1233
1234@app.route('/status')
1235def health_status():
1236    return 'OK'
1237