1""" 2TextFSM 3======= 4 5.. versionadded:: 2018.3.0 6 7Execution module that processes plain text and extracts data 8using TextFSM templates. The output is presented in JSON serializable 9data, and can be easily re-used in other modules, or directly 10inside the renderer (Jinja, Mako, Genshi, etc.). 11 12:depends: - textfsm Python library 13 14.. note:: 15 16 Install ``textfsm`` library: ``pip install textfsm``. 17""" 18 19import logging 20import os 21 22from salt.utils.files import fopen 23 24try: 25 import textfsm 26 27 HAS_TEXTFSM = True 28except ImportError: 29 HAS_TEXTFSM = False 30 31try: 32 from textfsm import clitable 33 34 HAS_CLITABLE = True 35except ImportError: 36 HAS_CLITABLE = False 37 38log = logging.getLogger(__name__) 39 40__virtualname__ = "textfsm" 41__proxyenabled__ = ["*"] 42 43 44def __virtual__(): 45 """ 46 Only load this execution module if TextFSM is installed. 47 """ 48 if HAS_TEXTFSM: 49 return __virtualname__ 50 return ( 51 False, 52 "The textfsm execution module failed to load: requires the textfsm library.", 53 ) 54 55 56def _clitable_to_dict(objects, fsm_handler): 57 """ 58 Converts TextFSM cli_table object to list of dictionaries. 59 """ 60 objs = [] 61 log.debug("Cli Table: %s; FSM handler: %s", objects, fsm_handler) 62 for row in objects: 63 temp_dict = {} 64 for index, element in enumerate(row): 65 temp_dict[fsm_handler.header[index].lower()] = element 66 objs.append(temp_dict) 67 log.debug("Extraction result: %s", objs) 68 return objs 69 70 71def extract(template_path, raw_text=None, raw_text_file=None, saltenv="base"): 72 r""" 73 Extracts the data entities from the unstructured 74 raw text sent as input and returns the data 75 mapping, processing using the TextFSM template. 76 77 template_path 78 The path to the TextFSM template. 79 This can be specified using the absolute path 80 to the file, or using one of the following URL schemes: 81 82 - ``salt://``, to fetch the template from the Salt fileserver. 83 - ``http://`` or ``https://`` 84 - ``ftp://`` 85 - ``s3://`` 86 - ``swift://`` 87 88 raw_text: ``None`` 89 The unstructured text to be parsed. 90 91 raw_text_file: ``None`` 92 Text file to read, having the raw text to be parsed using the TextFSM template. 93 Supports the same URL schemes as the ``template_path`` argument. 94 95 saltenv: ``base`` 96 Salt fileserver environment from which to retrieve the file. 97 Ignored if ``template_path`` is not a ``salt://`` URL. 98 99 CLI Example: 100 101 .. code-block:: bash 102 103 salt '*' textfsm.extract salt://textfsm/juniper_version_template raw_text_file=s3://junos_ver.txt 104 salt '*' textfsm.extract http://some-server/textfsm/juniper_version_template raw_text='Hostname: router.abc ... snip ...' 105 106 Jinja template example: 107 108 .. code-block:: jinja 109 110 {%- set raw_text = 'Hostname: router.abc ... snip ...' -%} 111 {%- set textfsm_extract = salt.textfsm.extract('https://some-server/textfsm/juniper_version_template', raw_text) -%} 112 113 Raw text example: 114 115 .. code-block:: text 116 117 Hostname: router.abc 118 Model: mx960 119 JUNOS Base OS boot [9.1S3.5] 120 JUNOS Base OS Software Suite [9.1S3.5] 121 JUNOS Kernel Software Suite [9.1S3.5] 122 JUNOS Crypto Software Suite [9.1S3.5] 123 JUNOS Packet Forwarding Engine Support (M/T Common) [9.1S3.5] 124 JUNOS Packet Forwarding Engine Support (MX Common) [9.1S3.5] 125 JUNOS Online Documentation [9.1S3.5] 126 JUNOS Routing Software Suite [9.1S3.5] 127 128 TextFSM Example: 129 130 .. code-block:: text 131 132 Value Chassis (\S+) 133 Value Required Model (\S+) 134 Value Boot (.*) 135 Value Base (.*) 136 Value Kernel (.*) 137 Value Crypto (.*) 138 Value Documentation (.*) 139 Value Routing (.*) 140 141 Start 142 # Support multiple chassis systems. 143 ^\S+:$$ -> Continue.Record 144 ^${Chassis}:$$ 145 ^Model: ${Model} 146 ^JUNOS Base OS boot \[${Boot}\] 147 ^JUNOS Software Release \[${Base}\] 148 ^JUNOS Base OS Software Suite \[${Base}\] 149 ^JUNOS Kernel Software Suite \[${Kernel}\] 150 ^JUNOS Crypto Software Suite \[${Crypto}\] 151 ^JUNOS Online Documentation \[${Documentation}\] 152 ^JUNOS Routing Software Suite \[${Routing}\] 153 154 Output example: 155 156 .. code-block:: json 157 158 { 159 "comment": "", 160 "result": true, 161 "out": [ 162 { 163 "kernel": "9.1S3.5", 164 "documentation": "9.1S3.5", 165 "boot": "9.1S3.5", 166 "crypto": "9.1S3.5", 167 "chassis": "", 168 "routing": "9.1S3.5", 169 "base": "9.1S3.5", 170 "model": "mx960" 171 } 172 ] 173 } 174 """ 175 ret = {"result": False, "comment": "", "out": None} 176 log.debug( 177 "Caching %s(saltenv: %s) using the Salt fileserver", template_path, saltenv 178 ) 179 tpl_cached_path = __salt__["cp.cache_file"](template_path, saltenv=saltenv) 180 if tpl_cached_path is False: 181 ret["comment"] = "Unable to read the TextFSM template from {}".format( 182 template_path 183 ) 184 log.error(ret["comment"]) 185 return ret 186 try: 187 log.debug("Reading TextFSM template from cache path: %s", tpl_cached_path) 188 # Disabling pylint W8470 to nto complain about fopen. 189 # Unfortunately textFSM needs the file handle rather than the content... 190 # pylint: disable=W8470 191 tpl_file_handle = fopen(tpl_cached_path, "r") 192 # pylint: disable=W8470 193 log.debug(tpl_file_handle.read()) 194 tpl_file_handle.seek(0) # move the object position back at the top of the file 195 fsm_handler = textfsm.TextFSM(tpl_file_handle) 196 except textfsm.TextFSMTemplateError as tfte: 197 log.error("Unable to parse the TextFSM template", exc_info=True) 198 ret[ 199 "comment" 200 ] = "Unable to parse the TextFSM template from {}: {}. Please check the logs.".format( 201 template_path, tfte 202 ) 203 return ret 204 if not raw_text and raw_text_file: 205 log.debug("Trying to read the raw input from %s", raw_text_file) 206 raw_text = __salt__["cp.get_file_str"](raw_text_file, saltenv=saltenv) 207 if raw_text is False: 208 ret[ 209 "comment" 210 ] = "Unable to read from {}. Please specify a valid input file or text.".format( 211 raw_text_file 212 ) 213 log.error(ret["comment"]) 214 return ret 215 if not raw_text: 216 ret["comment"] = "Please specify a valid input file or text." 217 log.error(ret["comment"]) 218 return ret 219 log.debug("Processing the raw text:\n%s", raw_text) 220 objects = fsm_handler.ParseText(raw_text) 221 ret["out"] = _clitable_to_dict(objects, fsm_handler) 222 ret["result"] = True 223 return ret 224 225 226def index( 227 command, 228 platform=None, 229 platform_grain_name=None, 230 platform_column_name=None, 231 output=None, 232 output_file=None, 233 textfsm_path=None, 234 index_file=None, 235 saltenv="base", 236 include_empty=False, 237 include_pat=None, 238 exclude_pat=None, 239): 240 """ 241 Dynamically identify the template required to extract the 242 information from the unstructured raw text. 243 244 The output has the same structure as the ``extract`` execution 245 function, the difference being that ``index`` is capable 246 to identify what template to use, based on the platform 247 details and the ``command``. 248 249 command 250 The command executed on the device, to get the output. 251 252 platform 253 The platform name, as defined in the TextFSM index file. 254 255 .. note:: 256 For ease of use, it is recommended to define the TextFSM 257 indexfile with values that can be matches using the grains. 258 259 platform_grain_name 260 The name of the grain used to identify the platform name 261 in the TextFSM index file. 262 263 .. note:: 264 This option can be also specified in the minion configuration 265 file or pillar as ``textfsm_platform_grain``. 266 267 .. note:: 268 This option is ignored when ``platform`` is specified. 269 270 platform_column_name: ``Platform`` 271 The column name used to identify the platform, 272 exactly as specified in the TextFSM index file. 273 Default: ``Platform``. 274 275 .. note:: 276 This is field is case sensitive, make sure 277 to assign the correct value to this option, 278 exactly as defined in the index file. 279 280 .. note:: 281 This option can be also specified in the minion configuration 282 file or pillar as ``textfsm_platform_column_name``. 283 284 output 285 The raw output from the device, to be parsed 286 and extract the structured data. 287 288 output_file 289 The path to a file that contains the raw output from the device, 290 used to extract the structured data. 291 This option supports the usual Salt-specific schemes: ``file://``, 292 ``salt://``, ``http://``, ``https://``, ``ftp://``, ``s3://``, ``swift://``. 293 294 textfsm_path 295 The path where the TextFSM templates can be found. This can be either 296 absolute path on the server, either specified using the following URL 297 schemes: ``file://``, ``salt://``, ``http://``, ``https://``, ``ftp://``, 298 ``s3://``, ``swift://``. 299 300 .. note:: 301 This needs to be a directory with a flat structure, having an 302 index file (whose name can be specified using the ``index_file`` option) 303 and a number of TextFSM templates. 304 305 .. note:: 306 This option can be also specified in the minion configuration 307 file or pillar as ``textfsm_path``. 308 309 index_file: ``index`` 310 The name of the TextFSM index file, under the ``textfsm_path``. Default: ``index``. 311 312 .. note:: 313 This option can be also specified in the minion configuration 314 file or pillar as ``textfsm_index_file``. 315 316 saltenv: ``base`` 317 Salt fileserver environment from which to retrieve the file. 318 Ignored if ``textfsm_path`` is not a ``salt://`` URL. 319 320 include_empty: ``False`` 321 Include empty files under the ``textfsm_path``. 322 323 include_pat 324 Glob or regex to narrow down the files cached from the given path. 325 If matching with a regex, the regex must be prefixed with ``E@``, 326 otherwise the expression will be interpreted as a glob. 327 328 exclude_pat 329 Glob or regex to exclude certain files from being cached from the given path. 330 If matching with a regex, the regex must be prefixed with ``E@``, 331 otherwise the expression will be interpreted as a glob. 332 333 .. note:: 334 If used with ``include_pat``, files matching this pattern will be 335 excluded from the subset of files defined by ``include_pat``. 336 337 CLI Example: 338 339 .. code-block:: bash 340 341 salt '*' textfsm.index 'sh ver' platform=Juniper output_file=salt://textfsm/juniper_version_example textfsm_path=salt://textfsm/ 342 salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example textfsm_path=ftp://textfsm/ platform_column_name=Vendor 343 salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example textfsm_path=https://some-server/textfsm/ platform_column_name=Vendor platform_grain_name=vendor 344 345 TextFSM index file example: 346 347 ``salt://textfsm/index`` 348 349 .. code-block:: text 350 351 Template, Hostname, Vendor, Command 352 juniper_version_template, .*, Juniper, sh[[ow]] ve[[rsion]] 353 354 The usage can be simplified, 355 by defining (some of) the following options: ``textfsm_platform_grain``, 356 ``textfsm_path``, ``textfsm_platform_column_name``, or ``textfsm_index_file``, 357 in the (proxy) minion configuration file or pillar. 358 359 Configuration example: 360 361 .. code-block:: yaml 362 363 textfsm_platform_grain: vendor 364 textfsm_path: salt://textfsm/ 365 textfsm_platform_column_name: Vendor 366 367 And the CLI usage becomes as simple as: 368 369 .. code-block:: bash 370 371 salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example 372 373 Usgae inside a Jinja template: 374 375 .. code-block:: jinja 376 377 {%- set command = 'sh ver' -%} 378 {%- set output = salt.net.cli(command) -%} 379 {%- set textfsm_extract = salt.textfsm.index(command, output=output) -%} 380 """ 381 ret = {"out": None, "result": False, "comment": ""} 382 if not HAS_CLITABLE: 383 ret["comment"] = "TextFSM does not seem that has clitable embedded." 384 log.error(ret["comment"]) 385 return ret 386 if not platform: 387 platform_grain_name = __opts__.get("textfsm_platform_grain") or __pillar__.get( 388 "textfsm_platform_grain", platform_grain_name 389 ) 390 if platform_grain_name: 391 log.debug( 392 "Using the %s grain to identify the platform name", platform_grain_name 393 ) 394 platform = __grains__.get(platform_grain_name) 395 if not platform: 396 ret[ 397 "comment" 398 ] = "Unable to identify the platform name using the {} grain.".format( 399 platform_grain_name 400 ) 401 return ret 402 log.info("Using platform: %s", platform) 403 else: 404 ret[ 405 "comment" 406 ] = "No platform specified, no platform grain identifier configured." 407 log.error(ret["comment"]) 408 return ret 409 if not textfsm_path: 410 log.debug( 411 "No TextFSM templates path specified, trying to look into the opts and" 412 " pillar" 413 ) 414 textfsm_path = __opts__.get("textfsm_path") or __pillar__.get("textfsm_path") 415 if not textfsm_path: 416 ret["comment"] = ( 417 "No TextFSM templates path specified. Please configure in" 418 " opts/pillar/function args." 419 ) 420 log.error(ret["comment"]) 421 return ret 422 log.debug( 423 "Caching %s(saltenv: %s) using the Salt fileserver", textfsm_path, saltenv 424 ) 425 textfsm_cachedir_ret = __salt__["cp.cache_dir"]( 426 textfsm_path, 427 saltenv=saltenv, 428 include_empty=include_empty, 429 include_pat=include_pat, 430 exclude_pat=exclude_pat, 431 ) 432 log.debug("Cache fun return:\n%s", textfsm_cachedir_ret) 433 if not textfsm_cachedir_ret: 434 ret[ 435 "comment" 436 ] = "Unable to fetch from {}. Is the TextFSM path correctly specified?".format( 437 textfsm_path 438 ) 439 log.error(ret["comment"]) 440 return ret 441 textfsm_cachedir = os.path.dirname(textfsm_cachedir_ret[0]) # first item 442 index_file = __opts__.get("textfsm_index_file") or __pillar__.get( 443 "textfsm_index_file", "index" 444 ) 445 index_file_path = os.path.join(textfsm_cachedir, index_file) 446 log.debug("Using the cached index file: %s", index_file_path) 447 log.debug("TextFSM templates cached under: %s", textfsm_cachedir) 448 textfsm_obj = clitable.CliTable(index_file_path, textfsm_cachedir) 449 attrs = {"Command": command} 450 platform_column_name = __opts__.get( 451 "textfsm_platform_column_name" 452 ) or __pillar__.get("textfsm_platform_column_name", "Platform") 453 log.info("Using the TextFSM platform idenfiticator: %s", platform_column_name) 454 attrs[platform_column_name] = platform 455 log.debug("Processing the TextFSM index file using the attributes: %s", attrs) 456 if not output and output_file: 457 log.debug("Processing the output from %s", output_file) 458 output = __salt__["cp.get_file_str"](output_file, saltenv=saltenv) 459 if output is False: 460 ret[ 461 "comment" 462 ] = "Unable to read from {}. Please specify a valid file or text.".format( 463 output_file 464 ) 465 log.error(ret["comment"]) 466 return ret 467 if not output: 468 ret["comment"] = "Please specify a valid output text or file" 469 log.error(ret["comment"]) 470 return ret 471 log.debug("Processing the raw text:\n%s", output) 472 try: 473 # Parse output through template 474 textfsm_obj.ParseCmd(output, attrs) 475 ret["out"] = _clitable_to_dict(textfsm_obj, textfsm_obj) 476 ret["result"] = True 477 except clitable.CliTableError as cterr: 478 log.error("Unable to proces the CliTable", exc_info=True) 479 ret["comment"] = "Unable to process the output: {}".format(cterr) 480 return ret 481