1import elasticsearch 2import curator 3import os 4import time 5from click import testing as clicktest 6import unittest 7from . import CuratorTestCase 8from . import testvars as testvars 9 10import logging 11logger = logging.getLogger(__name__) 12 13host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 14port = int(port) if port else 9200 15# ' - filtertype: {0}\n' 16# ' source: {1}\n' 17# ' direction: {2}\n' 18# ' timestring: {3}\n' 19# ' unit: {4}\n' 20# ' unit_count: {5}\n' 21# ' field: {6}\n' 22# ' stats_result: {7}\n' 23# ' epoch: {8}\n') 24 25global_client = elasticsearch.Elasticsearch(host=host, port=port) 26ILM_KEYS = ['ilm-history-1-000001', 'ilm-history-1'] 27 28def exclude_ilm_history(index_list): 29 """Remove any values from ILM_KEYS found in index_list""" 30 for val in ILM_KEYS: 31 if val in index_list: 32 index_list.remove(val) 33 return index_list 34class TestActionFileDeleteIndices(CuratorTestCase): 35 def test_retention_from_name_days(self): 36 # Test extraction of unit_count from index name 37 # Create indices for 10 days with retention time of 5 days in index name 38 # Expected: 5 oldest indices are deleted, 5 remain 39 self.args['prefix'] = 'logstash_5_' 40 self.create_indices(10) 41 self.write_config( 42 self.args['configfile'], testvars.client_config.format(host, port)) 43 self.write_config(self.args['actionfile'], 44 testvars.delete_pattern_proto.format( 45 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' ' 46 ) 47 ) 48 test = clicktest.CliRunner() 49 _ = test.invoke( 50 curator.cli, 51 [ 52 '--config', self.args['configfile'], 53 self.args['actionfile'] 54 ], 55 ) 56 57 self.assertEquals(5, len(exclude_ilm_history(curator.get_indices(self.client)))) 58 def test_retention_from_name_days_ignore_failed_match(self): 59 # Test extraction of unit_count from index name 60 # Create indices for 10 days with retention time of 5 days in index name 61 # Create indices for 10 days with no retention time in index name 62 # Expected: 5 oldest indices are deleted, 5 remain - 10 indices without retention time are ignored and remain 63 self.args['prefix'] = 'logstash_5_' 64 self.create_indices(10) 65 self.args['prefix'] = 'logstash_' 66 self.create_indices(10) 67 self.write_config( 68 self.args['configfile'], testvars.client_config.format(host, port)) 69 self.write_config(self.args['actionfile'], 70 testvars.delete_pattern_proto.format( 71 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' ' 72 ) 73 ) 74 test = clicktest.CliRunner() 75 _ = test.invoke( 76 curator.cli, 77 [ 78 '--config', self.args['configfile'], 79 self.args['actionfile'] 80 ], 81 ) 82 self.assertEquals(15, len(exclude_ilm_history(curator.get_indices(self.client)))) 83 def test_retention_from_name_days_keep_exclude_false_after_failed_match(self): 84 # Test extraction of unit_count from index name and confirm correct 85 # behavior after a failed regex match with no fallback time - see gh issue 1206 86 # Create indices for 30 days with retention time of 5 days in index name 87 # 88 # Create indices for 10 days with no retention time in index name 89 # that alphabetically sort before the 10 with retention time 90 # 91 # Create indices for 10 days with no retention time in index name 92 # that sort after the 10 with retention time 93 94 # Expected: 45 oldest matching indices are deleted, 5 matching indices remain 95 # 20 indices without retention time are ignored and remain 96 # overall 25 indices should remain 97 self.args['prefix'] = 'logstash-aanomatch-' 98 self.create_indices(10) 99 self.args['prefix'] = 'logstash-match-5-' 100 self.create_indices(30) 101 self.args['prefix'] = 'logstash-zznomatch-' 102 self.create_indices(10) 103 self.write_config( 104 self.args['configfile'], testvars.client_config.format(host, port)) 105 self.write_config(self.args['actionfile'], 106 testvars.delete_pattern_proto.format( 107 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', -1, r'logstash-\w+-([0-9]+)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}', ' ', ' ', ' ' 108 ) 109 ) 110 test = clicktest.CliRunner() 111 _ = test.invoke( 112 curator.cli, 113 [ 114 '--config', self.args['configfile'], 115 self.args['actionfile'] 116 ], 117 ) 118 self.assertEquals(25, len(exclude_ilm_history(curator.get_indices(self.client)))) 119 def test_retention_from_name_days_failed_match_with_fallback(self): 120 # Test extraction of unit_count from index name 121 # Create indices for 10 days with retention time of 5 days in index name 122 # Create indices for 10 days with no retention time in index name but configure fallback value of 7 123 # Expected: 5 oldest indices are deleted, 5 remain - 7 indices without retention time are ignored and remain due to the fallback value 124 self.args['prefix'] = 'logstash_5_' 125 self.create_indices(10) 126 self.args['prefix'] = 'logstash_' 127 self.create_indices(10) 128 self.write_config( 129 self.args['configfile'], testvars.client_config.format(host, port)) 130 self.write_config(self.args['actionfile'], 131 testvars.delete_pattern_proto.format( 132 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 7, '_([0-9]+)_', ' ', ' ', ' ' 133 ) 134 ) 135 test = clicktest.CliRunner() 136 _ = test.invoke( 137 curator.cli, 138 [ 139 '--config', self.args['configfile'], 140 self.args['actionfile'] 141 ], 142 ) 143 self.assertEquals(12, len(exclude_ilm_history(curator.get_indices(self.client)))) 144 def test_retention_from_name_no_capture_group(self): 145 # Test extraction of unit_count from index name when pattern contains no capture group 146 # Create indices for 10 months with retention time of 2 months in index name 147 # Expected: all indices remain as the pattern cannot be used to extract a retention time 148 self.args['prefix'] = 'logstash_2_' 149 self.args['time_unit'] = 'months' 150 self.create_indices(10) 151 self.write_config( 152 self.args['configfile'], testvars.client_config.format(host, port)) 153 self.write_config(self.args['actionfile'], 154 testvars.delete_pattern_proto.format( 155 'age', 'name', 'older', '\'%Y.%m\'', 'months', -1, '_[0-9]+_', ' ', ' ', ' ' 156 ) 157 ) 158 test = clicktest.CliRunner() 159 _ = test.invoke( 160 curator.cli, 161 [ 162 '--config', self.args['configfile'], 163 self.args['actionfile'] 164 ], 165 ) 166 self.assertEquals(10, len(exclude_ilm_history(curator.get_indices(self.client)))) 167 def test_retention_from_name_illegal_regex_no_fallback(self): 168 # Test extraction of unit_count from index name when pattern contains an illegal regular expression 169 # Create indices for 10 months with retention time of 2 months in index name 170 # Expected: all indices remain as the pattern cannot be used to extract a retention time 171 self.args['prefix'] = 'logstash_2_' 172 self.args['time_unit'] = 'months' 173 self.create_indices(10) 174 self.write_config( 175 self.args['configfile'], testvars.client_config.format(host, port)) 176 self.write_config(self.args['actionfile'], 177 testvars.delete_pattern_proto.format( 178 'age', 'name', 'older', '\'%Y.%m\'', 'months', -1, '_[0-9+_', ' ', ' ', ' ' 179 ) 180 ) 181 test = clicktest.CliRunner() 182 _ = test.invoke( 183 curator.cli, 184 [ 185 '--config', self.args['configfile'], 186 self.args['actionfile'] 187 ], 188 ) 189 self.assertEquals(10, len(exclude_ilm_history(curator.get_indices(self.client)))) 190 def test_retention_from_name_illegal_regex_with_fallback(self): 191 # Test extraction of unit_count from index name when pattern contains an illegal regular expression 192 # Create indices for 10 days with retention time of 2 days in index name 193 # Expected: Fallback value of 3 is used and 3 most recent indices remain in place 194 self.args['prefix'] = 'logstash_2_' 195 self.create_indices(10) 196 self.write_config( 197 self.args['configfile'], testvars.client_config.format(host, port)) 198 self.write_config(self.args['actionfile'], 199 testvars.delete_pattern_proto.format( 200 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 3, '_[0-9+_', ' ', ' ', ' ' 201 ) 202 ) 203 test = clicktest.CliRunner() 204 _ = test.invoke( 205 curator.cli, 206 [ 207 '--config', self.args['configfile'], 208 self.args['actionfile'] 209 ], 210 ) 211 self.assertEquals(3, len(exclude_ilm_history(curator.get_indices(self.client)))) 212 def test_name_older_than_now(self): 213 self.create_indices(10) 214 self.write_config( 215 self.args['configfile'], testvars.client_config.format(host, port)) 216 self.write_config(self.args['actionfile'], 217 testvars.delete_proto.format( 218 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' 219 ) 220 ) 221 test = clicktest.CliRunner() 222 _ = test.invoke( 223 curator.cli, 224 [ 225 '--config', self.args['configfile'], 226 self.args['actionfile'] 227 ], 228 ) 229 self.assertEquals(5, len(exclude_ilm_history(curator.get_indices(self.client)))) 230 def test_creation_date_newer_than_epoch(self): 231 self.create_indices(10) 232 self.write_config( 233 self.args['configfile'], testvars.client_config.format(host, port)) 234 self.write_config(self.args['actionfile'], 235 testvars.delete_proto.format( 236 'age', 'creation_date', 'younger', ' ', 'seconds', 0, 237 ' ', ' ', int(time.time()) - 60 238 ) 239 ) 240 test = clicktest.CliRunner() 241 _ = test.invoke( 242 curator.cli, 243 [ 244 '--config', self.args['configfile'], 245 self.args['actionfile'] 246 ], 247 ) 248 self.assertEquals(0, len(exclude_ilm_history(curator.get_indices(self.client)))) 249 def test_delete_in_period(self): 250 # filtertype: {0} 251 # source: {1} 252 # range_from: {2} 253 # range_to: {3} 254 # timestring: {4} 255 # unit: {5} 256 # field: {6} 257 # stats_result: {7} 258 # intersect: {8} 259 # epoch: {9} 260 # week_starts_on: {10} 261 self.create_indices(10) 262 self.write_config( 263 self.args['configfile'], testvars.client_config.format(host, port)) 264 self.write_config(self.args['actionfile'], 265 testvars.delete_period_proto.format( 266 'period', 'name', '-5', '-1', "'%Y.%m.%d'", 'days', 267 ' ', ' ', ' ', ' ', 'monday' 268 ) 269 ) 270 test = clicktest.CliRunner() 271 result = test.invoke( 272 curator.cli, 273 [ 274 '--config', self.args['configfile'], 275 self.args['actionfile'] 276 ], 277 ) 278 self.assertEqual(0, result.exit_code) 279 self.assertEquals(5, len(exclude_ilm_history(curator.get_indices(self.client)))) 280 def test_delete_in_period_absolute_date(self): 281 delete_period_abs = ('---\n' 282 'actions:\n' 283 ' 1:\n' 284 ' description: "Delete indices as filtered"\n' 285 ' action: delete_indices\n' 286 ' options:\n' 287 ' continue_if_exception: False\n' 288 ' disable_action: False\n' 289 ' filters:\n' 290 ' - filtertype: {0}\n' 291 ' period_type: absolute\n' 292 ' source: {1}\n' 293 ' date_from: {2}\n' 294 ' date_to: {3}\n' 295 ' timestring: {4}\n' 296 ' unit: {5}\n' 297 ' date_from_format: {6}\n' 298 ' date_to_format: {7}\n') 299 expected = 'index-2017.02.02' 300 self.create_index('index-2017.01.02') 301 self.create_index('index-2017.01.03') 302 self.create_index(expected) 303 self.write_config( 304 self.args['configfile'], testvars.client_config.format(host, port)) 305 self.write_config(self.args['actionfile'], 306 delete_period_abs.format( 307 'period', 'name', '2017.01.01', '2017.01.10', "'%Y.%m.%d'", 'days', 308 "'%Y.%m.%d'", "'%Y.%m.%d'" 309 ) 310 ) 311 test = clicktest.CliRunner() 312 result = test.invoke( 313 curator.cli, 314 [ 315 '--config', self.args['configfile'], 316 self.args['actionfile'] 317 ], 318 ) 319 indices = exclude_ilm_history(curator.get_indices(self.client)) 320 self.assertEqual(0, result.exit_code) 321 self.assertEqual(1, len(indices)) 322 self.assertEqual(expected, indices[0]) 323 def test_delete_in_period_intersect(self): 324 # filtertype: {0} 325 # source: {1} 326 # range_from: {2} 327 # range_to: {3} 328 # timestring: {4} 329 # unit: {5} 330 # field: {6} 331 # stats_result: {7} 332 # intersect: {8} 333 # epoch: {9} 334 # week_starts_on: {10} 335 # 2017-09-01T01:00:00 = 1504227600 336 # 2017-09-25T01:00:00 = 1506301200 337 # 2017-09-29T01:00:00 = 1506646800 338 self.create_index('intersecting') 339 self.create_index('notintersecting') 340 self.client.index(index='intersecting', doc_type='log', id='1', body={'@timestamp': '2017-09-25T01:00:00Z', 'doc' :'Earliest'}) 341 self.client.index(index='intersecting', doc_type='log', id='2', body={'@timestamp': '2017-09-29T01:00:00Z', 'doc' :'Latest'}) 342 self.client.index(index='notintersecting', doc_type='log', id='1', body={'@timestamp': '2017-09-01T01:00:00Z', 'doc' :'Earliest'}) 343 self.client.index(index='notintersecting', doc_type='log', id='2', body={'@timestamp': '2017-09-29T01:00:00Z', 'doc' :'Latest'}) 344 # Decorators cause this pylint error 345 # pylint: disable=E1123 346 self.client.indices.flush(index='_all', force=True) 347 self.client.indices.refresh(index='intersecting,notintersecting') 348 self.write_config( 349 self.args['configfile'], testvars.client_config.format(host, port)) 350 self.write_config(self.args['actionfile'], 351 testvars.delete_period_proto.format( 352 'period', 'field_stats', '0', '0', ' ', 'weeks', 353 "'@timestamp'", 'min_value', 'true', 1506716040, 'sunday' 354 ) 355 ) 356 test = clicktest.CliRunner() 357 result = test.invoke( 358 curator.cli, 359 ['--config', self.args['configfile'], self.args['actionfile']], 360 ) 361 self.assertEqual(0, result.exit_code) 362 indices = exclude_ilm_history(curator.get_indices(self.client)) 363 self.assertEquals(1, len(indices)) 364 self.assertEqual('notintersecting', indices[0]) 365 def test_empty_list(self): 366 self.create_indices(10) 367 self.write_config( 368 self.args['configfile'], testvars.client_config.format(host, port)) 369 self.write_config(self.args['actionfile'], 370 testvars.delete_proto.format( 371 'age', 'creation_date', 'older', ' ', 'days', 90, 372 ' ', ' ', int(time.time()) 373 ) 374 ) 375 test = clicktest.CliRunner() 376 result = test.invoke( 377 curator.cli, 378 [ 379 '--config', self.args['configfile'], 380 self.args['actionfile'] 381 ], 382 ) 383 self.assertEquals(10, len(exclude_ilm_history(curator.get_indices(self.client)))) 384 self.assertEqual(1, result.exit_code) 385 def test_ignore_empty_list(self): 386 self.create_indices(10) 387 self.write_config( 388 self.args['configfile'], testvars.client_config.format(host, port)) 389 self.write_config(self.args['actionfile'], 390 testvars.delete_ignore_proto.format( 391 'age', 'creation_date', 'older', ' ', 'days', 90, 392 ' ', ' ', int(time.time()) 393 ) 394 ) 395 test = clicktest.CliRunner() 396 result = test.invoke( 397 curator.cli, 398 [ 399 '--config', self.args['configfile'], 400 self.args['actionfile'] 401 ], 402 ) 403 self.assertEquals(10, len(exclude_ilm_history(curator.get_indices(self.client)))) 404 self.assertEqual(0, result.exit_code) 405 def test_extra_options(self): 406 self.write_config( 407 self.args['configfile'], testvars.client_config.format(host, port)) 408 self.write_config(self.args['actionfile'], 409 testvars.bad_option_proto_test.format('delete_indices')) 410 test = clicktest.CliRunner() 411 result = test.invoke( 412 curator.cli, 413 [ 414 '--config', self.args['configfile'], 415 self.args['actionfile'] 416 ], 417 ) 418 self.assertEqual(1, result.exit_code) 419 def test_945(self): 420 self.create_indices(10) 421 self.write_config( 422 self.args['configfile'], testvars.client_config.format(host, port)) 423 self.write_config(self.args['actionfile'], testvars.test_945) 424 test = clicktest.CliRunner() 425 result = test.invoke( 426 curator.cli, 427 [ 428 '--config', self.args['configfile'], 429 self.args['actionfile'] 430 ], 431 ) 432 self.assertEqual(1, result.exit_code) 433 def test_name_epoch_zero(self): 434 self.create_index('epoch_zero-1970.01.01') 435 self.write_config( 436 self.args['configfile'], testvars.client_config.format(host, port)) 437 self.write_config(self.args['actionfile'], 438 testvars.delete_proto.format( 439 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' 440 ) 441 ) 442 test = clicktest.CliRunner() 443 _ = test.invoke( 444 curator.cli, 445 [ 446 '--config', self.args['configfile'], 447 self.args['actionfile'] 448 ], 449 ) 450 self.assertEquals(0, len(exclude_ilm_history(curator.get_indices(self.client)))) 451 def test_name_negative_epoch(self): 452 self.create_index('index-1969.12.31') 453 self.write_config( 454 self.args['configfile'], testvars.client_config.format(host, port)) 455 self.write_config(self.args['actionfile'], 456 testvars.delete_proto.format( 457 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' 458 ) 459 ) 460 test = clicktest.CliRunner() 461 _ = test.invoke( 462 curator.cli, 463 [ 464 '--config', self.args['configfile'], 465 self.args['actionfile'] 466 ], 467 ) 468 self.assertEquals(0, len(exclude_ilm_history(curator.get_indices(self.client)))) 469 def test_allow_ilm_indices_true(self): 470 # ILM will not be added until 6.6 471 if curator.get_version(self.client) < (6,6,0): 472 self.assertTrue(True) 473 else: 474 import requests 475 name = 'test' 476 policy = { 477 'policy': { 478 'phases': { 479 'hot': { 480 'min_age': '0ms', 481 'actions': { 482 'rollover': { 483 'max_age': '2h', 484 'max_docs': 4 485 } 486 } 487 } 488 } 489 } 490 } 491 url = 'http://{0}:{1}/_ilm/policy/{2}'.format(host, port, name) 492 r = requests.put(url, json=policy) 493 # print(r.text) # logging reminder 494 self.create_indices(10, ilm_policy=name) 495 self.write_config( 496 self.args['configfile'], testvars.client_config.format(host, port)) 497 self.write_config(self.args['actionfile'], 498 testvars.ilm_delete_proto.format( 499 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'true' 500 ) 501 ) 502 test = clicktest.CliRunner() 503 _ = test.invoke( 504 curator.cli, 505 [ '--config', self.args['configfile'], self.args['actionfile'] ], 506 ) 507 self.assertEquals(5, len(exclude_ilm_history(curator.get_indices(self.client)))) 508 def test_allow_ilm_indices_false(self): 509 # ILM will not be added until 6.6 510 if curator.get_version(self.client) < (6,6,0): 511 512 self.assertTrue(True) 513 else: 514 import requests 515 name = 'test' 516 policy = { 517 'policy': { 518 'phases': { 519 'hot': { 520 'min_age': '0ms', 521 'actions': { 522 'rollover': { 523 'max_age': '2h', 524 'max_docs': 4 525 } 526 } 527 } 528 } 529 } 530 } 531 url = 'http://{0}:{1}/_ilm/policy/{2}'.format(host, port, name) 532 r = requests.put(url, json=policy) 533 # print(r.text) # logging reminder 534 self.create_indices(10, ilm_policy=name) 535 self.write_config( 536 self.args['configfile'], testvars.client_config.format(host, port)) 537 self.write_config(self.args['actionfile'], 538 testvars.ilm_delete_proto.format( 539 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false' 540 ) 541 ) 542 test = clicktest.CliRunner() 543 _ = test.invoke( 544 curator.cli, 545 [ '--config', self.args['configfile'], self.args['actionfile'] ], 546 ) 547 self.assertEquals(10, len(exclude_ilm_history(curator.get_indices(self.client)))) 548 549class TestCLIDeleteIndices(CuratorTestCase): 550 def test_name_older_than_now_cli(self): 551 self.create_indices(10) 552 args = self.get_runner_args() 553 args += [ 554 '--config', self.args['configfile'], 555 'delete-indices', 556 '--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}', 557 ] 558 self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli')) 559 self.assertEquals(5, len(exclude_ilm_history(curator.get_indices(self.client))))