1""" 2Compare lists of dictionaries by a specified key. 3 4The following can be retrieved: 5 (1) List of added, removed, intersect elements 6 (2) List of diffs having the following format: 7 <key_val>: {<elem_key: {'old': <old_value>, 'new': <new_value>}} 8 A recursive diff is done between the values (dicts) with the same 9 key 10 (3) List with the new values for each key 11 (4) List with the old values for each key 12 (5) List of changed items in the format 13 ('<key_val>.<elem_key>', {'old': <old_value>, 'new': <new_value>}) 14 (5) String representations of the list diff 15 16Note: All dictionaries keys are expected to be strings 17""" 18 19from salt.utils.dictdiffer import recursive_diff 20 21 22def list_diff(list_a, list_b, key): 23 return ListDictDiffer(list_a, list_b, key) 24 25 26class ListDictDiffer: 27 """ 28 Calculates the differences between two lists of dictionaries. 29 30 It matches the items based on a given key and uses the recursive_diff to 31 diff the two values. 32 """ 33 34 def __init__(self, current_list, next_list, key): 35 self._intersect = [] 36 self._removed = [] 37 self._added = [] 38 self._new = next_list 39 self._current = current_list 40 self._key = key 41 for current_item in current_list: 42 if key not in current_item: 43 raise ValueError( 44 "The supplied key '{}' does not " 45 "exist in item, the available keys are: {}" 46 "".format(key, current_item.keys()) 47 ) 48 for next_item in next_list: 49 if key not in next_item: 50 raise ValueError( 51 "The supplied key '{}' does not " 52 "exist in item, the available keys are: " 53 "{}".format(key, next_item.keys()) 54 ) 55 if next_item[key] == current_item[key]: 56 item = {key: next_item[key], "old": current_item, "new": next_item} 57 self._intersect.append(item) 58 break 59 else: 60 self._removed.append(current_item) 61 62 for next_item in next_list: 63 for current_item in current_list: 64 if next_item[key] == current_item[key]: 65 break 66 else: 67 self._added.append(next_item) 68 69 def _get_recursive_difference(self, type): 70 """Returns the recursive diff between dict values""" 71 if type == "intersect": 72 return [ 73 recursive_diff(item["old"], item["new"]) for item in self._intersect 74 ] 75 elif type == "added": 76 return [recursive_diff({}, item) for item in self._added] 77 elif type == "removed": 78 return [ 79 recursive_diff(item, {}, ignore_missing_keys=False) 80 for item in self._removed 81 ] 82 elif type == "all": 83 recursive_list = [] 84 recursive_list.extend( 85 [recursive_diff(item["old"], item["new"]) for item in self._intersect] 86 ) 87 recursive_list.extend([recursive_diff({}, item) for item in self._added]) 88 recursive_list.extend( 89 [ 90 recursive_diff(item, {}, ignore_missing_keys=False) 91 for item in self._removed 92 ] 93 ) 94 return recursive_list 95 else: 96 raise ValueError( 97 "The given type for recursive list matching is not supported." 98 ) 99 100 @property 101 def removed(self): 102 """Returns the objects which are removed from the list""" 103 return self._removed 104 105 @property 106 def added(self): 107 """Returns the objects which are added to the list""" 108 return self._added 109 110 @property 111 def intersect(self): 112 """Returns the intersect objects""" 113 return self._intersect 114 115 def remove_diff(self, diff_key=None, diff_list="intersect"): 116 """Deletes an attribute from all of the intersect objects""" 117 if diff_list == "intersect": 118 for item in self._intersect: 119 item["old"].pop(diff_key, None) 120 item["new"].pop(diff_key, None) 121 if diff_list == "removed": 122 for item in self._removed: 123 item.pop(diff_key, None) 124 125 @property 126 def diffs(self): 127 """ 128 Returns a list of dictionaries with key value pairs. 129 The values are the differences between the items identified by the key. 130 """ 131 differences = [] 132 for item in self._get_recursive_difference(type="all"): 133 if item.diffs: 134 if item.past_dict: 135 differences.append({item.past_dict[self._key]: item.diffs}) 136 elif item.current_dict: 137 differences.append({item.current_dict[self._key]: item.diffs}) 138 return differences 139 140 @property 141 def changes_str(self): 142 """Returns a string describing the changes""" 143 changes = "" 144 for item in self._get_recursive_difference(type="intersect"): 145 if item.diffs: 146 changes = "".join( 147 [ 148 changes, 149 # Tabulate comment deeper, show the key attribute and the value 150 # Next line should be tabulated even deeper, 151 # every change should be tabulated 1 deeper 152 "\tidentified by {} {}:\n\t{}\n".format( 153 self._key, 154 item.past_dict[self._key], 155 item.changes_str.replace("\n", "\n\t"), 156 ), 157 ] 158 ) 159 for item in self._get_recursive_difference(type="removed"): 160 if item.past_dict: 161 changes = "".join( 162 [ 163 changes, 164 # Tabulate comment deeper, show the key attribute and the value 165 "\tidentified by {} {}:\n\twill be removed\n".format( 166 self._key, item.past_dict[self._key] 167 ), 168 ] 169 ) 170 for item in self._get_recursive_difference(type="added"): 171 if item.current_dict: 172 changes = "".join( 173 [ 174 changes, 175 # Tabulate comment deeper, show the key attribute and the value 176 "\tidentified by {} {}:\n\twill be added\n".format( 177 self._key, item.current_dict[self._key] 178 ), 179 ] 180 ) 181 return changes 182 183 @property 184 def changes_str2(self, tab_string=" "): 185 """ 186 Returns a string in a more compact format describing the changes. 187 188 The output better alligns with the one in recursive_diff. 189 """ 190 changes = [] 191 for item in self._get_recursive_difference(type="intersect"): 192 if item.diffs: 193 changes.append( 194 "{tab}{0}={1} (updated):\n{tab}{tab}{2}".format( 195 self._key, 196 item.past_dict[self._key], 197 item.changes_str.replace("\n", "\n{0}{0}".format(tab_string)), 198 tab=tab_string, 199 ) 200 ) 201 for item in self._get_recursive_difference(type="removed"): 202 if item.past_dict: 203 changes.append( 204 "{tab}{0}={1} (removed)".format( 205 self._key, item.past_dict[self._key], tab=tab_string 206 ) 207 ) 208 for item in self._get_recursive_difference(type="added"): 209 if item.current_dict: 210 changes.append( 211 "{tab}{0}={1} (added): {2}".format( 212 self._key, 213 item.current_dict[self._key], 214 dict(item.current_dict), 215 tab=tab_string, 216 ) 217 ) 218 return "\n".join(changes) 219 220 @property 221 def new_values(self): 222 """Returns the new values from the diff""" 223 224 def get_new_values_and_key(item): 225 values = item.new_values 226 if item.past_dict: 227 values.update({self._key: item.past_dict[self._key]}) 228 else: 229 # This is a new item as it has no past_dict 230 values.update({self._key: item.current_dict[self._key]}) 231 return values 232 233 return [ 234 get_new_values_and_key(el) 235 for el in self._get_recursive_difference("all") 236 if el.diffs and el.current_dict 237 ] 238 239 @property 240 def old_values(self): 241 """Returns the old values from the diff""" 242 243 def get_old_values_and_key(item): 244 values = item.old_values 245 values.update({self._key: item.past_dict[self._key]}) 246 return values 247 248 return [ 249 get_old_values_and_key(el) 250 for el in self._get_recursive_difference("all") 251 if el.diffs and el.past_dict 252 ] 253 254 def changed(self, selection="all"): 255 """ 256 Returns the list of changed values. 257 The key is added to each item. 258 259 selection 260 Specifies the desired changes. 261 Supported values are 262 ``all`` - all changed items are included in the output 263 ``intersect`` - changed items present in both lists are included 264 """ 265 changed = [] 266 if selection == "all": 267 for recursive_item in self._get_recursive_difference(type="all"): 268 # We want the unset values as well 269 recursive_item.ignore_unset_values = False 270 key_val = ( 271 str(recursive_item.past_dict[self._key]) 272 if self._key in recursive_item.past_dict 273 else str(recursive_item.current_dict[self._key]) 274 ) 275 276 for change in recursive_item.changed(): 277 if change != self._key: 278 changed.append(".".join([self._key, key_val, change])) 279 return changed 280 elif selection == "intersect": 281 # We want the unset values as well 282 for recursive_item in self._get_recursive_difference(type="intersect"): 283 recursive_item.ignore_unset_values = False 284 key_val = ( 285 str(recursive_item.past_dict[self._key]) 286 if self._key in recursive_item.past_dict 287 else str(recursive_item.current_dict[self._key]) 288 ) 289 290 for change in recursive_item.changed(): 291 if change != self._key: 292 changed.append(".".join([self._key, key_val, change])) 293 return changed 294 295 @property 296 def current_list(self): 297 return self._current 298 299 @property 300 def new_list(self): 301 return self._new 302