1# Copyright 2013-2018 the openage authors. See copying.md for legal info. 2 3# TODO pylint: disable=C,R 4 5import pickle 6from zlib import decompress 7 8from . import civ 9from . import graphic 10from . import maps 11from . import playercolor 12from . import research 13from . import sound 14from . import tech 15from . import terrain 16from . import unit 17 18from ..game_versions import GameVersion 19from ..dataformat.exportable import Exportable 20from ..dataformat.members import SubdataMember 21from ..dataformat.member_access import READ, READ_EXPORT, READ_UNKNOWN 22 23from ...log import spam, dbg, info, warn 24 25 26# this file can parse and represent the empires2_x1_p1.dat file. 27# 28# the dat file contain all the information needed for running the game. 29# all units, buildings, terrains, whatever are defined in this dat file. 30# 31# documentation for this can be found in `doc/gamedata` 32# the binary structure, which the dat file has, is in `doc/gamedata.struct` 33 34 35class EmpiresDat(Exportable): 36 """ 37 class for fighting and beating the compressed empires2*.dat 38 39 represents the main game data file. 40 """ 41 42 name_struct_file = "gamedata" 43 name_struct = "empiresdat" 44 struct_description = "empires2_x1_p1.dat structure" 45 46 data_format = [(READ, "versionstr", "char[8]")] 47 48 # TODO: Enable conversion for SWGB 49 # =========================================================================== 50 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 51 # data_format.extend([ 52 # (READ, "civ_count_swgb", "uint16_t"), 53 # (READ_UNKNOWN, None, "int32_t"), 54 # (READ_UNKNOWN, None, "int32_t"), 55 # (READ_UNKNOWN, None, "int32_t"), 56 # (READ_UNKNOWN, None, "int32_t"), 57 # ]) 58 # =========================================================================== 59 60 # terrain header data 61 data_format.extend([ 62 (READ, "terrain_restriction_count", "uint16_t"), 63 (READ, "terrain_count", "uint16_t"), # number of "used" terrains 64 (READ, "float_ptr_terrain_tables", "int32_t[terrain_restriction_count]"), 65 ]) 66 67 # TODO: Enable conversion for AOE1; replace "terrain_pass_graphics_ptrs" 68 # =========================================================================== 69 # if (GameVersion.aoe_1 or GameVersion.aoe_ror) not in game_versions: 70 # data_format.append((READ, "terrain_pass_graphics_ptrs", "int32_t[terrain_restriction_count]")) 71 # =========================================================================== 72 data_format.append((READ, "terrain_pass_graphics_ptrs", "int32_t[terrain_restriction_count]")) 73 74 data_format.extend([ 75 (READ, "terrain_restrictions", SubdataMember( 76 ref_type=terrain.TerrainRestriction, 77 length="terrain_restriction_count", 78 passed_args={"terrain_count"}, 79 )), 80 81 # player color data 82 (READ, "player_color_count", "uint16_t"), 83 (READ, "player_colors", SubdataMember( 84 ref_type=playercolor.PlayerColor, 85 length="player_color_count", 86 )), 87 88 # sound data 89 (READ_EXPORT, "sound_count", "uint16_t"), 90 (READ_EXPORT, "sounds", SubdataMember( 91 ref_type=sound.Sound, 92 length="sound_count", 93 )), 94 95 # graphic data 96 (READ, "graphic_count", "uint16_t"), 97 (READ, "graphic_ptrs", "uint32_t[graphic_count]"), 98 (READ_EXPORT, "graphics", SubdataMember( 99 ref_type = graphic.Graphic, 100 length = "graphic_count", 101 offset_to = ("graphic_ptrs", lambda o: o > 0), 102 )), 103 104 # terrain data 105 (READ, "virt_function_ptr", "int32_t"), 106 (READ, "map_pointer", "int32_t"), 107 (READ, "map_width", "int32_t"), 108 (READ, "map_height", "int32_t"), 109 (READ, "world_width", "int32_t"), 110 (READ, "world_height", "int32_t"), 111 (READ_EXPORT, "tile_sizes", SubdataMember( 112 ref_type=terrain.TileSize, 113 length=19, # number of tile types 114 )), 115 (READ, "padding1", "int16_t"), 116 ]) 117 118 # TODO: Enable conversion for SWGB; replace "terrains" 119 # =========================================================================== 120 # # 42 terrains are stored (100 in African Kingdoms), but less are used. 121 # # TODO: maybe this number is defined somewhere. 122 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 123 # data_format.append((READ_EXPORT, "terrains", SubdataMember( 124 # ref_type=terrain.Terrain, 125 # length=55, 126 # ))) 127 # elif GameVersion.age2_hd_ak in game_versions: 128 # data_format.append((READ_EXPORT, "terrains", SubdataMember( 129 # ref_type=terrain.Terrain, 130 # length=100, 131 # ))) 132 # elif (GameVersion.aoe_1 or GameVersion.aoe_ror) in game_versions: 133 # data_format.append((READ_EXPORT, "terrains", SubdataMember( 134 # ref_type=terrain.Terrain, 135 # length=42, 136 # ))) 137 # else: 138 # data_format.append((READ_EXPORT, "terrains", SubdataMember( 139 # ref_type=terrain.Terrain, 140 # length=42, 141 # ))) 142 # =========================================================================== 143 data_format.append( 144 (READ_EXPORT, "terrains", SubdataMember( 145 ref_type=terrain.Terrain, 146 # 42 terrains are stored (100 in African Kingdoms), but less are used. 147 # TODO: maybe this number is defined somewhere. 148 length=(lambda self: 149 100 if GameVersion.age2_hd_ak in self.game_versions 150 else 42), 151 ))) 152 153 data_format.extend([ 154 (READ, "terrain_border", SubdataMember( 155 ref_type=terrain.TerrainBorder, 156 length=16, 157 )), 158 159 (READ, "map_row_offset", "int32_t"), 160 (READ, "map_min_x", "float"), 161 (READ, "map_min_y", "float"), 162 (READ, "map_max_x", "float"), 163 (READ, "map_max_y", "float"), 164 (READ, "map_max_xplus1", "float"), 165 (READ, "map_min_yplus1", "float"), 166 167 (READ, "terrain_count_additional", "uint16_t"), 168 (READ, "borders_used", "uint16_t"), 169 (READ, "max_terrain", "int16_t"), 170 (READ_EXPORT, "tile_width", "int16_t"), 171 (READ_EXPORT, "tile_height", "int16_t"), 172 (READ_EXPORT, "tile_half_height", "int16_t"), 173 (READ_EXPORT, "tile_half_width", "int16_t"), 174 (READ_EXPORT, "elev_height", "int16_t"), 175 (READ, "current_row", "int16_t"), 176 (READ, "current_column", "int16_t"), 177 (READ, "block_beginn_row", "int16_t"), 178 (READ, "block_end_row", "int16_t"), 179 (READ, "block_begin_column", "int16_t"), 180 (READ, "block_end_column", "int16_t"), 181 (READ, "search_map_ptr", "int32_t"), 182 (READ, "search_map_rows_ptr", "int32_t"), 183 (READ, "any_frame_change", "int8_t"), 184 (READ, "map_visible_flag", "int8_t"), 185 (READ, "fog_flag", "int8_t"), 186 ]) 187 188 # TODO: Enable conversion for SWGB; replace "terrain_blob0" 189 # =========================================================================== 190 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 191 # data_format.append((READ_UNKNOWN, "terrain_blob0", "uint8_t[25]")) 192 # else: 193 # data_format.append((READ_UNKNOWN, "terrain_blob0", "uint8_t[21]")) 194 # =========================================================================== 195 data_format.append((READ_UNKNOWN, "terrain_blob0", "uint8_t[21]")) 196 197 data_format.extend([ 198 (READ_UNKNOWN, "terrain_blob1", "uint32_t[157]"), 199 200 # random map config 201 (READ, "random_map_count", "uint32_t"), 202 (READ, "random_map_ptr", "uint32_t"), 203 (READ, "map_infos", SubdataMember( 204 ref_type=maps.MapInfo, 205 length="random_map_count", 206 )), 207 (READ, "maps", SubdataMember( 208 ref_type=maps.Map, 209 length="random_map_count", 210 )), 211 212 # technology data 213 (READ_EXPORT, "tech_count", "uint32_t"), 214 (READ_EXPORT, "techs", SubdataMember( 215 ref_type=tech.Tech, 216 length="tech_count", 217 )), 218 ]) 219 220 # TODO: Enable conversion for SWGB 221 # =========================================================================== 222 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 223 # data_format.extend([ 224 # (READ, "unit_line_count", "uint16_t"), 225 # (READ, "unit_lines", SubdataMember( 226 # ref_type=unit.UnitLine, 227 # length="unit_line_count", 228 # )), 229 # ]) 230 # =========================================================================== 231 232 # unit header data 233 # TODO: Enable conversion for AOE1; replace "unit_count", "unit_headers" 234 # =========================================================================== 235 # if (GameVersion.aoe_1 or GameVersion.aoe_ror) not in game_versions: 236 # data_format.extend([(READ_EXPORT, "unit_count", "uint32_t"), 237 # (READ_EXPORT, "unit_headers", SubdataMember( 238 # ref_type=unit.UnitHeader, 239 # length="unit_count", 240 # )), 241 # ]) 242 # =========================================================================== 243 data_format.extend([ 244 (READ_EXPORT, "unit_count", "uint32_t"), 245 (READ_EXPORT, "unit_headers", SubdataMember( 246 ref_type=unit.UnitHeader, 247 length="unit_count", 248 )), 249 ]) 250 251 # civilisation data 252 data_format.extend([ 253 (READ_EXPORT, "civ_count", "uint16_t"), 254 (READ_EXPORT, "civs", SubdataMember( 255 ref_type=civ.Civ, 256 length="civ_count" 257 )), 258 ]) 259 260 # TODO: Enable conversion for SWGB 261 # =========================================================================== 262 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 263 # data_format.append((READ_UNKNOWN, None, "int8_t")) 264 # =========================================================================== 265 266 # research data 267 data_format.extend([ 268 (READ_EXPORT, "research_count", "uint16_t"), 269 (READ_EXPORT, "researches", SubdataMember( 270 ref_type=research.Research, 271 length="research_count" 272 )), 273 ]) 274 275 # TODO: Enable conversion for SWGB 276 # =========================================================================== 277 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 278 # data_format.append((READ_UNKNOWN, None, "int8_t")) 279 # =========================================================================== 280 281 # TODO: Enable conversion for AOE1; replace the 7 values below 282 # =========================================================================== 283 # if (GameVersion.aoe_1 or GameVersion.aoe_ror) not in game_versions: 284 # data_format.extend([ 285 # (READ, "time_slice", "int32_t"), 286 # (READ, "unit_kill_rate", "int32_t"), 287 # (READ, "unit_kill_total", "int32_t"), 288 # (READ, "unit_hitpoint_rate", "int32_t"), 289 # (READ, "unit_hitpoint_total", "int32_t"), 290 # (READ, "razing_kill_rate", "int32_t"), 291 # (READ, "razing_kill_total", "int32_t"), 292 # ]) 293 # =========================================================================== 294 data_format.extend([ 295 (READ, "time_slice", "int32_t"), 296 (READ, "unit_kill_rate", "int32_t"), 297 (READ, "unit_kill_total", "int32_t"), 298 (READ, "unit_hitpoint_rate", "int32_t"), 299 (READ, "unit_hitpoint_total", "int32_t"), 300 (READ, "razing_kill_rate", "int32_t"), 301 (READ, "razing_kill_total", "int32_t"), 302 ]) 303 # =========================================================================== 304 305 # technology tree data 306 data_format.extend([ 307 (READ_EXPORT, "age_entry_count", "uint8_t"), 308 (READ_EXPORT, "building_connection_count", "uint8_t"), 309 ]) 310 311 # TODO: Enable conversion for SWGB; replace "unit_connection_count" 312 # =========================================================================== 313 # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions: 314 # data_format.append((READ_EXPORT, "unit_connection_count", "uint16_t")) 315 # else: 316 # data_format.append((READ_EXPORT, "unit_connection_count", "uint8_t")) 317 # =========================================================================== 318 data_format.append((READ_EXPORT, "unit_connection_count", "uint8_t")) 319 320 data_format.extend([ 321 (READ_EXPORT, "research_connection_count", "uint8_t"), 322 (READ_EXPORT, "age_tech_tree", SubdataMember( 323 ref_type=tech.AgeTechTree, 324 length="age_entry_count" 325 )), 326 # What is this? There shouldn't be something here 327 (READ_UNKNOWN, None, "int32_t"), 328 (READ_EXPORT, "building_connection", SubdataMember( 329 ref_type=tech.BuildingConnection, 330 length="building_connection_count" 331 )), 332 (READ_EXPORT, "unit_connection", SubdataMember( 333 ref_type=tech.UnitConnection, 334 length="unit_connection_count" 335 )), 336 (READ_EXPORT, "research_connection", SubdataMember( 337 ref_type=tech.ResearchConnection, 338 length="research_connection_count" 339 )), 340 ]) 341 342 @classmethod 343 def get_hash(cls): 344 """ return the unique hash for the data format tree """ 345 346 return cls.format_hash().hexdigest() 347 348 349class EmpiresDatWrapper(Exportable): 350 """ 351 This wrapper exists because the top-level element is discarded: 352 The gathered data fields are passed to the parent, 353 and are accumulated there to be written out. 354 355 This class acts as the parent for the "real" data values, 356 and has no parent itself. Thereby this class is discarded 357 and the child classes use this as parent for their return values. 358 """ 359 360 name_struct_file = "gamedata" 361 name_struct = "gamedata" 362 struct_description = "wrapper for empires2_x1_p1.dat structure" 363 364 # TODO: we could reference to other gamedata structures 365 data_format = [ 366 (READ_EXPORT, "empiresdat", SubdataMember( 367 ref_type=EmpiresDat, 368 length=1, 369 )), 370 ] 371 372 373def load_gamespec(fileobj, game_versions, cachefile_name=None, load_cache=False): 374 """ 375 Helper method that loads the contents of a 'empires.dat' gzipped gamespec 376 file. 377 378 If cachefile_name is given, this file is consulted before performing the 379 load. 380 """ 381 # try to use the cached result from a previous run 382 if cachefile_name and load_cache: 383 try: 384 with open(cachefile_name, "rb") as cachefile: 385 # pickle.load() can fail in many ways, we need to catch all. 386 # pylint: disable=broad-except 387 try: 388 gamespec = pickle.load(cachefile) 389 info("using cached gamespec: %s", cachefile_name) 390 return gamespec 391 except Exception: 392 warn("could not use cached gamespec:") 393 import traceback 394 traceback.print_exc() 395 warn("we will just skip the cache, no worries.") 396 397 except FileNotFoundError: 398 pass 399 400 # read the file ourselves 401 402 dbg("reading dat file") 403 compressed_data = fileobj.read() 404 fileobj.close() 405 406 dbg("decompressing dat file") 407 # -15: there's no header, window size is 15. 408 file_data = decompress(compressed_data, -15) 409 del compressed_data 410 411 spam("length of decompressed data: %d", len(file_data)) 412 413 gamespec = EmpiresDatWrapper(game_versions=game_versions) 414 gamespec.read(file_data, 0) 415 416 if cachefile_name: 417 dbg("dumping dat file contents to cache file: %s", cachefile_name) 418 with open(cachefile_name, "wb") as cachefile: 419 pickle.dump(gamespec, cachefile) 420 421 return gamespec 422