1# Copyright 2013-2018 the openage authors. See copying.md for legal info.
2
3# TODO pylint: disable=C,R
4
5import pickle
6from zlib import decompress
7
8from . import civ
9from . import graphic
10from . import maps
11from . import playercolor
12from . import research
13from . import sound
14from . import tech
15from . import terrain
16from . import unit
17
18from ..game_versions import GameVersion
19from ..dataformat.exportable import Exportable
20from ..dataformat.members import SubdataMember
21from ..dataformat.member_access import READ, READ_EXPORT, READ_UNKNOWN
22
23from ...log import spam, dbg, info, warn
24
25
26# this file can parse and represent the empires2_x1_p1.dat file.
27#
28# the dat file contain all the information needed for running the game.
29# all units, buildings, terrains, whatever are defined in this dat file.
30#
31# documentation for this can be found in `doc/gamedata`
32# the binary structure, which the dat file has, is in `doc/gamedata.struct`
33
34
35class EmpiresDat(Exportable):
36    """
37    class for fighting and beating the compressed empires2*.dat
38
39    represents the main game data file.
40    """
41
42    name_struct_file   = "gamedata"
43    name_struct        = "empiresdat"
44    struct_description = "empires2_x1_p1.dat structure"
45
46    data_format = [(READ, "versionstr", "char[8]")]
47
48    # TODO: Enable conversion for SWGB
49    # ===========================================================================
50    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
51    #     data_format.extend([
52    #         (READ, "civ_count_swgb", "uint16_t"),
53    #         (READ_UNKNOWN, None, "int32_t"),
54    #         (READ_UNKNOWN, None, "int32_t"),
55    #         (READ_UNKNOWN, None, "int32_t"),
56    #         (READ_UNKNOWN, None, "int32_t"),
57    #     ])
58    # ===========================================================================
59
60    # terrain header data
61    data_format.extend([
62        (READ, "terrain_restriction_count", "uint16_t"),
63        (READ, "terrain_count", "uint16_t"),   # number of "used" terrains
64        (READ, "float_ptr_terrain_tables", "int32_t[terrain_restriction_count]"),
65    ])
66
67    # TODO: Enable conversion for AOE1; replace "terrain_pass_graphics_ptrs"
68    # ===========================================================================
69    # if (GameVersion.aoe_1 or GameVersion.aoe_ror) not in game_versions:
70    #     data_format.append((READ, "terrain_pass_graphics_ptrs", "int32_t[terrain_restriction_count]"))
71    # ===========================================================================
72    data_format.append((READ, "terrain_pass_graphics_ptrs", "int32_t[terrain_restriction_count]"))
73
74    data_format.extend([
75        (READ, "terrain_restrictions", SubdataMember(
76            ref_type=terrain.TerrainRestriction,
77            length="terrain_restriction_count",
78            passed_args={"terrain_count"},
79        )),
80
81        # player color data
82        (READ, "player_color_count", "uint16_t"),
83        (READ, "player_colors", SubdataMember(
84            ref_type=playercolor.PlayerColor,
85            length="player_color_count",
86        )),
87
88        # sound data
89        (READ_EXPORT, "sound_count", "uint16_t"),
90        (READ_EXPORT, "sounds", SubdataMember(
91            ref_type=sound.Sound,
92            length="sound_count",
93        )),
94
95        # graphic data
96        (READ, "graphic_count", "uint16_t"),
97        (READ, "graphic_ptrs", "uint32_t[graphic_count]"),
98        (READ_EXPORT, "graphics", SubdataMember(
99            ref_type  = graphic.Graphic,
100            length    = "graphic_count",
101            offset_to = ("graphic_ptrs", lambda o: o > 0),
102        )),
103
104        # terrain data
105        (READ, "virt_function_ptr", "int32_t"),
106        (READ, "map_pointer", "int32_t"),
107        (READ, "map_width", "int32_t"),
108        (READ, "map_height", "int32_t"),
109        (READ, "world_width", "int32_t"),
110        (READ, "world_height", "int32_t"),
111        (READ_EXPORT,  "tile_sizes", SubdataMember(
112            ref_type=terrain.TileSize,
113            length=19,      # number of tile types
114        )),
115        (READ, "padding1", "int16_t"),
116    ])
117
118    # TODO: Enable conversion for SWGB; replace "terrains"
119    # ===========================================================================
120    # # 42 terrains are stored (100 in African Kingdoms), but less are used.
121    # # TODO: maybe this number is defined somewhere.
122    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
123    #     data_format.append((READ_EXPORT,  "terrains", SubdataMember(
124    #             ref_type=terrain.Terrain,
125    #             length=55,
126    #             )))
127    # elif GameVersion.age2_hd_ak in game_versions:
128    #     data_format.append((READ_EXPORT,  "terrains", SubdataMember(
129    #             ref_type=terrain.Terrain,
130    #             length=100,
131    #             )))
132    # elif (GameVersion.aoe_1 or GameVersion.aoe_ror) in game_versions:
133    #     data_format.append((READ_EXPORT,  "terrains", SubdataMember(
134    #             ref_type=terrain.Terrain,
135    #             length=42,
136    #             )))
137    # else:
138    #     data_format.append((READ_EXPORT,  "terrains", SubdataMember(
139    #             ref_type=terrain.Terrain,
140    #             length=42,
141    #             )))
142    # ===========================================================================
143    data_format.append(
144        (READ_EXPORT,  "terrains", SubdataMember(
145            ref_type=terrain.Terrain,
146            # 42 terrains are stored (100 in African Kingdoms), but less are used.
147            # TODO: maybe this number is defined somewhere.
148            length=(lambda self:
149                    100 if GameVersion.age2_hd_ak in self.game_versions
150                    else 42),
151        )))
152
153    data_format.extend([
154        (READ,         "terrain_border", SubdataMember(
155            ref_type=terrain.TerrainBorder,
156            length=16,
157        )),
158
159        (READ,         "map_row_offset", "int32_t"),
160        (READ,         "map_min_x", "float"),
161        (READ,         "map_min_y", "float"),
162        (READ,         "map_max_x", "float"),
163        (READ,         "map_max_y", "float"),
164        (READ,         "map_max_xplus1", "float"),
165        (READ,         "map_min_yplus1", "float"),
166
167        (READ,         "terrain_count_additional", "uint16_t"),
168        (READ,         "borders_used", "uint16_t"),
169        (READ,         "max_terrain", "int16_t"),
170        (READ_EXPORT,  "tile_width", "int16_t"),
171        (READ_EXPORT,  "tile_height", "int16_t"),
172        (READ_EXPORT,  "tile_half_height", "int16_t"),
173        (READ_EXPORT,  "tile_half_width", "int16_t"),
174        (READ_EXPORT,  "elev_height", "int16_t"),
175        (READ,         "current_row", "int16_t"),
176        (READ,         "current_column", "int16_t"),
177        (READ,         "block_beginn_row", "int16_t"),
178        (READ,         "block_end_row", "int16_t"),
179        (READ,         "block_begin_column", "int16_t"),
180        (READ,         "block_end_column", "int16_t"),
181        (READ,         "search_map_ptr", "int32_t"),
182        (READ,         "search_map_rows_ptr", "int32_t"),
183        (READ,         "any_frame_change", "int8_t"),
184        (READ,         "map_visible_flag", "int8_t"),
185        (READ,         "fog_flag", "int8_t"),
186    ])
187
188    # TODO: Enable conversion for SWGB; replace "terrain_blob0"
189    # ===========================================================================
190    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
191    #     data_format.append((READ_UNKNOWN, "terrain_blob0", "uint8_t[25]"))
192    # else:
193    #     data_format.append((READ_UNKNOWN, "terrain_blob0", "uint8_t[21]"))
194    # ===========================================================================
195    data_format.append((READ_UNKNOWN, "terrain_blob0", "uint8_t[21]"))
196
197    data_format.extend([
198        (READ_UNKNOWN, "terrain_blob1", "uint32_t[157]"),
199
200        # random map config
201        (READ, "random_map_count", "uint32_t"),
202        (READ, "random_map_ptr", "uint32_t"),
203        (READ, "map_infos", SubdataMember(
204            ref_type=maps.MapInfo,
205            length="random_map_count",
206        )),
207        (READ, "maps", SubdataMember(
208            ref_type=maps.Map,
209            length="random_map_count",
210        )),
211
212        # technology data
213        (READ_EXPORT, "tech_count", "uint32_t"),
214        (READ_EXPORT, "techs", SubdataMember(
215            ref_type=tech.Tech,
216            length="tech_count",
217        )),
218    ])
219
220    # TODO: Enable conversion for SWGB
221    # ===========================================================================
222    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
223    #     data_format.extend([
224    #         (READ, "unit_line_count", "uint16_t"),
225    #         (READ, "unit_lines", SubdataMember(
226    #             ref_type=unit.UnitLine,
227    #             length="unit_line_count",
228    #         )),
229    #     ])
230    # ===========================================================================
231
232    # unit header data
233    # TODO: Enable conversion for AOE1; replace "unit_count", "unit_headers"
234    # ===========================================================================
235    # if (GameVersion.aoe_1 or GameVersion.aoe_ror) not in game_versions:
236    #     data_format.extend([(READ_EXPORT, "unit_count", "uint32_t"),
237    #                         (READ_EXPORT, "unit_headers", SubdataMember(
238    #                             ref_type=unit.UnitHeader,
239    #                             length="unit_count",
240    #                         )),
241    #     ])
242    # ===========================================================================
243    data_format.extend([
244        (READ_EXPORT, "unit_count", "uint32_t"),
245        (READ_EXPORT, "unit_headers", SubdataMember(
246            ref_type=unit.UnitHeader,
247            length="unit_count",
248        )),
249    ])
250
251    # civilisation data
252    data_format.extend([
253        (READ_EXPORT, "civ_count", "uint16_t"),
254        (READ_EXPORT, "civs", SubdataMember(
255            ref_type=civ.Civ,
256            length="civ_count"
257        )),
258    ])
259
260    # TODO: Enable conversion for SWGB
261    # ===========================================================================
262    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
263    #     data_format.append((READ_UNKNOWN, None, "int8_t"))
264    # ===========================================================================
265
266    # research data
267    data_format.extend([
268        (READ_EXPORT, "research_count", "uint16_t"),
269        (READ_EXPORT, "researches", SubdataMember(
270            ref_type=research.Research,
271            length="research_count"
272        )),
273    ])
274
275    # TODO: Enable conversion for SWGB
276    # ===========================================================================
277    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
278    #     data_format.append((READ_UNKNOWN, None, "int8_t"))
279    # ===========================================================================
280
281    # TODO: Enable conversion for AOE1; replace the 7 values below
282    # ===========================================================================
283    # if (GameVersion.aoe_1 or GameVersion.aoe_ror) not in game_versions:
284    #     data_format.extend([
285    #         (READ, "time_slice", "int32_t"),
286    #         (READ, "unit_kill_rate", "int32_t"),
287    #         (READ, "unit_kill_total", "int32_t"),
288    #         (READ, "unit_hitpoint_rate", "int32_t"),
289    #         (READ, "unit_hitpoint_total", "int32_t"),
290    #         (READ, "razing_kill_rate", "int32_t"),
291    #         (READ, "razing_kill_total", "int32_t"),
292    #     ])
293    # ===========================================================================
294    data_format.extend([
295        (READ, "time_slice", "int32_t"),
296        (READ, "unit_kill_rate", "int32_t"),
297        (READ, "unit_kill_total", "int32_t"),
298        (READ, "unit_hitpoint_rate", "int32_t"),
299        (READ, "unit_hitpoint_total", "int32_t"),
300        (READ, "razing_kill_rate", "int32_t"),
301        (READ, "razing_kill_total", "int32_t"),
302    ])
303    # ===========================================================================
304
305    # technology tree data
306    data_format.extend([
307        (READ_EXPORT, "age_entry_count", "uint8_t"),
308        (READ_EXPORT, "building_connection_count", "uint8_t"),
309    ])
310
311    # TODO: Enable conversion for SWGB; replace "unit_connection_count"
312    # ===========================================================================
313    # if (GameVersion.swgb_10 or GameVersion.swgb_cc) in game_versions:
314    #     data_format.append((READ_EXPORT, "unit_connection_count", "uint16_t"))
315    # else:
316    #     data_format.append((READ_EXPORT, "unit_connection_count", "uint8_t"))
317    # ===========================================================================
318    data_format.append((READ_EXPORT, "unit_connection_count", "uint8_t"))
319
320    data_format.extend([
321        (READ_EXPORT, "research_connection_count", "uint8_t"),
322        (READ_EXPORT, "age_tech_tree", SubdataMember(
323            ref_type=tech.AgeTechTree,
324            length="age_entry_count"
325        )),
326        # What is this? There shouldn't be something here
327        (READ_UNKNOWN, None, "int32_t"),
328        (READ_EXPORT, "building_connection", SubdataMember(
329            ref_type=tech.BuildingConnection,
330            length="building_connection_count"
331        )),
332        (READ_EXPORT, "unit_connection", SubdataMember(
333            ref_type=tech.UnitConnection,
334            length="unit_connection_count"
335        )),
336        (READ_EXPORT, "research_connection", SubdataMember(
337            ref_type=tech.ResearchConnection,
338            length="research_connection_count"
339        )),
340    ])
341
342    @classmethod
343    def get_hash(cls):
344        """ return the unique hash for the data format tree """
345
346        return cls.format_hash().hexdigest()
347
348
349class EmpiresDatWrapper(Exportable):
350    """
351    This wrapper exists because the top-level element is discarded:
352    The gathered data fields are passed to the parent,
353    and are accumulated there to be written out.
354
355    This class acts as the parent for the "real" data values,
356    and has no parent itself. Thereby this class is discarded
357    and the child classes use this as parent for their return values.
358    """
359
360    name_struct_file   = "gamedata"
361    name_struct        = "gamedata"
362    struct_description = "wrapper for empires2_x1_p1.dat structure"
363
364    # TODO: we could reference to other gamedata structures
365    data_format = [
366        (READ_EXPORT, "empiresdat", SubdataMember(
367            ref_type=EmpiresDat,
368            length=1,
369        )),
370    ]
371
372
373def load_gamespec(fileobj, game_versions, cachefile_name=None, load_cache=False):
374    """
375    Helper method that loads the contents of a 'empires.dat' gzipped gamespec
376    file.
377
378    If cachefile_name is given, this file is consulted before performing the
379    load.
380    """
381    # try to use the cached result from a previous run
382    if cachefile_name and load_cache:
383        try:
384            with open(cachefile_name, "rb") as cachefile:
385                # pickle.load() can fail in many ways, we need to catch all.
386                # pylint: disable=broad-except
387                try:
388                    gamespec = pickle.load(cachefile)
389                    info("using cached gamespec: %s", cachefile_name)
390                    return gamespec
391                except Exception:
392                    warn("could not use cached gamespec:")
393                    import traceback
394                    traceback.print_exc()
395                    warn("we will just skip the cache, no worries.")
396
397        except FileNotFoundError:
398            pass
399
400    # read the file ourselves
401
402    dbg("reading dat file")
403    compressed_data = fileobj.read()
404    fileobj.close()
405
406    dbg("decompressing dat file")
407    # -15: there's no header, window size is 15.
408    file_data = decompress(compressed_data, -15)
409    del compressed_data
410
411    spam("length of decompressed data: %d", len(file_data))
412
413    gamespec = EmpiresDatWrapper(game_versions=game_versions)
414    gamespec.read(file_data, 0)
415
416    if cachefile_name:
417        dbg("dumping dat file contents to cache file: %s", cachefile_name)
418        with open(cachefile_name, "wb") as cachefile:
419            pickle.dump(gamespec, cachefile)
420
421    return gamespec
422