1 #include <iostream>
2 #include <vector>
3 #include <deque>
4 #include <list>
5 #include <unordered_set>
6 #include <map>
7 #include <set>
8 #include <fstream>
9 #include <cmath>
10
11 #include <boost/program_options.hpp>
12
13 #include <components/esm/esmreader.hpp>
14 #include <components/esm/esmwriter.hpp>
15 #include <components/esm/records.hpp>
16
17 #include "record.hpp"
18
19 #define ESMTOOL_VERSION 1.2
20
21 // Create a local alias for brevity
22 namespace bpo = boost::program_options;
23
24 struct ESMData
25 {
26 std::string author;
27 std::string description;
28 unsigned int version;
29 std::vector<ESM::Header::MasterData> masters;
30
31 std::deque<EsmTool::RecordBase *> mRecords;
32 // Value: (Reference, Deleted flag)
33 std::map<ESM::Cell *, std::deque<std::pair<ESM::CellRef, bool> > > mCellRefs;
34 std::map<int, int> mRecordStats;
35
36 static const std::set<int> sLabeledRec;
37 };
38
39 static const int sLabeledRecIds[] = {
40 ESM::REC_GLOB, ESM::REC_CLAS, ESM::REC_FACT, ESM::REC_RACE, ESM::REC_SOUN,
41 ESM::REC_REGN, ESM::REC_BSGN, ESM::REC_LTEX, ESM::REC_STAT, ESM::REC_DOOR,
42 ESM::REC_MISC, ESM::REC_WEAP, ESM::REC_CONT, ESM::REC_SPEL, ESM::REC_CREA,
43 ESM::REC_BODY, ESM::REC_LIGH, ESM::REC_ENCH, ESM::REC_NPC_, ESM::REC_ARMO,
44 ESM::REC_CLOT, ESM::REC_REPA, ESM::REC_ACTI, ESM::REC_APPA, ESM::REC_LOCK,
45 ESM::REC_PROB, ESM::REC_INGR, ESM::REC_BOOK, ESM::REC_ALCH, ESM::REC_LEVI,
46 ESM::REC_LEVC, ESM::REC_SNDG, ESM::REC_CELL, ESM::REC_DIAL
47 };
48
49 const std::set<int> ESMData::sLabeledRec =
50 std::set<int>(sLabeledRecIds, sLabeledRecIds + 34);
51
52 // Based on the legacy struct
53 struct Arguments
54 {
55 bool raw_given;
56 bool quiet_given;
57 bool loadcells_given;
58 bool plain_given;
59
60 std::string mode;
61 std::string encoding;
62 std::string filename;
63 std::string outname;
64
65 std::vector<std::string> types;
66 std::string name;
67
68 ESMData data;
69 ESM::ESMReader reader;
70 ESM::ESMWriter writer;
71 };
72
parseOptions(int argc,char ** argv,Arguments & info)73 bool parseOptions (int argc, char** argv, Arguments &info)
74 {
75 bpo::options_description desc("Inspect and extract from Morrowind ES files (ESM, ESP, ESS)\nSyntax: esmtool [options] mode infile [outfile]\nAllowed modes:\n dump\t Dumps all readable data from the input file.\n clone\t Clones the input file to the output file.\n comp\t Compares the given files.\n\nAllowed options");
76
77 desc.add_options()
78 ("help,h", "print help message.")
79 ("version,v", "print version information and quit.")
80 ("raw,r", "Show an unformatted list of all records and subrecords.")
81 // The intention is that this option would interact better
82 // with other modes including clone, dump, and raw.
83 ("type,t", bpo::value< std::vector<std::string> >(),
84 "Show only records of this type (four character record code). May "
85 "be specified multiple times. Only affects dump mode.")
86 ("name,n", bpo::value<std::string>(),
87 "Show only the record with this name. Only affects dump mode.")
88 ("plain,p", "Print contents of dialogs, books and scripts. "
89 "(skipped by default)"
90 "Only affects dump mode.")
91 ("quiet,q", "Suppress all record information. Useful for speed tests.")
92 ("loadcells,C", "Browse through contents of all cells.")
93
94 ( "encoding,e", bpo::value<std::string>(&(info.encoding))->
95 default_value("win1252"),
96 "Character encoding used in ESMTool:\n"
97 "\n\twin1250 - Central and Eastern European such as Polish, Czech, Slovak, Hungarian, Slovene, Bosnian, Croatian, Serbian (Latin script), Romanian and Albanian languages\n"
98 "\n\twin1251 - Cyrillic alphabet such as Russian, Bulgarian, Serbian Cyrillic and other languages\n"
99 "\n\twin1252 - Western European (Latin) alphabet, used by default")
100 ;
101
102 std::string finalText = "\nIf no option is given, the default action is to parse all records in the archive\nand display diagnostic information.";
103
104 // input-file is hidden and used as a positional argument
105 bpo::options_description hidden("Hidden Options");
106
107 hidden.add_options()
108 ( "mode,m", bpo::value<std::string>(), "esmtool mode")
109 ( "input-file,i", bpo::value< std::vector<std::string> >(), "input file")
110 ;
111
112 bpo::positional_options_description p;
113 p.add("mode", 1).add("input-file", 2);
114
115 // there might be a better way to do this
116 bpo::options_description all;
117 all.add(desc).add(hidden);
118 bpo::variables_map variables;
119
120 try
121 {
122 bpo::parsed_options valid_opts = bpo::command_line_parser(argc, argv)
123 .options(all).positional(p).run();
124
125 bpo::store(valid_opts, variables);
126 }
127 catch(std::exception &e)
128 {
129 std::cout << "ERROR parsing arguments: " << e.what() << std::endl;
130 return false;
131 }
132
133 bpo::notify(variables);
134
135 if (variables.count ("help"))
136 {
137 std::cout << desc << finalText << std::endl;
138 return false;
139 }
140 if (variables.count ("version"))
141 {
142 std::cout << "ESMTool version " << ESMTOOL_VERSION << std::endl;
143 return false;
144 }
145 if (!variables.count("mode"))
146 {
147 std::cout << "No mode specified!" << std::endl << std::endl
148 << desc << finalText << std::endl;
149 return false;
150 }
151
152 if (variables.count("type") > 0)
153 info.types = variables["type"].as< std::vector<std::string> >();
154 if (variables.count("name") > 0)
155 info.name = variables["name"].as<std::string>();
156
157 info.mode = variables["mode"].as<std::string>();
158 if (!(info.mode == "dump" || info.mode == "clone" || info.mode == "comp"))
159 {
160 std::cout << std::endl << "ERROR: invalid mode \"" << info.mode << "\"" << std::endl << std::endl
161 << desc << finalText << std::endl;
162 return false;
163 }
164
165 if ( !variables.count("input-file") )
166 {
167 std::cout << "\nERROR: missing ES file\n\n";
168 std::cout << desc << finalText << std::endl;
169 return false;
170 }
171
172 // handling gracefully the user adding multiple files
173 /* if (variables["input-file"].as< std::vector<std::string> >().size() > 1)
174 {
175 std::cout << "\nERROR: more than one ES file specified\n\n";
176 std::cout << desc << finalText << std::endl;
177 return false;
178 }*/
179
180 info.filename = variables["input-file"].as< std::vector<std::string> >()[0];
181 if (variables["input-file"].as< std::vector<std::string> >().size() > 1)
182 info.outname = variables["input-file"].as< std::vector<std::string> >()[1];
183
184 info.raw_given = variables.count ("raw") != 0;
185 info.quiet_given = variables.count ("quiet") != 0;
186 info.loadcells_given = variables.count ("loadcells") != 0;
187 info.plain_given = variables.count("plain") != 0;
188
189 // Font encoding settings
190 info.encoding = variables["encoding"].as<std::string>();
191 if(info.encoding != "win1250" && info.encoding != "win1251" && info.encoding != "win1252")
192 {
193 std::cout << info.encoding << " is not a valid encoding option." << std::endl;
194 info.encoding = "win1252";
195 }
196 std::cout << ToUTF8::encodingUsingMessage(info.encoding) << std::endl;
197
198 return true;
199 }
200
201 void printRaw(ESM::ESMReader &esm);
202 void loadCell(ESM::Cell &cell, ESM::ESMReader &esm, Arguments& info);
203
204 int load(Arguments& info);
205 int clone(Arguments& info);
206 int comp(Arguments& info);
207
main(int argc,char ** argv)208 int main(int argc, char**argv)
209 {
210 try
211 {
212 Arguments info;
213 if(!parseOptions (argc, argv, info))
214 return 1;
215
216 if (info.mode == "dump")
217 return load(info);
218 else if (info.mode == "clone")
219 return clone(info);
220 else if (info.mode == "comp")
221 return comp(info);
222 else
223 {
224 std::cout << "Invalid or no mode specified, dying horribly. Have a nice day." << std::endl;
225 return 1;
226 }
227 }
228 catch (std::exception& e)
229 {
230 std::cerr << "ERROR: " << e.what() << std::endl;
231 return 1;
232 }
233
234 return 0;
235 }
236
loadCell(ESM::Cell & cell,ESM::ESMReader & esm,Arguments & info)237 void loadCell(ESM::Cell &cell, ESM::ESMReader &esm, Arguments& info)
238 {
239 bool quiet = (info.quiet_given || info.mode == "clone");
240 bool save = (info.mode == "clone");
241
242 // Skip back to the beginning of the reference list
243 // FIXME: Changes to the references backend required to support multiple plugins have
244 // almost certainly broken this following line. I'll leave it as is for now, so that
245 // the compiler does not complain.
246 cell.restore(esm, 0);
247
248 // Loop through all the references
249 ESM::CellRef ref;
250 if(!quiet) std::cout << " References:\n";
251
252 bool deleted = false;
253 while(cell.getNextRef(esm, ref, deleted))
254 {
255 if (save) {
256 info.data.mCellRefs[&cell].push_back(std::make_pair(ref, deleted));
257 }
258
259 if(quiet) continue;
260
261 std::cout << " Refnum: " << ref.mRefNum.mIndex << std::endl;
262 std::cout << " ID: " << ref.mRefID << std::endl;
263 std::cout << " Position: (" << ref.mPos.pos[0] << ", " << ref.mPos.pos[1] << ", " << ref.mPos.pos[2] << ")" << std::endl;
264 if (ref.mScale != 1.f)
265 std::cout << " Scale: " << ref.mScale << std::endl;
266 if (!ref.mOwner.empty())
267 std::cout << " Owner: " << ref.mOwner << std::endl;
268 if (!ref.mGlobalVariable.empty())
269 std::cout << " Global: " << ref.mGlobalVariable << std::endl;
270 if (!ref.mFaction.empty())
271 std::cout << " Faction: " << ref.mFaction << std::endl;
272 if (!ref.mFaction.empty() || ref.mFactionRank != -2)
273 std::cout << " Faction rank: " << ref.mFactionRank << std::endl;
274 std::cout << " Enchantment charge: " << ref.mEnchantmentCharge << std::endl;
275 std::cout << " Uses/health: " << ref.mChargeInt << std::endl;
276 std::cout << " Gold value: " << ref.mGoldValue << std::endl;
277 std::cout << " Blocked: " << static_cast<int>(ref.mReferenceBlocked) << std::endl;
278 std::cout << " Deleted: " << deleted << std::endl;
279 if (!ref.mKey.empty())
280 std::cout << " Key: " << ref.mKey << std::endl;
281 std::cout << " Lock level: " << ref.mLockLevel << std::endl;
282 if (!ref.mTrap.empty())
283 std::cout << " Trap: " << ref.mTrap << std::endl;
284 if (!ref.mSoul.empty())
285 std::cout << " Soul: " << ref.mSoul << std::endl;
286 if (ref.mTeleport)
287 {
288 std::cout << " Destination position: (" << ref.mDoorDest.pos[0] << ", "
289 << ref.mDoorDest.pos[1] << ", " << ref.mDoorDest.pos[2] << ")" << std::endl;
290 if (!ref.mDestCell.empty())
291 std::cout << " Destination cell: " << ref.mDestCell << std::endl;
292 }
293 }
294 }
295
printRaw(ESM::ESMReader & esm)296 void printRaw(ESM::ESMReader &esm)
297 {
298 while(esm.hasMoreRecs())
299 {
300 ESM::NAME n = esm.getRecName();
301 std::cout << "Record: " << n.toString() << std::endl;
302 esm.getRecHeader();
303 while(esm.hasMoreSubs())
304 {
305 size_t offs = esm.getFileOffset();
306 esm.getSubName();
307 esm.skipHSub();
308 n = esm.retSubName();
309 std::ios::fmtflags f(std::cout.flags());
310 std::cout << " " << n.toString() << " - " << esm.getSubSize()
311 << " bytes @ 0x" << std::hex << offs << "\n";
312 std::cout.flags(f);
313 }
314 }
315 }
316
load(Arguments & info)317 int load(Arguments& info)
318 {
319 ESM::ESMReader& esm = info.reader;
320 ToUTF8::Utf8Encoder encoder (ToUTF8::calculateEncoding(info.encoding));
321 esm.setEncoder(&encoder);
322
323 std::string filename = info.filename;
324 std::cout << "Loading file: " << filename << std::endl;
325
326 std::unordered_set<uint32_t> skipped;
327
328 try {
329
330 if(info.raw_given && info.mode == "dump")
331 {
332 std::cout << "RAW file listing:\n";
333
334 esm.openRaw(filename);
335
336 printRaw(esm);
337
338 return 0;
339 }
340
341 bool quiet = (info.quiet_given || info.mode == "clone");
342 bool loadCells = (info.loadcells_given || info.mode == "clone");
343 bool save = (info.mode == "clone");
344
345 esm.open(filename);
346
347 info.data.author = esm.getAuthor();
348 info.data.description = esm.getDesc();
349 info.data.masters = esm.getGameFiles();
350
351 if (!quiet)
352 {
353 std::cout << "Author: " << esm.getAuthor() << std::endl
354 << "Description: " << esm.getDesc() << std::endl
355 << "File format version: " << esm.getFVer() << std::endl;
356 std::vector<ESM::Header::MasterData> masterData = esm.getGameFiles();
357 if (!masterData.empty())
358 {
359 std::cout << "Masters:" << std::endl;
360 for(const auto& master : masterData)
361 std::cout << " " << master.name << ", " << master.size << " bytes" << std::endl;
362 }
363 }
364
365 // Loop through all records
366 while(esm.hasMoreRecs())
367 {
368 const ESM::NAME n = esm.getRecName();
369 uint32_t flags;
370 esm.getRecHeader(flags);
371
372 EsmTool::RecordBase *record = EsmTool::RecordBase::create(n);
373 if (record == nullptr)
374 {
375 if (skipped.count(n.intval) == 0)
376 {
377 std::cout << "Skipping " << n.toString() << " records." << std::endl;
378 skipped.emplace(n.intval);
379 }
380
381 esm.skipRecord();
382 if (quiet) break;
383 std::cout << " Skipping\n";
384
385 continue;
386 }
387
388 record->setFlags(static_cast<int>(flags));
389 record->setPrintPlain(info.plain_given);
390 record->load(esm);
391
392 // Is the user interested in this record type?
393 bool interested = true;
394 if (!info.types.empty())
395 {
396 std::vector<std::string>::iterator match;
397 match = std::find(info.types.begin(), info.types.end(), n.toString());
398 if (match == info.types.end()) interested = false;
399 }
400
401 if (!info.name.empty() && !Misc::StringUtils::ciEqual(info.name, record->getId()))
402 interested = false;
403
404 if(!quiet && interested)
405 {
406 std::cout << "\nRecord: " << n.toString() << " '" << record->getId() << "'\n";
407 record->print();
408 }
409
410 if (record->getType().intval == ESM::REC_CELL && loadCells && interested)
411 {
412 loadCell(record->cast<ESM::Cell>()->get(), esm, info);
413 }
414
415 if (save)
416 {
417 info.data.mRecords.push_back(record);
418 }
419 else
420 {
421 delete record;
422 }
423 ++info.data.mRecordStats[n.intval];
424 }
425
426 } catch(std::exception &e) {
427 std::cout << "\nERROR:\n\n " << e.what() << std::endl;
428
429 for (const EsmTool::RecordBase* record : info.data.mRecords)
430 delete record;
431
432 info.data.mRecords.clear();
433 return 1;
434 }
435
436 return 0;
437 }
438
439 #include <iomanip>
440
clone(Arguments & info)441 int clone(Arguments& info)
442 {
443 if (info.outname.empty())
444 {
445 std::cout << "You need to specify an output name" << std::endl;
446 return 1;
447 }
448
449 if (load(info) != 0)
450 {
451 std::cout << "Failed to load, aborting." << std::endl;
452 return 1;
453 }
454
455 size_t recordCount = info.data.mRecords.size();
456
457 int digitCount = 1; // For a nicer output
458 if (recordCount > 0)
459 digitCount = (int)std::log10(recordCount) + 1;
460
461 std::cout << "Loaded " << recordCount << " records:" << std::endl << std::endl;
462
463 int i = 0;
464 for (std::pair<int, int> stat : info.data.mRecordStats)
465 {
466 ESM::NAME name;
467 name.intval = stat.first;
468 int amount = stat.second;
469 std::cout << std::setw(digitCount) << amount << " " << name.toString() << " ";
470 if (++i % 3 == 0)
471 std::cout << std::endl;
472 }
473
474 if (i % 3 != 0)
475 std::cout << std::endl;
476
477 std::cout << std::endl << "Saving records to: " << info.outname << "..." << std::endl;
478
479 ESM::ESMWriter& esm = info.writer;
480 ToUTF8::Utf8Encoder encoder (ToUTF8::calculateEncoding(info.encoding));
481 esm.setEncoder(&encoder);
482 esm.setAuthor(info.data.author);
483 esm.setDescription(info.data.description);
484 esm.setVersion(info.data.version);
485 esm.setRecordCount (recordCount);
486
487 for (const ESM::Header::MasterData &master : info.data.masters)
488 esm.addMaster(master.name, master.size);
489
490 std::fstream save(info.outname.c_str(), std::fstream::out | std::fstream::binary);
491 esm.save(save);
492
493 int saved = 0;
494 for (EsmTool::RecordBase* record : info.data.mRecords)
495 {
496 if (i <= 0)
497 break;
498
499 const ESM::NAME& typeName = record->getType();
500
501 esm.startRecord(typeName.toString(), record->getFlags());
502
503 record->save(esm);
504 if (typeName.intval == ESM::REC_CELL) {
505 ESM::Cell *ptr = &record->cast<ESM::Cell>()->get();
506 if (!info.data.mCellRefs[ptr].empty())
507 {
508 for (std::pair<ESM::CellRef, bool> &ref : info.data.mCellRefs[ptr])
509 ref.first.save(esm, ref.second);
510 }
511 }
512
513 esm.endRecord(typeName.toString());
514
515 saved++;
516 int perc = recordCount == 0 ? 100 : (int)((saved / (float)recordCount)*100);
517 if (perc % 10 == 0)
518 {
519 std::cerr << "\r" << perc << "%";
520 }
521 }
522
523 std::cout << "\rDone!" << std::endl;
524
525 esm.close();
526 save.close();
527
528 return 0;
529 }
530
comp(Arguments & info)531 int comp(Arguments& info)
532 {
533 if (info.filename.empty() || info.outname.empty())
534 {
535 std::cout << "You need to specify two input files" << std::endl;
536 return 1;
537 }
538
539 Arguments fileOne;
540 Arguments fileTwo;
541
542 fileOne.raw_given = false;
543 fileTwo.raw_given = false;
544
545 fileOne.mode = "clone";
546 fileTwo.mode = "clone";
547
548 fileOne.encoding = info.encoding;
549 fileTwo.encoding = info.encoding;
550
551 fileOne.filename = info.filename;
552 fileTwo.filename = info.outname;
553
554 if (load(fileOne) != 0)
555 {
556 std::cout << "Failed to load " << info.filename << ", aborting comparison." << std::endl;
557 return 1;
558 }
559
560 if (load(fileTwo) != 0)
561 {
562 std::cout << "Failed to load " << info.outname << ", aborting comparison." << std::endl;
563 return 1;
564 }
565
566 if (fileOne.data.mRecords.size() != fileTwo.data.mRecords.size())
567 {
568 std::cout << "Not equal, different amount of records." << std::endl;
569 return 1;
570 }
571
572 return 0;
573 }
574