1 /**
2 * @file mega/utils.h
3 * @brief Mega SDK various utilities and helper classes
4 *
5 * (c) 2013-2014 by Mega Limited, Auckland, New Zealand
6 *
7 * This file is part of the MEGA SDK - Client Access Engine.
8 *
9 * Applications using the MEGA API must present a valid application key
10 * and comply with the the rules set forth in the Terms of Service.
11 *
12 * The MEGA SDK is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
15 *
16 * @copyright Simplified (2-clause) BSD License.
17 *
18 * You should have received a copy of the license along with this
19 * program.
20 */
21
22 #ifndef MEGA_UTILS_H
23 #define MEGA_UTILS_H 1
24
25 #include <type_traits>
26 #include <condition_variable>
27 #include <thread>
28 #include <mutex>
29
30 #include "types.h"
31 #include "mega/logging.h"
32
33 // Needed for Windows Phone (MSVS 2013 - C++ version 9.8)
34 #if defined(_WIN32) && _MSC_VER <= 1800 && __cplusplus < 201103L && !defined(_TIMESPEC_DEFINED) && ! __struct_timespec_defined
35 struct timespec
36 {
37 long long tv_sec; /* seconds */
38 long tv_nsec; /* nanoseconds */
39 };
40 # define __struct_timespec_defined 1
41 #endif
42
43 namespace mega {
44 // convert 1...8 character ID to int64 integer (endian agnostic)
45 #define MAKENAMEID1(a) (nameid)(a)
46 #define MAKENAMEID2(a, b) (nameid)(((a) << 8) + (b))
47 #define MAKENAMEID3(a, b, c) (nameid)(((a) << 16) + ((b) << 8) + (c))
48 #define MAKENAMEID4(a, b, c, d) (nameid)(((a) << 24) + ((b) << 16) + ((c) << 8) + (d))
49 #define MAKENAMEID5(a, b, c, d, e) (nameid)((((uint64_t)a) << 32) + ((b) << 24) + ((c) << 16) + ((d) << 8) + (e))
50 #define MAKENAMEID6(a, b, c, d, e, f) (nameid)((((uint64_t)a) << 40) + (((uint64_t)b) << 32) + ((c) << 24) + ((d) << 16) + ((e) << 8) + (f))
51 #define MAKENAMEID7(a, b, c, d, e, f, g) (nameid)((((uint64_t)a) << 48) + (((uint64_t)b) << 40) + (((uint64_t)c) << 32) + ((d) << 24) + ((e) << 16) + ((f) << 8) + (g))
52 #define MAKENAMEID8(a, b, c, d, e, f, g, h) (nameid)((((uint64_t)a) << 56) + (((uint64_t)b) << 48) + (((uint64_t)c) << 40) + (((uint64_t)d) << 32) + ((e) << 24) + ((f) << 16) + ((g) << 8) + (h))
53
54 std::string toNodeHandle(handle nodeHandle);
55 std::string toHandle(handle h);
56 #define LOG_NODEHANDLE(x) toNodeHandle(x)
57 #define LOG_HANDLE(x) toHandle(x)
58
59 struct MEGA_API ChunkedHash
60 {
61 static const int SEGSIZE = 131072;
62
63 static m_off_t chunkfloor(m_off_t);
64 static m_off_t chunkceil(m_off_t, m_off_t limit = -1);
65 };
66
67 /**
68 * @brief Padded encryption using AES-128 in CBC mode.
69 */
70 struct MEGA_API PaddedCBC
71 {
72 /**
73 * @brief Encrypts a string after padding it to block length.
74 *
75 * Note: With an IV, only use the first 8 bytes.
76 *
77 * @param data Data buffer to be encrypted. Encryption is done in-place,
78 * so cipher text will be in `data` afterwards as well.
79 * @param key AES key for encryption.
80 * @param iv Optional initialisation vector for encryption. Will use a
81 * zero IV if not given. If `iv` is a zero length string, a new IV
82 * for encryption will be generated and available through the reference.
83 * @return Void.
84 */
85 static void encrypt(PrnGen &rng, string* data, SymmCipher* key, string* iv = NULL);
86
87 /**
88 * @brief Decrypts a string and strips the padding.
89 *
90 * Note: With an IV, only use the first 8 bytes.
91 *
92 * @param data Data buffer to be decrypted. Decryption is done in-place,
93 * so plain text will be in `data` afterwards as well.
94 * @param key AES key for decryption.
95 * @param iv Optional initialisation vector for encryption. Will use a
96 * zero IV if not given.
97 * @return Void.
98 */
99 static bool decrypt(string* data, SymmCipher* key, string* iv = NULL);
100 };
101
102 class MEGA_API HashSignature
103 {
104 Hash* hash;
105
106 public:
107 // add data
108 void add(const byte*, unsigned);
109
110 // generate signature
111 unsigned get(AsymmCipher*, byte*, unsigned);
112
113 // verify signature
114 bool checksignature(AsymmCipher*, const byte*, unsigned);
115
116 HashSignature(Hash*);
117 ~HashSignature();
118 };
119
120 /**
121 * @brief Crypto functions related to payments
122 */
123 class MEGA_API PayCrypter
124 {
125 /**
126 * @brief Length of the AES key
127 */
128 static const int ENC_KEY_BYTES = 16;
129
130 /**
131 * @brief Lenght of the key to generate the HMAC
132 */
133 static const int MAC_KEY_BYTES = 32;
134
135 /**
136 * @brief Length of the IV for AES-CBC
137 */
138 static const int IV_BYTES = 16;
139
140 /**
141 * @brief Buffer for the AES key and the HMAC key
142 */
143 byte keys[ENC_KEY_BYTES+MAC_KEY_BYTES];
144
145 /**
146 * @brief Pointer to the buffer with the AES key
147 */
148 byte *encKey;
149
150 /**
151 * @brief Pointer to the buffer with the HMAC key
152 */
153 byte *hmacKey;
154
155 /**
156 * @brief Buffer with the IV for AES-CBC
157 */
158 byte iv[IV_BYTES];
159
160 /**
161 * @brief Random blocks generator
162 */
163 PrnGen &rng;
164
165 public:
166
167 /**
168 * @brief Constructor. Initializes keys with random values.
169 */
170 PayCrypter(PrnGen &rng);
171
172 /**
173 * @brief Updates the crypto keys (mainly for testing)
174 * @param newEncKey New AES key (must contain ENC_KEY_BYTES bytes)
175 * @param newHmacKey New HMAC key (must contain MAC_KEY_BYTES bytes)
176 * @param newIv New IV for AES-CBC (must contain IV_BYTES bytes)
177 */
178 void setKeys(const byte *newEncKey, const byte *newHmacKey, const byte *newIv);
179
180 /**
181 * @brief Encrypts the cleartext and returns the payload string.
182 *
183 * The clear text is encrypted with AES-CBC, then a HMAC-SHA256 is generated for (IV + ciphertext)
184 * and finally returns (HMAC + IV + ciphertext)
185 *
186 * @param cleartext Clear text to generate the payload
187 * @param result The function will fill this string with the generated payload
188 * @return True if the funcion succeeds, otherwise false
189 */
190 bool encryptPayload(const string *cleartext, string *result);
191
192 /**
193 * @brief Encrypts the cleartext using RSA with random padding.
194 *
195 * A 2-byte header is inserted just before the clear text with the size in bytes.
196 * The result is padded with random bytes. Then RSA is applied and the result is returned
197 * in the third parameter, with a 2-byte header that contains the size of the result of RSA.
198 *
199 * @param cleartext Clear text to encrypt with RSA
200 * @param pubkdata Public key in binary format (result of AsymmCipher::serializekey)
201 * @param pubkdatalen Size (in bytes) of pubkdata
202 * @param result RSA encrypted text, with a 2-byte header with the size of the RSA buffer in bytes
203 * @param randompadding Enables padding with random bytes. Otherwise, the cleartext is 0-padded
204 * @return True if the funcion succeeds, otherwise false
205 */
206 bool rsaEncryptKeys(const string *cleartext, const byte *pubkdata, int pubkdatalen, string *result, bool randompadding = true);
207
208 /**
209 * @brief Encrypts clear text data to an authenticated ciphertext, authenticated with an HMAC.
210 * @param cleartext Clear text as byte string
211 * @param pubkdata Public key in binary format (result of AsymmCipher::serializekey)
212 * @param pubkdatalen Size (in bytes) of pubkdata
213 * @param result Encrypted data block as byte string.
214 * @param randompadding Enables padding with random bytes. Otherwise, the cleartext is 0-padded
215 * @return True if the funcion succeeds, otherwise false
216 */
217 bool hybridEncrypt(const string *cleartext, const byte *pubkdata, int pubkdatalen, string *result, bool randompadding = true);
218 };
219
220 // read/write multibyte words
221 struct MEGA_API MemAccess
222 {
223 #ifndef ALLOW_UNALIGNED_MEMORY_ACCESS
getMemAccess224 template<typename T> static T get(const char* ptr)
225 {
226 T val;
227 memcpy(&val,ptr,sizeof(T));
228 return val;
229 }
230
setMemAccess231 template<typename T> static void set(byte* ptr, T val)
232 {
233 memcpy(ptr,&val,sizeof val);
234 }
235 #else
236 template<typename T> static T get(const char* ptr)
237 {
238 return *(T*)ptr;
239 }
240
241 template<typename T> static void set(byte* ptr, T val)
242 {
243 *(T*)ptr = val;
244 }
245 #endif
246 };
247
248 #ifdef _WIN32
249 int mega_snprintf(char *s, size_t n, const char *format, ...);
250 #endif
251
252 struct MEGA_API TLVstore
253 {
254 private:
255 TLV_map tlv;
256
257 public:
258
259 /**
260 * @brief containerToTLVrecords Builds a TLV object with records from an encrypted container
261 * @param data Binary byte array representing the encrypted container
262 * @param key Master key to decrypt the container
263 * @return A new TLVstore object. You take the ownership of the object.
264 */
265 static TLVstore * containerToTLVrecords(const string *data, SymmCipher *key);
266
267 /**
268 * @brief Builds a TLV object with records from a container
269 * @param data Binary byte array representing the TLV records
270 * @return A new TLVstore object. You take the ownership of the object.
271 */
272 static TLVstore * containerToTLVrecords(const string *data);
273
274 /**
275 * @brief Converts the TLV records into an encrypted byte array
276 * @param key Master key to decrypt the container
277 * @param encSetting Block encryption mode to be used by AES
278 * @return A new string holding the encrypted byte array. You take the ownership of the string.
279 */
280 string *tlvRecordsToContainer(PrnGen &rng, SymmCipher *key, encryptionsetting_t encSetting = AES_GCM_12_16);
281
282 /**
283 * @brief Converts the TLV records into a byte array
284 * @return A new string holding the byte array. You take the ownership of the string.
285 */
286 string *tlvRecordsToContainer();
287
288 /**
289 * @brief get Get the value for a given key
290 * @param type Type of the value (without scope nor non-historic modifiers).
291 * @return String containing the array with the value, or NULL if error.
292 */
293 std::string get(string type) const;
294
295 /**
296 * @brief Get a reference to the TLV_map associated to this TLVstore
297 *
298 * The TLVstore object retains the ownership of the returned object. It will be
299 * valid until this TLVstore object is deleted.
300 *
301 * @return The TLV_map associated to this TLVstore
302 */
303 const TLV_map *getMap() const;
304
305 /**
306 * @brief Get a list of the keys contained in the TLV
307 *
308 * You take ownership of the returned value.
309 *
310 * @return A new vector with the keys included in the TLV
311 */
312 vector<string> *getKeys() const;
313
314 /**
315 * @brief find Checks whether a type of value is available in the TLV container.
316 * @param type Type of the value (without scope nor non-historic modifiers).
317 * @return True if the type of value is found, false otherwise.
318 */
319 bool find(string type) const;
320
321 /**
322 * @brief add Adds a new record to the container
323 * @param type Type for the new value (without scope nor non-historic modifiers).
324 * @param value New value to be set.
325 */
326 void set(string type, string value);
327
328 /**
329 * @brief Removes a record from the container
330 * @param type Type for the value to be removed (without scope nor non-historic modifiers).
331 */
332 void reset(string type);
333
334 size_t size();
335
336 static unsigned getTaglen(int mode);
337 static unsigned getIvlen(int mode);
338 static encryptionmode_t getMode(int mode);
339
340 ~TLVstore();
341 };
342
343 class Utils {
344 public:
345 /**
346 * @brief Converts a character string from UTF-8 to Unicode
347 * This method is a workaround for a legacy bug where Webclient used to encode
348 * each byte of the array in UTF-8, resulting in a wider string of variable length.
349 * @note The UTF-8 string should only contain characters encoded as 1 or 2 bytes.
350 * @param src Characters string encoded in UTF-8
351 * @param srclen Length of the string (in bytes)
352 * @param result String holding the byte array of Unicode characters
353 * @return True if success, false if the byte 'src' is not a valid UTF-8 string
354 */
355 static bool utf8toUnicode(const uint8_t *src, unsigned srclen, string *result);
356
357 /**
358 * @brief Determines size in bytes of a valid UTF-8 sequence.
359 * @param c first character of UTF-8 sequence
360 * @return the size of UTF-8 sequence if its valid, otherwise returns 0
361 */
362 static size_t utf8SequenceSize(unsigned char c);
363
364 /**
365 * @brief This function is analogous to a32_to_str in js version.
366 * Converts a vector of <T> elements into a std::string
367 *
368 * @param data a vector of <T> elements
369 * @note this function has been initially designed to work with <T> = uint32_t or <T> = int32_t
370 * This is a valid example: <t> = uint32_t, data = [1952805748] => return_value = "test"
371 *
372 * @return returns a std::string
373 */
a32_to_str(std::vector<T> data)374 template<typename T> static std::string a32_to_str(std::vector<T> data)
375 {
376 size_t size = data.size() * sizeof(T);
377 std::unique_ptr<char[]> result(new char[size]);
378 for (size_t i = 0; i < size; ++i)
379 {
380 result[i] = (data[i >> 2] >> (24 - (i & 3) * 8)) & 255;
381 }
382 return std::string (result.get(), size);
383 }
384
385 /**
386 * @brief This function is analogous to str_to_a32 in js version.
387 * Converts a std::string into a vector of <T> elements
388 *
389 * @param data a std::string
390 * @note this function has been initially designed to work with <T> = uint32_t or <T> = int32_t
391 * This is a valid example: <t> = uint32_t, data = "test" => return_value = [1952805748]
392 *
393 * @return returns a vector of <T> elements
394 */
str_to_a32(std::string data)395 template<typename T> static std::vector<T> str_to_a32(std::string data)
396 {
397 std::vector<T> data32((data.size() + 3) >> 2);
398 for (size_t i = 0; i < data.size(); ++i)
399 {
400 data32[i >> 2] |= (data[i] & 255) << (24 - (i & 3) * 8);
401 }
402 return data32;
403 }
404
405 static std::string stringToHex(const std::string& input);
406 static std::string hexToString(const std::string& input);
407 };
408
409 // for pre-c++11 where this version is not defined yet.
410 long long abs(long long n);
411
412 extern m_time_t m_time(m_time_t* tt = NULL);
413 extern struct tm* m_localtime(m_time_t, struct tm *dt);
414 extern struct tm* m_gmtime(m_time_t, struct tm *dt);
415 extern m_time_t m_mktime(struct tm*);
416 extern int m_clock_getmonotonictime(struct timespec *t);
417 // Similar behaviour to mktime but it receives a struct tm with a date in UTC and return mktime in UTC
418 extern m_time_t m_mktime_UTC(const struct tm *src);
419
420 std::string rfc1123_datetime( time_t time );
421 std::string webdavurlescape(const std::string &value);
422 std::string escapewebdavchar(const char c);
423 std::string webdavnameescape(const std::string &value);
424
425 void tolower_string(std::string& str);
426
427 #ifdef __APPLE__
428 int macOSmajorVersion();
429 #endif
430
431 // file chunk macs
432 class chunkmac_map : public map<m_off_t, ChunkMAC>
433 {
434 public:
435 int64_t macsmac(SymmCipher *cipher);
436 void serialize(string& d) const;
437 bool unserialize(const char*& ptr, const char* end);
438 void calcprogress(m_off_t size, m_off_t& chunkpos, m_off_t& completedprogress, m_off_t* lastblockprogress = nullptr);
439 m_off_t nextUnprocessedPosFrom(m_off_t pos);
440 m_off_t expandUnprocessedPiece(m_off_t pos, m_off_t npos, m_off_t fileSize, m_off_t maxReqSize);
441 void finishedUploadChunks(chunkmac_map& macs);
442 };
443
444 struct CacheableWriter
445 {
446 CacheableWriter(string& d);
447 string& dest;
448
449 void serializebinary(byte* data, size_t len);
450 void serializecstr(const char* field, bool storeNull); // may store the '\0' also for backward compatibility. Only use for utf8! (std::string storing double byte chars will only store 1 byte)
451 void serializepstr(const string* field); // uses string size() not strlen
452 void serializestring(const string& field);
453 void serializecompressed64(int64_t field);
454 void serializei64(int64_t field);
455 void serializeu32(uint32_t field);
456 void serializehandle(handle field);
457 void serializenodehandle(handle field);
458 void serializefsfp(fsfp_t field);
459 void serializebool(bool field);
460 void serializebyte(byte field);
461 void serializedouble(double field);
462 void serializechunkmacs(const chunkmac_map& m);
463
464 // Each class that might get extended should store expansion flags at the end
465 // When adding new fields to an existing class, set the next expansion flag true to indicate they are present.
466 // If you turn on the last flag, then you must also add another set of expansion flags (all false) after the new fields, for further expansion later.
467 void serializeexpansionflags(bool b1 = false, bool b2 = false, bool b3 = false, bool b4 = false, bool b5 = false, bool b6 = false, bool b7 = false, bool b8 = false);
468 };
469
470 struct CacheableReader
471 {
472 CacheableReader(const string& d);
473 const char* ptr;
474 const char* end;
475 unsigned fieldnum;
476
477 bool unserializebinary(byte* data, size_t len);
478 bool unserializecstr(string& s, bool removeNull); // set removeNull if this field stores the terminating '\0' at the end
479 bool unserializestring(string& s);
480 bool unserializecompressed64(uint64_t& field);
481 bool unserializei64(int64_t& s);
482 bool unserializeu32(uint32_t& s);
483 bool unserializebyte(byte& s);
484 bool unserializedouble(double& s);
485 bool unserializehandle(handle& s);
486 bool unserializenodehandle(handle& s);
487 bool unserializefsfp(fsfp_t& s);
488 bool unserializebool(bool& s);
489 bool unserializechunkmacs(chunkmac_map& m);
490
491 bool unserializeexpansionflags(unsigned char field[8], unsigned usedFlagCount);
492
493 void eraseused(string& d); // must be the same string, unchanged
hasdataleftCacheableReader494 bool hasdataleft() { return end > ptr; }
495 };
496
497 template<typename T, typename U>
hashCombine(T & seed,const U & v)498 void hashCombine(T& seed, const U& v)
499 {
500 static_assert(std::is_integral<T>::value, "T is not integral");
501 // the magic number is the twos complement version of the golden ratio
502 seed ^= std::hash<U>{}(v) + 0x9e3779b9 + (seed<<6) + (seed>>2);
503 }
504
505 struct FileAccess;
506 struct InputStreamAccess;
507 class SymmCipher;
508
509 std::pair<bool, int64_t> generateMetaMac(SymmCipher &cipher, FileAccess &ifAccess, const int64_t iv);
510
511 std::pair<bool, int64_t> generateMetaMac(SymmCipher &cipher, InputStreamAccess &isAccess, const int64_t iv);
512
513 // Helper class for MegaClient. Suitable for expansion/templatizing for other use caes.
514 // Maintains a small thread pool for executing independent operations such as encrypt/decrypt a block of data
515 // The number of threads can be 0 (eg. for helper MegaApi that deals with public folder links) in which case something queued is
516 // immediately executed synchronously on the caller's thread
517 struct MegaClientAsyncQueue
518 {
519 void push(std::function<void(SymmCipher&)> f, bool discardable);
520 void clearDiscardable();
521
522 MegaClientAsyncQueue(Waiter& w, unsigned threadCount);
523 ~MegaClientAsyncQueue();
524
525 private:
526 Waiter& mWaiter;
527 std::mutex mMutex;
528 std::condition_variable mConditionVariable;
529
530 struct Entry
531 {
532 bool discardable = false;
533 std::function<void(SymmCipher&)> f;
EntryMegaClientAsyncQueue::Entry534 Entry(bool disc, std::function<void(SymmCipher&)>&& func)
535 : discardable(disc), f(func)
536 {}
537 };
538
539 std::deque<Entry> mQueue;
540 std::vector<std::thread> mThreads;
541 SymmCipher mZeroThreadsCipher;
542
543 void asyncThreadLoop();
544 };
545
546 template<class T>
547 struct ThreadSafeDeque
548 {
549 // Just like a deque, but thread safe so that a separate thread can receive filesystem notifications as soon as they are available.
550 // When we try to do that on the same thread, the processing of queued notifications is too slow so more notifications bulid up than
551 // have been processed, so each time we get the outstanding ones from the buffer we gave to the OS, we need to give it an even
552 // larger buffer to write into, otherwise it runs out of space before this thread is idle and can get the next batch from the buffer.
553 protected:
554 std::deque<T> mNotifications;
555 std::mutex m;
556
557 public:
558
peekFrontThreadSafeDeque559 bool peekFront(T& t)
560 {
561 std::lock_guard<std::mutex> g(m);
562 if (!mNotifications.empty())
563 {
564 t = mNotifications.front();
565 return true;
566 }
567 return false;
568 }
569
popFrontThreadSafeDeque570 bool popFront(T& t)
571 {
572 std::lock_guard<std::mutex> g(m);
573 if (!mNotifications.empty())
574 {
575 t = std::move(mNotifications.front());
576 mNotifications.pop_front();
577 return true;
578 }
579 return false;
580 }
581
unpopFrontThreadSafeDeque582 void unpopFront(const T& t)
583 {
584 std::lock_guard<std::mutex> g(m);
585 mNotifications.push_front(t);
586 }
587
pushBackThreadSafeDeque588 void pushBack(T&& t)
589 {
590 std::lock_guard<std::mutex> g(m);
591 mNotifications.push_back(t);
592 }
593
emptyThreadSafeDeque594 bool empty()
595 {
596 std::lock_guard<std::mutex> g(m);
597 return mNotifications.empty();
598 }
599
sizeThreadSafeDeque600 bool size()
601 {
602 std::lock_guard<std::mutex> g(m);
603 return mNotifications.size();
604 }
605
606 };
607
608 } // namespace
609
610 #endif
611