1 #include "CppBase.h"
2 #include "Internal.h"
3
4 namespace Upp {
5
6 #define LTIMING(x) // DTIMING(x)
7 #define LLOG(x) // DLOG(x)
8
9 static std::atomic<int> s_PPserial;
10 static VectorMap<String, PPMacro> sAllMacros; // Only MakePP can write to this
11 static ArrayMap<String, PPFile> sPPfile; // Only MakePP can write to this
12
13 static VectorMap<String, Time> s_PathFileTime;
14 static StaticMutex s_PathFileTimeMutex;
15
16 static VectorMap<String, String> s_IncludePath;
17 static String s_Include_Path;
18 static StaticMutex s_IncludePathMutex;
19
20 static StaticMutex s_FlatPPMutex;
21 static ArrayMap<String, FlatPP> s_FlatPP; // ArrayMap to allow read access
22
NextPPSerial()23 int NextPPSerial()
24 {
25 return ++s_PPserial;
26 }
27
SweepPPFiles(const Index<String> & keep)28 void SweepPPFiles(const Index<String>& keep)
29 {
30 Index<int> pp_segment_id;
31 int unlinked_count = 0;
32 for(int i = 0; i < sPPfile.GetCount(); i++)
33 if(sPPfile.IsUnlinked(i))
34 unlinked_count++;
35 else
36 if(keep.Find(sPPfile.GetKey(i)) < 0) {
37 unlinked_count++;
38 sPPfile.Unlink(i);
39 }
40 else {
41 const PPFile& p = sPPfile[i];
42 for(int j = 0; j < p.item.GetCount(); j++)
43 pp_segment_id.FindAdd(p.item[j].segment_id);
44 }
45 if(unlinked_count > sPPfile.GetCount() / 2) {
46 CleanPP();
47 return;
48 }
49 unlinked_count = 0;
50 for(int i = 0; i < sAllMacros.GetCount(); i++) {
51 if(sAllMacros.IsUnlinked(i))
52 unlinked_count++;
53 else
54 if(sAllMacros[i].segment_id && pp_segment_id.Find(sAllMacros[i].segment_id) < 0) {
55 sAllMacros.Unlink(i);
56 unlinked_count++;
57 }
58 if(unlinked_count > sAllMacros.GetCount() / 2) {
59 CleanPP();
60 return;
61 }
62 }
63 }
64
GetSegmentFile(int segment_id)65 String GetSegmentFile(int segment_id)
66 {
67 for(int i = 0; i < sPPfile.GetCount(); i++) {
68 const Array<PPItem>& m = sPPfile[i].item;
69 for(int j = 0; j < m.GetCount(); j++)
70 if(m[j].type == PP_DEFINES && m[j].segment_id == segment_id)
71 return sPPfile.GetKey(i);
72 }
73 return "<not found>";
74 }
75
FindPPMacro(const String & id,Index<int> & segment_id,int & segmenti)76 PPMacro *FindPPMacro(const String& id, Index<int>& segment_id, int& segmenti)
77 {
78 Index<int> undef;
79 PPMacro *r;
80 int best;
81 for(int pass = 0; pass < 2; pass++) {
82 r = NULL;
83 best = segmenti;
84 int line = -1;
85 int q = sAllMacros.Find(id);
86 while(q >= 0) {
87 PPMacro& m = sAllMacros[q];
88 if(m.macro.IsUndef()) {
89 if(pass == 0 && segment_id.Find(m.segment_id) >= 0)
90 undef.FindAdd(m.segment_id); // cancel out undefined macro...
91 }
92 else
93 if(pass == 0 || m.segment_id == 0 || undef.Find(m.undef_segment_id) < 0) {
94 int si = m.segment_id == 0 ? INT_MAX : segment_id.Find(m.segment_id); // defs macros always override
95 if(si > best || si >= 0 && si == best && m.line > line) {
96 best = si;
97 line = m.line;
98 r = &m;
99 }
100 }
101 q = sAllMacros.FindNext(q);
102 }
103 if(undef.GetCount() == 0)
104 break;
105 }
106 segmenti = best;
107 return r;
108 }
109
FindMacro(const String & id,Index<int> & segment_id,int & segmenti)110 const CppMacro *FindMacro(const String& id, Index<int>& segment_id, int& segmenti)
111 {
112 PPMacro *m = FindPPMacro(id, segment_id, segmenti);
113 return m ? &m->macro : NULL;
114 }
115
CheckEndNamespace(Vector<int> & namespace_block,int level,Md5Stream & md5)116 void PPFile::CheckEndNamespace(Vector<int>& namespace_block, int level, Md5Stream& md5)
117 {
118 if(namespace_block.GetCount() && namespace_block.Top() == level) {
119 namespace_block.Drop();
120 item.Add().type = PP_NAMESPACE_END;
121 md5.Put('.');
122 }
123 }
124
Parse(Stream & in)125 void PPFile::Parse(Stream& in)
126 {
127 LTIMING("PPFile::Parse");
128 for(int i = 0; i < ppmacro.GetCount(); i++)
129 sAllMacros.Unlink(ppmacro[i]);
130 ppmacro.Clear();
131 item.Clear();
132 includes.Clear();
133 bool was_using = false;
134 bool was_namespace = false;
135 int level = 0;
136 bool incomment = false;
137 bool do_pp = true;
138 Vector<int> namespace_block;
139 bool next_segment = true;
140 Index<int> local_segments;
141 keywords.Clear();
142 int linei = 0;
143 Md5Stream md5;
144 int current_serial = 0;
145
146 VectorMap<String, PPMacro> local_macro; // gather all macros first to reduce locking
147
148 while(!in.IsEof()) {
149 String l = in.GetLine();
150 const char *ll = l;
151 while(*ll == ' ' || *ll == '\t')
152 ll++;
153 if(ll[0] == '/' && ll[1] == '/' && ll[2] == '$')
154 do_pp = decode(ll[3], '+', true, '-', false, do_pp);
155 while(*l.Last() == '\\' && !in.IsEof()) {
156 l.Trim(l.GetLength() - 1);
157 l.Cat(in.GetLine());
158 }
159 RemoveComments(l, incomment);
160 try {
161 CParser p(l);
162 if(p.Char('#')) {
163 if(do_pp) {
164 if(p.Id("define")) {
165 if(next_segment) {
166 PPItem& m = item.Add();
167 m.type = PP_DEFINES;
168 m.segment_id = current_serial = NextPPSerial();
169 next_segment = false;
170 local_segments.Add(current_serial);
171 }
172 CppMacro def;
173 String id = def.Define(p.GetPtr());
174 if(id.GetCount()) {
175 PPMacro& l = local_macro.Add(id);
176 l.segment_id = current_serial;
177 l.line = linei;
178 l.macro = def;
179 /*
180 PPMacro m;
181 m.segment_id = current_serial;
182 m.line = linei;
183 m.macro = def;
184 ppmacro.Add(sAllMacros.Put(id, m));
185 */
186 md5.Put("#", 1);
187 md5.Put(id);
188 md5.Put(0);
189 md5.Put(l.macro.md5, 16);
190 }
191 }
192 else
193 if(p.Id("undef")) {
194 if(p.IsId()) {
195 String id = p.ReadId();
196 if(id.GetCount()) {
197 md5.Put("#", 1);
198 md5.Put(id);
199 md5.Put(1);
200 int q = local_macro.FindLast(id); // heuristic: only local undefs are allowed
201 while(q >= 0) {
202 PPMacro& um = local_macro[q];
203 if(!um.macro.IsUndef()) { // found corresponding macro to undef
204 PPItem& m = item.Add();
205 m.type = PP_DEFINES;
206 m.segment_id = current_serial = NextPPSerial();
207 um.undef_segment_id = m.segment_id;
208 next_segment = true;
209 local_segments.Add(current_serial);
210 PPMacro& l = local_macro.Add(id);
211 l.segment_id = current_serial;
212 l.line = linei;
213 l.macro.SetUndef();
214 }
215 q = local_macro.FindPrev(q);
216 }
217 }
218 /*
219 int segmenti = -1;
220 PPMacro *um = FindPPMacro(id, local_segments, segmenti);
221 if(um && segmenti) {
222 PPItem& m = item.Add();
223 m.type = PP_DEFINES;
224 m.segment_id = current_serial = NextPPSerial();
225 um->undef_segment_id = m.segment_id;
226 next_segment = true;
227 local_segments.Add(current_serial);
228 if(id.GetCount()) {
229 PPMacro m;
230 m.segment_id = current_serial;
231 m.line = linei;
232 m.macro.SetUndef();
233 ppmacro.Add(sAllMacros.Put(id, m));
234 }
235 }
236 */
237 }
238 }
239 else
240 if(p.Id("include")) {
241 PPItem& m = item.Add();
242 next_segment = true;
243 m.type = PP_INCLUDE;
244 m.text = TrimBoth(p.GetPtr());
245 if(IsNull(m.text))
246 item.Drop();
247 else
248 includes.FindAdd(m.text);
249 md5.Put('@');
250 md5.Put(m.text);
251 }
252 }
253 }
254 else {
255 while(!p.IsEof()) {
256 if(was_namespace) {
257 int type = was_using ? PP_USING : PP_NAMESPACE;
258 String id;
259 while(p.Char2(':', ':'))
260 id = "::";
261 if(p.IsId()) {
262 id << p.ReadId();
263 while(p.Char2(':', ':') && p.IsId())
264 id << "::" << p.ReadId();
265 if(!was_using)
266 namespace_block.Add(level);
267 if(!was_using || level == 0) {
268 PPItem& m = item.Add();
269 next_segment = true;
270 m.type = type;
271 m.text = id;
272 }
273 md5.Put('$');
274 md5.Put(type);
275 md5.Put(id);
276 }
277 was_namespace = was_using = false;
278 }
279 else
280 if(p.Id("using"))
281 was_using = true;
282 else
283 if(p.Id("namespace"))
284 was_namespace = true;
285 else {
286 was_using = was_namespace = false;
287 if(p.IsId()) {
288 static const VectorMap<String, String>& namespace_macro = GetNamespaceMacros();
289 static const Index<String>& namespace_end_macro = GetNamespaceEndMacros();
290
291 String id = p.ReadId();
292 int q = namespace_macro.Find(id);
293 if(q > 0) {
294 PPItem& m = item.Add();
295 next_segment = true;
296 m.type = PP_NAMESPACE;
297 m.text = namespace_macro[q];
298 namespace_block.Add(level);
299 level++;
300 md5.Put('%');
301 md5.Put(id);
302 }
303 else {
304 q = namespace_end_macro.Find(id);
305 if(q >= 0) {
306 level--;
307 CheckEndNamespace(namespace_block, level, md5);
308 }
309 }
310 keywords.Add(id);
311 }
312 else
313 if(p.Char('}')) {
314 if(level > 0) {
315 level--;
316 CheckEndNamespace(namespace_block, level, md5);
317 }
318 }
319 else
320 if(p.Char('{'))
321 level++;
322 else
323 p.SkipTerm();
324 }
325 }
326 }
327 }
328 catch(...) {}
329 linei++;
330 }
331 md5sum = md5.FinishString();
332 Sort(keywords);
333 Vector<int> remove;
334 int i = 0;
335 while(i < keywords.GetCount()) { // Remove identical items
336 int ii = i;
337 i++;
338 while(i < keywords.GetCount() && keywords[ii] == keywords[i])
339 remove.Add(i++);
340 }
341 keywords.Remove(remove);
342 INTERLOCKED { // this is the only place that is allowed to write to sAllMacros
343 for(int i = 0; i < local_macro.GetCount(); i++)
344 ppmacro.Add(sAllMacros.Put(local_macro.GetKey(i), local_macro[i]));
345 }
346 }
347
Dump() const348 void PPFile::Dump() const
349 {
350 for(int i = 0; i < item.GetCount(); i++) {
351 const PPItem& m = item[i];
352 String ll;
353 ll << decode(m.type, PP_DEFINES, "#defines ", PP_INCLUDE, "#include ",
354 PP_USING, "using namespace ", PP_NAMESPACE, "namespace ",
355 PP_NAMESPACE_END, "}", "");
356 if(m.type == PP_DEFINES)
357 ll << m.segment_id;
358 else
359 ll << m.text;
360 if(m.type == PP_NAMESPACE)
361 ll << " {";
362 LOG(ll);
363 }
364 LOG("----- includes:");
365 DUMPC(includes);
366 }
367
PPSync(const String & include_path)368 void PPSync(const String& include_path)
369 {
370 LLOG("* PPSync");
371 {
372 Mutex::Lock __(s_IncludePathMutex);
373 s_IncludePath.Clear();
374 s_Include_Path = include_path;
375 }
376 {
377 Mutex::Lock __(s_FlatPPMutex);
378 s_FlatPP.Clear();
379 }
380 }
381
InvalidateFileTimeCache()382 void InvalidateFileTimeCache()
383 {
384 Mutex::Lock __(s_PathFileTimeMutex);
385 s_PathFileTime.Clear();
386 }
387
InvalidateFileTimeCache(const String & path)388 void InvalidateFileTimeCache(const String& path)
389 {
390 LLOG("InvalidateFileTimeCache " << path);
391 Mutex::Lock __(s_PathFileTimeMutex);
392 s_PathFileTime.UnlinkKey(path);
393 }
394
GetFileTimeCached(const String & p)395 Time GetFileTimeCached(const String& p)
396 {
397 LTIMING("GetFileTimeCached");
398 Mutex::Lock __(s_PathFileTimeMutex);
399 int q = s_PathFileTime.Find(p);
400 if(q >= 0)
401 return s_PathFileTime[q];
402 Time m = FileGetTime(p);
403 s_PathFileTime.Put(p, m);
404 return m;
405 }
406
GetIncludePath()407 String GetIncludePath()
408 {
409 Mutex::Lock __(s_IncludePathMutex);
410 return s_Include_Path;
411 }
412
GetIncludePath0(const char * s,const char * filedir)413 String GetIncludePath0(const char *s, const char *filedir)
414 {
415 LTIMING("GetIncludePath0");
416 while(IsSpace(*s))
417 s++;
418 int type = *s;
419 if(type == '<' || type == '\"' || type == '?') {
420 s++;
421 String name;
422 if(type == '<') type = '>';
423 while(*s != '\r' && *s != '\n') {
424 if(*s == type) {
425 if(type == '\"') {
426 String fn = NormalizeSourcePath(name, filedir);
427 if(FileExists(fn))
428 return fn;
429 }
430 String p = GetFileOnPath(name, GetIncludePath(), false);
431 if(p.GetCount())
432 return NormalizeSourcePath(p);
433 return Null;
434 }
435 name.Cat(*s++);
436 }
437 }
438 return Null;
439 }
440
GetIncludePath(const String & s,const String & filedir)441 String GetIncludePath(const String& s, const String& filedir)
442 {
443 LTIMING("GetIncludePath");
444 Mutex::Lock __(s_IncludePathMutex);
445 String key;
446 key << s << "#" << filedir;
447 int q = s_IncludePath.Find(key);
448 if(q >= 0)
449 return s_IncludePath[q];
450 LTIMING("GetIncludePath 2");
451 String p = GetIncludePath0(s, filedir);
452 s_IncludePath.Add(key, p);
453 LLOG("GetIncludePath " << s << " " << filedir << ": " << p);
454 return p;
455 }
456
MakePP(const Index<String> & paths)457 void MakePP(const Index<String>& paths)
458 {
459 LLOG("MakePP " << paths);
460 Vector<String> todo;
461 Vector<PPFile *> pp;
462 for(int i = 0; i < paths.GetCount(); i++) {
463 String path = paths[i];
464 PPFile& f = sPPfile.GetPut(path);
465 Time tm = GetFileTimeCached(path);
466 if(f.filetime != tm) {
467 f.filetime = tm;
468 pp.Add(&f);
469 todo.Add(path);
470 }
471 }
472 CoFor(todo.GetCount(), [&](int i) {
473 FileIn in(todo[i]);
474 pp[i]->Parse(in);
475 });
476 }
477
GetPPFile(const char * path)478 const PPFile& GetPPFile(const char *path)
479 {
480 LTIMING("GetPPFile");
481 LLOG("GetPPFile " << path);
482 static PPFile zero;
483 return sPPfile.Get(path, zero);
484 }
485
IsSameFile(const String & f1,const String & f2)486 bool IsSameFile(const String& f1, const String& f2)
487 {
488 return NormalizePath(f1) == NormalizePath(f2);
489 }
490
GetFlatPPFile(const char * path,Index<String> & visited)491 const FlatPP& GetFlatPPFile(const char *path, Index<String>& visited)
492 {
493 LTIMING("GetFlatPPFile");
494 LLOG("GetFlatPPFile " << path);
495 Mutex::Lock __(s_FlatPPMutex);
496 int q = s_FlatPP.Find(path);
497 if(q >= 0) {
498 LLOG("From cache");
499 return s_FlatPP[q];
500 }
501 FlatPP& fp = s_FlatPP.Add(path);
502 const PPFile& pp = GetPPFile(path);
503 int n = visited.GetCount();
504 visited.FindAdd(path);
505 for(int i = 0; i < pp.item.GetCount(); i++) {
506 const PPItem& m = pp.item[i];
507 if(m.type == PP_INCLUDE) {
508 String s = GetIncludePath(m.text, GetFileFolder(path));
509 LLOG("#include " << m.text << " -> " << s);
510 if(s.GetCount() && visited.Find(s) < 0) {
511 visited.Add(s);
512 const FlatPP& pp = GetFlatPPFile(s, visited);
513 for(int i = 0; i < pp.segment_id.GetCount(); i++)
514 fp.segment_id.FindAdd(pp.segment_id[i]);
515 for(int i = 0; i < pp.usings.GetCount(); i++)
516 fp.usings.FindAdd(pp.usings[i]);
517 }
518 }
519 else
520 if(m.type == PP_DEFINES)
521 fp.segment_id.FindAdd(m.segment_id);
522 else
523 if(m.type == PP_USING)
524 fp.usings.FindAdd(m.text);
525 }
526 visited.Trim(n);
527 return fp;
528 }
529
GetFlatPPFile(const char * path)530 const FlatPP& GetFlatPPFile(const char *path)
531 {
532 Index<String> visited;
533 visited.Add(path);
534 return GetFlatPPFile(path, visited);
535 }
536
GetAllMacros(Md5Stream & md5,const String & id,Index<int> & segment_id)537 void GetAllMacros(Md5Stream& md5, const String& id, Index<int>& segment_id)
538 {
539 Vector< Tuple2<int, int> > pos;
540 Vector<const CppMacro *> def;
541 String r;
542 int q = sAllMacros.Find(id);
543 while(q >= 0) {
544 const PPMacro& m = sAllMacros[q];
545 int si = segment_id.Find(m.segment_id);
546 if(si >= 0) {
547 pos.Add(MakeTuple(si, m.line));
548 def.Add(&m.macro);
549 }
550 q = sAllMacros.FindNext(q);
551 }
552 IndexSort(pos, def);
553 int n = def.GetCount();
554 if(n) {
555 md5.Put(&n, sizeof(int));
556 md5.Put(id);
557 for(int i = 0; i < n; i++)
558 md5.Put(def[i]->md5, 16);
559 }
560 }
561
562 static VectorMap<String, String> s_namespace_macro;
563 static Index<String> s_namespace_end_macro;
564
565 static String sDefs;
566
LoadPPConfig()567 void LoadPPConfig()
568 {
569 for(int i = 0; i < sAllMacros.GetCount(); i++)
570 if(sAllMacros[i].segment_id == 0 && !sAllMacros.IsUnlinked(i))
571 sAllMacros.Unlink(i);
572
573 s_namespace_macro.Clear();
574 s_namespace_end_macro.Clear();
575
576 StringStream ss(sDefs);
577 int linei = 0;
578 while(!ss.IsEof()) {
579 String l = ss.GetLine();
580 try {
581 CParser p(l);
582 if(p.Char('#')) {
583 if(p.Id("define")) {
584 CppMacro def;
585 String id = def.Define(p.GetPtr());
586 if(id.GetCount()) {
587 PPMacro m;
588 m.segment_id = 0;
589 m.line = linei;
590 m.macro = def;
591 sAllMacros.Put(id, m);
592 if(findarg(TrimBoth(def.body), "}", "};") >= 0)
593 s_namespace_end_macro.Add(id);
594 try {
595 CParser p(def.body);
596 if(p.Id("namespace") && p.IsId()) {
597 String n = p.ReadId();
598 if(p.Char('{') && p.IsEof())
599 s_namespace_macro.Add(id, n);
600 }
601 }
602 catch(CParser::Error) {}
603 }
604 }
605 }
606 }
607 catch(CParser::Error) {}
608 linei++;
609 }
610 }
611
GetNamespaceMacros()612 const VectorMap<String, String>& GetNamespaceMacros()
613 {
614 return s_namespace_macro;
615 }
616
GetNamespaceEndMacros()617 const Index<String>& GetNamespaceEndMacros()
618 {
619 return s_namespace_end_macro;
620 }
621
SetPPDefs(const String & defs)622 void SetPPDefs(const String& defs)
623 {
624 sDefs = defs;
625 LoadPPConfig();
626 }
627
CleanPP()628 void CleanPP()
629 {
630 sAllMacros.Clear();
631 sPPfile.Clear();
632 s_PPserial = 0;
633 LoadPPConfig();
634 }
635
SerializePPFiles(Stream & s)636 void SerializePPFiles(Stream& s)
637 {
638 int sPPserial = s_PPserial;
639 s % sAllMacros % sPPfile % sPPserial;
640 s_PPserial = sPPserial;
641 if(s.IsLoading())
642 LoadPPConfig();
643
644 #if 0
645 if(s.IsLoading()) { _DBG_
646 DDUMP(sPPfile.GetCount());
647 DDUMP(sAllMacros.GetCount());
648 DDUMP(sPPserial);
649
650 Index<int> psegment;
651 for(int i = 0; i < sPPfile.GetCount(); i++) {
652 const PPFile& p = sPPfile[i];
653 for(int j = 0; j < p.item.GetCount(); j++)
654 psegment.FindAdd(p.item[j].segment_id);
655 }
656 DDUMP(psegment.GetCount());
657
658 int n = 0; _DBG_
659 Index<int> msegment;
660 for(int i = 0; i < sAllMacros.GetCount(); i++) { _DBG_
661 if(sAllMacros.IsUnlinked(i))
662 n++;
663 else
664 msegment.FindAdd(sAllMacros[i].segment_id);
665 }
666 DLOG("UNLINKED " << n);
667 DLOG("Segments " << msegment.GetCount());
668 }
669 #endif
670 }
671
672 }
673