1 /*============================================================================= 2 Copyright (c) 2006 Joel de Guzman 3 http://spirit.sourceforge.net/ 4 5 Use, modification and distribution is subject to the Boost Software 6 License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at 7 http://www.boost.org/LICENSE_1_0.txt) 8 =============================================================================*/ 9 10 #include <boost/bind.hpp> 11 #include <boost/shared_ptr.hpp> 12 #include <boost/spirit/include/classic_actor.hpp> 13 #include <boost/spirit/include/classic_confix.hpp> 14 #include <boost/spirit/include/classic_core.hpp> 15 #include "actions.hpp" 16 #include "block_tags.hpp" 17 #include "files.hpp" 18 #include "state.hpp" 19 #include "stream.hpp" 20 #include "template_stack.hpp" 21 #include "values.hpp" 22 23 namespace quickbook 24 { 25 namespace cl = boost::spirit::classic; 26 27 struct code_snippet_actions 28 { code_snippet_actionsquickbook::code_snippet_actions29 code_snippet_actions( 30 std::vector<template_symbol>& storage_, 31 file_ptr source_file_, 32 char const* source_type_) 33 : last_code_pos(source_file_->source().begin()) 34 , in_code(false) 35 , snippet_stack() 36 , storage(storage_) 37 , source_file(source_file_) 38 , source_type(source_type_) 39 , error_count(0) 40 { 41 source_file->is_code_snippets = true; 42 content.start(source_file); 43 } 44 45 void mark(string_iterator first, string_iterator last); 46 void pass_thru(string_iterator first, string_iterator last); 47 void escaped_comment(string_iterator first, string_iterator last); 48 void start_snippet(string_iterator first, string_iterator last); 49 void start_snippet_impl(std::string const&, string_iterator); 50 void end_snippet(string_iterator first, string_iterator last); 51 void end_snippet_impl(string_iterator); 52 void end_file(string_iterator, string_iterator); 53 54 void append_code(string_iterator first, string_iterator last); 55 void close_code(); 56 57 struct snippet_data 58 { snippet_dataquickbook::code_snippet_actions::snippet_data59 snippet_data(std::string const& id_) : id(id_), start_code(false) {} 60 61 std::string id; 62 bool start_code; 63 string_iterator source_pos; 64 mapped_file_builder::pos_type start_pos; 65 boost::shared_ptr<snippet_data> next; 66 }; 67 push_snippet_dataquickbook::code_snippet_actions68 void push_snippet_data(std::string const& id, string_iterator pos) 69 { 70 boost::shared_ptr<snippet_data> new_snippet(new snippet_data(id)); 71 new_snippet->next = snippet_stack; 72 snippet_stack = new_snippet; 73 snippet_stack->start_code = in_code; 74 snippet_stack->source_pos = pos; 75 snippet_stack->start_pos = content.get_pos(); 76 } 77 pop_snippet_dataquickbook::code_snippet_actions78 boost::shared_ptr<snippet_data> pop_snippet_data() 79 { 80 boost::shared_ptr<snippet_data> snippet(snippet_stack); 81 snippet_stack = snippet->next; 82 snippet->next.reset(); 83 return snippet; 84 } 85 86 mapped_file_builder content; 87 string_iterator mark_begin, mark_end; 88 string_iterator last_code_pos; 89 bool in_code; 90 boost::shared_ptr<snippet_data> snippet_stack; 91 std::vector<template_symbol>& storage; 92 file_ptr source_file; 93 char const* const source_type; 94 int error_count; 95 }; 96 97 struct python_code_snippet_grammar 98 : cl::grammar<python_code_snippet_grammar> 99 { 100 typedef code_snippet_actions actions_type; 101 python_code_snippet_grammarquickbook::python_code_snippet_grammar102 python_code_snippet_grammar(actions_type& actions_) : actions(actions_) 103 { 104 } 105 106 template <typename Scanner> struct definition 107 { 108 typedef code_snippet_actions actions_type; 109 definitionquickbook::python_code_snippet_grammar::definition110 definition(python_code_snippet_grammar const& self) 111 { 112 // clang-format off 113 114 start_ = (*code_elements) [boost::bind(&actions_type::end_file, &self.actions, _1, _2)] 115 ; 116 117 identifier = 118 (cl::alpha_p | '_') >> *(cl::alnum_p | '_') 119 ; 120 121 code_elements = 122 start_snippet [boost::bind(&actions_type::start_snippet, &self.actions, _1, _2)] 123 | end_snippet [boost::bind(&actions_type::end_snippet, &self.actions, _1, _2)] 124 | escaped_comment [boost::bind(&actions_type::escaped_comment, &self.actions, _1, _2)] 125 | pass_thru_comment [boost::bind(&actions_type::pass_thru, &self.actions, _1, _2)] 126 | ignore [boost::bind(&actions_type::append_code, &self.actions, _1, _2)] 127 | cl::anychar_p 128 ; 129 130 start_snippet = 131 *cl::blank_p 132 >> !(cl::eol_p >> *cl::blank_p) 133 >> "#[" 134 >> *cl::blank_p 135 >> identifier [boost::bind(&actions_type::mark, &self.actions, _1, _2)] 136 >> *(cl::anychar_p - cl::eol_p) 137 ; 138 139 end_snippet = 140 *cl::blank_p 141 >> !(cl::eol_p >> *cl::blank_p) 142 >> "#]" 143 >> *(cl::anychar_p - cl::eol_p) 144 ; 145 146 ignore 147 = cl::confix_p( 148 *cl::blank_p >> "#<-", 149 *cl::anychar_p, 150 "#->" >> *cl::blank_p >> (cl::eol_p | cl::end_p) 151 ) 152 | cl::confix_p( 153 "\"\"\"<-\"\"\"", 154 *cl::anychar_p, 155 "\"\"\"->\"\"\"" 156 ) 157 | cl::confix_p( 158 "\"\"\"<-", 159 *cl::anychar_p, 160 "->\"\"\"" 161 ) 162 ; 163 164 escaped_comment = 165 cl::confix_p( 166 *cl::space_p >> "#`", 167 (*cl::anychar_p) [boost::bind(&actions_type::mark, &self.actions, _1, _2)], 168 (cl::eol_p | cl::end_p) 169 ) 170 | cl::confix_p( 171 *cl::space_p >> "\"\"\"`", 172 (*cl::anychar_p) [boost::bind(&actions_type::mark, &self.actions, _1, _2)], 173 "\"\"\"" 174 ) 175 ; 176 177 // Note: Unlike escaped_comment and ignore, this doesn't 178 // swallow preceeding whitespace. 179 pass_thru_comment 180 = "#=" >> (cl::eps_p - '=') 181 >> ( *(cl::anychar_p - cl::eol_p) 182 >> (cl::eol_p | cl::end_p) 183 ) [boost::bind(&actions_type::mark, &self.actions, _1, _2)] 184 | cl::confix_p( 185 "\"\"\"=" >> (cl::eps_p - '='), 186 (*cl::anychar_p) [boost::bind(&actions_type::mark, &self.actions, _1, _2)], 187 "\"\"\"" 188 ) 189 ; 190 191 // clang-format on 192 } 193 194 cl::rule<Scanner> start_, identifier, code_elements, start_snippet, 195 end_snippet, escaped_comment, pass_thru_comment, ignore; 196 startquickbook::python_code_snippet_grammar::definition197 cl::rule<Scanner> const& start() const { return start_; } 198 }; 199 200 actions_type& actions; 201 }; 202 203 struct cpp_code_snippet_grammar : cl::grammar<cpp_code_snippet_grammar> 204 { 205 typedef code_snippet_actions actions_type; 206 cpp_code_snippet_grammarquickbook::cpp_code_snippet_grammar207 cpp_code_snippet_grammar(actions_type& actions_) : actions(actions_) {} 208 209 template <typename Scanner> struct definition 210 { definitionquickbook::cpp_code_snippet_grammar::definition211 definition(cpp_code_snippet_grammar const& self) 212 { 213 // clang-format off 214 215 start_ = (*code_elements) [boost::bind(&actions_type::end_file, &self.actions, _1, _2)] 216 ; 217 218 identifier = 219 (cl::alpha_p | '_') >> *(cl::alnum_p | '_') 220 ; 221 222 code_elements = 223 start_snippet [boost::bind(&actions_type::start_snippet, &self.actions, _1, _2)] 224 | end_snippet [boost::bind(&actions_type::end_snippet, &self.actions, _1, _2)] 225 | escaped_comment [boost::bind(&actions_type::escaped_comment, &self.actions, _1, _2)] 226 | ignore [boost::bind(&actions_type::append_code, &self.actions, _1, _2)] 227 | pass_thru_comment [boost::bind(&actions_type::pass_thru, &self.actions, _1, _2)] 228 | cl::anychar_p 229 ; 230 231 start_snippet = 232 *cl::blank_p 233 >> !(cl::eol_p >> *cl::blank_p) 234 >> "//[" 235 >> *cl::blank_p 236 >> identifier [boost::bind(&actions_type::mark, &self.actions, _1, _2)] 237 >> *(cl::anychar_p - cl::eol_p) 238 | 239 *cl::blank_p 240 >> cl::eol_p 241 >> *cl::blank_p 242 >> "/*[" 243 >> *cl::space_p 244 >> identifier [boost::bind(&actions_type::mark, &self.actions, _1, _2)] 245 >> *cl::space_p 246 >> "*/" 247 >> *cl::blank_p 248 >> cl::eps_p(cl::eol_p) 249 | 250 "/*[" 251 >> *cl::space_p 252 >> identifier [boost::bind(&actions_type::mark, &self.actions, _1, _2)] 253 >> *cl::space_p 254 >> "*/" 255 ; 256 257 end_snippet = 258 *cl::blank_p 259 >> !(cl::eol_p >> *cl::blank_p) 260 >> "//]" 261 >> *(cl::anychar_p - cl::eol_p) 262 | 263 *cl::blank_p 264 >> cl::eol_p 265 >> *cl::blank_p 266 >> "/*]*/" 267 >> *cl::blank_p 268 >> cl::eps_p(cl::eol_p) 269 | 270 "/*[*/" 271 ; 272 273 ignore 274 = cl::confix_p( 275 *cl::blank_p >> "//<-", 276 *cl::anychar_p, 277 "//->" 278 ) 279 >> *cl::blank_p 280 >> cl::eol_p 281 | cl::confix_p( 282 "/*<-*/", 283 *cl::anychar_p, 284 "/*->*/" 285 ) 286 | cl::confix_p( 287 "/*<-", 288 *cl::anychar_p, 289 "->*/" 290 ) 291 ; 292 293 escaped_comment 294 = cl::confix_p( 295 *cl::space_p >> "//`", 296 (*cl::anychar_p) [boost::bind(&actions_type::mark, &self.actions, _1, _2)], 297 (cl::eol_p | cl::end_p) 298 ) 299 | cl::confix_p( 300 *cl::space_p >> "/*`", 301 (*cl::anychar_p) [boost::bind(&actions_type::mark, &self.actions, _1, _2)], 302 "*/" 303 ) 304 ; 305 306 // Note: Unlike escaped_comment and ignore, this doesn't 307 // swallow preceeding whitespace. 308 pass_thru_comment 309 = "//=" >> (cl::eps_p - '=') 310 >> ( *(cl::anychar_p - cl::eol_p) 311 >> (cl::eol_p | cl::end_p) 312 ) [boost::bind(&actions_type::mark, &self.actions, _1, _2)] 313 | cl::confix_p( 314 "/*=" >> (cl::eps_p - '='), 315 (*cl::anychar_p) [boost::bind(&actions_type::mark, &self.actions, _1, _2)], 316 "*/" 317 ) 318 ; 319 320 // clang-format on 321 } 322 323 cl::rule<Scanner> start_, identifier, code_elements, start_snippet, 324 end_snippet, escaped_comment, pass_thru_comment, ignore; 325 startquickbook::cpp_code_snippet_grammar::definition326 cl::rule<Scanner> const& start() const { return start_; } 327 }; 328 329 actions_type& actions; 330 }; 331 load_snippets(fs::path const & filename,std::vector<template_symbol> & storage,std::string const & extension,value::tag_type load_type)332 int load_snippets( 333 fs::path const& filename, 334 std::vector<template_symbol>& storage // snippets are stored in a 335 // vector of template_symbols 336 , 337 std::string const& extension, 338 value::tag_type load_type) 339 { 340 ignore_variable(&load_type); // Avoid unreferenced parameter warning. 341 assert( 342 load_type == block_tags::include || 343 load_type == block_tags::import); 344 345 bool is_python = extension == ".py" || extension == ".jam"; 346 code_snippet_actions a( 347 storage, load(filename, qbk_version_n), 348 is_python ? "[python]" : "[c++]"); 349 350 string_iterator first(a.source_file->source().begin()); 351 string_iterator last(a.source_file->source().end()); 352 353 cl::parse_info<string_iterator> info; 354 355 if (is_python) { 356 info = boost::spirit::classic::parse( 357 first, last, python_code_snippet_grammar(a)); 358 } 359 else { 360 info = boost::spirit::classic::parse( 361 first, last, cpp_code_snippet_grammar(a)); 362 } 363 364 assert(info.full); 365 return a.error_count; 366 } 367 append_code(string_iterator first,string_iterator last)368 void code_snippet_actions::append_code( 369 string_iterator first, string_iterator last) 370 { 371 assert(last_code_pos <= first); 372 373 if (snippet_stack) { 374 if (last_code_pos != first) { 375 if (!in_code) { 376 content.add_at_pos("\n\n", last_code_pos); 377 content.add_at_pos(source_type, last_code_pos); 378 content.add_at_pos("```\n", last_code_pos); 379 380 in_code = true; 381 } 382 383 content.add(quickbook::string_view( 384 last_code_pos, first - last_code_pos)); 385 } 386 } 387 388 last_code_pos = last; 389 } 390 close_code()391 void code_snippet_actions::close_code() 392 { 393 if (!snippet_stack) return; 394 395 if (in_code) { 396 content.add_at_pos("\n```\n\n", last_code_pos); 397 in_code = false; 398 } 399 } 400 mark(string_iterator first,string_iterator last)401 void code_snippet_actions::mark(string_iterator first, string_iterator last) 402 { 403 mark_begin = first; 404 mark_end = last; 405 } 406 pass_thru(string_iterator first,string_iterator last)407 void code_snippet_actions::pass_thru( 408 string_iterator first, string_iterator last) 409 { 410 if (!snippet_stack) return; 411 append_code(first, last); 412 413 if (!in_code) { 414 content.add_at_pos("\n\n", first); 415 content.add_at_pos(source_type, first); 416 content.add_at_pos("```\n", first); 417 in_code = true; 418 } 419 420 content.add(quickbook::string_view(mark_begin, mark_end - mark_begin)); 421 } 422 escaped_comment(string_iterator first,string_iterator last)423 void code_snippet_actions::escaped_comment( 424 string_iterator first, string_iterator last) 425 { 426 append_code(first, last); 427 close_code(); 428 429 if (mark_begin != mark_end) { 430 if (!snippet_stack) { 431 start_snippet_impl("!", first); 432 } 433 434 snippet_data& snippet = *snippet_stack; 435 436 content.add_at_pos("\n", mark_begin); 437 content.unindent_and_add( 438 quickbook::string_view(mark_begin, mark_end - mark_begin)); 439 440 if (snippet.id == "!") { 441 end_snippet_impl(last); 442 } 443 } 444 } 445 start_snippet(string_iterator first,string_iterator last)446 void code_snippet_actions::start_snippet( 447 string_iterator first, string_iterator last) 448 { 449 append_code(first, last); 450 start_snippet_impl(std::string(mark_begin, mark_end), first); 451 } 452 end_snippet(string_iterator first,string_iterator last)453 void code_snippet_actions::end_snippet( 454 string_iterator first, string_iterator last) 455 { 456 append_code(first, last); 457 458 if (!snippet_stack) { 459 if (qbk_version_n >= 106u) { 460 detail::outerr(source_file, first) 461 << "Mismatched end snippet." << std::endl; 462 ++error_count; 463 } 464 else { 465 detail::outwarn(source_file, first) 466 << "Mismatched end snippet." << std::endl; 467 } 468 return; 469 } 470 471 end_snippet_impl(first); 472 } 473 end_file(string_iterator,string_iterator pos)474 void code_snippet_actions::end_file(string_iterator, string_iterator pos) 475 { 476 append_code(pos, pos); 477 close_code(); 478 479 while (snippet_stack) { 480 if (qbk_version_n >= 106u) { 481 detail::outerr(source_file->path) 482 << "Unclosed snippet '" << snippet_stack->id << "'" 483 << std::endl; 484 ++error_count; 485 } 486 else { 487 detail::outwarn(source_file->path) 488 << "Unclosed snippet '" << snippet_stack->id << "'" 489 << std::endl; 490 } 491 492 end_snippet_impl(pos); 493 } 494 } 495 start_snippet_impl(std::string const & id,string_iterator position)496 void code_snippet_actions::start_snippet_impl( 497 std::string const& id, string_iterator position) 498 { 499 push_snippet_data(id, position); 500 } 501 end_snippet_impl(string_iterator position)502 void code_snippet_actions::end_snippet_impl(string_iterator position) 503 { 504 assert(snippet_stack); 505 506 boost::shared_ptr<snippet_data> snippet = pop_snippet_data(); 507 508 mapped_file_builder f; 509 f.start(source_file); 510 if (snippet->start_code) { 511 f.add_at_pos("\n\n", snippet->source_pos); 512 f.add_at_pos(source_type, snippet->source_pos); 513 f.add_at_pos("```\n", snippet->source_pos); 514 } 515 f.add(content, snippet->start_pos, content.get_pos()); 516 if (in_code) { 517 f.add_at_pos("\n```\n\n", position); 518 } 519 520 std::vector<std::string> params; 521 522 file_ptr body = f.release(); 523 524 storage.push_back(template_symbol( 525 snippet->id, params, 526 qbk_value( 527 body, body->source().begin(), body->source().end(), 528 template_tags::snippet))); 529 } 530 } 531