1 use anyhow::{anyhow, Context, Result};
2 use clap::{App, AppSettings, Arg, SubCommand};
3 use glob::glob;
4 use std::path::Path;
5 use std::{env, fs, u64};
6 use tree_sitter_cli::{
7 generate, highlight, logger, parse, playground, query, tags, test, test_highlight, util, wasm,
8 };
9 use tree_sitter_config::Config;
10 use tree_sitter_loader as loader;
11
12 const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
13 const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA");
14
main()15 fn main() {
16 let result = run();
17 if let Err(err) = &result {
18 // Ignore BrokenPipe errors
19 if let Some(error) = err.downcast_ref::<std::io::Error>() {
20 if error.kind() == std::io::ErrorKind::BrokenPipe {
21 return;
22 }
23 }
24 if !err.to_string().is_empty() {
25 eprintln!("{:?}", err);
26 }
27 std::process::exit(1);
28 }
29 }
30
run() -> Result<()>31 fn run() -> Result<()> {
32 let version = if let Some(build_sha) = BUILD_SHA {
33 format!("{} ({})", BUILD_VERSION, build_sha)
34 } else {
35 BUILD_VERSION.to_string()
36 };
37
38 let debug_arg = Arg::with_name("debug")
39 .help("Show parsing debug log")
40 .long("debug")
41 .short("d");
42
43 let debug_graph_arg = Arg::with_name("debug-graph")
44 .help("Produce the log.html file with debug graphs")
45 .long("debug-graph")
46 .short("D");
47
48 let debug_build_arg = Arg::with_name("debug-build")
49 .help("Compile a parser in debug mode")
50 .long("debug-build")
51 .short("0");
52
53 let paths_file_arg = Arg::with_name("paths-file")
54 .help("The path to a file with paths to source file(s)")
55 .long("paths")
56 .takes_value(true);
57
58 let paths_arg = Arg::with_name("paths")
59 .help("The source file(s) to use")
60 .multiple(true);
61
62 let scope_arg = Arg::with_name("scope")
63 .help("Select a language by the scope instead of a file extension")
64 .long("scope")
65 .takes_value(true);
66
67 let time_arg = Arg::with_name("time")
68 .help("Measure execution time")
69 .long("time")
70 .short("t");
71
72 let quiet_arg = Arg::with_name("quiet")
73 .help("Suppress main output")
74 .long("quiet")
75 .short("q");
76
77 let matches = App::new("tree-sitter")
78 .author("Max Brunsfeld <maxbrunsfeld@gmail.com>")
79 .about("Generates and tests parsers")
80 .version(version.as_str())
81 .setting(AppSettings::SubcommandRequiredElseHelp)
82 .global_setting(AppSettings::ColoredHelp)
83 .global_setting(AppSettings::DeriveDisplayOrder)
84 .global_setting(AppSettings::DisableHelpSubcommand)
85 .subcommand(SubCommand::with_name("init-config").about("Generate a default config file"))
86 .subcommand(
87 SubCommand::with_name("generate")
88 .alias("gen")
89 .alias("g")
90 .about("Generate a parser")
91 .arg(Arg::with_name("grammar-path").index(1))
92 .arg(Arg::with_name("log").long("log"))
93 .arg(Arg::with_name("prev-abi").long("prev-abi"))
94 .arg(Arg::with_name("no-bindings").long("no-bindings"))
95 .arg(
96 Arg::with_name("report-states-for-rule")
97 .long("report-states-for-rule")
98 .value_name("rule-name")
99 .takes_value(true),
100 )
101 .arg(Arg::with_name("no-minimize").long("no-minimize")),
102 )
103 .subcommand(
104 SubCommand::with_name("parse")
105 .alias("p")
106 .about("Parse files")
107 .arg(&paths_file_arg)
108 .arg(&paths_arg)
109 .arg(&scope_arg)
110 .arg(&debug_arg)
111 .arg(&debug_build_arg)
112 .arg(&debug_graph_arg)
113 .arg(Arg::with_name("debug-xml").long("xml").short("x"))
114 .arg(
115 Arg::with_name("stat")
116 .help("Show parsing statistic")
117 .long("stat")
118 .short("s"),
119 )
120 .arg(
121 Arg::with_name("timeout")
122 .help("Interrupt the parsing process by timeout (µs)")
123 .long("timeout")
124 .takes_value(true),
125 )
126 .arg(&time_arg)
127 .arg(&quiet_arg)
128 .arg(
129 Arg::with_name("edits")
130 .help("Apply edits in the format: \"row,col del_count insert_text\"")
131 .long("edit")
132 .short("edit")
133 .takes_value(true)
134 .multiple(true)
135 .number_of_values(1),
136 ),
137 )
138 .subcommand(
139 SubCommand::with_name("query")
140 .alias("q")
141 .about("Search files using a syntax tree query")
142 .arg(
143 Arg::with_name("query-path")
144 .help("Path to a file with queries")
145 .index(1)
146 .required(true),
147 )
148 .arg(&paths_file_arg)
149 .arg(&paths_arg.clone().index(2))
150 .arg(
151 Arg::with_name("byte-range")
152 .help("The range of byte offsets in which the query will be executed")
153 .long("byte-range")
154 .takes_value(true),
155 )
156 .arg(&scope_arg)
157 .arg(Arg::with_name("captures").long("captures").short("c"))
158 .arg(Arg::with_name("test").long("test")),
159 )
160 .subcommand(
161 SubCommand::with_name("tags")
162 .about("Generate a list of tags")
163 .arg(&scope_arg)
164 .arg(&time_arg)
165 .arg(&quiet_arg)
166 .arg(&paths_file_arg)
167 .arg(&paths_arg),
168 )
169 .subcommand(
170 SubCommand::with_name("test")
171 .alias("t")
172 .about("Run a parser's tests")
173 .arg(
174 Arg::with_name("filter")
175 .long("filter")
176 .short("f")
177 .takes_value(true)
178 .help("Only run corpus test cases whose name includes the given string"),
179 )
180 .arg(
181 Arg::with_name("update")
182 .long("update")
183 .short("u")
184 .help("Update all syntax trees in corpus files with current parser output"),
185 )
186 .arg(&debug_arg)
187 .arg(&debug_build_arg)
188 .arg(&debug_graph_arg),
189 )
190 .subcommand(
191 SubCommand::with_name("highlight")
192 .about("Highlight a file")
193 .arg(
194 Arg::with_name("html")
195 .help("Generate highlighting as an HTML document")
196 .long("html")
197 .short("H"),
198 )
199 .arg(&scope_arg)
200 .arg(&time_arg)
201 .arg(&quiet_arg)
202 .arg(&paths_file_arg)
203 .arg(&paths_arg),
204 )
205 .subcommand(
206 SubCommand::with_name("build-wasm")
207 .alias("bw")
208 .about("Compile a parser to WASM")
209 .arg(
210 Arg::with_name("docker")
211 .long("docker")
212 .help("Run emscripten via docker even if it is installed locally"),
213 )
214 .arg(Arg::with_name("path").index(1).multiple(true)),
215 )
216 .subcommand(
217 SubCommand::with_name("playground")
218 .alias("play")
219 .alias("pg")
220 .alias("web-ui")
221 .about("Start local playground for a parser in the browser")
222 .arg(
223 Arg::with_name("quiet")
224 .long("quiet")
225 .short("q")
226 .help("Don't open in default browser"),
227 ),
228 )
229 .subcommand(
230 SubCommand::with_name("dump-languages")
231 .about("Print info about all known language parsers"),
232 )
233 .get_matches();
234
235 let current_dir = env::current_dir().unwrap();
236 let config = Config::load()?;
237 let mut loader = loader::Loader::new()?;
238
239 match matches.subcommand() {
240 ("init-config", Some(_)) => {
241 if let Ok(Some(config_path)) = Config::find_config_file() {
242 return Err(anyhow!(
243 "Remove your existing config file first: {}",
244 config_path.to_string_lossy()
245 ));
246 }
247 let mut config = Config::initial()?;
248 config.add(tree_sitter_loader::Config::initial())?;
249 config.add(tree_sitter_cli::highlight::ThemeConfig::default())?;
250 config.save()?;
251 println!(
252 "Saved initial configuration to {}",
253 config.location.display()
254 );
255 }
256
257 ("generate", Some(matches)) => {
258 let grammar_path = matches.value_of("grammar-path");
259 let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| {
260 if matches.is_present("report-states") {
261 Some("")
262 } else {
263 None
264 }
265 });
266 if matches.is_present("log") {
267 logger::init();
268 }
269 let new_abi = !matches.is_present("prev-abi");
270 let generate_bindings = !matches.is_present("no-bindings");
271 generate::generate_parser_in_directory(
272 ¤t_dir,
273 grammar_path,
274 new_abi,
275 generate_bindings,
276 report_symbol_name,
277 )?;
278 }
279
280 ("test", Some(matches)) => {
281 let debug = matches.is_present("debug");
282 let debug_graph = matches.is_present("debug-graph");
283 let debug_build = matches.is_present("debug-build");
284 let update = matches.is_present("update");
285 let filter = matches.value_of("filter");
286
287 loader.use_debug_build(debug_build);
288
289 let languages = loader.languages_at_path(¤t_dir)?;
290 let language = languages
291 .first()
292 .ok_or_else(|| anyhow!("No language found"))?;
293 let test_dir = current_dir.join("test");
294
295 // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`.
296 let mut test_corpus_dir = test_dir.join("corpus");
297 if !test_corpus_dir.is_dir() {
298 test_corpus_dir = current_dir.join("corpus");
299 }
300 if test_corpus_dir.is_dir() {
301 test::run_tests_at_path(
302 *language,
303 &test_corpus_dir,
304 debug,
305 debug_graph,
306 filter,
307 update,
308 )?;
309 }
310
311 // Check that all of the queries are valid.
312 test::check_queries_at_path(*language, ¤t_dir.join("queries"))?;
313
314 // Run the syntax highlighting tests.
315 let test_highlight_dir = test_dir.join("highlight");
316 if test_highlight_dir.is_dir() {
317 test_highlight::test_highlights(&loader, &test_highlight_dir)?;
318 }
319 }
320
321 ("parse", Some(matches)) => {
322 let debug = matches.is_present("debug");
323 let debug_graph = matches.is_present("debug-graph");
324 let debug_build = matches.is_present("debug-build");
325 let debug_xml = matches.is_present("debug-xml");
326 let quiet = matches.is_present("quiet");
327 let time = matches.is_present("time");
328 let edits = matches
329 .values_of("edits")
330 .map_or(Vec::new(), |e| e.collect());
331 let cancellation_flag = util::cancel_on_stdin();
332
333 if debug {
334 // For augmenting debug logging in external scanners
335 env::set_var("TREE_SITTER_DEBUG", "1");
336 }
337
338 loader.use_debug_build(debug_build);
339
340 let timeout = matches
341 .value_of("timeout")
342 .map_or(0, |t| u64::from_str_radix(t, 10).unwrap());
343
344 let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
345
346 let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap_or(0);
347 let mut has_error = false;
348 let loader_config = config.get()?;
349 loader.find_all_languages(&loader_config)?;
350
351 let should_track_stats = matches.is_present("stat");
352 let mut stats = parse::Stats::default();
353
354 for path in paths {
355 let path = Path::new(&path);
356 let language =
357 loader.select_language(path, ¤t_dir, matches.value_of("scope"))?;
358
359 let this_file_errored = parse::parse_file_at_path(
360 language,
361 path,
362 &edits,
363 max_path_length,
364 quiet,
365 time,
366 timeout,
367 debug,
368 debug_graph,
369 debug_xml,
370 Some(&cancellation_flag),
371 )?;
372
373 if should_track_stats {
374 stats.total_parses += 1;
375 if !this_file_errored {
376 stats.successful_parses += 1;
377 }
378 }
379
380 has_error |= this_file_errored;
381 }
382
383 if should_track_stats {
384 println!("{}", stats)
385 }
386
387 if has_error {
388 return Err(anyhow!(""));
389 }
390 }
391
392 ("query", Some(matches)) => {
393 let ordered_captures = matches.values_of("captures").is_some();
394 let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
395 let loader_config = config.get()?;
396 loader.find_all_languages(&loader_config)?;
397 let language = loader.select_language(
398 Path::new(&paths[0]),
399 ¤t_dir,
400 matches.value_of("scope"),
401 )?;
402 let query_path = Path::new(matches.value_of("query-path").unwrap());
403 let range = matches.value_of("byte-range").map(|br| {
404 let r: Vec<&str> = br.split(":").collect();
405 r[0].parse().unwrap()..r[1].parse().unwrap()
406 });
407 let should_test = matches.is_present("test");
408 query::query_files_at_paths(
409 language,
410 paths,
411 query_path,
412 ordered_captures,
413 range,
414 should_test,
415 )?;
416 }
417
418 ("tags", Some(matches)) => {
419 let loader_config = config.get()?;
420 loader.find_all_languages(&loader_config)?;
421 let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
422 tags::generate_tags(
423 &loader,
424 matches.value_of("scope"),
425 &paths,
426 matches.is_present("quiet"),
427 matches.is_present("time"),
428 )?;
429 }
430
431 ("highlight", Some(matches)) => {
432 let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?;
433 loader.configure_highlights(&theme_config.theme.highlight_names);
434 let loader_config = config.get()?;
435 loader.find_all_languages(&loader_config)?;
436
437 let time = matches.is_present("time");
438 let quiet = matches.is_present("quiet");
439 let html_mode = quiet || matches.is_present("html");
440 let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
441
442 if html_mode && !quiet {
443 println!("{}", highlight::HTML_HEADER);
444 }
445
446 let cancellation_flag = util::cancel_on_stdin();
447
448 let mut lang = None;
449 if let Some(scope) = matches.value_of("scope") {
450 lang = loader.language_configuration_for_scope(scope)?;
451 if lang.is_none() {
452 return Err(anyhow!("Unknown scope '{}'", scope));
453 }
454 }
455
456 for path in paths {
457 let path = Path::new(&path);
458 let (language, language_config) = match lang {
459 Some(v) => v,
460 None => match loader.language_configuration_for_file_name(path)? {
461 Some(v) => v,
462 None => {
463 eprintln!("No language found for path {:?}", path);
464 continue;
465 }
466 },
467 };
468
469 if let Some(highlight_config) = language_config.highlight_config(language)? {
470 let source = fs::read(path)?;
471 if html_mode {
472 highlight::html(
473 &loader,
474 &theme_config.theme,
475 &source,
476 highlight_config,
477 quiet,
478 time,
479 )?;
480 } else {
481 highlight::ansi(
482 &loader,
483 &theme_config.theme,
484 &source,
485 highlight_config,
486 time,
487 Some(&cancellation_flag),
488 )?;
489 }
490 } else {
491 eprintln!("No syntax highlighting config found for path {:?}", path);
492 }
493 }
494
495 if html_mode && !quiet {
496 println!("{}", highlight::HTML_FOOTER);
497 }
498 }
499
500 ("build-wasm", Some(matches)) => {
501 let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
502 wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
503 }
504
505 ("playground", Some(matches)) => {
506 let open_in_browser = !matches.is_present("quiet");
507 playground::serve(¤t_dir, open_in_browser);
508 }
509
510 ("dump-languages", Some(_)) => {
511 let loader_config = config.get()?;
512 loader.find_all_languages(&loader_config)?;
513 for (configuration, language_path) in loader.get_all_language_configurations() {
514 println!(
515 concat!(
516 "scope: {}\n",
517 "parser: {:?}\n",
518 "highlights: {:?}\n",
519 "file_types: {:?}\n",
520 "content_regex: {:?}\n",
521 "injection_regex: {:?}\n",
522 ),
523 configuration.scope.as_ref().unwrap_or(&String::new()),
524 language_path,
525 configuration.highlights_filenames,
526 configuration.file_types,
527 configuration.content_regex,
528 configuration.injection_regex,
529 );
530 }
531 }
532
533 _ => unreachable!(),
534 }
535
536 Ok(())
537 }
538
collect_paths<'a>( paths_file: Option<&str>, paths: Option<impl Iterator<Item = &'a str>>, ) -> Result<Vec<String>>539 fn collect_paths<'a>(
540 paths_file: Option<&str>,
541 paths: Option<impl Iterator<Item = &'a str>>,
542 ) -> Result<Vec<String>> {
543 if let Some(paths_file) = paths_file {
544 return Ok(fs::read_to_string(paths_file)
545 .with_context(|| format!("Failed to read paths file {}", paths_file))?
546 .trim()
547 .lines()
548 .map(String::from)
549 .collect::<Vec<_>>());
550 }
551
552 if let Some(paths) = paths {
553 let mut result = Vec::new();
554
555 let mut incorporate_path = |path: &str, positive| {
556 if positive {
557 result.push(path.to_string());
558 } else {
559 if let Some(index) = result.iter().position(|p| p == path) {
560 result.remove(index);
561 }
562 }
563 };
564
565 for mut path in paths {
566 let mut positive = true;
567 if path.starts_with("!") {
568 positive = false;
569 path = path.trim_start_matches("!");
570 }
571
572 if Path::new(path).exists() {
573 incorporate_path(path, positive);
574 } else {
575 let paths =
576 glob(path).with_context(|| format!("Invalid glob pattern {:?}", path))?;
577 for path in paths {
578 if let Some(path) = path?.to_str() {
579 incorporate_path(path, positive);
580 }
581 }
582 }
583 }
584
585 if result.is_empty() {
586 return Err(anyhow!(
587 "No files were found at or matched by the provided pathname/glob"
588 ));
589 }
590
591 return Ok(result);
592 }
593
594 Err(anyhow!("Must provide one or more paths"))
595 }
596