Add the cache generator subcommand and docs for custom syntaxes

Author: notriddle

.gitignore

@@ -12,6 +12,8 @@ tests/dummy_book/book/
 # Ignore Jetbrains specific files.
 .idea/
 
+src/theme/syntaxes.bin
+
 # Ignore Vim temporary and swap files.
 *.sw?
 *~

Cargo.toml

@@ -14,6 +14,10 @@ license = "MPL-2.0"
 readme = "README.md"
 repository = "https://github.com/rust-lang/mdBook"
 description = "Creates a book from markdown files"
+build = "build.rs"
+
+[build-dependencies]
+syntect = { version = "4.6.0", default-features = false, features = ["regex-onig", "parsing", "html", "dump-load", "yaml-load", "dump-create", "assets"] }
 
 [dependencies]
 anyhow = "1.0.28"
@@ -55,10 +59,11 @@ pretty_assertions = "0.6"
 walkdir = "2.0"
 
 [features]
-default = ["watch", "serve", "search"]
+default = ["watch", "serve", "search", "gen-syntax-cache"]
 watch = ["notify", "gitignore"]
 serve = ["futures-util", "tokio", "warp"]
 search = ["elasticlunr-rs", "ammonia"]
+gen-syntax-cache = ["syntect/dump-create"]
 
 [[bin]]
 doc = false

build.rs

@@ -0,0 +1,18 @@
+use std::env;
+use std::error::Error;
+use syntect::dumps::dump_to_file;
+use syntect::parsing::SyntaxSet;
+
+pub fn main() -> Result<(), Box<dyn Error>> {
+    let src_dir = format!(
+        "{}/src/theme/syntaxes/",
+        env::var("CARGO_MANIFEST_DIR").unwrap()
+    );
+    let dest = format!("{}/syntaxes.bin", env::var("OUT_DIR").unwrap());
+
+    let mut builder = SyntaxSet::load_defaults_newlines().into_builder();
+    builder.add_from_folder(&src_dir, true)?;
+    dump_to_file(&builder.build(), dest)?;
+
+    Ok(())
+}

guide/src/format/theme/syntax-highlighting.md

@@ -1,6 +1,6 @@
 # Syntax Highlighting
 
-mdBook uses [Highlight.js](https://highlightjs.org) with a custom theme
+mdBook uses [syntect](https://docs.rs/syntect/4.6.0/syntect/) with a custom theme
 for syntax highlighting.
 
 Automatic language detection has been turned off, so you will probably want to
@@ -17,65 +17,103 @@ fn main() {
 ## Supported languages
 
 These languages are supported by default, but you can add more by supplying
-your own `highlight.js` file:
-
-- apache
-- armasm
-- bash
-- c
-- coffeescript
-- cpp
-- csharp
-- css
-- d
-- diff
-- go
-- handlebars
-- haskell
-- http
-- ini
-- java
-- javascript
-- json
-- julia
-- kotlin
-- less
-- lua
-- makefile
-- markdown
-- nginx
-- objectivec
-- perl
-- php
-- plaintext
-- properties
-- python
-- r
-- ruby
-- rust
-- scala
-- scss
-- shell
-- sql
-- swift
-- typescript
-- x86asm
-- xml
-- yaml
+your own `.sublime-syntax` file:
+
+<!-- this list is generated by running `cargo run -- gen-syntax-cache --syntaxes-only` -->
+<!-- which outputs a list of names -->
+
+ Name | Extensions
+ -----|-----------
+ Plain Text | txt
+ ASP | asa
+ HTML (ASP) | asp
+ ActionScript | as
+ AppleScript | applescript script editor
+ Batch File | bat cmd
+ NAnt Build File | build
+ C# | cs csx
+ C++ | cpp cc cp cxx c++ C h hh hpp hxx h++ inl ipp
+ C | c h
+ CSS | css css.erb css.liquid
+ Clojure | clj
+ D | d di
+ Diff | diff patch
+ Erlang | erl hrl Emakefile emakefile
+ HTML (Erlang) | yaws
+ Go | go
+ Graphviz (DOT) | dot DOT gv
+ Groovy | groovy gvy gradle
+ HTML | html htm shtml xhtml inc tmpl tpl
+ Haskell | hs
+ Literate Haskell | lhs
+ Java Server Page (JSP) | jsp
+ Java | java bsh
+ JavaDoc | 
+ Java Properties | properties
+ JSON | json sublime-settings sublime-menu sublime-keymap sublime-mousemap sublime-theme sublime-build sublime-project sublime-completions sublime-commands sublime-macro sublime-color-scheme
+ JavaScript | js htc
+ Regular Expressions (Javascript) | 
+ BibTeX | bib
+ LaTeX Log | 
+ LaTeX | tex ltx
+ TeX | sty cls
+ Lisp | lisp cl clisp l mud el scm ss lsp fasl
+ Lua | lua
+ Make Output | 
+ Makefile | make GNUmakefile makefile Makefile OCamlMakefile mak mk
+ Markdown | md mdown markdown markdn
+ MultiMarkdown | 
+ MATLAB | matlab
+ OCaml | ml mli
+ OCamllex | mll
+ OCamlyacc | mly
+ camlp4 | 
+ Objective-C++ | mm M h
+ Objective-C | m h
+ PHP Source | 
+ PHP | php php3 php4 php5 php7 phps phpt phtml
+ Pascal | pas p dpr
+ Perl | pl pm pod t PL
+ Python | py py3 pyw pyi pyx pyx.in pxd pxd.in pxi pxi.in rpy cpy SConstruct Sconstruct sconstruct SConscript gyp gypi Snakefile script
+ Regular Expressions (Python) | 
+ R Console | 
+ R | R r s S Rprofile
+ Rd (R Documentation) | rd
+ HTML (Rails) | rails rhtml erb html.erb
+ JavaScript (Rails) | js.erb
+ Ruby Haml | haml sass
+ Ruby on Rails | rxml builder
+ SQL (Rails) | erbsql sql.erb
+ Regular Expression | re
+ reStructuredText | rst rest
+ Ruby | rb Appfile Appraisals Berksfile Brewfile capfile cgi Cheffile config.ru Deliverfile Fastfile fcgi Gemfile gemspec Guardfile irbrc jbuilder podspec prawn rabl rake Rakefile Rantfile rbx rjs ruby.rail Scanfile simplecov Snapfile thor Thorfile Vagrantfile
+ Cargo Build Results | 
+ Rust | rs
+ SQL | sql ddl dml
+ Scala | scala sbt
+ Bourne Again Shell (bash) | sh bash zsh fish .bash_aliases .bash_completions .bash_functions .bash_login .bash_logout .bash_profile .bash_variables .bashrc .profile .textmate_init
+ Shell-Unix-Generic | 
+ commands-builtin-shell-bash | 
+ HTML (Tcl) | adp
+ Tcl | tcl
+ Textile | textile
+ XML | xml xsd xslt tld dtml rss opml svg
+ YAML | yaml yml sublime-syntax
+ Handlebars | handlebars handlebars.html hbr hbrs hbs hdbs hjs mu mustache rac stache template tmpl]
 
 ## Custom theme
-Like the rest of the theme, the files used for syntax highlighting can be
-overridden with your own.
 
-- ***highlight.js*** normally you shouldn't have to overwrite this file, unless
-  you want to use a more recent version.
-- ***highlight.css*** theme used by highlight.js for syntax highlighting.
+Use the `gen-syntax-cache` command to compile sublime text syntax definitions and textmate theme files for mdbook.
 
-If you want to use another theme for `highlight.js` download it from their
-website, or make it yourself, rename it to `highlight.css` and put it in
-the `theme` folder of your book.
+    mdbook gen-syntax-cache --dest-dir=./theme ./syntax
 
-Now your theme will be used instead of the default theme.
+The `gen-syntax-cache` will process every `tmTheme` file, and generate a similarly-named file in `[dest-dir]/css/syntax`,
+and will also process every `sublime-syntax` file and add it to the `syntaxes.bin` cache. When building your book,
+mdBook will look in the ./theme directory for the `syntaxes.bin` cache and use it, and will also copy across the generated
+CSS files.
+
+mdBook includes, by default, the above standard syntax definitions when generating the `syntaxes.bin` cache. If you want to
+override all of the syntax definitions, pass the `--no-default-syntaxes` option.
 
 ## Hiding code lines
 

src/cmd/gen_syntax_cache.rs

@@ -0,0 +1,97 @@
+use clap::{App, ArgMatches, SubCommand};
+use mdbook::errors::Result;
+use std::env;
+use std::io::Cursor;
+use syntect::dumps::dump_to_file;
+use syntect::highlighting::ThemeSet;
+use syntect::html::css_for_theme_with_class_style;
+use syntect::html::ClassStyle;
+use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
+
+// Create clap subcommand arguments
+pub fn make_subcommand<'a, 'b>() -> App<'a, 'b> {
+    SubCommand::with_name("gen-syntax-cache")
+        .about("Generate syntaxes.bin and css/syntax")
+        .arg_from_usage(
+            "-d, --dest-dir=[dir] 'Output directory for the syntax cache{n}\
+             Relative paths are interpreted relative to the current working directory.{n}\
+             If omitted, mdBook uses `.`.
+             This command outputs files [dir]/syntaxes.bin and [dir]/css/syntax/*.css'",
+        )
+        .arg_from_usage("--syntaxes-only 'Only generate syntaxes.bin, not css/syntax/*.css.'")
+        .arg_from_usage(
+            "--no-default-syntaxes 'Don't include Sublime Text's default open source syntaxes{n}\
+             If included, only syntaxes from [dir] are used.'",
+        )
+        .arg_from_usage("--themes-only 'Only generate themes, not syntaxes.bin.'")
+        .arg_from_usage(
+            "--no-default-themes 'Don't include mdbook's default light, dark, and ayu themes{n}\
+             If included, only themes from [dir] are used.'",
+        )
+        .arg_from_usage(
+            "[dir] 'Root directory for the syntax sources{n}\
+             (Defaults to the Current Directory when omitted)'",
+        )
+}
+
+// Generate Syntax Cache command implementation
+pub fn execute(args: &ArgMatches) -> Result<()> {
+    let src_dir = env::current_dir()
+        .unwrap()
+        .join(&format!("{}/", args.value_of("dir").unwrap_or(".")));
+    let dest_dir = env::current_dir()
+        .unwrap()
+        .join(&format!("{}/", args.value_of("dest-dir").unwrap_or(".")));
+
+    if !args.is_present("themes-only") {
+        let mut builder = if args.is_present("no-default-syntaxes") {
+            SyntaxSetBuilder::new()
+        } else {
+            syntect::dumps::from_binary::<SyntaxSet>(mdbook::theme::SYNTAXES_BIN).into_builder()
+        };
+        builder.add_from_folder(&src_dir, true)?;
+        let set = builder.build();
+        for syntax in set.syntaxes() {
+            info!(
+                "supports syntax: {} [{}]",
+                syntax.name,
+                syntax.file_extensions.join(" ")
+            );
+        }
+        dump_to_file(&set, dest_dir.join("syntaxes.bin"))?;
+    }
+
+    if !args.is_present("syntaxes-only") {
+        let mut builder = ThemeSet::load_from_folder(&src_dir)?;
+        if !args.is_present("no-default-themes") {
+            if !builder.themes.contains_key("light") {
+                let light = ThemeSet::load_from_reader(&mut Cursor::new(
+                    mdbook::theme::SYNTAX_THEME_LIGHT,
+                ))?;
+                builder.themes.insert(String::from("light"), light);
+            }
+            if !builder.themes.contains_key("dark") {
+                let dark =
+                    ThemeSet::load_from_reader(&mut Cursor::new(mdbook::theme::SYNTAX_THEME_DARK))?;
+                builder.themes.insert(String::from("dark"), dark);
+            }
+            if !builder.themes.contains_key("ayu") {
+                let ayu =
+                    ThemeSet::load_from_reader(&mut Cursor::new(mdbook::theme::SYNTAX_THEME_AYU))?;
+                builder.themes.insert(String::from("ayu"), ayu);
+            }
+        }
+        for (name, theme) in builder.themes.iter() {
+            info!("supports theme: {}", name);
+            std::fs::write(
+                dest_dir.join(format!("css/syntax/{}.css", name)),
+                css_for_theme_with_class_style(
+                    theme,
+                    ClassStyle::SpacedPrefixed { prefix: "syn-" },
+                ),
+            )?;
+        }
+    }
+
+    Ok(())
+}

src/cmd/mod.rs

@@ -2,6 +2,8 @@
 
 pub mod build;
 pub mod clean;
+#[cfg(feature = "gen-syntax-cache")]
+pub mod gen_syntax_cache;
 pub mod init;
 #[cfg(feature = "serve")]
 pub mod serve;

src/main.rs

@@ -33,6 +33,8 @@ fn main() {
         #[cfg(feature = "serve")]
         ("serve", Some(sub_matches)) => cmd::serve::execute(sub_matches),
         ("test", Some(sub_matches)) => cmd::test::execute(sub_matches),
+        #[cfg(feature = "gen_syntax_cache")]
+        ("gen-syntax-cache", Some(sub_matches)) => cmd::gen_syntax_cache::execute(sub_matches),
         ("completions", Some(sub_matches)) => (|| {
             let shell: Shell = sub_matches
                 .value_of("shell")
@@ -87,6 +89,8 @@ fn create_clap_app<'a, 'b>() -> App<'a, 'b> {
     let app = app.subcommand(cmd::watch::make_subcommand());
     #[cfg(feature = "serve")]
     let app = app.subcommand(cmd::serve::make_subcommand());
+    #[cfg(feature = "gen-syntax-cache")]
+    let app = app.subcommand(cmd::gen_syntax_cache::make_subcommand());
 
     app
 }