From 6cfd8c19dd5a93c4e7a8e5132a6abce0791f7442 Mon Sep 17 00:00:00 2001 From: Sytse Reitsma Date: Sun, 26 May 2019 21:03:56 +0200 Subject: [PATCH 1/5] Allow users to insert additional resources from other directories Adding one or more sections like this to book.toml: ```toml [[output.html.additional-resources]] output-dir="img" src="../uml/*.png" ``` Will copy the files to the book output directory. --- Cargo.lock | 9 ++ Cargo.toml | 1 + src/cmd/watch.rs | 23 +++- src/config.rs | 56 +++++++++ src/renderer/html_handlebars/hbs_renderer.rs | 119 ++++++++++++++++++- 5 files changed, 206 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d981de609..f9650ff9bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "aho-corasick" version = "0.6.8" @@ -272,6 +274,11 @@ dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "handlebars" version = "1.0.5" @@ -533,6 +540,7 @@ dependencies = [ "elasticlunr-rs 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "handlebars 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "iron 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1564,6 +1572,7 @@ dependencies = [ "checksum futf 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b" "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum getopts 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0a7292d30132fb5424b354f5dc02512a86e4c516fe544bb7a25e7f266951b797" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" "checksum handlebars 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3623110a77811256820e92df1b3b286f6f44f99d1f77a94b75e262c28d5034f4" "checksum html5ever 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a49d5001dd1bddf042ea41ed4e0a671d50b1bf187e66b349d7ec613bdce4ad90" "checksum html5ever 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b04478cf718862650a0bf66acaf8f2f8c906fbc703f35c916c1f4211b069a364" diff --git a/Cargo.toml b/Cargo.toml index aa5129e47d..c72891a7a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ tempfile = "3.0" itertools = "0.7" shlex = "0.1" toml-query = "0.7" +glob = "0.3.0" # Watch feature notify = { version = "4.0", optional = true } diff --git a/src/cmd/watch.rs b/src/cmd/watch.rs index d800ae7de5..6a0e365c61 100644 --- a/src/cmd/watch.rs +++ b/src/cmd/watch.rs @@ -1,11 +1,12 @@ extern crate notify; +extern crate glob; use self::notify::Watcher; use clap::{App, ArgMatches, SubCommand}; use mdbook::errors::Result; use mdbook::utils; use mdbook::MDBook; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; use std::time::Duration; use {get_book_dir, open}; @@ -47,6 +48,24 @@ pub fn execute(args: &ArgMatches) -> Result<()> { Ok(()) } +fn watch_additional_resources(book: &MDBook, watcher : &mut impl notify::Watcher) { + use self::glob::glob; + use self::notify::RecursiveMode::NonRecursive; + + book.config.html_config() + .and_then(|html_config| html_config.additional_resources) + .map(|additional_resources| { + for res in additional_resources { + let found_files = glob(res.src.as_str()) + .expect("Failed to read glob pattern for additional resource"); + for path in found_files.filter_map(std::result::Result::ok) { + let _ = watcher.watch(&path, NonRecursive); + debug!("Watching {:?}", path); + } + } + }); +} + /// Calls the closure when a book source file is changed, blocking indefinitely. pub fn trigger_on_change(book: &MDBook, closure: F) where @@ -77,6 +96,8 @@ where // Add the book.toml file to the watcher if it exists let _ = watcher.watch(book.root.join("book.toml"), NonRecursive); + watch_additional_resources(book, &mut watcher); + info!("Listening for changes..."); for event in rx.iter() { diff --git a/src/config.rs b/src/config.rs index c6b370949d..93aeac8036 100644 --- a/src/config.rs +++ b/src/config.rs @@ -385,6 +385,41 @@ impl Default for BookConfig { } } + +/// Additional files not normaly copied (like external images etc), which +/// you want to add to the book output. +/// In book.toml: +/// ```toml +/// [[output.html.additional-resources]] +/// output-dir="img" +/// src="../plantuml/*.png" +/// +/// [[output.html.additional-resources]] +/// output-dir="foo" +/// src="bar/*.xml" +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "kebab-case")] +pub struct AdditionalResource { + /// Glob string for the files to add (e.g. "../foo/*.png"), path is + /// relative to book root dir. The files found by the pattern + /// are flattened in the output dir (i.e. the directory structure + /// is not copied, only the files) + pub src: String, + /// Path relative to the book output dir + pub output_dir: PathBuf, +} + +impl Default for AdditionalResource { + fn default() -> AdditionalResource { + AdditionalResource { + src: String::from(""), + output_dir: PathBuf::from("src"), + } + } +} + + /// Configuration for the build procedure. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] @@ -447,6 +482,9 @@ pub struct HtmlConfig { /// FontAwesome icon class to use for the Git repository link. /// Defaults to `fa-github` if `None`. pub git_repository_icon: Option, + + /// The additional resources to copy + pub additional_resources: Option> } impl HtmlConfig { @@ -582,6 +620,14 @@ mod tests { git-repository-url = "https://foo.com/" git-repository-icon = "fa-code-fork" + [[output.html.additional-resources]] + src="foo*.*" + output-dir="bar" + + [[output.html.additional-resources]] + src="chuck*.*" + output-dir="norris" + [output.html.playpen] editable = true editor = "ace" @@ -621,6 +667,16 @@ mod tests { playpen: playpen_should_be, git_repository_url: Some(String::from("https://foo.com/")), git_repository_icon: Some(String::from("fa-code-fork")), + additional_resources: Some(vec![ + AdditionalResource { + src: String::from("foo*.*"), + output_dir: PathBuf::from("bar"), + }, + AdditionalResource { + src: String::from("chuck*.*"), + output_dir: PathBuf::from("norris"), + } + ]), ..Default::default() }; diff --git a/src/renderer/html_handlebars/hbs_renderer.rs b/src/renderer/html_handlebars/hbs_renderer.rs index b5ef228d34..fd5f69bd2b 100644 --- a/src/renderer/html_handlebars/hbs_renderer.rs +++ b/src/renderer/html_handlebars/hbs_renderer.rs @@ -1,5 +1,5 @@ use book::{Book, BookItem}; -use config::{Config, HtmlConfig, Playpen}; +use config::{Config, HtmlConfig, AdditionalResource, Playpen}; use errors::*; use renderer::html_handlebars::helpers; use renderer::{RenderContext, Renderer}; @@ -15,6 +15,9 @@ use handlebars::Handlebars; use regex::{Captures, Regex}; use serde_json; +extern crate glob; +use self::glob::glob; + #[derive(Default)] pub struct HtmlHandlebars; @@ -363,11 +366,67 @@ impl Renderer for HtmlHandlebars { // Copy all remaining files utils::fs::copy_files_except_ext(&src_dir, &destination, true, &["md"])?; + copy_additional_resources(&html_config.additional_resources, &destination)?; Ok(()) } } +/// Copy the additional resources specified in the config +fn copy_additional_resources( + resources: &Option>, + destination: &PathBuf, +) -> Result<()> { + + match resources { + Some(additional_resources) => { + for res in additional_resources { + copy_additional_resource(&res, &destination)?; + } + } + None => (), //Optional, so no issue here + } + + Ok(()) +} + +/// Copy the files from a single AdditionalResource entry in config to the book dir +/// The found files are copied by name only, the original directory structure is +/// flattened +fn copy_additional_resource(res: &AdditionalResource, destination: &PathBuf) -> Result<()> { + let dest_dir = destination.join(&res.output_dir); + + match fs::create_dir_all(&dest_dir) { + Ok(_) => debug!("Created: {}", dest_dir.display()), + Err(e) => { + error!( + "Failed to create output directory {} for additional resources.", + dest_dir.display() + ); + return Err(e.into()); + } + } + + let found_files = glob(res.src.as_str()) + .expect("Failed to read glob pattern for additional resource"); + for path in found_files.filter_map(std::result::Result::ok) { + let file_name = path.file_name().unwrap(); + let dest_file = dest_dir.join(file_name); + + match fs::copy(&path, &dest_file) { + Ok(_) => debug!("Copied {} to {}", path.display(), dest_file.display()), + Err(e) => warn!( + "Failed to copy {} to {} ({})", + path.display(), + dest_file.display(), + e + ), + } + } + + Ok(()) +} + fn make_data( root: &Path, book: &Book, @@ -669,4 +728,62 @@ mod tests { assert_eq!(got, should_be); } } + #[test] + fn test_copy_additional_resource_fails_to_create_output_dir() { + use tempfile::NamedTempFile; + + // Create a file with the same name as the directory we're + // about to create. This will prevent the system from creating + // the directory + let mask_file = NamedTempFile::new().expect("Failed to create dir mask file"); + + let destination = PathBuf::from(&mask_file.path()); + let resource = AdditionalResource { + src: String::from("*.foo"), + output_dir: PathBuf::from(&destination), + }; + + match copy_additional_resource(&resource, &destination) { + Ok(_) => assert!( + false, + "Expected a failure when creating the output dir" + ), + //Error is OS dependant, so not much use in checking the error text + Err(_e) => (), + } + } + + #[test] + fn test_copy_additional_resource() { + use tempfile::tempdir; + + let src_dir = tempdir().unwrap(); + let output_dir = tempdir().unwrap(); + + let destination = PathBuf::from(&output_dir.path()); + let resource = AdditionalResource { + src: String::from(src_dir.path().join("*.txt").to_str().unwrap()), + output_dir: PathBuf::from(&destination), + }; + + //Create some files + fs::File::create(src_dir.path().join("i_will_not_be_copied.doc")).unwrap(); + fs::File::create(src_dir.path().join("i_will_be_copied.txt")).unwrap(); + fs::File::create(src_dir.path().join("i_will_be_copied_too.txt")).unwrap(); + + match copy_additional_resource(&resource, &destination) { + Ok(_) => { + //Just test the most likely candidates have been copied + let test_file = destination.join("i_will_be_copied.txt"); + assert!(test_file.is_file(), "Expected 'i_will_be_copied.txt' to be copied"); + + let test_file = destination.join("i_will_be_copied_too.txt"); + assert!(test_file.is_file(), "Expected 'i_will_be_copied_too.txt' to be copied"); + + let test_file = destination.join("i_will_not_be_copied.doc"); + assert!(!test_file.is_file(), "Did not expect 'i_will_not_be_copied.doc' to be copied"); + }, + Err(e) => assert!(false, "Failed to copy additional resources ({})", e) + } + } } From cab921f490c32429b588aac0433fac98178caead Mon Sep 17 00:00:00 2001 From: Sytse Reitsma Date: Sun, 26 May 2019 21:51:28 +0200 Subject: [PATCH 2/5] fixedbuild --- src/cmd/watch.rs | 5 ++--- src/renderer/html_handlebars/hbs_renderer.rs | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/cmd/watch.rs b/src/cmd/watch.rs index f38a2046bb..e58a1652e7 100644 --- a/src/cmd/watch.rs +++ b/src/cmd/watch.rs @@ -1,5 +1,4 @@ extern crate notify; -extern crate glob; use self::notify::Watcher; use crate::{get_book_dir, open}; use clap::{App, ArgMatches, SubCommand}; @@ -10,6 +9,7 @@ use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; use std::thread::sleep; use std::time::Duration; +use glob::glob; // Create clap subcommand arguments pub fn make_subcommand<'a, 'b>() -> App<'a, 'b> { @@ -51,8 +51,7 @@ pub fn execute(args: &ArgMatches) -> Result<()> { } fn watch_additional_resources(book: &MDBook, watcher : &mut impl notify::Watcher) { - use self::glob::glob; - use self::notify::RecursiveMode::NonRecursive; + use notify::RecursiveMode::NonRecursive; book.config.html_config() .and_then(|html_config| html_config.additional_resources) diff --git a/src/renderer/html_handlebars/hbs_renderer.rs b/src/renderer/html_handlebars/hbs_renderer.rs index 30a7ce98df..17cc828c95 100644 --- a/src/renderer/html_handlebars/hbs_renderer.rs +++ b/src/renderer/html_handlebars/hbs_renderer.rs @@ -1,5 +1,5 @@ use crate::book::{Book, BookItem}; -use config::{Config, HtmlConfig, AdditionalResource, Playpen}; +use crate::config::{Config, HtmlConfig, AdditionalResource, Playpen}; use crate::errors::*; use crate::renderer::html_handlebars::helpers; use crate::renderer::{RenderContext, Renderer}; @@ -13,9 +13,8 @@ use std::path::{Path, PathBuf}; use handlebars::Handlebars; use regex::{Captures, Regex}; +use glob::glob; -extern crate glob; -use self::glob::glob; #[derive(Default)] pub struct HtmlHandlebars; From fa070fb4f32054fd60c9f212471590efe6b80b1c Mon Sep 17 00:00:00 2001 From: Sytse Reitsma Date: Mon, 27 May 2019 20:44:36 +0200 Subject: [PATCH 3/5] Corrected rustfmt errors --- src/cmd/watch.rs | 7 ++-- src/renderer/html_handlebars/hbs_renderer.rs | 34 +++++++++++--------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/cmd/watch.rs b/src/cmd/watch.rs index e58a1652e7..502fd7fd6c 100644 --- a/src/cmd/watch.rs +++ b/src/cmd/watch.rs @@ -2,6 +2,7 @@ extern crate notify; use self::notify::Watcher; use crate::{get_book_dir, open}; use clap::{App, ArgMatches, SubCommand}; +use glob::glob; use mdbook::errors::Result; use mdbook::utils; use mdbook::MDBook; @@ -9,7 +10,6 @@ use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; use std::thread::sleep; use std::time::Duration; -use glob::glob; // Create clap subcommand arguments pub fn make_subcommand<'a, 'b>() -> App<'a, 'b> { @@ -50,10 +50,11 @@ pub fn execute(args: &ArgMatches) -> Result<()> { Ok(()) } -fn watch_additional_resources(book: &MDBook, watcher : &mut impl notify::Watcher) { +fn watch_additional_resources(book: &MDBook, watcher: &mut impl notify::Watcher) { use notify::RecursiveMode::NonRecursive; - book.config.html_config() + book.config + .html_config() .and_then(|html_config| html_config.additional_resources) .map(|additional_resources| { for res in additional_resources { diff --git a/src/renderer/html_handlebars/hbs_renderer.rs b/src/renderer/html_handlebars/hbs_renderer.rs index 17cc828c95..72f0b73f49 100644 --- a/src/renderer/html_handlebars/hbs_renderer.rs +++ b/src/renderer/html_handlebars/hbs_renderer.rs @@ -1,5 +1,5 @@ use crate::book::{Book, BookItem}; -use crate::config::{Config, HtmlConfig, AdditionalResource, Playpen}; +use crate::config::{AdditionalResource, Config, HtmlConfig, Playpen}; use crate::errors::*; use crate::renderer::html_handlebars::helpers; use crate::renderer::{RenderContext, Renderer}; @@ -11,10 +11,9 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; +use glob::glob; use handlebars::Handlebars; use regex::{Captures, Regex}; -use glob::glob; - #[derive(Default)] pub struct HtmlHandlebars; @@ -379,7 +378,6 @@ fn copy_additional_resources( resources: &Option>, destination: &PathBuf, ) -> Result<()> { - match resources { Some(additional_resources) => { for res in additional_resources { @@ -409,8 +407,8 @@ fn copy_additional_resource(res: &AdditionalResource, destination: &PathBuf) -> } } - let found_files = glob(res.src.as_str()) - .expect("Failed to read glob pattern for additional resource"); + let found_files = + glob(res.src.as_str()).expect("Failed to read glob pattern for additional resource"); for path in found_files.filter_map(std::result::Result::ok) { let file_name = path.file_name().unwrap(); let dest_file = dest_dir.join(file_name); @@ -743,10 +741,7 @@ mod tests { }; match copy_additional_resource(&resource, &destination) { - Ok(_) => assert!( - false, - "Expected a failure when creating the output dir" - ), + Ok(_) => assert!(false, "Expected a failure when creating the output dir"), //Error is OS dependant, so not much use in checking the error text Err(_e) => (), } @@ -774,15 +769,24 @@ mod tests { Ok(_) => { //Just test the most likely candidates have been copied let test_file = destination.join("i_will_be_copied.txt"); - assert!(test_file.is_file(), "Expected 'i_will_be_copied.txt' to be copied"); + assert!( + test_file.is_file(), + "Expected 'i_will_be_copied.txt' to be copied" + ); let test_file = destination.join("i_will_be_copied_too.txt"); - assert!(test_file.is_file(), "Expected 'i_will_be_copied_too.txt' to be copied"); + assert!( + test_file.is_file(), + "Expected 'i_will_be_copied_too.txt' to be copied" + ); let test_file = destination.join("i_will_not_be_copied.doc"); - assert!(!test_file.is_file(), "Did not expect 'i_will_not_be_copied.doc' to be copied"); - }, - Err(e) => assert!(false, "Failed to copy additional resources ({})", e) + assert!( + !test_file.is_file(), + "Did not expect 'i_will_not_be_copied.doc' to be copied" + ); + } + Err(e) => assert!(false, "Failed to copy additional resources ({})", e), } } } From f27c44cd5ec538ed39ddb5a08f550f7a92017087 Mon Sep 17 00:00:00 2001 From: Sytse Reitsma Date: Thu, 30 May 2019 11:01:31 +0200 Subject: [PATCH 4/5] Applied rustfmt --- src/config.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/config.rs b/src/config.rs index cc3e2fe538..eff0505d4c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -388,7 +388,6 @@ impl Default for BookConfig { } } - /// Additional files not normaly copied (like external images etc), which /// you want to add to the book output. /// In book.toml: @@ -422,7 +421,6 @@ impl Default for AdditionalResource { } } - /// Configuration for the build procedure. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(default, rename_all = "kebab-case")] @@ -487,7 +485,7 @@ pub struct HtmlConfig { pub git_repository_icon: Option, /// The additional resources to copy - pub additional_resources: Option> + pub additional_resources: Option>, } impl HtmlConfig { @@ -670,15 +668,15 @@ mod tests { git_repository_url: Some(String::from("https://foo.com/")), git_repository_icon: Some(String::from("fa-code-fork")), additional_resources: Some(vec![ - AdditionalResource { - src: String::from("foo*.*"), - output_dir: PathBuf::from("bar"), - }, - AdditionalResource { - src: String::from("chuck*.*"), - output_dir: PathBuf::from("norris"), - } - ]), + AdditionalResource { + src: String::from("foo*.*"), + output_dir: PathBuf::from("bar"), + }, + AdditionalResource { + src: String::from("chuck*.*"), + output_dir: PathBuf::from("norris"), + }, + ]), ..Default::default() }; From 5355d819399eb4dbc7b442d84ad9fd4709356f49 Mon Sep 17 00:00:00 2001 From: Sytse Reitsma Date: Sat, 1 Jun 2019 09:56:55 +0200 Subject: [PATCH 5/5] Documented new feature in manual --- book-example/src/format/config.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/book-example/src/format/config.md b/book-example/src/format/config.md index 3f40dceae9..4d2fe6fff8 100644 --- a/book-example/src/format/config.md +++ b/book-example/src/format/config.md @@ -161,6 +161,10 @@ The following configuration options are available: - **additional-js:** If you need to add some behaviour to your book without removing the current behaviour, you can specify a set of JavaScript files that will be loaded alongside the default one. +- **additional-resources:** If you need to include some additional resources, + like for instance image files, that are located outside of the book + directory, you can specify one or more additional-resources sections to + include these. - **no-section-label:** mdBook by defaults adds section label in table of contents column. For example, "1.", "2.1". Set this option to true to disable those labels. Defaults to `false`. @@ -172,6 +176,16 @@ The following configuration options are available: - **git-repository-icon:** The FontAwesome icon class to use for the git repository link. Defaults to `fa-github`. +Available configuration options for the `[[output.html.additional-resources]]` +table array: + +- **output-dir:** The directory to copy the files to (relative to book output + directory). The directory is created recursively. +- **src:** The glob pattern to use when locating the additional resource files. + +Note that the found files are flattened in the output dircetory, meaning the +original directory structure is lost. + Available configuration options for the `[output.html.playpen]` table: - **editable:** Allow editing the source code. Defaults to `false`. @@ -238,6 +252,14 @@ boost-paragraph = 1 expand = true heading-split-level = 3 copy-js = true + +[[output.html.additional-resources]] +output-dir="img" +src="../img/*.png" + +[[output.html.additional-resources]] +output-dir="img/project" +src="../project/uml/*.png" ``` ### Custom Renderers