use hyphens: manual

Author: pietroalbini

Cargo.lock

@@ -16,3 +16,4 @@ fs_extra = "1.1.0"
 regex = "1.3"
 sass-rs = "0.2.1"
 chrono = "0.4.13"
+kl-hyphenate = "0.7.2"

Cargo.toml

@@ -1,4 +1,5 @@
 mod blogs;
+mod markdown;
 mod posts;
 
 use crate::blogs::Blog;

hyphenation-en-us.bincode

@@ -0,0 +1,80 @@
+use comrak::{
+    nodes::{AstNode, NodeValue},
+    Arena, ComrakExtensionOptions, ComrakOptions, ComrakRenderOptions,
+};
+use kl_hyphenate::{Hyphenator, Language, Load, Standard};
+use std::error::Error;
+
+const SOFT_HYPHEN: char = '\u{00AD}';
+const HYPHENATION_DICTIONARY: &str = "hyphenation-en-us.bincode";
+
+pub(crate) fn render(input: &str) -> Result<String, Box<dyn Error>> {
+    let options = ComrakOptions {
+        render: ComrakRenderOptions {
+            unsafe_: true, // Allow rendering of raw HTML
+            ..ComrakRenderOptions::default()
+        },
+        extension: ComrakExtensionOptions {
+            header_ids: Some(String::new()),
+            ..ComrakExtensionOptions::default()
+        },
+        ..ComrakOptions::default()
+    };
+
+    let hyphenator = Standard::from_path(Language::EnglishUS, HYPHENATION_DICTIONARY)?;
+
+    let arena = Arena::new();
+    let ast = comrak::parse_document(&arena, input, &options);
+
+    hyphenate(&ast, &hyphenator);
+
+    let mut output = Vec::new();
+    comrak::format_html(&ast, &options, &mut output)?;
+    Ok(String::from_utf8(output)?)
+}
+
+// Pre-compute points inside words where browsers can add hyphens during rendering.
+//
+// Support for the CSS rule `hyphens: auto`, which tells the browser to split words by adding
+// hyphens when there is no space left on the line, is quite low across browsers, preventing us
+// from using it on the blog.
+//
+// A widely supported alternative is the `hyphens: manual` rule, which moves the burden of deciding
+// *where* to break the word to the website. To properly use that rule, the website has to insert
+// the "soft hyphen" unicode character (U+00AD) in every position the browser is allowed to break
+// the word.
+//
+// The following piece of code walks through the Markdown AST adding those characters in every
+// suitable place, thanks to the kl-hyphenate library.
+
+fn hyphenate<'a>(node: &'a AstNode<'a>, hyphenator: &Standard) {
+    match &mut node.data.borrow_mut().value {
+        NodeValue::Text(content) => {
+            if let Ok(string) = std::str::from_utf8(&content) {
+                let hyphenated = add_soft_hyphens(string, hyphenator);
+                *content = hyphenated.as_bytes().to_vec();
+            }
+        }
+        _ => {}
+    }
+    for child in node.children() {
+        hyphenate(child, hyphenator);
+    }
+}
+
+fn add_soft_hyphens(content: &str, hyphenator: &Standard) -> String {
+    let mut output = String::with_capacity(content.len());
+    for (i, word) in content.split(' ').enumerate() {
+        if i != 0 {
+            output.push(' ');
+        }
+        let hyphenated = hyphenator.hyphenate(word);
+        for (j, segment) in hyphenated.into_iter().segments().enumerate() {
+            if j != 0 {
+                output.push(SOFT_HYPHEN);
+            }
+            output.push_str(&segment);
+        }
+    }
+    output
+}

src/main.rs

@@ -1,5 +1,4 @@
 use crate::blogs::Manifest;
-use comrak::{ComrakExtensionOptions, ComrakOptions, ComrakRenderOptions};
 use regex::Regex;
 use serde_derive::{Deserialize, Serialize};
 use std::error::Error;
@@ -63,19 +62,7 @@ impl Post {
             layout,
         } = serde_yaml::from_str(yaml)?;
         // next, the contents. we add + to get rid of the final "---\n\n"
-        let options = ComrakOptions {
-            render: ComrakRenderOptions {
-                unsafe_: true, // Allow rendering of raw HTML
-                ..ComrakRenderOptions::default()
-            },
-            extension: ComrakExtensionOptions {
-                header_ids: Some(String::new()),
-                ..ComrakExtensionOptions::default()
-            },
-            ..ComrakOptions::default()
-        };
-
-        let contents = comrak::markdown_to_html(&contents[end_of_yaml + 5..], &options);
+        let contents = crate::markdown::render(&contents[end_of_yaml + 5..])?;
 
         // finally, the url.
         let mut url = PathBuf::from(&*filename);

src/markdown.rs

@@ -56,6 +56,12 @@ blockquote {
 
 p {
   text-align: justify;
+
+  /* Use manual hyphenation, as automatic hyphenation is not widely
+   * supported (Chrome doesn't implement it on all platforms). */
+  -webkit-hyphens: manual;
+      -ms-hyphens: manual;
+          hyphens: manual;
 }
 
 code {