feat: preserve language attribute for code blocks (#247)

* test: add failing test for language attribute

Signed-off-by: Mike Fiedler <[email protected]>

* feat: preserve lang attribute in markdown

In order to allow the `lang` attribute on `pre` code blocks, add it to
the post-syntax highlighting step, and then allow through the sanitizer.

Signed-off-by: Mike Fiedler <[email protected]>

* test: update tests to include preserved `lang`

Signed-off-by: Mike Fiedler <[email protected]>

Co-authored-by: Dustin Ingram <[email protected]>

Author: miketheman

readme_renderer/clean.py

@@ -55,6 +55,7 @@
     "h6": ["align"],
     "code": ["class"],
     "p": ["align", "class"],
+    "pre": ["lang"],
     "ol": ["start"],
     "input": ["type", "checked", "disabled"],
 }

readme_renderer/markdown.py

@@ -116,7 +116,7 @@ def replacer(match: Match[Any]) -> str:
 
         highlighted = pygments.highlight(code, lexer, formatter)
 
-        return f'<pre>{highlighted}</pre>'
+        return f'<pre lang="{lang}">{highlighted}</pre>'
 
     result = code_expr.sub(replacer, html)
 

tests/fixtures/test_CommonMark_008.html

@@ -1,5 +1,5 @@
 <p>Here is some Python code for a <code>Dog</code>:</p>
-<pre><span class="k">class</span> <span class="nc">Dog</span><span class="p">(</span><span class="n">Animal</span><span class="p">):</span>
+<pre lang="python3"><span class="k">class</span> <span class="nc">Dog</span><span class="p">(</span><span class="n">Animal</span><span class="p">):</span>
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
 
@@ -9,7 +9,7 @@
 <span class="n">dog</span> <span class="o">=</span> <span class="n">Dog</span><span class="p">(</span><span class="s1">&#39;Fido&#39;</span><span class="p">)</span>
 </pre>
 <p>and then here is some bash:</p>
-<pre><span class="k">if</span> <span class="o">[</span> <span class="s2">&quot;</span><span class="nv">$1</span><span class="s2">&quot;</span> <span class="o">=</span> <span class="s2">&quot;--help&quot;</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span>
+<pre lang="bash"><span class="k">if</span> <span class="o">[</span> <span class="s2">&quot;</span><span class="nv">$1</span><span class="s2">&quot;</span> <span class="o">=</span> <span class="s2">&quot;--help&quot;</span> <span class="o">]</span><span class="p">;</span> <span class="k">then</span>
     <span class="nb">echo</span> <span class="s2">&quot;OK&quot;</span>
 <span class="k">fi</span>
 </pre>

tests/fixtures/test_GFM_doublequotes.html

@@ -1,11 +1,11 @@
 <p>This is normal text.</p>
 <pre><code>This is code text.
 </code></pre>
-<pre><span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
+<pre lang="python3"><span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
     <span class="sd">&quot;&quot;&quot;This is a docstring.&quot;&quot;&quot;</span>
     <span class="k">pass</span>
 </pre>
-<pre><span class="kd">func</span><span class="w"> </span><span class="nx">ThisIsGo</span><span class="p">(){</span><span class="w"></span>
+<pre lang="go"><span class="kd">func</span><span class="w"> </span><span class="nx">ThisIsGo</span><span class="p">(){</span><span class="w"></span>
 <span class="w">    </span><span class="k">return</span><span class="w"></span>
 <span class="p">}</span><span class="w"></span>
 </pre>

tests/fixtures/test_GFM_highlight.html

@@ -1,10 +1,12 @@
 <p>This is normal text.</p>
 <pre><code>This is code text.
 </code></pre>
-<pre><span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
+<pre lang="python3"><span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
     <span class="k">pass</span>
 </pre>
-<pre><span class="kd">func</span><span class="w"> </span><span class="nx">ThisIsGo</span><span class="p">(){</span><span class="w"></span>
+<pre lang="go"><span class="kd">func</span><span class="w"> </span><span class="nx">ThisIsGo</span><span class="p">(){</span><span class="w"></span>
 <span class="w">    </span><span class="k">return</span><span class="w"></span>
 <span class="p">}</span><span class="w"></span>
 </pre>
+<pre lang="abc">An unknown code fence block
+</pre>

tests/fixtures/test_GFM_highlight.md

@@ -14,3 +14,7 @@ func ThisIsGo(){
     return
 }
 ```
+
+```abc
+An unknown code fence block
+```

tests/fixtures/test_GFM_highlight_default_py.html

@@ -1,4 +1,4 @@
-<pre><span class="k">async</span> <span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
+<pre lang="python3"><span class="k">async</span> <span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
     <span class="k">pass</span>
 
 <span class="nb">print</span><span class="p">(</span><span class="k">await</span> <span class="n">this_is_python</span><span class="p">())</span>

tests/fixtures/test_GFM_malicious_pre.html

@@ -1,5 +1,5 @@
 <p>This is normal text.</p>
-<pre><span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
+<pre lang="python3"><span class="k">def</span> <span class="nf">this_is_python</span><span class="p">():</span>
     <span class="sd">&quot;&quot;&quot;This is a docstring.&quot;&quot;&quot;</span>
     <span class="k">pass</span>
 <span class="o">&lt;</span><span class="n">script</span> <span class="nb">type</span><span class="o">=</span><span class="s2">&quot;text/javascript&quot;</span><span class="o">&gt;</span><span class="n">alert</span><span class="p">(</span><span class="s1">&#39;I am evil.&#39;</span><span class="p">);</span><span class="o">&lt;/</span><span class="n">script</span><span class="o">&gt;</span>