Remove support for embedded JSON-LD inside XML comments and the use of script escape sequences. This supports w3c/json-ld-syntax#97.

Author: gkellogg

common/extract-examples.rb

@@ -15,6 +15,7 @@
 require 'fileutils'
 require 'colorize'
 require 'yaml'
+require 'cgi'
 
 PREFIXES = {
   dc:     "http://purl.org/dc/terms/",
@@ -49,8 +50,8 @@
 # Remove highlighting and commented out sections
 def justify(str)
   str = str.
-    sub(/^\s*<!--\s*$/, '').
-    sub(/^\s*-->\s*$/, '').
+    gsub(/^\s*<!--\s*$/, '').
+    gsub(/^\s*-->\s*$/, '').
     gsub('****', '').
     gsub(/####([^#]*)####/, '')
 
@@ -222,7 +223,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
         examples[title] = {
           title: title,
           filename: fn,
-          content: content,
+          content: content.to_s.gsub(/^\s*< !\s*-\s*-/, '<!--').gsub(/-\s*- >/, '-->'),
           content_type: element.attr('data-content-type'),
           number: example_number,
           ext: ext,
@@ -302,6 +303,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
     # Perform example syntactic validation based on extension
     case ex[:ext]
     when 'json', 'jsonld', 'jsonldf'
+      content = CGI.unescapeHTML(content)
       begin
         ::JSON.parse(content)
       rescue JSON::ParserError => exception
@@ -325,22 +327,16 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
         ex[:base] = html_base.to_s if html_base
 
         script_content = doc.at_xpath(xpath)
-        if script_content
-          # Remove (faked) XML comments and unescape sequences
-          content = script_content
-            .inner_html
-            .sub(/^\s*< !\s*-\s*-/, '')
-            .sub(/-\s*- >\s*$/, '')
-            .gsub(/&lt;/, '<')
-        end
-          
+        
+        # Remove (faked) XML comments and unescape sequences
+        content = CGI.unescapeHTML(script_content.inner_html) if script_content          
       rescue Nokogiri::XML::SyntaxError => exception
         errors << "Example #{ex[:number]} at line #{ex[:line]} parse error: #{exception.message}"
         $stdout.write "F".colorize(:red)
         next
       end
     when 'table'
-      # already in parsed form
+      content = Nokogiri::HTML.parse(content)
     when 'ttl', 'trig'
       begin
         reader_errors = []
@@ -443,10 +439,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
       # Set argument to referenced content to be parsed
       args[0] = if examples[ex[:result_for]][:ext] == 'html' && method == :expand
         # If we are expanding, and the reference is HTML, find the first script element.
-        doc = Nokogiri::HTML.parse(
-          examples[ex[:result_for]][:content]
-          .sub(/^\s*< !\s*-\s*-/, '')
-          .sub(/-\s*- >\s*$/, ''))
+        doc = Nokogiri::HTML.parse(examples[ex[:result_for]][:content])
 
         # Get base from document, if present
         html_base = doc.at_xpath('/html/head/base/@href')
@@ -458,15 +451,10 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
           $stdout.write "F".colorize(:red)
           next
         end
-        StringIO.new(script_content
-          .inner_html
-          .gsub(/&lt;/, '<'))
+        StringIO.new(CGI.unescapeHTML(script_content.inner_html))
       elsif examples[ex[:result_for]][:ext] == 'html' && ex[:target]
         # Only use the targeted script
-        doc = Nokogiri::HTML.parse(
-          examples[ex[:result_for]][:content]
-          .sub(/^\s*< !\s*-\s*-/, '')
-          .sub(/-\s*- >\s*$/, ''))
+        doc = Nokogiri::HTML.parse(examples[ex[:result_for]][:content])
         script_content = doc.at_xpath(xpath)
         unless script_content
           errors << "Example #{ex[:number]} at line #{ex[:line]} references example #{ex[:result_for].inspect} with no JSON-LD script element"
@@ -565,7 +553,7 @@ def save_example(examples:, element:, title:, example_number:, error:, warn:)
           $stderr.puts "expected:\n" + expected.to_trig if verbose
         when 'table'
           expected = begin
-            table_to_dataset(content)
+            table_to_dataset(content.xpath('/html/body/table'))
           rescue
             errors << "Example #{ex[:number]} at line #{ex[:line]} raised error reading table: #{$!}"
             RDF::Dataset.new

index.html

@@ -4671,40 +4671,15 @@ <h3>Extract Script Content Algorithm</h3>
 
     <section class="informative">
       <h4>Overview</h4>
-      <p>As a <a data-cite="HTML52/semantics-scripting.html#data-block">data block</a> 
-        may be inside a comment, and may be escaped, the algorithm extracts the JSON from any comment,
-        removes REVERSE SOLIDUS escapes,
-        and reverses <a data-cite="HTML5/syntax.html#character-references">HTML Character references</a>.
+      <p>The algorithm reverses <a data-cite="HTML5/syntax.html#character-references">HTML Character references</a>.
     </section>
 
     <section>
       <h4>Algorithm</h4>
       <p>The algorithm takes a single required input variable: <var>source</var>,
         the <a data-cite="DOM#dom-node-textcontent">textContent</a> of an HTML <a data-cite="HTML52/semantics-scripting.html#the-script-element">script element</a>.</p>
-      <p>For the purpose of this algorithm, the following tokens are defined in [[ABNF]]:</p>
-
-      <pre class="nohighlight">
-  <dfn>space-character</dfn> = %20 ; SPACE
-                  / %09 ; CHARACTER TABULATION (tab)
-                  / %0A ; LINE FEED (LF)
-                  / %0C ; FORM FEED (FF)
-                  / %0D ; CARRIAGE RETURN (CR)
-  <dfn>comment-open</dfn>    = *<a>space-character</a> <code>"&lt;!--"</code> *<a>space-character</a>
-  <dfn>comment-close</dfn>   = *<a>space-character</a> <code>"--&gt;"</code> *<a>space-character</a>
-      </pre>
 
       <ol>
-      <li>If <var>source</var> begins with <a>comment-open</a> and ends with <a>comment-close</a>,
-        remove those sequences from <var>source</var>.</li>
-      <li>If <var>source</var> contains <a>comment-open</a> or <a>comment-close</a>,
-        an <a data-link-for="JsonLdErrorCode">invalid script element</a> has been detected, and processing is aborted.</li>
-      <li>For all occurances of the any of the character sequences
-        <code>&lt;\script</code>,
-        <code>&lt;\/script</code>,
-        <code>&lt;\!--</code>,
-        or <code>--\&gt;</code>
-        in <var>source</var> using a case-insenstive match,
-        replace the sequence with the equivalent sequence excluding the REVERSE SOLIDUS (<code>\</code>).</li>
       <li>For all occurances of a <a data-cite="HTML5/syntax.html#character-references">HTML Character reference</a> in <var>source</var>,
         replace the sequence with the equivalent Unicode character as defined
         in <a data-cite="HTML52/syntax.html#named-character-references">Named character references</a> in [[HTML52]].</li>

tests/expand-manifest.jsonld

@@ -1353,22 +1353,6 @@
       "input": "expand/h007-in.html",
       "expect": "expand/h007-out.jsonld",
       "option": {"specVersion": "json-ld-1.1", "extractAllScripts": true}
-    }, {
-      "@id": "#th008",
-      "@type": ["jld:PositiveEvaluationTest", "jld:ExpandTest"],
-      "name": "Expands embedded JSON-LD script element with comments",
-      "purpose": "Tests embedded JSON-LD in HTML with comments",
-      "input": "expand/h008-in.html",
-      "expect": "expand/h008-out.jsonld",
-      "option": {"specVersion": "json-ld-1.1"}
-    }, {
-      "@id": "#th009",
-      "@type": ["jld:PositiveEvaluationTest", "jld:ExpandTest"],
-      "name": "Expands embedded JSON-LD script element with escaped tokens",
-      "purpose": "Tests embedded JSON-LD in HTML with escapes",
-      "input": "expand/h009-in.html",
-      "expect": "expand/h009-out.jsonld",
-      "option": {"specVersion": "json-ld-1.1"}
     }, {
       "@id": "#th010",
       "@type": ["jld:PositiveEvaluationTest", "jld:ExpandTest"],