Reduce in-memory size of named entities data from ~8.5Mb to ~250Kb (closes inikulin#52)

Author: inikulin

.eslintignore

@@ -1,3 +1,3 @@
-lib/tokenizer/named_entity_trie.js
+lib/tokenizer/named_entity_data.js
 test/benchmark/node_modules/**/*.js
 test/memory_benchmark/node_modules/**/*.js

Gulpfile.js

@@ -11,63 +11,13 @@ var fork = require('child_process').fork,
     through = require('through2'),
     concat = require('gulp-concat'),
     jsdoc = require('gulp-jsdoc-to-markdown'),
-    insert = require('gulp-insert');
+    insert = require('gulp-insert'),
+    generateNamedEntityData = require('./scripts/generate_named_entity_data'),
+    generateParserFeedbackTest = require('./scripts/generate_parser_feedback_test');
 
 
-gulp.task('generate-trie', function () {
-    function createTrie(entitiesData) {
-        return Object.keys(entitiesData).reduce(function (trie, entity) {
-            var resultCp = entitiesData[entity].codepoints;
-
-            entity = entity.replace(/^&/, '');
-
-            var entityLength = entity.length,
-                last = entityLength - 1,
-                leaf = trie;
-
-            for (var i = 0; i < entityLength; i++) {
-                var key = entity.charCodeAt(i);
-
-                if (!leaf[key])
-                    leaf[key] = {};
-
-                if (i === last)
-                    leaf[key].c = resultCp;
-
-                else {
-                    if (!leaf[key].l)
-                        leaf[key].l = {};
-
-                    leaf = leaf[key].l;
-                }
-            }
-
-            return trie;
-        }, {});
-    }
-
-    function trieCodeGen(file, encoding, callback) {
-        var entitiesData = JSON.parse(file.contents.toString()),
-            trie = createTrie(entitiesData),
-            out = '\'use strict\';\n\n' +
-                  '//NOTE: this file contains auto-generated trie structure that is used for named entity references consumption\n' +
-                  '//(see: http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#tokenizing-character-references and\n' +
-                  '//http://www.whatwg.org/specs/web-apps/current-work/multipage/named-character-references.html#named-character-references)\n' +
-                  'module.exports = ' + JSON.stringify(trie).replace(/"/g, '') + ';\n';
-
-
-        file.contents = new Buffer(out);
-
-        callback(null, file);
-    }
-
-    return download('https://html.spec.whatwg.org/multipage/entities.json')
-        .pipe(through.obj(trieCodeGen))
-        .pipe(rename('named_entity_trie.js'))
-        .pipe(gulp.dest('lib/tokenizer'));
-});
-
-gulp.task('generate-api-reference', function () {
+// Docs
+gulp.task('update-api-reference', function () {
     return gulp
         .src('lib/**/*.js')
         .pipe(concat('05_api_reference.md'))
@@ -76,13 +26,15 @@ gulp.task('generate-api-reference', function () {
         .pipe(gulp.dest('docs'));
 });
 
-gulp.task('docs', ['generate-api-reference'], function () {
+gulp.task('docs', ['update-api-reference'], function () {
     return gulp
         .src('docs/*.md')
         .pipe(concat('index.md'))
         .pipe(gulp.dest('docs/build'));
 });
 
+
+// Benchmarks
 gulp.task('install-upstream-parse5', function () {
     return gulp
         .src('test/benchmark/package.json')
@@ -112,99 +64,57 @@ gulp.task('named-entity-data-memory-benchmark', function (done) {
     fork('./test/memory_benchmark/named_entity_data').once('close', done);
 });
 
+
+// Test
 gulp.task('lint', function () {
     return gulp
         .src([
             'lib/**/*.js',
             'test/**/*.js',
+            'scripts/**/*.js',
             'Gulpfile.js'
         ])
         .pipe(eslint())
         .pipe(eslint.format())
         .pipe(eslint.failAfterError());
 });
 
-gulp.task('update-feedback-tests', function () {
-    var Parser = require('./lib/Parser');
-    var Tokenizer = require('./lib/tokenizer');
-    var defaultTreeAdapter = require('./lib/tree_adapters/default');
-    var testUtils = require('./test/test_utils');
-
-    function appendToken(dest, token) {
-        switch (token.type) {
-            case Tokenizer.EOF_TOKEN:
-                return false;
-            case Tokenizer.NULL_CHARACTER_TOKEN:
-            case Tokenizer.WHITESPACE_CHARACTER_TOKEN:
-                token.type = Tokenizer.CHARACTER_TOKEN;
-            /* falls through */
-            case Tokenizer.CHARACTER_TOKEN:
-                if (dest.length > 0 && dest[dest.length - 1].type === Tokenizer.CHARACTER_TOKEN) {
-                    dest[dest.length - 1].chars += token.chars;
-                    return true;
-                }
-                break;
-        }
-        dest.push(token);
-        return true;
-    }
-
-    function collectParserTokens(html) {
-        var tokens = [];
-        var parser = new Parser();
-
-        parser._processInputToken = function (token) {
-            Parser.prototype._processInputToken.call(this, token);
-
-            // Needed to split attributes of duplicate <html> and <body>
-            // which are otherwise merged as per tree constructor spec
-            if (token.type === Tokenizer.START_TAG_TOKEN)
-                token.attrs = token.attrs.slice();
-
-            appendToken(tokens, token);
-        };
-
-        parser.parse(html);
-
-        return tokens.map(testUtils.convertTokenToHtml5Lib);
-    }
+gulp.task('test', ['lint'], function () {
+    return gulp
+        .src('test/fixtures/*_test.js')
+        .pipe(mocha({
+            ui: 'exports',
+            reporter: 'progress',
+            timeout: typeof v8debug === 'undefined' ? 20000 : Infinity // NOTE: disable timeouts in debug
+        }));
+});
+
 
+// Scripts
+gulp.task('update-feedback-tests', function () {
     return gulp
         .src(['test/data/tree_construction/*.dat', 'test/data/tree_construction_regression/*.dat'])
         .pipe(through.obj(function (file, encoding, callback) {
-            var tests = testUtils.parseTreeConstructionTestData(
-                file.contents.toString(),
-                defaultTreeAdapter
-            );
-
-            var out = {
-                tests: tests.filter(function (test) {
-                    return !test.fragmentContext; // TODO
-                }).map(function (test) {
-                    var input = test.input;
-
-                    return {
-                        description: testUtils.addSlashes(input),
-                        input: input,
-                        output: collectParserTokens(input)
-                    };
-                })
-            };
-
-            file.contents = new Buffer(JSON.stringify(out, null, 4));
+            var test = generateParserFeedbackTest(file.contents.toString());
+
+            file.contents = new Buffer(test);
 
             callback(null, file);
         }))
         .pipe(rename({extname: '.test'}))
         .pipe(gulp.dest('test/data/parser_feedback'));
 });
 
-gulp.task('test', ['lint'], function () {
-    return gulp
-        .src('test/fixtures/*_test.js')
-        .pipe(mocha({
-            ui: 'exports',
-            reporter: 'progress',
-            timeout: typeof v8debug === 'undefined' ? 20000 : Infinity // NOTE: disable timeouts in debug
-        }));
+
+gulp.task('update-named-entities-data', function () {
+    return download('https://html.spec.whatwg.org/multipage/entities.json')
+        .pipe(through.obj(function (file, encoding, callback) {
+            var entitiesData = JSON.parse(file.contents.toString());
+
+            file.contents = new Buffer(generateNamedEntityData(entitiesData));
+
+            callback(null, file);
+        }))
+        .pipe(rename('named_entity_data.js'))
+        .pipe(gulp.dest('lib/tokenizer'));
 });

lib/tokenizer/index.js

@@ -3,7 +3,7 @@
 var Preprocessor = require('./preprocessor'),
     locationInfoMixin = require('../location_info/tokenizer_mixin'),
     UNICODE = require('../common/unicode'),
-    NAMED_ENTITY_TRIE = require('./named_entity_trie');
+    neTree = require('./named_entity_data');
 
 //Aliases
 var $ = UNICODE.CODE_POINTS,
@@ -18,6 +18,13 @@ var NUMERIC_ENTITY_REPLACEMENTS = {
     0x9A: 0x0161, 0x9B: 0x203A, 0x9C: 0x0153, 0x9D: 0x009D, 0x9E: 0x017E, 0x9F: 0x0178
 };
 
+// Named entity tree flags
+var HAS_DATA_FLAG = 1 << 0;
+var DATA_DUPLET_FLAG = 1 << 1;
+var HAS_BRANCHES_FLAG = 1 << 2;
+var MAX_BRANCH_MARKER_VALUE = HAS_DATA_FLAG | DATA_DUPLET_FLAG | HAS_BRANCHES_FLAG;
+
+
 //States
 var DATA_STATE = 'DATA_STATE',
     CHARACTER_REFERENCE_IN_DATA_STATE = 'CHARACTER_REFERENCE_IN_DATA_STATE',
@@ -142,6 +149,29 @@ function toAsciiLowerChar(cp) {
     return String.fromCharCode(toAsciiLowerCodePoint(cp));
 }
 
+function findNamedEntityTreeBranch(nodeIx, cp) {
+    var branchCount = neTree[++nodeIx],
+        lo = ++nodeIx,
+        hi = lo + branchCount - 1;
+
+    while (lo <= hi) {
+        var mid = lo + hi >>> 1,
+            midCp = neTree[mid];
+
+        if (midCp < cp)
+            lo = mid + 1;
+
+        else if (midCp > cp)
+            hi = mid - 1;
+
+        else
+            return neTree[mid + branchCount];
+    }
+
+    return -1;
+}
+
+
 //Tokenizer
 var Tokenizer = module.exports = function (options) {
     this.preprocessor = new Preprocessor();
@@ -482,33 +512,48 @@ Tokenizer.prototype._consumeNumericEntity = function (isHex) {
     return referencedCp;
 };
 
-Tokenizer.prototype._consumeNamedEntity = function (startCp, inAttr) {
+// NOTE: for the details on this algorithm see
+// https://github.com/inikulin/parse5/tree/master/scripts/generate_named_entity_data/README.md
+Tokenizer.prototype._consumeNamedEntity = function (inAttr) {
     var referencedCodePoints = null,
-        entityCodePointsCount = 0,
-        cp = startCp,
-        leaf = NAMED_ENTITY_TRIE[cp],
-        consumedCount = 1,
+        referenceSize = 0,
+        cp = null,
+        consumedCount = 0,
         semicolonTerminated = false;
 
-    for (; leaf && cp !== $.EOF; cp = this._consume(), consumedCount++, leaf = leaf.l && leaf.l[cp]) {
-        if (leaf.c) {
-            //NOTE: we have at least one named reference match. But we don't stop lookup at this point,
-            //because longer matches still can be found (e.g. '&not' and '&notin;') except the case
-            //then found match is terminated by semicolon.
-            referencedCodePoints = leaf.c;
-            entityCodePointsCount = consumedCount;
+    for (var i = 0; i > -1;) {
+        var current = neTree[i],
+            inNode = current < MAX_BRANCH_MARKER_VALUE,
+            nodeWithData = inNode && current & HAS_DATA_FLAG;
+
+        if (nodeWithData) {
+            referencedCodePoints = current & DATA_DUPLET_FLAG ? [neTree[++i], neTree[++i]] : [neTree[++i]];
+            referenceSize = consumedCount;
 
             if (cp === $.SEMICOLON) {
                 semicolonTerminated = true;
                 break;
             }
         }
+
+        cp = this._consume();
+        consumedCount++;
+
+        if (cp === $.EOF)
+            break;
+
+        if (inNode)
+            i = current & HAS_BRANCHES_FLAG ? findNamedEntityTreeBranch(i, cp) : -1;
+
+        else
+            i = cp === current ? ++i : -1;
     }
 
+
     if (referencedCodePoints) {
         if (!semicolonTerminated) {
             //NOTE: unconsume excess (e.g. 'it' in '&notit')
-            this._unconsumeSeveral(consumedCount - entityCodePointsCount);
+            this._unconsumeSeveral(consumedCount - referenceSize);
 
             //NOTE: If the character reference is being consumed as part of an attribute and the next character
             //is either a U+003D EQUALS SIGN character (=) or an alphanumeric ASCII character, then, for historical
@@ -521,7 +566,7 @@ Tokenizer.prototype._consumeNamedEntity = function (startCp, inAttr) {
                 var nextCp = this._lookahead();
 
                 if (nextCp === $.EQUALS_SIGN || isAsciiAlphaNumeric(nextCp)) {
-                    this._unconsumeSeveral(entityCodePointsCount);
+                    this._unconsumeSeveral(referenceSize);
                     return null;
                 }
             }
@@ -565,7 +610,9 @@ Tokenizer.prototype._consumeCharacterReference = function (startCp, inAttr) {
         return null;
     }
 
-    return this._consumeNamedEntity(startCp, inAttr);
+    this._unconsume();
+
+    return this._consumeNamedEntity(inAttr);
 };
 
 //State machine