Skip to content

page fragment loaders #1807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/index.md
Original file line number Diff line number Diff line change
@@ -2,6 +2,18 @@
index: false
---

```js server echo
console.log(`The current version of Node is: <b>${process.env.npm_package_version}.`);
```

```sh server echo
uname -a
```

```sh server echo
date
```

<style>

.hero {
10 changes: 7 additions & 3 deletions src/loader.ts
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@ export interface LoaderOptions {

export class LoaderResolver {
private readonly root: string;
private readonly interpreters: Map<string, string[]>;
public readonly interpreters: Map<string, string[]>; // TODO cleaner

constructor({root, interpreters}: {root: string; interpreters?: Record<string, string[] | null>}) {
this.root = root;
@@ -81,7 +81,7 @@ export class LoaderResolver {
const loader = this.findPage(path);
if (!loader) throw enoent(path);
const input = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
return parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
return await parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
}

/**
@@ -213,7 +213,7 @@ export class LoaderResolver {
const eext = fext.slice(0, -iext.length); // .zip
const loader = new CommandLoader({
command: command ?? commandPath,
args: params ? args.concat(defineParams(params)) : args,
args: withParams(args, params),
path,
params,
root: this.root,
@@ -332,6 +332,10 @@ export class LoaderResolver {
}
}

export function withParams(args: string[], params?: Params): string[] {
return params ? args.concat(defineParams(params)) : args;
}

function defineParams(params: Params): string[] {
return Object.entries(params)
.filter(([name]) => /^[a-z0-9_]+$/i.test(name)) // ignore non-ASCII parameters
96 changes: 88 additions & 8 deletions src/markdown.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint-disable import/no-named-as-default-member */
import {createHash} from "node:crypto";
import slugify from "@sindresorhus/slugify";
import {spawn} from "cross-spawn";
import he from "he";
import MarkdownIt from "markdown-it";
import type {RuleCore} from "markdown-it/lib/parser_core.mjs";
@@ -12,11 +13,12 @@ import type {Config} from "./config.js";
import {mergeStyle} from "./config.js";
import type {FrontMatter} from "./frontMatter.js";
import {readFrontMatter} from "./frontMatter.js";
import {html, rewriteHtmlPaths} from "./html.js";
import {html, parseHtml, rewriteHtmlPaths} from "./html.js";
import {parseInfo} from "./info.js";
import {transformJavaScriptSync} from "./javascript/module.js";
import type {JavaScriptNode} from "./javascript/parse.js";
import {parseJavaScript} from "./javascript/parse.js";
import {withParams} from "./loader.js";
import {isAssetPath, relativePath} from "./path.js";
import {parsePlaceholder} from "./placeholder.js";
import type {Params} from "./route.js";
@@ -31,6 +33,12 @@ export interface MarkdownCode {
mode: "inline" | "block" | "jsx";
}

export interface FragmentLoader {
id: string;
tag: string;
source: string;
}

export interface MarkdownPage {
title: string | null;
head: string | null;
@@ -46,6 +54,7 @@ export interface MarkdownPage {

interface ParseContext {
code: MarkdownCode[];
fragments: FragmentLoader[];
startLine: number;
currentLine: number;
path: string;
@@ -119,14 +128,25 @@ function makeFenceRenderer(baseRenderer: RenderRule): RenderRule {
let html = "";
let source: string | undefined;
try {
source = isFalse(attributes.run) ? undefined : getLiveSource(token.content, tag, attributes);
source =
attributes.server != null
? token.content
: isFalse(attributes.run)
? undefined
: getLiveSource(token.content, tag, attributes);
if (source != null) {
let loading = false;
const id = uniqueCodeId(context, source);
// TODO const sourceLine = context.startLine + context.currentLine;
const node = parseJavaScript(source, {path, params});
context.code.push({id, node, mode: tag === "jsx" || tag === "tsx" ? "jsx" : "block"});
if (attributes.server != null) {
context.fragments.push({id, tag, source});
} else {
const node = parseJavaScript(source, {path, params});
context.code.push({id, node, mode: tag === "jsx" || tag === "tsx" ? "jsx" : "block"});
loading = node.expression;
}
html += `<div class="observablehq observablehq--block">${
node.expression ? "<observablehq-loading></observablehq-loading>" : ""
loading ? "<observablehq-loading></observablehq-loading>" : ""
}<!--:${id}:--></div>\n`;
}
} catch (error) {
@@ -214,6 +234,7 @@ export interface ParseOptions {
path: string;
style?: Config["style"];
scripts?: Config["scripts"];
loaders?: Config["loaders"];
head?: Config["head"];
header?: Config["header"];
footer?: Config["footer"];
@@ -243,14 +264,57 @@ export function createMarkdownIt({
return markdownIt === undefined ? md : markdownIt(md);
}

export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
export async function parseMarkdown(input: string, options: ParseOptions): Promise<MarkdownPage> {
const {md, path, source = path, params} = options;
const {content, data} = readFrontMatter(input);
const code: MarkdownCode[] = [];
const context: ParseContext = {code, startLine: 0, currentLine: 0, path, params};
const fragments: FragmentLoader[] = [];
const context: ParseContext = {code, fragments, startLine: 0, currentLine: 0, path, params};
const tokens = md.parse(content, context);
const body = md.renderer.render(tokens, md.options, context); // Note: mutates code!
const title = data.title !== undefined ? data.title : findTitle(tokens);
let body = md.renderer.render(tokens, md.options, context); // Note: mutates code!

// Rewrite body to render fragments.
if (fragments.length) {
const {document} = parseHtml(body);
const roots = findRoots(document, document.body);
const interpreters = options.loaders!.interpreters;
for (const fragment of fragments) {
const root: Comment = roots.get(fragment.id);
const [command, ...args] = interpreters.get(`.${fragment.tag}`)!;
let target = "";
const subprocess = spawn(
command,
withParams(
command === "sh"
? args.concat("-s", "--") // TODO make this configurable
: args.concat("-"), // TODO make this configurable
params
),
{
windowsHide: true,
stdio: ["pipe", "pipe", "inherit"]
}
);
subprocess.stdin.write(fragment.source);
subprocess.stdin.end();
subprocess.stdout.on("data", (data) => {
target += data.toString();
});
const code = await new Promise((resolve, reject) => {
subprocess.on("error", reject);
subprocess.on("close", resolve);
});
if (code !== 0) {
throw new Error(`loader exited with code ${code}`);
}
const template = document.createElement("template");
template.innerHTML = target;
root.replaceWith(template.content.cloneNode(true));
}
body = document.body.innerHTML;
}

return {
head: getHead(title, data, options),
header: getHeader(title, data, options),
@@ -265,6 +329,22 @@ export function parseMarkdown(input: string, options: ParseOptions): MarkdownPag
};
}

function findRoots(document, root) {
const roots = new Map();
const iterator = document.createNodeIterator(root, 128, null);
let node;
while ((node = iterator.nextNode())) {
if (isRoot(node)) {
roots.set(node.data.slice(1, -1), node);
}
}
return roots;
}

function isRoot(node) {
return node.nodeType === 8 && /^:[0-9a-f]{8}(?:-\d+)?:$/.test(node.data);
}

/** Like parseMarkdown, but optimized to return only metadata. */
export function parseMarkdownMetadata(input: string, options: ParseOptions): Pick<MarkdownPage, "data" | "title"> {
const {md, path} = options;
2 changes: 1 addition & 1 deletion test/markdown-test.ts
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ describe("parseMarkdown(input)", () => {

(only ? it.only : skip ? it.skip : it)(`test/input/${name}`, async () => {
const source = await readFile(path, "utf8");
const snapshot = parseMarkdown(source, {path: name, md});
const snapshot = await parseMarkdown(source, {path: name, md});
let allequal = true;
for (const ext of ["html", "json"]) {
const actual = ext === "json" ? jsonMeta(snapshot) : snapshot.body;
30 changes: 15 additions & 15 deletions test/resolvers-test.ts
Original file line number Diff line number Diff line change
@@ -14,84 +14,84 @@ describe("getResolvers(page, {root, path})", () => {
const builtins = ["observablehq:runtime", "observablehq:stdlib", "observablehq:client"];
it("resolves directly-attached files", async () => {
const options = getOptions({root: "test/input", path: "attached.md"});
const page = parseMarkdown("${FileAttachment('foo.csv')}", options);
const page = await parseMarkdown("${FileAttachment('foo.csv')}", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.files, new Set(["./foo.csv"]));
});
it("ignores files that are outside of the source root", async () => {
const options = getOptions({root: "test/input", path: "attached.md"});
const page = parseMarkdown("${FileAttachment('../foo.csv')}", options);
const page = await parseMarkdown("${FileAttachment('../foo.csv')}", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.files, new Set([]));
});
it("detects file methods", async () => {
const options = getOptions({root: "test/input", path: "attached.md"});
const page = parseMarkdown("${FileAttachment('foo.csv').csv}", options);
const page = await parseMarkdown("${FileAttachment('foo.csv').csv}", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(["npm:d3-dsv", ...builtins]));
});
it("detects local static imports", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport './bar.js';\n```", options);
const page = await parseMarkdown("```js\nimport './bar.js';\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(["./bar.js", ...builtins]));
assert.deepStrictEqual(resolvers.localImports, new Set(["./bar.js"]));
});
it("detects local transitive static imports", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport './other/foo.js';\n```", options);
const page = await parseMarkdown("```js\nimport './other/foo.js';\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(["./other/foo.js", "./bar.js", ...builtins]));
assert.deepStrictEqual(resolvers.localImports, new Set(["./other/foo.js", "./bar.js"]));
});
it("detects local transitive static imports (2)", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
const page = await parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js", ...builtins])); // prettier-ignore
assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
});
it("detects local transitive dynamic imports", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
const page = await parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(["./dynamic-import.js", ...builtins]));
assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
});
it("detects local transitive dynamic imports (2)", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
const page = await parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
});
it("detects local transitive dynamic imports (3)", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
const page = await parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-dynamic-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
});
it("detects local transitive dynamic imports (4)", async () => {
const options = getOptions({root: "test/input/imports", path: "attached.md"});
const page = parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
const page = await parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
});
it("detects local dynamic imports", async () => {
const options = getOptions({root: "test/input", path: "attached.md"});
const page = parseMarkdown("${import('./foo.js')}", options);
const page = await parseMarkdown("${import('./foo.js')}", options);
const resolvers = await getResolvers(page, options);
assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
assert.deepStrictEqual(resolvers.localImports, new Set(["./foo.js"]));
});
});

describe("resolveLink(href) with {preserveExtension: true}", () => {
const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: true});
const page = parseMarkdown("", options);
async function getResolveLink() {
const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: true});
const page = await parseMarkdown("", options);
const resolvers = await getResolvers(page, options);
return resolvers.resolveLink;
}
@@ -164,9 +164,9 @@ describe("resolveLink(href) with {preserveExtension: true}", () => {
});

describe("resolveLink(href) with {preserveExtension: false}", () => {
const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: false});
const page = parseMarkdown("", options);
async function getResolveLink() {
const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: false});
const page = await parseMarkdown("", options);
const resolvers = await getResolvers(page, options);
return resolvers.resolveLink;
}