diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d3060caf70a..5e495067a245 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2061,6 +2061,15 @@ importers: specifier: 7.34.1 version: 7.34.1(eslint@8.57.1) + tools/generate-llms-txt: + devDependencies: + tsx: + specifier: 4.19.2 + version: 4.19.2 + typescript: + specifier: 5.6.3 + version: 5.6.3 + tools/tsconfig: {} packages: diff --git a/tools/generate-llms-txt/package.json b/tools/generate-llms-txt/package.json new file mode 100644 index 000000000000..721fdf39493b --- /dev/null +++ b/tools/generate-llms-txt/package.json @@ -0,0 +1,16 @@ +{ + "private": true, + "name": "generate-llms-txt", + "scripts": { + "generate-llms-txt": "tsx src/generate-llms-txt.ts" + }, + "lint-staged": { + "*": [ + "prettier --ignore-unknown --write" + ] + }, + "devDependencies": { + "tsx": "4.19.2", + "typescript": "5.6.3" + } +} diff --git a/tools/generate-llms-txt/src/generate-llms-txt.ts b/tools/generate-llms-txt/src/generate-llms-txt.ts new file mode 100644 index 000000000000..8b2aebe39dcf --- /dev/null +++ b/tools/generate-llms-txt/src/generate-llms-txt.ts @@ -0,0 +1,88 @@ +#!/usr/bin/env tsx + +import { readdir, readFile, writeFile } from 'fs/promises'; +import { join } from 'path'; + +async function getAllFiles(dir: string): Promise { + const files: string[] = []; + const entries = await readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await getAllFiles(fullPath))); + } else { + files.push(fullPath); + } + } + + return files; +} + +const exclusionPrefixes = [ + 'cookbook/01-next', + 'cookbook/05-node', + 'cookbook/15-api-servers/index', + 'cookbook/20-rsc', + 'docs/01-introduction', + 'docs/02-foundations', + 'docs/02-getting-started', + 'docs/02-guides', + 'docs/03-ai-sdk-core/index', + 'docs/04-ai-sdk-ui/index', + 'docs/04-ai-sdk-ui/50-stream-protocol', + 'docs/05-ai-sdk-rsc', + 'docs/06-advanced', + 'docs/07-reference/index', + 'docs/07-reference/01-ai-sdk-core', + 'docs/07-reference/02-ai-sdk-ui', + 'docs/07-reference/03-ai-sdk-rsc', + 'docs/07-reference/04-stream-helpers', + 'docs/07-reference/05-ai-sdk-errors/index', + 'docs/08-migration-guides', + 'docs/09-troubleshooting', + 'providers/01-ai-sdk-providers/index', + 'providers/02-openai-compatible-providers/01-custom-providers', + 'providers/02-openai-compatible-providers/40-baseten', + 'providers/03-community-providers', + 'providers/04-adapters', + 'providers/05-observability', +]; + +async function main() { + try { + const contentDir = join(process.cwd(), '../../content'); + const files = await getAllFiles(contentDir); + + const filteredFiles = files.filter(file => { + for (const prefix of exclusionPrefixes) { + if (file.includes(prefix)) { + return false; + } + } + return true; + }); + + console.log(filteredFiles.join('\n')); + console.log(); + + let fullContent = ''; + for (const file of filteredFiles) { + const content = await readFile(file, 'utf-8'); + + fullContent += content; + fullContent += '\n\n'; + } + + console.log(`Length (chars ): ${fullContent.length}`); + console.log(`Length (approx tokens): ${fullContent.length / 4}`); + + // write to llms.txt + await writeFile('llms.txt', fullContent); + } catch (error) { + console.error('Error:', error); + process.exit(1); + } +} + +main();