Add indexing script

Author: bcbogdan

scripts/sdk-indexer/ast/traverse.ts

@@ -1,14 +1,67 @@
-import { writeFileSync } from "fs";
+import { mkdir } from "node:fs/promises";
+import { $, file, write } from "bun";
 import { TypeScriptSymbolExtractor } from "./typescript-symbol-extractor";
+import { GoSymbolExtractor } from "./go-symbol-extractor";
+import { PythonSymbolExtractor } from "./python-symbol-extractor";
+import { SymbolExtractor, Symbol } from "./types";
 
-const extractor = new TypeScriptSymbolExtractor();
+type Repository = {
+  url: string;
+  files: { path: string; module: string }[];
+  name: string;
+  language: "typescript" | "go" | "python";
+};
+
+type IndexDocument = {
+  name: string;
+  type: "function" | "class" | "variable" | "type";
+  meta: Record<string, unknown>;
+  content: string;
+  file: string;
+  line: number;
+};
+
+(async () => {
+  const repositories: Repository[] = [];
+  const documents: IndexDocument[] = [];
+
+  try {
+    await init();
+
+    for (const repository of repositories) {
+      await cloneRepository(repository);
+      const symbols = await extractSymbols(repository);
+      const repositoryDocuments = await transformSymbolsToDocuments(symbols, repository);
+    }
+
+    await write("index.json", JSON.stringify(documents, null, 2));
+
+    // await updateIndex(documents);
+  } catch (e) {
+    console.error("Indexing failed");
+    console.error(e);
+    process.exit(1);
+  }
+})();
+
+async function init() {
+  const tmpDirPath = "./tmp";
+  const tmpDir = file(tmpDirPath);
+  if (!(await tmpDir.exists())) {
+    mkdir(tmpDirPath);
+  }
+  $.cwd("/tmp");
+}
+
+async function cloneRepository(repository: Repository) {
+  await $`git clone ${repository.url} ${repository.name}`;
+}
 
-const Config = {
-  repository: "https://github.com/supertokens/supertokens-node",
-  files: [],
+const ExtractorsMap: Record<Repository["language"], SymbolExtractor> = {
+  typescript: TypeScriptSymbolExtractor,
 };
 
-const symbols = extractor.extract("/Users/bogdan/src/supertokens/docs/scripts/sdk-indexer/samples");
+async function extractSymbols(repository: Repository): Symbol[] {}
 
-console.log(symbols);
-writeFileSync("./result.json", JSON.stringify(symbols, null, 2));
+async function transformSymbolsToDocuments(symbols: Symbol[], repository: Repository) {}
+async function updateIndex(documents: IndexDocument[]) {}

scripts/sdk-indexer/index.ts

@@ -2,10 +2,9 @@ import Parser from "tree-sitter";
 import TypeScript from "tree-sitter-typescript";
 
 import { BaseSymbolExtractor } from "./symbol-extractor";
-import { Symbol, VariableSymbol, FunctionSymbol, ClassSymbol, TypeSymbol } from "./types";
-import { writeFileSync } from "fs";
+import { Symbol, VariableSymbol, FunctionSymbol, ClassSymbol, TypeSymbol, SymbolExtractor } from "./types";
 
-export class TypeScriptSymbolExtractor extends BaseSymbolExtractor {
+export class TypeScriptSymbolExtractor extends BaseSymbolExtractor implements SymbolExtractor {
   language: "typescript" = "typescript";
   include = ["**/*.ts", "**/*.tsx"];
   exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/.git/**"];