diff --git a/package.json b/package.json index a14244a6..a663bf80 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "build": "yarn workspaces run build", "test": "yarn workspaces run test", "dev": "npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web", - "dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && zoekt-webserver -index .sourcebot/index -rpc", + "dev:mt": "npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web", + "dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc", + "dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc", "dev:backend": "yarn workspace @sourcebot/backend dev:watch", "dev:web": "yarn workspace @sourcebot/web dev" }, diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 59206730..c81ddff4 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -35,6 +35,7 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A const gitHubRepos = await getGitHubReposFromConfig(repoConfig, signal, ctx); const hostUrl = repoConfig.url ?? 'https://github.com'; const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : 'github.com'; + const tenantId = repoConfig.tenantId ?? 0; await Promise.all(gitHubRepos.map((repo) => { const repoName = `${hostname}/${repo.full_name}`; @@ -51,6 +52,7 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A name: repoName, isFork: repo.fork, isArchived: !!repo.archived, + tenantId: tenantId, metadata: { 'zoekt.web-url-type': 'github', 'zoekt.web-url': repo.html_url, @@ -101,6 +103,7 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A external_codeHostUrl: hostUrl, cloneUrl: cloneUrl.toString(), name: repoName, + tenantId: 0, // TODO: add support for tenantId in GitLab config isFork, isArchived: project.archived, metadata: { diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts index fbe05520..4ada1870 100644 --- a/packages/backend/src/schemas/v2.ts +++ b/packages/backend/src/schemas/v2.ts @@ -72,6 +72,10 @@ export interface GitHubConfig { * @minItems 1 */ topics?: string[]; + /** + * @nocheckin + */ + tenantId?: number; exclude?: { /** * Exclude forked repositories from syncing. diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 9b79f055..54563413 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -13,6 +13,7 @@ interface BaseRepository { codeHost?: string; topics?: string[]; sizeInBytes?: number; + tenantId?: number; } /** diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 359798a7..e5c7ffc5 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -10,9 +10,12 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => { const revisions = [ 'HEAD' ]; + + const tenantId = repo.tenantId ?? 0; + const shardPrefix = `${tenantId}_${repo.id}`; const repoPath = getRepoPath(repo, ctx); - const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -shard_prefix ${repo.id} ${repoPath}`; + const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${tenantId} -shard_prefix ${shardPrefix} ${repoPath}`; return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { exec(command, (error, stdout, stderr) => { diff --git a/packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql b/packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql new file mode 100644 index 00000000..d8008511 --- /dev/null +++ b/packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql @@ -0,0 +1,30 @@ +/* + Warnings: + + - Added the required column `tenantId` to the `Repo` table without a default value. This is not possible if the table is not empty. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Repo" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + "indexedAt" DATETIME, + "isFork" BOOLEAN NOT NULL, + "isArchived" BOOLEAN NOT NULL, + "metadata" JSONB NOT NULL, + "cloneUrl" TEXT NOT NULL, + "tenantId" INTEGER NOT NULL, + "external_id" TEXT NOT NULL, + "external_codeHostType" TEXT NOT NULL, + "external_codeHostUrl" TEXT NOT NULL +); +INSERT INTO "new_Repo" ("cloneUrl", "createdAt", "external_codeHostType", "external_codeHostUrl", "external_id", "id", "indexedAt", "isArchived", "isFork", "metadata", "name", "updatedAt") SELECT "cloneUrl", "createdAt", "external_codeHostType", "external_codeHostUrl", "external_id", "id", "indexedAt", "isArchived", "isFork", "metadata", "name", "updatedAt" FROM "Repo"; +DROP TABLE "Repo"; +ALTER TABLE "new_Repo" RENAME TO "Repo"; +CREATE UNIQUE INDEX "Repo_external_id_external_codeHostUrl_key" ON "Repo"("external_id", "external_codeHostUrl"); +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 26fad77a..ea4ca1e3 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -20,6 +20,7 @@ model Repo { isArchived Boolean metadata Json cloneUrl String + tenantId Int // The id of the repo in the external service external_id String diff --git a/packages/web/src/app/api/(server)/search/route.ts b/packages/web/src/app/api/(server)/search/route.ts index fa9496b4..9ba352a4 100644 --- a/packages/web/src/app/api/(server)/search/route.ts +++ b/packages/web/src/app/api/(server)/search/route.ts @@ -8,13 +8,21 @@ import { NextRequest } from "next/server"; export const POST = async (request: NextRequest) => { const body = await request.json(); - const parsed = await searchRequestSchema.safeParseAsync(body); + const tenantId = await request.headers.get("X-Tenant-ID"); + + console.log(`Search request received. Tenant ID: ${tenantId}`); + + const parsed = await searchRequestSchema.safeParseAsync({ + ...body, + ...(tenantId && { tenantId: parseInt(tenantId) }), + }); if (!parsed.success) { return serviceErrorResponse( schemaValidationError(parsed.error) ); } + const response = await search(parsed.data); if (isServiceError(response)) { return serviceErrorResponse(response); diff --git a/packages/web/src/lib/schemas.ts b/packages/web/src/lib/schemas.ts index 25526f6b..4c6ef988 100644 --- a/packages/web/src/lib/schemas.ts +++ b/packages/web/src/lib/schemas.ts @@ -4,6 +4,7 @@ export const searchRequestSchema = z.object({ query: z.string(), maxMatchDisplayCount: z.number(), whole: z.boolean().optional(), + tenantId: z.number().optional(), }); diff --git a/packages/web/src/lib/server/searchService.ts b/packages/web/src/lib/server/searchService.ts index bc37de84..933ca99a 100644 --- a/packages/web/src/lib/server/searchService.ts +++ b/packages/web/src/lib/server/searchService.ts @@ -34,7 +34,7 @@ const aliasPrefixMappings: Record = { "revision:": zoektPrefixes.branch, } -export const search = async ({ query, maxMatchDisplayCount, whole }: SearchRequest): Promise => { +export const search = async ({ query, maxMatchDisplayCount, whole, tenantId }: SearchRequest): Promise => { // Replace any alias prefixes with their corresponding zoekt prefixes. for (const [prefix, zoektPrefix] of Object.entries(aliasPrefixMappings)) { query = query.replaceAll(prefix, zoektPrefix); @@ -53,9 +53,17 @@ export const search = async ({ query, maxMatchDisplayCount, whole }: SearchReque } }); + let header: Record = {}; + if (tenantId) { + header = { + "X-Tenant-ID": tenantId.toString() + }; + } + const searchResponse = await zoektFetch({ path: "/api/search", body, + header, method: "POST", }); diff --git a/packages/web/src/lib/server/zoektClient.ts b/packages/web/src/lib/server/zoektClient.ts index 7f3979c3..dba28f0a 100644 --- a/packages/web/src/lib/server/zoektClient.ts +++ b/packages/web/src/lib/server/zoektClient.ts @@ -1,3 +1,4 @@ +import { headers } from "next/headers"; import { ZOEKT_WEBSERVER_URL } from "../environment" @@ -5,6 +6,7 @@ interface ZoektRequest { path: string, body: string, method: string, + header?: Record, cache?: RequestCache, } @@ -12,6 +14,7 @@ export const zoektFetch = async ({ path, body, method, + header, cache, }: ZoektRequest) => { const response = await fetch( @@ -19,6 +22,7 @@ export const zoektFetch = async ({ { method, headers: { + ...header, "Content-Type": "application/json", }, body, diff --git a/schemas/v2/index.json b/schemas/v2/index.json index ee70c0c2..11a58acb 100644 --- a/schemas/v2/index.json +++ b/schemas/v2/index.json @@ -141,6 +141,10 @@ ["docs", "core"] ] }, + "tenantId": { + "type": "number", + "description": "@nocheckin" + }, "exclude": { "type": "object", "properties": {