diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index a7e3a610..0df96298 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -39,3 +39,20 @@ export const fetchRepository = async (path: string, onProgress?: (event: SimpleG ] ); } + +export const getBranches = async (path: string) => { + const git = simpleGit(); + const branches = await git.cwd({ + path, + }).branch(); + + return branches.all; +} + +export const getTags = async (path: string) => { + const git = simpleGit(); + const tags = await git.cwd({ + path, + }).tags(); + return tags.all; +} \ No newline at end of file diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index d7e89c6a..475f0b0c 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -1,6 +1,6 @@ import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; -import { getTokenFromConfig, measure, fetchWithRetry } from './utils.js'; +import { getTokenFromConfig, measure } from './utils.js'; import fetch from 'cross-fetch'; import { createLogger } from './logger.js'; import micromatch from 'micromatch'; @@ -56,49 +56,6 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org return true; }); - - if (config.revisions) { - if (config.revisions.branches) { - const branchGlobs = config.revisions.branches; - allRepos = await Promise.all( - allRepos.map(async (repo) => { - const [owner, name] = repo.full_name!.split('/'); - let branches = (await fetchWithRetry( - () => getBranchesForRepo(owner, name, api), - `branches for ${owner}/${name}`, - logger - )).map(branch => branch.name!); - branches = micromatch.match(branches, branchGlobs); - - return { - ...repo, - branches, - }; - }) - ) - } - - if (config.revisions.tags) { - const tagGlobs = config.revisions.tags; - allRepos = await Promise.all( - allRepos.map(async (allRepos) => { - const [owner, name] = allRepos.full_name!.split('/'); - let tags = (await fetchWithRetry( - () => getTagsForRepo(owner, name, api), - `tags for ${owner}/${name}`, - logger - )).map(tag => tag.name!); - tags = micromatch.match(tags, tagGlobs); - - return { - ...allRepos, - tags, - }; - }) - ) - } - } - let repos = allRepos .filter((repo) => { const isExcluded = shouldExcludeRepo({ @@ -158,38 +115,6 @@ const shouldExcludeRepo = ({ return shouldExclude; } -const getTagsForRepo = async (owner: string, repo: string, api: Api) => { - try { - logger.debug(`Fetching tags for repo ${owner}/${repo}...`); - const { durationMs, data: tags } = await measure(() => - paginate((page) => api.repos.repoListTags(owner, repo, { - page - })) - ); - logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`); - return tags; - } catch (e) { - logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e); - throw e; - } -} - -const getBranchesForRepo = async (owner: string, repo: string, api: Api) => { - try { - logger.debug(`Fetching branches for repo ${owner}/${repo}...`); - const { durationMs, data: branches } = await measure(() => - paginate((page) => api.repos.repoListBranches(owner, repo, { - page - })) - ); - logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`); - return branches; - } catch (e) { - logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e); - throw e; - } -} - const getReposOwnedByUsers = async (users: string[], api: Api) => { const results = await Promise.allSettled(users.map(async (user) => { try { diff --git a/packages/backend/src/posthogEvents.ts b/packages/backend/src/posthogEvents.ts index d80d47d5..d60a7533 100644 --- a/packages/backend/src/posthogEvents.ts +++ b/packages/backend/src/posthogEvents.ts @@ -18,6 +18,7 @@ export type PosthogEventMap = { connectionId: number, repoCount: number, }, + revisions_truncated: {}, ////////////////////////////////////////////////////////////////// } diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index 70f75517..15462083 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -1,5 +1,5 @@ import express, { Request, Response } from 'express'; -import client, { Registry, Counter, Gauge, Histogram } from 'prom-client'; +import client, { Registry, Counter, Gauge } from 'prom-client'; export class PromClient { private registry: Registry; diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 4342faea..615b351a 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -7,6 +7,7 @@ import { Prisma, PrismaClient } from '@sourcebot/db'; import { WithRequired } from "./types.js" import { marshalBool } from "./utils.js"; import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { RepoMetadata } from './types.js'; export type RepoData = WithRequired; @@ -54,17 +55,21 @@ export const compileGithubConfig = async ( } }, metadata: { - 'zoekt.web-url-type': 'github', - 'zoekt.web-url': repo.html_url, - 'zoekt.name': repoName, - 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), - 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), - 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), - 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), - 'zoekt.archived': marshalBool(repo.archived), - 'zoekt.fork': marshalBool(repo.fork), - 'zoekt.public': marshalBool(repo.private === false) - }, + gitConfig: { + 'zoekt.web-url-type': 'github', + 'zoekt.web-url': repo.html_url, + 'zoekt.name': repoName, + 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), + 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), + 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), + 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), + 'zoekt.archived': marshalBool(repo.archived), + 'zoekt.fork': marshalBool(repo.fork), + 'zoekt.public': marshalBool(repo.private === false), + }, + branches: config.revisions?.branches ?? undefined, + tags: config.revisions?.tags ?? undefined, + } satisfies RepoMetadata, }; return record; @@ -113,15 +118,19 @@ export const compileGitlabConfig = async ( } }, metadata: { - 'zoekt.web-url-type': 'gitlab', - 'zoekt.web-url': projectUrl, - 'zoekt.name': project.path_with_namespace, - 'zoekt.gitlab-stars': (project.stargazers_count ?? 0).toString(), - 'zoekt.gitlab-forks': (project.forks_count ?? 0).toString(), - 'zoekt.archived': marshalBool(project.archived), - 'zoekt.fork': marshalBool(isFork), - 'zoekt.public': marshalBool(project.private === false) - }, + gitConfig: { + 'zoekt.web-url-type': 'gitlab', + 'zoekt.web-url': projectUrl, + 'zoekt.name': project.path_with_namespace, + 'zoekt.gitlab-stars': (project.stargazers_count ?? 0).toString(), + 'zoekt.gitlab-forks': (project.forks_count ?? 0).toString(), + 'zoekt.archived': marshalBool(project.archived), + 'zoekt.fork': marshalBool(isFork), + 'zoekt.public': marshalBool(project.private === false) + }, + branches: config.revisions?.branches ?? undefined, + tags: config.revisions?.tags ?? undefined, + } satisfies RepoMetadata, }; return record; @@ -168,13 +177,17 @@ export const compileGiteaConfig = async ( } }, metadata: { - 'zoekt.web-url-type': 'gitea', - 'zoekt.web-url': repo.html_url!, - 'zoekt.name': repo.full_name!, - 'zoekt.archived': marshalBool(repo.archived), - 'zoekt.fork': marshalBool(repo.fork!), - 'zoekt.public': marshalBool(repo.internal === false && repo.private === false), - }, + gitConfig: { + 'zoekt.web-url-type': 'gitea', + 'zoekt.web-url': repo.html_url!, + 'zoekt.name': repo.full_name!, + 'zoekt.archived': marshalBool(repo.archived), + 'zoekt.fork': marshalBool(repo.fork!), + 'zoekt.public': marshalBool(repo.internal === false && repo.private === false), + }, + branches: config.revisions?.branches ?? undefined, + tags: config.revisions?.tags ?? undefined, + } satisfies RepoMetadata, }; return record; @@ -227,13 +240,15 @@ export const compileGerritConfig = async ( } }, metadata: { - 'zoekt.web-url-type': 'gitiles', - 'zoekt.web-url': webUrl, - 'zoekt.name': repoId, - 'zoekt.archived': marshalBool(false), - 'zoekt.fork': marshalBool(false), - 'zoekt.public': marshalBool(true), - }, + gitConfig: { + 'zoekt.web-url-type': 'gitiles', + 'zoekt.web-url': webUrl, + 'zoekt.name': repoId, + 'zoekt.archived': marshalBool(false), + 'zoekt.fork': marshalBool(false), + 'zoekt.public': marshalBool(true), + }, + } satisfies RepoMetadata, }; return record; diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 88324408..0c8a90c1 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -3,7 +3,7 @@ import { Redis } from 'ioredis'; import { createLogger } from "./logger.js"; import { Connection, PrismaClient, Repo, RepoToConnection, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; -import { AppContext, Settings } from "./types.js"; +import { AppContext, Settings, RepoMetadata } from "./types.js"; import { getRepoPath, getTokenFromConfig, measure, getShardPrefix } from "./utils.js"; import { cloneRepository, fetchRepository } from "./git.js"; import { existsSync, rmSync, readdirSync, rm } from 'fs'; @@ -187,7 +187,7 @@ export class RepoManager implements IRepoManager { let cloneDuration_s: number | undefined = undefined; const repoPath = getRepoPath(repo, this.ctx); - const metadata = repo.metadata as Record; + const metadata = repo.metadata as RepoMetadata; // If the repo was already in the indexing state, this job was likely killed and picked up again. As a result, // to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone @@ -223,7 +223,7 @@ export class RepoManager implements IRepoManager { cloneUrl = url.toString(); } - const { durationMs } = await measure(() => cloneRepository(cloneUrl, repoPath, metadata, ({ method, stage, progress }) => { + const { durationMs } = await measure(() => cloneRepository(cloneUrl, repoPath, metadata.gitConfig, ({ method, stage, progress }) => { //this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) })); cloneDuration_s = durationMs / 1000; diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 590e2d88..847ca2b8 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -49,6 +49,29 @@ export type Settings = { gcGracePeriodMs: number; } +/** + * Structure of the `metadata` field in the `Repo` table. + */ +export type RepoMetadata = { + /** + * A set of key-value pairs that will be used as git config + * variables when cloning the repo. + * @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode + */ + gitConfig?: Record; + + /** + * A list of branches to index. Glob patterns are supported. + */ + branches?: string[]; + + /** + * A list of tags to index. Glob patterns are supported. + */ + tags?: string[]; +} + + // @see : https://stackoverflow.com/a/61132308 export type DeepPartial = T extends object ? { [P in keyof T]?: DeepPartial; diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 3155ea88..a3606418 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -1,17 +1,63 @@ import { exec } from "child_process"; -import { AppContext } from "./types.js"; +import { AppContext, RepoMetadata } from "./types.js"; import { Repo } from "@sourcebot/db"; import { getRepoPath } from "./utils.js"; import { DEFAULT_SETTINGS } from "./constants.js"; import { getShardPrefix } from "./utils.js"; +import { getBranches, getTags } from "./git.js"; +import micromatch from "micromatch"; +import { createLogger } from "./logger.js"; +import { captureEvent } from "./posthog.js"; + +const logger = createLogger('zoekt'); export const indexGitRepository = async (repo: Repo, ctx: AppContext) => { - const revisions = [ + let revisions = [ 'HEAD' ]; - - const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const repoPath = getRepoPath(repo, ctx); + const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const metadata = repo.metadata as RepoMetadata; + + if (metadata.branches) { + const branchGlobs = metadata.branches + const allBranches = await getBranches(repoPath); + const matchingBranches = + allBranches + .filter((branch) => micromatch.isMatch(branch, branchGlobs)) + .map((branch) => `refs/heads/${branch}`); + + revisions = [ + ...revisions, + ...matchingBranches + ]; + } + + if (metadata.tags) { + const tagGlobs = metadata.tags; + const allTags = await getTags(repoPath); + const matchingTags = + allTags + .filter((tag) => micromatch.isMatch(tag, tagGlobs)) + .map((tag) => `refs/tags/${tag}`); + + revisions = [ + ...revisions, + ...matchingTags + ]; + } + + // zoekt has a limit of 64 branches/tags to index. + if (revisions.length > 64) { + logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); + captureEvent('backend_revisions_truncated', { + repoId: repo.id, + revisionCount: revisions.length, + }); + revisions = revisions.slice(0, 64); + } + const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`; return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { diff --git a/packages/schemas/src/v3/connection.schema.ts b/packages/schemas/src/v3/connection.schema.ts index 0df57812..8fa52cc5 100644 --- a/packages/schemas/src/v3/connection.schema.ts +++ b/packages/schemas/src/v3/connection.schema.ts @@ -154,11 +154,11 @@ const schema = { }, "revisions": { "type": "object", - "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", "properties": { "branches": { "type": "array", - "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", "items": { "type": "string" }, @@ -175,7 +175,7 @@ const schema = { }, "tags": { "type": "array", - "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", "items": { "type": "string" }, diff --git a/packages/schemas/src/v3/connection.type.ts b/packages/schemas/src/v3/connection.type.ts index c3588cbb..fa46a8cc 100644 --- a/packages/schemas/src/v3/connection.type.ts +++ b/packages/schemas/src/v3/connection.type.ts @@ -77,15 +77,15 @@ export interface Token { secret: string; } /** - * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ export interface GitRevisions { /** - * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. */ branches?: string[]; /** - * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. */ tags?: string[]; } diff --git a/packages/schemas/src/v3/gitea.schema.ts b/packages/schemas/src/v3/gitea.schema.ts index 34c1cf96..c9c5b7d6 100644 --- a/packages/schemas/src/v3/gitea.schema.ts +++ b/packages/schemas/src/v3/gitea.schema.ts @@ -97,11 +97,11 @@ const schema = { }, "revisions": { "type": "object", - "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", "properties": { "branches": { "type": "array", - "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", "items": { "type": "string" }, @@ -118,7 +118,7 @@ const schema = { }, "tags": { "type": "array", - "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", "items": { "type": "string" }, diff --git a/packages/schemas/src/v3/gitea.type.ts b/packages/schemas/src/v3/gitea.type.ts index cba934a9..012c7808 100644 --- a/packages/schemas/src/v3/gitea.type.ts +++ b/packages/schemas/src/v3/gitea.type.ts @@ -48,15 +48,15 @@ export interface Token { secret: string; } /** - * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ export interface GitRevisions { /** - * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. */ branches?: string[]; /** - * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. */ tags?: string[]; } diff --git a/packages/schemas/src/v3/github.schema.ts b/packages/schemas/src/v3/github.schema.ts index 31ec3b87..66e04b57 100644 --- a/packages/schemas/src/v3/github.schema.ts +++ b/packages/schemas/src/v3/github.schema.ts @@ -150,11 +150,11 @@ const schema = { }, "revisions": { "type": "object", - "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", "properties": { "branches": { "type": "array", - "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", "items": { "type": "string" }, @@ -171,7 +171,7 @@ const schema = { }, "tags": { "type": "array", - "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", "items": { "type": "string" }, diff --git a/packages/schemas/src/v3/github.type.ts b/packages/schemas/src/v3/github.type.ts index 3bfeed1f..412811ef 100644 --- a/packages/schemas/src/v3/github.type.ts +++ b/packages/schemas/src/v3/github.type.ts @@ -71,15 +71,15 @@ export interface Token { secret: string; } /** - * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ export interface GitRevisions { /** - * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. */ branches?: string[]; /** - * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. */ tags?: string[]; } diff --git a/packages/schemas/src/v3/gitlab.schema.ts b/packages/schemas/src/v3/gitlab.schema.ts index f633abc4..1aaedde5 100644 --- a/packages/schemas/src/v3/gitlab.schema.ts +++ b/packages/schemas/src/v3/gitlab.schema.ts @@ -139,11 +139,11 @@ const schema = { }, "revisions": { "type": "object", - "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", "properties": { "branches": { "type": "array", - "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", "items": { "type": "string" }, @@ -160,7 +160,7 @@ const schema = { }, "tags": { "type": "array", - "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", "items": { "type": "string" }, diff --git a/packages/schemas/src/v3/gitlab.type.ts b/packages/schemas/src/v3/gitlab.type.ts index e0f77188..8d0cb305 100644 --- a/packages/schemas/src/v3/gitlab.type.ts +++ b/packages/schemas/src/v3/gitlab.type.ts @@ -62,15 +62,15 @@ export interface Token { secret: string; } /** - * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ export interface GitRevisions { /** - * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. */ branches?: string[]; /** - * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. */ tags?: string[]; } diff --git a/packages/schemas/src/v3/shared.schema.ts b/packages/schemas/src/v3/shared.schema.ts index e094a232..daef67ba 100644 --- a/packages/schemas/src/v3/shared.schema.ts +++ b/packages/schemas/src/v3/shared.schema.ts @@ -18,11 +18,11 @@ const schema = { }, "GitRevisions": { "type": "object", - "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", "properties": { "branches": { "type": "array", - "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", "items": { "type": "string" }, @@ -39,7 +39,7 @@ const schema = { }, "tags": { "type": "array", - "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", "items": { "type": "string" }, diff --git a/packages/schemas/src/v3/shared.type.ts b/packages/schemas/src/v3/shared.type.ts index c785e32a..a8d14eff 100644 --- a/packages/schemas/src/v3/shared.type.ts +++ b/packages/schemas/src/v3/shared.type.ts @@ -14,18 +14,18 @@ export interface Token { secret: string; } /** - * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. * * This interface was referenced by `Shared`'s JSON-Schema * via the `definition` "GitRevisions". */ export interface GitRevisions { /** - * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. */ branches?: string[]; /** - * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. */ tags?: string[]; } diff --git a/packages/web/src/app/[domain]/search/page.tsx b/packages/web/src/app/[domain]/search/page.tsx index 7ea5dfd5..36253ce8 100644 --- a/packages/web/src/app/[domain]/search/page.tsx +++ b/packages/web/src/app/[domain]/search/page.tsx @@ -134,30 +134,11 @@ const SearchPageInternal = () => { }; } - const isBranchFilteringEnabled = searchResponse.isBranchFilteringEnabled; - let fileMatches = searchResponse.Result.Files ?? []; - - // We only want to show matches for the default branch when - // the user isn't explicitly filtering by branch. - - measureSync(() => { - if (!isBranchFilteringEnabled) { - fileMatches = fileMatches.filter(match => { - // @note : this case handles local repos that don't have any branches. - if (!match.Branches) { - return true; - } - - return match.Branches.includes("HEAD"); - }); - } - }, "search.branchFiltering"); - return { - fileMatches, + fileMatches: searchResponse.Result.Files ?? [], searchDurationMs: Math.round(searchResponse.Result.Duration / 1000000), totalMatchCount: searchResponse.Result.MatchCount, - isBranchFilteringEnabled, + isBranchFilteringEnabled: searchResponse.isBranchFilteringEnabled, repoUrlTemplates: searchResponse.Result.RepoURLs, } }, [searchResponse]); diff --git a/packages/web/src/lib/server/searchService.ts b/packages/web/src/lib/server/searchService.ts index b6353b9b..e120f1c7 100644 --- a/packages/web/src/lib/server/searchService.ts +++ b/packages/web/src/lib/server/searchService.ts @@ -40,6 +40,17 @@ export const search = async ({ query, maxMatchDisplayCount, whole}: SearchReques query = query.replaceAll(prefix, zoektPrefix); } + const isBranchFilteringEnabled = ( + query.includes(zoektPrefixes.branch) || + query.includes(zoektPrefixes.branchShort) + ); + + // We only want to show matches for the default branch when + // the user isn't explicitly filtering by branch. + if (!isBranchFilteringEnabled) { + query = query.concat(` branch:HEAD`); + } + const body = JSON.stringify({ q: query, // @see: https://github.com/sourcebot-dev/zoekt/blob/main/api.go#L892 @@ -76,11 +87,6 @@ export const search = async ({ query, maxMatchDisplayCount, whole}: SearchReques return unexpectedError(`Something went wrong while parsing the response from zoekt`); } - const isBranchFilteringEnabled = ( - query.includes(zoektPrefixes.branch) || - query.includes(zoektPrefixes.branchShort) - ) - return { ...parsedSearchResponse.data, isBranchFilteringEnabled, diff --git a/schemas/v3/shared.json b/schemas/v3/shared.json index 97f807ff..79578ee9 100644 --- a/schemas/v3/shared.json +++ b/schemas/v3/shared.json @@ -17,11 +17,11 @@ }, "GitRevisions": { "type": "object", - "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", "properties": { "branches": { "type": "array", - "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", "items": { "type": "string" }, @@ -38,7 +38,7 @@ }, "tags": { "type": "array", - "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", "items": { "type": "string" },