diff --git a/.env.development b/.env.development index d01b5c09..4fa97cae 100644 --- a/.env.development +++ b/.env.development @@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict # You can generate a new secret with: # openssl rand -base64 33 # @see: https://authjs.dev/getting-started/deployment#auth_secret -AUTH_SECRET="secret" +AUTH_SECRET="00000000000000000000000000000000000000000000" AUTH_URL="http://localhost:3000" # AUTH_CREDENTIALS_LOGIN_ENABLED=true # AUTH_GITHUB_CLIENT_ID="" @@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379" # Generated using: # openssl rand -base64 24 -SOURCEBOT_ENCRYPTION_KEY="secret" +SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000" SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection @@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection # NEXT_PUBLIC_SOURCEBOT_VERSION= # CONFIG_MAX_REPOS_NO_TOKEN= -# SOURCEBOT_ROOT_DOMAIN= # NODE_ENV= -# SOURCEBOT_TENANCY_MODE=mutli \ No newline at end of file +# SOURCEBOT_TENANCY_MODE=single \ No newline at end of file diff --git a/package.json b/package.json index 3f573804..ebc26eb2 100644 --- a/package.json +++ b/package.json @@ -7,13 +7,13 @@ "build": "yarn workspaces run build", "test": "yarn workspaces run test", - "dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web", + "dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web", "with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --", "dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc", "dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch", "dev:web": "yarn with-env yarn workspace @sourcebot/web dev", - "dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev", + "dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev", "dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio", "dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset" }, diff --git a/packages/backend/package.json b/packages/backend/package.json index e63c0789..37055afa 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,6 +34,7 @@ "@sourcebot/schemas": "^0.1.0", "@t3-oss/env-core": "^0.12.0", "@types/express": "^5.0.0", + "ajv": "^8.17.1", "argparse": "^2.0.1", "bullmq": "^5.34.10", "cross-fetch": "^4.0.0", diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index e52e2724..e513ea6c 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -41,8 +41,8 @@ export const env = createEnv({ LOGTAIL_TOKEN: z.string().optional(), LOGTAIL_HOST: z.string().url().optional(), - INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(), - DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres") + DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"), + CONFIG_PATH: z.string().optional(), }, runtimeEnv: process.env, emptyStringAsUndefined: true, diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index e43ae5de..61843ed2 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -12,8 +12,7 @@ import { env } from './env.js'; const logger = createLogger('Gitea'); export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => { - const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; - const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN; + const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN; const api = giteaApi(config.url ?? 'https://gitea.com', { token: token, diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index cdf6fa02..a27180a9 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => { } export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => { - const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; - const token = tokenResult?.token; - const secretKey = tokenResult?.secretKey; + const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN; const octokit = new Octokit({ - auth: token ?? env.FALLBACK_GITHUB_TOKEN, + auth: token, ...(config.url ? { baseUrl: `${config.url}/api/v3` } : {}), @@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o if (isHttpError(error, 401)) { const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, { - secretKey, + ...(config.token && 'secret' in config.token ? { + secretKey: config.token.secret, + } : {}), }); Sentry.captureException(e); throw e; diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index bffccfa1..8f4899e7 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -12,8 +12,7 @@ const logger = createLogger("GitLab"); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => { - const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; - const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN; + const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN; const api = new Gitlab({ ...(token ? { diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index fc4c612a..702cb0e5 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -42,7 +42,6 @@ const parser = new ArgumentParser({ }); type Arguments = { - configPath: string; cacheDir: string; } @@ -67,7 +66,6 @@ const context: AppContext = { indexPath, reposPath, cachePath: cacheDir, - configPath: args.configPath, } const prisma = new PrismaClient(); diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 01d9a29b..7a22e9e5 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js'; import { RepoManager } from './repoManager.js'; import { env } from './env.js'; import { PromClient } from './promClient.js'; +import { isRemotePath } from './utils.js'; +import { readFile } from 'fs/promises'; +import stripJsonComments from 'strip-json-comments'; +import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type'; +import { indexSchema } from '@sourcebot/schemas/v3/index.schema'; +import { Ajv } from "ajv"; const logger = createLogger('main'); +const ajv = new Ajv({ + validateFormats: false, +}); + +const getSettings = async (configPath?: string) => { + if (!configPath) { + return DEFAULT_SETTINGS; + } + + const configContent = await (async () => { + if (isRemotePath(configPath)) { + const response = await fetch(configPath); + if (!response.ok) { + throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`); + } + return response.text(); + } else { + return readFile(configPath, { encoding: 'utf-8' }); + } + })(); + + const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig; + const isValidConfig = ajv.validate(indexSchema, config); + if (!isValidConfig) { + throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`); + } + + return { + ...DEFAULT_SETTINGS, + ...config.settings, + } +} export const main = async (db: PrismaClient, context: AppContext) => { const redis = new Redis(env.REDIS_URL, { @@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => { process.exit(1); }); - const settings = DEFAULT_SETTINGS; - if (env.INDEX_CONCURRENCY_MULTIPLE) { - settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE; - } + const settings = await getSettings(env.CONFIG_PATH); const promClient = new PromClient(); diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 8b161c32..231a32c6 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js"; import os from 'os'; import { PromClient } from './promClient.js'; import * as Sentry from "@sentry/node"; + interface IRepoManager { blockingPollLoop: () => void; dispose: () => void; @@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager { const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig; if (config.token) { - const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db); - token = tokenResult?.token; + token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger); if (token) { break; } @@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager { this.logger.info(`Fetching ${repo.id}...`); const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => { - //this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) + this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) })); fetchDuration_s = durationMs / 1000; @@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager { } const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => { - //this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) + this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) })); cloneDuration_s = durationMs / 1000; @@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager { } this.logger.info(`Indexing ${repo.id}...`); - const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx)); + const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); const indexDuration_s = durationMs / 1000; this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 2c16515d..117555a7 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,3 +1,5 @@ +import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; + export type AppContext = { /** * Path to the repos cache directory. @@ -10,52 +12,9 @@ export type AppContext = { indexPath: string; cachePath: string; - - configPath: string; } -export type Settings = { - /** - * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed. - */ - maxFileSize: number; - /** - * The maximum number of trigrams per document. Files that exceed this maximum will not be indexed. - */ - maxTrigramCount: number; - /** - * The interval (in milliseconds) at which the indexer should re-index all repositories. - */ - reindexIntervalMs: number; - /** - * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. - */ - resyncConnectionPollingIntervalMs: number; - /** - * The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed. - */ - reindexRepoPollingIntervalMs: number; - /** - * The multiple of the number of CPUs to use for indexing. - */ - indexConcurrencyMultiple: number; - /** - * The multiple of the number of CPUs to use for syncing the configuration. - */ - configSyncConcurrencyMultiple: number; - /** - * The multiple of the number of CPUs to use for garbage collection. - */ - gcConcurrencyMultiple: number; - /** - * The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded. - */ - gcGracePeriodMs: number; - /** - * The timeout (in milliseconds) for a repo indexing to timeout. - */ - repoIndexTimeoutMs: number; -} +export type Settings = Required; /** * Structure of the `metadata` field in the `Repo` table. diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 39dc2e40..64d1dce9 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -21,43 +21,48 @@ export const marshalBool = (value?: boolean) => { return !!value ? '1' : '0'; } -export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => { - if (!db) { - const e = new BackendException(BackendError.CONNECTION_SYNC_SYSTEM_ERROR, { - message: `No database connection provided.`, - }); - Sentry.captureException(e); - throw e; - } +export const isRemotePath = (path: string) => { + return path.startsWith('https://') || path.startsWith('http://'); +} - const secretKey = token.secret; - const secret = await db.secret.findUnique({ - where: { - orgId_key: { - key: secretKey, - orgId +export const getTokenFromConfig = async (token: Token, orgId: number, db: PrismaClient, logger?: Logger) => { + if ('secret' in token) { + const secretKey = token.secret; + const secret = await db.secret.findUnique({ + where: { + orgId_key: { + key: secretKey, + orgId + } } + }); + + if (!secret) { + const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, { + message: `Secret with key ${secretKey} not found for org ${orgId}`, + }); + Sentry.captureException(e); + logger?.error(e.metadata.message); + throw e; } - }); - if (!secret) { - const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, { - message: `Secret with key ${secretKey} not found for org ${orgId}`, - }); - Sentry.captureException(e); - throw e; - } + const decryptedToken = decrypt(secret.iv, secret.encryptedValue); + return decryptedToken; + } else { + const envToken = process.env[token.env]; + if (!envToken) { + const e = new BackendException(BackendError.CONNECTION_SYNC_SECRET_DNE, { + message: `Environment variable ${token.env} not found.`, + }); + Sentry.captureException(e); + logger?.error(e.metadata.message); + throw e; + } - const decryptedSecret = decrypt(secret.iv, secret.encryptedValue); - return { - token: decryptedSecret, - secretKey, - }; + return envToken; + } } -export const isRemotePath = (path: string) => { - return path.startsWith('https://') || path.startsWith('http://'); -} export const resolvePathRelativeToConfig = (localPath: string, configPath: string) => { let absolutePath = localPath; diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 974fb1b3..57080e32 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -1,8 +1,7 @@ import { exec } from "child_process"; -import { AppContext, RepoMetadata } from "./types.js"; +import { AppContext, RepoMetadata, Settings } from "./types.js"; import { Repo } from "@sourcebot/db"; import { getRepoPath } from "./utils.js"; -import { DEFAULT_SETTINGS } from "./constants.js"; import { getShardPrefix } from "./utils.js"; import { getBranches, getTags } from "./git.js"; import micromatch from "micromatch"; @@ -11,7 +10,7 @@ import { captureEvent } from "./posthog.js"; const logger = createLogger('zoekt'); -export const indexGitRepository = async (repo: Repo, ctx: AppContext) => { +export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => { let revisions = [ 'HEAD' ]; @@ -58,7 +57,7 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => { revisions = revisions.slice(0, 64); } - const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${DEFAULT_SETTINGS.maxTrigramCount} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`; + const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -max_trigram_count ${settings.maxTrigramCount} -file_limit ${settings.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`; return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { exec(command, (error, stdout, stderr) => { diff --git a/packages/db/prisma/migrations/20250320215449_unique_connection_name_constraint_within_org/migration.sql b/packages/db/prisma/migrations/20250320215449_unique_connection_name_constraint_within_org/migration.sql new file mode 100644 index 00000000..89de79b3 --- /dev/null +++ b/packages/db/prisma/migrations/20250320215449_unique_connection_name_constraint_within_org/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - A unique constraint covering the columns `[name,orgId]` on the table `Connection` will be added. If there are existing duplicate values, this will fail. + +*/ +-- CreateIndex +CREATE UNIQUE INDEX "Connection_name_orgId_key" ON "Connection"("name", "orgId"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index cec632fd..629fbe2f 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -79,6 +79,8 @@ model Connection { // The organization that owns this connection org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) orgId Int + + @@unique([name, orgId]) } model RepoToConnection { diff --git a/packages/schemas/src/v3/connection.schema.ts b/packages/schemas/src/v3/connection.schema.ts index 8fa52cc5..3ab0c8db 100644 --- a/packages/schemas/src/v3/connection.schema.ts +++ b/packages/schemas/src/v3/connection.schema.ts @@ -19,17 +19,34 @@ const schema = { "secret": "SECRET_KEY" } ], - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false } - }, - "required": [ - "secret" - ], - "additionalProperties": false + ] }, "url": { "type": "string", diff --git a/packages/schemas/src/v3/connection.type.ts b/packages/schemas/src/v3/connection.type.ts index fa46a8cc..8b1e479e 100644 --- a/packages/schemas/src/v3/connection.type.ts +++ b/packages/schemas/src/v3/connection.type.ts @@ -11,7 +11,22 @@ export interface GithubConnectionConfig { * GitHub Configuration */ type: "github"; - token?: Token; + /** + * A Personal Access Token (PAT). + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; /** * The URL of the GitHub host. Defaults to https://github.com */ @@ -67,15 +82,6 @@ export interface GithubConnectionConfig { }; revisions?: GitRevisions; } -/** - * A Personal Access Token (PAT). - */ -export interface Token { - /** - * The name of the secret that contains the token. - */ - secret: string; -} /** * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ @@ -94,7 +100,22 @@ export interface GitlabConnectionConfig { * GitLab Configuration */ type: "gitlab"; - token?: Token1; + /** + * An authentication token. + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; /** * The URL of the GitLab host. Defaults to https://gitlab.com */ @@ -141,21 +162,27 @@ export interface GitlabConnectionConfig { }; revisions?: GitRevisions; } -/** - * An authentication token. - */ -export interface Token1 { - /** - * The name of the secret that contains the token. - */ - secret: string; -} export interface GiteaConnectionConfig { /** * Gitea Configuration */ type: "gitea"; - token?: Token2; + /** + * A Personal Access Token (PAT). + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; /** * The URL of the Gitea host. Defaults to https://gitea.com */ @@ -188,15 +215,6 @@ export interface GiteaConnectionConfig { }; revisions?: GitRevisions; } -/** - * A Personal Access Token (PAT). - */ -export interface Token2 { - /** - * The name of the secret that contains the token. - */ - secret: string; -} export interface GerritConnectionConfig { /** * Gerrit Configuration diff --git a/packages/schemas/src/v3/gitea.schema.ts b/packages/schemas/src/v3/gitea.schema.ts index c9c5b7d6..1e1283ee 100644 --- a/packages/schemas/src/v3/gitea.schema.ts +++ b/packages/schemas/src/v3/gitea.schema.ts @@ -15,17 +15,34 @@ const schema = { "secret": "SECRET_KEY" } ], - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false } - }, - "required": [ - "secret" - ], - "additionalProperties": false + ] }, "url": { "type": "string", diff --git a/packages/schemas/src/v3/gitea.type.ts b/packages/schemas/src/v3/gitea.type.ts index 012c7808..ec9e3046 100644 --- a/packages/schemas/src/v3/gitea.type.ts +++ b/packages/schemas/src/v3/gitea.type.ts @@ -5,7 +5,22 @@ export interface GiteaConnectionConfig { * Gitea Configuration */ type: "gitea"; - token?: Token; + /** + * A Personal Access Token (PAT). + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; /** * The URL of the Gitea host. Defaults to https://gitea.com */ @@ -38,15 +53,6 @@ export interface GiteaConnectionConfig { }; revisions?: GitRevisions; } -/** - * A Personal Access Token (PAT). - */ -export interface Token { - /** - * The name of the secret that contains the token. - */ - secret: string; -} /** * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ diff --git a/packages/schemas/src/v3/github.schema.ts b/packages/schemas/src/v3/github.schema.ts index 66e04b57..c29e1c08 100644 --- a/packages/schemas/src/v3/github.schema.ts +++ b/packages/schemas/src/v3/github.schema.ts @@ -15,17 +15,34 @@ const schema = { "secret": "SECRET_KEY" } ], - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false } - }, - "required": [ - "secret" - ], - "additionalProperties": false + ] }, "url": { "type": "string", diff --git a/packages/schemas/src/v3/github.type.ts b/packages/schemas/src/v3/github.type.ts index 412811ef..4cb73c9b 100644 --- a/packages/schemas/src/v3/github.type.ts +++ b/packages/schemas/src/v3/github.type.ts @@ -5,7 +5,22 @@ export interface GithubConnectionConfig { * GitHub Configuration */ type: "github"; - token?: Token; + /** + * A Personal Access Token (PAT). + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; /** * The URL of the GitHub host. Defaults to https://github.com */ @@ -61,15 +76,6 @@ export interface GithubConnectionConfig { }; revisions?: GitRevisions; } -/** - * A Personal Access Token (PAT). - */ -export interface Token { - /** - * The name of the secret that contains the token. - */ - secret: string; -} /** * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ diff --git a/packages/schemas/src/v3/gitlab.schema.ts b/packages/schemas/src/v3/gitlab.schema.ts index 1aaedde5..891ca4eb 100644 --- a/packages/schemas/src/v3/gitlab.schema.ts +++ b/packages/schemas/src/v3/gitlab.schema.ts @@ -15,17 +15,34 @@ const schema = { "secret": "SECRET_KEY" } ], - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false } - }, - "required": [ - "secret" - ], - "additionalProperties": false + ] }, "url": { "type": "string", diff --git a/packages/schemas/src/v3/gitlab.type.ts b/packages/schemas/src/v3/gitlab.type.ts index 8d0cb305..f5a293ce 100644 --- a/packages/schemas/src/v3/gitlab.type.ts +++ b/packages/schemas/src/v3/gitlab.type.ts @@ -5,7 +5,22 @@ export interface GitlabConnectionConfig { * GitLab Configuration */ type: "gitlab"; - token?: Token; + /** + * An authentication token. + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; /** * The URL of the GitLab host. Defaults to https://gitlab.com */ @@ -52,15 +67,6 @@ export interface GitlabConnectionConfig { }; revisions?: GitRevisions; } -/** - * An authentication token. - */ -export interface Token { - /** - * The name of the secret that contains the token. - */ - secret: string; -} /** * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. */ diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts new file mode 100644 index 00000000..24890a4b --- /dev/null +++ b/packages/schemas/src/v3/index.schema.ts @@ -0,0 +1,580 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "SourcebotConfig", + "definitions": { + "Settings": { + "type": "object", + "description": "Defines the globabl settings for Sourcebot.", + "properties": { + "maxFileSize": { + "type": "number", + "description": "The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed." + }, + "maxTrigramCount": { + "type": "number", + "description": "The maximum number of trigrams per document. Files that exceed this maximum will not be indexed." + }, + "reindexIntervalMs": { + "type": "number", + "description": "The interval (in milliseconds) at which the indexer should re-index all repositories." + }, + "resyncConnectionPollingIntervalMs": { + "type": "number", + "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced." + }, + "reindexRepoPollingIntervalMs": { + "type": "number", + "description": "The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed." + }, + "indexConcurrencyMultiple": { + "type": "number", + "description": "The multiple of the number of CPUs to use for indexing." + }, + "configSyncConcurrencyMultiple": { + "type": "number", + "description": "The multiple of the number of CPUs to use for syncing the configuration." + }, + "gcConcurrencyMultiple": { + "type": "number", + "description": "The multiple of the number of CPUs to use for garbage collection." + }, + "gcGracePeriodMs": { + "type": "number", + "description": "The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded." + }, + "repoIndexTimeoutMs": { + "type": "number", + "description": "The timeout (in milliseconds) for a repo indexing to timeout." + } + }, + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "settings": { + "$ref": "#/definitions/Settings" + }, + "connections": { + "type": "object", + "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ConnectionConfig", + "oneOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GithubConnectionConfig", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ], + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://github.com", + "description": "The URL of the GitHub host. Defaults to https://github.com", + "examples": [ + "https://github.com", + "https://github.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "users": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "torvalds", + "DHH" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "default": [], + "examples": [ + [ + "my-org-name" + ], + [ + "sourcebot-dev", + "commaai" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "default": [], + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "default": [], + "description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GitlabConnectionConfig", + "properties": { + "type": { + "const": "gitlab", + "description": "GitLab Configuration" + }, + "token": { + "$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/token", + "description": "An authentication token.", + "examples": [ + { + "secret": "SECRET_KEY" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://gitlab.com", + "description": "The URL of the GitLab host. Defaults to https://gitlab.com", + "examples": [ + "https://gitlab.com", + "https://gitlab.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "all": { + "type": "boolean", + "default": false, + "description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ." + }, + "users": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "groups": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-group" + ], + [ + "my-group/sub-group-a", + "my-group/sub-group-b" + ] + ], + "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-group/my-project" + ], + [ + "my-group/my-sub-group/my-project" + ] + ], + "description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked projects from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived projects from syncing." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + [ + "my-group/my-project" + ] + ], + "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/revisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GiteaConnectionConfig", + "properties": { + "type": { + "const": "gitea", + "description": "Gitea Configuration" + }, + "token": { + "$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/token", + "description": "A Personal Access Token (PAT).", + "examples": [ + { + "secret": "SECRET_KEY" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://gitea.com", + "description": "The URL of the Gitea host. Defaults to https://gitea.com", + "examples": [ + "https://gitea.com", + "https://gitea.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "orgs": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-org-name" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "users": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "username-1", + "username-2" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope." + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/properties/connections/patternProperties/%5E%5Ba-zA-Z0-9_-%5D%2B%24/oneOf/0/properties/revisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GerritConnectionConfig", + "properties": { + "type": { + "const": "gerrit", + "description": "Gerrit Configuration" + }, + "url": { + "type": "string", + "format": "url", + "description": "The URL of the Gerrit host.", + "examples": [ + "https://gerrit.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported", + "examples": [ + [ + "project1/repo1", + "project2/**" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "project1/repo1", + "project2/**" + ] + ], + "description": "List of specific projects to exclude from syncing." + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "url" + ], + "additionalProperties": false + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} as const; +export { schema as indexSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts new file mode 100644 index 00000000..7692d609 --- /dev/null +++ b/packages/schemas/src/v3/index.type.ts @@ -0,0 +1,299 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * This interface was referenced by `undefined`'s JSON-Schema definition + * via the `patternProperty` "^[a-zA-Z0-9_-]+$". + */ +export type ConnectionConfig = + | GithubConnectionConfig + | GitlabConnectionConfig + | GiteaConnectionConfig + | GerritConnectionConfig; + +export interface SourcebotConfig { + $schema?: string; + settings?: Settings; + /** + * Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode. + */ + connections?: { + [k: string]: ConnectionConfig; + }; +} +/** + * Defines the globabl settings for Sourcebot. + * + * This interface was referenced by `SourcebotConfig`'s JSON-Schema + * via the `definition` "Settings". + */ +export interface Settings { + /** + * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed. + */ + maxFileSize?: number; + /** + * The maximum number of trigrams per document. Files that exceed this maximum will not be indexed. + */ + maxTrigramCount?: number; + /** + * The interval (in milliseconds) at which the indexer should re-index all repositories. + */ + reindexIntervalMs?: number; + /** + * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. + */ + resyncConnectionPollingIntervalMs?: number; + /** + * The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed. + */ + reindexRepoPollingIntervalMs?: number; + /** + * The multiple of the number of CPUs to use for indexing. + */ + indexConcurrencyMultiple?: number; + /** + * The multiple of the number of CPUs to use for syncing the configuration. + */ + configSyncConcurrencyMultiple?: number; + /** + * The multiple of the number of CPUs to use for garbage collection. + */ + gcConcurrencyMultiple?: number; + /** + * The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded. + */ + gcGracePeriodMs?: number; + /** + * The timeout (in milliseconds) for a repo indexing to timeout. + */ + repoIndexTimeoutMs?: number; +} +export interface GithubConnectionConfig { + /** + * GitHub Configuration + */ + type: "github"; + /** + * A Personal Access Token (PAT). + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + /** + * The URL of the GitHub host. Defaults to https://github.com + */ + url?: string; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + /** + * Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. A maximum of 64 branches can be indexed, with any additional branches being ignored. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. A maximum of 64 tags can be indexed, with any additional tags being ignored. + */ + tags?: string[]; +} +export interface GitlabConnectionConfig { + /** + * GitLab Configuration + */ + type: "gitlab"; + /** + * An authentication token. + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + /** + * The URL of the GitLab host. Defaults to https://gitlab.com + */ + url?: string; + /** + * Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com . + */ + all?: boolean; + /** + * List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`). + */ + groups?: string[]; + /** + * List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/ + */ + projects?: string[]; + /** + * List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + exclude?: { + /** + * Exclude forked projects from syncing. + */ + forks?: boolean; + /** + * Exclude archived projects from syncing. + */ + archived?: boolean; + /** + * List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/ + */ + projects?: string[]; + /** + * List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + }; + revisions?: GitRevisions; +} +export interface GiteaConnectionConfig { + /** + * Gitea Configuration + */ + type: "gitea"; + /** + * A Personal Access Token (PAT). + */ + token?: + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + /** + * The URL of the Gitea host. Defaults to https://gitea.com + */ + url?: string; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope. + */ + users?: string[]; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + }; + revisions?: GitRevisions; +} +export interface GerritConnectionConfig { + /** + * Gerrit Configuration + */ + type: "gerrit"; + /** + * The URL of the Gerrit host. + */ + url: string; + /** + * List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported + */ + projects?: string[]; + exclude?: { + /** + * List of specific projects to exclude from syncing. + */ + projects?: string[]; + }; +} diff --git a/packages/schemas/src/v3/shared.schema.ts b/packages/schemas/src/v3/shared.schema.ts index daef67ba..0c1792ae 100644 --- a/packages/schemas/src/v3/shared.schema.ts +++ b/packages/schemas/src/v3/shared.schema.ts @@ -4,17 +4,34 @@ const schema = { "type": "object", "definitions": { "Token": { - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false } - }, - "required": [ - "secret" - ], - "additionalProperties": false + ] }, "GitRevisions": { "type": "object", diff --git a/packages/schemas/src/v3/shared.type.ts b/packages/schemas/src/v3/shared.type.ts index a8d14eff..eeec734c 100644 --- a/packages/schemas/src/v3/shared.type.ts +++ b/packages/schemas/src/v3/shared.type.ts @@ -1,17 +1,25 @@ // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! -export interface Shared { - [k: string]: unknown; -} /** * This interface was referenced by `Shared`'s JSON-Schema * via the `definition` "Token". */ -export interface Token { - /** - * The name of the secret that contains the token. - */ - secret: string; +export type Token = + | { + /** + * The name of the secret that contains the token. + */ + secret: string; + } + | { + /** + * The name of the environment variable that contains the token. Only supported in declarative connection configs. + */ + env: string; + }; + +export interface Shared { + [k: string]: unknown; } /** * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. A maximum of 64 revisions can be indexed, with any additional revisions being ignored. diff --git a/packages/web/package.json b/packages/web/package.json index 9664a83d..160195e6 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -129,6 +129,7 @@ "react-resizable-panels": "^2.1.1", "server-only": "^0.0.1", "sharp": "^0.33.5", + "strip-json-comments": "^5.0.1", "stripe": "^17.6.0", "tailwind-merge": "^2.5.2", "tailwind-scrollbar-hide": "^1.1.7", diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index b82ef598..2fce05d5 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -1,6 +1,7 @@ 'use server'; import Ajv from "ajv"; +import * as Sentry from '@sentry/nextjs'; import { auth } from "./auth"; import { notAuthenticated, notFound, ServiceError, unexpectedError, orgInvalidSubscription, secretAlreadyExists, stripeClientNotInitialized } from "@/lib/serviceError"; import { prisma } from "@/prisma"; @@ -11,7 +12,7 @@ import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema"; import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema"; import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema"; -import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, GerritConnectionConfig, ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { decrypt, encrypt } from "@sourcebot/crypto" import { getConnection } from "./data/connection"; import { ConnectionSyncStatus, Prisma, OrgRole, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; @@ -32,6 +33,22 @@ const ajv = new Ajv({ validateFormats: false, }); +/** + * "Service Error Wrapper". + * + * Captures any thrown exceptions and converts them to a unexpected + * service error. Also logs them with Sentry. + */ +export const sew = async (fn: () => Promise): Promise => { + try { + return await fn(); + } catch (e) { + Sentry.captureException(e); + console.error(e); + return unexpectedError(`An unexpected error occurred. Please try again later.`); + } +} + export const withAuth = async (fn: (session: Session) => Promise, allowSingleTenantUnauthedAccess: boolean = false) => { const session = await auth(); if (!session) { @@ -117,7 +134,9 @@ export const withTenancyModeEnforcement = async(mode: TenancyMode, fn: () => return fn(); } -export const createOrg = (name: string, domain: string): Promise<{ id: number } | ServiceError> => +////// Actions /////// + +export const createOrg = (name: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() => withTenancyModeEnforcement('multi', () => withAuth(async (session) => { const org = await prisma.org.create({ @@ -140,9 +159,9 @@ export const createOrg = (name: string, domain: string): Promise<{ id: number } return { id: org.id, } - })); + }))); -export const updateOrgName = async (name: string, domain: string) => +export const updateOrgName = async (name: string, domain: string) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const { success } = orgNameSchema.safeParse(name); @@ -163,9 +182,9 @@ export const updateOrgName = async (name: string, domain: string) => success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const updateOrgDomain = async (newDomain: string, existingDomain: string) => +export const updateOrgDomain = async (newDomain: string, existingDomain: string) => sew(() => withTenancyModeEnforcement('multi', () => withAuth((session) => withOrgMembership(session, existingDomain, async ({ orgId }) => { @@ -187,9 +206,9 @@ export const updateOrgDomain = async (newDomain: string, existingDomain: string) success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - )); + ))); -export const completeOnboarding = async (domain: string): Promise<{ success: boolean } | ServiceError> => +export const completeOnboarding = async (domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const org = await prisma.org.findUnique({ @@ -230,9 +249,9 @@ export const completeOnboarding = async (domain: string): Promise<{ success: boo success: true, } }) - ); + )); -export const getSecrets = (domain: string): Promise<{ createdAt: Date; key: string; }[] | ServiceError> => +export const getSecrets = (domain: string): Promise<{ createdAt: Date; key: string; }[] | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const secrets = await prisma.secret.findMany({ @@ -249,44 +268,41 @@ export const getSecrets = (domain: string): Promise<{ createdAt: Date; key: stri key: secret.key, createdAt: secret.createdAt, })); - })); + }))); -export const createSecret = async (key: string, value: string, domain: string): Promise<{ success: boolean } | ServiceError> => +export const createSecret = async (key: string, value: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { - try { - const encrypted = encrypt(value); - const existingSecret = await prisma.secret.findUnique({ - where: { - orgId_key: { - orgId, - key, - } - } - }); - - if (existingSecret) { - return secretAlreadyExists(); - } - - await prisma.secret.create({ - data: { + const encrypted = encrypt(value); + const existingSecret = await prisma.secret.findUnique({ + where: { + orgId_key: { orgId, key, - encryptedValue: encrypted.encryptedData, - iv: encrypted.iv, } - }); - } catch { - return unexpectedError(`Failed to create secret`); + } + }); + + if (existingSecret) { + return secretAlreadyExists(); } + await prisma.secret.create({ + data: { + orgId, + key, + encryptedValue: encrypted.encryptedData, + iv: encrypted.iv, + } + }); + + return { success: true, } - })); + }))); -export const checkIfSecretExists = async (key: string, domain: string): Promise => +export const checkIfSecretExists = async (key: string, domain: string): Promise => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const secret = await prisma.secret.findUnique({ @@ -299,9 +315,9 @@ export const checkIfSecretExists = async (key: string, domain: string): Promise< }); return !!secret; - })); + }))); -export const deleteSecret = async (key: string, domain: string): Promise<{ success: boolean } | ServiceError> => +export const deleteSecret = async (key: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { await prisma.secret.delete({ @@ -316,10 +332,10 @@ export const deleteSecret = async (key: string, domain: string): Promise<{ succe return { success: true, } - })); + }))); -export const getConnections = async (domain: string, filter: { status?: ConnectionSyncStatus[] } = {}) => +export const getConnections = async (domain: string, filter: { status?: ConnectionSyncStatus[] } = {}) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const connections = await prisma.connection.findMany({ @@ -352,10 +368,9 @@ export const getConnections = async (domain: string, filter: { status?: Connecti repoIndexingStatus: repo.repoIndexingStatus, })), })); - }) - ); + }))); -export const getConnectionInfo = async (connectionId: number, domain: string) => +export const getConnectionInfo = async (connectionId: number, domain: string) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const connection = await prisma.connection.findUnique({ @@ -382,10 +397,9 @@ export const getConnectionInfo = async (connectionId: number, domain: string) => syncedAt: connection.syncedAt ?? undefined, numLinkedRepos: connection.repos.length, } - }) - ); + }))); -export const getRepos = async (domain: string, filter: { status?: RepoIndexingStatus[], connectionId?: number } = {}) => +export const getRepos = async (domain: string, filter: { status?: RepoIndexingStatus[], connectionId?: number } = {}) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const repos = await prisma.repo.findMany({ @@ -425,9 +439,9 @@ export const getRepos = async (domain: string, filter: { status?: RepoIndexingSt repoIndexingStatus: repo.repoIndexingStatus, })); } - ), /* allowSingleTenantUnauthedAccess = */ true); + ), /* allowSingleTenantUnauthedAccess = */ true)); -export const createConnection = async (name: string, type: string, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> => +export const createConnection = async (name: string, type: string, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const parsedConfig = parseConnectionConfig(type, connectionConfig); @@ -435,6 +449,23 @@ export const createConnection = async (name: string, type: string, connectionCon return parsedConfig; } + const existingConnectionWithName = await prisma.connection.findUnique({ + where: { + name_orgId: { + orgId, + name, + } + } + }); + + if (existingConnectionWithName) { + return { + statusCode: StatusCodes.BAD_REQUEST, + errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS, + message: "A connection with this name already exists.", + } satisfies ServiceError; + } + const connection = await prisma.connection.create({ data: { orgId, @@ -448,9 +479,9 @@ export const createConnection = async (name: string, type: string, connectionCon id: connection.id, } }) - ); + )); -export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> => +export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const connection = await getConnection(connectionId, orgId); @@ -458,6 +489,23 @@ export const updateConnectionDisplayName = async (connectionId: number, name: st return notFound(); } + const existingConnectionWithName = await prisma.connection.findUnique({ + where: { + name_orgId: { + orgId, + name, + } + } + }); + + if (existingConnectionWithName) { + return { + statusCode: StatusCodes.BAD_REQUEST, + errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS, + message: "A connection with this name already exists.", + } satisfies ServiceError; + } + await prisma.connection.update({ where: { id: connectionId, @@ -471,9 +519,10 @@ export const updateConnectionDisplayName = async (connectionId: number, name: st return { success: true, } - })); + }) + )); -export const updateConnectionConfigAndScheduleSync = async (connectionId: number, config: string, domain: string): Promise<{ success: boolean } | ServiceError> => +export const updateConnectionConfigAndScheduleSync = async (connectionId: number, config: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const connection = await getConnection(connectionId, orgId); @@ -510,9 +559,10 @@ export const updateConnectionConfigAndScheduleSync = async (connectionId: number return { success: true, } - })); + }) + )); -export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => +export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const connection = await getConnection(connectionId, orgId); @@ -532,9 +582,10 @@ export const flagConnectionForSync = async (connectionId: number, domain: string return { success: true, } - })); + }) + )); -export const flagReposForIndex = async (repoIds: number[], domain: string) => +export const flagReposForIndex = async (repoIds: number[], domain: string) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { await prisma.repo.updateMany({ @@ -551,9 +602,9 @@ export const flagReposForIndex = async (repoIds: number[], domain: string) => success: true, } }) - ); + )); -export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => +export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const connection = await getConnection(connectionId, orgId); @@ -571,16 +622,17 @@ export const deleteConnection = async (connectionId: number, domain: string): Pr return { success: true, } - })); + }) + )); -export const getCurrentUserRole = async (domain: string): Promise => +export const getCurrentUserRole = async (domain: string): Promise => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ userRole }) => { return userRole; }) - ); + )); -export const createInvites = async (emails: string[], domain: string): Promise<{ success: boolean } | ServiceError> => +export const createInvites = async (emails: string[], domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { // Check for existing invites @@ -691,9 +743,9 @@ export const createInvites = async (emails: string[], domain: string): Promise<{ success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const cancelInvite = async (inviteId: string, domain: string): Promise<{ success: boolean } | ServiceError> => +export const cancelInvite = async (inviteId: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const invite = await prisma.invite.findUnique({ @@ -717,9 +769,9 @@ export const cancelInvite = async (inviteId: string, domain: string): Promise<{ success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const getMe = async () => +export const getMe = async () => sew(() => withAuth(async (session) => { const user = await prisma.user.findUnique({ where: { @@ -749,9 +801,9 @@ export const getMe = async () => name: org.org.name, })) } - }); + })); -export const redeemInvite = async (inviteId: string): Promise<{ success: boolean } | ServiceError> => +export const redeemInvite = async (inviteId: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth(async () => { const invite = await prisma.invite.findUnique({ where: { @@ -818,9 +870,9 @@ export const redeemInvite = async (inviteId: string): Promise<{ success: boolean return { success: true, } - }); + })); -export const getInviteInfo = async (inviteId: string) => +export const getInviteInfo = async (inviteId: string) => sew(() => withAuth(async () => { const user = await getMe(); if (isServiceError(user)) { @@ -860,9 +912,9 @@ export const getInviteInfo = async (inviteId: string) => email: user.email!, } } - }); + })); -export const transferOwnership = async (newOwnerId: string, domain: string): Promise<{ success: boolean } | ServiceError> => +export const transferOwnership = async (newOwnerId: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const currentUserId = session.user.id; @@ -921,9 +973,9 @@ export const transferOwnership = async (newOwnerId: string, domain: string): Pro success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const createOnboardingSubscription = async (domain: string) => +export const createOnboardingSubscription = async (domain: string) => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { const org = await prisma.org.findUnique({ @@ -1028,9 +1080,9 @@ export const createOnboardingSubscription = async (domain: string) => }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const createStripeCheckoutSession = async (domain: string) => +export const createStripeCheckoutSession = async (domain: string) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const org = await prisma.org.findUnique({ @@ -1090,9 +1142,9 @@ export const createStripeCheckoutSession = async (domain: string) => url: stripeSession.url, } }) - ); + )); -export const getCustomerPortalSessionLink = async (domain: string): Promise => +export const getCustomerPortalSessionLink = async (domain: string): Promise => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const org = await prisma.org.findUnique({ @@ -1117,16 +1169,16 @@ export const getCustomerPortalSessionLink = async (domain: string): Promise => +export const fetchSubscription = (domain: string): Promise => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { return _fetchSubscriptionForOrg(orgId, prisma); }) - ); + )); -export const getSubscriptionBillingEmail = async (domain: string): Promise => +export const getSubscriptionBillingEmail = async (domain: string): Promise => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { const org = await prisma.org.findUnique({ @@ -1149,9 +1201,9 @@ export const getSubscriptionBillingEmail = async (domain: string): Promise => +export const changeSubscriptionBillingEmail = async (domain: string, newEmail: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const org = await prisma.org.findUnique({ @@ -1176,9 +1228,9 @@ export const changeSubscriptionBillingEmail = async (domain: string, newEmail: s success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const checkIfOrgDomainExists = async (domain: string): Promise => +export const checkIfOrgDomainExists = async (domain: string): Promise => sew(() => withAuth(async () => { const org = await prisma.org.findFirst({ where: { @@ -1187,9 +1239,9 @@ export const checkIfOrgDomainExists = async (domain: string): Promise => +export const removeMemberFromOrg = async (memberId: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { const targetMember = await prisma.userToOrg.findUnique({ @@ -1246,9 +1298,9 @@ export const removeMemberFromOrg = async (memberId: string, domain: string): Pro success: true, } }, /* minRequiredRole = */ OrgRole.OWNER) - ); + )); -export const leaveOrg = async (domain: string): Promise<{ success: boolean } | ServiceError> => +export const leaveOrg = async (domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId, userRole }) => { if (userRole === OrgRole.OWNER) { @@ -1300,9 +1352,9 @@ export const leaveOrg = async (domain: string): Promise<{ success: boolean } | S success: true, } }) - ); + )); -export const getSubscriptionData = async (domain: string) => +export const getSubscriptionData = async (domain: string) => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async () => { const subscription = await fetchSubscription(domain); @@ -1322,9 +1374,9 @@ export const getSubscriptionData = async (domain: string) => status: subscription.status, } }) - ); + )); -export const getOrgMembers = async (domain: string) => +export const getOrgMembers = async (domain: string) => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { const members = await prisma.userToOrg.findMany({ @@ -1345,9 +1397,9 @@ export const getOrgMembers = async (domain: string) => joinedAt: member.joinedAt, })); }) - ); + )); -export const getOrgInvites = async (domain: string) => +export const getOrgInvites = async (domain: string) => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { const invites = await prisma.invite.findMany({ @@ -1362,11 +1414,12 @@ export const getOrgInvites = async (domain: string) => createdAt: invite.createdAt, })); }) - ); + )); -export const dismissMobileUnsupportedSplashScreen = async () => { +export const dismissMobileUnsupportedSplashScreen = async () => sew(async () => { await cookies().set(MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, 'true'); -} + return true; +}); ////// Helpers /////// @@ -1442,41 +1495,35 @@ const parseConnectionConfig = (connectionType: string, config: string) => { } satisfies ServiceError; } + if ('token' in parsedConfig && parsedConfig.token && 'env' in parsedConfig.token) { + return { + statusCode: StatusCodes.BAD_REQUEST, + errorCode: ErrorCode.INVALID_REQUEST_BODY, + message: "Environment variables are not supported for connections created in the web UI. Please use a secret instead.", + } satisfies ServiceError; + } + const { numRepos, hasToken } = (() => { - switch (connectionType) { + switch (parsedConfig.type) { + case "gitea": case "github": { - const githubConfig = parsedConfig as GithubConnectionConfig; return { - numRepos: githubConfig.repos?.length, - hasToken: !!githubConfig.token, + numRepos: parsedConfig.repos?.length, + hasToken: !!parsedConfig.token, } } case "gitlab": { - const gitlabConfig = parsedConfig as GitlabConnectionConfig; return { - numRepos: gitlabConfig.projects?.length, - hasToken: !!gitlabConfig.token, - } - } - case "gitea": { - const giteaConfig = parsedConfig as GiteaConnectionConfig; - return { - numRepos: giteaConfig.repos?.length, - hasToken: !!giteaConfig.token, + numRepos: parsedConfig.projects?.length, + hasToken: !!parsedConfig.token, } } case "gerrit": { - const gerritConfig = parsedConfig as GerritConnectionConfig; return { - numRepos: gerritConfig.projects?.length, + numRepos: parsedConfig.projects?.length, hasToken: true, // gerrit doesn't use a token atm } } - default: - return { - numRepos: undefined, - hasToken: true - } } })(); diff --git a/packages/web/src/app/[domain]/components/navigationMenu.tsx b/packages/web/src/app/[domain]/components/navigationMenu.tsx index 3fdd7fe1..4ce65874 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu.tsx @@ -27,7 +27,7 @@ export const NavigationMenu = async ({ const subscription = IS_BILLING_ENABLED ? await getSubscriptionData(domain) : null; return ( -
+
) { + const session = await auth(); + if (!session) { + return redirect(`/${domain}`); + } return (
diff --git a/packages/web/src/app/[domain]/settings/(general)/page.tsx b/packages/web/src/app/[domain]/settings/(general)/page.tsx index 90ad8884..d028ab0e 100644 --- a/packages/web/src/app/[domain]/settings/(general)/page.tsx +++ b/packages/web/src/app/[domain]/settings/(general)/page.tsx @@ -3,9 +3,9 @@ import { isServiceError } from "@/lib/utils"; import { getCurrentUserRole } from "@/actions"; import { getOrgFromDomain } from "@/data/org"; import { ChangeOrgDomainCard } from "./components/changeOrgDomainCard"; -import { env } from "@/env.mjs"; import { ServiceErrorException } from "@/lib/serviceError"; import { ErrorCode } from "@/lib/errorCodes"; +import { headers } from "next/headers"; interface GeneralSettingsPageProps { params: { domain: string; @@ -27,6 +27,8 @@ export default async function GeneralSettingsPage({ params: { domain } }: Genera }); } + const host = (await headers()).get('host') ?? ''; + return (
@@ -41,7 +43,7 @@ export default async function GeneralSettingsPage({ params: { domain } }: Genera
) diff --git a/packages/web/src/app/[domain]/settings/layout.tsx b/packages/web/src/app/[domain]/settings/layout.tsx index 89a659e4..0f15f9bc 100644 --- a/packages/web/src/app/[domain]/settings/layout.tsx +++ b/packages/web/src/app/[domain]/settings/layout.tsx @@ -3,17 +3,24 @@ import { SidebarNav } from "./components/sidebar-nav" import { NavigationMenu } from "../components/navigationMenu" import { Header } from "./components/header"; import { IS_BILLING_ENABLED } from "@/lib/stripe"; +import { redirect } from "next/navigation"; +import { auth } from "@/auth"; + export const metadata: Metadata = { title: "Settings", } -export default function SettingsLayout({ +export default async function SettingsLayout({ children, params: { domain }, }: Readonly<{ children: React.ReactNode; params: { domain: string }; }>) { + const session = await auth(); + if (!session) { + return redirect(`/${domain}`); + } const sidebarNavItems = [ { @@ -37,9 +44,9 @@ export default function SettingsLayout({ ] return ( -
+
-
+

Settings

diff --git a/packages/web/src/app/api/(server)/repos/route.ts b/packages/web/src/app/api/(server)/repos/route.ts index bab9bc9d..829d4f7c 100644 --- a/packages/web/src/app/api/(server)/repos/route.ts +++ b/packages/web/src/app/api/(server)/repos/route.ts @@ -2,7 +2,7 @@ import { listRepositories } from "@/lib/server/searchService"; import { NextRequest } from "next/server"; -import { withAuth, withOrgMembership } from "@/actions"; +import { sew, withAuth, withOrgMembership } from "@/actions"; import { isServiceError } from "@/lib/utils"; import { serviceErrorResponse } from "@/lib/serviceError"; @@ -17,10 +17,10 @@ export const GET = async (request: NextRequest) => { } -const getRepos = (domain: string) => +const getRepos = (domain: string) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const response = await listRepositories(orgId); return response; } - ), /* allowSingleTenantUnauthedAccess */ true); \ No newline at end of file + ), /* allowSingleTenantUnauthedAccess */ true)); \ No newline at end of file diff --git a/packages/web/src/app/api/(server)/search/route.ts b/packages/web/src/app/api/(server)/search/route.ts index 813640e7..54161b0d 100644 --- a/packages/web/src/app/api/(server)/search/route.ts +++ b/packages/web/src/app/api/(server)/search/route.ts @@ -4,7 +4,7 @@ import { search } from "@/lib/server/searchService"; import { searchRequestSchema } from "@/lib/schemas"; import { isServiceError } from "@/lib/utils"; import { NextRequest } from "next/server"; -import { withAuth, withOrgMembership } from "@/actions"; +import { sew, withAuth, withOrgMembership } from "@/actions"; import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; import { SearchRequest } from "@/lib/types"; @@ -25,10 +25,10 @@ export const POST = async (request: NextRequest) => { return Response.json(response); } -const postSearch = (request: SearchRequest, domain: string) => +const postSearch = (request: SearchRequest, domain: string) => sew(() => withAuth((session) => withOrgMembership(session, domain, async ({ orgId }) => { const response = await search(request, orgId); return response; } - ), /* allowSingleTenantUnauthedAccess */ true); \ No newline at end of file + ), /* allowSingleTenantUnauthedAccess */ true)); \ No newline at end of file diff --git a/packages/web/src/app/api/(server)/source/route.ts b/packages/web/src/app/api/(server)/source/route.ts index 1c9318b4..19962d50 100644 --- a/packages/web/src/app/api/(server)/source/route.ts +++ b/packages/web/src/app/api/(server)/source/route.ts @@ -5,7 +5,7 @@ import { getFileSource } from "@/lib/server/searchService"; import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; import { NextRequest } from "next/server"; -import { withAuth, withOrgMembership } from "@/actions"; +import { sew, withAuth, withOrgMembership } from "@/actions"; import { FileSourceRequest } from "@/lib/types"; export const POST = async (request: NextRequest) => { @@ -27,10 +27,10 @@ export const POST = async (request: NextRequest) => { } -const postSource = (request: FileSourceRequest, domain: string) => +const postSource = (request: FileSourceRequest, domain: string) => sew(() => withAuth(async (session) => withOrgMembership(session, domain, async ({ orgId }) => { const response = await getFileSource(request, orgId); return response; } - ), /* allowSingleTenantUnauthedAccess */ true); + ), /* allowSingleTenantUnauthedAccess */ true)); diff --git a/packages/web/src/app/login/components/credentialsForm.tsx b/packages/web/src/app/login/components/credentialsForm.tsx index b49ca12f..ac9b4bbb 100644 --- a/packages/web/src/app/login/components/credentialsForm.tsx +++ b/packages/web/src/app/login/components/credentialsForm.tsx @@ -35,9 +35,10 @@ export const CredentialsForm = ({ callbackUrl }: CredentialsFormProps) => { password: values.password, redirectTo: callbackUrl ?? "/" }) - .finally(() => { + .catch(() => { setIsLoading(false); }); + // signIn will redirect on success, so don't set isLoading to false } return ( diff --git a/packages/web/src/app/onboard/page.tsx b/packages/web/src/app/onboard/page.tsx index 009c003b..9389a794 100644 --- a/packages/web/src/app/onboard/page.tsx +++ b/packages/web/src/app/onboard/page.tsx @@ -4,7 +4,7 @@ import { redirect } from "next/navigation"; import { OnboardHeader } from "./components/onboardHeader"; import { OnboardingSteps } from "@/lib/constants"; import { LogoutEscapeHatch } from "../components/logoutEscapeHatch"; -import { env } from "@/env.mjs"; +import { headers } from "next/headers"; export default async function Onboarding() { const session = await auth(); @@ -12,6 +12,8 @@ export default async function Onboarding() { redirect("/login"); } + const host = (await headers()).get('host') ?? ''; + return (
- +
); diff --git a/packages/web/src/env.mjs b/packages/web/src/env.mjs index 612b77f5..acf41389 100644 --- a/packages/web/src/env.mjs +++ b/packages/web/src/env.mjs @@ -38,13 +38,13 @@ export const env = createEnv({ // Misc CONFIG_MAX_REPOS_NO_TOKEN: numberSchema.default(Number.MAX_SAFE_INTEGER), - SOURCEBOT_ROOT_DOMAIN: z.string().default("localhost:3000"), NODE_ENV: z.enum(["development", "test", "production"]), SOURCEBOT_TELEMETRY_DISABLED: booleanSchema.default('false'), DATABASE_URL: z.string().url(), SOURCEBOT_TENANCY_MODE: tenancyModeSchema.default("single"), SOURCEBOT_AUTH_ENABLED: booleanSchema.default('true'), + CONFIG_PATH: z.string().optional(), }, // @NOTE: Make sure you destructure all client variables in the // `experimental__runtimeEnv` block below. diff --git a/packages/web/src/initialize.ts b/packages/web/src/initialize.ts index 6ef83e49..e1fbba0e 100644 --- a/packages/web/src/initialize.ts +++ b/packages/web/src/initialize.ts @@ -1,12 +1,26 @@ -import { OrgRole } from '@sourcebot/db'; +import { ConnectionSyncStatus, OrgRole, Prisma } from '@sourcebot/db'; import { env } from './env.mjs'; import { prisma } from "@/prisma"; import { SINGLE_TENANT_USER_ID, SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_DOMAIN, SINGLE_TENANT_ORG_NAME, SINGLE_TENANT_USER_EMAIL } from './lib/constants'; +import { readFile } from 'fs/promises'; +import stripJsonComments from 'strip-json-comments'; +import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type"; +import { ConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { indexSchema } from '@sourcebot/schemas/v3/index.schema'; +import Ajv from 'ajv'; + +const ajv = new Ajv({ + validateFormats: false, +}); if (env.SOURCEBOT_AUTH_ENABLED === 'false' && env.SOURCEBOT_TENANCY_MODE === 'multi') { throw new Error('SOURCEBOT_AUTH_ENABLED must be true when SOURCEBOT_TENANCY_MODE is multi'); } +const isRemotePath = (path: string) => { + return path.startsWith('https://') || path.startsWith('http://'); +} + const initSingleTenancy = async () => { await prisma.org.upsert({ where: { @@ -50,6 +64,74 @@ const initSingleTenancy = async () => { } }); } + + // Load any connections defined declaratively in the config file. + const configPath = env.CONFIG_PATH; + if (configPath) { + const configContent = await (async () => { + if (isRemotePath(configPath)) { + const response = await fetch(configPath); + if (!response.ok) { + throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`); + } + return response.text(); + } else { + return readFile(configPath, { + encoding: 'utf-8', + }); + } + })(); + + const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig; + const isValidConfig = ajv.validate(indexSchema, config); + if (!isValidConfig) { + throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`); + } + + if (config.connections) { + for (const [key, newConnectionConfig] of Object.entries(config.connections)) { + const currentConnection = await prisma.connection.findUnique({ + where: { + name_orgId: { + name: key, + orgId: SINGLE_TENANT_ORG_ID, + } + }, + select: { + config: true, + } + }); + + const currentConnectionConfig = currentConnection ? currentConnection.config as unknown as ConnectionConfig : undefined; + const syncNeededOnUpdate = currentConnectionConfig && JSON.stringify(currentConnectionConfig) !== JSON.stringify(newConnectionConfig); + + const connectionDb = await prisma.connection.upsert({ + where: { + name_orgId: { + name: key, + orgId: SINGLE_TENANT_ORG_ID, + } + }, + update: { + config: newConnectionConfig as unknown as Prisma.InputJsonValue, + syncStatus: syncNeededOnUpdate ? ConnectionSyncStatus.SYNC_NEEDED : undefined, + }, + create: { + name: key, + connectionType: newConnectionConfig.type, + config: newConnectionConfig as unknown as Prisma.InputJsonValue, + org: { + connect: { + id: SINGLE_TENANT_ORG_ID, + } + } + } + }); + + console.log(`Upserted connection with name '${key}'. Connection ID: ${connectionDb.id}`); + } + } + } } if (env.SOURCEBOT_TENANCY_MODE === 'single') { diff --git a/packages/web/src/lib/errorCodes.ts b/packages/web/src/lib/errorCodes.ts index 65449088..66229e7b 100644 --- a/packages/web/src/lib/errorCodes.ts +++ b/packages/web/src/lib/errorCodes.ts @@ -14,6 +14,7 @@ export enum ErrorCode { INVALID_CREDENTIALS = 'INVALID_CREDENTIALS', INSUFFICIENT_PERMISSIONS = 'INSUFFICIENT_PERMISSIONS', CONNECTION_NOT_FAILED = 'CONNECTION_NOT_FAILED', + CONNECTION_ALREADY_EXISTS = 'CONNECTION_ALREADY_EXISTS', OWNER_CANNOT_LEAVE_ORG = 'OWNER_CANNOT_LEAVE_ORG', INVALID_INVITE = 'INVALID_INVITE', STRIPE_CHECKOUT_ERROR = 'STRIPE_CHECKOUT_ERROR', diff --git a/schemas/v3/index.json b/schemas/v3/index.json new file mode 100644 index 00000000..21baac5e --- /dev/null +++ b/schemas/v3/index.json @@ -0,0 +1,73 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "SourcebotConfig", + "definitions": { + "Settings": { + "type": "object", + "description": "Defines the globabl settings for Sourcebot.", + "properties": { + "maxFileSize": { + "type": "number", + "description": "The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be indexed." + }, + "maxTrigramCount": { + "type": "number", + "description": "The maximum number of trigrams per document. Files that exceed this maximum will not be indexed." + }, + "reindexIntervalMs": { + "type": "number", + "description": "The interval (in milliseconds) at which the indexer should re-index all repositories." + }, + "resyncConnectionPollingIntervalMs": { + "type": "number", + "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced." + }, + "reindexRepoPollingIntervalMs": { + "type": "number", + "description": "The polling rate (in milliseconds) at which the db should be checked for repos that should be re-indexed." + }, + "indexConcurrencyMultiple": { + "type": "number", + "description": "The multiple of the number of CPUs to use for indexing." + }, + "configSyncConcurrencyMultiple": { + "type": "number", + "description": "The multiple of the number of CPUs to use for syncing the configuration." + }, + "gcConcurrencyMultiple": { + "type": "number", + "description": "The multiple of the number of CPUs to use for garbage collection." + }, + "gcGracePeriodMs": { + "type": "number", + "description": "The grace period (in milliseconds) for garbage collection. Used to prevent deleting shards while they're being loaded." + }, + "repoIndexTimeoutMs": { + "type": "number", + "description": "The timeout (in milliseconds) for a repo indexing to timeout." + } + }, + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "settings": { + "$ref": "#/definitions/Settings" + }, + "connections": { + "type": "object", + "description": "Defines a collection of connections from varying code hosts that Sourcebot should sync with. This is only available in single-tenancy mode.", + "patternProperties": { + "^[a-zA-Z0-9_-]+$": { + "$ref": "./connection.json" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/schemas/v3/shared.json b/schemas/v3/shared.json index 79578ee9..8af4cbdc 100644 --- a/schemas/v3/shared.json +++ b/schemas/v3/shared.json @@ -3,17 +3,34 @@ "type": "object", "definitions": { "Token": { - "type": "object", - "properties": { - "secret": { - "type": "string", - "description": "The name of the secret that contains the token." + "anyOf": [ + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token. Only supported in declarative connection configs." + } + }, + "required": [ + "env" + ], + "additionalProperties": false } - }, - "required": [ - "secret" - ], - "additionalProperties": false + ] }, "GitRevisions": { "type": "object", diff --git a/yarn.lock b/yarn.lock index fea2a244..56ba9f78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4175,7 +4175,7 @@ ajv@^6.12.4: ajv@^8.17.1: version "8.17.1" - resolved "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz" + resolved "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz#37d9a5c776af6bc92d7f4f9510eba4c0a60d11a6" integrity sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g== dependencies: fast-deep-equal "^3.1.3" @@ -9600,7 +9600,7 @@ strip-json-comments@^3.1.1: strip-json-comments@^5.0.1: version "5.0.1" - resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.1.tgz" + resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.1.tgz#0d8b7d01b23848ed7dbdf4baaaa31a8250d8cfa0" integrity sha512-0fk9zBqO67Nq5M/m45qHCJxylV/DhBlIOVExqgOMiCCrzrhU6tCibRXNqE3jwJLftzE9SNuZtYbpzcO+i9FiKw== stripe@^17.6.0: