From b52dfb01ebf68136aa957ceee259e2802b633fb3 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 23 Jan 2025 19:26:39 -0800 Subject: [PATCH 01/21] add @sourcebot/schemas package --- packages/schemas/package.json | 21 + packages/schemas/src/v1/index.schema.ts | 137 +++++ packages/schemas/src/v1/index.type.ts | 88 ++++ packages/schemas/src/v2/index.schema.ts | 640 +++++++++++++++++++++++ packages/schemas/src/v2/index.type.ts | 331 ++++++++++++ packages/schemas/src/v3/github.schema.ts | 205 ++++++++ packages/schemas/src/v3/github.type.ts | 90 ++++ packages/schemas/src/v3/shared.schema.ts | 69 +++ packages/schemas/src/v3/shared.type.ts | 34 ++ packages/schemas/tools/generate.ts | 48 ++ packages/schemas/tsconfig.json | 27 + schemas/index.json | 135 ----- schemas/v3/github.json | 1 + 13 files changed, 1691 insertions(+), 135 deletions(-) create mode 100644 packages/schemas/package.json create mode 100644 packages/schemas/src/v1/index.schema.ts create mode 100644 packages/schemas/src/v1/index.type.ts create mode 100644 packages/schemas/src/v2/index.schema.ts create mode 100644 packages/schemas/src/v2/index.type.ts create mode 100644 packages/schemas/src/v3/github.schema.ts create mode 100644 packages/schemas/src/v3/github.type.ts create mode 100644 packages/schemas/src/v3/shared.schema.ts create mode 100644 packages/schemas/src/v3/shared.type.ts create mode 100644 packages/schemas/tools/generate.ts create mode 100644 packages/schemas/tsconfig.json delete mode 100644 schemas/index.json diff --git a/packages/schemas/package.json b/packages/schemas/package.json new file mode 100644 index 00000000..01fadaee --- /dev/null +++ b/packages/schemas/package.json @@ -0,0 +1,21 @@ +{ + "name": "@sourcebot/schemas", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "yarn generate && tsc", + "generate": "tsx tools/generate.ts", + "postinstall": "yarn build" + }, + "devDependencies": { + "@apidevtools/json-schema-ref-parser": "^11.7.3", + "glob": "^11.0.1", + "json-schema-to-typescript": "^15.0.4", + "tsx": "^4.19.2", + "typescript": "^5.7.3" + }, + "exports": { + "./v2/*": "./dist/v2/*.js", + "./v3/*": "./dist/v3/*.js" + } +} diff --git a/packages/schemas/src/v1/index.schema.ts b/packages/schemas/src/v1/index.schema.ts new file mode 100644 index 00000000..f6c3d547 --- /dev/null +++ b/packages/schemas/src/v1/index.schema.ts @@ -0,0 +1,137 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "definitions": { + "RepoNameRegexIncludeFilter": { + "type": "string", + "description": "Only clone repos whose name matches the given regexp.", + "format": "regexp", + "default": "^(foo|bar)$" + }, + "RepoNameRegexExcludeFilter": { + "type": "string", + "description": "Don't mirror repos whose names match this regexp.", + "format": "regexp", + "default": "^(fizz|buzz)$" + }, + "ZoektConfig": { + "anyOf": [ + { + "$ref": "#/definitions/GitHubConfig" + }, + { + "$ref": "#/definitions/GitLabConfig" + } + ] + }, + "GitHubConfig": { + "type": "object", + "properties": { + "Type": { + "const": "github" + }, + "GitHubUrl": { + "type": "string", + "description": "GitHub Enterprise url. If not set github.com will be used as the host." + }, + "GitHubUser": { + "type": "string", + "description": "The GitHub user to mirror" + }, + "GitHubOrg": { + "type": "string", + "description": "The GitHub organization to mirror" + }, + "Name": { + "$ref": "#/definitions/RepoNameRegexIncludeFilter" + }, + "Exclude": { + "$ref": "#/definitions/RepoNameRegexExcludeFilter" + }, + "CredentialPath": { + "type": "string", + "description": "Path to a file containing a GitHub access token.", + "default": "~/.github-token" + }, + "Topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Only mirror repos that have one of the given topics" + }, + "ExcludeTopics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Don't mirror repos that have one of the given topics" + }, + "NoArchived": { + "type": "boolean", + "description": "Mirror repos that are _not_ archived", + "default": false + }, + "IncludeForks": { + "type": "boolean", + "description": "Also mirror forks", + "default": false + } + }, + "required": [ + "Type" + ], + "additionalProperties": false + }, + "GitLabConfig": { + "type": "object", + "properties": { + "Type": { + "const": "gitlab" + }, + "GitLabURL": { + "type": "string", + "description": "The GitLab API url.", + "default": "https://gitlab.com/api/v4/" + }, + "Name": { + "$ref": "#/definitions/RepoNameRegexIncludeFilter" + }, + "Exclude": { + "$ref": "#/definitions/RepoNameRegexExcludeFilter" + }, + "OnlyPublic": { + "type": "boolean", + "description": "Only mirror public repos", + "default": false + }, + "CredentialPath": { + "type": "string", + "description": "Path to a file containing a GitLab access token.", + "default": "~/.gitlab-token" + } + }, + "required": [ + "Type" + ], + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "Configs": { + "type": "array", + "items": { + "$ref": "#/definitions/ZoektConfig" + } + } + }, + "required": [ + "Configs" + ], + "additionalProperties": false +} as const; +export { schema as indexSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v1/index.type.ts b/packages/schemas/src/v1/index.type.ts new file mode 100644 index 00000000..2f0bcc1d --- /dev/null +++ b/packages/schemas/src/v1/index.type.ts @@ -0,0 +1,88 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "ZoektConfig". + */ +export type ZoektConfig = GitHubConfig | GitLabConfig; +/** + * Only clone repos whose name matches the given regexp. + * + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "RepoNameRegexIncludeFilter". + */ +export type RepoNameRegexIncludeFilter = string; +/** + * Don't mirror repos whose names match this regexp. + * + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "RepoNameRegexExcludeFilter". + */ +export type RepoNameRegexExcludeFilter = string; + +export interface Index { + $schema?: string; + Configs: ZoektConfig[]; +} +/** + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "GitHubConfig". + */ +export interface GitHubConfig { + Type: "github"; + /** + * GitHub Enterprise url. If not set github.com will be used as the host. + */ + GitHubUrl?: string; + /** + * The GitHub user to mirror + */ + GitHubUser?: string; + /** + * The GitHub organization to mirror + */ + GitHubOrg?: string; + Name?: RepoNameRegexIncludeFilter; + Exclude?: RepoNameRegexExcludeFilter; + /** + * Path to a file containing a GitHub access token. + */ + CredentialPath?: string; + /** + * Only mirror repos that have one of the given topics + */ + Topics?: string[]; + /** + * Don't mirror repos that have one of the given topics + */ + ExcludeTopics?: string[]; + /** + * Mirror repos that are _not_ archived + */ + NoArchived?: boolean; + /** + * Also mirror forks + */ + IncludeForks?: boolean; +} +/** + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "GitLabConfig". + */ +export interface GitLabConfig { + Type: "gitlab"; + /** + * The GitLab API url. + */ + GitLabURL?: string; + Name?: RepoNameRegexIncludeFilter; + Exclude?: RepoNameRegexExcludeFilter; + /** + * Only mirror public repos + */ + OnlyPublic?: boolean; + /** + * Path to a file containing a GitLab access token. + */ + CredentialPath?: string; +} diff --git a/packages/schemas/src/v2/index.schema.ts b/packages/schemas/src/v2/index.schema.ts new file mode 100644 index 00000000..ed475f73 --- /dev/null +++ b/packages/schemas/src/v2/index.schema.ts @@ -0,0 +1,640 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Sourcebot configuration schema", + "description": "A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.", + "definitions": { + "Token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "GitRevisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + }, + "GitHubConfig": { + "type": "object", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "$ref": "#/definitions/Token", + "description": "A Personal Access Token (PAT).", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://github.com", + "description": "The URL of the GitHub host. Defaults to https://github.com", + "examples": [ + "https://github.com", + "https://github.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "users": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "torvalds", + "DHH" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "my-org-name" + ], + [ + "sourcebot-dev", + "commaai" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "tenantId": { + "type": "number", + "description": "@nocheckin" + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "GitLabConfig": { + "type": "object", + "properties": { + "type": { + "const": "gitlab", + "description": "GitLab Configuration" + }, + "token": { + "$ref": "#/definitions/Token", + "description": "An authentication token.", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://gitlab.com", + "description": "The URL of the GitLab host. Defaults to https://gitlab.com", + "examples": [ + "https://gitlab.com", + "https://gitlab.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "all": { + "type": "boolean", + "default": false, + "description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ." + }, + "users": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "groups": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-group" + ], + [ + "my-group/sub-group-a", + "my-group/sub-group-b" + ] + ], + "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-group/my-project" + ], + [ + "my-group/my-sub-group/my-project" + ] + ], + "description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked projects from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived projects from syncing." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + [ + "my-group/my-project" + ] + ], + "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "GiteaConfig": { + "type": "object", + "properties": { + "type": { + "const": "gitea", + "description": "Gitea Configuration" + }, + "token": { + "$ref": "#/definitions/Token", + "description": "An access token.", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://gitea.com", + "description": "The URL of the Gitea host. Defaults to https://gitea.com", + "examples": [ + "https://gitea.com", + "https://gitea.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "orgs": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-org-name" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "users": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "username-1", + "username-2" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope." + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "GerritConfig": { + "type": "object", + "properties": { + "type": { + "const": "gerrit", + "description": "Gerrit Configuration" + }, + "url": { + "type": "string", + "format": "url", + "description": "The URL of the Gerrit host.", + "examples": [ + "https://gerrit.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported", + "examples": [ + [ + "project1/repo1", + "project2/**" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "project1/repo1", + "project2/**" + ] + ], + "description": "List of specific projects to exclude from syncing." + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "url" + ], + "additionalProperties": false + }, + "LocalConfig": { + "type": "object", + "properties": { + "type": { + "const": "local", + "description": "Local Configuration" + }, + "path": { + "type": "string", + "description": "Path to the local directory to sync with. Relative paths are relative to the configuration file's directory.", + "pattern": ".+" + }, + "watch": { + "type": "boolean", + "default": true, + "description": "Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true." + }, + "exclude": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { + "type": "string", + "pattern": ".+" + }, + "description": "List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded.", + "default": [], + "examples": [ + [ + "node_modules", + "bin", + "dist", + "build", + "out" + ] + ] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "path" + ], + "additionalProperties": false + }, + "GitConfig": { + "type": "object", + "properties": { + "type": { + "const": "git", + "description": "Git Configuration" + }, + "url": { + "type": "string", + "format": "url", + "description": "The URL to the git repository." + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type", + "url" + ], + "additionalProperties": false + }, + "Repos": { + "anyOf": [ + { + "$ref": "#/definitions/GitHubConfig" + }, + { + "$ref": "#/definitions/GitLabConfig" + }, + { + "$ref": "#/definitions/GiteaConfig" + }, + { + "$ref": "#/definitions/GerritConfig" + }, + { + "$ref": "#/definitions/LocalConfig" + }, + { + "$ref": "#/definitions/GitConfig" + } + ] + }, + "Settings": { + "type": "object", + "description": "Global settings. These settings are applied to all repositories.", + "properties": { + "maxFileSize": { + "type": "integer", + "description": "The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. Defaults to 2MB (2097152 bytes).", + "default": 2097152, + "minimum": 1 + }, + "autoDeleteStaleRepos": { + "type": "boolean", + "description": "Automatically delete stale repositories from the index. Defaults to true.", + "default": true + }, + "reindexInterval": { + "type": "integer", + "description": "The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).", + "default": 3600000, + "minimum": 1 + }, + "resyncInterval": { + "type": "integer", + "description": "The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).", + "default": 86400000, + "minimum": 1 + } + }, + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "settings": { + "$ref": "#/definitions/Settings" + }, + "repos": { + "type": "array", + "description": "Defines a collection of repositories from varying code hosts that Sourcebot should sync with.", + "items": { + "$ref": "#/definitions/Repos" + } + } + }, + "additionalProperties": false +} as const; +export { schema as indexSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v2/index.type.ts b/packages/schemas/src/v2/index.type.ts new file mode 100644 index 00000000..1449d43c --- /dev/null +++ b/packages/schemas/src/v2/index.type.ts @@ -0,0 +1,331 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "Repos". + */ +export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig | GitConfig; +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "Token". + */ +export type Token = + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + +/** + * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index. + */ +export interface SourcebotConfigurationSchema { + $schema?: string; + settings?: Settings; + /** + * Defines a collection of repositories from varying code hosts that Sourcebot should sync with. + */ + repos?: Repos[]; +} +/** + * Global settings. These settings are applied to all repositories. + * + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "Settings". + */ +export interface Settings { + /** + * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. Defaults to 2MB (2097152 bytes). + */ + maxFileSize?: number; + /** + * Automatically delete stale repositories from the index. Defaults to true. + */ + autoDeleteStaleRepos?: boolean; + /** + * The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds). + */ + reindexInterval?: number; + /** + * The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds). + */ + resyncInterval?: number; +} +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitHubConfig". + */ +export interface GitHubConfig { + /** + * GitHub Configuration + */ + type: "github"; + /** + * A Personal Access Token (PAT). + */ + token?: + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + /** + * The URL of the GitHub host. Defaults to https://github.com + */ + url?: string; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + /** + * @nocheckin + */ + tenantId?: number; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + /** + * Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitRevisions". + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + */ + tags?: string[]; +} +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitLabConfig". + */ +export interface GitLabConfig { + /** + * GitLab Configuration + */ + type: "gitlab"; + /** + * An authentication token. + */ + token?: + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + /** + * The URL of the GitLab host. Defaults to https://gitlab.com + */ + url?: string; + /** + * Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com . + */ + all?: boolean; + /** + * List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`). + */ + groups?: string[]; + /** + * List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/ + */ + projects?: string[]; + /** + * List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + exclude?: { + /** + * Exclude forked projects from syncing. + */ + forks?: boolean; + /** + * Exclude archived projects from syncing. + */ + archived?: boolean; + /** + * List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/ + */ + projects?: string[]; + /** + * List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + }; + revisions?: GitRevisions; +} +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GiteaConfig". + */ +export interface GiteaConfig { + /** + * Gitea Configuration + */ + type: "gitea"; + /** + * An access token. + */ + token?: + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + /** + * The URL of the Gitea host. Defaults to https://gitea.com + */ + url?: string; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope. + */ + users?: string[]; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + }; + revisions?: GitRevisions; +} +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GerritConfig". + */ +export interface GerritConfig { + /** + * Gerrit Configuration + */ + type: "gerrit"; + /** + * The URL of the Gerrit host. + */ + url: string; + /** + * List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported + */ + projects?: string[]; + exclude?: { + /** + * List of specific projects to exclude from syncing. + */ + projects?: string[]; + }; +} +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "LocalConfig". + */ +export interface LocalConfig { + /** + * Local Configuration + */ + type: "local"; + /** + * Path to the local directory to sync with. Relative paths are relative to the configuration file's directory. + */ + path: string; + /** + * Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true. + */ + watch?: boolean; + exclude?: { + /** + * List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded. + */ + paths?: string[]; + }; +} +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitConfig". + */ +export interface GitConfig { + /** + * Git Configuration + */ + type: "git"; + /** + * The URL to the git repository. + */ + url: string; + revisions?: GitRevisions; +} diff --git a/packages/schemas/src/v3/github.schema.ts b/packages/schemas/src/v3/github.schema.ts new file mode 100644 index 00000000..0c88f81b --- /dev/null +++ b/packages/schemas/src/v3/github.schema.ts @@ -0,0 +1,205 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GitHubConfig", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ], + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://github.com", + "description": "The URL of the GitHub host. Defaults to https://github.com", + "examples": [ + "https://github.com", + "https://github.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "users": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "torvalds", + "DHH" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "my-org-name" + ], + [ + "sourcebot-dev", + "commaai" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "tenantId": { + "type": "number", + "description": "@nocheckin" + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type" + ], + "additionalProperties": false +} as const; +export { schema as githubSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/github.type.ts b/packages/schemas/src/v3/github.type.ts new file mode 100644 index 00000000..d9c7e1aa --- /dev/null +++ b/packages/schemas/src/v3/github.type.ts @@ -0,0 +1,90 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +export interface GitHubConfig { + /** + * GitHub Configuration + */ + type: "github"; + /** + * A Personal Access Token (PAT). + */ + token?: + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + /** + * The URL of the GitHub host. Defaults to https://github.com + */ + url?: string; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + /** + * @nocheckin + */ + tenantId?: number; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + /** + * Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + */ + tags?: string[]; +} diff --git a/packages/schemas/src/v3/shared.schema.ts b/packages/schemas/src/v3/shared.schema.ts new file mode 100644 index 00000000..ccb21856 --- /dev/null +++ b/packages/schemas/src/v3/shared.schema.ts @@ -0,0 +1,69 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "definitions": { + "Token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "GitRevisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + } +} as const; +export { schema as sharedSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/shared.type.ts b/packages/schemas/src/v3/shared.type.ts new file mode 100644 index 00000000..6dcd54e7 --- /dev/null +++ b/packages/schemas/src/v3/shared.type.ts @@ -0,0 +1,34 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * This interface was referenced by `Shared`'s JSON-Schema + * via the `definition` "Token". + */ +export type Token = + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + +export interface Shared { + [k: string]: unknown; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * + * This interface was referenced by `Shared`'s JSON-Schema + * via the `definition` "GitRevisions". + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + */ + tags?: string[]; +} diff --git a/packages/schemas/tools/generate.ts b/packages/schemas/tools/generate.ts new file mode 100644 index 00000000..a5911e27 --- /dev/null +++ b/packages/schemas/tools/generate.ts @@ -0,0 +1,48 @@ +import path, { dirname } from "path"; +import { mkdir, rm, writeFile } from "fs/promises"; +import $RefParser from "@apidevtools/json-schema-ref-parser"; +import { compileFromFile } from "json-schema-to-typescript"; +import { glob } from "glob"; + + +const BANNER_COMMENT = '// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!\n'; +// const SCHEMAS: string[] = ["github.json", "shared.json"]; + +(async () => { + const cwd = process.cwd(); + const schemasBasePath = path.resolve(`${cwd}/../../schemas`); + const schemas = await glob(`${schemasBasePath}/**/*.json`) + + await Promise.all(schemas.map(async (schemaPath) => { + const name = path.parse(schemaPath).name; + const version = path.basename(path.dirname(schemaPath)); + const outDir = path.join(cwd, `src/${version}`); + + // Clean output directory first + await rm(outDir, { recursive: true, force: true }); + await mkdir(outDir, { recursive: true }); + + // Generate schema + const schema = JSON.stringify(await $RefParser.bundle(schemaPath), null, 2); + await writeFile( + path.join(outDir, `${name}.schema.ts`), + BANNER_COMMENT + + 'const schema = ' + + schema + + ` as const;\nexport { schema as ${name}Schema };`, + ); + + // Generate types + const content = await compileFromFile(schemaPath, { + bannerComment: BANNER_COMMENT, + cwd: dirname(schemaPath), + ignoreMinAndMaxItems: true, + declareExternallyReferenced: true, + unreachableDefinitions: true, + }); + await writeFile( + path.join(outDir, `${name}.type.ts`), + content, + ) + })); +})(); \ No newline at end of file diff --git a/packages/schemas/tsconfig.json b/packages/schemas/tsconfig.json new file mode 100644 index 00000000..9a1ea53a --- /dev/null +++ b/packages/schemas/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "outDir": "dist", + "incremental": true, + "declaration": true, + "emitDecoratorMetadata": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": false, + "module": "CommonJS", + "moduleResolution": "node", + "noEmitOnError": false, + "noImplicitAny": true, + "noUnusedLocals": true, + "pretty": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "strict": true, + "sourceMap": true, + "target": "ES2017", + "rootDir": "src", + "baseUrl": "." + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules"] +} \ No newline at end of file diff --git a/schemas/index.json b/schemas/index.json deleted file mode 100644 index 2fe2195a..00000000 --- a/schemas/index.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "definitions": { - "RepoNameRegexIncludeFilter": { - "type": "string", - "description": "Only clone repos whose name matches the given regexp.", - "format": "regexp", - "default": "^(foo|bar)$" - }, - "RepoNameRegexExcludeFilter": { - "type": "string", - "description": "Don't mirror repos whose names match this regexp.", - "format": "regexp", - "default": "^(fizz|buzz)$" - }, - "ZoektConfig": { - "anyOf": [ - { - "$ref": "#/definitions/GitHubConfig" - }, - { - "$ref": "#/definitions/GitLabConfig" - } - ] - }, - "GitHubConfig": { - "type": "object", - "properties": { - "Type": { - "const": "github" - }, - "GitHubUrl": { - "type": "string", - "description": "GitHub Enterprise url. If not set github.com will be used as the host." - }, - "GitHubUser": { - "type": "string", - "description": "The GitHub user to mirror" - }, - "GitHubOrg": { - "type": "string", - "description": "The GitHub organization to mirror" - }, - "Name": { - "$ref": "#/definitions/RepoNameRegexIncludeFilter" - }, - "Exclude": { - "$ref": "#/definitions/RepoNameRegexExcludeFilter" - }, - "CredentialPath": { - "type": "string", - "description": "Path to a file containing a GitHub access token.", - "default": "~/.github-token" - }, - "Topics": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Only mirror repos that have one of the given topics" - }, - "ExcludeTopics": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Don't mirror repos that have one of the given topics" - }, - "NoArchived": { - "type": "boolean", - "description": "Mirror repos that are _not_ archived", - "default": false - }, - "IncludeForks": { - "type": "boolean", - "description": "Also mirror forks", - "default": false - } - }, - "required": [ - "Type" - ], - "additionalProperties": false - }, - "GitLabConfig": { - "type": "object", - "properties": { - "Type": { - "const": "gitlab" - }, - "GitLabURL": { - "type": "string", - "description": "The GitLab API url.", - "default": "https://gitlab.com/api/v4/" - }, - "Name": { - "$ref": "#/definitions/RepoNameRegexIncludeFilter" - }, - "Exclude": { - "$ref": "#/definitions/RepoNameRegexExcludeFilter" - }, - "OnlyPublic": { - "type": "boolean", - "description": "Only mirror public repos", - "default": false - }, - "CredentialPath": { - "type": "string", - "description": "Path to a file containing a GitLab access token.", - "default": "~/.gitlab-token" - } - }, - "required": [ - "Type" - ], - "additionalProperties": false - } - }, - "properties": { - "$schema": { - "type": "string" - }, - "Configs": { - "type": "array", - "items": { - "$ref": "#/definitions/ZoektConfig" - } - } - }, - "required": [ - "Configs" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/schemas/v3/github.json b/schemas/v3/github.json index 469d8b16..bc0c1d44 100644 --- a/schemas/v3/github.json +++ b/schemas/v3/github.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "title": "GitHubConfig", "properties": { "type": { "const": "github", From 5a53a41052b0385a86047b5a7a96b97d254efae7 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 23 Jan 2025 19:28:44 -0800 Subject: [PATCH 02/21] migrate things to use the schemas package --- Makefile | 2 + packages/backend/package.json | 5 +- packages/backend/src/config.ts | 2 +- packages/backend/src/gerrit.ts | 2 +- packages/backend/src/git.ts | 2 +- packages/backend/src/gitea.ts | 2 +- packages/backend/src/github.ts | 2 +- packages/backend/src/gitlab.ts | 2 +- packages/backend/src/local.ts | 2 +- packages/backend/src/main.ts | 2 +- packages/backend/src/schemas/v2.ts | 285 ------------------ packages/backend/tools/generateTypes.ts | 23 -- packages/schemas/tsconfig.json | 5 +- packages/web/package.json | 11 +- packages/web/src/actions.ts | 2 +- packages/web/src/app/connections/new/page.tsx | 2 +- packages/web/src/schemas/github.schema.ts | 1 + packages/web/tools/generateSchemas.ts | 30 -- yarn.lock | 45 +++ 19 files changed, 69 insertions(+), 358 deletions(-) delete mode 100644 packages/backend/src/schemas/v2.ts delete mode 100644 packages/backend/tools/generateTypes.ts delete mode 100644 packages/web/tools/generateSchemas.ts diff --git a/Makefile b/Makefile index 3e78a0b7..03a610e6 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ clean: packages/backend/node_modules \ packages/db/node_modules \ packages/db/dist \ + packages/schemas/node_modules \ + packages/schemas/dist \ .sourcebot .PHONY: bin diff --git a/packages/backend/package.json b/packages/backend/package.json index 244d6223..6a66d099 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -5,10 +5,8 @@ "main": "index.js", "type": "module", "scripts": { - "dev:watch": "yarn generate:types && tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"", + "dev:watch": "tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"", "dev": "export PATH=\"$PWD/../../bin:$PATH\" && export CTAGS_COMMAND=ctags && node ./dist/index.js", - "build": "yarn generate:types && tsc", - "generate:types": "tsx tools/generateTypes.ts", "test": "vitest --config ./vitest.config.ts" }, "devDependencies": { @@ -33,6 +31,7 @@ "micromatch": "^4.0.8", "posthog-node": "^4.2.1", "@sourcebot/db": "^0.1.0", + "@sourcebot/schemas": "^0.1.0", "simple-git": "^3.27.0", "strip-json-comments": "^5.0.1", "winston": "^3.15.0", diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 2434380c..a885a61a 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -3,7 +3,7 @@ import { readFile } from 'fs/promises'; import stripJsonComments from 'strip-json-comments'; import { getGitHubReposFromConfig } from "./github.js"; import { getGitLabReposFromConfig, GITLAB_CLOUD_HOSTNAME } from "./gitlab.js"; -import { SourcebotConfigurationSchema } from "./schemas/v2.js"; +import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type"; import { AppContext } from "./types.js"; import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js"; diff --git a/packages/backend/src/gerrit.ts b/packages/backend/src/gerrit.ts index 882f0218..21bd8e8b 100644 --- a/packages/backend/src/gerrit.ts +++ b/packages/backend/src/gerrit.ts @@ -1,5 +1,5 @@ import fetch from 'cross-fetch'; -import { GerritConfig } from './schemas/v2.js'; +import { GerritConfig } from "@sourcebot/schemas/v2/index.type" import { AppContext, GitRepository } from './types.js'; import { createLogger } from './logger.js'; import path from 'path'; diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index 77bcab10..40097dac 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,7 +1,7 @@ import { GitRepository, AppContext } from './types.js'; import { simpleGit, SimpleGitProgressEvent } from 'simple-git'; import { createLogger } from './logger.js'; -import { GitConfig } from './schemas/v2.js'; +import { GitConfig } from "@sourcebot/schemas/v2/index.type" import path from 'path'; const logger = createLogger('git'); diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index 831d38a6..f3a6b514 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -1,5 +1,5 @@ import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; -import { GiteaConfig } from './schemas/v2.js'; +import { GiteaConfig } from "@sourcebot/schemas/v2/index.type" import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js'; import { AppContext, GitRepository } from './types.js'; import fetch from 'cross-fetch'; diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index baa98512..104aff5c 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -1,5 +1,5 @@ import { Octokit } from "@octokit/rest"; -import { GitHubConfig } from "./schemas/v2.js"; +import { GitHubConfig } from "@sourcebot/schemas/v2/index.type" import { createLogger } from "./logger.js"; import { AppContext } from "./types.js"; import { getTokenFromConfig, measure } from "./utils.js"; diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 20d4e5b1..2501fd05 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -1,7 +1,7 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest"; import micromatch from "micromatch"; import { createLogger } from "./logger.js"; -import { GitLabConfig } from "./schemas/v2.js"; +import { GitLabConfig } from "@sourcebot/schemas/v2/index.type" import { AppContext } from "./types.js"; import { getTokenFromConfig, measure } from "./utils.js"; diff --git a/packages/backend/src/local.ts b/packages/backend/src/local.ts index 6e415546..0bddac6a 100644 --- a/packages/backend/src/local.ts +++ b/packages/backend/src/local.ts @@ -1,6 +1,6 @@ import { existsSync, FSWatcher, statSync, watch } from "fs"; import { createLogger } from "./logger.js"; -import { LocalConfig } from "./schemas/v2.js"; +import { LocalConfig } from "@sourcebot/schemas/v2/index.type" import { AppContext, LocalRepository } from "./types.js"; import { resolvePathRelativeToConfig } from "./utils.js"; import path from "path"; diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index c77ab0d1..34f0e28c 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -12,7 +12,7 @@ import { Queue, Worker, Job } from 'bullmq'; import { Redis } from 'ioredis'; import * as os from 'os'; import { SOURCEBOT_TENANT_MODE } from './environment.js'; -import { SourcebotConfigurationSchema } from './schemas/v2.js'; +import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type" const logger = createLogger('main'); diff --git a/packages/backend/src/schemas/v2.ts b/packages/backend/src/schemas/v2.ts deleted file mode 100644 index 4ada1870..00000000 --- a/packages/backend/src/schemas/v2.ts +++ /dev/null @@ -1,285 +0,0 @@ -// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! - -export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig | GitConfig; - -/** - * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index. - */ -export interface SourcebotConfigurationSchema { - $schema?: string; - settings?: Settings; - /** - * Defines a collection of repositories from varying code hosts that Sourcebot should sync with. - */ - repos?: Repos[]; -} -/** - * Global settings. These settings are applied to all repositories. - */ -export interface Settings { - /** - * The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. Defaults to 2MB (2097152 bytes). - */ - maxFileSize?: number; - /** - * Automatically delete stale repositories from the index. Defaults to true. - */ - autoDeleteStaleRepos?: boolean; - /** - * The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds). - */ - reindexInterval?: number; - /** - * The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds). - */ - resyncInterval?: number; -} -export interface GitHubConfig { - /** - * GitHub Configuration - */ - type: "github"; - /** - * A Personal Access Token (PAT). - */ - token?: - | string - | { - /** - * The name of the environment variable that contains the token. - */ - env: string; - }; - /** - * The URL of the GitHub host. Defaults to https://github.com - */ - url?: string; - /** - * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. - */ - users?: string[]; - /** - * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. - */ - orgs?: string[]; - /** - * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. - */ - repos?: string[]; - /** - * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. - * - * @minItems 1 - */ - topics?: string[]; - /** - * @nocheckin - */ - tenantId?: number; - exclude?: { - /** - * Exclude forked repositories from syncing. - */ - forks?: boolean; - /** - * Exclude archived repositories from syncing. - */ - archived?: boolean; - /** - * List of individual repositories to exclude from syncing. Glob patterns are supported. - */ - repos?: string[]; - /** - * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. - */ - topics?: string[]; - /** - * Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned. - */ - size?: { - /** - * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. - */ - min?: number; - /** - * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. - */ - max?: number; - }; - }; - revisions?: GitRevisions; -} -/** - * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. - */ -export interface GitRevisions { - /** - * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. - */ - branches?: string[]; - /** - * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. - */ - tags?: string[]; -} -export interface GitLabConfig { - /** - * GitLab Configuration - */ - type: "gitlab"; - /** - * An authentication token. - */ - token?: - | string - | { - /** - * The name of the environment variable that contains the token. - */ - env: string; - }; - /** - * The URL of the GitLab host. Defaults to https://gitlab.com - */ - url?: string; - /** - * Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com . - */ - all?: boolean; - /** - * List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. - */ - users?: string[]; - /** - * List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`). - */ - groups?: string[]; - /** - * List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/ - */ - projects?: string[]; - /** - * List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. - * - * @minItems 1 - */ - topics?: string[]; - exclude?: { - /** - * Exclude forked projects from syncing. - */ - forks?: boolean; - /** - * Exclude archived projects from syncing. - */ - archived?: boolean; - /** - * List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/ - */ - projects?: string[]; - /** - * List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. - */ - topics?: string[]; - }; - revisions?: GitRevisions; -} -export interface GiteaConfig { - /** - * Gitea Configuration - */ - type: "gitea"; - /** - * An access token. - */ - token?: - | string - | { - /** - * The name of the environment variable that contains the token. - */ - env: string; - }; - /** - * The URL of the Gitea host. Defaults to https://gitea.com - */ - url?: string; - /** - * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope. - */ - orgs?: string[]; - /** - * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. - */ - repos?: string[]; - /** - * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope. - */ - users?: string[]; - exclude?: { - /** - * Exclude forked repositories from syncing. - */ - forks?: boolean; - /** - * Exclude archived repositories from syncing. - */ - archived?: boolean; - /** - * List of individual repositories to exclude from syncing. Glob patterns are supported. - */ - repos?: string[]; - }; - revisions?: GitRevisions; -} -export interface GerritConfig { - /** - * Gerrit Configuration - */ - type: "gerrit"; - /** - * The URL of the Gerrit host. - */ - url: string; - /** - * List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported - */ - projects?: string[]; - exclude?: { - /** - * List of specific projects to exclude from syncing. - */ - projects?: string[]; - }; -} -export interface LocalConfig { - /** - * Local Configuration - */ - type: "local"; - /** - * Path to the local directory to sync with. Relative paths are relative to the configuration file's directory. - */ - path: string; - /** - * Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true. - */ - watch?: boolean; - exclude?: { - /** - * List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded. - */ - paths?: string[]; - }; -} -export interface GitConfig { - /** - * Git Configuration - */ - type: "git"; - /** - * The URL to the git repository. - */ - url: string; - revisions?: GitRevisions; -} diff --git a/packages/backend/tools/generateTypes.ts b/packages/backend/tools/generateTypes.ts deleted file mode 100644 index 9e78e137..00000000 --- a/packages/backend/tools/generateTypes.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { compileFromFile } from 'json-schema-to-typescript' -import path from 'path'; -import fs from 'fs'; - -const BANNER_COMMENT = '// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!\n'; - -(async () => { - const cwd = process.cwd(); - const schemaPath = path.resolve(`${cwd}/../../schemas/v2/index.json`); - const outputPath = path.resolve(`${cwd}/src/schemas/v2.ts`); - - const content = await compileFromFile(schemaPath, { - bannerComment: BANNER_COMMENT, - cwd, - ignoreMinAndMaxItems: true, - }); - - await fs.promises.writeFile( - outputPath, - content, - "utf-8" - ); -})(); \ No newline at end of file diff --git a/packages/schemas/tsconfig.json b/packages/schemas/tsconfig.json index 9a1ea53a..cbb69b61 100644 --- a/packages/schemas/tsconfig.json +++ b/packages/schemas/tsconfig.json @@ -20,7 +20,10 @@ "sourceMap": true, "target": "ES2017", "rootDir": "src", - "baseUrl": "." + "baseUrl": ".", + // This is needed otherwise .tsbuildinfo wasn't being written + // into the dist folder for some reason. + "tsBuildInfoFile": "./dist/.tsbuildinfo" }, "include": ["src/**/*.ts"], "exclude": ["node_modules"] diff --git a/packages/web/package.json b/packages/web/package.json index 6e7f0658..e35ae88e 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -3,12 +3,11 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "yarn generate:schemas && next dev", - "build": "yarn generate:schemas && next build", + "dev": "next dev", + "build": "next build", "start": "next start", "lint": "next lint", - "test": "vitest", - "generate:schemas": "tsx tools/generateSchemas.ts" + "test": "vitest" }, "dependencies": { "@auth/prisma-adapter": "^2.7.4", @@ -59,6 +58,8 @@ "@replit/codemirror-lang-svelte": "^6.0.0", "@replit/codemirror-vim": "^6.2.1", "@shopify/lang-jsonc": "^1.0.0", + "@sourcebot/schemas": "^0.1.0", + "@sourcebot/db": "^0.1.0", "@ssddanbrown/codemirror-lang-twig": "^1.0.0", "@tanstack/react-query": "^5.53.3", "@tanstack/react-table": "^8.20.5", @@ -115,8 +116,6 @@ "zod": "^3.23.8" }, "devDependencies": { - "@apidevtools/json-schema-ref-parser": "^11.7.3", - "@sourcebot/db": "^0.1.0", "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index dc20dbc9..6027ac96 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -5,9 +5,9 @@ import { getUser } from "./data/user"; import { auth } from "./auth"; import { notAuthenticated, notFound, ServiceError, unexpectedError } from "./lib/serviceError"; import { prisma } from "@/prisma"; -import { githubSchema } from "./schemas/github.schema"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "./lib/errorCodes"; +import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; const ajv = new Ajv({ validateFormats: false, diff --git a/packages/web/src/app/connections/new/page.tsx b/packages/web/src/app/connections/new/page.tsx index d986b1ef..e5239658 100644 --- a/packages/web/src/app/connections/new/page.tsx +++ b/packages/web/src/app/connections/new/page.tsx @@ -21,12 +21,12 @@ import { import { useCallback, useRef } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; -import { githubSchema } from "@/schemas/github.schema"; import { Input } from "@/components/ui/input"; import { createConnection } from "@/actions"; import { useToast } from "@/components/hooks/use-toast"; import { isServiceError } from "@/lib/utils"; import { useRouter } from "next/navigation"; +import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; const ajv = new Ajv({ validateFormats: false, diff --git a/packages/web/src/schemas/github.schema.ts b/packages/web/src/schemas/github.schema.ts index 0a784b22..0c88f81b 100644 --- a/packages/web/src/schemas/github.schema.ts +++ b/packages/web/src/schemas/github.schema.ts @@ -2,6 +2,7 @@ const schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "title": "GitHubConfig", "properties": { "type": { "const": "github", diff --git a/packages/web/tools/generateSchemas.ts b/packages/web/tools/generateSchemas.ts deleted file mode 100644 index 858765dc..00000000 --- a/packages/web/tools/generateSchemas.ts +++ /dev/null @@ -1,30 +0,0 @@ -import $RefParser from "@apidevtools/json-schema-ref-parser"; -import path from "path"; -import { writeFile } from "fs/promises"; - -const BANNER_COMMENT = '// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!\n'; -const SCHEMAS: string[] = ["github.json"]; - -(async () => { - - const cwd = process.cwd(); - const schemasPath = path.resolve(`${cwd}/../../schemas/v3`); - const outDir = path.resolve(`${cwd}/src/schemas`); - console.log(outDir); - - SCHEMAS.forEach(async (schemaName) => { - const schemaPath = path.resolve(`${schemasPath}/${schemaName}`); - const name = path.parse(schemaPath).name; - console.log(name); - - const schema = JSON.stringify(await $RefParser.bundle(schemaPath), null, 2); - - await writeFile( - path.join(outDir, `${name}.schema.ts`), - BANNER_COMMENT + - 'const schema = ' + - schema + - ` as const;\nexport { schema as ${name}Schema };`, - ); - }); -})(); \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 77536fcc..c7e2e797 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4172,6 +4172,11 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fdir@^6.4.2: + version "6.4.3" + resolved "https://registry.yarnpkg.com/fdir/-/fdir-6.4.3.tgz#011cdacf837eca9b811c89dbb902df714273db72" + integrity sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw== + fecha@^4.2.0: version "4.2.3" resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" @@ -4378,6 +4383,18 @@ glob@^11.0.0: package-json-from-dist "^1.0.0" path-scurry "^2.0.0" +glob@^11.0.1: + version "11.0.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-11.0.1.tgz#1c3aef9a59d680e611b53dcd24bb8639cef064d9" + integrity sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw== + dependencies: + foreground-child "^3.1.0" + jackspeak "^4.0.1" + minimatch "^10.0.0" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^2.0.0" + glob@^7.1.3: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -4985,6 +5002,21 @@ json-schema-to-typescript@^15.0.2: minimist "^1.2.8" prettier "^3.2.5" +json-schema-to-typescript@^15.0.4: + version "15.0.4" + resolved "https://registry.yarnpkg.com/json-schema-to-typescript/-/json-schema-to-typescript-15.0.4.tgz#a530c7f17312503b262ae12233749732171840f3" + integrity sha512-Su9oK8DR4xCmDsLlyvadkXzX6+GGXJpbhwoLtOGArAG61dvbW4YQmSEno2y66ahpIdmLMg6YUf/QHLgiwvkrHQ== + dependencies: + "@apidevtools/json-schema-ref-parser" "^11.5.5" + "@types/json-schema" "^7.0.15" + "@types/lodash" "^4.17.7" + is-glob "^4.0.3" + js-yaml "^4.1.0" + lodash "^4.17.21" + minimist "^1.2.8" + prettier "^3.2.5" + tinyglobby "^0.2.9" + json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" @@ -5756,6 +5788,11 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== +picomatch@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.2.tgz#77c742931e8f3b8820946c76cd0c1f13730d1dab" + integrity sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg== + pidtree@^0.3.0: version "0.3.1" resolved "https://registry.yarnpkg.com/pidtree/-/pidtree-0.3.1.tgz#ef09ac2cc0533df1f3250ccf2c4d366b0d12114a" @@ -6868,6 +6905,14 @@ tinyexec@^0.3.1: resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-0.3.1.tgz#0ab0daf93b43e2c211212396bdb836b468c97c98" integrity sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ== +tinyglobby@^0.2.9: + version "0.2.10" + resolved "https://registry.yarnpkg.com/tinyglobby/-/tinyglobby-0.2.10.tgz#e712cf2dc9b95a1f5c5bbd159720e15833977a0f" + integrity sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew== + dependencies: + fdir "^6.4.2" + picomatch "^4.0.2" + tinypool@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-1.0.1.tgz#c64233c4fac4304e109a64340178760116dbe1fe" From 46c38baa00636b6f41f7b6b79d311c160ffa124c Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 23 Jan 2025 19:43:04 -0800 Subject: [PATCH 03/21] Dockerfile support --- Dockerfile | 25 +++++++++++++++---------- packages/backend/package.json | 1 + 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8f74ec17..3720da60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,13 +10,15 @@ RUN go mod download COPY vendor/zoekt ./ RUN CGO_ENABLED=0 GOOS=linux go build -o /cmd/ ./cmd/... -# ------ Build Database ------ -FROM node-alpine AS database-builder +# ------ Build shared libraries ------ +FROM node-alpine AS shared-libs-builder WORKDIR /app COPY package.json yarn.lock* ./ COPY ./packages/db ./packages/db +COPY ./packages/schemas ./packages/schemas RUN yarn workspace @sourcebot/db install --frozen-lockfile +RUN yarn workspace @sourcebot/schemas install --frozen-lockfile # ------ Build Web ------ FROM node-alpine AS web-builder @@ -25,8 +27,9 @@ WORKDIR /app COPY package.json yarn.lock* ./ COPY ./packages/web ./packages/web -COPY --from=database-builder /app/node_modules ./node_modules -COPY --from=database-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/node_modules ./node_modules +COPY --from=shared-libs-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas # Fixes arm64 timeouts RUN yarn config set registry https://registry.npmjs.org/ @@ -54,8 +57,9 @@ WORKDIR /app COPY package.json yarn.lock* ./ COPY ./schemas ./schemas COPY ./packages/backend ./packages/backend -COPY --from=database-builder /app/node_modules ./node_modules -COPY --from=database-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/node_modules ./node_modules +COPY --from=shared-libs-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas RUN yarn workspace @sourcebot/backend install --frozen-lockfile RUN yarn workspace @sourcebot/backend build @@ -114,21 +118,22 @@ COPY --from=zoekt-builder \ /cmd/zoekt-index \ /usr/local/bin/ -# Configure the webapp +# Copy all of the things COPY --from=web-builder /app/packages/web/public ./packages/web/public COPY --from=web-builder /app/packages/web/.next/standalone ./ COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/static -# Configure the backend COPY --from=backend-builder /app/node_modules ./node_modules COPY --from=backend-builder /app/packages/backend ./packages/backend +COPY --from=shared-libs-builder /app/node_modules ./node_modules +COPY --from=shared-libs-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas + # Configure the database RUN mkdir -p /run/postgresql && \ chown -R postgres:postgres /run/postgresql && \ chmod 775 /run/postgresql -COPY --from=database-builder /app/node_modules ./node_modules -COPY --from=database-builder /app/packages/db ./packages/db COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY prefix-output.sh ./prefix-output.sh diff --git a/packages/backend/package.json b/packages/backend/package.json index 6a66d099..3bc4cae6 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -7,6 +7,7 @@ "scripts": { "dev:watch": "tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"", "dev": "export PATH=\"$PWD/../../bin:$PATH\" && export CTAGS_COMMAND=ctags && node ./dist/index.js", + "build": "tsc", "test": "vitest --config ./vitest.config.ts" }, "devDependencies": { From c68035656c94ca62f5c22aa27c3ee3b7551f6d83 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 24 Jan 2025 13:13:54 -0800 Subject: [PATCH 04/21] add secret table to schema --- .../migration.sql | 13 +++++++++ packages/db/prisma/schema.prisma | 14 +++++++++ .../web/src/app/api/(server)/secret/route.ts | 29 +++++++++++++++++++ packages/web/src/lib/schemas.ts | 9 ++++++ 4 files changed, 65 insertions(+) create mode 100644 packages/db/prisma/migrations/20250124190846_add_secret_table/migration.sql create mode 100644 packages/web/src/app/api/(server)/secret/route.ts diff --git a/packages/db/prisma/migrations/20250124190846_add_secret_table/migration.sql b/packages/db/prisma/migrations/20250124190846_add_secret_table/migration.sql new file mode 100644 index 00000000..faf5be73 --- /dev/null +++ b/packages/db/prisma/migrations/20250124190846_add_secret_table/migration.sql @@ -0,0 +1,13 @@ +-- CreateTable +CREATE TABLE "Secret" ( + "orgId" INTEGER NOT NULL, + "key" TEXT NOT NULL, + "encryptedValue" TEXT NOT NULL, + "iv" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "Secret_pkey" PRIMARY KEY ("orgId","key") +); + +-- AddForeignKey +ALTER TABLE "Secret" ADD CONSTRAINT "Secret_orgId_fkey" FOREIGN KEY ("orgId") REFERENCES "Org"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 7490155d..7909f25b 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -75,6 +75,7 @@ model Org { members UserToOrg[] configs Config[] repos Repo[] + secrets Secret[] } enum OrgRole { @@ -98,6 +99,19 @@ model UserToOrg { @@id([orgId, userId]) } +model Secret { + orgId Int + key String + encryptedValue String + iv String + + createdAt DateTime @default(now()) + + org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) + + @@id([orgId, key]) +} + // @see : https://authjs.dev/concepts/database-models#user model User { id String @id @default(cuid()) diff --git a/packages/web/src/app/api/(server)/secret/route.ts b/packages/web/src/app/api/(server)/secret/route.ts new file mode 100644 index 00000000..72ea388d --- /dev/null +++ b/packages/web/src/app/api/(server)/secret/route.ts @@ -0,0 +1,29 @@ +'user server'; + +import { secretCreateRequestSchema, secreteDeleteRequestSchema } from "@/lib/schemas"; +import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; +import { NextRequest } from "next/server"; + +export const POST = async (request: NextRequest) => { + const body = await request.json(); + const parsed = await secretCreateRequestSchema.safeParseAsync(body); + if (!parsed.success) { + return serviceErrorResponse( + schemaValidationError(parsed.error) + ) + } + + return Response.json({ success: true }); +} + +export const DELETE = async (request: NextRequest) => { + const body = await request.json(); + const parsed = await secreteDeleteRequestSchema.safeParseAsync(body); + if (!parsed.success) { + return serviceErrorResponse( + schemaValidationError(parsed.error) + ) + } + + return Response.json({ success: true }); +} \ No newline at end of file diff --git a/packages/web/src/lib/schemas.ts b/packages/web/src/lib/schemas.ts index 4c6ef988..edfe4eb1 100644 --- a/packages/web/src/lib/schemas.ts +++ b/packages/web/src/lib/schemas.ts @@ -98,6 +98,15 @@ export const fileSourceResponseSchema = z.object({ language: z.string(), }); +export const secretCreateRequestSchema = z.object({ + key: z.string(), + value: z.string(), +}); + +export const secreteDeleteRequestSchema = z.object({ + key: z.string(), +}); + // @see : https://github.com/sourcebot-dev/zoekt/blob/3780e68cdb537d5a7ed2c84d9b3784f80c7c5d04/api.go#L728 const repoStatsSchema = z.object({ From 71b9d5c8bbc7e1c2590dceb13b1c00390a66bfc2 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 23 Jan 2025 22:50:02 -0800 Subject: [PATCH 05/21] Add concept of connection manager --- packages/backend/src/config.ts | 139 ------------- packages/backend/src/connectionManager.ts | 147 +++++++++++++ packages/backend/src/constants.ts | 2 +- packages/backend/src/github.ts | 8 +- packages/backend/src/main.ts | 176 ++-------------- packages/backend/src/types.ts | 4 +- packages/schemas/src/v3/connection.schema.ts | 207 +++++++++++++++++++ packages/schemas/src/v3/connection.type.ts | 88 ++++++++ packages/schemas/src/v3/github.schema.ts | 6 +- packages/schemas/src/v3/github.type.ts | 6 +- schemas/v3/connection.json | 9 + schemas/v3/github.json | 6 +- 12 files changed, 476 insertions(+), 322 deletions(-) delete mode 100644 packages/backend/src/config.ts create mode 100644 packages/backend/src/connectionManager.ts create mode 100644 packages/schemas/src/v3/connection.schema.ts create mode 100644 packages/schemas/src/v3/connection.type.ts create mode 100644 schemas/v3/connection.json diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts deleted file mode 100644 index a885a61a..00000000 --- a/packages/backend/src/config.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { PrismaClient } from '@sourcebot/db'; -import { readFile } from 'fs/promises'; -import stripJsonComments from 'strip-json-comments'; -import { getGitHubReposFromConfig } from "./github.js"; -import { getGitLabReposFromConfig, GITLAB_CLOUD_HOSTNAME } from "./gitlab.js"; -import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type"; -import { AppContext } from "./types.js"; -import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js"; - -export const fetchConfigFromPath = async (configPath: string, signal: AbortSignal) => { - const configContent = await (async () => { - if (isRemotePath(configPath)) { - const response = await fetch(configPath, { - signal, - }); - if (!response.ok) { - throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`); - } - return response.text(); - } else { - return readFile(configPath, { - encoding: 'utf-8', - signal, - }); - } - })(); - - const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfigurationSchema; - return config; -} - -export const syncConfig = async (config: SourcebotConfigurationSchema, db: PrismaClient, signal: AbortSignal, ctx: AppContext) => { - for (const repoConfig of config.repos ?? []) { - switch (repoConfig.type) { - case 'github': { - const token = repoConfig.token ? getTokenFromConfig(repoConfig.token, ctx) : undefined; - const gitHubRepos = await getGitHubReposFromConfig(repoConfig, signal, ctx); - const hostUrl = repoConfig.url ?? 'https://github.com'; - const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : 'github.com'; - const tenantId = repoConfig.tenantId ?? 0; - - await Promise.all(gitHubRepos.map((repo) => { - const repoName = `${hostname}/${repo.full_name}`; - const cloneUrl = new URL(repo.clone_url!); - if (token) { - cloneUrl.username = token; - } - - const data = { - external_id: repo.id.toString(), - external_codeHostType: 'github', - external_codeHostUrl: hostUrl, - cloneUrl: cloneUrl.toString(), - name: repoName, - isFork: repo.fork, - isArchived: !!repo.archived, - tenantId: tenantId, - metadata: { - 'zoekt.web-url-type': 'github', - 'zoekt.web-url': repo.html_url, - 'zoekt.name': repoName, - 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), - 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), - 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), - 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), - 'zoekt.archived': marshalBool(repo.archived), - 'zoekt.fork': marshalBool(repo.fork), - 'zoekt.public': marshalBool(repo.private === false) - }, - }; - - return db.repo.upsert({ - where: { - external_id_external_codeHostUrl: { - external_id: repo.id.toString(), - external_codeHostUrl: hostUrl, - }, - }, - create: data, - update: data, - }) - })); - - break; - } - case 'gitlab': { - const hostUrl = repoConfig.url ?? 'https://gitlab.com'; - const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : GITLAB_CLOUD_HOSTNAME; - const token = repoConfig.token ? getTokenFromConfig(repoConfig.token, ctx) : undefined; - const gitLabRepos = await getGitLabReposFromConfig(repoConfig, ctx); - - await Promise.all(gitLabRepos.map((project) => { - const repoName = `${hostname}/${project.path_with_namespace}`; - const isFork = project.forked_from_project !== undefined; - - const cloneUrl = new URL(project.http_url_to_repo); - if (token) { - cloneUrl.username = 'oauth2'; - cloneUrl.password = token; - } - - const data = { - external_id: project.id.toString(), - external_codeHostType: 'gitlab', - external_codeHostUrl: hostUrl, - cloneUrl: cloneUrl.toString(), - name: repoName, - tenantId: 0, // TODO: add support for tenantId in GitLab config - isFork, - isArchived: !!project.archived, - metadata: { - 'zoekt.web-url-type': 'gitlab', - 'zoekt.web-url': project.web_url, - 'zoekt.name': repoName, - 'zoekt.gitlab-stars': project.star_count?.toString() ?? '0', - 'zoekt.gitlab-forks': project.forks_count?.toString() ?? '0', - 'zoekt.archived': marshalBool(project.archived), - 'zoekt.fork': marshalBool(isFork), - 'zoekt.public': marshalBool(project.visibility === 'public'), - } - } - - return db.repo.upsert({ - where: { - external_id_external_codeHostUrl: { - external_id: project.id.toString(), - external_codeHostUrl: hostUrl, - }, - }, - create: data, - update: data, - }) - })); - - break; - } - } - } -} \ No newline at end of file diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts new file mode 100644 index 00000000..df50e54a --- /dev/null +++ b/packages/backend/src/connectionManager.ts @@ -0,0 +1,147 @@ +import { Connection, ConnectionSyncStatus, PrismaClient } from "@sourcebot/db"; +import { Job, Queue, Worker } from 'bullmq'; +import { AppContext, Settings } from "./types.js"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; +import { createLogger } from "./logger.js"; +import os from 'os'; +import { Redis } from 'ioredis'; +import { getTokenFromConfig, marshalBool } from "./utils.js"; +import { getGitHubReposFromConfig } from "./github.js"; + +interface IConnectionManager { + scheduleConnectionSync: (connection: Connection) => Promise; + dispose: () => void; +} + +const QUEUE_NAME = 'configSyncQueue'; + +type JobPayload = { + connectionId: number, + orgId: number, + config: ConnectionConfig, +}; + +export class ConnectionManager implements IConnectionManager { + private queue = new Queue(QUEUE_NAME); + private worker: Worker; + private logger = createLogger('ConnectionManager'); + + constructor( + private db: PrismaClient, + settings: Settings, + redis: Redis, + private context: AppContext, + ) { + const numCores = os.cpus().length; + this.worker = new Worker(QUEUE_NAME, this.runSyncJob.bind(this), { + connection: redis, + concurrency: numCores * settings.configSyncConcurrencyMultiple, + }); + this.worker.on('completed', this.onSyncJobCompleted.bind(this)); + this.worker.on('failed', this.onSyncJobFailed.bind(this)); + } + + public async scheduleConnectionSync(connection: Connection) { + await this.db.$transaction(async (tx) => { + await tx.connection.update({ + where: { id: connection.id }, + data: { syncStatus: ConnectionSyncStatus.IN_SYNC_QUEUE }, + }); + + const connectionConfig = connection.config as unknown as ConnectionConfig; + + await this.queue.add('connectionSyncJob', { + connectionId: connection.id, + orgId: connection.orgId, + config: connectionConfig, + }); + this.logger.info(`Added job to queue for connection ${connection.id}`); + }).catch((err: unknown) => { + this.logger.error(`Failed to add job to queue for connection ${connection.id}: ${err}`); + }); + } + + private async runSyncJob(job: Job) { + const { config, orgId } = job.data; + // @note: We aren't actually doing anything with this atm. + const abortController = new AbortController(); + + switch (config.type) { + case 'github': { + const token = config.token ? getTokenFromConfig(config.token, this.context) : undefined; + const gitHubRepos = await getGitHubReposFromConfig(config, abortController.signal, this.context); + const hostUrl = config.url ?? 'https://github.com'; + const hostname = config.url ? new URL(config.url).hostname : 'github.com'; + + await Promise.all(gitHubRepos.map((repo) => { + const repoName = `${hostname}/${repo.full_name}`; + const cloneUrl = new URL(repo.clone_url!); + if (token) { + cloneUrl.username = token; + } + + const data = { + external_id: repo.id.toString(), + external_codeHostType: 'github', + external_codeHostUrl: hostUrl, + cloneUrl: cloneUrl.toString(), + name: repoName, + isFork: repo.fork, + isArchived: !!repo.archived, + orgId, + metadata: { + 'zoekt.web-url-type': 'github', + 'zoekt.web-url': repo.html_url, + 'zoekt.name': repoName, + 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), + 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), + 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), + 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), + 'zoekt.archived': marshalBool(repo.archived), + 'zoekt.fork': marshalBool(repo.fork), + 'zoekt.public': marshalBool(repo.private === false) + }, + }; + + return this.db.repo.upsert({ + where: { + external_id_external_codeHostUrl: { + external_id: repo.id.toString(), + external_codeHostUrl: hostUrl, + }, + }, + create: data, + update: data, + }) + })); + break; + } + } + } + + + private async onSyncJobCompleted(job: Job) { + this.logger.info(`Config sync job ${job.id} completed`); + const { connectionId } = job.data; + + await this.db.connection.update({ + where: { + id: connectionId, + }, + data: { + syncStatus: ConnectionSyncStatus.SYNCED, + syncedAt: new Date() + } + }) + } + + private async onSyncJobFailed(_job: Job | undefined, err: unknown) { + this.logger.info(`Config sync job failed with error: ${err}`); + } + + public dispose() { + this.worker.close(); + this.queue.close(); + } +} + diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index 0c983120..407ec462 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -7,7 +7,7 @@ export const DEFAULT_SETTINGS: Settings = { maxFileSize: 2 * 1024 * 1024, // 2MB in bytes autoDeleteStaleRepos: true, reindexIntervalMs: 1000 * 60, - resyncIntervalMs: 1000 * 60 * 60 * 24, // 1 day in milliseconds + resyncConnectionPollingIntervalMs: 1000, indexConcurrencyMultiple: 3, configSyncConcurrencyMultiple: 3, } \ No newline at end of file diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 104aff5c..f344832a 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -1,5 +1,5 @@ import { Octokit } from "@octokit/rest"; -import { GitHubConfig } from "@sourcebot/schemas/v2/index.type" +import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { createLogger } from "./logger.js"; import { AppContext } from "./types.js"; import { getTokenFromConfig, measure } from "./utils.js"; @@ -25,7 +25,7 @@ export type OctokitRepository = { size?: number, } -export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: AbortSignal, ctx: AppContext) => { +export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, signal: AbortSignal, ctx: AppContext) => { const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined; const octokit = new Octokit({ @@ -93,9 +93,9 @@ export const shouldExcludeRepo = ({ } : { repo: OctokitRepository, include?: { - topics?: GitHubConfig['topics'] + topics?: GithubConnectionConfig['topics'] }, - exclude?: GitHubConfig['exclude'] + exclude?: GithubConnectionConfig['exclude'] }) => { let reason = ''; const repoName = repo.full_name; diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 34f0e28c..7e9d2f80 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -1,18 +1,16 @@ -import { ConfigSyncStatus, PrismaClient, Repo, Config, RepoIndexingStatus, Prisma } from '@sourcebot/db'; -import { existsSync, watch } from 'fs'; -import { fetchConfigFromPath, syncConfig } from "./config.js"; +import { ConnectionSyncStatus, PrismaClient, Repo, RepoIndexingStatus } from '@sourcebot/db'; +import { existsSync } from 'fs'; import { cloneRepository, fetchRepository } from "./git.js"; import { createLogger } from "./logger.js"; import { captureEvent } from "./posthog.js"; import { AppContext } from "./types.js"; -import { getRepoPath, isRemotePath, measure } from "./utils.js"; +import { getRepoPath, measure } from "./utils.js"; import { indexGitRepository } from "./zoekt.js"; import { DEFAULT_SETTINGS } from './constants.js'; import { Queue, Worker, Job } from 'bullmq'; import { Redis } from 'ioredis'; import * as os from 'os'; -import { SOURCEBOT_TENANT_MODE } from './environment.js'; -import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type" +import { ConnectionManager } from './connectionManager.js'; const logger = createLogger('main'); @@ -58,23 +56,6 @@ const syncGitRepository = async (repo: Repo, ctx: AppContext) => { } } -async function addConfigsToQueue(db: PrismaClient, queue: Queue, configs: Config[]) { - for (const config of configs) { - await db.$transaction(async (tx) => { - await tx.config.update({ - where: { id: config.id }, - data: { syncStatus: ConfigSyncStatus.IN_SYNC_QUEUE }, - }); - - // Add the job to the queue - await queue.add('configSyncJob', config); - logger.info(`Added job to queue for config ${config.id}`); - }).catch((err: unknown) => { - logger.error(`Failed to add job to queue for config ${config.id}: ${err}`); - }); - } -} - async function addReposToQueue(db: PrismaClient, queue: Queue, repos: Repo[]) { for (const repo of repos) { await db.$transaction(async (tx) => { @@ -93,79 +74,6 @@ async function addReposToQueue(db: PrismaClient, queue: Queue, repos: Repo[]) { } export const main = async (db: PrismaClient, context: AppContext) => { - let abortController = new AbortController(); - let isSyncing = false; - const _syncConfig = async (dbConfig?: Config | undefined) => { - - // Fetch config object and update syncing status - let config: SourcebotConfigurationSchema; - switch (SOURCEBOT_TENANT_MODE) { - case 'single': - logger.info(`Syncing configuration file ${context.configPath} ...`); - - if (isSyncing) { - abortController.abort(); - abortController = new AbortController(); - } - config = await fetchConfigFromPath(context.configPath, abortController.signal); - isSyncing = true; - break; - case 'multi': - if(!dbConfig) { - throw new Error('config object is required in multi tenant mode'); - } - config = dbConfig.data as SourcebotConfigurationSchema - db.config.update({ - where: { - id: dbConfig.id, - }, - data: { - syncStatus: ConfigSyncStatus.SYNCING, - } - }) - break; - default: - throw new Error(`Invalid SOURCEBOT_TENANT_MODE: ${SOURCEBOT_TENANT_MODE}`); - } - - // Attempt to sync the config, handle failure cases - try { - const { durationMs } = await measure(() => syncConfig(config, db, abortController.signal, context)) - logger.info(`Synced configuration in ${durationMs / 1000}s`); - isSyncing = false; - } catch (err: any) { - switch(SOURCEBOT_TENANT_MODE) { - case 'single': - if (err.name === "AbortError") { - // @note: If we're aborting, we don't want to set isSyncing to false - // since it implies another sync is in progress. - } else { - isSyncing = false; - logger.error(`Failed to sync configuration file with error:`); - console.log(err); - } - break; - case 'multi': - if (dbConfig) { - await db.config.update({ - where: { - id: dbConfig.id, - }, - data: { - syncStatus: ConfigSyncStatus.FAILED, - } - }) - logger.error(`Failed to sync configuration ${dbConfig.id} with error: ${err}`); - } else { - logger.error(`DB config undefined. Failed to sync configuration with error: ${err}`); - } - break; - default: - throw new Error(`Invalid SOURCEBOT_TENANT_MODE: ${SOURCEBOT_TENANT_MODE}`); - } - } - } - ///////////////////////////// // Init Redis ///////////////////////////// @@ -182,71 +90,18 @@ export const main = async (db: PrismaClient, context: AppContext) => { process.exit(1); }); - ///////////////////////////// - // Setup config sync watchers - ///////////////////////////// - switch (SOURCEBOT_TENANT_MODE) { - case 'single': - // Re-sync on file changes if the config file is local - if (!isRemotePath(context.configPath)) { - watch(context.configPath, () => { - logger.info(`Config file ${context.configPath} changed. Re-syncing...`); - _syncConfig(); - }); + const connectionManager = new ConnectionManager(db, DEFAULT_SETTINGS, redis, context); + setInterval(async () => { + const configs = await db.connection.findMany({ + where: { + syncStatus: ConnectionSyncStatus.SYNC_NEEDED, } - - // Re-sync at a fixed interval - setInterval(() => { - _syncConfig(); - }, DEFAULT_SETTINGS.resyncIntervalMs); - - // Sync immediately on startup - await _syncConfig(); - break; - case 'multi': - // Setup config sync queue and workers - const configSyncQueue = new Queue('configSyncQueue'); - const numCores = os.cpus().length; - const numWorkers = numCores * DEFAULT_SETTINGS.configSyncConcurrencyMultiple; - logger.info(`Detected ${numCores} cores. Setting config sync max concurrency to ${numWorkers}`); - const configSyncWorker = new Worker('configSyncQueue', async (job: Job) => { - const config = job.data as Config; - await _syncConfig(config); - }, { connection: redis, concurrency: numWorkers }); - configSyncWorker.on('completed', async (job: Job) => { - logger.info(`Config sync job ${job.id} completed`); - - const config = job.data as Config; - await db.config.update({ - where: { - id: config.id, - }, - data: { - syncStatus: ConfigSyncStatus.SYNCED, - syncedAt: new Date() - } - }) - }); - configSyncWorker.on('failed', (job: Job | undefined, err: unknown) => { - logger.info(`Config sync job failed with error: ${err}`); - }); - - setInterval(async () => { - const configs = await db.config.findMany({ - where: { - syncStatus: ConfigSyncStatus.SYNC_NEEDED, - } - }); - - logger.info(`Found ${configs.length} configs to sync...`); - addConfigsToQueue(db, configSyncQueue, configs); - }, 1000); - break; - default: - throw new Error(`Invalid SOURCEBOT_TENANT_MODE: ${SOURCEBOT_TENANT_MODE}`); - } - - + }); + for (const config of configs) { + await connectionManager.scheduleConnectionSync(config); + } + }, DEFAULT_SETTINGS.resyncConnectionPollingIntervalMs); + ///////////////////////// // Setup repo indexing ///////////////////////// @@ -318,7 +173,6 @@ export const main = async (db: PrismaClient, context: AppContext) => { ] } }); - logger.info(`Found ${repos.length} repos to index...`); addReposToQueue(db, indexQueue, repos); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index a6e80b81..9e714ccb 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -71,9 +71,9 @@ export type Settings = { */ reindexIntervalMs: number; /** - * The interval (in milliseconds) at which the configuration file should be re-synced. + * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. */ - resyncIntervalMs: number; + resyncConnectionPollingIntervalMs: number; /** * The multiple of the number of CPUs to use for indexing. */ diff --git a/packages/schemas/src/v3/connection.schema.ts b/packages/schemas/src/v3/connection.schema.ts new file mode 100644 index 00000000..0a86ecf2 --- /dev/null +++ b/packages/schemas/src/v3/connection.schema.ts @@ -0,0 +1,207 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ConnectionConfig", + "oneOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GithubConnectionConfig", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ], + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://github.com", + "description": "The URL of the GitHub host. Defaults to https://github.com", + "examples": [ + "https://github.com", + "https://github.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "users": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "torvalds", + "DHH" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "my-org-name" + ], + [ + "sourcebot-dev", + "commaai" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type" + ], + "additionalProperties": false + } + ] +} as const; +export { schema as connectionSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/connection.type.ts b/packages/schemas/src/v3/connection.type.ts new file mode 100644 index 00000000..30c0ff27 --- /dev/null +++ b/packages/schemas/src/v3/connection.type.ts @@ -0,0 +1,88 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +export type ConnectionConfig = GithubConnectionConfig; + +export interface GithubConnectionConfig { + /** + * GitHub Configuration + */ + type: "github"; + /** + * A Personal Access Token (PAT). + */ + token?: + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + /** + * The URL of the GitHub host. Defaults to https://github.com + */ + url?: string; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + /** + * Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + */ + tags?: string[]; +} diff --git a/packages/schemas/src/v3/github.schema.ts b/packages/schemas/src/v3/github.schema.ts index 0c88f81b..546e10a8 100644 --- a/packages/schemas/src/v3/github.schema.ts +++ b/packages/schemas/src/v3/github.schema.ts @@ -2,7 +2,7 @@ const schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "title": "GitHubConfig", + "title": "GithubConnectionConfig", "properties": { "type": { "const": "github", @@ -99,10 +99,6 @@ const schema = { ] ] }, - "tenantId": { - "type": "number", - "description": "@nocheckin" - }, "exclude": { "type": "object", "properties": { diff --git a/packages/schemas/src/v3/github.type.ts b/packages/schemas/src/v3/github.type.ts index d9c7e1aa..d7355c5b 100644 --- a/packages/schemas/src/v3/github.type.ts +++ b/packages/schemas/src/v3/github.type.ts @@ -1,6 +1,6 @@ // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! -export interface GitHubConfig { +export interface GithubConnectionConfig { /** * GitHub Configuration */ @@ -38,10 +38,6 @@ export interface GitHubConfig { * @minItems 1 */ topics?: string[]; - /** - * @nocheckin - */ - tenantId?: number; exclude?: { /** * Exclude forked repositories from syncing. diff --git a/schemas/v3/connection.json b/schemas/v3/connection.json new file mode 100644 index 00000000..a40c76e1 --- /dev/null +++ b/schemas/v3/connection.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ConnectionConfig", + "oneOf": [ + { + "$ref": "./github.json" + } + ] +} \ No newline at end of file diff --git a/schemas/v3/github.json b/schemas/v3/github.json index bc0c1d44..dad32b1c 100644 --- a/schemas/v3/github.json +++ b/schemas/v3/github.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "title": "GitHubConfig", + "title": "GithubConnectionConfig", "properties": { "type": { "const": "github", @@ -81,10 +81,6 @@ ] ] }, - "tenantId": { - "type": "number", - "description": "@nocheckin" - }, "exclude": { "type": "object", "properties": { From 20b09b05b7aca6ae79c20bff14aff484d095f4f2 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 23 Jan 2025 22:50:41 -0800 Subject: [PATCH 06/21] Rename Config->Connection --- packages/backend/src/zoekt.ts | 6 +-- .../migration.sql | 33 +++++++++++++++ .../migration.sql | 10 +++++ packages/db/prisma/schema.prisma | 41 +++++++++---------- packages/web/src/actions.ts | 4 +- 5 files changed, 67 insertions(+), 27 deletions(-) create mode 100644 packages/db/prisma/migrations/20250124045320_rename_config_to_connection/migration.sql create mode 100644 packages/db/prisma/migrations/20250124063518_remove_repo_tenant_id/migration.sql diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index e5c7ffc5..088ae03c 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -11,11 +11,9 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => { 'HEAD' ]; - const tenantId = repo.tenantId ?? 0; - const shardPrefix = `${tenantId}_${repo.id}`; - + const shardPrefix = `${repo.orgId}_${repo.id}`; const repoPath = getRepoPath(repo, ctx); - const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${tenantId} -shard_prefix ${shardPrefix} ${repoPath}`; + const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${repo.orgId} -shard_prefix ${shardPrefix} ${repoPath}`; return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { exec(command, (error, stdout, stderr) => { diff --git a/packages/db/prisma/migrations/20250124045320_rename_config_to_connection/migration.sql b/packages/db/prisma/migrations/20250124045320_rename_config_to_connection/migration.sql new file mode 100644 index 00000000..059b5ccd --- /dev/null +++ b/packages/db/prisma/migrations/20250124045320_rename_config_to_connection/migration.sql @@ -0,0 +1,33 @@ +/* + Warnings: + + - You are about to drop the `Config` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- CreateEnum +CREATE TYPE "ConnectionSyncStatus" AS ENUM ('SYNC_NEEDED', 'IN_SYNC_QUEUE', 'SYNCING', 'SYNCED', 'FAILED'); + +-- DropForeignKey +ALTER TABLE "Config" DROP CONSTRAINT "Config_orgId_fkey"; + +-- DropTable +DROP TABLE "Config"; + +-- DropEnum +DROP TYPE "ConfigSyncStatus"; + +-- CreateTable +CREATE TABLE "Connection" ( + "id" SERIAL NOT NULL, + "config" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "syncedAt" TIMESTAMP(3), + "syncStatus" "ConnectionSyncStatus" NOT NULL DEFAULT 'SYNC_NEEDED', + "orgId" INTEGER NOT NULL, + + CONSTRAINT "Connection_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "Connection" ADD CONSTRAINT "Connection_orgId_fkey" FOREIGN KEY ("orgId") REFERENCES "Org"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20250124063518_remove_repo_tenant_id/migration.sql b/packages/db/prisma/migrations/20250124063518_remove_repo_tenant_id/migration.sql new file mode 100644 index 00000000..4a61724d --- /dev/null +++ b/packages/db/prisma/migrations/20250124063518_remove_repo_tenant_id/migration.sql @@ -0,0 +1,10 @@ +/* + Warnings: + + - You are about to drop the column `tenantId` on the `Repo` table. All the data in the column will be lost. + - Made the column `orgId` on table `Repo` required. This step will fail if there are existing NULL values in that column. + +*/ +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "tenantId", +ALTER COLUMN "orgId" SET NOT NULL; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 7909f25b..ab3cada5 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -18,7 +18,7 @@ enum RepoIndexingStatus { FAILED } -enum ConfigSyncStatus { +enum ConnectionSyncStatus { SYNC_NEEDED IN_SYNC_QUEUE SYNCING @@ -36,7 +36,6 @@ model Repo { isArchived Boolean metadata Json cloneUrl String - tenantId Int repoIndexingStatus RepoIndexingStatus @default(NEW) @@ -47,35 +46,35 @@ model Repo { // The base url of the external service (e.g., https://github.com) external_codeHostUrl String - org Org? @relation(fields: [orgId], references: [id], onDelete: Cascade) - orgId Int? + org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) + orgId Int @@unique([external_id, external_codeHostUrl]) } -model Config { - id Int @id @default(autoincrement()) - data Json - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt +model Connection { + id Int @id @default(autoincrement()) + config Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt syncedAt DateTime? - syncStatus ConfigSyncStatus @default(SYNC_NEEDED) + syncStatus ConnectionSyncStatus @default(SYNC_NEEDED) - // The organization that owns this config - org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) - orgId Int + // The organization that owns this connection + org Org @relation(fields: [orgId], references: [id], onDelete: Cascade) + orgId Int } model Org { - id Int @id @default(autoincrement()) - name String - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - members UserToOrg[] - configs Config[] - repos Repo[] - secrets Secret[] + id Int @id @default(autoincrement()) + name String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + members UserToOrg[] + connections Connection[] + repos Repo[] + secrets Secret[] } enum OrgRole { diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 6027ac96..717a74fe 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -121,10 +121,10 @@ export const createConnection = async (config: string): Promise<{ id: number } | } satisfies ServiceError; } - const connection = await prisma.config.create({ + const connection = await prisma.connection.create({ data: { orgId: orgId, - data: parsedConfig, + config: parsedConfig, } }); From cc6bf12f27045795f4e957d9d78f67e89a668a13 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 23 Jan 2025 23:04:31 -0800 Subject: [PATCH 07/21] Handle job failures --- packages/backend/src/connectionManager.ts | 14 +++++++++++++- packages/backend/src/github.ts | 9 ++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index df50e54a..5c24362d 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -135,8 +135,20 @@ export class ConnectionManager implements IConnectionManager { }) } - private async onSyncJobFailed(_job: Job | undefined, err: unknown) { + private async onSyncJobFailed(job: Job | undefined, err: unknown) { this.logger.info(`Config sync job failed with error: ${err}`); + if (job) { + const { connectionId } = job.data; + await this.db.connection.update({ + where: { + id: connectionId, + }, + data: { + syncStatus: ConnectionSyncStatus.FAILED, + syncedAt: new Date() + } + }) + } } public dispose() { diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index f344832a..6680ce15 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -202,8 +202,9 @@ const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, o logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`); return data; } catch (e) { + // @todo: handle rate limiting errors logger.error(`Failed to fetch repository info for user ${user}.`, e); - return []; + throw e; } }))).flat(); @@ -226,8 +227,9 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`); return data; } catch (e) { + // @todo: handle rate limiting errors logger.error(`Failed to fetch repository info for org ${org}.`, e); - return []; + throw e; } }))).flat(); @@ -252,8 +254,9 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna return [result.data]; } catch (e) { + // @todo: handle rate limiting errors logger.error(`Failed to fetch repository info for ${repo}.`, e); - return []; + throw e; } }))).flat(); From c836adbb4be62baffd1d787851434a566209ac27 Mon Sep 17 00:00:00 2001 From: bkellam Date: Fri, 24 Jan 2025 10:45:42 -0800 Subject: [PATCH 08/21] Add join table between repo and connection --- packages/backend/src/connectionManager.ts | 136 +++++++++++------- packages/backend/src/types.ts | 5 +- .../migration.sql | 14 ++ packages/db/prisma/schema.prisma | 14 ++ 4 files changed, 117 insertions(+), 52 deletions(-) create mode 100644 packages/db/prisma/migrations/20250124173816_relate_connection_and_repo/migration.sql diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index 5c24362d..f1530937 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -1,6 +1,6 @@ -import { Connection, ConnectionSyncStatus, PrismaClient } from "@sourcebot/db"; +import { Connection, ConnectionSyncStatus, PrismaClient, Prisma } from "@sourcebot/db"; import { Job, Queue, Worker } from 'bullmq'; -import { AppContext, Settings } from "./types.js"; +import { AppContext, Settings, WithRequired } from "./types.js"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { createLogger } from "./logger.js"; import os from 'os'; @@ -66,62 +66,96 @@ export class ConnectionManager implements IConnectionManager { // @note: We aren't actually doing anything with this atm. const abortController = new AbortController(); - switch (config.type) { - case 'github': { - const token = config.token ? getTokenFromConfig(config.token, this.context) : undefined; - const gitHubRepos = await getGitHubReposFromConfig(config, abortController.signal, this.context); - const hostUrl = config.url ?? 'https://github.com'; - const hostname = config.url ? new URL(config.url).hostname : 'github.com'; - - await Promise.all(gitHubRepos.map((repo) => { - const repoName = `${hostname}/${repo.full_name}`; - const cloneUrl = new URL(repo.clone_url!); - if (token) { - cloneUrl.username = token; + type RepoData = WithRequired; + const repoData: RepoData[] = await (async () => { + switch (config.type) { + case 'github': { + const token = config.token ? getTokenFromConfig(config.token, this.context) : undefined; + const gitHubRepos = await getGitHubReposFromConfig(config, abortController.signal, this.context); + const hostUrl = config.url ?? 'https://github.com'; + const hostname = config.url ? new URL(config.url).hostname : 'github.com'; + + return Promise.all(gitHubRepos.map((repo) => { + const repoName = `${hostname}/${repo.full_name}`; + const cloneUrl = new URL(repo.clone_url!); + if (token) { + cloneUrl.username = token; + } + + const record: RepoData = { + external_id: repo.id.toString(), + external_codeHostType: 'github', + external_codeHostUrl: hostUrl, + cloneUrl: cloneUrl.toString(), + name: repoName, + isFork: repo.fork, + isArchived: !!repo.archived, + org: { + connect: { + id: orgId, + }, + }, + connections: { + create: { + connectionId: job.data.connectionId, + } + }, + metadata: { + 'zoekt.web-url-type': 'github', + 'zoekt.web-url': repo.html_url, + 'zoekt.name': repoName, + 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), + 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), + 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), + 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), + 'zoekt.archived': marshalBool(repo.archived), + 'zoekt.fork': marshalBool(repo.fork), + 'zoekt.public': marshalBool(repo.private === false) + }, + }; + + return record; + })); + } + } + })(); + + // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, + // and then recreate them when we upsert the repos. For example, if a repo is no-longer + // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't + // appear in the repoData array above, and so the RepoToConnection record will be deleted. + // Repos that have no RepoToConnection records are considered orphaned and can be deleted. + await this.db.$transaction(async (tx) => { + await tx.connection.update({ + where: { + id: job.data.connectionId, + }, + data: { + repos: { + deleteMany: {} } + } + }); - const data = { - external_id: repo.id.toString(), - external_codeHostType: 'github', - external_codeHostUrl: hostUrl, - cloneUrl: cloneUrl.toString(), - name: repoName, - isFork: repo.fork, - isArchived: !!repo.archived, - orgId, - metadata: { - 'zoekt.web-url-type': 'github', - 'zoekt.web-url': repo.html_url, - 'zoekt.name': repoName, - 'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(), - 'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(), - 'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(), - 'zoekt.github-forks': (repo.forks_count ?? 0).toString(), - 'zoekt.archived': marshalBool(repo.archived), - 'zoekt.fork': marshalBool(repo.fork), - 'zoekt.public': marshalBool(repo.private === false) + await Promise.all(repoData.map((repo) => { + return tx.repo.upsert({ + where: { + external_id_external_codeHostUrl: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, }, - }; + }, + create: repo, + update: repo as Prisma.RepoUpdateInput, + }); + })); - return this.db.repo.upsert({ - where: { - external_id_external_codeHostUrl: { - external_id: repo.id.toString(), - external_codeHostUrl: hostUrl, - }, - }, - create: data, - update: data, - }) - })); - break; - } - } + }); } private async onSyncJobCompleted(job: Job) { - this.logger.info(`Config sync job ${job.id} completed`); + this.logger.info(`Connection sync job ${job.id} completed`); const { connectionId } = job.data; await this.db.connection.update({ @@ -136,7 +170,7 @@ export class ConnectionManager implements IConnectionManager { } private async onSyncJobFailed(job: Job | undefined, err: unknown) { - this.logger.info(`Config sync job failed with error: ${err}`); + this.logger.info(`Connection sync job failed with error: ${err}`); if (job) { const { connectionId } = job.data; await this.db.connection.update({ diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 9e714ccb..71d8fff8 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -87,4 +87,7 @@ export type Settings = { // @see : https://stackoverflow.com/a/61132308 export type DeepPartial = T extends object ? { [P in keyof T]?: DeepPartial; -} : T; \ No newline at end of file +} : T; + +// @see: https://stackoverflow.com/a/69328045 +export type WithRequired = T & { [P in K]-?: T[P] }; \ No newline at end of file diff --git a/packages/db/prisma/migrations/20250124173816_relate_connection_and_repo/migration.sql b/packages/db/prisma/migrations/20250124173816_relate_connection_and_repo/migration.sql new file mode 100644 index 00000000..b61d111f --- /dev/null +++ b/packages/db/prisma/migrations/20250124173816_relate_connection_and_repo/migration.sql @@ -0,0 +1,14 @@ +-- CreateTable +CREATE TABLE "RepoToConnection" ( + "addedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "connectionId" INTEGER NOT NULL, + "repoId" INTEGER NOT NULL, + + CONSTRAINT "RepoToConnection_pkey" PRIMARY KEY ("connectionId","repoId") +); + +-- AddForeignKey +ALTER TABLE "RepoToConnection" ADD CONSTRAINT "RepoToConnection_connectionId_fkey" FOREIGN KEY ("connectionId") REFERENCES "Connection"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "RepoToConnection" ADD CONSTRAINT "RepoToConnection_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index ab3cada5..4f071a1b 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -36,6 +36,7 @@ model Repo { isArchived Boolean metadata Json cloneUrl String + connections RepoToConnection[] repoIndexingStatus RepoIndexingStatus @default(NEW) @@ -58,6 +59,7 @@ model Connection { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt syncedAt DateTime? + repos RepoToConnection[] syncStatus ConnectionSyncStatus @default(SYNC_NEEDED) @@ -66,6 +68,18 @@ model Connection { orgId Int } +model RepoToConnection { + addedAt DateTime @default(now()) + + connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) + connectionId Int + + repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) + repoId Int + + @@id([connectionId, repoId]) +} + model Org { id Int @id @default(autoincrement()) name String From 80de9f7af45cdad2b3dc5320c83d12da53bda401 Mon Sep 17 00:00:00 2001 From: bkellam Date: Fri, 24 Jan 2025 11:52:22 -0800 Subject: [PATCH 09/21] nits --- packages/backend/src/connectionManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index f1530937..c4b25c72 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -13,7 +13,7 @@ interface IConnectionManager { dispose: () => void; } -const QUEUE_NAME = 'configSyncQueue'; +const QUEUE_NAME = 'connectionSyncQueue'; type JobPayload = { connectionId: number, @@ -75,7 +75,7 @@ export class ConnectionManager implements IConnectionManager { const hostUrl = config.url ?? 'https://github.com'; const hostname = config.url ? new URL(config.url).hostname : 'github.com'; - return Promise.all(gitHubRepos.map((repo) => { + return gitHubRepos.map((repo) => { const repoName = `${hostname}/${repo.full_name}`; const cloneUrl = new URL(repo.clone_url!); if (token) { @@ -115,7 +115,7 @@ export class ConnectionManager implements IConnectionManager { }; return record; - })); + }) } } })(); @@ -123,7 +123,7 @@ export class ConnectionManager implements IConnectionManager { // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, // and then recreate them when we upsert the repos. For example, if a repo is no-longer // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record will be deleted. + // appear in the repoData array above, and so the RepoToConnection record won't be re-created. // Repos that have no RepoToConnection records are considered orphaned and can be deleted. await this.db.$transaction(async (tx) => { await tx.connection.update({ From b452d6241d01e2f5a8d8389a38ea58a5a78bce9c Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 24 Jan 2025 14:07:25 -0800 Subject: [PATCH 10/21] create first version of crypto package --- packages/crypto/.gitignore | 1 + packages/crypto/package.json | 7 ++++++ packages/crypto/src/environment.ts | 17 +++++++++++++++ packages/crypto/src/utils.ts | 35 ++++++++++++++++++++++++++++++ packages/crypto/tsconfig.json | 27 +++++++++++++++++++++++ 5 files changed, 87 insertions(+) create mode 100644 packages/crypto/.gitignore create mode 100644 packages/crypto/package.json create mode 100644 packages/crypto/src/environment.ts create mode 100644 packages/crypto/src/utils.ts create mode 100644 packages/crypto/tsconfig.json diff --git a/packages/crypto/.gitignore b/packages/crypto/.gitignore new file mode 100644 index 00000000..3a8fe5ed --- /dev/null +++ b/packages/crypto/.gitignore @@ -0,0 +1 @@ +.env.local \ No newline at end of file diff --git a/packages/crypto/package.json b/packages/crypto/package.json new file mode 100644 index 00000000..82295ea2 --- /dev/null +++ b/packages/crypto/package.json @@ -0,0 +1,7 @@ +{ + "name": "crypto", + "version": "0.1.0", + "scripts": { + "build": "tsc" + } +} diff --git a/packages/crypto/src/environment.ts b/packages/crypto/src/environment.ts new file mode 100644 index 00000000..c1f72210 --- /dev/null +++ b/packages/crypto/src/environment.ts @@ -0,0 +1,17 @@ +import dotenv from 'dotenv'; + +export const getEnv = (env: string | undefined, defaultValue?: string, required?: boolean) => { + if (required && !env && !defaultValue) { + throw new Error(`Missing required environment variable`); + } + + return env ?? defaultValue; +} + +dotenv.config({ + path: './.env.local', + override: true +}); + +// @note: You can use https://generate-random.org/encryption-key-generator to create a new 32 byte key +export const SOURCEBOT_ENCRYPTION_KEY = getEnv(process.env.SOURCEBOT_ENCRYPTION_KEY, undefined, true)!; \ No newline at end of file diff --git a/packages/crypto/src/utils.ts b/packages/crypto/src/utils.ts new file mode 100644 index 00000000..fc5626b8 --- /dev/null +++ b/packages/crypto/src/utils.ts @@ -0,0 +1,35 @@ +import crypto from 'crypto'; +import { SOURCEBOT_ENCRYPTION_KEY } from './environment'; + +const algorithm = 'aes-256-cbc'; +const ivLength = 16; // 16 bytes for CBC + +const generateIV = (): Buffer => { + return crypto.randomBytes(ivLength); +}; + +export function encrypt(text: string): { iv: string; encryptedData: string } { + const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'hex'); + + const iv = generateIV(); + const cipher = crypto.createCipheriv(algorithm, encryptionKey, iv); + + let encrypted = cipher.update(text, 'utf8', 'hex'); + encrypted += cipher.final('hex'); + + return { iv: iv.toString('hex'), encryptedData: encrypted }; +} + +export function decrypt(iv: string, encryptedText: string): string { + const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'hex'); + + const ivBuffer = Buffer.from(iv, 'hex'); + const encryptedBuffer = Buffer.from(encryptedText, 'hex'); + + const decipher = crypto.createDecipheriv(algorithm, encryptionKey, ivBuffer); + + let decrypted = decipher.update(encryptedBuffer, undefined, 'utf8'); + decrypted += decipher.final('utf8'); + + return decrypted; +} diff --git a/packages/crypto/tsconfig.json b/packages/crypto/tsconfig.json new file mode 100644 index 00000000..c38ce16c --- /dev/null +++ b/packages/crypto/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "outDir": "dist", + "incremental": true, + "declaration": true, + "emitDecoratorMetadata": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "module": "Node16", + "moduleResolution": "Node16", + "target": "ES2022", + "noEmitOnError": false, + "noImplicitAny": true, + "noUnusedLocals": false, + "pretty": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "lib": ["ES2023"], + "strict": true, + "sourceMap": true, + "inlineSources": true + }, + "include": ["src/utils.ts", "src/environment.ts"], + "exclude": ["node_modules"] +} \ No newline at end of file From 28ae6feabd4820b8ce8a3c2fb291a8675d5b8593 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 24 Jan 2025 15:03:22 -0800 Subject: [PATCH 11/21] add crypto package as deps to others --- packages/crypto/package.json | 3 +- packages/crypto/src/{utils.ts => index.ts} | 4 +- packages/crypto/tsconfig.json | 48 ++++++++++------------ 3 files changed, 26 insertions(+), 29 deletions(-) rename packages/crypto/src/{utils.ts => index.ts} (98%) diff --git a/packages/crypto/package.json b/packages/crypto/package.json index 82295ea2..a9070bc8 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -1,5 +1,6 @@ { - "name": "crypto", + "name": "@sourcebot/crypto", + "main": "dist/index.js", "version": "0.1.0", "scripts": { "build": "tsc" diff --git a/packages/crypto/src/utils.ts b/packages/crypto/src/index.ts similarity index 98% rename from packages/crypto/src/utils.ts rename to packages/crypto/src/index.ts index fc5626b8..fc63f764 100644 --- a/packages/crypto/src/utils.ts +++ b/packages/crypto/src/index.ts @@ -9,7 +9,7 @@ const generateIV = (): Buffer => { }; export function encrypt(text: string): { iv: string; encryptedData: string } { - const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'hex'); + const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'ascii'); const iv = generateIV(); const cipher = crypto.createCipheriv(algorithm, encryptionKey, iv); @@ -21,7 +21,7 @@ export function encrypt(text: string): { iv: string; encryptedData: string } { } export function decrypt(iv: string, encryptedText: string): string { - const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'hex'); + const encryptionKey = Buffer.from(SOURCEBOT_ENCRYPTION_KEY, 'ascii'); const ivBuffer = Buffer.from(iv, 'hex'); const encryptedBuffer = Buffer.from(encryptedText, 'hex'); diff --git a/packages/crypto/tsconfig.json b/packages/crypto/tsconfig.json index c38ce16c..39b3533d 100644 --- a/packages/crypto/tsconfig.json +++ b/packages/crypto/tsconfig.json @@ -1,27 +1,23 @@ { - "compilerOptions": { - "outDir": "dist", - "incremental": true, - "declaration": true, - "emitDecoratorMetadata": true, - "esModuleInterop": true, - "experimentalDecorators": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "module": "Node16", - "moduleResolution": "Node16", - "target": "ES2022", - "noEmitOnError": false, - "noImplicitAny": true, - "noUnusedLocals": false, - "pretty": true, - "resolveJsonModule": true, - "skipLibCheck": true, - "lib": ["ES2023"], - "strict": true, - "sourceMap": true, - "inlineSources": true - }, - "include": ["src/utils.ts", "src/environment.ts"], - "exclude": ["node_modules"] -} \ No newline at end of file + "compilerOptions": { + "target": "ES6", + "module": "CommonJS", + "lib": ["ES6"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "moduleResolution": "node", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "isolatedModules": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] + } + \ No newline at end of file From e1d0a87bb2ec648d104f243c5b14d1799ff3bc33 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 24 Jan 2025 15:03:50 -0800 Subject: [PATCH 12/21] forgot to add package changes --- packages/backend/package.json | 1 + packages/web/package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/backend/package.json b/packages/backend/package.json index 3bc4cae6..02a6f772 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -31,6 +31,7 @@ "lowdb": "^7.0.1", "micromatch": "^4.0.8", "posthog-node": "^4.2.1", + "@sourcebot/crypto": "^0.1.0", "@sourcebot/db": "^0.1.0", "@sourcebot/schemas": "^0.1.0", "simple-git": "^3.27.0", diff --git a/packages/web/package.json b/packages/web/package.json index e35ae88e..d269aa94 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -58,6 +58,7 @@ "@replit/codemirror-lang-svelte": "^6.0.0", "@replit/codemirror-vim": "^6.2.1", "@shopify/lang-jsonc": "^1.0.0", + "@sourcebot/crypto": "^0.1.0", "@sourcebot/schemas": "^0.1.0", "@sourcebot/db": "^0.1.0", "@ssddanbrown/codemirror-lang-twig": "^1.0.0", From 741cc45c1f839463a7d15178f3891983f276ecf2 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 24 Jan 2025 17:03:47 -0800 Subject: [PATCH 13/21] add server action for adding and listing secrets, create test page for it --- packages/crypto/package.json | 3 +- packages/web/src/actions.ts | 93 ++++++++++++++++ packages/web/src/app/secrets/columns.tsx | 40 +++++++ packages/web/src/app/secrets/page.tsx | 13 +++ packages/web/src/app/secrets/secretsTable.tsx | 103 ++++++++++++++++++ 5 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 packages/web/src/app/secrets/columns.tsx create mode 100644 packages/web/src/app/secrets/page.tsx create mode 100644 packages/web/src/app/secrets/secretsTable.tsx diff --git a/packages/crypto/package.json b/packages/crypto/package.json index a9070bc8..3339a5df 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -3,6 +3,7 @@ "main": "dist/index.js", "version": "0.1.0", "scripts": { - "build": "tsc" + "build": "tsc", + "postinstall": "yarn build" } } diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 717a74fe..bc47904b 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -8,11 +8,104 @@ import { prisma } from "@/prisma"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "./lib/errorCodes"; import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; +import { encrypt } from "@sourcebot/crypto" const ajv = new Ajv({ validateFormats: false, }); +export const createSecret = async (key: string, value: string): Promise<{ success: boolean } | ServiceError> => { + const session = await auth(); + if (!session) { + return notAuthenticated(); + } + + const user = await getUser(session.user.id); + if (!user) { + return unexpectedError("User not found"); + } + const orgId = user.activeOrgId; + if (!orgId) { + return unexpectedError("User has no active org"); + } + + // @todo: refactor this into a shared function + const membership = await prisma.userToOrg.findUnique({ + where: { + orgId_userId: { + userId: session.user.id, + orgId, + } + }, + }); + if (!membership) { + return notFound(); + } + + try { + + const encrypted = encrypt(value); + await prisma.secret.create({ + data: { + orgId, + key, + encryptedValue: encrypted.encryptedData, + iv: encrypted.iv, + } + }); + } catch (e) { + console.error(e); + return { + success: false, + } + } + + return { + success: true, + } +} + +export const getSecrets = async (): Promise<{ createdAt: Date; key: string; }[] | ServiceError> => { + const session = await auth(); + if (!session) { + return notAuthenticated(); + } + + const user = await getUser(session.user.id); + if (!user) { + return unexpectedError("User not found"); + } + const orgId = user.activeOrgId; + if (!orgId) { + return unexpectedError("User has no active org"); + } + + const membership = await prisma.userToOrg.findUnique({ + where: { + orgId_userId: { + userId: session.user.id, + orgId, + } + }, + }); + if (!membership) { + return notFound(); + } + + const secrets = await prisma.secret.findMany({ + where: { + orgId, + }, + select: { + key: true, + createdAt: true + } + }); + + return secrets; +} + + export const createOrg = async (name: string): Promise<{ id: number } | ServiceError> => { const session = await auth(); if (!session) { diff --git a/packages/web/src/app/secrets/columns.tsx b/packages/web/src/app/secrets/columns.tsx new file mode 100644 index 00000000..1de99327 --- /dev/null +++ b/packages/web/src/app/secrets/columns.tsx @@ -0,0 +1,40 @@ +'use client'; + +import { Button } from "@/components/ui/button"; +import { Column, ColumnDef } from "@tanstack/react-table" +import { ArrowUpDown } from "lucide-react" + +export type SecretColumnInfo = { + key: string; + createdAt: string; +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "key", + cell: ({ row }) => { + const secret = row.original; + return
{secret.key}
; + } + }, + { + accessorKey: "createdAt", + header: ({ column }) => createSortHeader("Created At", column), + cell: ({ row }) => { + const secret = row.original; + return
{secret.createdAt}
; + } + } +]; + +const createSortHeader = (name: string, column: Column) => { + return ( + + ) +} \ No newline at end of file diff --git a/packages/web/src/app/secrets/page.tsx b/packages/web/src/app/secrets/page.tsx new file mode 100644 index 00000000..ccb04ab1 --- /dev/null +++ b/packages/web/src/app/secrets/page.tsx @@ -0,0 +1,13 @@ +import { NavigationMenu } from "../components/navigationMenu"; +import { SecretsTable } from "./secretsTable"; + +export default function SecretsPage() { + return ( +
+ +
+ +
+
+ ) +} \ No newline at end of file diff --git a/packages/web/src/app/secrets/secretsTable.tsx b/packages/web/src/app/secrets/secretsTable.tsx new file mode 100644 index 00000000..1a771f3d --- /dev/null +++ b/packages/web/src/app/secrets/secretsTable.tsx @@ -0,0 +1,103 @@ +'use client'; +import { useEffect, useState } from "react"; +import { getSecrets, createSecret } from "../../actions" +import { Button } from "@/components/ui/button"; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; +import { columns, SecretColumnInfo } from "./columns"; +import { DataTable } from "@/components/ui/data-table"; + +const formSchema = z.object({ + key: z.string().min(2).max(40), + value: z.string().min(2).max(40), +}); + +export const SecretsTable = () => { + const [secrets, setSecrets] = useState<{ createdAt: Date; key: string; }[]>([]); + + useEffect(() => { + const fetchSecretKeys = async () => { + const keys = await getSecrets(); + if ('keys' in keys) { + setSecrets(keys); + } else { + console.error(keys); + } + }; + + fetchSecretKeys(); + }, []); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + key: "", + value: "", + }, + }); + + const handleCreateSecret = async (values: { key: string, value: string }) => { + await createSecret(values.key, values.value); + const keys = await getSecrets(); + if ('keys' in keys) { + setSecrets(keys); + } else { + console.error(keys); + } + }; + + + const keys = secrets.map((secret): SecretColumnInfo => { + return { + key: secret.key, + createdAt: secret.createdAt.toISOString(), + } + }).sort((a, b) => { + return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(); + }); + + return ( +
+
+ + ( + + Key + + + + + + )} + /> + ( + + Value + + + + + + )} + /> + + + + +
+ ); +}; \ No newline at end of file From b3fb7693d68b2edcc7584028008531b14ab5441b Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 11:06:50 -0800 Subject: [PATCH 14/21] add secrets page to nav menu --- packages/web/src/app/components/navigationMenu.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/web/src/app/components/navigationMenu.tsx b/packages/web/src/app/components/navigationMenu.tsx index 871259ab..aeefd81c 100644 --- a/packages/web/src/app/components/navigationMenu.tsx +++ b/packages/web/src/app/components/navigationMenu.tsx @@ -56,6 +56,13 @@ export const NavigationMenu = async () => { + + + + Secrets + + + From f62a6cf1e1fc46e570f10d0d067951b75d7d44a5 Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 12:51:37 -0800 Subject: [PATCH 15/21] add secret to config and support fetching it in backend --- packages/backend/src/connectionManager.ts | 10 +--- packages/backend/src/gitea.ts | 5 +- packages/backend/src/github.ts | 5 +- packages/backend/src/gitlab.ts | 5 +- packages/backend/src/main.ts | 60 +++++++++++++++++--- packages/backend/src/utils.ts | 37 ++++++++++-- packages/schemas/src/v3/connection.schema.ts | 13 +++++ packages/schemas/src/v3/connection.type.ts | 6 ++ packages/schemas/src/v3/github.schema.ts | 13 +++++ packages/schemas/src/v3/github.type.ts | 6 ++ packages/schemas/src/v3/shared.schema.ts | 13 +++++ packages/schemas/src/v3/shared.type.ts | 6 ++ schemas/v3/shared.json | 13 +++++ 13 files changed, 164 insertions(+), 28 deletions(-) diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index c4b25c72..456c5d15 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -5,7 +5,7 @@ import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { createLogger } from "./logger.js"; import os from 'os'; import { Redis } from 'ioredis'; -import { getTokenFromConfig, marshalBool } from "./utils.js"; +import { marshalBool } from "./utils.js"; import { getGitHubReposFromConfig } from "./github.js"; interface IConnectionManager { @@ -70,17 +70,13 @@ export class ConnectionManager implements IConnectionManager { const repoData: RepoData[] = await (async () => { switch (config.type) { case 'github': { - const token = config.token ? getTokenFromConfig(config.token, this.context) : undefined; - const gitHubRepos = await getGitHubReposFromConfig(config, abortController.signal, this.context); + const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal); const hostUrl = config.url ?? 'https://github.com'; const hostname = config.url ? new URL(config.url).hostname : 'github.com'; - + return gitHubRepos.map((repo) => { const repoName = `${hostname}/${repo.full_name}`; const cloneUrl = new URL(repo.clone_url!); - if (token) { - cloneUrl.username = token; - } const record: RepoData = { external_id: repo.id.toString(), diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index f3a6b514..8701b523 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -9,8 +9,9 @@ import micromatch from 'micromatch'; const logger = createLogger('Gitea'); -export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppContext) => { - const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined; +export const getGiteaReposFromConfig = async (config: GiteaConfig, orgId: number, ctx: AppContext) => { + // TODO: pass in DB here to fetch secret properly + const token = config.token ? await getTokenFromConfig(config.token, orgId) : undefined; const api = giteaApi(config.url ?? 'https://gitea.com', { token, diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 6680ce15..bef5e6bc 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -4,6 +4,7 @@ import { createLogger } from "./logger.js"; import { AppContext } from "./types.js"; import { getTokenFromConfig, measure } from "./utils.js"; import micromatch from "micromatch"; +import { PrismaClient } from "@sourcebot/db"; const logger = createLogger("GitHub"); @@ -25,8 +26,8 @@ export type OctokitRepository = { size?: number, } -export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, signal: AbortSignal, ctx: AppContext) => { - const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined; +export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => { + const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined; const octokit = new Octokit({ auth: token, diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 2501fd05..73ed5e92 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -8,8 +8,9 @@ import { getTokenFromConfig, measure } from "./utils.js"; const logger = createLogger("GitLab"); export const GITLAB_CLOUD_HOSTNAME = "gitlab.com"; -export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppContext) => { - const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined; +export const getGitLabReposFromConfig = async (config: GitLabConfig, orgId: number, ctx: AppContext) => { + // TODO: pass in DB here to fetch secret properly + const token = config.token ? await getTokenFromConfig(config.token, orgId) : undefined; const api = new Gitlab({ ...(config.token ? { token, diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 7e9d2f80..9b89506e 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -1,20 +1,47 @@ -import { ConnectionSyncStatus, PrismaClient, Repo, RepoIndexingStatus } from '@sourcebot/db'; +import { ConnectionSyncStatus, PrismaClient, Repo, RepoIndexingStatus, RepoToConnection, Connection } from '@sourcebot/db'; import { existsSync } from 'fs'; import { cloneRepository, fetchRepository } from "./git.js"; import { createLogger } from "./logger.js"; import { captureEvent } from "./posthog.js"; import { AppContext } from "./types.js"; -import { getRepoPath, measure } from "./utils.js"; +import { getRepoPath, getTokenFromConfig, measure } from "./utils.js"; import { indexGitRepository } from "./zoekt.js"; import { DEFAULT_SETTINGS } from './constants.js'; import { Queue, Worker, Job } from 'bullmq'; import { Redis } from 'ioredis'; import * as os from 'os'; import { ConnectionManager } from './connectionManager.js'; +import { ConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; const logger = createLogger('main'); -const syncGitRepository = async (repo: Repo, ctx: AppContext) => { +type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection})[] }; + +// TODO: do this better? ex: try using the tokens from all the connections +// We can no longer use repo.cloneUrl directly since it doesn't contain the token for security reasons. As a result, we need to +// fetch the token here using the connections from the repo. Multiple connections could be referencing this repo, and each +// may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This +// may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referrencing. +const getTokenForRepo = async (repo: RepoWithConnections, db: PrismaClient) => { + const repoConnections = repo.connections; + if (repoConnections.length === 0) { + logger.error(`Repo ${repo.id} has no connections`); + return; + } + + let token: string | undefined; + for (const repoConnection of repoConnections) { + const connection = repoConnection.connection; + const config = connection.config as unknown as ConnectionConfig; + if (config.token) { + token = await getTokenFromConfig(config.token, connection.orgId, db); + } + } + + return token; +} + +const syncGitRepository = async (repo: RepoWithConnections, ctx: AppContext, db: PrismaClient) => { let fetchDuration_s: number | undefined = undefined; let cloneDuration_s: number | undefined = undefined; @@ -35,7 +62,15 @@ const syncGitRepository = async (repo: Repo, ctx: AppContext) => { } else { logger.info(`Cloning ${repo.id}...`); - const { durationMs } = await measure(() => cloneRepository(repo.cloneUrl, repoPath, metadata, ({ method, stage, progress }) => { + const token = await getTokenForRepo(repo, db); + let cloneUrl = repo.cloneUrl; + if (token) { + const url = new URL(cloneUrl); + url.username = token; + cloneUrl = url.toString(); + } + + const { durationMs } = await measure(() => cloneRepository(cloneUrl, repoPath, metadata, ({ method, stage, progress }) => { logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`) })); cloneDuration_s = durationMs / 1000; @@ -92,13 +127,13 @@ export const main = async (db: PrismaClient, context: AppContext) => { const connectionManager = new ConnectionManager(db, DEFAULT_SETTINGS, redis, context); setInterval(async () => { - const configs = await db.connection.findMany({ + const connections = await db.connection.findMany({ where: { syncStatus: ConnectionSyncStatus.SYNC_NEEDED, } }); - for (const config of configs) { - await connectionManager.scheduleConnectionSync(config); + for (const connection of connections) { + await connectionManager.scheduleConnectionSync(connection); } }, DEFAULT_SETTINGS.resyncConnectionPollingIntervalMs); @@ -111,13 +146,13 @@ export const main = async (db: PrismaClient, context: AppContext) => { const numWorkers = numCores * DEFAULT_SETTINGS.indexConcurrencyMultiple; logger.info(`Detected ${numCores} cores. Setting repo index max concurrency to ${numWorkers}`); const worker = new Worker('indexQueue', async (job: Job) => { - const repo = job.data as Repo; + const repo = job.data as RepoWithConnections; let indexDuration_s: number | undefined; let fetchDuration_s: number | undefined; let cloneDuration_s: number | undefined; - const stats = await syncGitRepository(repo, context); + const stats = await syncGitRepository(repo, context, db); indexDuration_s = stats.indexDuration_s; fetchDuration_s = stats.fetchDuration_s; cloneDuration_s = stats.cloneDuration_s; @@ -171,6 +206,13 @@ export const main = async (db: PrismaClient, context: AppContext) => { { indexedAt: { lt: thresholdDate } }, { repoIndexingStatus: RepoIndexingStatus.NEW } ] + }, + include: { + connections: { + include: { + connection: true + } + } } }); addReposToQueue(db, indexQueue, repos); diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 1b2d365a..7cfdee8d 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -2,7 +2,8 @@ import { Logger } from "winston"; import { AppContext, Repository } from "./types.js"; import path from 'path'; import micromatch from "micromatch"; -import { Repo } from "@sourcebot/db"; +import { PrismaClient, Repo } from "@sourcebot/db"; +import { decrypt } from "@sourcebot/crypto"; export const measure = async (cb : () => Promise) => { const start = Date.now(); @@ -86,15 +87,39 @@ export const excludeReposByTopic = (repos: T[], excludedRe }); } -export const getTokenFromConfig = (token: string | { env: string }, ctx: AppContext) => { +export const getTokenFromConfig = async (token: string | { env: string } | { secret: string }, orgId: number, db?: PrismaClient) => { if (typeof token === 'string') { return token; } - const tokenValue = process.env[token.env]; - if (!tokenValue) { - throw new Error(`The environment variable '${token.env}' was referenced in ${ctx.configPath}, but was not set.`); + if ('env' in token) { + const tokenValue = process.env[token.env]; + if (!tokenValue) { + throw new Error(`The environment variable '${token.env}' was referenced in the config but was not set.`); + } + return tokenValue; + } else if ('secret' in token) { + if (!db) { + throw new Error(`Database connection required to retrieve secret`); + } + + const secretKey = token.secret; + const secret = await db.secret.findUnique({ + where: { + orgId_key: { + key: secretKey, + orgId + } + } + }); + + if (!secret) { + throw new Error(`Secret with key ${secretKey} not found for org ${orgId}`); + } + + const decryptedSecret = decrypt(secret.iv, secret.encryptedValue); + return decryptedSecret; } - return tokenValue; + throw new Error(`Invalid token configuration in config`); } export const isRemotePath = (path: string) => { diff --git a/packages/schemas/src/v3/connection.schema.ts b/packages/schemas/src/v3/connection.schema.ts index 0a86ecf2..77a85423 100644 --- a/packages/schemas/src/v3/connection.schema.ts +++ b/packages/schemas/src/v3/connection.schema.ts @@ -36,6 +36,19 @@ const schema = { "env" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false } ] }, diff --git a/packages/schemas/src/v3/connection.type.ts b/packages/schemas/src/v3/connection.type.ts index 30c0ff27..3d07a7c2 100644 --- a/packages/schemas/src/v3/connection.type.ts +++ b/packages/schemas/src/v3/connection.type.ts @@ -17,6 +17,12 @@ export interface GithubConnectionConfig { * The name of the environment variable that contains the token. */ env: string; + } + | { + /** + * The name of the secret that contains the token. + */ + secret: string; }; /** * The URL of the GitHub host. Defaults to https://github.com diff --git a/packages/schemas/src/v3/github.schema.ts b/packages/schemas/src/v3/github.schema.ts index 546e10a8..9d53a13a 100644 --- a/packages/schemas/src/v3/github.schema.ts +++ b/packages/schemas/src/v3/github.schema.ts @@ -32,6 +32,19 @@ const schema = { "env" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false } ] }, diff --git a/packages/schemas/src/v3/github.type.ts b/packages/schemas/src/v3/github.type.ts index d7355c5b..d9d75f4c 100644 --- a/packages/schemas/src/v3/github.type.ts +++ b/packages/schemas/src/v3/github.type.ts @@ -15,6 +15,12 @@ export interface GithubConnectionConfig { * The name of the environment variable that contains the token. */ env: string; + } + | { + /** + * The name of the secret that contains the token. + */ + secret: string; }; /** * The URL of the GitHub host. Defaults to https://github.com diff --git a/packages/schemas/src/v3/shared.schema.ts b/packages/schemas/src/v3/shared.schema.ts index ccb21856..ff9dbab1 100644 --- a/packages/schemas/src/v3/shared.schema.ts +++ b/packages/schemas/src/v3/shared.schema.ts @@ -20,6 +20,19 @@ const schema = { "env" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false } ] }, diff --git a/packages/schemas/src/v3/shared.type.ts b/packages/schemas/src/v3/shared.type.ts index 6dcd54e7..3347d6a6 100644 --- a/packages/schemas/src/v3/shared.type.ts +++ b/packages/schemas/src/v3/shared.type.ts @@ -11,6 +11,12 @@ export type Token = * The name of the environment variable that contains the token. */ env: string; + } + | { + /** + * The name of the secret that contains the token. + */ + secret: string; }; export interface Shared { diff --git a/schemas/v3/shared.json b/schemas/v3/shared.json index fbb19e72..fcb1db75 100644 --- a/schemas/v3/shared.json +++ b/schemas/v3/shared.json @@ -19,6 +19,19 @@ "env" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "The name of the secret that contains the token." + } + }, + "required": [ + "secret" + ], + "additionalProperties": false } ] }, From 8de69b2107fc768155aa32c52ade187d726fe0f4 Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 13:01:57 -0800 Subject: [PATCH 16/21] reset secret form on successful submission --- packages/web/src/app/secrets/secretsTable.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/web/src/app/secrets/secretsTable.tsx b/packages/web/src/app/secrets/secretsTable.tsx index 1a771f3d..7ec73f28 100644 --- a/packages/web/src/app/secrets/secretsTable.tsx +++ b/packages/web/src/app/secrets/secretsTable.tsx @@ -44,6 +44,10 @@ export const SecretsTable = () => { const keys = await getSecrets(); if ('keys' in keys) { setSecrets(keys); + + form.reset(); + form.resetField("key"); + form.resetField("value"); } else { console.error(keys); } From 9adaeb5e78fbe44f0245a5322308847c80fae7e3 Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 13:07:34 -0800 Subject: [PATCH 17/21] add toast feedback for secrets form --- packages/web/src/actions.ts | 5 +---- packages/web/src/app/secrets/secretsTable.tsx | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index bc47904b..481094fe 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -43,7 +43,6 @@ export const createSecret = async (key: string, value: string): Promise<{ succes } try { - const encrypted = encrypt(value); await prisma.secret.create({ data: { @@ -55,9 +54,7 @@ export const createSecret = async (key: string, value: string): Promise<{ succes }); } catch (e) { console.error(e); - return { - success: false, - } + return unexpectedError(`Failed to create secret: ${e}`); } return { diff --git a/packages/web/src/app/secrets/secretsTable.tsx b/packages/web/src/app/secrets/secretsTable.tsx index 7ec73f28..18b80de5 100644 --- a/packages/web/src/app/secrets/secretsTable.tsx +++ b/packages/web/src/app/secrets/secretsTable.tsx @@ -9,6 +9,8 @@ import { useForm } from "react-hook-form"; import { z } from "zod"; import { columns, SecretColumnInfo } from "./columns"; import { DataTable } from "@/components/ui/data-table"; +import { isServiceError } from "@/lib/utils"; +import { useToast } from "@/components/hooks/use-toast"; const formSchema = z.object({ key: z.string().min(2).max(40), @@ -17,6 +19,7 @@ const formSchema = z.object({ export const SecretsTable = () => { const [secrets, setSecrets] = useState<{ createdAt: Date; key: string; }[]>([]); + const { toast } = useToast(); useEffect(() => { const fetchSecretKeys = async () => { @@ -40,7 +43,18 @@ export const SecretsTable = () => { }); const handleCreateSecret = async (values: { key: string, value: string }) => { - await createSecret(values.key, values.value); + const res = await createSecret(values.key, values.value); + if (isServiceError(res)) { + toast({ + description: `❌ Failed to create secret` + }); + return; + } else { + toast({ + description: `✅ Secret created successfully!` + }); + } + const keys = await getSecrets(); if ('keys' in keys) { setSecrets(keys); From 4d0737b0bd076e3f33e77c98871bbc3f2eaf1851 Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 13:34:28 -0800 Subject: [PATCH 18/21] add instructions for adding encryption key to dev instructions --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5084377b..c59329fb 100644 --- a/README.md +++ b/README.md @@ -374,14 +374,20 @@ docker run -v /path/to/my-repo:/repos/my-repo /* additional args */ ghcr. 5. Create a `config.json` file at the repository root. See [Configuring Sourcebot](#configuring-sourcebot) for more information. -6. Start Sourcebot with the command: +6. Create `.env.local` files in the `packages/backend` and `packages/web` directories with the following contents: + ```sh + # You can use https://acte.ltd/utils/randomkeygen to generate a key ("Encryption key 256") + SOURCEBOT_ENCRYPTION_KEY="32-byte-secret-key" + ``` + +7. Start Sourcebot with the command: ```sh yarn dev ``` A `.sourcebot` directory will be created and zoekt will begin to index the repositories found given `config.json`. -7. Start searching at `http://localhost:3000`. +8. Start searching at `http://localhost:3000`. ## Telemetry From c9e10a54f2665640bfa2bbfe7244c521e1b25592 Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 14:28:48 -0800 Subject: [PATCH 19/21] add encryption key support in docker file --- Dockerfile | 9 +++++- entrypoint.sh | 16 ++++++++++ packages/crypto/package.json | 7 +++++ .../web/src/app/api/(server)/secret/route.ts | 29 ------------------- 4 files changed, 31 insertions(+), 30 deletions(-) delete mode 100644 packages/web/src/app/api/(server)/secret/route.ts diff --git a/Dockerfile b/Dockerfile index 3720da60..4c9ff7a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,10 @@ WORKDIR /app COPY package.json yarn.lock* ./ COPY ./packages/db ./packages/db COPY ./packages/schemas ./packages/schemas +COPY ./packages/crypto ./packages/crypto RUN yarn workspace @sourcebot/db install --frozen-lockfile RUN yarn workspace @sourcebot/schemas install --frozen-lockfile +RUN yarn workspace @sourcebot/crypto install --frozen-lockfile # ------ Build Web ------ FROM node-alpine AS web-builder @@ -30,6 +32,7 @@ COPY ./packages/web ./packages/web COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas +COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto # Fixes arm64 timeouts RUN yarn config set registry https://registry.npmjs.org/ @@ -60,6 +63,7 @@ COPY ./packages/backend ./packages/backend COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas +COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto RUN yarn workspace @sourcebot/backend install --frozen-lockfile RUN yarn workspace @sourcebot/backend build @@ -100,7 +104,7 @@ ENV POSTHOG_PAPIK=$POSTHOG_PAPIK # ENV SOURCEBOT_TELEMETRY_DISABLED=1 # Configure dependencies -RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib +RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib openssl # Configure zoekt COPY vendor/zoekt/install-ctags-alpine.sh . @@ -129,6 +133,7 @@ COPY --from=backend-builder /app/packages/backend ./packages/backend COPY --from=shared-libs-builder /app/node_modules ./node_modules COPY --from=shared-libs-builder /app/packages/db ./packages/db COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas +COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto # Configure the database RUN mkdir -p /run/postgresql && \ @@ -143,6 +148,8 @@ RUN chmod +x ./entrypoint.sh COPY default-config.json . +ENV SOURCEBOT_ENCRYPTION_KEY="" + EXPOSE 3000 ENV PORT=3000 ENV HOSTNAME="0.0.0.0" diff --git a/entrypoint.sh b/entrypoint.sh index 7d497055..b0883428 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -26,6 +26,22 @@ if [ ! -d "$DB_DATA_DIR" ]; then su postgres -c "initdb -D $DB_DATA_DIR" fi +if [ -z "$SOURCEBOT_ENCRYPTION_KEY" ]; then + echo -e "\e[31m[Error] SOURCEBOT_ENCRYPTION_KEY is not set.\e[0m" + + if [ -f "$DATA_CACHE_DIR/.secret" ]; then + echo -e "\e[34m[Info] Loading environment variables from $DATA_CACHE_DIR/.secret\e[0m" + else + echo -e "\e[34m[Info] Generating a new encryption key...\e[0m" + SOURCEBOT_ENCRYPTION_KEY=$(openssl rand -base64 24) + echo "SOURCEBOT_ENCRYPTION_KEY=\"$SOURCEBOT_ENCRYPTION_KEY\"" >> "$DATA_CACHE_DIR/.secret" + fi + + set -a + . "$DATA_CACHE_DIR/.secret" + set +a +fi + # In order to detect if this is the first run, we create a `.installed` file in # the cache directory. FIRST_RUN_FILE="$DATA_CACHE_DIR/.installedv2" diff --git a/packages/crypto/package.json b/packages/crypto/package.json index 3339a5df..cc32c0b4 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -5,5 +5,12 @@ "scripts": { "build": "tsc", "postinstall": "yarn build" + }, + "dependencies": { + "dotenv": "^16.4.5" + }, + "devDependencies": { + "@types/node": "^22.7.5", + "typescript": "^5.7.3" } } diff --git a/packages/web/src/app/api/(server)/secret/route.ts b/packages/web/src/app/api/(server)/secret/route.ts deleted file mode 100644 index 72ea388d..00000000 --- a/packages/web/src/app/api/(server)/secret/route.ts +++ /dev/null @@ -1,29 +0,0 @@ -'user server'; - -import { secretCreateRequestSchema, secreteDeleteRequestSchema } from "@/lib/schemas"; -import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; -import { NextRequest } from "next/server"; - -export const POST = async (request: NextRequest) => { - const body = await request.json(); - const parsed = await secretCreateRequestSchema.safeParseAsync(body); - if (!parsed.success) { - return serviceErrorResponse( - schemaValidationError(parsed.error) - ) - } - - return Response.json({ success: true }); -} - -export const DELETE = async (request: NextRequest) => { - const body = await request.json(); - const parsed = await secreteDeleteRequestSchema.safeParseAsync(body); - if (!parsed.success) { - return serviceErrorResponse( - schemaValidationError(parsed.error) - ) - } - - return Response.json({ success: true }); -} \ No newline at end of file From ef948e5a562ef03ff2426bd32358c259aa678413 Mon Sep 17 00:00:00 2001 From: msukkari Date: Sat, 25 Jan 2025 14:50:41 -0800 Subject: [PATCH 20/21] add delete secret button --- packages/web/src/actions.ts | 41 +++++++++++++++ packages/web/src/app/secrets/columns.tsx | 51 +++++++++++++------ packages/web/src/app/secrets/secretsTable.tsx | 42 +++++++++++---- 3 files changed, 108 insertions(+), 26 deletions(-) diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 481094fe..6ba5b365 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -102,6 +102,47 @@ export const getSecrets = async (): Promise<{ createdAt: Date; key: string; }[] return secrets; } +export const deleteSecret = async (key: string): Promise<{ success: boolean } | ServiceError> => { + const session = await auth(); + if (!session) { + return notAuthenticated(); + } + + const user = await getUser(session.user.id); + if (!user) { + return unexpectedError("User not found"); + } + const orgId = user.activeOrgId; + if (!orgId) { + return unexpectedError("User has no active org"); + } + + const membership = await prisma.userToOrg.findUnique({ + where: { + orgId_userId: { + userId: session.user.id, + orgId, + } + }, + }); + if (!membership) { + return notFound(); + } + + await prisma.secret.delete({ + where: { + orgId_key: { + orgId, + key, + } + } + }); + + return { + success: true, + } +} + export const createOrg = async (name: string): Promise<{ id: number } | ServiceError> => { const session = await auth(); diff --git a/packages/web/src/app/secrets/columns.tsx b/packages/web/src/app/secrets/columns.tsx index 1de99327..53e23e31 100644 --- a/packages/web/src/app/secrets/columns.tsx +++ b/packages/web/src/app/secrets/columns.tsx @@ -9,23 +9,42 @@ export type SecretColumnInfo = { createdAt: string; } -export const columns: ColumnDef[] = [ - { - accessorKey: "key", - cell: ({ row }) => { - const secret = row.original; - return
{secret.key}
; +export const columns = (handleDelete: (key: string) => void): ColumnDef[] => { + return [ + { + accessorKey: "key", + cell: ({ row }) => { + const secret = row.original; + return
{secret.key}
; + } + }, + { + accessorKey: "createdAt", + header: ({ column }) => createSortHeader("Created At", column), + cell: ({ row }) => { + const secret = row.original; + return
{secret.createdAt}
; + } + }, + { + accessorKey: "delete", + cell: ({ row }) => { + const secret = row.original; + return ( + + ) + } } - }, - { - accessorKey: "createdAt", - header: ({ column }) => createSortHeader("Created At", column), - cell: ({ row }) => { - const secret = row.original; - return
{secret.createdAt}
; - } - } -]; + ] + +} const createSortHeader = (name: string, column: Column) => { return ( diff --git a/packages/web/src/app/secrets/secretsTable.tsx b/packages/web/src/app/secrets/secretsTable.tsx index 18b80de5..13fb3c6b 100644 --- a/packages/web/src/app/secrets/secretsTable.tsx +++ b/packages/web/src/app/secrets/secretsTable.tsx @@ -11,6 +11,7 @@ import { columns, SecretColumnInfo } from "./columns"; import { DataTable } from "@/components/ui/data-table"; import { isServiceError } from "@/lib/utils"; import { useToast } from "@/components/hooks/use-toast"; +import { deleteSecret } from "../../actions" const formSchema = z.object({ key: z.string().min(2).max(40), @@ -21,16 +22,16 @@ export const SecretsTable = () => { const [secrets, setSecrets] = useState<{ createdAt: Date; key: string; }[]>([]); const { toast } = useToast(); - useEffect(() => { - const fetchSecretKeys = async () => { - const keys = await getSecrets(); - if ('keys' in keys) { - setSecrets(keys); - } else { - console.error(keys); - } - }; + const fetchSecretKeys = async () => { + const keys = await getSecrets(); + if ('keys' in keys) { + setSecrets(keys); + } else { + console.error(keys); + } + }; + useEffect(() => { fetchSecretKeys(); }, []); @@ -67,6 +68,27 @@ export const SecretsTable = () => { } }; + const handleDelete = async (key: string) => { + const res = await deleteSecret(key); + if (isServiceError(res)) { + toast({ + description: `❌ Failed to delete secret` + }); + return; + } else { + toast({ + description: `✅ Secret deleted successfully!` + }); + } + + const keys = await getSecrets(); + if ('keys' in keys) { + setSecrets(keys); + } else { + console.error(keys); + } + }; + const keys = secrets.map((secret): SecretColumnInfo => { return { @@ -111,7 +133,7 @@ export const SecretsTable = () => { Date: Mon, 27 Jan 2025 13:57:17 -0800 Subject: [PATCH 21/21] fix nits from pr review --- packages/backend/src/main.ts | 3 ++ packages/backend/src/utils.ts | 3 +- packages/web/src/actions.ts | 8 +++-- packages/web/src/app/secrets/page.tsx | 18 +++++++--- packages/web/src/app/secrets/secretsTable.tsx | 34 +++++++++++-------- 5 files changed, 43 insertions(+), 23 deletions(-) diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 9b89506e..d74e6d93 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -35,6 +35,9 @@ const getTokenForRepo = async (repo: RepoWithConnections, db: PrismaClient) => { const config = connection.config as unknown as ConnectionConfig; if (config.token) { token = await getTokenFromConfig(config.token, connection.orgId, db); + if (token) { + break; + } } } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 7cfdee8d..f85ea10a 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -4,6 +4,7 @@ import path from 'path'; import micromatch from "micromatch"; import { PrismaClient, Repo } from "@sourcebot/db"; import { decrypt } from "@sourcebot/crypto"; +import { Token } from "@sourcebot/schemas/v3/shared.type"; export const measure = async (cb : () => Promise) => { const start = Date.now(); @@ -87,7 +88,7 @@ export const excludeReposByTopic = (repos: T[], excludedRe }); } -export const getTokenFromConfig = async (token: string | { env: string } | { secret: string }, orgId: number, db?: PrismaClient) => { +export const getTokenFromConfig = async (token: Token, orgId: number, db?: PrismaClient) => { if (typeof token === 'string') { return token; } diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 6ba5b365..ab4c3f9f 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -53,8 +53,7 @@ export const createSecret = async (key: string, value: string): Promise<{ succes } }); } catch (e) { - console.error(e); - return unexpectedError(`Failed to create secret: ${e}`); + return unexpectedError(`Failed to create secret`); } return { @@ -99,7 +98,10 @@ export const getSecrets = async (): Promise<{ createdAt: Date; key: string; }[] } }); - return secrets; + return secrets.map((secret) => ({ + key: secret.key, + createdAt: secret.createdAt, + })); } export const deleteSecret = async (key: string): Promise<{ success: boolean } | ServiceError> => { diff --git a/packages/web/src/app/secrets/page.tsx b/packages/web/src/app/secrets/page.tsx index ccb04ab1..0a594873 100644 --- a/packages/web/src/app/secrets/page.tsx +++ b/packages/web/src/app/secrets/page.tsx @@ -1,13 +1,23 @@ import { NavigationMenu } from "../components/navigationMenu"; import { SecretsTable } from "./secretsTable"; +import { getSecrets, createSecret } from "../../actions" +import { isServiceError } from "@/lib/utils"; -export default function SecretsPage() { +export interface SecretsTableProps { + initialSecrets: { createdAt: Date; key: string; }[]; +} + +export default async function SecretsPage() { + const secrets = await getSecrets(); + return (
-
- -
+ { !isServiceError(secrets) && ( +
+ +
+ )}
) } \ No newline at end of file diff --git a/packages/web/src/app/secrets/secretsTable.tsx b/packages/web/src/app/secrets/secretsTable.tsx index 13fb3c6b..94623bf5 100644 --- a/packages/web/src/app/secrets/secretsTable.tsx +++ b/packages/web/src/app/secrets/secretsTable.tsx @@ -1,5 +1,5 @@ 'use client'; -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { getSecrets, createSecret } from "../../actions" import { Button } from "@/components/ui/button"; import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; @@ -12,14 +12,16 @@ import { DataTable } from "@/components/ui/data-table"; import { isServiceError } from "@/lib/utils"; import { useToast } from "@/components/hooks/use-toast"; import { deleteSecret } from "../../actions" +import { SecretsTableProps } from "./page"; const formSchema = z.object({ key: z.string().min(2).max(40), value: z.string().min(2).max(40), }); -export const SecretsTable = () => { - const [secrets, setSecrets] = useState<{ createdAt: Date; key: string; }[]>([]); + +export const SecretsTable = ({ initialSecrets }: SecretsTableProps) => { + const [secrets, setSecrets] = useState<{ createdAt: Date; key: string; }[]>(initialSecrets); const { toast } = useToast(); const fetchSecretKeys = async () => { @@ -57,14 +59,14 @@ export const SecretsTable = () => { } const keys = await getSecrets(); - if ('keys' in keys) { + if (isServiceError(keys)) { + console.error("Failed to fetch secrets"); + } else { setSecrets(keys); - + form.reset(); form.resetField("key"); form.resetField("value"); - } else { - console.error(keys); } }; @@ -90,14 +92,16 @@ export const SecretsTable = () => { }; - const keys = secrets.map((secret): SecretColumnInfo => { - return { - key: secret.key, - createdAt: secret.createdAt.toISOString(), - } - }).sort((a, b) => { - return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(); - }); + const keys = useMemo(() => { + return secrets.map((secret): SecretColumnInfo => { + return { + key: secret.key, + createdAt: secret.createdAt.toISOString(), + } + }).sort((a, b) => { + return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(); + }); + }, [secrets]); return (