diff --git a/Dockerfile b/Dockerfile index 8f74ec17..3720da60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,13 +10,15 @@ RUN go mod download COPY vendor/zoekt ./ RUN CGO_ENABLED=0 GOOS=linux go build -o /cmd/ ./cmd/... -# ------ Build Database ------ -FROM node-alpine AS database-builder +# ------ Build shared libraries ------ +FROM node-alpine AS shared-libs-builder WORKDIR /app COPY package.json yarn.lock* ./ COPY ./packages/db ./packages/db +COPY ./packages/schemas ./packages/schemas RUN yarn workspace @sourcebot/db install --frozen-lockfile +RUN yarn workspace @sourcebot/schemas install --frozen-lockfile # ------ Build Web ------ FROM node-alpine AS web-builder @@ -25,8 +27,9 @@ WORKDIR /app COPY package.json yarn.lock* ./ COPY ./packages/web ./packages/web -COPY --from=database-builder /app/node_modules ./node_modules -COPY --from=database-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/node_modules ./node_modules +COPY --from=shared-libs-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas # Fixes arm64 timeouts RUN yarn config set registry https://registry.npmjs.org/ @@ -54,8 +57,9 @@ WORKDIR /app COPY package.json yarn.lock* ./ COPY ./schemas ./schemas COPY ./packages/backend ./packages/backend -COPY --from=database-builder /app/node_modules ./node_modules -COPY --from=database-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/node_modules ./node_modules +COPY --from=shared-libs-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas RUN yarn workspace @sourcebot/backend install --frozen-lockfile RUN yarn workspace @sourcebot/backend build @@ -114,21 +118,22 @@ COPY --from=zoekt-builder \ /cmd/zoekt-index \ /usr/local/bin/ -# Configure the webapp +# Copy all of the things COPY --from=web-builder /app/packages/web/public ./packages/web/public COPY --from=web-builder /app/packages/web/.next/standalone ./ COPY --from=web-builder /app/packages/web/.next/static ./packages/web/.next/static -# Configure the backend COPY --from=backend-builder /app/node_modules ./node_modules COPY --from=backend-builder /app/packages/backend ./packages/backend +COPY --from=shared-libs-builder /app/node_modules ./node_modules +COPY --from=shared-libs-builder /app/packages/db ./packages/db +COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas + # Configure the database RUN mkdir -p /run/postgresql && \ chown -R postgres:postgres /run/postgresql && \ chmod 775 /run/postgresql -COPY --from=database-builder /app/node_modules ./node_modules -COPY --from=database-builder /app/packages/db ./packages/db COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY prefix-output.sh ./prefix-output.sh diff --git a/Makefile b/Makefile index 3e78a0b7..03a610e6 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ clean: packages/backend/node_modules \ packages/db/node_modules \ packages/db/dist \ + packages/schemas/node_modules \ + packages/schemas/dist \ .sourcebot .PHONY: bin diff --git a/packages/backend/package.json b/packages/backend/package.json index 244d6223..3bc4cae6 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -5,10 +5,9 @@ "main": "index.js", "type": "module", "scripts": { - "dev:watch": "yarn generate:types && tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"", + "dev:watch": "tsc-watch --preserveWatchOutput --onSuccess \"yarn dev --configPath ../../config.json --cacheDir ../../.sourcebot\"", "dev": "export PATH=\"$PWD/../../bin:$PATH\" && export CTAGS_COMMAND=ctags && node ./dist/index.js", - "build": "yarn generate:types && tsc", - "generate:types": "tsx tools/generateTypes.ts", + "build": "tsc", "test": "vitest --config ./vitest.config.ts" }, "devDependencies": { @@ -33,6 +32,7 @@ "micromatch": "^4.0.8", "posthog-node": "^4.2.1", "@sourcebot/db": "^0.1.0", + "@sourcebot/schemas": "^0.1.0", "simple-git": "^3.27.0", "strip-json-comments": "^5.0.1", "winston": "^3.15.0", diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 2434380c..a885a61a 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -3,7 +3,7 @@ import { readFile } from 'fs/promises'; import stripJsonComments from 'strip-json-comments'; import { getGitHubReposFromConfig } from "./github.js"; import { getGitLabReposFromConfig, GITLAB_CLOUD_HOSTNAME } from "./gitlab.js"; -import { SourcebotConfigurationSchema } from "./schemas/v2.js"; +import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type"; import { AppContext } from "./types.js"; import { getTokenFromConfig, isRemotePath, marshalBool } from "./utils.js"; diff --git a/packages/backend/src/gerrit.ts b/packages/backend/src/gerrit.ts index 882f0218..21bd8e8b 100644 --- a/packages/backend/src/gerrit.ts +++ b/packages/backend/src/gerrit.ts @@ -1,5 +1,5 @@ import fetch from 'cross-fetch'; -import { GerritConfig } from './schemas/v2.js'; +import { GerritConfig } from "@sourcebot/schemas/v2/index.type" import { AppContext, GitRepository } from './types.js'; import { createLogger } from './logger.js'; import path from 'path'; diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index 77bcab10..40097dac 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,7 +1,7 @@ import { GitRepository, AppContext } from './types.js'; import { simpleGit, SimpleGitProgressEvent } from 'simple-git'; import { createLogger } from './logger.js'; -import { GitConfig } from './schemas/v2.js'; +import { GitConfig } from "@sourcebot/schemas/v2/index.type" import path from 'path'; const logger = createLogger('git'); diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index 831d38a6..f3a6b514 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -1,5 +1,5 @@ import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js'; -import { GiteaConfig } from './schemas/v2.js'; +import { GiteaConfig } from "@sourcebot/schemas/v2/index.type" import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js'; import { AppContext, GitRepository } from './types.js'; import fetch from 'cross-fetch'; diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index baa98512..104aff5c 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -1,5 +1,5 @@ import { Octokit } from "@octokit/rest"; -import { GitHubConfig } from "./schemas/v2.js"; +import { GitHubConfig } from "@sourcebot/schemas/v2/index.type" import { createLogger } from "./logger.js"; import { AppContext } from "./types.js"; import { getTokenFromConfig, measure } from "./utils.js"; diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 20d4e5b1..2501fd05 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -1,7 +1,7 @@ import { Gitlab, ProjectSchema } from "@gitbeaker/rest"; import micromatch from "micromatch"; import { createLogger } from "./logger.js"; -import { GitLabConfig } from "./schemas/v2.js"; +import { GitLabConfig } from "@sourcebot/schemas/v2/index.type" import { AppContext } from "./types.js"; import { getTokenFromConfig, measure } from "./utils.js"; diff --git a/packages/backend/src/local.ts b/packages/backend/src/local.ts index 6e415546..0bddac6a 100644 --- a/packages/backend/src/local.ts +++ b/packages/backend/src/local.ts @@ -1,6 +1,6 @@ import { existsSync, FSWatcher, statSync, watch } from "fs"; import { createLogger } from "./logger.js"; -import { LocalConfig } from "./schemas/v2.js"; +import { LocalConfig } from "@sourcebot/schemas/v2/index.type" import { AppContext, LocalRepository } from "./types.js"; import { resolvePathRelativeToConfig } from "./utils.js"; import path from "path"; diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index c77ab0d1..34f0e28c 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -12,7 +12,7 @@ import { Queue, Worker, Job } from 'bullmq'; import { Redis } from 'ioredis'; import * as os from 'os'; import { SOURCEBOT_TENANT_MODE } from './environment.js'; -import { SourcebotConfigurationSchema } from './schemas/v2.js'; +import { SourcebotConfigurationSchema } from "@sourcebot/schemas/v2/index.type" const logger = createLogger('main'); diff --git a/packages/backend/tools/generateTypes.ts b/packages/backend/tools/generateTypes.ts deleted file mode 100644 index 9e78e137..00000000 --- a/packages/backend/tools/generateTypes.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { compileFromFile } from 'json-schema-to-typescript' -import path from 'path'; -import fs from 'fs'; - -const BANNER_COMMENT = '// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!\n'; - -(async () => { - const cwd = process.cwd(); - const schemaPath = path.resolve(`${cwd}/../../schemas/v2/index.json`); - const outputPath = path.resolve(`${cwd}/src/schemas/v2.ts`); - - const content = await compileFromFile(schemaPath, { - bannerComment: BANNER_COMMENT, - cwd, - ignoreMinAndMaxItems: true, - }); - - await fs.promises.writeFile( - outputPath, - content, - "utf-8" - ); -})(); \ No newline at end of file diff --git a/packages/schemas/package.json b/packages/schemas/package.json new file mode 100644 index 00000000..01fadaee --- /dev/null +++ b/packages/schemas/package.json @@ -0,0 +1,21 @@ +{ + "name": "@sourcebot/schemas", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "yarn generate && tsc", + "generate": "tsx tools/generate.ts", + "postinstall": "yarn build" + }, + "devDependencies": { + "@apidevtools/json-schema-ref-parser": "^11.7.3", + "glob": "^11.0.1", + "json-schema-to-typescript": "^15.0.4", + "tsx": "^4.19.2", + "typescript": "^5.7.3" + }, + "exports": { + "./v2/*": "./dist/v2/*.js", + "./v3/*": "./dist/v3/*.js" + } +} diff --git a/packages/schemas/src/v1/index.schema.ts b/packages/schemas/src/v1/index.schema.ts new file mode 100644 index 00000000..f6c3d547 --- /dev/null +++ b/packages/schemas/src/v1/index.schema.ts @@ -0,0 +1,137 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "definitions": { + "RepoNameRegexIncludeFilter": { + "type": "string", + "description": "Only clone repos whose name matches the given regexp.", + "format": "regexp", + "default": "^(foo|bar)$" + }, + "RepoNameRegexExcludeFilter": { + "type": "string", + "description": "Don't mirror repos whose names match this regexp.", + "format": "regexp", + "default": "^(fizz|buzz)$" + }, + "ZoektConfig": { + "anyOf": [ + { + "$ref": "#/definitions/GitHubConfig" + }, + { + "$ref": "#/definitions/GitLabConfig" + } + ] + }, + "GitHubConfig": { + "type": "object", + "properties": { + "Type": { + "const": "github" + }, + "GitHubUrl": { + "type": "string", + "description": "GitHub Enterprise url. If not set github.com will be used as the host." + }, + "GitHubUser": { + "type": "string", + "description": "The GitHub user to mirror" + }, + "GitHubOrg": { + "type": "string", + "description": "The GitHub organization to mirror" + }, + "Name": { + "$ref": "#/definitions/RepoNameRegexIncludeFilter" + }, + "Exclude": { + "$ref": "#/definitions/RepoNameRegexExcludeFilter" + }, + "CredentialPath": { + "type": "string", + "description": "Path to a file containing a GitHub access token.", + "default": "~/.github-token" + }, + "Topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Only mirror repos that have one of the given topics" + }, + "ExcludeTopics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Don't mirror repos that have one of the given topics" + }, + "NoArchived": { + "type": "boolean", + "description": "Mirror repos that are _not_ archived", + "default": false + }, + "IncludeForks": { + "type": "boolean", + "description": "Also mirror forks", + "default": false + } + }, + "required": [ + "Type" + ], + "additionalProperties": false + }, + "GitLabConfig": { + "type": "object", + "properties": { + "Type": { + "const": "gitlab" + }, + "GitLabURL": { + "type": "string", + "description": "The GitLab API url.", + "default": "https://gitlab.com/api/v4/" + }, + "Name": { + "$ref": "#/definitions/RepoNameRegexIncludeFilter" + }, + "Exclude": { + "$ref": "#/definitions/RepoNameRegexExcludeFilter" + }, + "OnlyPublic": { + "type": "boolean", + "description": "Only mirror public repos", + "default": false + }, + "CredentialPath": { + "type": "string", + "description": "Path to a file containing a GitLab access token.", + "default": "~/.gitlab-token" + } + }, + "required": [ + "Type" + ], + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "Configs": { + "type": "array", + "items": { + "$ref": "#/definitions/ZoektConfig" + } + } + }, + "required": [ + "Configs" + ], + "additionalProperties": false +} as const; +export { schema as indexSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v1/index.type.ts b/packages/schemas/src/v1/index.type.ts new file mode 100644 index 00000000..2f0bcc1d --- /dev/null +++ b/packages/schemas/src/v1/index.type.ts @@ -0,0 +1,88 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "ZoektConfig". + */ +export type ZoektConfig = GitHubConfig | GitLabConfig; +/** + * Only clone repos whose name matches the given regexp. + * + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "RepoNameRegexIncludeFilter". + */ +export type RepoNameRegexIncludeFilter = string; +/** + * Don't mirror repos whose names match this regexp. + * + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "RepoNameRegexExcludeFilter". + */ +export type RepoNameRegexExcludeFilter = string; + +export interface Index { + $schema?: string; + Configs: ZoektConfig[]; +} +/** + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "GitHubConfig". + */ +export interface GitHubConfig { + Type: "github"; + /** + * GitHub Enterprise url. If not set github.com will be used as the host. + */ + GitHubUrl?: string; + /** + * The GitHub user to mirror + */ + GitHubUser?: string; + /** + * The GitHub organization to mirror + */ + GitHubOrg?: string; + Name?: RepoNameRegexIncludeFilter; + Exclude?: RepoNameRegexExcludeFilter; + /** + * Path to a file containing a GitHub access token. + */ + CredentialPath?: string; + /** + * Only mirror repos that have one of the given topics + */ + Topics?: string[]; + /** + * Don't mirror repos that have one of the given topics + */ + ExcludeTopics?: string[]; + /** + * Mirror repos that are _not_ archived + */ + NoArchived?: boolean; + /** + * Also mirror forks + */ + IncludeForks?: boolean; +} +/** + * This interface was referenced by `Index`'s JSON-Schema + * via the `definition` "GitLabConfig". + */ +export interface GitLabConfig { + Type: "gitlab"; + /** + * The GitLab API url. + */ + GitLabURL?: string; + Name?: RepoNameRegexIncludeFilter; + Exclude?: RepoNameRegexExcludeFilter; + /** + * Only mirror public repos + */ + OnlyPublic?: boolean; + /** + * Path to a file containing a GitLab access token. + */ + CredentialPath?: string; +} diff --git a/packages/schemas/src/v2/index.schema.ts b/packages/schemas/src/v2/index.schema.ts new file mode 100644 index 00000000..ed475f73 --- /dev/null +++ b/packages/schemas/src/v2/index.schema.ts @@ -0,0 +1,640 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Sourcebot configuration schema", + "description": "A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.", + "definitions": { + "Token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "GitRevisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + }, + "GitHubConfig": { + "type": "object", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "$ref": "#/definitions/Token", + "description": "A Personal Access Token (PAT).", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://github.com", + "description": "The URL of the GitHub host. Defaults to https://github.com", + "examples": [ + "https://github.com", + "https://github.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "users": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "torvalds", + "DHH" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "my-org-name" + ], + [ + "sourcebot-dev", + "commaai" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "tenantId": { + "type": "number", + "description": "@nocheckin" + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "GitLabConfig": { + "type": "object", + "properties": { + "type": { + "const": "gitlab", + "description": "GitLab Configuration" + }, + "token": { + "$ref": "#/definitions/Token", + "description": "An authentication token.", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://gitlab.com", + "description": "The URL of the GitLab host. Defaults to https://gitlab.com", + "examples": [ + "https://gitlab.com", + "https://gitlab.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "all": { + "type": "boolean", + "default": false, + "description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ." + }, + "users": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "groups": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-group" + ], + [ + "my-group/sub-group-a", + "my-group/sub-group-b" + ] + ], + "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-group/my-project" + ], + [ + "my-group/my-sub-group/my-project" + ] + ], + "description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked projects from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived projects from syncing." + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "examples": [ + [ + "my-group/my-project" + ] + ], + "description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "GiteaConfig": { + "type": "object", + "properties": { + "type": { + "const": "gitea", + "description": "Gitea Configuration" + }, + "token": { + "$ref": "#/definitions/Token", + "description": "An access token.", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://gitea.com", + "description": "The URL of the Gitea host. Defaults to https://gitea.com", + "examples": [ + "https://gitea.com", + "https://gitea.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "orgs": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "my-org-name" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "users": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "username-1", + "username-2" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope." + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + } + }, + "additionalProperties": false + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + "GerritConfig": { + "type": "object", + "properties": { + "type": { + "const": "gerrit", + "description": "Gerrit Configuration" + }, + "url": { + "type": "string", + "format": "url", + "description": "The URL of the Gerrit host.", + "examples": [ + "https://gerrit.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported", + "examples": [ + [ + "project1/repo1", + "project2/**" + ] + ] + }, + "exclude": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "project1/repo1", + "project2/**" + ] + ], + "description": "List of specific projects to exclude from syncing." + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "url" + ], + "additionalProperties": false + }, + "LocalConfig": { + "type": "object", + "properties": { + "type": { + "const": "local", + "description": "Local Configuration" + }, + "path": { + "type": "string", + "description": "Path to the local directory to sync with. Relative paths are relative to the configuration file's directory.", + "pattern": ".+" + }, + "watch": { + "type": "boolean", + "default": true, + "description": "Enables a file watcher that will automatically re-sync when changes are made within `path` (recursively). Defaults to true." + }, + "exclude": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { + "type": "string", + "pattern": ".+" + }, + "description": "List of paths relative to the provided `path` to exclude from the index. .git, .hg, and .svn are always exluded.", + "default": [], + "examples": [ + [ + "node_modules", + "bin", + "dist", + "build", + "out" + ] + ] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type", + "path" + ], + "additionalProperties": false + }, + "GitConfig": { + "type": "object", + "properties": { + "type": { + "const": "git", + "description": "Git Configuration" + }, + "url": { + "type": "string", + "format": "url", + "description": "The URL to the git repository." + }, + "revisions": { + "$ref": "#/definitions/GitRevisions" + } + }, + "required": [ + "type", + "url" + ], + "additionalProperties": false + }, + "Repos": { + "anyOf": [ + { + "$ref": "#/definitions/GitHubConfig" + }, + { + "$ref": "#/definitions/GitLabConfig" + }, + { + "$ref": "#/definitions/GiteaConfig" + }, + { + "$ref": "#/definitions/GerritConfig" + }, + { + "$ref": "#/definitions/LocalConfig" + }, + { + "$ref": "#/definitions/GitConfig" + } + ] + }, + "Settings": { + "type": "object", + "description": "Global settings. These settings are applied to all repositories.", + "properties": { + "maxFileSize": { + "type": "integer", + "description": "The maximum size of a file (in bytes) to be indexed. Files that exceed this maximum will not be inexed. Defaults to 2MB (2097152 bytes).", + "default": 2097152, + "minimum": 1 + }, + "autoDeleteStaleRepos": { + "type": "boolean", + "description": "Automatically delete stale repositories from the index. Defaults to true.", + "default": true + }, + "reindexInterval": { + "type": "integer", + "description": "The interval (in milliseconds) at which the indexer should re-index all repositories. Repositories are always indexed when first added. Defaults to 1 hour (3600000 milliseconds).", + "default": 3600000, + "minimum": 1 + }, + "resyncInterval": { + "type": "integer", + "description": "The interval (in milliseconds) at which the configuration file should be re-synced. The configuration file is always synced on startup. Defaults to 24 hours (86400000 milliseconds).", + "default": 86400000, + "minimum": 1 + } + }, + "additionalProperties": false + } + }, + "properties": { + "$schema": { + "type": "string" + }, + "settings": { + "$ref": "#/definitions/Settings" + }, + "repos": { + "type": "array", + "description": "Defines a collection of repositories from varying code hosts that Sourcebot should sync with.", + "items": { + "$ref": "#/definitions/Repos" + } + } + }, + "additionalProperties": false +} as const; +export { schema as indexSchema }; \ No newline at end of file diff --git a/packages/backend/src/schemas/v2.ts b/packages/schemas/src/v2/index.type.ts similarity index 86% rename from packages/backend/src/schemas/v2.ts rename to packages/schemas/src/v2/index.type.ts index 4ada1870..1449d43c 100644 --- a/packages/backend/src/schemas/v2.ts +++ b/packages/schemas/src/v2/index.type.ts @@ -1,6 +1,22 @@ // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "Repos". + */ export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig | GitConfig; +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "Token". + */ +export type Token = + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; /** * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index. @@ -15,6 +31,9 @@ export interface SourcebotConfigurationSchema { } /** * Global settings. These settings are applied to all repositories. + * + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "Settings". */ export interface Settings { /** @@ -34,6 +53,10 @@ export interface Settings { */ resyncInterval?: number; } +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitHubConfig". + */ export interface GitHubConfig { /** * GitHub Configuration @@ -111,6 +134,9 @@ export interface GitHubConfig { } /** * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitRevisions". */ export interface GitRevisions { /** @@ -122,6 +148,10 @@ export interface GitRevisions { */ tags?: string[]; } +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitLabConfig". + */ export interface GitLabConfig { /** * GitLab Configuration @@ -184,6 +214,10 @@ export interface GitLabConfig { }; revisions?: GitRevisions; } +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GiteaConfig". + */ export interface GiteaConfig { /** * Gitea Configuration @@ -232,6 +266,10 @@ export interface GiteaConfig { }; revisions?: GitRevisions; } +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GerritConfig". + */ export interface GerritConfig { /** * Gerrit Configuration @@ -252,6 +290,10 @@ export interface GerritConfig { projects?: string[]; }; } +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "LocalConfig". + */ export interface LocalConfig { /** * Local Configuration @@ -272,6 +314,10 @@ export interface LocalConfig { paths?: string[]; }; } +/** + * This interface was referenced by `SourcebotConfigurationSchema`'s JSON-Schema + * via the `definition` "GitConfig". + */ export interface GitConfig { /** * Git Configuration diff --git a/packages/schemas/src/v3/github.schema.ts b/packages/schemas/src/v3/github.schema.ts new file mode 100644 index 00000000..0c88f81b --- /dev/null +++ b/packages/schemas/src/v3/github.schema.ts @@ -0,0 +1,205 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "GitHubConfig", + "properties": { + "type": { + "const": "github", + "description": "GitHub Configuration" + }, + "token": { + "description": "A Personal Access Token (PAT).", + "examples": [ + "secret-token", + { + "env": "ENV_VAR_CONTAINING_TOKEN" + } + ], + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "url": { + "type": "string", + "format": "url", + "default": "https://github.com", + "description": "The URL of the GitHub host. Defaults to https://github.com", + "examples": [ + "https://github.com", + "https://github.example.com" + ], + "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$" + }, + "users": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "torvalds", + "DHH" + ] + ], + "description": "List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property." + }, + "orgs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+$" + }, + "examples": [ + [ + "my-org-name" + ], + [ + "sourcebot-dev", + "commaai" + ] + ], + "description": "List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property." + }, + "repos": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[\\w.-]+\\/[\\w.-]+$" + }, + "description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.", + "examples": [ + [ + "docs", + "core" + ] + ] + }, + "tenantId": { + "type": "number", + "description": "@nocheckin" + }, + "exclude": { + "type": "object", + "properties": { + "forks": { + "type": "boolean", + "default": false, + "description": "Exclude forked repositories from syncing." + }, + "archived": { + "type": "boolean", + "default": false, + "description": "Exclude archived repositories from syncing." + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "List of individual repositories to exclude from syncing. Glob patterns are supported." + }, + "topics": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.", + "examples": [ + [ + "tests", + "ci" + ] + ] + }, + "size": { + "type": "object", + "description": "Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned.", + "properties": { + "min": { + "type": "integer", + "description": "Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing." + }, + "max": { + "type": "integer", + "description": "Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "revisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + }, + "required": [ + "type" + ], + "additionalProperties": false +} as const; +export { schema as githubSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/github.type.ts b/packages/schemas/src/v3/github.type.ts new file mode 100644 index 00000000..d9c7e1aa --- /dev/null +++ b/packages/schemas/src/v3/github.type.ts @@ -0,0 +1,90 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +export interface GitHubConfig { + /** + * GitHub Configuration + */ + type: "github"; + /** + * A Personal Access Token (PAT). + */ + token?: + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + /** + * The URL of the GitHub host. Defaults to https://github.com + */ + url?: string; + /** + * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. + */ + users?: string[]; + /** + * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. + */ + orgs?: string[]; + /** + * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'. + */ + repos?: string[]; + /** + * List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported. + * + * @minItems 1 + */ + topics?: string[]; + /** + * @nocheckin + */ + tenantId?: number; + exclude?: { + /** + * Exclude forked repositories from syncing. + */ + forks?: boolean; + /** + * Exclude archived repositories from syncing. + */ + archived?: boolean; + /** + * List of individual repositories to exclude from syncing. Glob patterns are supported. + */ + repos?: string[]; + /** + * List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported. + */ + topics?: string[]; + /** + * Exclude repositories based on their disk usage. Note: the disk usage is calculated by GitHub and may not reflect the actual disk usage when cloned. + */ + size?: { + /** + * Minimum repository size (in bytes) to sync (inclusive). Repositories less than this size will be excluded from syncing. + */ + min?: number; + /** + * Maximum repository size (in bytes) to sync (inclusive). Repositories greater than this size will be excluded from syncing. + */ + max?: number; + }; + }; + revisions?: GitRevisions; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + */ + tags?: string[]; +} diff --git a/packages/schemas/src/v3/shared.schema.ts b/packages/schemas/src/v3/shared.schema.ts new file mode 100644 index 00000000..ccb21856 --- /dev/null +++ b/packages/schemas/src/v3/shared.schema.ts @@ -0,0 +1,69 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! +const schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "definitions": { + "Token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "env": { + "type": "string", + "description": "The name of the environment variable that contains the token." + } + }, + "required": [ + "env" + ], + "additionalProperties": false + } + ] + }, + "GitRevisions": { + "type": "object", + "description": "The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed.", + "properties": { + "branches": { + "type": "array", + "description": "List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "main", + "release/*" + ], + [ + "**" + ] + ], + "default": [] + }, + "tags": { + "type": "array", + "description": "List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported.", + "items": { + "type": "string" + }, + "examples": [ + [ + "latest", + "v2.*.*" + ], + [ + "**" + ] + ], + "default": [] + } + }, + "additionalProperties": false + } + } +} as const; +export { schema as sharedSchema }; \ No newline at end of file diff --git a/packages/schemas/src/v3/shared.type.ts b/packages/schemas/src/v3/shared.type.ts new file mode 100644 index 00000000..6dcd54e7 --- /dev/null +++ b/packages/schemas/src/v3/shared.type.ts @@ -0,0 +1,34 @@ +// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY! + +/** + * This interface was referenced by `Shared`'s JSON-Schema + * via the `definition` "Token". + */ +export type Token = + | string + | { + /** + * The name of the environment variable that contains the token. + */ + env: string; + }; + +export interface Shared { + [k: string]: unknown; +} +/** + * The revisions (branches, tags) that should be included when indexing. The default branch (HEAD) is always indexed. + * + * This interface was referenced by `Shared`'s JSON-Schema + * via the `definition` "GitRevisions". + */ +export interface GitRevisions { + /** + * List of branches to include when indexing. For a given repo, only the branches that exist on the repo's remote *and* match at least one of the provided `branches` will be indexed. The default branch (HEAD) is always indexed. Glob patterns are supported. + */ + branches?: string[]; + /** + * List of tags to include when indexing. For a given repo, only the tags that exist on the repo's remote *and* match at least one of the provided `tags` will be indexed. Glob patterns are supported. + */ + tags?: string[]; +} diff --git a/packages/schemas/tools/generate.ts b/packages/schemas/tools/generate.ts new file mode 100644 index 00000000..a5911e27 --- /dev/null +++ b/packages/schemas/tools/generate.ts @@ -0,0 +1,48 @@ +import path, { dirname } from "path"; +import { mkdir, rm, writeFile } from "fs/promises"; +import $RefParser from "@apidevtools/json-schema-ref-parser"; +import { compileFromFile } from "json-schema-to-typescript"; +import { glob } from "glob"; + + +const BANNER_COMMENT = '// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!\n'; +// const SCHEMAS: string[] = ["github.json", "shared.json"]; + +(async () => { + const cwd = process.cwd(); + const schemasBasePath = path.resolve(`${cwd}/../../schemas`); + const schemas = await glob(`${schemasBasePath}/**/*.json`) + + await Promise.all(schemas.map(async (schemaPath) => { + const name = path.parse(schemaPath).name; + const version = path.basename(path.dirname(schemaPath)); + const outDir = path.join(cwd, `src/${version}`); + + // Clean output directory first + await rm(outDir, { recursive: true, force: true }); + await mkdir(outDir, { recursive: true }); + + // Generate schema + const schema = JSON.stringify(await $RefParser.bundle(schemaPath), null, 2); + await writeFile( + path.join(outDir, `${name}.schema.ts`), + BANNER_COMMENT + + 'const schema = ' + + schema + + ` as const;\nexport { schema as ${name}Schema };`, + ); + + // Generate types + const content = await compileFromFile(schemaPath, { + bannerComment: BANNER_COMMENT, + cwd: dirname(schemaPath), + ignoreMinAndMaxItems: true, + declareExternallyReferenced: true, + unreachableDefinitions: true, + }); + await writeFile( + path.join(outDir, `${name}.type.ts`), + content, + ) + })); +})(); \ No newline at end of file diff --git a/packages/schemas/tsconfig.json b/packages/schemas/tsconfig.json new file mode 100644 index 00000000..cbb69b61 --- /dev/null +++ b/packages/schemas/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "outDir": "dist", + "incremental": true, + "declaration": true, + "emitDecoratorMetadata": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": false, + "module": "CommonJS", + "moduleResolution": "node", + "noEmitOnError": false, + "noImplicitAny": true, + "noUnusedLocals": true, + "pretty": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "strict": true, + "sourceMap": true, + "target": "ES2017", + "rootDir": "src", + "baseUrl": ".", + // This is needed otherwise .tsbuildinfo wasn't being written + // into the dist folder for some reason. + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules"] +} \ No newline at end of file diff --git a/packages/web/package.json b/packages/web/package.json index 6e7f0658..e35ae88e 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -3,12 +3,11 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "yarn generate:schemas && next dev", - "build": "yarn generate:schemas && next build", + "dev": "next dev", + "build": "next build", "start": "next start", "lint": "next lint", - "test": "vitest", - "generate:schemas": "tsx tools/generateSchemas.ts" + "test": "vitest" }, "dependencies": { "@auth/prisma-adapter": "^2.7.4", @@ -59,6 +58,8 @@ "@replit/codemirror-lang-svelte": "^6.0.0", "@replit/codemirror-vim": "^6.2.1", "@shopify/lang-jsonc": "^1.0.0", + "@sourcebot/schemas": "^0.1.0", + "@sourcebot/db": "^0.1.0", "@ssddanbrown/codemirror-lang-twig": "^1.0.0", "@tanstack/react-query": "^5.53.3", "@tanstack/react-table": "^8.20.5", @@ -115,8 +116,6 @@ "zod": "^3.23.8" }, "devDependencies": { - "@apidevtools/json-schema-ref-parser": "^11.7.3", - "@sourcebot/db": "^0.1.0", "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index dc20dbc9..6027ac96 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -5,9 +5,9 @@ import { getUser } from "./data/user"; import { auth } from "./auth"; import { notAuthenticated, notFound, ServiceError, unexpectedError } from "./lib/serviceError"; import { prisma } from "@/prisma"; -import { githubSchema } from "./schemas/github.schema"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "./lib/errorCodes"; +import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; const ajv = new Ajv({ validateFormats: false, diff --git a/packages/web/src/app/connections/new/page.tsx b/packages/web/src/app/connections/new/page.tsx index d986b1ef..e5239658 100644 --- a/packages/web/src/app/connections/new/page.tsx +++ b/packages/web/src/app/connections/new/page.tsx @@ -21,12 +21,12 @@ import { import { useCallback, useRef } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; -import { githubSchema } from "@/schemas/github.schema"; import { Input } from "@/components/ui/input"; import { createConnection } from "@/actions"; import { useToast } from "@/components/hooks/use-toast"; import { isServiceError } from "@/lib/utils"; import { useRouter } from "next/navigation"; +import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; const ajv = new Ajv({ validateFormats: false, diff --git a/packages/web/src/schemas/github.schema.ts b/packages/web/src/schemas/github.schema.ts index 0a784b22..0c88f81b 100644 --- a/packages/web/src/schemas/github.schema.ts +++ b/packages/web/src/schemas/github.schema.ts @@ -2,6 +2,7 @@ const schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "title": "GitHubConfig", "properties": { "type": { "const": "github", diff --git a/packages/web/tools/generateSchemas.ts b/packages/web/tools/generateSchemas.ts deleted file mode 100644 index 858765dc..00000000 --- a/packages/web/tools/generateSchemas.ts +++ /dev/null @@ -1,30 +0,0 @@ -import $RefParser from "@apidevtools/json-schema-ref-parser"; -import path from "path"; -import { writeFile } from "fs/promises"; - -const BANNER_COMMENT = '// THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!\n'; -const SCHEMAS: string[] = ["github.json"]; - -(async () => { - - const cwd = process.cwd(); - const schemasPath = path.resolve(`${cwd}/../../schemas/v3`); - const outDir = path.resolve(`${cwd}/src/schemas`); - console.log(outDir); - - SCHEMAS.forEach(async (schemaName) => { - const schemaPath = path.resolve(`${schemasPath}/${schemaName}`); - const name = path.parse(schemaPath).name; - console.log(name); - - const schema = JSON.stringify(await $RefParser.bundle(schemaPath), null, 2); - - await writeFile( - path.join(outDir, `${name}.schema.ts`), - BANNER_COMMENT + - 'const schema = ' + - schema + - ` as const;\nexport { schema as ${name}Schema };`, - ); - }); -})(); \ No newline at end of file diff --git a/schemas/index.json b/schemas/index.json deleted file mode 100644 index 2fe2195a..00000000 --- a/schemas/index.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "definitions": { - "RepoNameRegexIncludeFilter": { - "type": "string", - "description": "Only clone repos whose name matches the given regexp.", - "format": "regexp", - "default": "^(foo|bar)$" - }, - "RepoNameRegexExcludeFilter": { - "type": "string", - "description": "Don't mirror repos whose names match this regexp.", - "format": "regexp", - "default": "^(fizz|buzz)$" - }, - "ZoektConfig": { - "anyOf": [ - { - "$ref": "#/definitions/GitHubConfig" - }, - { - "$ref": "#/definitions/GitLabConfig" - } - ] - }, - "GitHubConfig": { - "type": "object", - "properties": { - "Type": { - "const": "github" - }, - "GitHubUrl": { - "type": "string", - "description": "GitHub Enterprise url. If not set github.com will be used as the host." - }, - "GitHubUser": { - "type": "string", - "description": "The GitHub user to mirror" - }, - "GitHubOrg": { - "type": "string", - "description": "The GitHub organization to mirror" - }, - "Name": { - "$ref": "#/definitions/RepoNameRegexIncludeFilter" - }, - "Exclude": { - "$ref": "#/definitions/RepoNameRegexExcludeFilter" - }, - "CredentialPath": { - "type": "string", - "description": "Path to a file containing a GitHub access token.", - "default": "~/.github-token" - }, - "Topics": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Only mirror repos that have one of the given topics" - }, - "ExcludeTopics": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Don't mirror repos that have one of the given topics" - }, - "NoArchived": { - "type": "boolean", - "description": "Mirror repos that are _not_ archived", - "default": false - }, - "IncludeForks": { - "type": "boolean", - "description": "Also mirror forks", - "default": false - } - }, - "required": [ - "Type" - ], - "additionalProperties": false - }, - "GitLabConfig": { - "type": "object", - "properties": { - "Type": { - "const": "gitlab" - }, - "GitLabURL": { - "type": "string", - "description": "The GitLab API url.", - "default": "https://gitlab.com/api/v4/" - }, - "Name": { - "$ref": "#/definitions/RepoNameRegexIncludeFilter" - }, - "Exclude": { - "$ref": "#/definitions/RepoNameRegexExcludeFilter" - }, - "OnlyPublic": { - "type": "boolean", - "description": "Only mirror public repos", - "default": false - }, - "CredentialPath": { - "type": "string", - "description": "Path to a file containing a GitLab access token.", - "default": "~/.gitlab-token" - } - }, - "required": [ - "Type" - ], - "additionalProperties": false - } - }, - "properties": { - "$schema": { - "type": "string" - }, - "Configs": { - "type": "array", - "items": { - "$ref": "#/definitions/ZoektConfig" - } - } - }, - "required": [ - "Configs" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/schemas/v3/github.json b/schemas/v3/github.json index 469d8b16..bc0c1d44 100644 --- a/schemas/v3/github.json +++ b/schemas/v3/github.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "title": "GitHubConfig", "properties": { "type": { "const": "github", diff --git a/yarn.lock b/yarn.lock index 77536fcc..c7e2e797 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4172,6 +4172,11 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fdir@^6.4.2: + version "6.4.3" + resolved "https://registry.yarnpkg.com/fdir/-/fdir-6.4.3.tgz#011cdacf837eca9b811c89dbb902df714273db72" + integrity sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw== + fecha@^4.2.0: version "4.2.3" resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" @@ -4378,6 +4383,18 @@ glob@^11.0.0: package-json-from-dist "^1.0.0" path-scurry "^2.0.0" +glob@^11.0.1: + version "11.0.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-11.0.1.tgz#1c3aef9a59d680e611b53dcd24bb8639cef064d9" + integrity sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw== + dependencies: + foreground-child "^3.1.0" + jackspeak "^4.0.1" + minimatch "^10.0.0" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^2.0.0" + glob@^7.1.3: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" @@ -4985,6 +5002,21 @@ json-schema-to-typescript@^15.0.2: minimist "^1.2.8" prettier "^3.2.5" +json-schema-to-typescript@^15.0.4: + version "15.0.4" + resolved "https://registry.yarnpkg.com/json-schema-to-typescript/-/json-schema-to-typescript-15.0.4.tgz#a530c7f17312503b262ae12233749732171840f3" + integrity sha512-Su9oK8DR4xCmDsLlyvadkXzX6+GGXJpbhwoLtOGArAG61dvbW4YQmSEno2y66ahpIdmLMg6YUf/QHLgiwvkrHQ== + dependencies: + "@apidevtools/json-schema-ref-parser" "^11.5.5" + "@types/json-schema" "^7.0.15" + "@types/lodash" "^4.17.7" + is-glob "^4.0.3" + js-yaml "^4.1.0" + lodash "^4.17.21" + minimist "^1.2.8" + prettier "^3.2.5" + tinyglobby "^0.2.9" + json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" @@ -5756,6 +5788,11 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== +picomatch@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.2.tgz#77c742931e8f3b8820946c76cd0c1f13730d1dab" + integrity sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg== + pidtree@^0.3.0: version "0.3.1" resolved "https://registry.yarnpkg.com/pidtree/-/pidtree-0.3.1.tgz#ef09ac2cc0533df1f3250ccf2c4d366b0d12114a" @@ -6868,6 +6905,14 @@ tinyexec@^0.3.1: resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-0.3.1.tgz#0ab0daf93b43e2c211212396bdb836b468c97c98" integrity sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ== +tinyglobby@^0.2.9: + version "0.2.10" + resolved "https://registry.yarnpkg.com/tinyglobby/-/tinyglobby-0.2.10.tgz#e712cf2dc9b95a1f5c5bbd159720e15833977a0f" + integrity sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew== + dependencies: + fdir "^6.4.2" + picomatch "^4.0.2" + tinypool@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-1.0.1.tgz#c64233c4fac4304e109a64340178760116dbe1fe"