add retries for 429 issues

Makefile

@@ -28,4 +28,9 @@ clean:
 		packages/crypto/dist \
 		.sourcebot
 
+soft-reset:
+	rm -rf .sourcebot
+	redis-cli FLUSHALL
+
+
 .PHONY: bin

demo-site-config.json

@@ -18,7 +18,7 @@
             },
             "repos": [
                 "torvalds/linux",
-                "pytorch/pytorch",
+                "pytorch/pytorch", 
                 "commaai/openpilot",
                 "ggerganov/whisper.cpp",
                 "ggerganov/llama.cpp",
@@ -42,7 +42,6 @@
                 "TheAlgorithms/Python",
                 "trekhleb/javascript-algorithms",
                 "tensorflow/tensorflow",
-                "torvalds/linux",
                 "getify/You-Dont-Know-JS",
                 "CyC2018/CS-Notes",
                 "ohmyzsh/ohmyzsh",
@@ -106,10 +105,8 @@
                 "Hack-with-Github/Awesome-Hacking",
                 "nvbn/thefuck",
                 "mtdvio/every-programmer-should-know",
-                "pytorch/pytorch",
                 "storybookjs/storybook",
                 "neovim/neovim",
-                "tailwindlabs/tailwindcss",
                 "microsoft/Web-Dev-For-Beginners",
                 "django/django",
                 "florinpop17/app-ideas",
@@ -153,7 +150,6 @@
                 "fighting41love/funNLP",
                 "vitejs/vite",
                 "thedaviddias/Front-End-Checklist",
-                "ggerganov/llama.cpp",
                 "coder/code-server",
                 "moby/moby",
                 "CompVis/stable-diffusion",

packages/backend/src/constants.ts

@@ -6,7 +6,7 @@ import { Settings } from "./types.js";
 export const DEFAULT_SETTINGS: Settings = {
     maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
     autoDeleteStaleRepos: true,
-    reindexIntervalMs: 1000 * 60,
+    reindexIntervalMs: 1000 * 60 * 60, // 1 hour
     resyncConnectionPollingIntervalMs: 1000,
     reindexRepoPollingIntervalMs: 1000,
     indexConcurrencyMultiple: 3,

packages/backend/src/environment.ts

@@ -18,6 +18,10 @@ export const getEnvBoolean = (env: string | undefined, defaultValue: boolean) =>
 dotenv.config({
 	path: './.env',
 });
+dotenv.config({
+	path: './.env.local',
+	override: true
+});
 
 
 export const SOURCEBOT_LOG_LEVEL = getEnv(process.env.SOURCEBOT_LOG_LEVEL, 'info')!;
@@ -26,3 +30,7 @@ export const SOURCEBOT_INSTALL_ID = getEnv(process.env.SOURCEBOT_INSTALL_ID, 'un
 export const SOURCEBOT_VERSION = getEnv(process.env.SOURCEBOT_VERSION, 'unknown')!;
 export const POSTHOG_PAPIK = getEnv(process.env.POSTHOG_PAPIK);
 export const POSTHOG_HOST = getEnv(process.env.POSTHOG_HOST);
+
+export const FALLBACK_GITHUB_TOKEN = getEnv(process.env.FALLBACK_GITHUB_TOKEN);
+export const FALLBACK_GITLAB_TOKEN = getEnv(process.env.FALLBACK_GITLAB_TOKEN);
+export const FALLBACK_GITEA_TOKEN = getEnv(process.env.FALLBACK_GITEA_TOKEN);

packages/backend/src/gerrit.ts

@@ -2,7 +2,7 @@ import fetch from 'cross-fetch';
 import { GerritConfig } from "@sourcebot/schemas/v2/index.type"
 import { createLogger } from './logger.js';
 import micromatch from "micromatch";
-import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
+import { measure, marshalBool, excludeReposByName, includeReposByName, fetchWithRetry } from './utils.js';
 
 // https://gerrit-review.googlesource.com/Documentation/rest-api.html
 interface GerritProjects {
@@ -30,21 +30,21 @@ interface GerritWebLink {
 const logger = createLogger('Gerrit');
 
 export const getGerritReposFromConfig = async (config: GerritConfig): Promise<GerritProject[]> => {
-
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
    const hostname = new URL(config.url).hostname;
 
    let { durationMs, data: projects } = await measure(async () => {
       try {
-         return fetchAllProjects(url)
+         const fetchFn = () => fetchAllProjects(url);
+         return fetchWithRetry(fetchFn, `projects from ${url}`, logger);
       } catch (err) {
          logger.error(`Failed to fetch projects from ${url}`, err);
          return null;
       }
    });
 
    if (!projects) {
-      return [];
+      throw new Error(`Failed to fetch projects from ${url}`);
    }
 
    // exclude "All-Projects" and "All-Users" projects

packages/backend/src/git.ts

@@ -99,7 +99,7 @@ export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) =
                 .filter(Boolean)
                 .map(branch => branch.replace('refs/heads/', ''));
 
-            repo.branches = branches.filter(branch => 
+            repo.branches = branches.filter(branch =>
                 branchGlobs.some(glob => new RegExp(glob).test(branch))
             );
         }
@@ -114,7 +114,7 @@ export const getGitRepoFromConfig = async (config: GitConfig, ctx: AppContext) =
                 .filter(Boolean)
                 .map(tag => tag.replace('refs/tags/', ''));
 
-            repo.tags = tags.filter(tag => 
+            repo.tags = tags.filter(tag =>
                 tagGlobs.some(glob => new RegExp(glob).test(tag))
             );
         }

packages/backend/src/gitea.ts

@@ -1,36 +1,47 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
 import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
-import { getTokenFromConfig, measure } from './utils.js';
+import { getTokenFromConfig, measure, fetchWithRetry } from './utils.js';
 import fetch from 'cross-fetch';
 import { createLogger } from './logger.js';
 import micromatch from 'micromatch';
 import { PrismaClient } from '@sourcebot/db';
-
+import { FALLBACK_GITEA_TOKEN } from './environment.js';
 const logger = createLogger('Gitea');
 
 export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
-    // TODO: pass in DB here to fetch secret properly
     const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
 
     const api = giteaApi(config.url ?? 'https://gitea.com', {
-        token,
+        token: token ?? FALLBACK_GITEA_TOKEN,
         customFetch: fetch,
     });
 
     let allRepos: GiteaRepository[] = [];
 
     if (config.orgs) {
-        const _repos = await getReposForOrgs(config.orgs, api);
+        const _repos = await fetchWithRetry(
+            () => getReposForOrgs(config.orgs!, api),
+            `orgs ${config.orgs.join(', ')}`,
+            logger
+        );
         allRepos = allRepos.concat(_repos);
     }
 
     if (config.repos) {
-        const _repos = await getRepos(config.repos, api);
+        const _repos = await fetchWithRetry(
+            () => getRepos(config.repos!, api),
+            `repos ${config.repos.join(', ')}`,
+            logger
+        );
         allRepos = allRepos.concat(_repos);
     }
 
     if (config.users) {
-        const _repos = await getReposOwnedByUsers(config.users, api);
+        const _repos = await fetchWithRetry(
+            () => getReposOwnedByUsers(config.users!, api),
+            `users ${config.users.join(', ')}`,
+            logger
+        );
         allRepos = allRepos.concat(_repos);
     }
 
@@ -50,7 +61,11 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
             allRepos = await Promise.all(
                 allRepos.map(async (repo) => {
                     const [owner, name] = repo.full_name!.split('/');
-                    let branches = (await getBranchesForRepo(owner, name, api)).map(branch => branch.name!);
+                    let branches = (await fetchWithRetry(
+                        () => getBranchesForRepo(owner, name, api),
+                        `branches for ${owner}/${name}`,
+                        logger
+                    )).map(branch => branch.name!);
                     branches = micromatch.match(branches, branchGlobs);
 
                     return {
@@ -66,7 +81,11 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, org
             allRepos = await Promise.all(
                 allRepos.map(async (allRepos) => {
                     const [owner, name] = allRepos.name!.split('/');
-                    let tags = (await getTagsForRepo(owner, name, api)).map(tag => tag.name!);
+                    let tags = (await fetchWithRetry(
+                        () => getTagsForRepo(owner, name, api),
+                        `tags for ${owner}/${name}`,
+                        logger
+                    )).map(tag => tag.name!);
                     tags = micromatch.match(tags, tagGlobs);
 
                     return {
@@ -146,7 +165,7 @@ const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
         return tags;
     } catch (e) {
         logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
-        return [];
+        throw e;
     }
 }
 
@@ -162,7 +181,7 @@ const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) =
         return branches;
     } catch (e) {
         logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
-        return [];
+        throw e;
     }
 }
 
@@ -181,7 +200,7 @@ const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
             return data;
         } catch (e) {
             logger.error(`Failed to fetch repos for user ${user}.`, e);
-            return [];
+            throw e;
         }
     }))).flat();
 
@@ -204,7 +223,7 @@ const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
             return data;
         } catch (e) {
             logger.error(`Failed to fetch repos for org ${org}.`, e);
-            return [];
+            throw e;
         }
     }))).flat();
 }
@@ -224,7 +243,7 @@ const getRepos = async <T>(repos: string[], api: Api<T>) => {
             return [response.data];
         } catch (e) {
             logger.error(`Failed to fetch repository info for ${repo}.`, e);
-            return [];
+            throw e;
         }
     }))).flat();
 }