Adding getSuggestedRepositories method (#18681)

* Adding getSuggestedRepositories method

* accounting for lastUse better

* limit to org

* remove repoName for now

* add checks

* cleanup

* remove repositoryName for now

* cleanup

* removing repo name for now

* adding tracing

Author: selfcontained

components/dashboard/src/components/RepositoryFinder.tsx

@@ -4,10 +4,12 @@
  * See License.AGPL.txt in the project root for license information.
  */
 
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { getGitpodService } from "../service/service";
 import { DropDown2, DropDown2Element, DropDown2SelectedElement } from "./DropDown2";
 import Repository from "../icons/Repository.svg";
+import { useSuggestedRepositories } from "../data/git-providers/suggested-repositories-query";
+import { useFeatureFlag } from "../data/featureflag-query";
 
 const LOCAL_STORAGE_KEY = "open-in-gitpod-search-data";
 
@@ -28,7 +30,22 @@ function stripOffProtocol(url: string): string {
 }
 
 export default function RepositoryFinder(props: RepositoryFinderProps) {
+    const includeProjectsOnCreateWorkspace = useFeatureFlag("includeProjectsOnCreateWorkspace");
+
     const [suggestedContextURLs, setSuggestedContextURLs] = useState<string[]>(loadSearchData());
+    const { data: suggestedRepos } = useSuggestedRepositories();
+
+    const suggestedRepoURLs = useMemo(() => {
+        // If the flag is disabled continue to use suggestedContextURLs
+        if (!includeProjectsOnCreateWorkspace) {
+            return suggestedContextURLs;
+        }
+
+        // For now, convert the suggestedRepos to a list of URLs
+        // We'll follow up with updating the UI with the new data
+        return suggestedRepos?.map((repo) => repo.url) || [];
+    }, [suggestedContextURLs, suggestedRepos, includeProjectsOnCreateWorkspace]);
+
     useEffect(() => {
         getGitpodService()
             .server.getSuggestedContextURLs()
@@ -43,21 +60,21 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
             let result: string[];
             searchString = searchString.trim();
             if (searchString.length > 1) {
-                result = suggestedContextURLs.filter((e) => e.toLowerCase().indexOf(searchString.toLowerCase()) !== -1);
+                result = suggestedRepoURLs.filter((e) => e.toLowerCase().indexOf(searchString.toLowerCase()) !== -1);
                 if (result.length > 200) {
                     result = result.slice(0, 200);
                 }
                 if (result.length === 0) {
                     try {
                         // If the searchString is a URL, and it's not present in the proposed results, "artificially" add it here.
                         new URL(searchString);
-                        if (!suggestedContextURLs.includes(searchString)) {
+                        if (!suggestedRepoURLs.includes(searchString)) {
                             result.push(searchString);
                         }
                     } catch {}
                 }
             } else {
-                result = suggestedContextURLs.slice(0, 200);
+                result = suggestedRepoURLs.slice(0, 200);
             }
 
             return result.map(
@@ -79,7 +96,7 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
                     } as DropDown2Element),
             );
         },
-        [suggestedContextURLs],
+        [suggestedRepoURLs],
     );
 
     const element = (

components/dashboard/src/data/featureflag-query.ts

@@ -27,6 +27,7 @@ const featureFlags = {
     supervisor_live_git_status: false,
     enabledOrbitalDiscoveries: "",
     newProjectIncrementalRepoSearchBBS: false,
+    includeProjectsOnCreateWorkspace: false,
 };
 
 type FeatureFlags = typeof featureFlags;

components/dashboard/src/data/git-providers/suggested-repositories-query.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { useQuery } from "@tanstack/react-query";
+import { useCurrentOrg } from "../organizations/orgs-query";
+import { getGitpodService } from "../../service/service";
+
+export const useSuggestedRepositories = () => {
+    const { data: org } = useCurrentOrg();
+
+    return useQuery(["suggested-repositories", { orgId: org?.id }], async () => {
+        if (!org) {
+            throw new Error("No org selected");
+        }
+
+        return await getGitpodService().server.getSuggestedRepositories(org.id);
+    });
+};

components/gitpod-protocol/src/gitpod-service.ts

@@ -30,6 +30,7 @@ import {
     WorkspaceTimeoutSetting,
     WorkspaceContext,
     LinkedInProfile,
+    SuggestedRepository,
 } from "./protocol";
 import {
     Team,
@@ -98,6 +99,7 @@ export interface GitpodServer extends JsonRpcServer<GitpodClient>, AdminServer,
     getWorkspaceUsers(workspaceId: string): Promise<WorkspaceInstanceUser[]>;
     getFeaturedRepositories(): Promise<WhitelistedRepository[]>;
     getSuggestedContextURLs(): Promise<string[]>;
+    getSuggestedRepositories(organizationId: string): Promise<SuggestedRepository[]>;
     /**
      * **Security:**
      * Sensitive information like an owner token is erased, since it allows access for all team members.

components/gitpod-protocol/src/protocol.ts

@@ -1615,3 +1615,9 @@ export interface LinkedInProfile {
     profilePicture: string;
     emailAddress: string;
 }
+
+export type SuggestedRepository = {
+    url: string;
+    projectId?: string;
+    projectName?: string;
+};

components/server/src/auth/rate-limiter.ts

@@ -65,6 +65,7 @@ const defaultFunctions: FunctionsConfig = {
     getWorkspaceUsers: { group: "default", points: 1 },
     getFeaturedRepositories: { group: "default", points: 1 },
     getSuggestedContextURLs: { group: "default", points: 1 },
+    getSuggestedRepositories: { group: "default", points: 1 },
     getWorkspace: { group: "default", points: 1 },
     isWorkspaceOwner: { group: "default", points: 1 },
     getOwnerToken: { group: "default", points: 1 },

components/server/src/workspace/gitpod-server-impl.ts

@@ -67,6 +67,7 @@ import {
     RoleOrPermission,
     WorkspaceInstanceRepoStatus,
     GetProviderRepositoriesParams,
+    SuggestedRepository,
 } from "@gitpod/gitpod-protocol";
 import { BlockedRepository } from "@gitpod/gitpod-protocol/lib/blocked-repositories-protocol";
 import {
@@ -1739,6 +1740,151 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
             .map((s) => s.url);
     }
 
+    public async getSuggestedRepositories(ctx: TraceContext, organizationId: string): Promise<SuggestedRepository[]> {
+        traceAPIParams(ctx, { organizationId });
+
+        const user = await this.checkAndBlockUser("getSuggestedRepositories");
+
+        if (!uuidValidate(organizationId)) {
+            throw new ApplicationError(ErrorCodes.BAD_REQUEST, "organizationId must be a valid UUID");
+        }
+
+        const logCtx: LogContext = { userId: user.id };
+
+        type SuggestedRepositoryWithSorting = SuggestedRepository & {
+            priority: number;
+            lastUse?: string;
+        };
+
+        const fetchProjects = async (): Promise<SuggestedRepositoryWithSorting[]> => {
+            const span = TraceContext.startSpan("getSuggestedRepositories.fetchProjects", ctx);
+            const projects = await this.projectsService.getProjects(user.id, organizationId);
+
+            const projectRepos = projects.map((project) => ({
+                url: project.cloneUrl.replace(/\.git$/, ""),
+                projectId: project.id,
+                projectName: project.name,
+                priority: 1,
+            }));
+
+            span.finish();
+
+            return projectRepos;
+        };
+
+        // Load user repositories (from Git hosts directly)
+        const fetchUserRepos = async (): Promise<SuggestedRepositoryWithSorting[]> => {
+            const span = TraceContext.startSpan("getSuggestedRepositories.fetchUserRepos", ctx);
+            const authProviders = await this.getAuthProviders(ctx);
+
+            const providerRepos = await Promise.all(
+                authProviders.map(async (p): Promise<SuggestedRepositoryWithSorting[]> => {
+                    try {
+                        span.setTag("host", p.host);
+
+                        const hostContext = this.hostContextProvider.get(p.host);
+                        const services = hostContext?.services;
+                        if (!services) {
+                            log.error(logCtx, "Unsupported repository host: " + p.host);
+                            return [];
+                        }
+                        const userRepos = await services.repositoryProvider.getUserRepos(user);
+
+                        return userRepos.map((r) => ({
+                            url: r.replace(/\.git$/, ""),
+                            priority: 5,
+                        }));
+                    } catch (error) {
+                        log.debug(logCtx, "Could not get user repositories from host " + p.host, error);
+                    }
+
+                    return [];
+                }),
+            );
+
+            span.finish();
+
+            return providerRepos.flat();
+        };
+
+        const fetchRecentRepos = async (): Promise<SuggestedRepositoryWithSorting[]> => {
+            const span = TraceContext.startSpan("getSuggestedRepositories.fetchRecentRepos", ctx);
+
+            const workspaces = await this.getWorkspaces(ctx, { organizationId });
+            const recentRepos: SuggestedRepositoryWithSorting[] = [];
+
+            for (const ws of workspaces) {
+                let repoUrl;
+                if (CommitContext.is(ws.workspace.context)) {
+                    repoUrl = ws.workspace.context?.repository?.cloneUrl?.replace(/\.git$/, "");
+                }
+                if (!repoUrl) {
+                    repoUrl = ws.workspace.contextURL;
+                }
+                if (repoUrl) {
+                    const lastUse = WorkspaceInfo.lastActiveISODate(ws);
+
+                    recentRepos.push({
+                        url: repoUrl,
+                        projectId: ws.workspace.projectId,
+                        priority: 10,
+                        lastUse,
+                    });
+                }
+            }
+
+            span.finish();
+
+            return recentRepos;
+        };
+
+        const repoResults = await Promise.allSettled([
+            fetchProjects().catch((e) => log.error(logCtx, "Could not fetch projects", e)),
+            fetchUserRepos().catch((e) => log.error(logCtx, "Could not fetch user repositories", e)),
+            fetchRecentRepos().catch((e) => log.error(logCtx, "Could not fetch recent repositories", e)),
+        ]);
+
+        const sortedRepos = repoResults
+            .map((r) => (r.status === "fulfilled" ? r.value || [] : []))
+            .flat()
+            .sort((a, b) => {
+                // priority first
+                if (a.priority !== b.priority) {
+                    return a.priority < b.priority ? 1 : -1;
+                }
+                // Most recently used second
+                if (b.lastUse || a.lastUse) {
+                    const la = a.lastUse || "";
+                    const lb = b.lastUse || "";
+                    return la < lb ? 1 : la === lb ? 0 : -1;
+                }
+                // Otherwise, alphasort
+                const ua = a.url.toLowerCase();
+                const ub = b.url.toLowerCase();
+                return ua > ub ? 1 : ua === ub ? 0 : -1;
+            });
+
+        const uniqueRepositories = new Map<string, SuggestedRepositoryWithSorting>();
+
+        for (const repo of sortedRepos) {
+            const existingRepo = uniqueRepositories.get(repo.url);
+
+            uniqueRepositories.set(repo.url, {
+                ...(existingRepo || {}),
+                ...repo,
+            });
+        }
+
+        // Convert to return type
+        return Array.from(uniqueRepositories.values()).map(
+            (repo): SuggestedRepository => ({
+                url: repo.url,
+                projectId: repo.projectId,
+                projectName: repo.projectName,
+            }),
+        );
+    }
+
     public async setWorkspaceTimeout(
         ctx: TraceContext,
         workspaceId: string,