From e489e114bf99666e9589a4337d8cab953b817f2e Mon Sep 17 00:00:00 2001 From: Axel Schneider Date: Sun, 11 May 2025 02:17:21 +0200 Subject: [PATCH 1/5] Support self-hosted gitlab instance --- api/data_pipeline.py | 116 +++++++----------- api/rag.py | 3 +- api/simple_chat.py | 36 +----- src/app/[owner]/[repo]/page.tsx | 197 +++++++++++-------------------- src/app/page.tsx | 157 +++++++++++------------- src/components/Ask.tsx | 66 ++++------- src/types/repoinfo.tsx | 10 ++ src/types/wiki/wikipage.tsx | 13 ++ src/types/wiki/wikistructure.tsx | 9 ++ src/utils/getRepoUrl.tsx | 10 ++ src/utils/urlDecoder.tsx | 21 ++++ 11 files changed, 276 insertions(+), 362 deletions(-) create mode 100644 src/types/repoinfo.tsx create mode 100644 src/types/wiki/wikipage.tsx create mode 100644 src/types/wiki/wikistructure.tsx create mode 100644 src/utils/getRepoUrl.tsx create mode 100644 src/utils/urlDecoder.tsx diff --git a/api/data_pipeline.py b/api/data_pipeline.py index 4be6a31d..7298acca 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -13,6 +13,7 @@ from adalflow.core.db import LocalDB from api.config import configs from api.ollama_patch import OllamaDocumentProcessor +from urllib.parse import urlparse, quote # Configure logging logger = logging.getLogger(__name__) @@ -44,7 +45,7 @@ def count_tokens(text: str, local_ollama: bool = False) -> int: # Rough approximation: 4 characters per token return len(text) // 4 -def download_repo(repo_url: str, local_path: str, access_token: str = None): +def download_repo(repo_url: str, local_path: str, type: str = "github", access_token: str = None) -> str: """ Downloads a Git repository (GitHub, GitLab, or Bitbucket) to a specified local path. @@ -79,13 +80,17 @@ def download_repo(repo_url: str, local_path: str, access_token: str = None): clone_url = repo_url if access_token: # Determine the repository type and format the URL accordingly - if "github.com" in repo_url: + if type == "github": # Format: https://{token}@github.com/owner/repo.git clone_url = repo_url.replace("https://", f"https://{access_token}@") - elif "gitlab.com" in repo_url: + elif type == "gitlab": # Format: https://oauth2:{token}@gitlab.com/owner/repo.git - clone_url = repo_url.replace("https://", f"https://oauth2:{access_token}@") - elif "bitbucket.org" in repo_url: + if(repo_url.startswith("https://")): + clone_url = repo_url.replace("https://", f"https://oauth2:{access_token}@") + else: + # Handle self-hosted GitLab URLs + clone_url = repo_url.replace("http://", f"http://oauth2:{access_token}@") + elif type == "bitbucket": # Format: https://{token}@bitbucket.org/owner/repo.git clone_url = repo_url.replace("https://", f"https://{access_token}@") logger.info("Using access token for authentication") @@ -370,46 +375,38 @@ def get_github_file_content(repo_url: str, file_path: str, access_token: str = N def get_gitlab_file_content(repo_url: str, file_path: str, access_token: str = None) -> str: """ - Retrieves the content of a file from a GitLab repository using the GitLab API. + Retrieves the content of a file from a GitLab repository (cloud or self-hosted). Args: - repo_url (str): The URL of the GitLab repository (e.g., "https://gitlab.com/username/repo") - file_path (str): The path to the file within the repository (e.g., "src/main.py") - access_token (str, optional): GitLab personal access token for private repositories + repo_url (str): The GitLab repo URL (e.g., "https://gitlab.com/username/repo" or "http://localhost/group/project") + file_path (str): File path within the repository (e.g., "src/main.py") + access_token (str, optional): GitLab personal access token Returns: - str: The content of the file as a string + str: File content Raises: - ValueError: If the file cannot be fetched or if the URL is not a valid GitLab URL + ValueError: If anything fails """ try: - # Extract owner and repo name from GitLab URL - if not (repo_url.startswith("https://gitlab.com/") or repo_url.startswith("http://gitlab.com/")): + # Parse and validate the URL + parsed_url = urlparse(repo_url) + if not parsed_url.scheme or not parsed_url.netloc: raise ValueError("Not a valid GitLab repository URL") - parts = repo_url.rstrip('/').split('/') - if len(parts) < 5: - raise ValueError("Invalid GitLab URL format") + gitlab_domain = f"{parsed_url.scheme}://{parsed_url.netloc}" + path_parts = parsed_url.path.strip("/").split("/") + if len(path_parts) < 2: + raise ValueError("Invalid GitLab URL format — expected something like https://gitlab.domain.com/group/project") - # For GitLab, the URL format can be: - # - https://gitlab.com/username/repo - # - https://gitlab.com/group/subgroup/repo - # We need to extract the project path with namespace + # Build project path and encode for API + project_path = "/".join(path_parts).replace(".git", "") + encoded_project_path = quote(project_path, safe='') - # Remove the domain part - path_parts = parts[3:] - # Join the remaining parts to get the project path with namespace - project_path = '/'.join(path_parts).replace(".git", "") - # URL encode the path for API use - encoded_project_path = project_path.replace('/', '%2F') + # Encode file path + encoded_file_path = quote(file_path, safe='') - # Use GitLab API to get file content - # The API endpoint for getting file content is: /api/v4/projects/{encoded_project_path}/repository/files/{encoded_file_path}/raw - encoded_file_path = file_path.replace('/', '%2F') - api_url = f"https://gitlab.com/api/v4/projects/{encoded_project_path}/repository/files/{encoded_file_path}/raw?ref=main" - - # Prepare curl command with authentication if token is provided + api_url = f"{gitlab_domain}/api/v4/projects/{encoded_project_path}/repository/files/{encoded_file_path}/raw?ref={default_branch}" curl_cmd = ["curl", "-s"] if access_token: curl_cmd.extend(["-H", f"PRIVATE-TOKEN: {access_token}"]) @@ -423,37 +420,14 @@ def get_gitlab_file_content(repo_url: str, file_path: str, access_token: str = N stderr=subprocess.PIPE, ) - # GitLab API returns the raw file content directly content = result.stdout.decode("utf-8") - # Check if we got an error response (GitLab returns JSON for errors) - if content.startswith('{') and '"message":' in content: + # Check for GitLab error response (JSON instead of raw file) + if content.startswith("{") and '"message":' in content: try: error_data = json.loads(content) if "message" in error_data: - # Try with 'master' branch if 'main' failed - api_url = f"https://gitlab.com/api/v4/projects/{encoded_project_path}/repository/files/{encoded_file_path}/raw?ref=master" - logger.info(f"Retrying with master branch: {api_url}") - - # Prepare curl command for retry - curl_cmd = ["curl", "-s"] - if access_token: - curl_cmd.extend(["-H", f"PRIVATE-TOKEN: {access_token}"]) - curl_cmd.append(api_url) - - result = subprocess.run( - curl_cmd, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - content = result.stdout.decode("utf-8") - - # Check again for error - if content.startswith('{') and '"message":' in content: - error_data = json.loads(content) - if "message" in error_data: - raise ValueError(f"GitLab API error: {error_data['message']}") + raise ValueError(f"GitLab API error: {error_data['message']}") except json.JSONDecodeError: # If it's not valid JSON, it's probably the file content pass @@ -532,7 +506,7 @@ def get_bitbucket_file_content(repo_url: str, file_path: str, access_token: str raise ValueError(f"Failed to get file content: {str(e)}") -def get_file_content(repo_url: str, file_path: str, access_token: str = None) -> str: +def get_file_content(repo_url: str, file_path: str, type: str = "github", access_token: str = None) -> str: """ Retrieves the content of a file from a Git repository (GitHub or GitLab). @@ -547,11 +521,11 @@ def get_file_content(repo_url: str, file_path: str, access_token: str = None) -> Raises: ValueError: If the file cannot be fetched or if the URL is not valid """ - if "github.com" in repo_url: + if type == "github": return get_github_file_content(repo_url, file_path, access_token) - elif "gitlab.com" in repo_url: + elif type == "gitlab": return get_gitlab_file_content(repo_url, file_path, access_token) - elif "bitbucket.org" in repo_url: + elif type == "bitbucket": return get_bitbucket_file_content(repo_url, file_path, access_token) else: raise ValueError("Unsupported repository URL. Only GitHub and GitLab are supported.") @@ -566,7 +540,7 @@ def __init__(self): self.repo_url_or_path = None self.repo_paths = None - def prepare_database(self, repo_url_or_path: str, access_token: str = None, local_ollama: bool = False, + def prepare_database(self, repo_url_or_path: str, type: str = "github", access_token: str = None, local_ollama: bool = False, excluded_dirs: List[str] = None, excluded_files: List[str] = None) -> List[Document]: """ Create a new database from the repository. @@ -582,7 +556,7 @@ def prepare_database(self, repo_url_or_path: str, access_token: str = None, loca List[Document]: List of Document objects """ self.reset_database() - self._create_repo(repo_url_or_path, access_token) + self._create_repo(repo_url_or_path, type, access_token) return self.prepare_db_index(local_ollama=local_ollama, excluded_dirs=excluded_dirs, excluded_files=excluded_files) def reset_database(self): @@ -593,7 +567,7 @@ def reset_database(self): self.repo_url_or_path = None self.repo_paths = None - def _create_repo(self, repo_url_or_path: str, access_token: str = None) -> None: + def _create_repo(self, repo_url_or_path: str, type: str = "github", access_token: str = None) -> None: """ Download and prepare all paths. Paths: @@ -613,14 +587,14 @@ def _create_repo(self, repo_url_or_path: str, access_token: str = None) -> None: # url if repo_url_or_path.startswith("https://") or repo_url_or_path.startswith("http://"): # Extract repo name based on the URL format - if "github.com" in repo_url_or_path: + if type == "github": # GitHub URL format: https://github.com/owner/repo repo_name = repo_url_or_path.split("/")[-1].replace(".git", "") - elif "gitlab.com" in repo_url_or_path: + elif type == "gitlab": # GitLab URL format: https://gitlab.com/owner/repo or https://gitlab.com/group/subgroup/repo # Use the last part of the URL as the repo name repo_name = repo_url_or_path.split("/")[-1].replace(".git", "") - elif "bitbucket.org" in repo_url_or_path: + elif type == "bitbucket": # Bitbucket URL format: https://bitbucket.org/owner/repo repo_name = repo_url_or_path.split("/")[-1].replace(".git", "") else: @@ -632,7 +606,7 @@ def _create_repo(self, repo_url_or_path: str, access_token: str = None) -> None: # Check if the repository directory already exists and is not empty if not (os.path.exists(save_repo_dir) and os.listdir(save_repo_dir)): # Only download if the repository doesn't exist or is empty - download_repo(repo_url_or_path, save_repo_dir, access_token) + download_repo(repo_url_or_path, save_repo_dir, type, access_token) else: logger.info(f"Repository already exists at {save_repo_dir}. Using existing repository.") else: # local path @@ -695,7 +669,7 @@ def prepare_db_index(self, local_ollama: bool = False, excluded_dirs: List[str] logger.info(f"Total transformed documents: {len(transformed_docs)}") return transformed_docs - def prepare_retriever(self, repo_url_or_path: str, access_token: str = None): + def prepare_retriever(self, repo_url_or_path: str, type: str = "github", access_token: str = None): """ Prepare the retriever for a repository. This is a compatibility method for the isolated API. @@ -707,4 +681,4 @@ def prepare_retriever(self, repo_url_or_path: str, access_token: str = None): Returns: List[Document]: List of Document objects """ - return self.prepare_database(repo_url_or_path, access_token) + return self.prepare_database(repo_url_or_path, type, access_token) diff --git a/api/rag.py b/api/rag.py index cbeef246..332fb2dd 100644 --- a/api/rag.py +++ b/api/rag.py @@ -287,7 +287,7 @@ def initialize_db_manager(self): self.db_manager = DatabaseManager() self.transformed_docs = [] - def prepare_retriever(self, repo_url_or_path: str, access_token: str = None, local_ollama: bool = False, + def prepare_retriever(self, repo_url_or_path: str, type: str = "github", access_token: str = None, local_ollama: bool = False, excluded_dirs: List[str] = None, excluded_files: List[str] = None): """ Prepare the retriever for a repository. @@ -304,6 +304,7 @@ def prepare_retriever(self, repo_url_or_path: str, access_token: str = None, loc self.repo_url_or_path = repo_url_or_path self.transformed_docs = self.db_manager.prepare_database( repo_url_or_path, + type, access_token, local_ollama=local_ollama, excluded_dirs=excluded_dirs, diff --git a/api/simple_chat.py b/api/simple_chat.py index 735b0521..159f1ccc 100644 --- a/api/simple_chat.py +++ b/api/simple_chat.py @@ -60,9 +60,8 @@ class ChatCompletionRequest(BaseModel): repo_url: str = Field(..., description="URL of the repository to query") messages: List[ChatMessage] = Field(..., description="List of chat messages") filePath: Optional[str] = Field(None, description="Optional path to a file in the repository to include in the prompt") - github_token: Optional[str] = Field(None, description="GitHub personal access token for private repositories") - gitlab_token: Optional[str] = Field(None, description="GitLab personal access token for private repositories") - bitbucket_token: Optional[str] = Field(None, description="Bitbucket personal access token for private repositories") + token: Optional[str] = Field(None, description="Personal access token for private repositories") + type: Optional[str] = Field("github", description="Type of repository (e.g., 'github', 'gitlab', 'bitbucket')") # model parameters provider: str = Field("google", description="Model provider (google, openai, openrouter, ollama)") @@ -91,18 +90,6 @@ async def chat_completions_stream(request: ChatCompletionRequest): try: request_rag = RAG(provider=request.provider, model=request.model) - # Determine which access token to use based on the repository URL - access_token = None - if "github.com" in request.repo_url and request.github_token: - access_token = request.github_token - logger.info("Using GitHub token for authentication") - elif "gitlab.com" in request.repo_url and request.gitlab_token: - access_token = request.gitlab_token - logger.info("Using GitLab token for authentication") - elif "bitbucket.org" in request.repo_url and request.bitbucket_token: - access_token = request.bitbucket_token - logger.info("Using Bitbucket token for authentication") - # Extract custom file filter parameters if provided excluded_dirs = None excluded_files = None @@ -113,7 +100,7 @@ async def chat_completions_stream(request: ChatCompletionRequest): excluded_files = [unquote(file_pattern) for file_pattern in request.excluded_files.split('\n') if file_pattern.strip()] logger.info(f"Using custom excluded files: {excluded_files}") - request_rag.prepare_retriever(request.repo_url, access_token, False, excluded_dirs, excluded_files) + request_rag.prepare_retriever(request.repo_url, request.type, request.token, False, excluded_dirs, excluded_files) logger.info(f"Retriever prepared for {request.repo_url}") except Exception as e: logger.error(f"Error preparing retriever: {str(e)}") @@ -233,11 +220,7 @@ async def chat_completions_stream(request: ChatCompletionRequest): repo_name = repo_url.split("/")[-1] if "/" in repo_url else repo_url # Determine repository type - repo_type = "GitHub" - if "gitlab.com" in repo_url: - repo_type = "GitLab" - elif "bitbucket.org" in repo_url: - repo_type = "Bitbucket" + repo_type = request.type # Get language information language_code = request.language or "en" @@ -396,16 +379,7 @@ async def chat_completions_stream(request: ChatCompletionRequest): file_content = "" if request.filePath: try: - # Determine which access token to use - access_token = None - if "github.com" in request.repo_url and request.github_token: - access_token = request.github_token - elif "gitlab.com" in request.repo_url and request.gitlab_token: - access_token = request.gitlab_token - elif "bitbucket.org" in request.repo_url and request.bitbucket_token: - access_token = request.bitbucket_token - - file_content = get_file_content(request.repo_url, request.filePath, access_token) + file_content = get_file_content(request.repo_url, request.filePath, request.type, request.token) logger.info(f"Successfully retrieved content for file: {request.filePath}") except Exception as e: logger.error(f"Error retrieving file content: {str(e)}") diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index fc3e16f5..16701423 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -10,23 +10,9 @@ import Markdown from '@/components/Markdown'; import Ask from '@/components/Ask'; import ModelSelectionModal from '@/components/ModelSelectionModal'; import { useLanguage } from '@/contexts/LanguageContext'; - -// Wiki Interfaces -interface WikiPage { - id: string; - title: string; - content: string; - filePaths: string[]; - importance: 'high' | 'medium' | 'low'; - relatedPages: string[]; -} - -interface WikiStructure { - id: string; - title: string; - description: string; - pages: WikiPage[]; -} +import { RepoInfo } from '@/types/repoinfo'; +import { extractUrlDomain, extractUrlPath } from '@/utils/urlDecoder'; +import getRepoUrl from '@/utils/getRepoUrl'; // Add CSS styles for wiki with Japanese aesthetic const wikiStyles = ` @@ -71,18 +57,6 @@ const wikiStyles = ` } `; -// Helper functions for token handling and API requests -const getRepoUrl = (owner: string, repo: string, repoType: string, localPath?: string): string => { - if (repoType === 'local' && localPath) { - return localPath; - } - return repoType === 'github' - ? `https://github.com/${owner}/${repo}` - : repoType === 'gitlab' - ? `https://gitlab.com/${owner}/${repo}` - : `https://bitbucket.org/${owner}/${repo}`; -}; - // Helper function to generate cache key for localStorage const getCacheKey = (owner: string, repo: string, repoType: string, language: string): string => { return `deepwiki_cache_${repoType}_${owner}_${repo}_${language}`; @@ -92,9 +66,7 @@ const getCacheKey = (owner: string, repo: string, repoType: string, language: st const addTokensToRequestBody = ( // eslint-disable-next-line @typescript-eslint/no-explicit-any requestBody: Record, - githubToken: string, - gitlabToken: string, - bitbucketToken: string, + token: string, repoType: string, provider: string = '', model: string = '', @@ -104,14 +76,8 @@ const addTokensToRequestBody = ( excludedDirs?: string, excludedFiles?: string, ): void => { - if (githubToken && repoType === 'github') { - requestBody.github_token = githubToken; - } - if (gitlabToken && repoType === 'gitlab') { - requestBody.gitlab_token = gitlabToken; - } - if (bitbucketToken && repoType === 'bitbucket') { - requestBody.bitbucket_token = bitbucketToken; + if (token !== '') { + requestBody.token = token; } // Add provider-based model selection parameters @@ -179,11 +145,10 @@ export default function RepoWikiPage() { const repo = params.repo as string; // Extract tokens from search params - const githubToken = searchParams.get('github_token') || ''; - const gitlabToken = searchParams.get('gitlab_token') || ''; - const bitbucketToken = searchParams.get('bitbucket_token') || ''; + const token = searchParams.get('token') || ''; const repoType = searchParams.get('type') || 'github'; const localPath = searchParams.get('local_path') ? decodeURIComponent(searchParams.get('local_path') || '') : undefined; + const repoUrl = searchParams.get('repo_url') ? decodeURIComponent(searchParams.get('repo_url') || '') : undefined; const providerParam = searchParams.get('provider') || ''; const modelParam = searchParams.get('model') || ''; const isCustomModelParam = searchParams.get('is_custom_model') === 'true'; @@ -194,12 +159,14 @@ export default function RepoWikiPage() { const { messages } = useLanguage(); // Initialize repo info - const repoInfo = useMemo(() => ({ + const repoInfo = useMemo(() => ({ owner, repo, type: repoType, - localPath - }), [owner, repo, repoType, localPath]); + token: token || null, + localPath: localPath || null, + repoUrl: repoUrl || null + }), [owner, repo, repoType, localPath, repoUrl, token]); // State variables const [isLoading, setIsLoading] = useState(true); @@ -296,7 +263,7 @@ export default function RepoWikiPage() { console.log(`Starting content generation for page: ${page.title}`); // Get repository URL - const repoUrl = getRepoUrl(owner, repo, repoInfo.type, repoInfo.localPath); + const repoUrl = getRepoUrl(repoInfo); // Create the prompt content - simplified to avoid message dialogs const promptContent = @@ -363,6 +330,7 @@ Use proper markdown formatting for code blocks and include a vertical Mermaid di // eslint-disable-next-line @typescript-eslint/no-explicit-any const requestBody: Record = { repo_url: repoUrl, + type: repoInfo.type, messages: [{ role: 'user', content: promptContent @@ -370,7 +338,7 @@ Use proper markdown formatting for code blocks and include a vertical Mermaid di }; // Add tokens if available - addTokensToRequestBody(requestBody, githubToken, gitlabToken, bitbucketToken, repoInfo.type, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, language, modelExcludedDirs, modelExcludedFiles); + addTokensToRequestBody(requestBody, token, repoInfo.type, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, language, modelExcludedDirs, modelExcludedFiles); const response = await fetch(`/api/chat/stream`, { method: 'POST', @@ -445,7 +413,7 @@ Use proper markdown formatting for code blocks and include a vertical Mermaid di setLoadingMessage(undefined); // Clear specific loading message } }); - }, [generatedPages, githubToken, gitlabToken, bitbucketToken, repoInfo.type, repoInfo.localPath, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, activeContentRequests]); + }, [generatedPages, token, repoInfo.type, repoInfo.localPath, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, activeContentRequests]); // Determine the wiki structure from repository data const determineWikiStructure = useCallback(async (fileTree: string, readme: string, owner: string, repo: string) => { @@ -466,12 +434,13 @@ Use proper markdown formatting for code blocks and include a vertical Mermaid di setLoadingMessage(messages.loading?.determiningStructure || 'Determining wiki structure...'); // Get repository URL - const repoUrl = getRepoUrl(owner, repo, repoInfo.type, repoInfo.localPath); + const repoUrl = getRepoUrl(repoInfo); // Prepare request body // eslint-disable-next-line @typescript-eslint/no-explicit-any const requestBody: Record = { repo_url: repoUrl, + type: repoInfo.type, messages: [{ role: 'user', content: `Analyze this GitHub repository ${owner}/${repo} and create a wiki structure for it. @@ -542,7 +511,7 @@ IMPORTANT: }; // Add tokens if available - addTokensToRequestBody(requestBody, githubToken, gitlabToken, bitbucketToken, repoInfo.type, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, language, modelExcludedDirs, modelExcludedFiles); + addTokensToRequestBody(requestBody, token, repoInfo.type, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, language, modelExcludedDirs, modelExcludedFiles); const response = await fetch(`/api/chat/stream`, { method: 'POST', @@ -738,7 +707,7 @@ IMPORTANT: } finally { setStructureRequestInProgress(false); } - }, [generatePageContent, githubToken, gitlabToken, bitbucketToken, repoInfo.type, repoInfo.localPath, pagesInProgress.size, structureRequestInProgress, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, messages.loading]); + }, [generatePageContent, token, repoInfo.type, repoInfo.localPath, pagesInProgress.size, structureRequestInProgress, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, messages.loading]); // Fetch repository structure using GitHub or GitLab API const fetchRepositoryStructure = useCallback(async () => { @@ -789,7 +758,7 @@ IMPORTANT: for (const branch of ['main', 'master']) { const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`; - const headers = createGithubHeaders(githubToken); + const headers = createGithubHeaders(token); console.log(`Fetching repository structure from branch: ${branch}`); try { @@ -827,7 +796,7 @@ IMPORTANT: // Try to fetch README.md content try { - const headers = createGithubHeaders(githubToken); + const headers = createGithubHeaders(token); const readmeResponse = await fetch(`https://api.github.com/repos/${owner}/${repo}/readme`, { headers @@ -845,100 +814,84 @@ IMPORTANT: } else if (repoInfo.type === 'gitlab') { // GitLab API approach - const projectPath = `${owner}/${repo}`; + const projectPath = extractUrlPath(repoInfo.repoUrl ?? '') ?? `${owner}/${repo}`; + const projectDomain = extractUrlDomain(repoInfo.repoUrl ?? "https://gitlab.com"); const encodedProjectPath = encodeURIComponent(projectPath); - // Try to get the file tree for common branch names - let filesData = null; - let apiErrorDetails = ''; - let defaultBranch = ''; - const headers = createGitlabHeaders(gitlabToken); + const headers = createGitlabHeaders(token); + + let filesData: any[] = []; - // First get project info to determine default branch - const projectInfoUrl = `https://gitlab.com/api/v4/projects/${encodedProjectPath}`; - console.log(`Fetching GitLab project info: ${projectInfoUrl}`); try { - const response = await fetch(projectInfoUrl, { headers }); + // Step 1: Get project info to determine default branch + const projectInfoUrl = `${projectDomain}/api/v4/projects/${encodedProjectPath}`; + console.log(`Fetching GitLab project info: ${projectInfoUrl}`); + const projectInfoRes = await fetch(projectInfoUrl, { headers }); - if (response.ok) { - const projectData = await response.json(); - defaultBranch = projectData.default_branch; + if (!projectInfoRes.ok) { + const errorData = await projectInfoRes.text(); + throw new Error(`GitLab project info error: Status ${projectInfoRes.status}, Response: ${errorData}`); + } - const apiUrl = `https://gitlab.com/api/v4/projects/${encodedProjectPath}/repository/tree?recursive=true&ref=${defaultBranch}&per_page=100`; - try { - const response = await fetch(apiUrl, { - headers - }); + // Step 2: Paginate to fetch full file tree + let page = 1; + let morePages = true; - if (response.ok) { - filesData = await response.json(); - } else { + while (morePages) { + const apiUrl = `${projectDomain}/api/v4/projects/${encodedProjectPath}/repository/tree?recursive=true&per_page=100&page=${page}`; + const response = await fetch(apiUrl, { headers }); + + if (!response.ok) { const errorData = await response.text(); - apiErrorDetails = `Status: ${response.status}, Response: ${errorData}`; - console.error(`Error fetching GitLab repository structure: ${apiErrorDetails}`); - } - } catch (err) { - console.error(`Network error fetching GitLab branch ${defaultBranch}:`, err); + throw new Error(`Error fetching GitLab repository structure (page ${page}): ${errorData}`); } - } else { - const errorData = await response.text(); - apiErrorDetails = `Status: ${response.status}, Response: ${errorData}`; - console.error(`Error fetching GitLab project info: ${apiErrorDetails}`); - } - } catch (err) { - console.error("Network error fetching GitLab project info:", err); + + const pageData = await response.json(); + filesData.push(...pageData); + + const nextPage = response.headers.get('x-next-page'); + morePages = !!nextPage; + page = nextPage ? parseInt(nextPage, 10) : page + 1; } - if (!filesData || !Array.isArray(filesData) || filesData.length === 0) { - if (apiErrorDetails) { - throw new Error(`Could not fetch repository structure. GitLab API Error: ${apiErrorDetails}`); - } else { - throw new Error('Could not fetch repository structure. Repository might not exist, be empty or private.'); - } + if (!Array.isArray(filesData) || filesData.length === 0) { + throw new Error('Could not fetch repository structure. Repository might be empty or inaccessible.'); } - // Convert files data to a string representation + // Step 3: Format file paths fileTreeData = filesData .filter((item: { type: string; path: string }) => item.type === 'blob') .map((item: { type: string; path: string }) => item.path) .join('\n'); - // Try to fetch README.md content - try { - for (const branch of ['main', 'master']) { - const readmeUrl = `https://gitlab.com/api/v4/projects/${encodedProjectPath}/repository/files/README.md/raw?ref=${branch}`; - const headers = createGitlabHeaders(gitlabToken); - + // Step 4: Try to fetch README.md content + const readmeUrl = `${projectDomain}/api/v4/projects/${encodedProjectPath}/repository/files/README.md/raw`; try { - const readmeResponse = await fetch(readmeUrl, { - headers - }); - + const readmeResponse = await fetch(readmeUrl, { headers }); if (readmeResponse.ok) { readmeContent = await readmeResponse.text(); console.log('Successfully fetched GitLab README.md'); - break; } else { - console.warn(`Could not fetch GitLab README.md for branch ${branch}, status: ${readmeResponse.status}`); + console.warn(`Could not fetch GitLab README.md status: ${readmeResponse.status}`); } } catch (err) { - console.warn(`Error fetching GitLab README.md for branch ${branch}:`, err); + console.warn(`Error fetching GitLab README.md:`, err); } - } } catch (err) { - console.warn('Could not fetch GitLab README.md, continuing with empty README', err); + console.error("Error during GitLab repository tree retrieval:", err); + throw err; } } else if (repoInfo.type === 'bitbucket') { // Bitbucket API approach - const repoPath = `${owner}/${repo}`; + const repoPath = extractUrlPath(repoInfo.repoUrl ?? '') ?? `${owner}/${repo}`; const encodedRepoPath = encodeURIComponent(repoPath); // Try to get the file tree for common branch names let filesData = null; let apiErrorDetails = ''; let defaultBranch = ''; - const headers = createBitbucketHeaders(bitbucketToken); + const headers = createBitbucketHeaders(token); // First get project info to determine default branch const projectInfoUrl = `https://api.bitbucket.org/2.0/repositories/${encodedRepoPath}`; @@ -992,7 +945,7 @@ IMPORTANT: // Try to fetch README.md content try { - const headers = createBitbucketHeaders(bitbucketToken); + const headers = createBitbucketHeaders(token); const readmeResponse = await fetch(`https://api.bitbucket.org/2.0/repositories/${encodedRepoPath}/src/${defaultBranch}/README.md`, { headers @@ -1020,7 +973,7 @@ IMPORTANT: // Reset the request in progress flag setRequestInProgress(false); } - }, [owner, repo, determineWikiStructure, githubToken, gitlabToken, bitbucketToken, repoInfo.type, repoInfo.localPath, requestInProgress, messages.loading]); + }, [owner, repo, determineWikiStructure, token, repoInfo.type, repoInfo.localPath, requestInProgress, messages.loading]); // Function to export wiki content const exportWiki = useCallback(async (format: 'markdown' | 'json') => { @@ -1045,7 +998,7 @@ IMPORTANT: }); // Get repository URL - const repoUrl = getRepoUrl(repoInfo.owner, repoInfo.repo, repoInfo.type, repoInfo.localPath); + const repoUrl = getRepoUrl(repoInfo); // Make API call to export wiki const response = await fetch(`/export/wiki`, { @@ -1055,6 +1008,7 @@ IMPORTANT: }, body: JSON.stringify({ repo_url: repoUrl, + type: repoInfo.type, pages: pagesToExport, format }) @@ -1414,12 +1368,7 @@ IMPORTANT: )} {isAskSectionVisible && ( = {}): string => { @@ -108,6 +109,8 @@ export default function Home() { // Handle Windows absolute paths (e.g., C:\path\to\folder) const windowsPathRegex = /^[a-zA-Z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*$/; + const customGitRegex = /^(?:https?:\/\/)?([^\/]+)\/(.+?)\/([^\/]+)(?:\.git)?\/?$/; + if (windowsPathRegex.test(input)) { type = 'local'; localPath = input; @@ -121,40 +124,23 @@ export default function Home() { repo = input.split('/').filter(Boolean).pop() || 'local-repo'; owner = 'local'; } - // Handle GitHub URL format - else if (input.startsWith('https://github.com/')) { - type = 'github'; - const parts = input.replace('https://github.com/', '').split('/'); - owner = parts[0] || ''; - repo = parts[1] || ''; - } - // Handle GitLab URL format - else if (input.startsWith('https://gitlab.com/')) { - type = 'gitlab'; - const parts = input.replace('https://gitlab.com/', '').split('/'); - - // GitLab can have nested groups, so the repo is the last part - // and the owner/group is everything before that + else if (customGitRegex.test(input)) { + type = 'web'; + fullPath = extractUrlPath(input)?.replace(/\.git$/, ''); + const parts = fullPath?.split('/') ?? []; if (parts.length >= 2) { repo = parts[parts.length - 1] || ''; - owner = parts[0] || ''; - - // For GitLab, we also need to keep track of the full path for API calls - fullPath = parts.join('/'); + owner = parts[parts.length - 2] || ''; } } - // Handle Bitbucket URL format - else if (input.startsWith('https://bitbucket.org/')) { - type = 'bitbucket'; - const parts = input.replace('https://bitbucket.org/', '').split('/'); - owner = parts[0] || ''; - repo = parts[1] || ''; - } - // Handle owner/repo format (assume GitHub by default) + // Unsupported URL formats else { - const parts = input.split('/'); - owner = parts[0] || ''; - repo = parts[1] || ''; + console.error('Unsupported URL format:', input); + return null; + } + + if (!owner || !repo) { + return null; } // Clean values @@ -166,11 +152,7 @@ export default function Home() { repo = repo.slice(0, -4); } - if (!owner || !repo) { - return null; - } - - return {owner, repo, type, fullPath, localPath}; + return { owner, repo, type, fullPath, localPath }; }; const handleFormSubmit = (e: React.FormEvent) => { @@ -193,24 +175,20 @@ export default function Home() { return; } - const {owner, repo, type, localPath} = parsedRepo; + const { owner, repo, type, localPath } = parsedRepo; // Store tokens in query params if they exist const params = new URLSearchParams(); if (accessToken) { - if (selectedPlatform === 'github') { - params.append('github_token', accessToken); - } else if (selectedPlatform === 'gitlab') { - params.append('gitlab_token', accessToken); - } else if (selectedPlatform === 'bitbucket') { - params.append('bitbucket_token', accessToken); - } + params.append('token', accessToken); } // Always include the type parameter - params.append('type', type); + params.append('type', (type == 'local' ? type : selectedPlatform) || 'github'); // Add local path if it exists if (localPath) { params.append('local_path', encodeURIComponent(localPath)); + } else { + params.append('repo_url', encodeURIComponent(repositoryInput)); } // Add model parameters params.append('provider', provider); @@ -244,7 +222,7 @@ export default function Home() { className="flex flex-col md:flex-row md:items-center md:justify-between gap-4 bg-[var(--card-bg)] rounded-lg shadow-custom border border-[var(--border-color)] p-4">
- +

{t('common.appName')}

@@ -252,7 +230,7 @@ export default function Home() {

{t('common.tagline')}

+ className="text-xs font-medium text-[var(--accent-primary)] hover:text-[var(--highlight)] hover:underline whitespace-nowrap"> {t('nav.wikiProjects')}
@@ -263,7 +241,7 @@ export default function Home() {
{/* Repository URL input and submit button */}
-
+
setAccessToken(e.target.value)} - placeholder={t('form.tokenPlaceholder', {platform: selectedPlatform})} + placeholder={t('form.tokenPlaceholder', { platform: selectedPlatform })} className="input-japanese block w-full px-3 py-2 rounded-md bg-transparent text-[var(--foreground)] focus:outline-none focus:border-[var(--accent-primary)] text-sm" />
+ fill="none" viewBox="0 0 24 24" stroke="currentColor"> + d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" /> {t('form.tokenSecurityNote')}
@@ -452,7 +427,7 @@ export default function Home() {
- +

{t('home.welcome')}

@@ -470,9 +445,9 @@ export default function Home() { className="w-full max-w-2xl mb-10 bg-[var(--accent-primary)]/5 border border-[var(--accent-primary)]/20 rounded-lg p-5">

+ stroke="currentColor"> + d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" /> {t('home.quickStart')}

@@ -502,10 +477,10 @@ export default function Home() { className="w-full max-w-2xl mb-8 bg-[var(--background)]/70 rounded-lg p-6 border border-[var(--border-color)]">
+ className="h-5 w-5 text-[var(--accent-primary)] flex-shrink-0 mt-0.5 sm:mt-0" fill="none" + viewBox="0 0 24 24" stroke="currentColor"> + d="M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z" />

{t('home.advancedVisualization')}

@@ -517,12 +492,12 @@ export default function Home() {

{t('home.flowDiagram')}

- +

{t('home.sequenceDiagram')}

- +
@@ -537,19 +512,19 @@ export default function Home() {
diff --git a/src/components/Ask.tsx b/src/components/Ask.tsx index f30c994d..54288c16 100644 --- a/src/components/Ask.tsx +++ b/src/components/Ask.tsx @@ -4,6 +4,8 @@ import React, { useState, useRef, useEffect } from 'react'; import {FaChevronLeft, FaChevronRight } from 'react-icons/fa'; import Markdown from './Markdown'; import { useLanguage } from '@/contexts/LanguageContext'; +import RepoInfo from '@/types/repoinfo'; +import getRepoUrl from '@/utils/getRepoUrl'; import ModelSelectionModal from './ModelSelectionModal'; interface Message { @@ -19,10 +21,7 @@ interface ResearchStage { } interface AskProps { - repoUrl: string; - githubToken?: string; - gitlabToken?: string; - bitbucketToken?: string; + repoInfo: RepoInfo; provider?: string; model?: string; isCustomModel?: boolean; @@ -31,10 +30,7 @@ interface AskProps { } const Ask: React.FC = ({ - repoUrl, - githubToken, - gitlabToken, - bitbucketToken, + repoInfo, provider = '', model = '', isCustomModel = false, @@ -101,27 +97,27 @@ const Ask: React.FC = ({ // Check for conclusion sections that don't indicate further research if ((content.includes('## Conclusion') || content.includes('## Summary')) && - !content.includes('I will now proceed to') && - !content.includes('Next Steps') && - !content.includes('next iteration')) { + !content.includes('I will now proceed to') && + !content.includes('Next Steps') && + !content.includes('next iteration')) { return true; } // Check for phrases that explicitly indicate completion if (content.includes('This concludes our research') || - content.includes('This completes our investigation') || - content.includes('This concludes the deep research process') || - content.includes('Key Findings and Implementation Details') || - content.includes('In conclusion,') || - (content.includes('Final') && content.includes('Conclusion'))) { + content.includes('This completes our investigation') || + content.includes('This concludes the deep research process') || + content.includes('Key Findings and Implementation Details') || + content.includes('In conclusion,') || + (content.includes('Final') && content.includes('Conclusion'))) { return true; } // Check for topic-specific completion indicators if (content.includes('Dockerfile') && - (content.includes('This Dockerfile') || content.includes('The Dockerfile')) && - !content.includes('Next Steps') && - !content.includes('In the next iteration')) { + (content.includes('This Dockerfile') || content.includes('The Dockerfile')) && + !content.includes('Next Steps') && + !content.includes('In the next iteration')) { return true; } @@ -229,7 +225,7 @@ const Ask: React.FC = ({ // Prepare the request body const requestBody: Record = { - repo_url: repoUrl, + repo_url: getRepoUrl(repoInfo), messages: newHistory, provider: selectedProvider, model: isCustomSelectedModel ? customSelectedModel : selectedModel, @@ -237,14 +233,8 @@ const Ask: React.FC = ({ }; // Add tokens if available - if (githubToken && repoUrl.includes('github.com')) { - requestBody.github_token = githubToken; - } - if (gitlabToken && repoUrl.includes('gitlab.com')) { - requestBody.gitlab_token = gitlabToken; - } - if (bitbucketToken && repoUrl.includes('bitbucket.org')) { - requestBody.bitbucket_token = bitbucketToken; + if (repoInfo?.token) { + requestBody.token = repoInfo.token; } // Make the API call @@ -349,7 +339,7 @@ const Ask: React.FC = ({ return () => clearTimeout(timer); } } - // eslint-disable-next-line react-hooks/exhaustive-deps + // eslint-disable-next-line react-hooks/exhaustive-deps }, [response, isLoading, deepResearch, researchComplete, researchIteration]); // Effect to update research stages when the response changes @@ -381,7 +371,7 @@ const Ask: React.FC = ({ } } - // eslint-disable-next-line react-hooks/exhaustive-deps + // eslint-disable-next-line react-hooks/exhaustive-deps }, [response, isLoading, deepResearch, researchIteration]); const handleSubmit = async (e: React.FormEvent) => { @@ -412,7 +402,7 @@ const Ask: React.FC = ({ // Prepare request body const requestBody: Record = { - repo_url: repoUrl, + repo_url: getRepoUrl(repoInfo), messages: newHistory, provider: selectedProvider, model: isCustomSelectedModel ? customSelectedModel : selectedModel, @@ -420,14 +410,8 @@ const Ask: React.FC = ({ }; // Add tokens if available - if (githubToken && repoUrl.includes('github.com')) { - requestBody.github_token = githubToken; - } - if (gitlabToken && repoUrl.includes('gitlab.com')) { - requestBody.gitlab_token = gitlabToken; - } - if (bitbucketToken && repoUrl.includes('bitbucket.org')) { - requestBody.bitbucket_token = bitbucketToken; + if (repoInfo?.token) { + requestBody.token = repoInfo.token; } const apiResponse = await fetch(`/api/chat/stream`, { @@ -649,8 +633,8 @@ const Ask: React.FC = ({ {deepResearch ? (researchIteration === 0 - ? "Planning research approach..." - : `Research iteration ${researchIteration} in progress...`) + ? "Planning research approach..." + : `Research iteration ${researchIteration} in progress...`) : "Thinking..."}
diff --git a/src/types/repoinfo.tsx b/src/types/repoinfo.tsx new file mode 100644 index 00000000..ec4f87e6 --- /dev/null +++ b/src/types/repoinfo.tsx @@ -0,0 +1,10 @@ +export interface RepoInfo { + owner: string; + repo: string; + type: string; + token: string | null; + localPath: string | null; + repoUrl: string | null; +} + +export default RepoInfo; \ No newline at end of file diff --git a/src/types/wiki/wikipage.tsx b/src/types/wiki/wikipage.tsx new file mode 100644 index 00000000..45eff1cd --- /dev/null +++ b/src/types/wiki/wikipage.tsx @@ -0,0 +1,13 @@ +// Wiki Interfaces +interface WikiPage { + id: string; + title: string; + content: string; + filePaths: string[]; + importance: 'high' | 'medium' | 'low'; + relatedPages: string[]; + // New fields for hierarchy + parentId?: string; + isSection?: boolean; + children?: string[]; // IDs of child pages +} \ No newline at end of file diff --git a/src/types/wiki/wikistructure.tsx b/src/types/wiki/wikistructure.tsx new file mode 100644 index 00000000..5125971e --- /dev/null +++ b/src/types/wiki/wikistructure.tsx @@ -0,0 +1,9 @@ +/** + * @fileoverview This file defines the structure of a wiki page and its sections. + */ +interface WikiStructure { + id: string; + title: string; + description: string; + pages: WikiPage[]; +} \ No newline at end of file diff --git a/src/utils/getRepoUrl.tsx b/src/utils/getRepoUrl.tsx new file mode 100644 index 00000000..433814a1 --- /dev/null +++ b/src/utils/getRepoUrl.tsx @@ -0,0 +1,10 @@ +import RepoInfo from "@/types/repoinfo"; + +export default function getRepoUrl(repoInfo: RepoInfo): string { + console.log('getRepoUrl', repoInfo); + if (repoInfo.type === 'local' && repoInfo.localPath) { + return repoInfo.localPath; + } else { + return repoInfo.repoUrl || ''; + } +}; \ No newline at end of file diff --git a/src/utils/urlDecoder.tsx b/src/utils/urlDecoder.tsx new file mode 100644 index 00000000..c9074a6b --- /dev/null +++ b/src/utils/urlDecoder.tsx @@ -0,0 +1,21 @@ +export function extractUrlDomain(input: string): string | null { + try { + const normalizedInput = input.startsWith('http') ? input : `https://${input}`; + const url = new URL(normalizedInput); + return `${url.protocol}//${url.hostname}`; // Inclut le protocole et le domaine + } catch { + return null; // Not a valid URL + } +} + +export function extractUrlPath(input: string): string | null { + try { + const normalizedInput = input.startsWith('http') ? input : `https://${input}`; + const url = new URL(normalizedInput); + return url.pathname.replace(/^\/|\/$/g, ''); // Remove leading and trailing slashes + } catch { + return null; // Not a valid URL + } +} + +export default { extractUrlDomain, extractUrlPath }; \ No newline at end of file From 5de45bfbd7e53f05530b9ae04f602c6b6b3571d2 Mon Sep 17 00:00:00 2001 From: Axel Schneider Date: Sun, 11 May 2025 03:25:45 +0200 Subject: [PATCH 2/5] fix review from gemini-code-assist --- api/data_pipeline.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/api/data_pipeline.py b/api/data_pipeline.py index 7298acca..fc61a928 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -13,7 +13,7 @@ from adalflow.core.db import LocalDB from api.config import configs from api.ollama_patch import OllamaDocumentProcessor -from urllib.parse import urlparse, quote +from urllib.parse import urlparse, urlunparse, quote # Configure logging logger = logging.getLogger(__name__) @@ -79,20 +79,17 @@ def download_repo(repo_url: str, local_path: str, type: str = "github", access_t # Prepare the clone URL with access token if provided clone_url = repo_url if access_token: + parsed = urlparse(repo_url) # Determine the repository type and format the URL accordingly if type == "github": # Format: https://{token}@github.com/owner/repo.git - clone_url = repo_url.replace("https://", f"https://{access_token}@") + clone_url = urlunparse((parsed.scheme, f"{access_token}@", parsed.path, '', '', '')) elif type == "gitlab": - # Format: https://oauth2:{token}@gitlab.com/owner/repo.git - if(repo_url.startswith("https://")): - clone_url = repo_url.replace("https://", f"https://oauth2:{access_token}@") - else: - # Handle self-hosted GitLab URLs - clone_url = repo_url.replace("http://", f"http://oauth2:{access_token}@") + # Format: https://oauth2:{token}@gitlab.com/owner/repo.git + clone_url = urlunparse((parsed.scheme, f"oauth2:{access_token}@{parsed.netloc}", parsed.path, '', '', '')) elif type == "bitbucket": # Format: https://{token}@bitbucket.org/owner/repo.git - clone_url = repo_url.replace("https://", f"https://{access_token}@") + clone_url = urlunparse((parsed.scheme, f"{access_token}@", parsed.path, '', '', '')) logger.info("Using access token for authentication") # Clone the repository From b586cae84a18a30e3a1f1fb80f4918a9756a9058 Mon Sep 17 00:00:00 2001 From: Axel Schneider Date: Sun, 11 May 2025 03:44:33 +0200 Subject: [PATCH 3/5] Adding checks asking from review --- src/app/[owner]/[repo]/page.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 16701423..10107e26 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -824,8 +824,13 @@ IMPORTANT: try { // Step 1: Get project info to determine default branch - const projectInfoUrl = `${projectDomain}/api/v4/projects/${encodedProjectPath}`; - console.log(`Fetching GitLab project info: ${projectInfoUrl}`); + let projectInfoUrl: string; + try { + const validatedUrl = new URL(projectDomain ?? ''); // Validate domain + projectInfoUrl = `${validatedUrl.origin}/api/v4/projects/${encodedProjectPath}`; + } catch (err) { + throw new Error(`Invalid project domain URL: ${projectDomain}`); + } const projectInfoRes = await fetch(projectInfoUrl, { headers }); if (!projectInfoRes.ok) { @@ -838,7 +843,7 @@ IMPORTANT: let morePages = true; while (morePages) { - const apiUrl = `${projectDomain}/api/v4/projects/${encodedProjectPath}/repository/tree?recursive=true&per_page=100&page=${page}`; + const apiUrl = `${projectInfoUrl}/repository/tree?recursive=true&per_page=100&page=${page}`; const response = await fetch(apiUrl, { headers }); if (!response.ok) { @@ -865,7 +870,7 @@ IMPORTANT: .join('\n'); // Step 4: Try to fetch README.md content - const readmeUrl = `${projectDomain}/api/v4/projects/${encodedProjectPath}/repository/files/README.md/raw`; + const readmeUrl = `${projectInfoUrl}/repository/files/README.md/raw`; try { const readmeResponse = await fetch(readmeUrl, { headers }); if (readmeResponse.ok) { From 1e26979b0e074159ed6227d229a94ca3498a2ede Mon Sep 17 00:00:00 2001 From: Axel Schneider Date: Sun, 11 May 2025 17:01:16 +0200 Subject: [PATCH 4/5] Fixing eslint --- src/app/[owner]/[repo]/page.tsx | 11 +++++++---- src/types/wiki/wikipage.tsx | 2 +- src/types/wiki/wikistructure.tsx | 4 +++- src/utils/urlDecoder.tsx | 4 +--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 10107e26..1f200778 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -13,6 +13,8 @@ import { useLanguage } from '@/contexts/LanguageContext'; import { RepoInfo } from '@/types/repoinfo'; import { extractUrlDomain, extractUrlPath } from '@/utils/urlDecoder'; import getRepoUrl from '@/utils/getRepoUrl'; +import { WikiStructure } from '@/types/wiki/wikistructure'; +import { WikiPage } from '@/types/wiki/wikipage'; // Add CSS styles for wiki with Japanese aesthetic const wikiStyles = ` @@ -413,7 +415,7 @@ Use proper markdown formatting for code blocks and include a vertical Mermaid di setLoadingMessage(undefined); // Clear specific loading message } }); - }, [generatedPages, token, repoInfo.type, repoInfo.localPath, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, activeContentRequests]); + }, [generatedPages, token, repoInfo, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, activeContentRequests]); // Determine the wiki structure from repository data const determineWikiStructure = useCallback(async (fileTree: string, readme: string, owner: string, repo: string) => { @@ -707,7 +709,7 @@ IMPORTANT: } finally { setStructureRequestInProgress(false); } - }, [generatePageContent, token, repoInfo.type, repoInfo.localPath, pagesInProgress.size, structureRequestInProgress, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, messages.loading]); + }, [generatePageContent, token, repoInfo, pagesInProgress.size, structureRequestInProgress, selectedProviderState, selectedModelState, isCustomSelectedModelState, customSelectedModelState, modelExcludedDirs, modelExcludedFiles, language, messages.loading]); // Fetch repository structure using GitHub or GitLab API const fetchRepositoryStructure = useCallback(async () => { @@ -820,7 +822,8 @@ IMPORTANT: const headers = createGitlabHeaders(token); - let filesData: any[] = []; + /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ + const filesData: any[] = []; try { // Step 1: Get project info to determine default branch @@ -978,7 +981,7 @@ IMPORTANT: // Reset the request in progress flag setRequestInProgress(false); } - }, [owner, repo, determineWikiStructure, token, repoInfo.type, repoInfo.localPath, requestInProgress, messages.loading]); + }, [owner, repo, determineWikiStructure, token, repoInfo, requestInProgress, messages.loading]); // Function to export wiki content const exportWiki = useCallback(async (format: 'markdown' | 'json') => { diff --git a/src/types/wiki/wikipage.tsx b/src/types/wiki/wikipage.tsx index 45eff1cd..2621eedc 100644 --- a/src/types/wiki/wikipage.tsx +++ b/src/types/wiki/wikipage.tsx @@ -1,5 +1,5 @@ // Wiki Interfaces -interface WikiPage { +export interface WikiPage { id: string; title: string; content: string; diff --git a/src/types/wiki/wikistructure.tsx b/src/types/wiki/wikistructure.tsx index 5125971e..4bda2347 100644 --- a/src/types/wiki/wikistructure.tsx +++ b/src/types/wiki/wikistructure.tsx @@ -1,7 +1,9 @@ +import { WikiPage } from "./wikipage"; + /** * @fileoverview This file defines the structure of a wiki page and its sections. */ -interface WikiStructure { +export interface WikiStructure { id: string; title: string; description: string; diff --git a/src/utils/urlDecoder.tsx b/src/utils/urlDecoder.tsx index c9074a6b..0ec9248f 100644 --- a/src/utils/urlDecoder.tsx +++ b/src/utils/urlDecoder.tsx @@ -16,6 +16,4 @@ export function extractUrlPath(input: string): string | null { } catch { return null; // Not a valid URL } -} - -export default { extractUrlDomain, extractUrlPath }; \ No newline at end of file +} \ No newline at end of file From 64f493671bf9a02a314227c5427338052d86e15a Mon Sep 17 00:00:00 2001 From: Axel Schneider Date: Sun, 11 May 2025 23:35:09 +0200 Subject: [PATCH 5/5] Missing domain name with GitHub & Bitbucket --- api/data_pipeline.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/data_pipeline.py b/api/data_pipeline.py index fc61a928..b3195734 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -83,13 +83,13 @@ def download_repo(repo_url: str, local_path: str, type: str = "github", access_t # Determine the repository type and format the URL accordingly if type == "github": # Format: https://{token}@github.com/owner/repo.git - clone_url = urlunparse((parsed.scheme, f"{access_token}@", parsed.path, '', '', '')) + clone_url = urlunparse((parsed.scheme, f"{access_token}@{parsed.netloc}", parsed.path, '', '', '')) elif type == "gitlab": # Format: https://oauth2:{token}@gitlab.com/owner/repo.git clone_url = urlunparse((parsed.scheme, f"oauth2:{access_token}@{parsed.netloc}", parsed.path, '', '', '')) elif type == "bitbucket": # Format: https://{token}@bitbucket.org/owner/repo.git - clone_url = urlunparse((parsed.scheme, f"{access_token}@", parsed.path, '', '', '')) + clone_url = urlunparse((parsed.scheme, f"{access_token}@{parsed.netloc}", parsed.path, '', '', '')) logger.info("Using access token for authentication") # Clone the repository @@ -392,6 +392,8 @@ def get_gitlab_file_content(repo_url: str, file_path: str, access_token: str = N raise ValueError("Not a valid GitLab repository URL") gitlab_domain = f"{parsed_url.scheme}://{parsed_url.netloc}" + if parsed_url.port not in (None, 80, 443): + gitlab_domain += f":{parsed_url.port}" path_parts = parsed_url.path.strip("/").split("/") if len(path_parts) < 2: raise ValueError("Invalid GitLab URL format — expected something like https://gitlab.domain.com/group/project")