fix: introduce our own cors proxy for git import to fix 403 errors on isometric git cors proxy (#924)

* Exploration of improving git import

* Fix our own git proxy

* Clean out file counting for progress, does not seem to work well anyways

app/components/chat/GitCloneButton.tsx

@@ -3,6 +3,9 @@ import { useGit } from '~/lib/hooks/useGit';
 import type { Message } from 'ai';
 import { detectProjectCommands, createCommandsMessage } from '~/utils/projectCommands';
 import { generateId } from '~/utils/fileUtils';
+import { useState } from 'react';
+import { toast } from 'react-toastify';
+import { LoadingOverlay } from '~/components/ui/LoadingOverlay';
 
 const IGNORE_PATTERNS = [
   'node_modules/**',
@@ -37,6 +40,8 @@ interface GitCloneButtonProps {
 
 export default function GitCloneButton({ importChat }: GitCloneButtonProps) {
   const { ready, gitClone } = useGit();
+  const [loading, setLoading] = useState(false);
+
   const onClick = async (_e: any) => {
     if (!ready) {
       return;
@@ -45,33 +50,34 @@ export default function GitCloneButton({ importChat }: GitCloneButtonProps) {
     const repoUrl = prompt('Enter the Git url');
 
     if (repoUrl) {
-      const { workdir, data } = await gitClone(repoUrl);
-
-      if (importChat) {
-        const filePaths = Object.keys(data).filter((filePath) => !ig.ignores(filePath));
-        console.log(filePaths);
-
-        const textDecoder = new TextDecoder('utf-8');
-
-        // Convert files to common format for command detection
-        const fileContents = filePaths
-          .map((filePath) => {
-            const { data: content, encoding } = data[filePath];
-            return {
-              path: filePath,
-              content: encoding === 'utf8' ? content : content instanceof Uint8Array ? textDecoder.decode(content) : '',
-            };
-          })
-          .filter((f) => f.content);
-
-        // Detect and create commands message
-        const commands = await detectProjectCommands(fileContents);
-        const commandsMessage = createCommandsMessage(commands);
-
-        // Create files message
-        const filesMessage: Message = {
-          role: 'assistant',
-          content: `Cloning the repo ${repoUrl} into ${workdir}
+      setLoading(true);
+
+      try {
+        const { workdir, data } = await gitClone(repoUrl);
+
+        if (importChat) {
+          const filePaths = Object.keys(data).filter((filePath) => !ig.ignores(filePath));
+          console.log(filePaths);
+
+          const textDecoder = new TextDecoder('utf-8');
+
+          const fileContents = filePaths
+            .map((filePath) => {
+              const { data: content, encoding } = data[filePath];
+              return {
+                path: filePath,
+                content:
+                  encoding === 'utf8' ? content : content instanceof Uint8Array ? textDecoder.decode(content) : '',
+              };
+            })
+            .filter((f) => f.content);
+
+          const commands = await detectProjectCommands(fileContents);
+          const commandsMessage = createCommandsMessage(commands);
+
+          const filesMessage: Message = {
+            role: 'assistant',
+            content: `Cloning the repo ${repoUrl} into ${workdir}
 <boltArtifact id="imported-files" title="Git Cloned Files" type="bundled">
 ${fileContents
   .map(
@@ -82,29 +88,38 @@ ${file.content}
   )
   .join('\n')}
 </boltArtifact>`,
-          id: generateId(),
-          createdAt: new Date(),
-        };
+            id: generateId(),
+            createdAt: new Date(),
+          };
 
-        const messages = [filesMessage];
+          const messages = [filesMessage];
 
-        if (commandsMessage) {
-          messages.push(commandsMessage);
-        }
+          if (commandsMessage) {
+            messages.push(commandsMessage);
+          }
 
-        await importChat(`Git Project:${repoUrl.split('/').slice(-1)[0]}`, messages);
+          await importChat(`Git Project:${repoUrl.split('/').slice(-1)[0]}`, messages);
+        }
+      } catch (error) {
+        console.error('Error during import:', error);
+        toast.error('Failed to import repository');
+      } finally {
+        setLoading(false);
       }
     }
   };
 
   return (
-    <button
-      onClick={onClick}
-      title="Clone a Git Repo"
-      className="px-4 py-2 rounded-lg border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary hover:bg-bolt-elements-background-depth-3 transition-all flex items-center gap-2"
-    >
-      <span className="i-ph:git-branch" />
-      Clone a Git Repo
-    </button>
+    <>
+      <button
+        onClick={onClick}
+        title="Clone a Git Repo"
+        className="px-4 py-2 rounded-lg border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary hover:bg-bolt-elements-background-depth-3 transition-all flex items-center gap-2"
+      >
+        <span className="i-ph:git-branch" />
+        Clone a Git Repo
+      </button>
+      {loading && <LoadingOverlay message="Please wait while we clone the repository..." />}
+    </>
   );
 }

app/components/git/GitUrlImport.client.tsx

@@ -49,33 +49,32 @@ export function GitUrlImport() {
 
     if (repoUrl) {
       const ig = ignore().add(IGNORE_PATTERNS);
-      const { workdir, data } = await gitClone(repoUrl);
-
-      if (importChat) {
-        const filePaths = Object.keys(data).filter((filePath) => !ig.ignores(filePath));
-
-        const textDecoder = new TextDecoder('utf-8');
-
-        // Convert files to common format for command detection
-        const fileContents = filePaths
-          .map((filePath) => {
-            const { data: content, encoding } = data[filePath];
-            return {
-              path: filePath,
-              content: encoding === 'utf8' ? content : content instanceof Uint8Array ? textDecoder.decode(content) : '',
-            };
-          })
-          .filter((f) => f.content);
-
-        // Detect and create commands message
-        const commands = await detectProjectCommands(fileContents);
-        const commandsMessage = createCommandsMessage(commands);
-
-        // Create files message
-        const filesMessage: Message = {
-          role: 'assistant',
-          content: `Cloning the repo ${repoUrl} into ${workdir}
-<boltArtifact id="imported-files" title="Git Cloned Files" type="bundled">           
+
+      try {
+        const { workdir, data } = await gitClone(repoUrl);
+
+        if (importChat) {
+          const filePaths = Object.keys(data).filter((filePath) => !ig.ignores(filePath));
+          const textDecoder = new TextDecoder('utf-8');
+
+          const fileContents = filePaths
+            .map((filePath) => {
+              const { data: content, encoding } = data[filePath];
+              return {
+                path: filePath,
+                content:
+                  encoding === 'utf8' ? content : content instanceof Uint8Array ? textDecoder.decode(content) : '',
+              };
+            })
+            .filter((f) => f.content);
+
+          const commands = await detectProjectCommands(fileContents);
+          const commandsMessage = createCommandsMessage(commands);
+
+          const filesMessage: Message = {
+            role: 'assistant',
+            content: `Cloning the repo ${repoUrl} into ${workdir}
+<boltArtifact id="imported-files" title="Git Cloned Files" type="bundled">
 ${fileContents
   .map(
     (file) =>
@@ -85,17 +84,25 @@ ${file.content}
   )
   .join('\n')}
 </boltArtifact>`,
-          id: generateId(),
-          createdAt: new Date(),
-        };
+            id: generateId(),
+            createdAt: new Date(),
+          };
+
+          const messages = [filesMessage];
 
-        const messages = [filesMessage];
+          if (commandsMessage) {
+            messages.push(commandsMessage);
+          }
 
-        if (commandsMessage) {
-          messages.push(commandsMessage);
+          await importChat(`Git Project:${repoUrl.split('/').slice(-1)[0]}`, messages);
         }
+      } catch (error) {
+        console.error('Error during import:', error);
+        toast.error('Failed to import repository');
+        setLoading(false);
+        window.location.href = '/';
 
-        await importChat(`Git Project:${repoUrl.split('/').slice(-1)[0]}`, messages);
+        return;
       }
     }
   };

app/components/ui/LoadingOverlay.tsx

@@ -1,13 +1,31 @@
-export const LoadingOverlay = ({ message = 'Loading...' }) => {
+export const LoadingOverlay = ({
+  message = 'Loading...',
+  progress,
+  progressText,
+}: {
+  message?: string;
+  progress?: number;
+  progressText?: string;
+}) => {
   return (
     <div className="fixed inset-0 flex items-center justify-center bg-black/80 z-50 backdrop-blur-sm">
-      {/* Loading content */}
       <div className="relative flex flex-col items-center gap-4 p-8 rounded-lg bg-bolt-elements-background-depth-2 shadow-lg">
         <div
           className={'i-svg-spinners:90-ring-with-bg text-bolt-elements-loader-progress'}
           style={{ fontSize: '2rem' }}
         ></div>
         <p className="text-lg text-bolt-elements-textTertiary">{message}</p>
+        {progress !== undefined && (
+          <div className="w-64 flex flex-col gap-2">
+            <div className="w-full h-2 bg-bolt-elements-background-depth-1 rounded-full overflow-hidden">
+              <div
+                className="h-full bg-bolt-elements-loader-progress transition-all duration-300 ease-out rounded-full"
+                style={{ width: `${Math.min(100, Math.max(0, progress))}%` }}
+              />
+            </div>
+            {progressText && <p className="text-sm text-bolt-elements-textTertiary text-center">{progressText}</p>}
+          </div>
+        )}
       </div>
     </div>
   );

app/lib/hooks/useGit.ts

@@ -49,50 +49,54 @@ export function useGit() {
       }
 
       fileData.current = {};
-      await git.clone({
-        fs,
-        http,
-        dir: webcontainer.workdir,
-        url,
-        depth: 1,
-        singleBranch: true,
-        corsProxy: 'https://cors.isomorphic-git.org',
-        onAuth: (url) => {
-          // let domain=url.split("/")[2]
-
-          let auth = lookupSavedPassword(url);
-
-          if (auth) {
-            return auth;
-          }
-
-          if (confirm('This repo is password protected. Ready to enter a username & password?')) {
-            auth = {
-              username: prompt('Enter username'),
-              password: prompt('Enter password'),
-            };
-            return auth;
-          } else {
-            return { cancel: true };
-          }
-        },
-        onAuthFailure: (url, _auth) => {
-          toast.error(`Error Authenticating with ${url.split('/')[2]}`);
-        },
-        onAuthSuccess: (url, auth) => {
-          saveGitAuth(url, auth);
-        },
-      });
-
-      const data: Record<string, { data: any; encoding?: string }> = {};
-
-      for (const [key, value] of Object.entries(fileData.current)) {
-        data[key] = value;
-      }
 
-      return { workdir: webcontainer.workdir, data };
+      try {
+        await git.clone({
+          fs,
+          http,
+          dir: webcontainer.workdir,
+          url,
+          depth: 1,
+          singleBranch: true,
+          corsProxy: '/api/git-proxy',
+          onAuth: (url) => {
+            let auth = lookupSavedPassword(url);
+
+            if (auth) {
+              return auth;
+            }
+
+            if (confirm('This repo is password protected. Ready to enter a username & password?')) {
+              auth = {
+                username: prompt('Enter username'),
+                password: prompt('Enter password'),
+              };
+              return auth;
+            } else {
+              return { cancel: true };
+            }
+          },
+          onAuthFailure: (url, _auth) => {
+            toast.error(`Error Authenticating with ${url.split('/')[2]}`);
+          },
+          onAuthSuccess: (url, auth) => {
+            saveGitAuth(url, auth);
+          },
+        });
+
+        const data: Record<string, { data: any; encoding?: string }> = {};
+
+        for (const [key, value] of Object.entries(fileData.current)) {
+          data[key] = value;
+        }
+
+        return { workdir: webcontainer.workdir, data };
+      } catch (error) {
+        console.error('Git clone error:', error);
+        throw error;
+      }
     },
-    [webcontainer],
+    [webcontainer, fs, ready],
   );
 
   return { ready, gitClone };
@@ -104,55 +108,86 @@ const getFs = (
 ) => ({
   promises: {
     readFile: async (path: string, options: any) => {
-      const encoding = options.encoding;
+      const encoding = options?.encoding;
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      console.log('readFile', relativePath, encoding);
 
-      return await webcontainer.fs.readFile(relativePath, encoding);
+      try {
+        const result = await webcontainer.fs.readFile(relativePath, encoding);
+
+        return result;
+      } catch (error) {
+        throw error;
+      }
     },
     writeFile: async (path: string, data: any, options: any) => {
       const encoding = options.encoding;
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      console.log('writeFile', { relativePath, data, encoding });
 
       if (record.current) {
         record.current[relativePath] = { data, encoding };
       }
 
-      return await webcontainer.fs.writeFile(relativePath, data, { ...options, encoding });
+      try {
+        const result = await webcontainer.fs.writeFile(relativePath, data, { ...options, encoding });
+
+        return result;
+      } catch (error) {
+        throw error;
+      }
     },
     mkdir: async (path: string, options: any) => {
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      console.log('mkdir', relativePath, options);
 
-      return await webcontainer.fs.mkdir(relativePath, { ...options, recursive: true });
+      try {
+        const result = await webcontainer.fs.mkdir(relativePath, { ...options, recursive: true });
+
+        return result;
+      } catch (error) {
+        throw error;
+      }
     },
     readdir: async (path: string, options: any) => {
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      console.log('readdir', relativePath, options);
 
-      return await webcontainer.fs.readdir(relativePath, options);
+      try {
+        const result = await webcontainer.fs.readdir(relativePath, options);
+
+        return result;
+      } catch (error) {
+        throw error;
+      }
     },
     rm: async (path: string, options: any) => {
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      console.log('rm', relativePath, options);
 
-      return await webcontainer.fs.rm(relativePath, { ...(options || {}) });
+      try {
+        const result = await webcontainer.fs.rm(relativePath, { ...(options || {}) });
+
+        return result;
+      } catch (error) {
+        throw error;
+      }
     },
     rmdir: async (path: string, options: any) => {
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      console.log('rmdir', relativePath, options);
 
-      return await webcontainer.fs.rm(relativePath, { recursive: true, ...options });
-    },
+      try {
+        const result = await webcontainer.fs.rm(relativePath, { recursive: true, ...options });
 
-    // Mock implementations for missing functions
+        return result;
+      } catch (error) {
+        throw error;
+      }
+    },
     unlink: async (path: string) => {
-      // unlink is just removing a single file
       const relativePath = pathUtils.relative(webcontainer.workdir, path);
-      return await webcontainer.fs.rm(relativePath, { recursive: false });
-    },
 
+      try {
+        return await webcontainer.fs.rm(relativePath, { recursive: false });
+      } catch (error) {
+        throw error;
+      }
+    },
     stat: async (path: string) => {
       try {
         const relativePath = pathUtils.relative(webcontainer.workdir, path);
@@ -185,23 +220,12 @@ const getFs = (
         throw err;
       }
     },
-
     lstat: async (path: string) => {
-      /*
-       * For basic usage, lstat can return the same as stat
-       * since we're not handling symbolic links
-       */
       return await getFs(webcontainer, record).promises.stat(path);
     },
-
     readlink: async (path: string) => {
-      /*
-       * Since WebContainer doesn't support symlinks,
-       * we'll throw a "not a symbolic link" error
-       */
       throw new Error(`EINVAL: invalid argument, readlink '${path}'`);
     },
-
     symlink: async (target: string, path: string) => {
       /*
        * Since WebContainer doesn't support symlinks,

app/routes/api.git-proxy.$.ts

@@ -0,0 +1,65 @@
+import { json } from '@remix-run/cloudflare';
+import type { ActionFunctionArgs, LoaderFunctionArgs } from '@remix-run/cloudflare';
+
+// Handle all HTTP methods
+export async function action({ request, params }: ActionFunctionArgs) {
+  return handleProxyRequest(request, params['*']);
+}
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  return handleProxyRequest(request, params['*']);
+}
+
+async function handleProxyRequest(request: Request, path: string | undefined) {
+  try {
+    if (!path) {
+      return json({ error: 'Invalid proxy URL format' }, { status: 400 });
+    }
+
+    const url = new URL(request.url);
+
+    // Reconstruct the target URL
+    const targetURL = `https://${path}${url.search}`;
+
+    // Forward the request to the target URL
+    const response = await fetch(targetURL, {
+      method: request.method,
+      headers: {
+        ...Object.fromEntries(request.headers),
+
+        // Override host header with the target host
+        host: new URL(targetURL).host,
+      },
+      body: ['GET', 'HEAD'].includes(request.method) ? null : await request.arrayBuffer(),
+    });
+
+    // Create response with CORS headers
+    const corsHeaders = {
+      'Access-Control-Allow-Origin': '*',
+      'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
+      'Access-Control-Allow-Headers': '*',
+    };
+
+    // Handle preflight requests
+    if (request.method === 'OPTIONS') {
+      return new Response(null, {
+        headers: corsHeaders,
+        status: 204,
+      });
+    }
+
+    // Forward the response with CORS headers
+    const responseHeaders = new Headers(response.headers);
+    Object.entries(corsHeaders).forEach(([key, value]) => {
+      responseHeaders.set(key, value);
+    });
+
+    return new Response(response.body, {
+      status: response.status,
+      headers: responseHeaders,
+    });
+  } catch (error) {
+    console.error('Git proxy error:', error);
+    return json({ error: 'Proxy error' }, { status: 500 });
+  }
+}