v0.0.1

.gitignore

@@ -0,0 +1,5 @@
+*.pyd
+
+build/
+dist/
+*.egg-info/

LICENSE

@@ -0,0 +1,7 @@
+Copyright 2024 blaisewf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -1,13 +1,63 @@
 ---
-title: Repo2txt
-emoji: 📚
+title: repo2txt
+emoji: 🌐
 colorFrom: green
 colorTo: red
 sdk: gradio
 sdk_version: 4.39.0
 app_file: app.py
-pinned: false
+pinned: true
+header: mini
 license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# repo2txt
+
+`repo2txt` is a Python package that clones a GitHub repository, generates a text file containing the repository's directory structure and the contents of all its files, and handles cleanup.
+
+## Installation
+
+You can install `repo2txt` using pip:
+
+```sh
+pip install git+https://github.com/blaisewf/repo2txt.git
+```
+
+Alternatively, you can clone the repository and install it locally:
+
+```sh
+git clone https://github.com/blaisewf/repo2txt.git
+cd repo2txt
+pip install .
+```
+
+> [!WARNING]  
+> Git is required to clone the repository. If you don't have Git installed, you can download it from [git-scm.com](https://git-scm.com/).
+
+## Usage
+
+Once installed, you can use the CLI command `repo2txt` to process a GitHub repository. Here’s the basic syntax:
+
+```sh
+repo2txt --repo-url <repository_url> --output-file <output_file_path>
+```
+
+### Example
+
+```sh
+repo2txt --repo-url https://github.com/example/repository.git --output-file output.txt
+```
+
+This command will:
+
+1. Clone the repository from `https://github.com/example/repository.git`.
+2. Generate a text file `output.txt` containing the directory structure and contents of all files in the repository.
+3. Clean up the cloned repository directory.
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+## References
+- https://github.com/kirill-markin/repo-to-text
+

app.py

@@ -0,0 +1,65 @@
+import gradio as gr
+from repo2txt.decoder import (
+    clone_repo,
+    get_directory_structure,
+    extract_all_files_contents,
+    write_output_file,
+    cleanup,
+)
+
+
+def process_repository(repo_url_or_shorthand):
+    """Process the GitHub repository and return the content of the output file."""
+    # Define the directory to clone into
+    clone_dir = "temp_repo"
+    output_file = "output.txt"
+
+    try:
+        # Clone the repository
+        clone_repo(repo_url_or_shorthand, clone_dir)
+
+        # Get directory structure and file contents
+        directory_structure = get_directory_structure(clone_dir)
+        file_contents = extract_all_files_contents(clone_dir)
+
+        # Write output to file
+        write_output_file(output_file, directory_structure, file_contents)
+
+        # Read the content of the output file
+        with open(output_file, "r", encoding="utf-8") as file:
+            output_content = file.read()
+
+        # Cleanup
+        cleanup(clone_dir)
+        # Return the output file path for Gradio
+        return output_content, output_file
+
+    except Exception as e:
+        return f"An error occurred: {e}", None
+
+
+# Define Gradio interface
+with gr.Blocks(title="repo2txt") as demo:
+    gr.Markdown("# repo2txt")
+
+    with gr.Row():
+        repo_url_input = gr.Textbox(
+            label="GitHub Repository URL or Shorthand",
+            placeholder="e.g., user/repo or https://github.com/user/repo",
+        )
+
+    process_button = gr.Button("Process Repository")
+    txt_output = gr.File(label="Download txt file")
+    result_output = gr.Textbox(
+        label="Result",
+        lines=1,
+        placeholder="Processing result will be shown here",
+    )
+
+    process_button.click(
+        process_repository, inputs=repo_url_input, outputs=[result_output, txt_output]
+    )
+
+# Launch the interface
+if __name__ == "__main__":
+    demo.launch()

main.py

@@ -0,0 +1,131 @@
+import os
+import shutil
+import subprocess
+import argparse
+
+
+def clone_repo(repo_url, clone_dir):
+    """Clone the GitHub repository into the specified directory."""
+    subprocess.run(["git", "clone", repo_url, clone_dir], check=True)
+
+
+def extract_repo_name_from_url(repo_url):
+    """Extract the repository name from the GitHub URL."""
+    repo_name = repo_url.rstrip("/").split("/")[-1]
+    return repo_name.split(".")[0] if "." in repo_name else repo_name
+
+
+def get_directory_structure(root_dir):
+    """Get the directory structure in a tree format, ignoring .git directory."""
+    lines = []
+    for root, dirs, files in os.walk(root_dir):
+        if ".git" in dirs:
+            dirs.remove(".git")  # Avoid walking into .git directory
+
+        level = root.replace(root_dir, "").count(os.sep)
+        indent = " " * 4 * level
+        lines.append(f"{indent}├── {os.path.basename(root)}/")
+
+        subindent = " " * 4 * (level + 1)
+        for file in files:
+            lines.append(f"{subindent}├── {file}")
+    return "\n".join(lines)
+
+
+def read_file_contents(file_path):
+    """Read the contents of a file, ignore if in .git directory."""
+    if ".git" in file_path:
+        return "[Ignored .git directory]"
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as file:
+            return file.read()
+    except (UnicodeDecodeError, OSError) as e:
+        return f"[Error reading file: {e}]"
+
+
+def extract_all_files_contents(root_dir):
+    """Extract contents of all files in the directory, ignoring .git directory."""
+    file_contents = {}
+    for root, _, files in os.walk(root_dir):
+        if ".git" in root:
+            continue
+
+        for file_name in files:
+            file_path = os.path.join(root, file_name)
+            relative_path = os.path.relpath(file_path, root_dir)
+            file_contents[relative_path] = read_file_contents(file_path)
+    return file_contents
+
+
+def count_tokens(text):
+    """Count the number of tokens in a given text."""
+    return len(text.split())
+
+
+def write_output_file(output_file, directory_structure, file_contents):
+    """Write the directory structure and file contents to the output file with metadata."""
+    total_lines = directory_structure.count("\n") + sum(
+        content.count("\n") for content in file_contents.values()
+    )
+    total_chars = len(directory_structure) + sum(
+        len(content) for content in file_contents.values()
+    )
+
+    with open(output_file, "w", encoding="utf-8") as file:
+        file.write(f"Lines: {total_lines}\nCharacters: {total_chars}\n\n")
+        file.write("Directory Structure:\n```\n")
+        file.write(directory_structure)
+        file.write("\n```\n")
+
+        for file_path, content in file_contents.items():
+            file.write(f"\nContents of {file_path}:\n```\n")
+            file.write(content)
+            file.write("\n```\n")
+
+
+def cleanup(clone_dir):
+    """Remove the cloned repository directory with error handling."""
+    if os.path.exists(clone_dir):
+        try:
+            shutil.rmtree(clone_dir, onerror=handle_remove_error)
+        except Exception as e:
+            print(f"An error occurred while cleaning up: {e}")
+
+
+def handle_remove_error(func, path, exc_info):
+    """Error handler for shutil.rmtree to handle permission errors."""
+    import stat
+
+    if isinstance(exc_info[1], PermissionError):
+        os.chmod(path, stat.S_IWRITE)
+        func(path)
+    else:
+        print(f"Error removing {path}: {exc_info[1]}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate a text file with repository structure and all file contents."
+    )
+    parser.add_argument("repo_url", help="URL of the GitHub repository to process.")
+    parser.add_argument("output_file", help="Path to the output text file.")
+    args = parser.parse_args()
+
+    repo_url = args.repo_url
+    output_file = args.output_file
+
+    repo_name = extract_repo_name_from_url(repo_url)
+    clone_dir = repo_name
+
+    clone_repo(repo_url, clone_dir)
+
+    directory_structure = get_directory_structure(clone_dir)
+    file_contents = extract_all_files_contents(clone_dir)
+
+    write_output_file(output_file, directory_structure, file_contents)
+    cleanup(clone_dir)
+
+
+if __name__ == "__main__":
+    main()

repo2txt/cli.py

@@ -0,0 +1,45 @@
+import click
+from repo2txt.decoder import (
+    clone_repo,
+    extract_repo_name_from_url,
+    get_directory_structure,
+    extract_all_files_contents,
+    write_output_file,
+    cleanup,
+)
+
+
+@click.command()
+@click.option(
+    "--repo-url",
+    prompt="Repository URL",
+    help="URL of the GitHub repository to process.",
+)
+@click.option(
+    "--output-file", prompt="Output file path", help="Path to the output text file."
+)
+def cli(repo_url, output_file):
+    """CLI entry point for generating a text file with repository structure and all file contents."""
+    repo_name = extract_repo_name_from_url(repo_url)
+    clone_dir = repo_name
+
+    click.echo(f"Cloning repository {repo_url} into directory {clone_dir}...")
+    clone_repo(repo_url, clone_dir)
+
+    click.echo("Generating directory structure...")
+    directory_structure = get_directory_structure(clone_dir)
+
+    click.echo("Extracting file contents...")
+    file_contents = extract_all_files_contents(clone_dir)
+
+    click.echo(f"Writing output to {output_file}...")
+    write_output_file(output_file, directory_structure, file_contents)
+
+    click.echo("Cleaning up...")
+    cleanup(clone_dir)
+
+    click.echo("Process completed successfully.")
+
+
+if __name__ == "__main__":
+    cli()

repo2txt/decoder.py

@@ -0,0 +1,130 @@
+import os
+import shutil
+import subprocess
+
+
+def is_git_installed():
+    """Check if Git is installed."""
+    try:
+        subprocess.run(
+            ["git", "--version"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=True,
+        )
+        return True
+    except FileNotFoundError:
+        return False
+
+
+def convert_to_full_url(repo_url_or_shorthand):
+    """Convert shorthand 'user/repo' format to full URL format."""
+    if repo_url_or_shorthand.startswith("http"):
+        return repo_url_or_shorthand  # Already a full URL
+    return f"https://github.com/{repo_url_or_shorthand}"
+
+
+def clone_repo(repo_url_or_shorthand, clone_dir):
+    """Clone the GitHub repository into the specified directory."""
+    repo_url = convert_to_full_url(repo_url_or_shorthand)
+    if is_git_installed():
+        subprocess.run(["git", "clone", repo_url, clone_dir], check=True)
+    else:
+        raise RuntimeError(
+            "Git is not installed. Please install Git to clone repositories."
+        )
+
+
+def extract_repo_name_from_url(repo_url):
+    """Extract the repository name from the GitHub URL."""
+    repo_name = repo_url.rstrip("/").split("/")[-1]
+    return repo_name.split(".")[0] if "." in repo_name else repo_name
+
+
+def get_directory_structure(root_dir):
+    """Get the directory structure in a tree format, ignoring .git directory."""
+    lines = []
+    for root, dirs, files in os.walk(root_dir):
+        if ".git" in dirs:
+            dirs.remove(".git")  # Avoid walking into .git directory
+
+        level = root.replace(root_dir, "").count(os.sep)
+        indent = " " * 4 * level
+        lines.append(f"{indent}├── {os.path.basename(root)}/")
+
+        subindent = " " * 4 * (level + 1)
+        for file in files:
+            lines.append(f"{subindent}├── {file}")
+    return "\n".join(lines)
+
+
+def read_file_contents(file_path):
+    """Read the contents of a file, ignore if in .git directory."""
+    if ".git" in file_path:
+        return "[Ignored .git directory]"
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as file:
+            return file.read()
+    except (UnicodeDecodeError, OSError) as e:
+        return f"[Error reading file: {e}]"
+
+
+def extract_all_files_contents(root_dir):
+    """Extract contents of all files in the directory, ignoring .git directory."""
+    file_contents = {}
+    for root, _, files in os.walk(root_dir):
+        if ".git" in root:
+            continue
+
+        for file_name in files:
+            file_path = os.path.join(root, file_name)
+            relative_path = os.path.relpath(file_path, root_dir)
+            file_contents[relative_path] = read_file_contents(file_path)
+    return file_contents
+
+
+def count_tokens(text):
+    """Count the number of tokens in a given text."""
+    return len(text.split())
+
+
+def write_output_file(output_file, directory_structure, file_contents):
+    """Write the directory structure and file contents to the output file with metadata."""
+    total_lines = directory_structure.count("\n") + sum(
+        content.count("\n") for content in file_contents.values()
+    )
+    total_chars = len(directory_structure) + sum(
+        len(content) for content in file_contents.values()
+    )
+
+    with open(output_file, "w", encoding="utf-8") as file:
+        file.write(f"Lines: {total_lines}\nCharacters: {total_chars}\n\n")
+        file.write("Directory Structure:\n```\n")
+        file.write(directory_structure)
+        file.write("\n```\n")
+
+        for file_path, content in file_contents.items():
+            file.write(f"\nContents of {file_path}:\n```\n")
+            file.write(content)
+            file.write("\n```\n")
+
+
+def cleanup(clone_dir):
+    """Remove the cloned repository directory with error handling."""
+    if os.path.exists(clone_dir):
+        try:
+            shutil.rmtree(clone_dir, onerror=handle_remove_error)
+        except Exception as e:
+            print(f"An error occurred while cleaning up: {e}")
+
+
+def handle_remove_error(func, path, exc_info):
+    """Error handler for shutil.rmtree to handle permission errors."""
+    import stat
+
+    if isinstance(exc_info[1], PermissionError):
+        os.chmod(path, stat.S_IWRITE)
+        func(path)
+    else:
+        print(f"Error removing {path}: {exc_info[1]}")

requirements.txt

@@ -0,0 +1 @@
+gradio

setup.py

@@ -0,0 +1,47 @@
+from setuptools import setup, find_packages
+import codecs
+import os
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with codecs.open(os.path.join(here, "README.md"), encoding="utf-8") as fh:
+    long_description = "\n" + fh.read()
+
+setup(
+    name="repo2txt",
+    version="0.0.1",
+    author="Blaise",
+    author_email="[email protected]",
+    description="A tool to clone GitHub repositories, document their directory structure, and extract file contents into a text file.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    license="MIT",
+    packages=find_packages(),
+    install_requires=[
+        "click",
+    ],
+    entry_points={
+        "console_scripts": [
+            "repo2txt=repo2txt.cli:cli",
+        ],
+    },
+    url="https://github.com/blaise-tk/repo2txt",
+    keywords=[
+        "GitHub",
+        "repository",
+        "ai",
+        "clone",
+        "directory structure",
+        "file extraction",
+        "documentation",
+        "CLI tool",
+    ],
+    classifiers=[
+        "Development Status :: Release",
+        "Intended Audience :: Developers",
+        "Programming Language :: Python :: 3",
+        "Operating System :: MacOS :: MacOS X",
+        "Operating System :: Microsoft :: Windows :: Windows 10",
+        "Operating System :: POSIX :: Linux",
+    ],
+)