From 70a798b2d4c58e603e66508277adb511e1073247 Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 09:17:52 +0530 Subject: [PATCH 1/7] fixing no module named "gh_processor" & no module named "gh_explainer" --- Project-Explainer | 1 + README.md | 2 +- project_explainer/requirements.txt | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 160000 Project-Explainer create mode 100644 project_explainer/requirements.txt diff --git a/Project-Explainer b/Project-Explainer new file mode 160000 index 0000000..3af0213 --- /dev/null +++ b/Project-Explainer @@ -0,0 +1 @@ +Subproject commit 3af0213bf6f52afdb8e99a58fac061e87505c143 diff --git a/README.md b/README.md index c2884a7..e3a2044 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ A python module that is capable of providing different levels of summary for the #### Installation ``` -pip install git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer +pip install -r project_explainer/requirements.txt ``` #### Example usage diff --git a/project_explainer/requirements.txt b/project_explainer/requirements.txt new file mode 100644 index 0000000..7125520 --- /dev/null +++ b/project_explainer/requirements.txt @@ -0,0 +1,2 @@ +-e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer +-e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor \ No newline at end of file From c538d43292c7380d76fc30ddf1208272b5eb35ae Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 11:22:00 +0530 Subject: [PATCH 2/7] repeating file error --- Project-Explainer | 1 - project_explainer_ui/ui.py | 2 + project_processor/gh_processor/__init__.py | 39 +++++++++-------- project_processor/gh_processor/downloader.py | 43 +++++++++++++++++++ .../gh_processor/github_downloader.py | 23 +++++++--- 5 files changed, 81 insertions(+), 27 deletions(-) delete mode 160000 Project-Explainer create mode 100644 project_processor/gh_processor/downloader.py diff --git a/Project-Explainer b/Project-Explainer deleted file mode 160000 index 3af0213..0000000 --- a/Project-Explainer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3af0213bf6f52afdb8e99a58fac061e87505c143 diff --git a/project_explainer_ui/ui.py b/project_explainer_ui/ui.py index a836666..f41c4af 100644 --- a/project_explainer_ui/ui.py +++ b/project_explainer_ui/ui.py @@ -3,7 +3,9 @@ def summarize(summarization_type, github_project_url, github_project_branch="main", huggingface_model_id="gpt2"): gptExplainer = Explainer(huggingface_model_id) + print("hello.") if summarization_type == "brief": + print("hello..") return gptExplainer.brief(github_url=github_project_url, branch=github_project_branch)["summary"] return gptExplainer.outline(github_url=github_project_url, branch=github_project_branch)["summary"] diff --git a/project_processor/gh_processor/__init__.py b/project_processor/gh_processor/__init__.py index 65b83c7..49660df 100644 --- a/project_processor/gh_processor/__init__.py +++ b/project_processor/gh_processor/__init__.py @@ -1,20 +1,21 @@ -from .github_downloader import download_github_repo +# from .github_downloader import download_github_repo +from .downloader import download_github_repo -from .file_utils import (extract_code_blocks_from_markdown, - extract_headings_with_paragraphs_from_markdown, - extract_images_from_markdown, - extract_links_from_markdown, - extract_project_description_from_readme, - extract_tables_from_markdown, - get_files_by_extension, - get_elements_from_markdown_file, - remove_headings_from_markdown_file, - remove_sections_from_markdown, - convert_markdown_file_to_html, - convert_markdown_to_html, - check_phrase_similarity_using_spacyweb, - check_similarity, - remove_code_blocks_from_markdown, - remove_images_from_markdown, - remove_links_from_markdown, - remove_tables_from_markdown) +# from .file_utils import (extract_code_blocks_from_markdown, +# extract_headings_with_paragraphs_from_markdown, +# extract_images_from_markdown, +# extract_links_from_markdown, +# extract_project_description_from_readme, +# extract_tables_from_markdown, +# get_files_by_extension, +# get_elements_from_markdown_file, +# remove_headings_from_markdown_file, +# remove_sections_from_markdown, +# convert_markdown_file_to_html, +# convert_markdown_to_html, +# check_phrase_similarity_using_spacyweb, +# check_similarity, +# remove_code_blocks_from_markdown, +# remove_images_from_markdown, +# remove_links_from_markdown, +# remove_tables_from_markdown) diff --git a/project_processor/gh_processor/downloader.py b/project_processor/gh_processor/downloader.py new file mode 100644 index 0000000..0210573 --- /dev/null +++ b/project_processor/gh_processor/downloader.py @@ -0,0 +1,43 @@ +import logging +from git import Repo +import os + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +console_handler = logging.StreamHandler() +console_handler.setLevel(logging.INFO) + +formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) + + +def download_github_repo(repo_url: str, branch: str = "main") -> str: + """ + Download a GitHub repository from the provided URL. + + Args: + repo_url (str): The URL of the GitHub repository. + branch (str): The branch of the GitHub repository. + + Returns: + repo_path (str): Absolute path to downloaded repo + """ + # Check if the repository directory already exists + if os.path.exists(repo_path): + logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") + return repo_path + + try: + repo_name = repo_url.split("/")[-1].split(".")[0] + repo_path = os.path.abspath(repo_name) + + Repo.clone_from(repo_url, repo_name, branch=branch) + + logger.info(f"Repository '{repo_name}' downloaded successy!") + return repo_path + except Exception as e: + logger.error(f"Error whule donwloading repository: {str(e)}") + raise diff --git a/project_processor/gh_processor/github_downloader.py b/project_processor/gh_processor/github_downloader.py index 51ea2d7..053dceb 100644 --- a/project_processor/gh_processor/github_downloader.py +++ b/project_processor/gh_processor/github_downloader.py @@ -25,10 +25,19 @@ def download_github_repo(repo_url: str, branch: str = "main") -> str: Returns: repo_path (str): Absolute path to downloaded repo """ - repo_name = repo_url.split("/")[-1].split(".")[0] - repo_path = os.path.abspath(repo_name) - - Repo.clone_from(repo_url, repo_name, branch=branch) - - logger.info(f"Repository '{repo_name}' downloaded successfully!") - return repo_path + # Check if the repository directory already exists + if os.path.exists(repo_path): + logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") + return repo_path + + try: + repo_name = repo_url.split("/")[-1].split(".")[0] + repo_path = os.path.abspath(repo_name) + + Repo.clone_from(repo_url, repo_name, branch=branch) + + logger.info(f"Repository '{repo_name}' downloaded successfully!") + return repo_path + except Exception as e: + logger.error(f"Error whule donwloading repository: {str(e)}") + raise From 6dc14af99c3278d56c69cc86669a292c19089bf5 Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 12:22:30 +0530 Subject: [PATCH 3/7] init file change --- project_processor/gh_processor/__init__.py | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/project_processor/gh_processor/__init__.py b/project_processor/gh_processor/__init__.py index 49660df..4bc0d0b 100644 --- a/project_processor/gh_processor/__init__.py +++ b/project_processor/gh_processor/__init__.py @@ -1,21 +1,21 @@ # from .github_downloader import download_github_repo from .downloader import download_github_repo -# from .file_utils import (extract_code_blocks_from_markdown, -# extract_headings_with_paragraphs_from_markdown, -# extract_images_from_markdown, -# extract_links_from_markdown, -# extract_project_description_from_readme, -# extract_tables_from_markdown, -# get_files_by_extension, -# get_elements_from_markdown_file, -# remove_headings_from_markdown_file, -# remove_sections_from_markdown, -# convert_markdown_file_to_html, -# convert_markdown_to_html, -# check_phrase_similarity_using_spacyweb, -# check_similarity, -# remove_code_blocks_from_markdown, -# remove_images_from_markdown, -# remove_links_from_markdown, -# remove_tables_from_markdown) +from .file_utils import (extract_code_blocks_from_markdown, + extract_headings_with_paragraphs_from_markdown, + extract_images_from_markdown, + extract_links_from_markdown, + extract_project_description_from_readme, + extract_tables_from_markdown, + get_files_by_extension, + get_elements_from_markdown_file, + remove_headings_from_markdown_file, + remove_sections_from_markdown, + convert_markdown_file_to_html, + convert_markdown_to_html, + check_phrase_similarity_using_spacyweb, + check_similarity, + remove_code_blocks_from_markdown, + remove_images_from_markdown, + remove_links_from_markdown, + remove_tables_from_markdown) From 4f2d6c9e946d68d2005f6583da953ef0109fea3d Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 12:24:31 +0530 Subject: [PATCH 4/7] requirement file edit --- project_explainer/requirements.txt | 4 ++-- project_explainer_ui/requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project_explainer/requirements.txt b/project_explainer/requirements.txt index 7125520..18e58c1 100644 --- a/project_explainer/requirements.txt +++ b/project_explainer/requirements.txt @@ -1,2 +1,2 @@ --e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer --e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor \ No newline at end of file +-e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer +-e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor \ No newline at end of file diff --git a/project_explainer_ui/requirements.txt b/project_explainer_ui/requirements.txt index 44678c9..d8310ff 100644 --- a/project_explainer_ui/requirements.txt +++ b/project_explainer_ui/requirements.txt @@ -1,3 +1,3 @@ gradio --e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor --e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer \ No newline at end of file +-e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor +-e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer \ No newline at end of file From 68f18d5a9ef0b461f05a0384f297d1db8dbada94 Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 12:30:29 +0530 Subject: [PATCH 5/7] do changes --- Project-Explainer | 1 + project_processor/gh_processor/__init__.py | 3 +- project_processor/gh_processor/downloader.py | 43 ------------------- .../gh_processor/github_downloader.py | 17 +++++--- 4 files changed, 12 insertions(+), 52 deletions(-) create mode 160000 Project-Explainer delete mode 100644 project_processor/gh_processor/downloader.py diff --git a/Project-Explainer b/Project-Explainer new file mode 160000 index 0000000..3af0213 --- /dev/null +++ b/Project-Explainer @@ -0,0 +1 @@ +Subproject commit 3af0213bf6f52afdb8e99a58fac061e87505c143 diff --git a/project_processor/gh_processor/__init__.py b/project_processor/gh_processor/__init__.py index 4bc0d0b..65b83c7 100644 --- a/project_processor/gh_processor/__init__.py +++ b/project_processor/gh_processor/__init__.py @@ -1,5 +1,4 @@ -# from .github_downloader import download_github_repo -from .downloader import download_github_repo +from .github_downloader import download_github_repo from .file_utils import (extract_code_blocks_from_markdown, extract_headings_with_paragraphs_from_markdown, diff --git a/project_processor/gh_processor/downloader.py b/project_processor/gh_processor/downloader.py deleted file mode 100644 index 0210573..0000000 --- a/project_processor/gh_processor/downloader.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -from git import Repo -import os - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -console_handler = logging.StreamHandler() -console_handler.setLevel(logging.INFO) - -formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') -console_handler.setFormatter(formatter) - -logger.addHandler(console_handler) - - -def download_github_repo(repo_url: str, branch: str = "main") -> str: - """ - Download a GitHub repository from the provided URL. - - Args: - repo_url (str): The URL of the GitHub repository. - branch (str): The branch of the GitHub repository. - - Returns: - repo_path (str): Absolute path to downloaded repo - """ - # Check if the repository directory already exists - if os.path.exists(repo_path): - logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") - return repo_path - - try: - repo_name = repo_url.split("/")[-1].split(".")[0] - repo_path = os.path.abspath(repo_name) - - Repo.clone_from(repo_url, repo_name, branch=branch) - - logger.info(f"Repository '{repo_name}' downloaded successy!") - return repo_path - except Exception as e: - logger.error(f"Error whule donwloading repository: {str(e)}") - raise diff --git a/project_processor/gh_processor/github_downloader.py b/project_processor/gh_processor/github_downloader.py index 053dceb..d954231 100644 --- a/project_processor/gh_processor/github_downloader.py +++ b/project_processor/gh_processor/github_downloader.py @@ -25,19 +25,22 @@ def download_github_repo(repo_url: str, branch: str = "main") -> str: Returns: repo_path (str): Absolute path to downloaded repo """ - # Check if the repository directory already exists - if os.path.exists(repo_path): - logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") - return repo_path + try: repo_name = repo_url.split("/")[-1].split(".")[0] repo_path = os.path.abspath(repo_name) - Repo.clone_from(repo_url, repo_name, branch=branch) + # Check if the repository directory already exists + if os.path.exists(repo_path): + logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") + return repo_path + else: + + Repo.clone_from(repo_url, repo_name, branch=branch) - logger.info(f"Repository '{repo_name}' downloaded successfully!") - return repo_path + logger.info(f"Repository '{repo_name}' downloaded successfully!") + return repo_path except Exception as e: logger.error(f"Error whule donwloading repository: {str(e)}") raise From dbc037efeb307da56d424a887f3726a990cb20dc Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 12:46:17 +0530 Subject: [PATCH 6/7] do some changes to code --- Project-Explainer | 1 - project_explainer_ui/requirements.txt | 4 +-- .../gh_processor/github_downloader.py | 33 +++++++++---------- 3 files changed, 18 insertions(+), 20 deletions(-) delete mode 160000 Project-Explainer diff --git a/Project-Explainer b/Project-Explainer deleted file mode 160000 index 3af0213..0000000 --- a/Project-Explainer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3af0213bf6f52afdb8e99a58fac061e87505c143 diff --git a/project_explainer_ui/requirements.txt b/project_explainer_ui/requirements.txt index d8310ff..f2c852d 100644 --- a/project_explainer_ui/requirements.txt +++ b/project_explainer_ui/requirements.txt @@ -1,3 +1,3 @@ gradio --e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor --e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer \ No newline at end of file +# -e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor +# -e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer \ No newline at end of file diff --git a/project_processor/gh_processor/github_downloader.py b/project_processor/gh_processor/github_downloader.py index d954231..8e460a4 100644 --- a/project_processor/gh_processor/github_downloader.py +++ b/project_processor/gh_processor/github_downloader.py @@ -27,20 +27,19 @@ def download_github_repo(repo_url: str, branch: str = "main") -> str: """ - try: - repo_name = repo_url.split("/")[-1].split(".")[0] - repo_path = os.path.abspath(repo_name) - - # Check if the repository directory already exists - if os.path.exists(repo_path): - logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") - return repo_path - else: - - Repo.clone_from(repo_url, repo_name, branch=branch) - - logger.info(f"Repository '{repo_name}' downloaded successfully!") - return repo_path - except Exception as e: - logger.error(f"Error whule donwloading repository: {str(e)}") - raise + + repo_name = repo_url.split("/")[-1].split(".")[0] + repo_path = os.path.abspath(repo_name) + + # Check if the repository directory already exists + if os.path.exists(repo_path): + logger.info(f"Repository '{repo_name}' already exists at '{repo_path}'. Skipping cloning.") + return repo_path + + else: + + Repo.clone_from(repo_url, repo_name, branch=branch) + + logger.info(f"Repository '{repo_name}' downloaded successfully! and repo path {repo_path}") + return repo_path + From f9633f4b7ace636b4086db979a34224cb10a8fcf Mon Sep 17 00:00:00 2001 From: tharoosha Date: Sun, 10 Mar 2024 12:54:20 +0530 Subject: [PATCH 7/7] change before merging --- Project-Explainer | 1 + project_explainer/requirements.txt | 4 ++-- project_explainer_ui/requirements.txt | 4 ++-- project_explainer_ui/ui.py | 2 -- 4 files changed, 5 insertions(+), 6 deletions(-) create mode 160000 Project-Explainer diff --git a/Project-Explainer b/Project-Explainer new file mode 160000 index 0000000..3af0213 --- /dev/null +++ b/Project-Explainer @@ -0,0 +1 @@ +Subproject commit 3af0213bf6f52afdb8e99a58fac061e87505c143 diff --git a/project_explainer/requirements.txt b/project_explainer/requirements.txt index 18e58c1..7125520 100644 --- a/project_explainer/requirements.txt +++ b/project_explainer/requirements.txt @@ -1,2 +1,2 @@ --e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer --e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor \ No newline at end of file +-e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer +-e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor \ No newline at end of file diff --git a/project_explainer_ui/requirements.txt b/project_explainer_ui/requirements.txt index f2c852d..44678c9 100644 --- a/project_explainer_ui/requirements.txt +++ b/project_explainer_ui/requirements.txt @@ -1,3 +1,3 @@ gradio -# -e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor -# -e git+https://github.com/tharoosha/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer \ No newline at end of file +-e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_processor&egg=gh_processor +-e git+https://github.com/c2siorg/Project-Explainer.git@main#subdirectory=project_explainer&egg=gh_explainer&egg=gh_explainer \ No newline at end of file diff --git a/project_explainer_ui/ui.py b/project_explainer_ui/ui.py index f41c4af..a836666 100644 --- a/project_explainer_ui/ui.py +++ b/project_explainer_ui/ui.py @@ -3,9 +3,7 @@ def summarize(summarization_type, github_project_url, github_project_branch="main", huggingface_model_id="gpt2"): gptExplainer = Explainer(huggingface_model_id) - print("hello.") if summarization_type == "brief": - print("hello..") return gptExplainer.brief(github_url=github_project_url, branch=github_project_branch)["summary"] return gptExplainer.outline(github_url=github_project_url, branch=github_project_branch)["summary"]