diff --git a/.DS_Store b/.DS_Store index 823ebd1..a7ab7a8 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.env.template b/.env.template deleted file mode 100644 index db0cb2d..0000000 --- a/.env.template +++ /dev/null @@ -1,17 +0,0 @@ -# csv backend configuration -CSV_DATASET_FOLDER_PATH="xxx" - -# sql backend configuration -HOST="xxx" -PORT="xxx" -DBNAME="xxx" -USER="xxx" - -# openai keys -SEMANTIC_SCHOLAR_API_KEY="xxx" - -# local storage information (e.g., store latex/figures for arxiv tables) -PAPER_FOLDER_PATH="xxx" - -# local storage information (e.g., store pdfs for openreview tables) -PDF_FOLDER_PATH="xxx" \ No newline at end of file diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index f97abff..0000000 --- a/.gitattributes +++ /dev/null @@ -1,7 +0,0 @@ -*.pt filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.pdf filter=lfs diff=lfs merge=lfs -text -*.csv filter=lfs diff=lfs merge=lfs -text -*.jsonl filter=lfs diff=lfs merge=lfs -text -data/openreview/*.csv filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 36c9109..0000000 --- a/.gitignore +++ /dev/null @@ -1,181 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -data/ -# Django stuff: -*.log -*.py[cod] -*$py.class -.vscode -json/ -# C extensions -*.so -!./data/openreview/figures.csv -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST -openreview_log/ -*.bash -logs/ -csv/ -jsonl/ -*.jsonl -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Downloaded latex files -/download/ -download/ -# System output files storing the printed information when executing python files -system_output/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -#.idea/ -*.csv - -experiment_output/ -final_paragraph_generation_model/ -jingjun/ -data/ -json/ -paragraph_generation_models/ -paragraph_generation_models/* -token_embedding_cache/ -wandb/ -*.json \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 7bba29e..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-toml - - id: check-added-large-files - - id: check-json - - id: detect-private-key - - id: mixed-line-ending - - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.1 - hooks: - - id: prettier - types_or: [html, javascript, css, markdown, yaml, json] - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.0 - hooks: - - id: ruff - args: [--fix] - - id: ruff-format - - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 - hooks: - - id: mypy - additional_dependencies: - - pydantic>=2.8.2 - - types-setuptools - - types-pyyaml - - - repo: https://github.com/commitizen-tools/commitizen - rev: v3.14.1 - hooks: - - id: commitizen - stages: [commit-msg] diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a8c2003 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 2c2f97c..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Jingjun Xu - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md deleted file mode 100644 index e07c047..0000000 --- a/README.md +++ /dev/null @@ -1,193 +0,0 @@ -

🎡 ResearchArcade: Graph Interface for Academic Tasks

- -

- 🌐 Project Page | - 📜 arXiv -

- -

- -
- -# 🗞️ News - - - -# 🧭 Motivation - -Academic data is distributed across multiple platforms (e.g., ArXiv, OpenReview) and modalities (text, figures, tables, reviews). ResearchArcade unifies these heterogeneous data sources into a single graph-based interface to enable large-scale, structured, and temporal analysis of academic dataset. - -### Core Features - -- **Multi-Source**: ArXiv (Academic Corpora) & OpenReview (Peer Reviews and Manuscript Revisions) -- **Multi-Modal**: Figures and Tables in Academic Corpora -- **Highly Structural and Heterogeneous**: Data can be intuitively viewed as heterogeneous graphs with multi-table format -- **Dynamically Evolving**: Manuscript (Intra-paper) Level (e.g., Paper Revision) & Community (Inter-paper) Level (e.g., Paper Citation with Timestamp) - -### Data Illustration - -![data_description](./assets/research_arcade.png) - -Tables are classified into **node tables** (colored) or **edge tables** (black and white). The blue (denoting the OpenReview part) or red (denoting the ArXiv part) columns represent the unique identification of each node or edge, and the remaining columns represent the features of the nodes or edges. The conversion from the multiple tables to heterogeneous graphs is straightforward. - -# 🚀 Get started - -### Supported Features - -- **Dual Backend Support**: CSV backend & PostgreSQL backend -- **Comprehensive Data** - - **OpenReview**: Support for papers, authors, reviews, revisions, paragraphs, and their interconnections - - **ArXiv**: Support for papers, authors, paragraphs, sections, figures, tables and their interconnections -- **Flexible Data Import**: Load data from OpenReview API, Arxiv API, CSV files, or JSON files -- **Graph-like Operations**: Navigate relationships between entities - - **OpenReview**: authorship (paper-author), comment-under-paper (paper-review), revision-of-paper (paper-revision), revision-caused-by-review (revision-review), etc. - - **ArXiv**: citationship (paper-paper), authorship (paper-author), paragraph-of-paper (paper-paragraph), figure-of-paper (paper-figure), table-of-paper (paper-table), etc. -- **CRUD Operations**: Full support for Create, Read, Update, and Delete operations on all entities - -### ⚙️ Environment Requirements -- Python ≥ 3.9 (tested on 3.12) -- PostgreSQL ≥ 14 (for SQL backend) -- Conda ≥ 22.0 (recommended) -- API keys: - - Semantic Scholar API - - -```python -# create a new environment -conda create -n research_arcade python=3.12 -conda activate research_arcade - -# install related libraries -pip install -r requirements.txt -``` - -### Configure Environment Variables - -To run the code, you’ll need to set up environment variables such as your **Semantic Scholar API key** and database configurations. - -Copy the template file into the project root directory: -```bash -cp .env.template .env -``` - -### Backend Selection - -#### Initialize with CSV Backend - -```python -from research_arcade import ResearchArcade - -research_arcade = ResearchArcade( - db_type="csv", - config={"csv_dir": "/path/to/csv/data/"} -) -``` - -#### Initialize with SQL Backend - -```python -from research_arcade import ResearchArcade - -research_arcade = ResearchArcade( - db_type="sql", - config={ - "host": "localhost", - "dbname": "conference_db", - "user": "username", - "password": "password", - "port": "5432" - } -) -``` - -### Core Operations - -The following examples demonstrate the core operations available in ResearchArcade. For comprehensive examples covering all supported tables and operations, please refer to the `examples/tutorials.ipynb` file in the repository. - -#### Table Construction - -```python -# From API -config = {"venue": "ICLR.cc/2025/Conference"} -research_arcade.construct_table_from_api("openreview_papers", config) - -# From CSV file -config = {"csv_file": "/path/to/papers.csv"} -research_arcade.construct_table_from_csv("openreview_papers", config) - -# From JSON file -config = {"json_file": "/path/to/papers.json"} -research_arcade.construct_table_from_json("openreview_papers", config) -``` - -#### Query Operations - -```python -# Get all entities -papers_df = research_arcade.get_all_node_features("openreview_papers") - -# Get specific entity by ID -paper_id = {"paper_openreview_id": "zGej22CBnS"} -paper = research_arcade.get_node_features_by_id("openreview_papers", paper_id) - -# Get relationships -paper_authors = research_arcade.get_neighborhood("openreview_papers_authors", paper_id) -``` - -#### Node Manipulation - -```python -# Insert new node -new_author = { - 'venue': 'ICLR.cc/2025/Conference', - 'author_openreview_id': '~john_doe1', - 'author_full_name': 'John Doe', - 'email': 'john@university.edu', - 'affiliation': 'University Name' -} -research_arcade.insert_node("openreview_authors", node_features=new_author) - -# Update existing node -updated_paper = { - 'paper_openreview_id': 'paper123', - 'title': 'Updated Title', - # ... other fields -} -research_arcade.update_node("openreview_papers", node_features=updated_paper) - -# Delete a node -review_id = {"review_openreview_id": "review456"} -research_arcade.delete_node_by_id("openreview_reviews", review_id) -``` - -#### Edge Manipulation - -```python -# Create an edge -paper_author_edge = { - 'venue': 'ICLR.cc/2025/Conference', - 'paper_openreview_id': 'paper123', - 'author_openreview_id': '~john_doe1' -} -research_arcade.insert_edge("openreview_papers_authors", paper_author_edge) - -# Delete an edge -research_arcade.delete_edge_by_id("openreview_papers_authors", paper_author_edge) -``` - -# Contribution - -We’re working on extending support for data and operations. Contributions welcome! - -# Acknowledgements -This project builds on open academic infrastructures such as ArXiv and OpenReview. - -# License - -This project is licensed under the MIT License – see the [LICENSE](./LICENSE) file for details. - - -# Citation - diff --git a/assets/research_arcade.png b/assets/research_arcade.png deleted file mode 100644 index b768d14..0000000 Binary files a/assets/research_arcade.png and /dev/null differ diff --git a/convert.sh b/convert.sh deleted file mode 100644 index e69de29..0000000 diff --git a/docs/.DS_Store b/docs/.DS_Store index 2c61505..269477f 100644 Binary files a/docs/.DS_Store and b/docs/.DS_Store differ diff --git a/docs/index.html b/docs/index.html index 9a47c62..3c90bdf 100644 --- a/docs/index.html +++ b/docs/index.html @@ -82,7 +82,7 @@
Learn Tutorials - arXiv + arXiv Code
@@ -96,9 +96,11 @@

ResearchArcade: Graph Interface for Academic Tasks

+ Jingjun Xu + Chongshan Lin + Tao Feng + Haofei Yu Jiaxuan You -
@@ -216,8 +218,7 @@

3.2 Ablation Study

BibTeX

@inproceedings{researcharcade2025,
   title     = {ResearchArcade: Graph Interface for Academic Tasks},
-  author    = {You, Jiaxuan and others},
-  booktitle = {Proceedings of ...},
+  author    = {Xu, Jingjun and Lin, Chongshan and Feng, Tao and Yu, Haofei and You, Jiaxuan},
   year      = {2025},
   url       = {https://github.com/ulab-uiuc/research-arcade}
 }
@@ -268,4 +269,4 @@

BibTeX

})(); - + \ No newline at end of file diff --git a/docs/tutorials/continuous-crawling.html b/docs/tutorials/continuous-crawling.html new file mode 100644 index 0000000..5bf0b33 --- /dev/null +++ b/docs/tutorials/continuous-crawling.html @@ -0,0 +1,245 @@ + + + + + + Continuous Crawling • ResearchArcade + + + + + + + + + + + + + + + + + + + +
+
+
+ +

Continuous Crawling

+

Automatically fetch new arXiv papers at some regular interval to keep the dataset up-to-date.

+
+
+
+ + +
+
+
+ +
+ +

Introduction

+

+ ResearchArcade supports continuous crawling, which automatically fetches new papers from arXiv at some fixed intervals. This maintains an up-to-date dataset of research papers. +

+ +

Prerequisites

+

Before setting up continuous crawling, ensure you have:

+
    +
  • ResearchArcade installed and configured
    • +
  • A database backend (CSV or SQL) properly set up
    • +
  • Sufficient disk space for downloaded papers
    • +
  • Stable internet connection
    • +
+ +

Setup

+

First, initialize your ResearchArcade instance with your backend:

+ +

CSV Backend

+
from research_arcade import ResearchArcade
+
+db_type = "csv"
+config = {
+    "csv_dir": "./csv_data",
+}
+
+research_arcade = ResearchArcade(db_type=db_type, config=config)
+ +

SQL Backend

+
from research_arcade import ResearchArcade
+
+db_type = "sql"
+config = {
+    "host": "localhost",
+    "dbname": "DATABASE_NAME",
+    "user": "USER_NAME",
+    "password": "PASSWORD",
+    "port": "5432"
+}
+
+research_arcade = ResearchArcade(db_type=db_type, config=config)
+ +

Basic Usage

+

The continuous_crawling method starts an automated process that periodically fetches new papers from arXiv:

+
research_arcade.continuous_crawling(
+    interval_days=2,
+    delay_days=2,
+    paper_category='All',
+    dest_dir="./download",
+    arxiv_id_dest="./data"
+)
+ +

Parameters

+

The continuous_crawling method accepts the following parameters:

+ +

interval_days

+

+ Specifies how frequently the crawler should run, measured in days. For example, interval_days=2 means the crawler will check for new papers every 2 days. +

+ +

delay_days

+

+ Sets a delay before processing papers. This accounts for the fact that newly submitted arXiv papers may take some time to become fully available. A value of delay_days=2 means the crawler will only process papers that were submitted at least 2 days ago. +

+ +

paper_category

+

+ Filters papers by arXiv category. Set to 'All' to crawl all categories, or specify a particular category like 'cs.LG' (Machine Learning), 'cs.CL' (Computation and Language), or 'cs.AI' (Artificial Intelligence). +

+ +

dest_dir

+

+ The directory where downloaded PDF files will be stored. Ensure this directory has sufficient space and proper write permissions. +

+ +

arxiv_id_dest

+

+ The directory where arXiv ID tracking data will be stored. This helps the crawler keep track of which papers have already been processed. +

+ +

Example Configurations

+ +

Daily Machine Learning Papers

+
# Crawl ML papers daily
+research_arcade.continuous_crawling(
+    interval_days=1,
+    delay_days=1,
+    paper_category='cs.LG',
+    dest_dir="./ml_papers",
+    arxiv_id_dest="./ml_data"
+)
+ +

Weekly All CS Papers

+
# Crawl all CS papers weekly
+research_arcade.continuous_crawling(
+    interval_days=7,
+    delay_days=3,
+    paper_category='cs.*',
+    dest_dir="./cs_papers",
+    arxiv_id_dest="./cs_data"
+)
+ +

NLP-Focused Crawling

+
# Focus on NLP and AI papers
+research_arcade.continuous_crawling(
+    interval_days=2,
+    delay_days=2,
+    paper_category='cs.CL',
+    dest_dir="./nlp_papers",
+    arxiv_id_dest="./nlp_data"
+)
+ +

Best Practices

+
    +
  • Storage management: Regularly monitor disk space usage, especially when crawling all categories.
    • +
  • Rate limiting: The crawler respects arXiv's rate limits automatically, but avoid running multiple crawlers simultaneously.
    • +
  • Error handling: Check log files periodically for any failed downloads or parsing errors.
    • +
  • Backup: Regularly backup your arxiv_id_dest directory to avoid re-processing papers after a system failure.
    • +
  • Category selection: Start with specific categories to test your setup before expanding to broader crawls.
    • +
+ +

Next Steps

+ +
+ + +
+ +
+
+
+
+ + + + + + + + \ No newline at end of file diff --git a/docs/tutorials/crud-delete.html b/docs/tutorials/crud-delete.html index 6a645ef..620c058 100644 --- a/docs/tutorials/crud-delete.html +++ b/docs/tutorials/crud-delete.html @@ -114,12 +114,12 @@

Deleting Nodes

-

Deleting a ArXiv Papers Node

-

Example code for deleting a arxiv papers entry:

+

Deleting an ArXiv Papers Node

+

Example code for deleting an arxiv papers entry:

# Delete a paper by ID
-paper_id = {"arxiv_id": "1706.03762v7"}
-deleted_paper = research_arcade.delete_node_by_id("arxiv_papers", paper_id)
-print("Deleted paper:")
+paper_id = {"arxiv_id": "1706.03762v7"}
+deleted_paper = research_arcade.delete_node_by_id("arxiv_papers", paper_id)
+print("Deleted paper:")
 print(deleted_paper)
@@ -138,88 +138,88 @@

Deleting a ArXiv Papers Node

-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Sections Node

+

Example code for deleting an arxiv sections entry:

+ 
section_id = {"id": 544409}
+research_arcade.delete_node_by_id("arxiv_sections", section_id)
-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Paragraphs Node

+

Example code for deleting an arxiv paragraphs entry:

+ 
paragraph_id = {"id": 27428267}
+research_arcade.delete_node_by_id("arxiv_paragraphs", paragraph_id)
-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Figures Node

+

Example code for deleting an arxiv figures entry:

+ 
figure_id = {"id": 836691}
+research_arcade.delete_node_by_id("arxiv_figures", figure_id)
-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Tables Node

+

Example code for deleting an arxiv tables entry:

+ 
table_id = {"id": 299024}
+research_arcade.delete_node_by_id("arxiv_tables", table_id)
-

Deleting a OpenReview Authors Node

-

Example code for deleting a openreview authors entry:

- 
author_id = {"author_openreview_id": "~ishmam_zabir1"}
-author_features = research_arcade.delete_node_by_id("openreview_authors", author_id)
-print(author_features.to_dict(orient="records")[0])
+

Deleting an OpenReview Authors Node

+

Example code for deleting an openreview authors entry:

+ 
author_id = {"author_openreview_id": "~ishmam_zabir1"}
+author_features = research_arcade.delete_node_by_id("openreview_authors", author_id)
+print(author_features.to_dict(orient="records")[0])
-

Deleting a OpenReview Papers Node

-

Example code for deleting a openreview papers entry:

- 
paper_id = {"paper_openreview_id": "zGej22CBnS"}
-paper_features = research_arcade.delete_node_by_id("openreview_papers", paper_id)
-print(paper_features.to_dict(orient="records")[0])
+

Deleting an OpenReview Papers Node

+

Example code for deleting an openreview papers entry:

+ 
paper_id = {"paper_openreview_id": "zGej22CBnS"}
+paper_features = research_arcade.delete_node_by_id("openreview_papers", paper_id)
+print(paper_features.to_dict(orient="records")[0])
-

Deleting a OpenReview Reviews Node

-

Example code for deleting a openreview reviews entry:

- 
review_id = {"review_openreview_id": "DHwZxFryth"}
-review_features = research_arcade.delete_node_by_id("openreview_reviews", review_id)
-print(review_features.to_dict(orient="records")[0])
+

Deleting an OpenReview Reviews Node

+

Example code for deleting an openreview reviews entry:

+ 
review_id = {"review_openreview_id": "DHwZxFryth"}
+review_features = research_arcade.delete_node_by_id("openreview_reviews", review_id)
+print(review_features.to_dict(orient="records")[0])
-

Deleting a OpenReview Revisions Node

-

Example code for deleting a openreview revisions entry:

- 
revision_id = {"revision_openreview_id": "yfHQOp5zWc"}
-revision_feature = research_arcade.delete_node_by_id("openreview_revisions", revision_id)
-print(revision_feature.to_dict(orient="records")[0])
+

Deleting an OpenReview Revisions Node

+

Example code for deleting an openreview revisions entry:

+ 
revision_id = {"revision_openreview_id": "yfHQOp5zWc"}
+revision_feature = research_arcade.delete_node_by_id("openreview_revisions", revision_id)
+print(revision_feature.to_dict(orient="records")[0])
-

Deleting a OpenReview Paragraphs Node

-

Example code for deleting a openreview paragraphs entry:

- 
paper_id = {"paper_openreview_id": "xujj_test"}
-paragraph_feature = research_arcade.delete_node_by_id("openreview_paragraphs", paper_id)
+          
Deleting an OpenReview Paragraphs Node

+          
Example code for deleting an openreview paragraphs entry:

+          
paper_id = {"paper_openreview_id": "xujj_test"}
+paragraph_feature = research_arcade.delete_node_by_id("openreview_paragraphs", paper_id)
 print(len(paragraph_feature))
-print(paragraph_feature.to_dict(orient="records")[0])

+print(paragraph_feature.to_dict(orient="records")[0])
-

Deleting a OpenReview-ArXiv Link Node

-

Example code for deleting a openreview-arxiv link entry:

- 
openreview_id = {"paper_openreview_id": "zkNCWtw2fd"}
-openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_id)
-print(openreview_arxiv_df.to_dict(orient="records")[0])
-
-arxiv_id = {"arxiv_id": "http://arxiv.org/abs/2408.10536v1"}
-openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", arxiv_id)
-print(openreview_arxiv_df.to_dict(orient="records")[0])
-
-openreview_arxiv_id = {"paper_openreview_id": "zkNCWtw2fd", "arxiv_id": "http://arxiv.org/abs/2408.10536v1"}
-openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_arxiv_id)
-print(openreview_arxiv_df.to_dict(orient="records")[0])
+

Deleting an OpenReview-ArXiv Link Node

+

Example code for deleting an openreview-arxiv link entry:

+ 
openreview_id = {"paper_openreview_id": "zkNCWtw2fd"}
+openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_id)
+print(openreview_arxiv_df.to_dict(orient="records")[0])
+
+arxiv_id = {"arxiv_id": "http://arxiv.org/abs/2408.10536v1"}
+openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", arxiv_id)
+print(openreview_arxiv_df.to_dict(orient="records")[0])
+
+openreview_arxiv_id = {"paper_openreview_id": "zkNCWtw2fd", "arxiv_id": "http://arxiv.org/abs/2408.10536v1"}
+openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_arxiv_id)
+print(openreview_arxiv_df.to_dict(orient="records")[0])
@@ -256,87 +256,116 @@

Deleting Edges

Deleting an ArXiv Author-Paper Relationship

-

Example code for deleting a arxiv author-paper entry:

- 
relation_id = {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani'}
-research_arcade.delete_edge_by_id("arxiv_paper_author", primary_key=relation_id)
-print("Relationship deleted!")
+

Example code for deleting an arxiv author-paper entry:

+ 
relation_id = {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani'}
+research_arcade.delete_edge_by_id("arxiv_paper_author", primary_key=relation_id)
+print("Relationship deleted!")
-

Deleting a ArXiv Paper-Category Relationship

-

Example code for deleting a arxiv paper-category entry:

- 
relation_id = {'paper_arxiv_id': '1706.03762v7', 'category_id': 'cs.AI'}
-research_arcade.delete_edge_by_id("arxiv_paper_category", primary_key=relation_id)
-print("Relationship deleted!")
+

Deleting an ArXiv Paper-Category Relationship

+

Example code for deleting an arxiv paper-category entry:

+ 
relation_id = {'paper_arxiv_id': '1706.03762v7', 'category_id': 'cs.AI'}
+research_arcade.delete_edge_by_id("arxiv_paper_category", primary_key=relation_id)
+print("Relationship deleted!")
-

Deleting a ArXiv Citation

-

Example code for deleting a arxiv citation entry:

+

Deleting an ArXiv Citation

+

Example code for deleting an arxiv citation entry:

citation_id = {
-    'citing_paper_id': '1810.04805v2',
-    'cited_paper_id': '1706.03762v7'
+    'citing_paper_id': '1810.04805v2',
+    'cited_paper_id': '1706.03762v7'
 }
-research_arcade.delete_edge_by_id("arxiv_citation", primary_key=citation_id)
-print("Citation deleted!")
+research_arcade.delete_edge_by_id("arxiv_citation", primary_key=citation_id) +print("Citation deleted!")
-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Paper-Figures Relationship

+

Example code for deleting an arxiv paper-figures entry:

+ 
# Delete by paper_arxiv_id
+paper_id = {"paper_arxiv_id": "2507.13024"}
+result = research_arcade.delete_edge_by_id("arxiv_paper_figure", paper_id)
+
+# Delete by figure_id
+figure_id = {"figure_id": 1}
+result = research_arcade.delete_edge_by_id("arxiv_paper_figure", figure_id)
+
+# Delete by both ids
+paper_figure_id = {
+    "paper_arxiv_id": "2410.23123v2",
+    "figure_id": 476323
+}
+result = research_arcade.delete_edge_by_id("arxiv_paper_figure", paper_figure_id)
-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Paper-Tables Relationship

+

Example code for deleting an arxiv paper-tables entry:

+ 
# Delete by paper_arxiv_id
+paper_id = {"paper_arxiv_id": "1706.03762v7"}
+result = research_arcade.delete_edge_by_id("arxiv_paper_table", paper_id)
+
+# Delete by table_id
+table_id = {"table_id": 1}
+result = research_arcade.delete_edge_by_id("arxiv_paper_table", table_id)
+
+# Delete by both ids
+paper_table_id = {
+    "paper_arxiv_id": "1706.03762v7",
+    "table_id": 1
+}
+result = research_arcade.delete_edge_by_id("arxiv_paper_table", paper_table_id)
-

Deleting arxiv_paragraph_citations Edges

-

Code example placeholder - Add your Python/API code here for deleting arxiv_paragraph_citations edges

+

Deleting an ArXiv Paragraph-Citations Relationship

+

Example code for deleting an arxiv paragraph-citations entry:

+ 
paragraph_citation = {
+    'paragraph_id': 1
+}
+count = research_arcade.delete_edge_by_id('arxiv_paragraph_citation', paragraph_citation)
+print(f"Deleted {count} paragraph citations")
-
- -

No example code available for this operation in the tutorial.

-
+

Deleting an ArXiv Paragraph-References Relationship

+

Example code for deleting an arxiv paragraph-references entry:

+ 
paragraph_id = {"paragraph_id": 1}
+result = research_arcade.delete_edge_by_id("arxiv_paragraph_reference", paragraph_id)
-

Deleting a OpenReview Papers-Authors Relationship

-

Example code for deleting a openreview papers-authors entry:

- 
paper_id = {"paper_openreview_id": "00SnKBGTsz"}
-openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", paper_id)
-print(openreview_papers_authors.to_dict(orient="records"))
+

Deleting an OpenReview Papers-Authors Relationship

+

Example code for deleting an openreview papers-authors entry:

+ 
paper_id = {"paper_openreview_id": "00SnKBGTsz"}
+openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", paper_id)
+print(openreview_papers_authors.to_dict(orient="records"))
-

Deleting a OpenReview Papers-Reviews Relationship

-

Example code for deleting a openreview papers-reviews entry:

- 
paper_review_id = {"paper_openreview_id": "00SnKBGTsz", "review_openreview_id": "13mj0Rtn5W"}
-openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", paper_review_id)
-print(openreview_papers_reviews.to_dict(orient="records"))
+

Deleting an OpenReview Papers-Reviews Relationship

+

Example code for deleting an openreview papers-reviews entry:

+ 
paper_review_id = {"paper_openreview_id": "00SnKBGTsz", "review_openreview_id": "13mj0Rtn5W"}
+openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", paper_review_id)
+print(openreview_papers_reviews.to_dict(orient="records"))
-

Deleting a OpenReview Papers-Revisions Relationship

-

Example code for deleting a openreview papers-revisions entry:

- 
paper_revision_id = {"paper_openreview_id": "00SnKBGTsz", "revision_openreview_id": "Cn0twOEX1T"}
-openreview_papers_revisions = research_arcade.delete_edge_by_id("openreview_papers_revisions", paper_revision_id)
-print(openreview_papers_revisions.to_dict(orient="records"))
+

Deleting an OpenReview Papers-Revisions Relationship

+

Example code for deleting an openreview papers-revisions entry:

+ 
paper_revision_id = {"paper_openreview_id": "00SnKBGTsz", "revision_openreview_id": "Cn0twOEX1T"}
+openreview_papers_revisions = research_arcade.delete_edge_by_id("openreview_papers_revisions", paper_revision_id)
+print(openreview_papers_revisions.to_dict(orient="records"))
-

Deleting a OpenReview Revisions-Reviews Relationship

-

Example code for deleting a openreview revisions-reviews entry:

- 
revision_review_id = {"revision_openreview_id": "Cn0twOEX1T", "review_openreview_id": "13mj0Rtn5W"}
-openreview_revisions_reviews = research_arcade.delete_edge_by_id("openreview_revisions_reviews", revision_review_id)
-print(openreview_revisions_reviews.to_dict(orient="records"))
+

Deleting an OpenReview Revisions-Reviews Relationship

+

Example code for deleting an openreview revisions-reviews entry:

+ 
revision_review_id = {"revision_openreview_id": "Cn0twOEX1T", "review_openreview_id": "13mj0Rtn5W"}
+openreview_revisions_reviews = research_arcade.delete_edge_by_id("openreview_revisions_reviews", revision_review_id)
+print(openreview_revisions_reviews.to_dict(orient="records"))
diff --git a/docs/tutorials/crud-edit.html b/docs/tutorials/crud-edit.html index 4551a65..7606b63 100644 --- a/docs/tutorials/crud-edit.html +++ b/docs/tutorials/crud-edit.html @@ -162,28 +162,58 @@

Updating a ArXiv Authors Node

-

No example code available for this operation in the tutorial.

+

Example code for updating an existing arxiv sections entry:

+ 
updated_section = {
+                'id': 544409,
+                'title': 'Updated Section Title',
+                'content': 'Updated content'
+            }
+            research_arcade.update_node("arxiv_sections", node_features=updated_section)
+
-

No example code available for this operation in the tutorial.

+

Example code for updating an existing arxiv paragraph entry:

+ 
updated_paragraph = {
+                'id': 27428268,
+                'content': 'Updated paragraph content'
+            }
+            research_arcade.update_node("arxiv_paragraphs", node_features=updated_paragraph)
+
-

No example code available for this operation in the tutorial.

+

Example code for updating an existing arxiv figure entry:

+ 
updated_figure = {
+                'id': 836693,
+                'paper_arxiv_id': 1453.1644,
+                'path': 'path',
+                'caption': 'Updated figure caption'
+            }
+            research_arcade.update_node("arxiv_figures", node_features=updated_figure)
+
-

No example code available for this operation in the tutorial.

+

Example code for updating an existing arxiv table entry:

+ 
updated_table = {
+                'id': 299024,
+                'paper_arxiv_id': '1706.03762v7',
+                'caption': 'Performance comparison',
+                'label': 'label',
+                'table_text': 'Table content here'
+            }
+            research_arcade.update_node("arxiv_tables", node_features=updated_table)
+
diff --git a/examples/.DS_Store b/examples/.DS_Store deleted file mode 100644 index c1f3806..0000000 Binary files a/examples/.DS_Store and /dev/null differ diff --git a/examples/advanced_querying.ipynb b/examples/advanced_querying.ipynb deleted file mode 100644 index 89c2083..0000000 --- a/examples/advanced_querying.ipynb +++ /dev/null @@ -1,778 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "5d5f816e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked paper 1706.03762v7 to category 1\n", - "Linked paper 1706.03762v7 to category 2\n", - "Linked paper 1810.04805v2 to category 1\n", - "Linked paper 1810.04805v2 to category 3\n" - ] - } - ], - "source": [ - "# Link papers to categories\n", - "paper_categories = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'category_id': 1 # cs.CL\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'category_id': 2 # cs.LG\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1810.04805v2',\n", - " 'category_id': 1 # cs.CL\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1810.04805v2',\n", - " 'category_id': 3 # cs.AI\n", - " }\n", - "]\n", - "\n", - "for relation in paper_categories:\n", - " research_arcade.insert_edge(\"arxiv_paper_category\", edge_features=relation)\n", - " print(f\"Linked paper {relation['paper_arxiv_id']} to category {relation['category_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "fa72703f", - "metadata": {}, - "source": [ - "### Citation Relationships (arxiv_citations)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b00b3318", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table arxiv_citations not found.\n", - "Created citation: 1810.04805v2 → 1706.03762v7\n" - ] - } - ], - "source": [ - "# Create citation relationships (BERT cites Transformer)\n", - "citations = [\n", - " {\n", - " 'citing_arxiv_id': '1810.04805v2', # BERT\n", - " 'cited_arxiv_id': '1706.03762v7', # Transformer\n", - " 'citing_sections': ['Introduction', 'Related Work', 'Model Architecture']\n", - " }\n", - "]\n", - "\n", - "for citation in citations:\n", - " research_arcade.insert_edge(\"arxiv_citations\", edge_features=citation)\n", - " print(f\"Created citation: {citation['citing_arxiv_id']} → {citation['cited_arxiv_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "badc585c", - "metadata": {}, - "source": [ - "### Paper-Figure Relationships (arxiv_paper_figures)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ef99bab7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table arxiv_paper_figures not found.\n", - "Linked figure 1 to paper 1706.03762v7\n", - "Table arxiv_paper_figures not found.\n", - "Linked figure 2 to paper 1706.03762v7\n", - "Table arxiv_paper_figures not found.\n", - "Linked figure 3 to paper 1706.03762v7\n" - ] - } - ], - "source": [ - "# Link figures to papers\n", - "paper_figures = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'figure_id': 1\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'figure_id': 2\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'figure_id': 3\n", - " }\n", - "]\n", - "\n", - "for relation in paper_figures:\n", - " research_arcade.insert_edge(\"arxiv_paper_figures\", edge_features=relation)\n", - " print(f\"Linked figure {relation['figure_id']} to paper {relation['paper_arxiv_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "260c3bf0", - "metadata": {}, - "source": [ - "### Paper-Table Relationships (arxiv_paper_tables)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8fddc636", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table arxiv_paper_tables not found.\n", - "Linked table 1 to paper 1706.03762v7\n", - "Table arxiv_paper_tables not found.\n", - "Linked table 2 to paper 1706.03762v7\n", - "Table arxiv_paper_tables not found.\n", - "Linked table 3 to paper 1706.03762v7\n" - ] - } - ], - "source": [ - "# Link tables to papers\n", - "paper_tables = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'table_id': 1\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'table_id': 2\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'table_id': 3\n", - " }\n", - "]\n", - "\n", - "for relation in paper_tables:\n", - " research_arcade.insert_edge(\"arxiv_paper_tables\", edge_features=relation)\n", - " print(f\"Linked table {relation['table_id']} to paper {relation['paper_arxiv_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "1384e5a6", - "metadata": {}, - "source": [ - "### Paragraph Citation Relationships (arxiv_paragraph_citations)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b7a93dc1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table arxiv_paragraph_citations not found.\n", - "Paragraph 1 cites 1706.03762v7\n", - "Table arxiv_paragraph_citations not found.\n", - "Paragraph 4 cites 1706.03762v7\n" - ] - } - ], - "source": [ - "# Link specific paragraphs to cited papers\n", - "paragraph_citations = [\n", - " {\n", - " 'paragraph_id': 1,\n", - " 'paper_section': 'Introduction',\n", - " 'citing_arxiv_id': '1810.04805v2',\n", - " 'cited_arxiv_id': '1706.03762v7',\n", - " 'bib_key': 'vaswani2017attention'\n", - " },\n", - " {\n", - " 'paragraph_id': 4,\n", - " 'paper_section': 'Related Work',\n", - " 'citing_arxiv_id': '1810.04805v2',\n", - " 'cited_arxiv_id': '1706.03762v7',\n", - " 'bib_key': 'vaswani2017attention'\n", - " }\n", - "]\n", - "\n", - "for relation in paragraph_citations:\n", - " research_arcade.insert_edge(\"arxiv_paragraph_citations\", edge_features=relation)\n", - " print(f\"Paragraph {relation['paragraph_id']} cites {relation['cited_arxiv_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "7b0e861c", - "metadata": {}, - "source": [ - "### Paragraph References (arxiv_paragraph_references)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "630758e5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table arxiv_paragraph_references not found.\n", - "Paragraph 10 references figure: fig:architecture\n", - "Table arxiv_paragraph_references not found.\n", - "Paragraph 12 references figure: fig:attention\n", - "Table arxiv_paragraph_references not found.\n", - "Paragraph 20 references table: tab:wmt\n" - ] - } - ], - "source": [ - "# Link paragraphs to internal references (figures, tables, equations)\n", - "paragraph_references = [\n", - " {\n", - " 'paragraph_id': 10,\n", - " 'paper_section': 'Model Architecture',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'reference_label': 'fig:architecture',\n", - " 'reference_type': 'figure'\n", - " },\n", - " {\n", - " 'paragraph_id': 12,\n", - " 'paper_section': 'Model Architecture',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'reference_label': 'fig:attention',\n", - " 'reference_type': 'figure'\n", - " },\n", - " {\n", - " 'paragraph_id': 20,\n", - " 'paper_section': 'Results',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'reference_label': 'tab:wmt',\n", - " 'reference_type': 'table'\n", - " }\n", - "]\n", - "\n", - "for relation in paragraph_references:\n", - " research_arcade.insert_edge(\"arxiv_paragraph_references\", edge_features=relation)\n", - " print(f\"Paragraph {relation['paragraph_id']} references {relation['reference_type']}: {relation['reference_label']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "9429f34d", - "metadata": {}, - "source": [ - "### Paragraph-Table Relationships (arxiv_paragraph_tables)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0aab5263", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table paragraph_tables not found.\n", - "Linked paragraph 18 to table 1\n", - "Table paragraph_tables not found.\n", - "Linked paragraph 20 to table 2\n", - "Table paragraph_tables not found.\n", - "Linked paragraph 25 to table 3\n" - ] - } - ], - "source": [ - "# Link paragraphs that discuss specific tables\n", - "paragraph_tables = [\n", - " {\n", - " 'paragraph_id': 18,\n", - " 'table_id': 1\n", - " },\n", - " {\n", - " 'paragraph_id': 20,\n", - " 'table_id': 2\n", - " },\n", - " {\n", - " 'paragraph_id': 25,\n", - " 'table_id': 3\n", - " }\n", - "]\n", - "\n", - "for relation in paragraph_tables:\n", - " research_arcade.insert_edge(\"paragraph_tables\", edge_features=relation)\n", - " print(f\"Linked paragraph {relation['paragraph_id']} to table {relation['table_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "8b0b8223", - "metadata": {}, - "source": [ - "### Paragraph-Figure Relationships (arxiv_paragraph_figures)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6da65abf", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table paragraph_figures not found.\n", - "Linked paragraph 10 to figure 1\n", - "Table paragraph_figures not found.\n", - "Linked paragraph 12 to figure 2\n", - "Table paragraph_figures not found.\n", - "Linked paragraph 15 to figure 3\n" - ] - } - ], - "source": [ - "# Link paragraphs that discuss specific figures\n", - "paragraph_figures = [\n", - " {\n", - " 'paragraph_id': 10,\n", - " 'figure_id': 1\n", - " },\n", - " {\n", - " 'paragraph_id': 12,\n", - " 'figure_id': 2\n", - " },\n", - " {\n", - " 'paragraph_id': 15,\n", - " 'figure_id': 3\n", - " }\n", - "]\n", - "\n", - "for relation in paragraph_figures:\n", - " research_arcade.insert_edge(\"paragraph_figures\", edge_features=relation)\n", - " print(f\"Linked paragraph {relation['paragraph_id']} to figure {relation['figure_id']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "40f3e422", - "metadata": {}, - "source": [ - "## 11. Advanced Queries \n", - "\n", - "Examples of more complex operations." - ] - }, - { - "cell_type": "markdown", - "id": "b87f71c5", - "metadata": {}, - "source": [ - "### Batch Insert Multiple Papers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "91319fc2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Batch inserted 2 papers\n" - ] - } - ], - "source": [ - "# Batch insert multiple papers at once\n", - "papers_batch = [\n", - " {\n", - " 'arxiv_id': '1409.0473v7',\n", - " 'base_arxiv_id': '1409.0473',\n", - " 'version': 7,\n", - " 'title': 'Neural Machine Translation by Jointly Learning to Align and Translate',\n", - " 'abstract': 'Neural machine translation is a recently proposed approach to machine translation...',\n", - " 'submit_date': '2014-09-01',\n", - " 'metadata': {'venue': 'ICLR 2015'}\n", - " },\n", - " {\n", - " 'arxiv_id': '1512.03385v1',\n", - " 'base_arxiv_id': '1512.03385',\n", - " 'version': 1,\n", - " 'title': 'Deep Residual Learning for Image Recognition',\n", - " 'abstract': 'Deeper neural networks are more difficult to train...',\n", - " 'submit_date': '2015-12-10',\n", - " 'metadata': {'venue': 'CVPR 2016'}\n", - " }\n", - "]\n", - "\n", - "for paper in papers_batch:\n", - " research_arcade.insert_node(\"arxiv_papers\", node_features=paper)\n", - " \n", - "print(f\"Batch inserted {len(papers_batch)} papers\")" - ] - }, - { - "cell_type": "markdown", - "id": "4e21df88", - "metadata": {}, - "source": [ - "### Query Papers by a Specific Author" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aff35dba", - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'ResearchArcade' object has no attribute 'query_edges'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m\n", - "\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)\n", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[36]\u001b[39m\u001b[32m, line 3\u001b[39m\n", - "\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Get all papers by a specific author\u001b[39;00m\n", - "\u001b[32m 2\u001b[39m author_id = \u001b[33m\"\u001b[39m\u001b[33mss_ashish_vaswani\u001b[39m\u001b[33m\"\u001b[39m\n", - "\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m author_papers = \u001b[43mresearch_arcade\u001b[49m\u001b[43m.\u001b[49m\u001b[43mquery_edges\u001b[49m(\n", - "\u001b[32m 4\u001b[39m \u001b[33m\"\u001b[39m\u001b[33marxiv_paper_authors\u001b[39m\u001b[33m\"\u001b[39m,\n", - "\u001b[32m 5\u001b[39m filters={\u001b[33m\"\u001b[39m\u001b[33mauthor_id\u001b[39m\u001b[33m\"\u001b[39m: author_id}\n", - "\u001b[32m 6\u001b[39m )\n", - "\u001b[32m 8\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mPapers by \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mauthor_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m:\u001b[39m\u001b[33m\"\u001b[39m)\n", - "\u001b[32m 9\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m _, paper \u001b[38;5;129;01min\u001b[39;00m author_papers.iterrows():\n", - "\n", - "\u001b[31mAttributeError\u001b[39m: 'ResearchArcade' object has no attribute 'query_edges'" - ] - } - ], - "source": [ - "# Get all papers by a specific author\n", - "author_id = \"ss_ashish_vaswani\"\n", - "author_papers = research_arcade.query_edges(\n", - " \"arxiv_paper_authors\",\n", - " filters={\"author_id\": author_id}\n", - ")\n", - "\n", - "print(f\"Papers by {author_id}:\")\n", - "for _, paper in author_papers.iterrows():\n", - " paper_details = research_arcade.get_node_features_by_id(\n", - " \"arxiv_papers\",\n", - " {\"arxiv_id\": paper['paper_arxiv_id']}\n", - " )\n", - " print(f\" - {paper_details['title'].iloc[0]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "d7755779", - "metadata": {}, - "source": [ - "### Find All Papers Citing a Specific Paper" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "01c1c436", - "metadata": {}, - "outputs": [], - "source": [ - "# Find all papers that cite the Transformer paper\n", - "cited_paper = \"1706.03762v7\"\n", - "citations = research_arcade.query_edges(\n", - " \"arxiv_citations\",\n", - " filters={\"cited_arxiv_id\": cited_paper}\n", - ")\n", - "\n", - "print(f\"Papers citing {cited_paper}:\")\n", - "for _, citation in citations.iterrows():\n", - " citing_paper = research_arcade.get_node_features_by_id(\n", - " \"arxiv_papers\",\n", - " {\"arxiv_id\": citation['citing_arxiv_id']}\n", - " )\n", - " print(f\" - {citing_paper['title'].iloc[0]}\")\n", - " print(f\" Cited in sections: {citation['citing_sections']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "01b786b1", - "metadata": {}, - "source": [ - "### Get Complete Paper Structure (Sections + Paragraphs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0a0c98d3", - "metadata": {}, - "outputs": [], - "source": [ - "# Get the complete structure of a paper\n", - "paper_id = \"1706.03762v7\"\n", - "\n", - "# Get paper\n", - "paper = research_arcade.get_node_features_by_id(\"arxiv_papers\", {\"arxiv_id\": paper_id})\n", - "print(f\"Paper: {paper['title'].iloc[0]}\\n\")\n", - "\n", - "# Get sections\n", - "sections = research_arcade.query_nodes(\n", - " \"arxiv_sections\",\n", - " filters={\"paper_arxiv_id\": paper_id}\n", - ")\n", - "sections = sections.sort_values('section_in_paper_id')\n", - "\n", - "print(\"Paper Structure:\")\n", - "for _, section in sections.iterrows():\n", - " print(f\"\\n{section['section_in_paper_id']}. {section['title']}\")\n", - " \n", - " # Get paragraphs for this section\n", - " paragraphs = research_arcade.query_nodes(\n", - " \"arxiv_paragraphs\",\n", - " filters={\n", - " \"paper_arxiv_id\": paper_id,\n", - " \"section_id\": section['section_in_paper_id']\n", - " }\n", - " )\n", - " paragraphs = paragraphs.sort_values('paragraph_id')\n", - " \n", - " for _, para in paragraphs.iterrows():\n", - " print(f\" Para {para['paragraph_id']}: {para['content'][:100]}...\")" - ] - }, - { - "cell_type": "markdown", - "id": "15d45569", - "metadata": {}, - "source": [ - "### Get All Figures and Tables for a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6009bcf", - "metadata": {}, - "outputs": [], - "source": [ - "# Get all figures and tables for a specific paper\n", - "paper_id = \"1706.03762v7\"\n", - "\n", - "# Get figures\n", - "figures = research_arcade.query_nodes(\n", - " \"arxiv_figures\",\n", - " filters={\"paper_arxiv_id\": paper_id}\n", - ")\n", - "\n", - "print(\"Figures:\")\n", - "for _, fig in figures.iterrows():\n", - " print(f\" {fig['name']}: {fig['caption']}\")\n", - "\n", - "# Get tables\n", - "tables = research_arcade.query_nodes(\n", - " \"arxiv_tables\",\n", - " filters={\"paper_arxiv_id\": paper_id}\n", - ")\n", - "\n", - "print(\"\\nTables:\")\n", - "for _, tab in tables.iterrows():\n", - " print(f\" {tab['label']}: {tab['caption']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "4c3efe64", - "metadata": {}, - "source": [ - "### Find All Papers in a Specific Category" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ae789b1", - "metadata": {}, - "outputs": [], - "source": [ - "# Find all papers in the cs.CL category\n", - "category_id = 1 # cs.CL\n", - "\n", - "paper_categories = research_arcade.query_edges(\n", - " \"arxiv_paper_category\",\n", - " filters={\"category_id\": category_id}\n", - ")\n", - "\n", - "print(\"Papers in cs.CL category:\")\n", - "for _, pc in paper_categories.iterrows():\n", - " paper = research_arcade.get_node_features_by_id(\n", - " \"arxiv_papers\",\n", - " {\"arxiv_id\": pc['paper_arxiv_id']}\n", - " )\n", - " print(f\" - {paper['title'].iloc[0]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "06f71b71", - "metadata": {}, - "source": [ - "### Find Author Collaborations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4283a295", - "metadata": {}, - "outputs": [], - "source": [ - "# Find all co-authors of a specific author\n", - "author_id = \"ss_ashish_vaswani\"\n", - "\n", - "# Get papers by this author\n", - "author_papers = research_arcade.query_edges(\n", - " \"arxiv_paper_authors\",\n", - " filters={\"author_id\": author_id}\n", - ")\n", - "\n", - "# Get all co-authors\n", - "coauthors = set()\n", - "for _, paper_relation in author_papers.iterrows():\n", - " paper_id = paper_relation['paper_arxiv_id']\n", - " all_authors = research_arcade.query_edges(\n", - " \"arxiv_paper_authors\",\n", - " filters={\"paper_arxiv_id\": paper_id}\n", - " )\n", - " for _, author_relation in all_authors.iterrows():\n", - " if author_relation['author_id'] != author_id:\n", - " coauthors.add(author_relation['author_id'])\n", - "\n", - "print(f\"Co-authors of {author_id}:\")\n", - "for coauthor_id in coauthors:\n", - " author = research_arcade.get_node_features_by_id(\n", - " \"arxiv_authors\",\n", - " {\"semantic_scholar_id\": coauthor_id}\n", - " )\n", - " print(f\" - {author['name'].iloc[0]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "d3e59370", - "metadata": {}, - "source": [ - "### Database Statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f77da988", - "metadata": {}, - "outputs": [], - "source": [ - "# Get statistics about the database\n", - "stats = {\n", - " 'papers': len(research_arcade.get_all_node_features(\"arxiv_papers\")),\n", - " 'authors': len(research_arcade.get_all_node_features(\"arxiv_authors\")),\n", - " 'categories': len(research_arcade.get_all_node_features(\"arxiv_categories\")),\n", - " 'figures': len(research_arcade.get_all_node_features(\"arxiv_figures\")),\n", - " 'tables': len(research_arcade.get_all_node_features(\"arxiv_tables\")),\n", - " 'sections': len(research_arcade.get_all_node_features(\"arxiv_sections\")),\n", - " 'paragraphs': len(research_arcade.get_all_node_features(\"arxiv_paragraphs\")),\n", - "}\n", - "\n", - "print(\"Database Statistics:\")\n", - "print(\"=\" * 40)\n", - "for entity, count in stats.items():\n", - " print(f\"{entity.capitalize():15s}: {count:5d}\")\n", - "print(\"=\" * 40)" - ] - }, - { - "cell_type": "markdown", - "id": "94b625f8", - "metadata": {}, - "source": [ - "## Cleanup and Best Practices\n", - "\n", - "### Validation Before Insertion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bb748b3d", - "metadata": {}, - "outputs": [], - "source": [ - "# Always validate data before insertion\n", - "def validate_paper(paper_data):\n", - " required_fields = ['arxiv_id', 'base_arxiv_id', 'version', 'title', 'abstract']\n", - " for field in required_fields:\n", - " if field not in paper_data or not paper_data[field]:\n", - " raise ValueError(f\"Missing required field: {field}\")\n", - " return True\n", - "\n", - "# Example usage\n", - "try:\n", - " new_paper = {\n", - " 'arxiv_id': '2023.12345v1',\n", - " 'base_arxiv_id': '2023.12345',\n", - " 'version': 1,\n", - " 'title': 'New Research Paper',\n", - " 'abstract': 'This is an abstract...'\n", - " }\n", - " \n", - " if validate_paper(new_paper):\n", - " research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)\n", - " print(\"Paper inserted successfully!\")\n", - "except ValueError as e:\n", - " print(f\"Validation error: {e}\")" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/convert.sh b/examples/convert.sh deleted file mode 100644 index e69de29..0000000 diff --git a/examples/convert_sql_to_csv.py b/examples/convert_sql_to_csv.py deleted file mode 100644 index 9796a4d..0000000 --- a/examples/convert_sql_to_csv.py +++ /dev/null @@ -1,77 +0,0 @@ -import sys -from pathlib import Path -# 添加项目根目录到路径 -from research_arcade import ResearchArcade -project_root = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(project_root)) - -db_type = "sql" -config = { - "host": "localhost", - "dbname": "iclr_openreview_database", - "user": "jingjunx", - "password": "", - "port": "5432" -} - -research_arcade = ResearchArcade(db_type=db_type, config=config) - -########## openreview_authors ########## -# get_all_node_features -# openreview_authors_df = research_arcade.get_all_node_features("openreview_authors") -# openreview_authors_df.to_csv("/data/jingjunx/my_research_arcade_data/openreview_authors.csv", index=False) -# print(len(openreview_authors_df)) - -########## openreview_papers ########## -# get_all_node_features -# openreview_papers_df = research_arcade.get_all_node_features("openreview_papers") -# openreview_papers_df.to_csv("/data/jingjunx/my_research_arcade_data/openreview_papers.csv", index=False) -# print(len(openreview_papers_df)) - -########## openreview_reviews ########## -# get_all_node_features -# openreview_reviews_df = research_arcade.get_all_node_features("openreview_reviews") -# openreview_reviews_df.to_csv("/data/jingjunx/my_research_arcade_data/openreview_reviews.csv", index=False) -# print(len(openreview_reviews_df)) - -########## openreview_revisions ########## -# get_all_node_features -# openreview_revisions_df = research_arcade.get_all_node_features("openreview_revisions") -# openreview_revisions_df.to_csv("/data/jingjunx/my_research_arcade_data/openreview_revisions.csv", index=False) -# print(len(openreview_revisions_df)) - -########## openreview_paragraphs ########## -# get_all_node_features -# openreview_paragraphs_df = research_arcade.get_all_node_features("openreview_paragraphs") -# openreview_paragraphs_df.to_csv("/data/jingjunx/my_research_arcade_data/openreview_paragraphs.csv", index=False) -# print(len(openreview_paragraphs_df)) - -########## openreview_arxiv ########## -# get_all_edge_features -# openreview_arxiv_df = research_arcade.get_all_edge_features("openreview_arxiv") -# openreview_arxiv_df.to_csv("/data/jingjunx/my_research_arcade_data/openreview_arxiv.csv", index=False) -# print(len(openreview_arxiv_df)) - -########## openreview_papers_authors ########## -# get_all_edge_features -# openreview_papers_authors = research_arcade.get_all_edge_features("openreview_papers_authors") -# openreview_papers_authors.to_csv("/data/jingjunx/my_research_arcade_data/openreview_papers_authors.csv", index=False) -# print(len(openreview_papers_authors)) - -########## openreview_papers_reviews ########## -# get_all_edge_features -# openreview_papers_reviews = research_arcade.get_all_edge_features("openreview_papers_reviews") -# openreview_papers_reviews.to_csv("/data/jingjunx/my_research_arcade_data/openreview_papers_reviews.csv", index=False) -# print(len(openreview_papers_reviews)) - -########## openreview_papers_revisions ########## -# get_all_edge_features -# openreview_papers_revisions = research_arcade.get_all_edge_features("openreview_papers_revisions") -# openreview_papers_revisions.to_csv("/data/jingjunx/my_research_arcade_data/openreview_papers_revisions.csv", index=False) -# print(len(openreview_papers_revisions)) - -########## openreview_revisions_reviews ########## -# get_all_edge_features -# openreview_revisions_reviews = research_arcade.get_all_edge_features("openreview_revisions_reviews") -# openreview_revisions_reviews.to_csv("/data/jingjunx/my_research_arcade_data/openreview_revisions_reviews.csv", index=False) -# print(len(openreview_revisions_reviews)) \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_arxiv_example.csv b/examples/csv_data/csv_openreview_arxiv_example.csv deleted file mode 100644 index 5285fb9..0000000 --- a/examples/csv_data/csv_openreview_arxiv_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,paper_openreview_id,arxiv_id,title -xujj_test,xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_author_example.csv b/examples/csv_data/csv_openreview_author_example.csv deleted file mode 100644 index a4e0b51..0000000 --- a/examples/csv_data/csv_openreview_author_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,author_openreview_id,author_full_name,email,affiliation,homepage,dblp -xujj_test,xujj_test,xujj_test,xujj_test,xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_paper_example.csv b/examples/csv_data/csv_openreview_paper_example.csv deleted file mode 100644 index add92d3..0000000 --- a/examples/csv_data/csv_openreview_paper_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,paper_openreview_id,title,abstract,paper_decision,paper_pdf_link -xujj_test,xujj_test,xujj_test,xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_papers_authors_example.csv b/examples/csv_data/csv_openreview_papers_authors_example.csv deleted file mode 100644 index 43ca617..0000000 --- a/examples/csv_data/csv_openreview_papers_authors_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,paper_openreview_id,author_openreview_id -xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_papers_reviews_example.csv b/examples/csv_data/csv_openreview_papers_reviews_example.csv deleted file mode 100644 index 43ab98f..0000000 --- a/examples/csv_data/csv_openreview_papers_reviews_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,paper_openreview_id,review_openreview_id,title,time -xujj_test,xujj_test,xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_papers_revisions_example.csv b/examples/csv_data/csv_openreview_papers_revisions_example.csv deleted file mode 100644 index 01e12dc..0000000 --- a/examples/csv_data/csv_openreview_papers_revisions_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,paper_openreview_id,revision_openreview_id,title,time -xujj_test,xujj_test,xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_paragraphs_example.csv b/examples/csv_data/csv_openreview_paragraphs_example.csv deleted file mode 100644 index d1a5335..0000000 --- a/examples/csv_data/csv_openreview_paragraphs_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,paper_openreview_id,paragraph_idx,section,content -xujj_test,xujj_test,1,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_review_example.csv b/examples/csv_data/csv_openreview_review_example.csv deleted file mode 100644 index 7a37b2a..0000000 --- a/examples/csv_data/csv_openreview_review_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,review_openreview_id,replyto_openreview_id,writer,title,content,time -xujj_test,xujj_test,xujj_test,xujj_test,xujj_test,"{'title': 'xujj_test', 'content': 'xujj_test'}",xujj_test \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_revision_example.csv b/examples/csv_data/csv_openreview_revision_example.csv deleted file mode 100644 index aed23bb..0000000 --- a/examples/csv_data/csv_openreview_revision_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,original_openreview_id,revision_openreview_id,time,content -xujj_test,xujj_test,xujj_test,xujj_test,"{'title': 'xujj_test', 'content': 'xujj_test'}" \ No newline at end of file diff --git a/examples/csv_data/csv_openreview_revisions_reviews_example.csv b/examples/csv_data/csv_openreview_revisions_reviews_example.csv deleted file mode 100644 index fe1fe40..0000000 --- a/examples/csv_data/csv_openreview_revisions_reviews_example.csv +++ /dev/null @@ -1,2 +0,0 @@ -venue,revision_openreview_id,review_openreview_id -xujj_test,xujj_test,xujj_test \ No newline at end of file diff --git a/examples/get_pdfs.py b/examples/get_pdfs.py deleted file mode 100644 index c5ec6cb..0000000 --- a/examples/get_pdfs.py +++ /dev/null @@ -1,131 +0,0 @@ -import os -import openreview -import requests -from tqdm import tqdm -import time - -def get_paper_pdf(link, pdf_path, log_file): - pdf_url = "https://openreview.net"+link - - headers = { - "User-Agent": "Mozilla/5.0" - } - try: - response = requests.get(pdf_url, headers=headers, timeout=15) - if response.status_code == 200: - with open(pdf_path, "wb") as f: - f.write(response.content) - print(f"✅ PDF downloaded: {pdf_path}") - else: - print(f"❌ Download failed ({response.status_code}) for ID: {id}") - with open(log_file, "a") as log: - log.write(f"{link}\n") - except Exception as e: - print(f"❌ Exception for ID {link}: {e}") - with open(log_file, "a") as log: - log.write(f"{link}\n") - -def get_revision_pdf(venue, id, pdf_path, log_file): - if "2024" in venue or "2025" in venue: - pdf_url = "https://openreview.net/notes/edits/attachment?id="+id+"&name=pdf" - elif "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - pdf_url = "https://openreview.net/references/pdf?id="+id - - headers = { - "User-Agent": "Mozilla/5.0" - } - try: - response = requests.get(pdf_url, headers=headers, timeout=15) - if response.status_code == 200: - with open(pdf_path, "wb") as f: - f.write(response.content) - print(f"✅ PDF downloaded: {pdf_path}") - else: - print(f"❌ Download failed ({response.status_code}) for ID: {id}") - with open(log_file, "a") as log: - log.write(f"{id}\n") - except Exception as e: - print(f"❌ Exception for ID {id}: {e}") - with open(log_file, "a") as log: - log.write(f"{id}\n") - -if __name__ == "__main__": - client_v1 = openreview.Client(baseurl='https://api.openreview.net') - client_v2 = openreview.api.OpenReviewClient(baseurl='https://api2.openreview.net') - venue = 'ICLR.cc/2017/conference' - pdf_dir = "/data/jingjunx/openreview_pdfs_2017/" - log_file = "./download_failed_ids_revisions_2017.log" - start_idx = 0 - end_idx = 5 - - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='revisions') - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='revisions') - - if submissions is None: - print(f"No submissions found for venue: {venue}") - else: - for submission in tqdm(submissions[start_idx:end_idx]): - # get paper openreview id - paper_id = submission.id - if "pdf" in submission.content: - pdf_link = submission.content["pdf"] - pdf_path = str(pdf_dir)+str(paper_id)+".pdf" - if os.path.isfile(pdf_path): - continue - else: - get_paper_pdf(pdf_link, pdf_path, log_file) - - revisions = client_v1.get_references(referent=paper_id, original=True) - time.sleep(1) - - pdf_revisions_ids = [] - for revision in revisions: - if "pdf" in revision.content: - pdf_revisions_ids.append(revision.id) - - if len(pdf_revisions_ids) <= 1: - continue - else: - for pdf_revision_id in pdf_revisions_ids: - pdf_path = str(pdf_dir)+str(pdf_revision_id)+".pdf" - if os.path.isfile(pdf_path): - continue - else: - get_revision_pdf(venue, pdf_revision_id, pdf_path, log_file) - time.sleep(1) - else: - submissions = client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - else: - for submission in tqdm(submissions[start_idx:end_idx]): - decision = submission.content["venueid"]["value"].split('/')[-1] - if decision == "Withdrawn_Submission": - continue - else: - # get paper openreview id - paper_id = submission.id - if "pdf" in submission.content: - pdf_link = submission.content["pdf"]["value"] - pdf_path = str(pdf_dir)+str(paper_id)+".pdf" - if os.path.isfile(pdf_path): - continue - else: - get_paper_pdf(pdf_link, pdf_path, log_file) - - revisions = client_v2.get_note_edits(note_id=paper_id) - if len(revisions) <= 1: - continue - else: - for revision in revisions: - pdf_revision_id = revision.id - pdf_path = str(pdf_dir)+str(pdf_revision_id)+".pdf" - if os.path.isfile(pdf_path): - continue - else: - time.sleep(1) - get_revision_pdf(venue, pdf_revision_id, pdf_path, log_file) - time.sleep(1) \ No newline at end of file diff --git a/examples/json_data/json_openreview_arxiv_example.json b/examples/json_data/json_openreview_arxiv_example.json deleted file mode 100644 index 9a5f2f2..0000000 --- a/examples/json_data/json_openreview_arxiv_example.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "venue": "xujj_test", - "paper_openreview_id": "xujj_test", - "arxiv_id": "xujj_test", - "title": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_author_example.json b/examples/json_data/json_openreview_author_example.json deleted file mode 100644 index a1e20c6..0000000 --- a/examples/json_data/json_openreview_author_example.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "venue": "xujj_test", - "author_openreview_id": "xujj_test", - "author_full_name": "xujj_test", - "email": "xujj_test", - "affiliation": "xujj_test", - "homepage": "xujj_test", - "dblp": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_paper_example.json b/examples/json_data/json_openreview_paper_example.json deleted file mode 100644 index af2b079..0000000 --- a/examples/json_data/json_openreview_paper_example.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "venue": "xujj_test", - "paper_openreview_id": "xujj_test", - "title": "xujj_test", - "abstract": "xujj_test", - "paper_decision": "xujj_test", - "paper_pdf_link": "xujj_test" - } -] diff --git a/examples/json_data/json_openreview_papers_authors_example.json b/examples/json_data/json_openreview_papers_authors_example.json deleted file mode 100644 index 1ff22d3..0000000 --- a/examples/json_data/json_openreview_papers_authors_example.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "venue": "xujj_test", - "paper_openreview_id": "xujj_test", - "author_openreview_id": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_papers_reviews_example.json b/examples/json_data/json_openreview_papers_reviews_example.json deleted file mode 100644 index 38b3146..0000000 --- a/examples/json_data/json_openreview_papers_reviews_example.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "venue": "xujj_test", - "paper_openreview_id": "xujj_test", - "review_openreview_id": "xujj_test", - "title": "xujj_test", - "time": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_papers_revisions_example.json b/examples/json_data/json_openreview_papers_revisions_example.json deleted file mode 100644 index b080fea..0000000 --- a/examples/json_data/json_openreview_papers_revisions_example.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "venue": "xujj_test", - "paper_openreview_id": "xujj_test", - "revision_openreview_id": "xujj_test", - "title": "xujj_test", - "time": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_paragraphs_example.json b/examples/json_data/json_openreview_paragraphs_example.json deleted file mode 100644 index 20d1f15..0000000 --- a/examples/json_data/json_openreview_paragraphs_example.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "venue": "xujj_test", - "paper_openreview_id": "xujj_test", - "paragraph_idx": 1, - "section": "xujj_test", - "content": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_review_example.json b/examples/json_data/json_openreview_review_example.json deleted file mode 100644 index 75ea711..0000000 --- a/examples/json_data/json_openreview_review_example.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "venue": "xujj_test", - "review_openreview_id": "xujj_test", - "replyto_openreview_id": "xujj_test", - "writer": "xujj_test", - "title": "xujj_test", - "content": { - "title": "xujj_test", - "content": "xujj_test" - }, - "time": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_revision_example.json b/examples/json_data/json_openreview_revision_example.json deleted file mode 100644 index d4bc125..0000000 --- a/examples/json_data/json_openreview_revision_example.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "venue": "xujj_test", - "original_openreview_id": "xujj_test", - "revision_openreview_id": "xujj_test", - "content": { - "title": "xujj_test", - "content": "xujj_test" - }, - "time": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/json_data/json_openreview_revisions_reviews_example.json b/examples/json_data/json_openreview_revisions_reviews_example.json deleted file mode 100644 index 44075d1..0000000 --- a/examples/json_data/json_openreview_revisions_reviews_example.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "venue": "xujj_test", - "revision_openreview_id": "xujj_test", - "review_openreview_id": "xujj_test" - } -] \ No newline at end of file diff --git a/examples/research_arcade_complete_tutorial.ipynb b/examples/research_arcade_complete_tutorial.ipynb deleted file mode 100644 index d13e91c..0000000 --- a/examples/research_arcade_complete_tutorial.ipynb +++ /dev/null @@ -1,2090 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "header", - "metadata": {}, - "source": [ - "# ResearchArcade Complete Tutorial\n", - "\n", - "This tutorial demonstrates how to work with the ResearchArcade database, covering all node types and edge relationships.\n", - "\n", - "## Table of Contents\n", - "1. [Setup](#setup)\n", - "2. [OpenReview Data](#openreview)\n", - "3. [ArXiv Papers](#arxiv-papers)\n", - "4. [ArXiv Authors](#arxiv-authors)\n", - "5. [ArXiv Categories](#arxiv-categories)\n", - "6. [ArXiv Figures](#arxiv-figures)\n", - "7. [ArXiv Tables](#arxiv-tables)\n", - "8. [ArXiv Sections](#arxiv-sections)\n", - "9. [ArXiv Paragraphs](#arxiv-paragraphs)\n", - "10. [Relationships/Edges](#relationships)\n", - "11. [Advanced Queries](#advanced-queries)" - ] - }, - { - "cell_type": "markdown", - "id": "setup-section", - "metadata": {}, - "source": [ - "## 1. Setup " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "imports", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "import os\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..')))\n", - "from research_arcade.research_arcade import ResearchArcade\n", - "import pandas as pd\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "markdown", - "id": "db-backend", - "metadata": {}, - "source": [ - "### Choose Database Backend" - ] - }, - { - "cell_type": "markdown", - "id": "csv-backend", - "metadata": {}, - "source": [ - "#### CSV Based" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "csv-config", - "metadata": {}, - "outputs": [], - "source": [ - "db_type = \"csv\"\n", - "config = {\n", - " \"csv_dir\": \"../data/my_research_arcade_data/\"\n", - "}\n", - "\n", - "research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "sql-backend", - "metadata": {}, - "source": [ - "#### SQL Based (PostgreSQL)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "sql-config", - "metadata": {}, - "outputs": [], - "source": [ - "# db_type = \"sql\"\n", - "# config = {\n", - "# \"host\": \"localhost\",\n", - "# \"dbname\": \"iclr_openreview_database\",\n", - "# \"user\": \"jingjunx\",\n", - "# \"password\": \"\",\n", - "# \"port\": \"5432\"\n", - "# }\n", - "\n", - "# research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-section", - "metadata": {}, - "source": [ - "## 3. ArXiv Papers \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `arxiv_id` (VARCHAR, unique) - e.g., 1802.08773v3\n", - "- `base_arxiv_id` (VARCHAR) - e.g., 1802.08773\n", - "- `version` (INT) - e.g., 3\n", - "- `title` (TEXT)\n", - "- `abstract` (TEXT)\n", - "- `submit_date` (DATE)\n", - "- `metadata` (JSONB)" - ] - }, - { - "cell_type": "markdown", - "id": "91b8e215", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "ccaeefb3", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1806.08804v4\", \"1903.03894v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-insert", - "metadata": {}, - "source": [ - "### Insert a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "insert-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 1: Insert the famous \"Attention is All You Need\" paper\n", - "new_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'base_arxiv_id': '1706.03762',\n", - " 'version': 7,\n", - " 'title': 'Attention Is All You Need',\n", - " 'abstract': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.',\n", - " 'submit_date': '2017-06-12',\n", - " 'metadata': {'venue': 'NeurIPS 2017', 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf'}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)\n", - "print(\"Paper inserted successfully!\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "insert-paper-bert", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BERT paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 2: Insert BERT paper\n", - "bert_paper = {\n", - " 'arxiv_id': '1810.04805v2',\n", - " 'base_arxiv_id': '1810.04805',\n", - " 'version': 2,\n", - " 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding',\n", - " 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.',\n", - " 'submit_date': '2018-10-11',\n", - " 'metadata': {'venue': 'NAACL 2019', 'citations': 50000}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=bert_paper)\n", - "print(\"BERT paper inserted successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-all", - "metadata": {}, - "source": [ - "### Get All Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "get-all-papers", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total papers in database: 6\n", - "\n", - "First 5 papers:\n", - " id arxiv_id base_arxiv_id version \\\n", - "0 2 1810.04805v2 1810.04805 2 \n", - "1 3 1409.0473v7 1409.04730 7 \n", - "2 4 1512.03385v1 1512.03385 1 \n", - "3 5 1806.08804v4 1806.08804 4 \n", - "4 6 1903.03894v4 1903.03894 4 \n", - "\n", - " title \\\n", - "0 BERT: Pre-training of Deep Bidirectional Trans... \n", - "1 Neural Machine Translation by Jointly Learning... \n", - "2 Deep Residual Learning for Image Recognition \n", - "3 Hierarchical Graph Representation Learning wit... \n", - "4 GNNExplainer: Generating Explanations for Grap... \n", - "\n", - " abstract \\\n", - "0 We introduce a new language representation mod... \n", - "1 Neural machine translation is a recently propo... \n", - "2 Deeper neural networks are more difficult to t... \n", - "3 Recently, graph neural networks (GNNs) have re... \n", - "4 Graph Neural Networks (GNNs) are a powerful to... \n", - "\n", - " submit_date \\\n", - "0 2018-10-11 \n", - "1 2014-09-01 \n", - "2 2015-12-10 \n", - "3 2018-06-22 18:04:46+00:00 \n", - "4 2019-03-10 00:56:26+00:00 \n", - "\n", - " metadata \n", - "0 {\"venue\": \"NAACL 2019\", \"citations\": 50000} \n", - "1 {\"venue\": \"ICLR 2015\"} \n", - "2 {\"venue\": \"CVPR 2016\"} \n", - "3 {\"id\": \"1806.08804v4\", \"title\": \"Hierarchical ... \n", - "4 {\"id\": \"1903.03894v4\", \"title\": \"GNNExplainer:... \n" - ] - } - ], - "source": [ - "arxiv_papers_df = research_arcade.get_all_node_features(\"arxiv_papers\")\n", - "print(f\"Total papers in database: {len(arxiv_papers_df)}\")\n", - "print(\"\\nFirst 5 papers:\")\n", - "print(arxiv_papers_df.head())" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Paper by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "get-paper-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper details:\n", - "{'id': 2, 'arxiv_id': '1810.04805v2', 'base_arxiv_id': 1810.04805, 'version': 2, 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding', 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.', 'submit_date': '2018-10-11', 'metadata': '{\"venue\": \"NAACL 2019\", \"citations\": 50000}'}\n" - ] - } - ], - "source": [ - "paper_id = {\"arxiv_id\": \"1810.04805v2\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Paper details:\")\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-update", - "metadata": {}, - "source": [ - "### Update a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "update-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper updated successfully!\n" - ] - } - ], - "source": [ - "# Update metadata for a paper\n", - "updated_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'metadata': {\n", - " 'venue': 'NeurIPS 2017',\n", - " 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf',\n", - " 'citations': 75000,\n", - " 'influential': True\n", - " }\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_papers\", node_features=updated_paper)\n", - "print(\"Paper updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-delete", - "metadata": {}, - "source": [ - "### Delete a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "delete-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted paper:\n", - "True\n" - ] - } - ], - "source": [ - "# Delete a paper by ID\n", - "paper_id = {\"arxiv_id\": \"1706.03762v7\"}\n", - "deleted_paper = research_arcade.delete_node_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Deleted paper:\")\n", - "print(deleted_paper)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-section", - "metadata": {}, - "source": [ - "## 4. ArXiv Authors \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `semantic_scholar_id` (VARCHAR, unique)\n", - "- `name` (VARCHAR)\n", - "- `homepage` (VARCHAR)" - ] - }, - { - "cell_type": "markdown", - "id": "6e14ad06", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "c18c7737", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "base_arxiv_id: 1903.03894\n", - "base_arxiv_id: 1806.08804\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-insert", - "metadata": {}, - "source": [ - "### Insert Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "insert-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted author: Ashish Vaswani\n", - "Inserted author: Noam Shazeer\n", - "Inserted author: Niki Parmar\n", - "Inserted author: Jakob Uszkoreit\n", - "Inserted author: Llion Jones\n" - ] - } - ], - "source": [ - "# Insert authors from the Transformer paper\n", - "authors = [\n", - " {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'name': 'Ashish Vaswani',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_noam_shazeer',\n", - " 'name': 'Noam Shazeer',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_niki_parmar',\n", - " 'name': 'Niki Parmar',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_jakob_uszkoreit',\n", - " 'name': 'Jakob Uszkoreit',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_llion_jones',\n", - " 'name': 'Llion Jones',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " }\n", - "]\n", - "\n", - "for author in authors:\n", - " research_arcade.insert_node(\"arxiv_authors\", node_features=author)\n", - " print(f\"Inserted author: {author['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-all", - "metadata": {}, - "source": [ - "### Get All Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "get-all-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total authors in database: 13\n", - "\n", - "All authors:\n", - " id semantic_scholar_id name \\\n", - "0 1 ss_ashish_vaswani Ashish Vaswani \n", - "1 2 ss_noam_shazeer Noam Shazeer \n", - "2 3 ss_niki_parmar Niki Parmar \n", - "3 4 ss_jakob_uszkoreit Jakob Uszkoreit \n", - "4 5 ss_llion_jones Llion Jones \n", - "5 6 83539859 Rex Ying \n", - "6 7 40974349 Dylan Bourgeois \n", - "7 8 145829303 Jiaxuan You \n", - "8 9 2095762 M. Zitnik \n", - "9 10 1702139 J. Leskovec \n", - "10 11 143622465 Christopher Morris \n", - "11 12 145201124 Xiang Ren \n", - "12 13 49437682 William L. Hamilton \n", - "\n", - " homepage \n", - "0 https://ashishvaswani.com \n", - "1 https://scholar.google.com/citations?user=oR9s... \n", - "2 https://scholar.google.com/citations?user=oR9s... \n", - "3 https://scholar.google.com/citations?user=oR9s... \n", - "4 https://scholar.google.com/citations?user=oR9s... \n", - "5 https://www.semanticscholar.org/author/83539859 \n", - "6 https://www.semanticscholar.org/author/40974349 \n", - "7 https://www.semanticscholar.org/author/145829303 \n", - "8 https://www.semanticscholar.org/author/2095762 \n", - "9 https://www.semanticscholar.org/author/1702139 \n", - "10 https://www.semanticscholar.org/author/143622465 \n", - "11 https://www.semanticscholar.org/author/145201124 \n", - "12 https://www.semanticscholar.org/author/49437682 \n" - ] - } - ], - "source": [ - "authors_df = research_arcade.get_all_node_features(\"arxiv_authors\")\n", - "print(f\"Total authors in database: {len(authors_df)}\")\n", - "print(\"\\nAll authors:\")\n", - "print(authors_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Author by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "get-author-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author details:\n", - "None\n" - ] - } - ], - "source": [ - "author_id = {\"semantic_scholar_id\": \"ss_ashish_vaswani\"}\n", - "author_features = research_arcade.get_node_features_by_id(\"arxiv_authors\", author_id)\n", - "print(\"Author details:\")\n", - "print(author_features)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-update", - "metadata": {}, - "source": [ - "### Update an Author" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "update-author", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author updated successfully!\n" - ] - } - ], - "source": [ - "updated_author = {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'homepage': 'https://ashishvaswani.com'\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_authors\", node_features=updated_author)\n", - "print(\"Author updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-section", - "metadata": {}, - "source": [ - "## 5. ArXiv Categories \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `name` (VARCHAR, unique)\n", - "- `description` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "3e9eeea6", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "168633f2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': '1903.03894v4', 'title': 'GNNExplainer: Generating Explanations for Graph Neural Networks', 'abstract': \"Graph Neural Networks (GNNs) are a powerful tool for machine learning on\\ngraphs.GNNs combine node feature information with the graph structure by\\nrecursively passing neural messages along edges of the input graph. However,\\nincorporating both graph structure and feature information leads to complex\\nmodels, and explaining predictions made by GNNs remains unsolved. Here we\\npropose GNNExplainer, the first general, model-agnostic approach for providing\\ninterpretable explanations for predictions of any GNN-based model on any\\ngraph-based machine learning task. Given an instance, GNNExplainer identifies a\\ncompact subgraph structure and a small subset of node features that have a\\ncrucial role in GNN's prediction. Further, GNNExplainer can generate consistent\\nand concise explanations for an entire class of instances. We formulate\\nGNNExplainer as an optimization task that maximizes the mutual information\\nbetween a GNN's prediction and distribution of possible subgraph structures.\\nExperiments on synthetic and real-world graphs show that our approach can\\nidentify important graph structures as well as node features, and outperforms\\nbaselines by 17.1% on average. GNNExplainer provides a variety of benefits,\\nfrom the ability to visualize semantically relevant structures to\\ninterpretability, to giving insights into errors of faulty GNNs.\", 'authors': ['Rex Ying', 'Dylan Bourgeois', 'Jiaxuan You', 'Marinka Zitnik', 'Jure Leskovec'], 'published': '2019-03-10 00:56:26+00:00', 'categories': ['cs.LG', 'stat.ML'], 'url': 'http://arxiv.org/abs/1903.03894v4'}\n", - "{'id': '1806.08804v4', 'title': 'Hierarchical Graph Representation Learning with Differentiable Pooling', 'abstract': 'Recently, graph neural networks (GNNs) have revolutionized the field of graph\\nrepresentation learning through effectively learned node embeddings, and\\nachieved state-of-the-art results in tasks such as node classification and link\\nprediction. However, current GNN methods are inherently flat and do not learn\\nhierarchical representations of graphs---a limitation that is especially\\nproblematic for the task of graph classification, where the goal is to predict\\nthe label associated with an entire graph. Here we propose DiffPool, a\\ndifferentiable graph pooling module that can generate hierarchical\\nrepresentations of graphs and can be combined with various graph neural network\\narchitectures in an end-to-end fashion. DiffPool learns a differentiable soft\\ncluster assignment for nodes at each layer of a deep GNN, mapping nodes to a\\nset of clusters, which then form the coarsened input for the next GNN layer.\\nOur experimental results show that combining existing GNN methods with DiffPool\\nyields an average improvement of 5-10% accuracy on graph classification\\nbenchmarks, compared to all existing pooling approaches, achieving a new\\nstate-of-the-art on four out of five benchmark data sets.', 'authors': ['Rex Ying', 'Jiaxuan You', 'Christopher Morris', 'Xiang Ren', 'William L. Hamilton', 'Jure Leskovec'], 'published': '2018-06-22 18:04:46+00:00', 'categories': ['cs.LG', 'cs.NE', 'cs.SI', 'stat.ML'], 'url': 'http://arxiv.org/abs/1806.08804v4'}\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce02c495", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-insert", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "insert-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-get-all", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "get-all-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 7\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.CL Computation and Language (Natural Language Pro...\n", - "1 2 cs.LG Machine Learning\n", - "2 3 cs.AI Artificial Intelligence\n", - "3 4 cs.CV Computer Vision and Pattern Recognition\n", - "4 5 stat.ML Machine Learning (Statistics)\n", - "5 6 cs.NE NaN\n", - "6 7 cs.SI NaN\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-section", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-insert", - "metadata": {}, - "source": [ - "### Insert Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "insert-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted Figure 1\n", - "Inserted Figure 2\n", - "Inserted Figure 3\n" - ] - } - ], - "source": [ - "# Insert figures for the Transformer paper\n", - "figures = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/transformer_architecture.png',\n", - " 'caption': 'The Transformer model architecture. The left side shows the encoder stack and the right side shows the decoder stack.',\n", - " 'label': 'fig:architecture',\n", - " 'name': 'Figure 1'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/scaled_dot_product_attention.png',\n", - " 'caption': 'Scaled Dot-Product Attention and Multi-Head Attention mechanisms.',\n", - " 'label': 'fig:attention',\n", - " 'name': 'Figure 2'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/positional_encoding.png',\n", - " 'caption': 'Positional encoding visualization showing sine and cosine functions of different frequencies.',\n", - " 'label': 'fig:positional',\n", - " 'name': 'Figure 3'\n", - " }\n", - "]\n", - "\n", - "for figure in figures:\n", - " research_arcade.insert_node(\"arxiv_figures\", node_features=figure)\n", - " print(f\"Inserted {figure['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-get-all", - "metadata": {}, - "source": [ - "### Get All Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "get-all-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total figures: 3\n", - "\n", - "All figures:\n", - " name caption \\\n", - "0 Figure 1 The Transformer model architecture. The left s... \n", - "1 Figure 2 Scaled Dot-Product Attention and Multi-Head At... \n", - "2 Figure 3 Positional encoding visualization showing sine... \n", - "\n", - " label \n", - "0 fig:architecture \n", - "1 fig:attention \n", - "2 fig:positional \n" - ] - } - ], - "source": [ - "figures_df = research_arcade.get_all_node_features(\"arxiv_figures\")\n", - "print(f\"Total figures: {len(figures_df)}\")\n", - "print(\"\\nAll figures:\")\n", - "print(figures_df[['name', 'caption', 'label']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-section", - "metadata": {}, - "source": [ - "## 7. ArXiv Tables \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `table_text` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "c240602b", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "54a13d98", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed: ['1903.03894v4']\n", - "BFS_que.qsize(): 1\n", - "current paper: 1903.03894v4\n", - "Thread 13208330240 Processing 1903.03894v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "x 000abstract.tex\n", - "x 010intro.tex\n", - "x 020related.tex\n", - "x 030background.tex\n", - "x 030formulation.tex\n", - "x 030proposed.tex\n", - "x 040experiments.tex\n", - "x 050conclusion.tex\n", - "x 060supplement.tex\n", - "x acmart.bib\n", - "x acmart.cls\n", - "x acmart.dtx\n", - "x acmart.ins\n", - "x ACM-Reference-Format.bbx\n", - "x ACM-Reference-Format.bst\n", - "x ACM-Reference-Format.cbx\n", - "x ACM-Reference-Format.dbx\n", - "x figs/\n", - "x figs/explainer-introduction_v2.pdf\n", - "x figs/explainer-motivation.pdf\n", - "x figs/explainer.pdf\n", - "x figs/feature_importance_v2.pdf\n", - "x figs/fig3-graph-cls-v2.pdf\n", - "x figs/fig3-graph-cls.pdf\n", - "x figs/fig3-node-cls-v3.pdf\n", - "x figs/fig3-node-cls.pdf\n", - "x figs/fig3-v4.pdf\n", - "x figs/fig3-v5.pdf\n", - "x figs/including-node-features.pdf\n", - "x figs/local_subgraph.png\n", - "x figs/motivation-node-features.pdf\n", - "x figs/prototype.png\n", - "x figs/prototype1.png\n", - "x figs/single-instance-explanation-final.pdf\n", - "x figs/single-instance-explanation2.pdf\n", - "x figs/single-instance-explanations.pdf\n", - "x figs/synth_dataset-v7.pdf\n", - "x GNN-Explainer.bbl\n", - "x GNN-Explainer.tex\n", - "x micro.tex\n", - "x neurips_2019_author_response.sty\n", - "x neurips_2019.sty\n", - "x refs.bib\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 13208330240 Processing file GNN-Explainer.tex\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:bibtexparser.bparser:Overwriting existing string for key: amstrans.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: bullams.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: procams.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: transams.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: cacm.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: acmmathsoft.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: signum.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: amerstatassoc.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: applmathcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: amermathmonthly.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: britstatpsych.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: canmathbull.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compapplmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compstruct.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compjour.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compsyssci.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: contempmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: giornalemath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ieeetranscomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ieeetransac.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: procieee.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ieeetransaeroelec.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: imanumerana.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: infproclet.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: instmathapp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: intcontrol.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: intnumereng.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: intsuper.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: jresnatburstand.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: linalgapp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathanaappl.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathannalen.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathscand.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: tablesaidscomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: numermath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: pacificmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: pardistcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: parcomputing.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: philmag.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: procnas.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: quartmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: quartapplmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: revueinststat.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: jsiam.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: jsiamb.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamalgmeth.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamappmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siammatrix.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamnumanal.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamreview.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamscistat.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: softpracexp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: statscience.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ussrcompmathphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: vlsicompsys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: zangewmathmech.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: zangewmathphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: amstrans.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: bullams.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: procams.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: transams.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: cacm.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: acmmathsoft.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: signum.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: amerstatassoc.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: applmathcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: amermathmonthly.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: britstatpsych.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: canmathbull.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compapplmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compstruct.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compjour.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: compsyssci.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: contempmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: giornalemath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ieeetranscomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ieeetransac.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: procieee.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ieeetransaeroelec.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: imanumerana.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: infproclet.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: instmathapp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: intcontrol.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: intnumereng.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: intsuper.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: jresnatburstand.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: linalgapp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathanaappl.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathannalen.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: mathscand.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: tablesaidscomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: numermath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: pacificmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: pardistcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: parcomputing.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: philmag.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: procnas.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: quartmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: quartapplmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: revueinststat.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: jsiam.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: jsiamb.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamalgmeth.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamappmath.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamcomp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siammatrix.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamnumanal.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamreview.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: siamscistat.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: softpracexp.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: statscience.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: ussrcompmathphys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: vlsicompsys.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: zangewmathmech.\n", - "WARNING:bibtexparser.bparser:Overwriting existing string for key: zangewmathphys.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "numbmer of citations in node info method: 0\n", - "bib_path: ./download/working_folder/13208330240/refs.bib\n", - "key2id: {'alvarez-melis_robustness_2018': '1806.08049', 'goldstein_peeking_2013': '1309.6392', 'greenwell_simple_2018': '1805.04755', 'Kang2019explaine': '1904.12694', 'weiss_extracting_2017': '1711.09576', 'smilkov2017smoothgrad': '1706.03825', 'poursabzi-sangdeh_manipulating_2018': '1802.07810', 'gilpin_explaining_2018': '1806.00069', 'battaglia': '1806.01261', 'zhou_graph_2018': '1812.08434', 'zhang_deep_2018': '1812.04202', 'zhang2018deep': '1812.04202', 'vinyals2015order': '1511.06391', 'li2015gated': '1511.05493', 'rahimi2018semi': '1804.08049', 'peng2017cross': '1708.03743', 'chen2018learning': '1802.07814'}\n", - "Cannot find the figure figs/grad_subgraph\n", - "Thread 13208330240 Finished processing 1903.03894v4 (1/999999999) Time elapsed: 2.18s\n", - "fromisoformat: argument must be str\n", - "Thread 13208330240 Failed to process 1903.03894v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "cp: ./download/working_folder/13208330240/figs/grad_subgraph: No such file or directory\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 8670343744 Finished processing 1 papers\n", - "seed: ['1806.08804v4']\n", - "BFS_que.qsize(): 1\n", - "current paper: 1806.08804v4\n", - "Thread 13208330240 Processing 1806.08804v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "x 000abstract.tex\n", - "x 010intro.tex\n", - "x 010intro-ws.tex\n", - "x 020related.tex\n", - "x 030proposed.tex\n", - "x 040experiments.tex\n", - "x 040experiments-ws.tex\n", - "x 050conclusion.tex\n", - "x 060appendix.tex\n", - "x acmart.cls\n", - "x acmart.dtx\n", - "x acmart.ins\n", - "x figs/\n", - "x figs/assignment1.png\n", - "x figs/diffpool.pdf\n", - "x figs/differentiable-pooling.pdf\n", - "x figs/differentiable-pooling-V2.pdf\n", - "x figs/vis_1.jpg\n", - "x figs/vis_2.jpg\n", - "x figs/vis_3.png\n", - "x figs/diffpool_vis.pdf\n", - "x figs/vis_2_ell.png\n", - "x figs/vis_1_ell.png\n", - "x figs/vis_5.png\n", - "x micro.tex\n", - "x neurips_2018.sty\n", - "x nips_2018_author_response.sty\n", - "x paper-diffpool.bbl\n", - "x paper-diffpool.tex\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 13208330240 Processing file paper-diffpool.tex\n", - "numbmer of citations in node info method: 0\n", - "Cannot find the bib file refs.bib\n", - "Thread 13208330240 Finished processing 1806.08804v4 (1/999999999) Time elapsed: 1.08s\n", - "Thread 8670343744 Finished processing 1 papers\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "b7002162", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "b809fdd9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3771de83", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "1f1357fc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 7\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.CL Computation and Language (Natural Language Pro...\n", - "1 2 cs.LG Machine Learning\n", - "2 3 cs.AI Artificial Intelligence\n", - "3 4 cs.CV Computer Vision and Pattern Recognition\n", - "4 5 stat.ML Machine Learning (Statistics)\n", - "5 6 cs.NE NaN\n", - "6 7 cs.SI NaN\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "827d6714", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "1a7d33eb", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "195e218d", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-insert", - "metadata": {}, - "source": [ - "### Insert Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "insert-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted table: tab:variations\n", - "Inserted table: tab:wmt\n", - "Inserted table: tab:parsing\n" - ] - } - ], - "source": [ - "# Insert tables for the Transformer paper\n", - "tables = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/model_variations.tex',\n", - " 'caption': 'Variations on the Transformer architecture with different hyperparameters.',\n", - " 'label': 'tab:variations',\n", - " 'table_text': 'Model | N | d_model | d_ff | h | d_k | d_v | P_drop | train time\\nbase | 6 | 512 | 2048 | 8 | 64 | 64 | 0.1 | 12 hrs'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/wmt_results.tex',\n", - " 'caption': 'Performance of the Transformer on WMT 2014 English-German and English-French translation tasks.',\n", - " 'label': 'tab:wmt',\n", - " 'table_text': 'Model | EN-DE BLEU | EN-FR BLEU\\nTransformer (base) | 27.3 | 38.1\\nTransformer (big) | 28.4 | 41.8'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/parsing_results.tex',\n", - " 'caption': 'English constituency parsing results on WSJ test set.',\n", - " 'label': 'tab:parsing',\n", - " 'table_text': 'Model | WSJ 23 F1\\nTransformer | 91.3'\n", - " }\n", - "]\n", - "\n", - "for table in tables:\n", - " research_arcade.insert_node(\"arxiv_tables\", node_features=table)\n", - " print(f\"Inserted table: {table['label']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-get-all", - "metadata": {}, - "source": [ - "### Get All Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "get-all-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total tables: 11\n", - "\n", - "All tables:\n", - " label \\\n", - "0 tab:variations \n", - "1 tab:wmt \n", - "2 tab:parsing \n", - "3 \\label{fig:synth_datasets} \n", - "4 \\label{tab:results_pr} \n", - "5 \\label{tab:results_pr} \n", - "6 \\label{tab:results} \n", - "7 \\label{tab:results2} \n", - "8 tab:variations \n", - "9 tab:wmt \n", - "10 tab:parsing \n", - "\n", - " caption \n", - "0 Variations on the Transformer architecture wit... \n", - "1 Performance of the Transformer on WMT 2014 Eng... \n", - "2 English constituency parsing results on WSJ te... \n", - "3 \\caption{Illustration of synthetic datasets (r... \n", - "4 \\caption{\\namelong compared to baselines in id... \n", - "5 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "6 \\caption{Classification accuracies in percent.... \n", - "7 \\caption{Accuracy results of applying \\name to... \n", - "8 Variations on the Transformer architecture wit... \n", - "9 Performance of the Transformer on WMT 2014 Eng... \n", - "10 English constituency parsing results on WSJ te... \n" - ] - } - ], - "source": [ - "tables_df = research_arcade.get_all_node_features(\"arxiv_tables\")\n", - "print(f\"Total tables: {len(tables_df)}\")\n", - "print(\"\\nAll tables:\")\n", - "print(tables_df[['label', 'caption']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-section", - "metadata": {}, - "source": [ - "## 8. ArXiv Sections \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `content` (TEXT)\n", - "- `title` (TEXT)\n", - "- `appendix` (BOOLEAN)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `section_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "d9890560", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "c1735fc6", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-insert", - "metadata": {}, - "source": [ - "### Insert Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "insert-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted section: Introduction\n", - "Inserted section: Background\n", - "Inserted section: Model Architecture\n", - "Inserted section: Training\n", - "Inserted section: Results\n", - "Inserted section: Conclusion\n" - ] - } - ], - "source": [ - "# Insert sections for the Transformer paper\n", - "sections = [\n", - " {\n", - " 'content': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder...',\n", - " 'title': 'Introduction',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 1\n", - " },\n", - " {\n", - " 'content': 'Most competitive neural sequence transduction models have an encoder-decoder structure. Here, the encoder maps an input sequence of symbol representations...',\n", - " 'title': 'Background',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 2\n", - " },\n", - " {\n", - " 'content': 'Most neural sequence transduction models have an encoder-decoder structure. The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers...',\n", - " 'title': 'Model Architecture',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 3\n", - " },\n", - " {\n", - " 'content': 'In this section we describe the training regime for our models...',\n", - " 'title': 'Training',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 4\n", - " },\n", - " {\n", - " 'content': 'On the WMT 2014 English-to-German translation task, the big transformer model outperforms the best previously reported models...',\n", - " 'title': 'Results',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 5\n", - " },\n", - " {\n", - " 'content': 'In this work, we presented the Transformer, the first sequence transduction model based entirely on attention, replacing the recurrent layers...',\n", - " 'title': 'Conclusion',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 6\n", - " }\n", - "]\n", - "\n", - "for section in sections:\n", - " research_arcade.insert_node(\"arxiv_sections\", node_features=section)\n", - " print(f\"Inserted section: {section['title']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-get-all", - "metadata": {}, - "source": [ - "### Get All Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "get-all-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total sections: id content \\\n", - "0 1 The dominant sequence transduction models are ... \n", - "1 2 Most competitive neural sequence transduction ... \n", - "2 3 Most neural sequence transduction models have ... \n", - "3 4 In this section we describe the training regim... \n", - "4 5 On the WMT 2014 English-to-German translation ... \n", - "5 6 In this work, we presented the Transformer, th... \n", - "6 7 \\n\\n\\n\\hide{\\n\\begin{figure}[h]\\n \\centerin... \n", - "7 8 \\n\\label{sec:related}\\n\\n\\n\\nAlthough the prob... \n", - "8 9 \\n\\label{sec:explainer}\\n\\n\\n\\begin{figure*}[t... \n", - "9 10 \\n\\label{sec:exp}\\n\\n\\n\\n\\n\\n\\hide{\\nResults i... \n", - "10 11 \\n\\label{sec:conclusion}\\n\\nWe present \\longna... \n", - "11 12 \\n\\nThe problem of multi-instance explanations... \n", - "12 13 \\n\\nIn the context of multi-instance explanati... \n", - "13 14 \\n\\n\\xhdr{Training details}Training details\\nW... \n", - "14 15 \\n\\label{sec:intro}\\nIn recent years there has... \n", - "15 16 \\n\\nOur work builds upon a rich line of recent... \n", - "16 17 \\n\\label{sec:proposed}\\n\\nThe key idea of \\nam... \n", - "17 18 \\n\\label{sec:ex}\\n\\nWe evaluate the benefits o... \n", - "18 19 \\n\\nWe introduced a differentiable pooling met... \n", - "19 20 \\nThis research has been supported in part by ... \n", - "20 21 The dominant sequence transduction models are ... \n", - "21 22 Most competitive neural sequence transduction ... \n", - "22 23 Most neural sequence transduction models have ... \n", - "23 24 In this section we describe the training regim... \n", - "24 25 On the WMT 2014 English-to-German translation ... \n", - "25 26 In this work, we presented the Transformer, th... \n", - "\n", - " title appendix \\\n", - "0 Introduction False \n", - "1 Background False \n", - "2 Model Architecture False \n", - "3 Training False \n", - "4 Results False \n", - "5 Conclusion False \n", - "6 Introduction False \n", - "7 Related work False \n", - "8 Formulating explanations for graph neural netw... False \n", - "9 Experiments False \n", - "10 Conclusion False \n", - "11 Multi-instance explanations False \n", - "12 Experiments on multi-instance explanations and... False \n", - "13 Further implementation details False \n", - "14 Introduction False \n", - "15 Related Work False \n", - "16 Proposed Method False \n", - "17 Experiments False \n", - "18 Conclusion False \n", - "19 Acknowledgement False \n", - "20 Introduction False \n", - "21 Background False \n", - "22 Model Architecture False \n", - "23 Training False \n", - "24 Results False \n", - "25 Conclusion False \n", - "\n", - " paper_arxiv_id section_in_paper_id \n", - "0 1706.03762v7 1.0 \n", - "1 1706.03762v7 2.0 \n", - "2 1706.03762v7 3.0 \n", - "3 1706.03762v7 4.0 \n", - "4 1706.03762v7 5.0 \n", - "5 1706.03762v7 6.0 \n", - "6 1903.03894v4 1.0 \n", - "7 1903.03894v4 2.0 \n", - "8 1903.03894v4 3.0 \n", - "9 1903.03894v4 4.0 \n", - "10 1903.03894v4 5.0 \n", - "11 1903.03894v4 6.0 \n", - "12 1903.03894v4 7.0 \n", - "13 1903.03894v4 8.0 \n", - "14 1806.08804v4 1.0 \n", - "15 1806.08804v4 2.0 \n", - "16 1806.08804v4 3.0 \n", - "17 1806.08804v4 4.0 \n", - "18 1806.08804v4 5.0 \n", - "19 1806.08804v4 6.0 \n", - "20 1706.03762v7 1.0 \n", - "21 1706.03762v7 2.0 \n", - "22 1706.03762v7 3.0 \n", - "23 1706.03762v7 4.0 \n", - "24 1706.03762v7 5.0 \n", - "25 1706.03762v7 6.0 \n", - "\n", - "All sections:\n", - " title section_in_paper_id \\\n", - "0 Introduction 1.0 \n", - "1 Background 2.0 \n", - "2 Model Architecture 3.0 \n", - "3 Training 4.0 \n", - "4 Results 5.0 \n", - "5 Conclusion 6.0 \n", - "6 Introduction 1.0 \n", - "7 Related work 2.0 \n", - "8 Formulating explanations for graph neural netw... 3.0 \n", - "9 Experiments 4.0 \n", - "10 Conclusion 5.0 \n", - "11 Multi-instance explanations 6.0 \n", - "12 Experiments on multi-instance explanations and... 7.0 \n", - "13 Further implementation details 8.0 \n", - "14 Introduction 1.0 \n", - "15 Related Work 2.0 \n", - "16 Proposed Method 3.0 \n", - "17 Experiments 4.0 \n", - "18 Conclusion 5.0 \n", - "19 Acknowledgement 6.0 \n", - "20 Introduction 1.0 \n", - "21 Background 2.0 \n", - "22 Model Architecture 3.0 \n", - "23 Training 4.0 \n", - "24 Results 5.0 \n", - "25 Conclusion 6.0 \n", - "\n", - " appendix \n", - "0 False \n", - "1 False \n", - "2 False \n", - "3 False \n", - "4 False \n", - "5 False \n", - "6 False \n", - "7 False \n", - "8 False \n", - "9 False \n", - "10 False \n", - "11 False \n", - "12 False \n", - "13 False \n", - "14 False \n", - "15 False \n", - "16 False \n", - "17 False \n", - "18 False \n", - "19 False \n", - "20 False \n", - "21 False \n", - "22 False \n", - "23 False \n", - "24 False \n", - "25 False \n" - ] - } - ], - "source": [ - "sections_df = research_arcade.get_all_node_features(\"arxiv_sections\")\n", - "print(f\"Total sections: {sections_df}\")\n", - "print(\"\\nAll sections:\")\n", - "print(sections_df[['title', 'section_in_paper_id', 'appendix']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-section", - "metadata": {}, - "source": [ - "## 9. ArXiv Paragraphs \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paragraph_id` (INT)\n", - "- `content` (TEXT)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `paper_section` (TEXT)\n", - "- `section_id` (INT)\n", - "- `paragraph_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "1b3fd1cc", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "480dabdf", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 879.86it/s]\n", - "100%|██████████| 2/2 [00:00<00:00, 137.13it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1903.03894v4\n", - "Key to References: {'fig:explainer-motivation': 'figures_3', 'fig:explainer-intro': 'figures_4', 'fig:definition-node-features': 'figures_5', 'fig:including-node-features': 'figures_7', 'fig:subgraph_node': 'figures_8', 'fig:subgraph_graph': 'figures_9', 'fig:prototype': 'figures_12', 'fig:my_label': 'figures_11', 'fig:synth_datasets': 'table_13', 'tab:results_pr': 'table_15'}\n", - "No paper found for cho2011friendship\n", - "No paper found for you2018graph\n", - "No paper found for zitnik2018decagon\n", - "No paper found for zhang_deep_2018\n", - "No paper found for zhou_graph_2018\n", - "No paper found for graphsage\n", - "No paper found for kipf2016semi\n", - "No paper found for ying2018hierarchical\n", - "No paper found for zhang2018link\n", - "No paper found for doshi-velez_towards_2017\n", - "No paper found for lakkaraju_interpretable_2017\n", - "No paper found for ribeiro_why_2016\n", - "No paper found for schmitz_ann-dt:_1999\n", - "No paper found for chen2018learning\n", - "No paper found for Erhan2009VisualizingHF\n", - "No paper found for lundberg_unified_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for koh_understanding_2017\n", - "No paper found for DBLP:journals/corr/abs-1811-09720\n", - "No paper found for ribeiro_why_2016\n", - "No paper found for augasta_reverse_2012\n", - "No paper found for lakkaraju_interpretable_2017\n", - "No paper found for calders_deepred_2016\n", - "No paper found for Erhan2009VisualizingHF\n", - "No paper found for fleet_visualizing_2014\n", - "No paper found for chen2018learning\n", - "No paper found for shrikumar_learning_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for Kang2019explaine\n", - "No paper found for fleet_visualizing_2014\n", - "No paper found for 2018sanity\n", - "No paper found for shrikumar_learning_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for adadi_peeking_2018\n", - "No paper found for fisher_all_2018\n", - "No paper found for guidotti_survey_2018\n", - "No paper found for hooker_discovering_2004\n", - "No paper found for koh_understanding_2017\n", - "No paper found for DBLP:journals/corr/abs-1811-09720\n", - "No paper found for mutag\n", - "No paper found for duvenaud_convolutional_2015\n", - "No paper found for neil2018interpretable\n", - "No paper found for velickovic2018graph\n", - "No paper found for PhysRevLett.120.145301\n", - "No paper found for battaglia\n", - "No paper found for zhang_deep_2018\n", - "No paper found for zhou_graph_2018\n", - "No paper found for graphsage\n", - "No paper found for xu2018powerful\n", - "No paper found for kipf2016semi\n", - "No paper found for xujumping\n", - "No paper found for chen2018supervised\n", - "No paper found for kipf2016semi\n", - "No paper found for xujumping\n", - "No paper found for chen2018supervised\n", - "No paper found for mutag\n", - "No paper found for yanardag2015deep\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "No paper found for velickovic2018graph\n", - "No paper found for velickovic2018graph\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "No paper found for kumar2018community\n", - "No paper found for kumar2018community\n", - "1806.08804v4\n", - "Key to References: {'fig:assignment_vis': 'figures_157', 'tab:results': 'table_158', 'tab:results2': 'table_159'}\n", - "tab:results\n", - "tab:results2\n", - "Paper count: 2\n", - "Total nodes: 254\n", - "Total edges: 474\n", - "Paper nodes: 2\n", - "Figure nodes: 0\n", - "Table nodes: 2\n", - "Text nodes: 250\n", - "0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'CSVArxivParagraphs' object has no attribute 'db'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m config = {\u001b[33m\"\u001b[39m\u001b[33marxiv_ids\u001b[39m\u001b[33m\"\u001b[39m: [\u001b[33m\"\u001b[39m\u001b[33m1903.03894v4\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33m1806.08804v4\u001b[39m\u001b[33m\"\u001b[39m], \u001b[33m\"\u001b[39m\u001b[33mdest_dir\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33m./download\u001b[39m\u001b[33m\"\u001b[39m}\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43mresearch_arcade\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconstruct_table_from_api\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43marxiv_paragraphs\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/Research/uiuc/research-arcade/research_arcade/research_arcade.py:585\u001b[39m, in \u001b[36mResearchArcade.construct_table_from_api\u001b[39m\u001b[34m(self, table, config)\u001b[39m\n\u001b[32m 583\u001b[39m \u001b[38;5;28mself\u001b[39m.arxiv_sections.construct_sections_table_from_api(**config)\n\u001b[32m 584\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m table == \u001b[33m\"\u001b[39m\u001b[33marxiv_paragraphs\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m--> \u001b[39m\u001b[32m585\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43marxiv_paragraphs\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconstruct_paragraphs_table_from_api\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 586\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m table == \u001b[33m\"\u001b[39m\u001b[33marxiv_categories\u001b[39m\u001b[33m\"\u001b[39m:\n\u001b[32m 587\u001b[39m \u001b[38;5;28mself\u001b[39m.arxiv_categories.construct_category_table_from_api(**config)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/Research/uiuc/research-arcade/research_arcade/csv_database/csv_arxiv_paragraphs.py:297\u001b[39m, in \u001b[36mCSVArxivParagraphs.construct_paragraphs_table_from_api\u001b[39m\u001b[34m(self, arxiv_ids, dest_dir)\u001b[39m\n\u001b[32m 295\u001b[39m paragraph_cite_bib_keys = paragraph.get(\u001b[33m'\u001b[39m\u001b[33mcites\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m 296\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m bib_key \u001b[38;5;129;01min\u001b[39;00m paragraph_cite_bib_keys:\n\u001b[32m--> \u001b[39m\u001b[32m297\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mdb\u001b[49m.insert_paragraph_citations(paragraph_id=id_zero_based, paper_section=paper_section, citing_arxiv_id=paper_arxiv_id, bib_key=bib_key)\n", - "\u001b[31mAttributeError\u001b[39m: 'CSVArxivParagraphs' object has no attribute 'db'" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-insert", - "metadata": {}, - "source": [ - "### Insert Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "insert-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted paragraph 1 from Introduction\n", - "Inserted paragraph 2 from Introduction\n", - "Inserted paragraph 3 from Introduction\n", - "Inserted paragraph 4 from Introduction\n", - "Inserted paragraph 5 from Introduction\n" - ] - } - ], - "source": [ - "# Insert paragraphs from the Introduction section\n", - "paragraphs = [\n", - " {\n", - " 'paragraph_id': 1,\n", - " 'content': 'Recurrent neural networks, long short-term memory and gated recurrent neural networks in particular, have been firmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 1\n", - " },\n", - " {\n", - " 'paragraph_id': 2,\n", - " 'content': 'Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures. Recurrent models typically factor computation along the symbol positions of the input and output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 2\n", - " },\n", - " {\n", - " 'paragraph_id': 3,\n", - " 'content': 'Aligning the positions to steps in computation time, they generate a sequence of hidden states h_t, as a function of the previous hidden state h_{t-1} and the input for position t. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 3\n", - " },\n", - " {\n", - " 'paragraph_id': 4,\n", - " 'content': 'Attention mechanisms have become an integral part of compelling sequence modeling and transduction models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 4\n", - " },\n", - " {\n", - " 'paragraph_id': 5,\n", - " 'content': 'In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for significantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 5\n", - " }\n", - "]\n", - "\n", - "for paragraph in paragraphs:\n", - " research_arcade.insert_node(\"arxiv_paragraphs\", node_features=paragraph)\n", - " print(f\"Inserted paragraph {paragraph['paragraph_id']} from {paragraph['paper_section']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-get-all", - "metadata": {}, - "source": [ - "### Get All Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "get-all-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total paragraphs: 5\n", - "\n", - "First 3 paragraphs:\n", - " paragraph_id paper_section \\\n", - "0 1 Introduction \n", - "1 2 Introduction \n", - "2 3 Introduction \n", - "\n", - " content \n", - "0 Recurrent neural networks, long short-term mem... \n", - "1 Numerous efforts have since continued to push ... \n", - "2 Aligning the positions to steps in computation... \n" - ] - } - ], - "source": [ - "paragraphs_df = research_arcade.get_all_node_features(\"arxiv_paragraphs\")\n", - "print(f\"Total paragraphs: {len(paragraphs_df)}\")\n", - "print(\"\\nFirst 3 paragraphs:\")\n", - "print(paragraphs_df[['paragraph_id', 'paper_section', 'content']].head(3))" - ] - }, - { - "cell_type": "markdown", - "id": "relationships-section", - "metadata": {}, - "source": [ - "## 10. Relationships/Edges \n", - "\n", - "This section demonstrates how to create relationships between different entities." - ] - }, - { - "cell_type": "markdown", - "id": "paper-authors-edge", - "metadata": {}, - "source": [ - "### Paper-Author Relationships (arxiv_paper_authors)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "insert-paper-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table arxiv_paper_authors not found.\n", - "Linked author ss_ashish_vaswani to paper (position 1)\n", - "Table arxiv_paper_authors not found.\n", - "Linked author ss_noam_shazeer to paper (position 2)\n", - "Table arxiv_paper_authors not found.\n", - "Linked author ss_niki_parmar to paper (position 3)\n", - "Table arxiv_paper_authors not found.\n", - "Linked author ss_jakob_uszkoreit to paper (position 4)\n", - "Table arxiv_paper_authors not found.\n", - "Linked author ss_llion_jones to paper (position 5)\n" - ] - } - ], - "source": [ - "paper_authors = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'author_id': 'ss_ashish_vaswani',\n", - " 'author_sequence': 1\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'author_id': 'ss_noam_shazeer',\n", - " 'author_sequence': 2\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'author_id': 'ss_niki_parmar',\n", - " 'author_sequence': 3\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'author_id': 'ss_jakob_uszkoreit',\n", - " 'author_sequence': 4\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'author_id': 'ss_llion_jones',\n", - " 'author_sequence': 5\n", - " }\n", - "]\n", - "\n", - "for relation in paper_authors:\n", - " research_arcade.insert_edge(\"arxiv_paper_authors\", edge_features=relation)\n", - " print(f\"Linked author {relation['author_id']} to paper (position {relation['author_sequence']})\")" - ] - }, - { - "cell_type": "markdown", - "id": "paper-categories-edge", - "metadata": {}, - "source": [ - "### Paper-Category Relationships (arxiv_paper_category)" - ] - }, - { - "cell_type": "markdown", - "id": "conclusion", - "metadata": {}, - "source": [ - "## Conclusion\n", - "\n", - "This tutorial has covered:\n", - "\n", - "1. Setting up the ResearchArcade database connection\n", - "2. Working with OpenReview data\n", - "3. CRUD operations for all ArXiv entity types:\n", - " - Papers\n", - " - Authors\n", - " - Categories\n", - " - Figures\n", - " - Tables\n", - " - Sections\n", - " - Paragraphs\n", - "4. Creating relationships between entities:\n", - " - Authorship\n", - " - Citations\n", - " - Paper-Category links\n", - " - Paper-Figure/Table links\n", - " - Paragraph-level references\n", - "5. Advanced querying patterns\n", - "6. Best practices for data validation\n", - "\n", - "For more information, refer to the ResearchArcade documentation." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "research_arcade", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/research_arcade_complete_tutorial_integrated.ipynb b/examples/research_arcade_complete_tutorial_integrated.ipynb deleted file mode 100644 index 732abdb..0000000 --- a/examples/research_arcade_complete_tutorial_integrated.ipynb +++ /dev/null @@ -1,2389 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "header", - "metadata": {}, - "source": [ - "# ResearchArcade Complete Tutorial\n", - "\n", - "This tutorial demonstrates how to work with the ResearchArcade database, covering all node types and edge relationships.\n", - "\n", - "## Table of Contents\n", - "1. [Setup](#setup)\n", - "2. [OpenReview Data](#openreview)\n", - "3. [ArXiv Papers](#arxiv-papers)\n", - "4. [ArXiv Authors](#arxiv-authors)\n", - "5. [ArXiv Categories](#arxiv-categories)\n", - "6. [ArXiv Figures](#arxiv-figures)\n", - "7. [ArXiv Tables](#arxiv-tables)\n", - "8. [ArXiv Sections](#arxiv-sections)\n", - "9. [ArXiv Paragraphs](#arxiv-paragraphs)\n", - "10. [Relationships/Edges](#relationships)\n", - "11. [Advanced Queries](#advanced-queries)" - ] - }, - { - "cell_type": "markdown", - "id": "setup-section", - "metadata": {}, - "source": [ - "## 1. Setup " - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "id": "05b9b352", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "import os\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..')))\n", - "from research_arcade.research_arcade import ResearchArcade\n", - "import pandas as pd\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "markdown", - "id": "c4f1a1e4", - "metadata": {}, - "source": [ - "### Choose Database Backend" - ] - }, - { - "cell_type": "markdown", - "id": "c9672a27", - "metadata": {}, - "source": [ - "#### CSV Based" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "169f7a6d", - "metadata": {}, - "outputs": [], - "source": [ - "db_type = \"csv\"\n", - "config = {\n", - " \"csv_dir\": \"../data/my_research_arcade_data/\"\n", - "}\n", - "\n", - "research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-section", - "metadata": {}, - "source": [ - "## 3. ArXiv Papers \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `arxiv_id` (VARCHAR, unique) - e.g., 1802.08773v3\n", - "- `base_arxiv_id` (VARCHAR) - e.g., 1802.08773\n", - "- `version` (INT) - e.g., 3\n", - "- `title` (TEXT)\n", - "- `abstract` (TEXT)\n", - "- `submit_date` (DATE)\n", - "- `metadata` (JSONB)" - ] - }, - { - "cell_type": "markdown", - "id": "91b8e215", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ccaeefb3", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1806.08804v4\", \"1903.03894v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-insert", - "metadata": {}, - "source": [ - "### Insert a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "insert-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 1: Insert the famous \"Attention is All You Need\" paper\n", - "new_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'base_arxiv_id': '1706.03762',\n", - " 'version': 7,\n", - " 'title': 'Attention Is All You Need',\n", - " 'abstract': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.',\n", - " 'submit_date': '2017-06-12',\n", - " 'metadata': {'venue': 'NeurIPS 2017', 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf'}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)\n", - "print(\"Paper inserted successfully!\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "insert-paper-bert", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BERT paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 2: Insert BERT paper\n", - "bert_paper = {\n", - " 'arxiv_id': '1810.04805v2',\n", - " 'base_arxiv_id': '1810.04805',\n", - " 'version': 2,\n", - " 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding',\n", - " 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.',\n", - " 'submit_date': '2018-10-11',\n", - " 'metadata': {'venue': 'NAACL 2019', 'citations': 50000}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=bert_paper)\n", - "print(\"BERT paper inserted successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-all", - "metadata": {}, - "source": [ - "### Get All Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "get-all-papers", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total papers in database: 6\n", - "\n", - "First 5 papers:\n", - " id arxiv_id base_arxiv_id version \\\n", - "0 2 1810.04805v2 1810.04805 2 \n", - "1 3 1409.0473v7 1409.04730 7 \n", - "2 4 1512.03385v1 1512.03385 1 \n", - "3 5 1806.08804v4 1806.08804 4 \n", - "4 6 1903.03894v4 1903.03894 4 \n", - "\n", - " title \\\n", - "0 BERT: Pre-training of Deep Bidirectional Trans... \n", - "1 Neural Machine Translation by Jointly Learning... \n", - "2 Deep Residual Learning for Image Recognition \n", - "3 Hierarchical Graph Representation Learning wit... \n", - "4 GNNExplainer: Generating Explanations for Grap... \n", - "\n", - " abstract \\\n", - "0 We introduce a new language representation mod... \n", - "1 Neural machine translation is a recently propo... \n", - "2 Deeper neural networks are more difficult to t... \n", - "3 Recently, graph neural networks (GNNs) have re... \n", - "4 Graph Neural Networks (GNNs) are a powerful to... \n", - "\n", - " submit_date \\\n", - "0 2018-10-11 \n", - "1 2014-09-01 \n", - "2 2015-12-10 \n", - "3 2018-06-22 18:04:46+00:00 \n", - "4 2019-03-10 00:56:26+00:00 \n", - "\n", - " metadata \n", - "0 {\"venue\": \"NAACL 2019\", \"citations\": 50000} \n", - "1 {\"venue\": \"ICLR 2015\"} \n", - "2 {\"venue\": \"CVPR 2016\"} \n", - "3 {\"id\": \"1806.08804v4\", \"title\": \"Hierarchical ... \n", - "4 {\"id\": \"1903.03894v4\", \"title\": \"GNNExplainer:... \n" - ] - } - ], - "source": [ - "arxiv_papers_df = research_arcade.get_all_node_features(\"arxiv_papers\")\n", - "print(f\"Total papers in database: {len(arxiv_papers_df)}\")\n", - "print(\"\\nFirst 5 papers:\")\n", - "print(arxiv_papers_df.head())" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Paper by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "get-paper-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper details:\n", - "{'id': 2, 'arxiv_id': '1810.04805v2', 'base_arxiv_id': 1810.04805, 'version': 2, 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding', 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.', 'submit_date': '2018-10-11', 'metadata': '{\"venue\": \"NAACL 2019\", \"citations\": 50000}'}\n" - ] - } - ], - "source": [ - "paper_id = {\"arxiv_id\": \"1810.04805v2\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Paper details:\")\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-update", - "metadata": {}, - "source": [ - "### Update a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "update-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper updated successfully!\n" - ] - } - ], - "source": [ - "# Update metadata for a paper\n", - "updated_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'metadata': {\n", - " 'venue': 'NeurIPS 2017',\n", - " 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf',\n", - " 'citations': 75000,\n", - " 'influential': True\n", - " }\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_papers\", node_features=updated_paper)\n", - "print(\"Paper updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-delete", - "metadata": {}, - "source": [ - "### Delete a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "delete-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted paper:\n", - "True\n" - ] - } - ], - "source": [ - "# Delete a paper by ID\n", - "paper_id = {\"arxiv_id\": \"1706.03762v7\"}\n", - "deleted_paper = research_arcade.delete_node_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Deleted paper:\")\n", - "print(deleted_paper)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-section", - "metadata": {}, - "source": [ - "## 4. ArXiv Authors \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `semantic_scholar_id` (VARCHAR, unique)\n", - "- `name` (VARCHAR)\n", - "- `homepage` (VARCHAR)" - ] - }, - { - "cell_type": "markdown", - "id": "6e14ad06", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "c18c7737", - "metadata": {}, - "outputs": [], - "source": [ - "# config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "# research_arcade.construct_table_from_api(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-insert", - "metadata": {}, - "source": [ - "### Insert Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "insert-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted author: Ashish Vaswani\n", - "Inserted author: Noam Shazeer\n", - "Inserted author: Niki Parmar\n", - "Inserted author: Jakob Uszkoreit\n", - "Inserted author: Llion Jones\n" - ] - } - ], - "source": [ - "# Insert authors from the Transformer paper\n", - "authors = [\n", - " {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'name': 'Ashish Vaswani',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_noam_shazeer',\n", - " 'name': 'Noam Shazeer',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_niki_parmar',\n", - " 'name': 'Niki Parmar',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_jakob_uszkoreit',\n", - " 'name': 'Jakob Uszkoreit',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_llion_jones',\n", - " 'name': 'Llion Jones',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " }\n", - "]\n", - "\n", - "for author in authors:\n", - " research_arcade.insert_node(\"arxiv_authors\", node_features=author)\n", - " print(f\"Inserted author: {author['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-all", - "metadata": {}, - "source": [ - "### Get All Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "get-all-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total authors in database: 13\n", - "\n", - "All authors:\n", - " id semantic_scholar_id name \\\n", - "0 1 ss_ashish_vaswani Ashish Vaswani \n", - "1 2 ss_noam_shazeer Noam Shazeer \n", - "2 3 ss_niki_parmar Niki Parmar \n", - "3 4 ss_jakob_uszkoreit Jakob Uszkoreit \n", - "4 5 ss_llion_jones Llion Jones \n", - "5 6 83539859 Rex Ying \n", - "6 7 40974349 Dylan Bourgeois \n", - "7 8 145829303 Jiaxuan You \n", - "8 9 2095762 M. Zitnik \n", - "9 10 1702139 J. Leskovec \n", - "10 11 143622465 Christopher Morris \n", - "11 12 145201124 Xiang Ren \n", - "12 13 49437682 William L. Hamilton \n", - "\n", - " homepage \n", - "0 https://ashishvaswani.com \n", - "1 https://scholar.google.com/citations?user=oR9s... \n", - "2 https://scholar.google.com/citations?user=oR9s... \n", - "3 https://scholar.google.com/citations?user=oR9s... \n", - "4 https://scholar.google.com/citations?user=oR9s... \n", - "5 https://www.semanticscholar.org/author/83539859 \n", - "6 https://www.semanticscholar.org/author/40974349 \n", - "7 https://www.semanticscholar.org/author/145829303 \n", - "8 https://www.semanticscholar.org/author/2095762 \n", - "9 https://www.semanticscholar.org/author/1702139 \n", - "10 https://www.semanticscholar.org/author/143622465 \n", - "11 https://www.semanticscholar.org/author/145201124 \n", - "12 https://www.semanticscholar.org/author/49437682 \n" - ] - } - ], - "source": [ - "authors_df = research_arcade.get_all_node_features(\"arxiv_authors\")\n", - "print(f\"Total authors in database: {len(authors_df)}\")\n", - "print(\"\\nAll authors:\")\n", - "print(authors_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Author by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "get-author-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author details:\n", - "None\n" - ] - } - ], - "source": [ - "author_id = {\"semantic_scholar_id\": \"ss_ashish_vaswani\"}\n", - "author_features = research_arcade.get_node_features_by_id(\"arxiv_authors\", author_id)\n", - "print(\"Author details:\")\n", - "print(author_features)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-update", - "metadata": {}, - "source": [ - "### Update an Author" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "update-author", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author updated successfully!\n" - ] - } - ], - "source": [ - "updated_author = {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'homepage': 'https://ashishvaswani.com'\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_authors\", node_features=updated_author)\n", - "print(\"Author updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-section", - "metadata": {}, - "source": [ - "## 5. ArXiv Categories \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `name` (VARCHAR, unique)\n", - "- `description` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "3e9eeea6", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "168633f2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': '1903.03894v4', 'title': 'GNNExplainer: Generating Explanations for Graph Neural Networks', 'abstract': \"Graph Neural Networks (GNNs) are a powerful tool for machine learning on\\ngraphs.GNNs combine node feature information with the graph structure by\\nrecursively passing neural messages along edges of the input graph. However,\\nincorporating both graph structure and feature information leads to complex\\nmodels, and explaining predictions made by GNNs remains unsolved. Here we\\npropose GNNExplainer, the first general, model-agnostic approach for providing\\ninterpretable explanations for predictions of any GNN-based model on any\\ngraph-based machine learning task. Given an instance, GNNExplainer identifies a\\ncompact subgraph structure and a small subset of node features that have a\\ncrucial role in GNN's prediction. Further, GNNExplainer can generate consistent\\nand concise explanations for an entire class of instances. We formulate\\nGNNExplainer as an optimization task that maximizes the mutual information\\nbetween a GNN's prediction and distribution of possible subgraph structures.\\nExperiments on synthetic and real-world graphs show that our approach can\\nidentify important graph structures as well as node features, and outperforms\\nbaselines by 17.1% on average. GNNExplainer provides a variety of benefits,\\nfrom the ability to visualize semantically relevant structures to\\ninterpretability, to giving insights into errors of faulty GNNs.\", 'authors': ['Rex Ying', 'Dylan Bourgeois', 'Jiaxuan You', 'Marinka Zitnik', 'Jure Leskovec'], 'published': '2019-03-10 00:56:26+00:00', 'categories': ['cs.LG', 'stat.ML'], 'url': 'http://arxiv.org/abs/1903.03894v4'}\n", - "{'id': '1806.08804v4', 'title': 'Hierarchical Graph Representation Learning with Differentiable Pooling', 'abstract': 'Recently, graph neural networks (GNNs) have revolutionized the field of graph\\nrepresentation learning through effectively learned node embeddings, and\\nachieved state-of-the-art results in tasks such as node classification and link\\nprediction. However, current GNN methods are inherently flat and do not learn\\nhierarchical representations of graphs---a limitation that is especially\\nproblematic for the task of graph classification, where the goal is to predict\\nthe label associated with an entire graph. Here we propose DiffPool, a\\ndifferentiable graph pooling module that can generate hierarchical\\nrepresentations of graphs and can be combined with various graph neural network\\narchitectures in an end-to-end fashion. DiffPool learns a differentiable soft\\ncluster assignment for nodes at each layer of a deep GNN, mapping nodes to a\\nset of clusters, which then form the coarsened input for the next GNN layer.\\nOur experimental results show that combining existing GNN methods with DiffPool\\nyields an average improvement of 5-10% accuracy on graph classification\\nbenchmarks, compared to all existing pooling approaches, achieving a new\\nstate-of-the-art on four out of five benchmark data sets.', 'authors': ['Rex Ying', 'Jiaxuan You', 'Christopher Morris', 'Xiang Ren', 'William L. Hamilton', 'Jure Leskovec'], 'published': '2018-06-22 18:04:46+00:00', 'categories': ['cs.LG', 'cs.NE', 'cs.SI', 'stat.ML'], 'url': 'http://arxiv.org/abs/1806.08804v4'}\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-insert", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "insert-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-get-all", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "get-all-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 7\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.CL Computation and Language (Natural Language Pro...\n", - "1 2 cs.LG Machine Learning\n", - "2 3 cs.AI Artificial Intelligence\n", - "3 4 cs.CV Computer Vision and Pattern Recognition\n", - "4 5 stat.ML Machine Learning (Statistics)\n", - "5 6 cs.NE NaN\n", - "6 7 cs.SI NaN\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-section", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-insert", - "metadata": {}, - "source": [ - "### Insert Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "insert-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted Figure 1\n", - "Inserted Figure 2\n", - "Inserted Figure 3\n" - ] - } - ], - "source": [ - "# Insert figures for the Transformer paper\n", - "figures = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/transformer_architecture.png',\n", - " 'caption': 'The Transformer model architecture. The left side shows the encoder stack and the right side shows the decoder stack.',\n", - " 'label': 'fig:architecture',\n", - " 'name': 'Figure 1'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/scaled_dot_product_attention.png',\n", - " 'caption': 'Scaled Dot-Product Attention and Multi-Head Attention mechanisms.',\n", - " 'label': 'fig:attention',\n", - " 'name': 'Figure 2'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/positional_encoding.png',\n", - " 'caption': 'Positional encoding visualization showing sine and cosine functions of different frequencies.',\n", - " 'label': 'fig:positional',\n", - " 'name': 'Figure 3'\n", - " }\n", - "]\n", - "\n", - "for figure in figures:\n", - " research_arcade.insert_node(\"arxiv_figures\", node_features=figure)\n", - " print(f\"Inserted {figure['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-get-all", - "metadata": {}, - "source": [ - "### Get All Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "get-all-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total figures: 135\n", - "\n", - "All figures:\n", - " name caption \\\n", - "0 Figure 1 The Transformer model architecture. The left s... \n", - "1 Figure 2 Scaled Dot-Product Attention and Multi-Head At... \n", - "2 Figure 3 Positional encoding visualization showing sine... \n", - "3 NaN \\caption{\\gnn computation graph $G_c$ for maki... \n", - "4 NaN \\caption{\\name provides interpretable explanat... \n", - ".. ... ... \n", - "130 NaN \\caption{} \n", - "131 NaN \\caption{} \n", - "132 NaN \\caption{} \n", - "133 NaN \\caption{} \n", - "134 NaN \\caption{} \n", - "\n", - " label \n", - "0 fig:architecture \n", - "1 fig:attention \n", - "2 fig:positional \n", - "3 \\label{fig:explainer-motivation} \n", - "4 \\label{fig:explainer-intro} \n", - ".. ... \n", - "130 NaN \n", - "131 NaN \n", - "132 NaN \n", - "133 NaN \n", - "134 NaN \n", - "\n", - "[135 rows x 3 columns]\n" - ] - } - ], - "source": [ - "figures_df = research_arcade.get_all_node_features(\"arxiv_figures\")\n", - "print(f\"Total figures: {len(figures_df)}\")\n", - "print(\"\\nAll figures:\")\n", - "print(figures_df[['name', 'caption', 'label']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-section", - "metadata": {}, - "source": [ - "## 7. ArXiv Tables \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `table_text` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "c240602b", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "54a13d98", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "b7002162", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "b809fdd9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3771de83", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "1f1357fc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 7\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.CL Computation and Language (Natural Language Pro...\n", - "1 2 cs.LG Machine Learning\n", - "2 3 cs.AI Artificial Intelligence\n", - "3 4 cs.CV Computer Vision and Pattern Recognition\n", - "4 5 stat.ML Machine Learning (Statistics)\n", - "5 6 cs.NE NaN\n", - "6 7 cs.SI NaN\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "827d6714", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "1a7d33eb", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "195e218d", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-insert", - "metadata": {}, - "source": [ - "### Insert Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "insert-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted table: tab:variations\n", - "Inserted table: tab:wmt\n", - "Inserted table: tab:parsing\n" - ] - } - ], - "source": [ - "# Insert tables for the Transformer paper\n", - "tables = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/model_variations.tex',\n", - " 'caption': 'Variations on the Transformer architecture with different hyperparameters.',\n", - " 'label': 'tab:variations',\n", - " 'table_text': 'Model | N | d_model | d_ff | h | d_k | d_v | P_drop | train time\\nbase | 6 | 512 | 2048 | 8 | 64 | 64 | 0.1 | 12 hrs'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/wmt_results.tex',\n", - " 'caption': 'Performance of the Transformer on WMT 2014 English-German and English-French translation tasks.',\n", - " 'label': 'tab:wmt',\n", - " 'table_text': 'Model | EN-DE BLEU | EN-FR BLEU\\nTransformer (base) | 27.3 | 38.1\\nTransformer (big) | 28.4 | 41.8'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/parsing_results.tex',\n", - " 'caption': 'English constituency parsing results on WSJ test set.',\n", - " 'label': 'tab:parsing',\n", - " 'table_text': 'Model | WSJ 23 F1\\nTransformer | 91.3'\n", - " }\n", - "]\n", - "\n", - "for table in tables:\n", - " research_arcade.insert_node(\"arxiv_tables\", node_features=table)\n", - " print(f\"Inserted table: {table['label']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-get-all", - "metadata": {}, - "source": [ - "### Get All Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "get-all-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total tables: 59\n", - "\n", - "All tables:\n", - " label \\\n", - "0 tab:variations \n", - "1 tab:wmt \n", - "2 tab:parsing \n", - "3 \\label{fig:synth_datasets} \n", - "4 \\label{tab:results_pr} \n", - "5 \\label{tab:results_pr} \n", - "6 \\label{tab:results} \n", - "7 \\label{tab:results2} \n", - "8 tab:variations \n", - "9 tab:wmt \n", - "10 tab:parsing \n", - "11 \\label{fig:synth_datasets} \n", - "12 \\label{tab:results_pr} \n", - "13 \\label{tab:results_pr} \n", - "14 \\label{tab:results} \n", - "15 \\label{tab:results2} \n", - "16 tab:variations \n", - "17 tab:wmt \n", - "18 tab:parsing \n", - "19 \\label{fig:synth_datasets} \n", - "20 \\label{tab:results_pr} \n", - "21 \\label{tab:results_pr} \n", - "22 \\label{tab:results} \n", - "23 \\label{tab:results2} \n", - "24 tab:variations \n", - "25 tab:wmt \n", - "26 tab:parsing \n", - "27 \\label{fig:synth_datasets} \n", - "28 \\label{tab:results_pr} \n", - "29 \\label{tab:results_pr} \n", - "30 \\label{tab:results} \n", - "31 \\label{tab:results2} \n", - "32 tab:variations \n", - "33 tab:wmt \n", - "34 tab:parsing \n", - "35 \\label{fig:synth_datasets} \n", - "36 \\label{tab:results_pr} \n", - "37 \\label{tab:results_pr} \n", - "38 \\label{tab:results} \n", - "39 \\label{tab:results2} \n", - "40 tab:variations \n", - "41 tab:wmt \n", - "42 tab:parsing \n", - "43 \\label{fig:synth_datasets} \n", - "44 \\label{tab:results_pr} \n", - "45 \\label{tab:results_pr} \n", - "46 \\label{tab:results} \n", - "47 \\label{tab:results2} \n", - "48 tab:variations \n", - "49 tab:wmt \n", - "50 tab:parsing \n", - "51 \\label{fig:synth_datasets} \n", - "52 \\label{tab:results_pr} \n", - "53 \\label{tab:results_pr} \n", - "54 \\label{tab:results} \n", - "55 \\label{tab:results2} \n", - "56 tab:variations \n", - "57 tab:wmt \n", - "58 tab:parsing \n", - "\n", - " caption \n", - "0 Variations on the Transformer architecture wit... \n", - "1 Performance of the Transformer on WMT 2014 Eng... \n", - "2 English constituency parsing results on WSJ te... \n", - "3 \\caption{Illustration of synthetic datasets (r... \n", - "4 \\caption{\\namelong compared to baselines in id... \n", - "5 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "6 \\caption{Classification accuracies in percent.... \n", - "7 \\caption{Accuracy results of applying \\name to... \n", - "8 Variations on the Transformer architecture wit... \n", - "9 Performance of the Transformer on WMT 2014 Eng... \n", - "10 English constituency parsing results on WSJ te... \n", - "11 \\caption{Illustration of synthetic datasets (r... \n", - "12 \\caption{\\namelong compared to baselines in id... \n", - "13 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "14 \\caption{Classification accuracies in percent.... \n", - "15 \\caption{Accuracy results of applying \\name to... \n", - "16 Variations on the Transformer architecture wit... \n", - "17 Performance of the Transformer on WMT 2014 Eng... \n", - "18 English constituency parsing results on WSJ te... \n", - "19 \\caption{Illustration of synthetic datasets (r... \n", - "20 \\caption{\\namelong compared to baselines in id... \n", - "21 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "22 \\caption{Classification accuracies in percent.... \n", - "23 \\caption{Accuracy results of applying \\name to... \n", - "24 Variations on the Transformer architecture wit... \n", - "25 Performance of the Transformer on WMT 2014 Eng... \n", - "26 English constituency parsing results on WSJ te... \n", - "27 \\caption{Illustration of synthetic datasets (r... \n", - "28 \\caption{\\namelong compared to baselines in id... \n", - "29 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "30 \\caption{Classification accuracies in percent.... \n", - "31 \\caption{Accuracy results of applying \\name to... \n", - "32 Variations on the Transformer architecture wit... \n", - "33 Performance of the Transformer on WMT 2014 Eng... \n", - "34 English constituency parsing results on WSJ te... \n", - "35 \\caption{Illustration of synthetic datasets (r... \n", - "36 \\caption{\\namelong compared to baselines in id... \n", - "37 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "38 \\caption{Classification accuracies in percent.... \n", - "39 \\caption{Accuracy results of applying \\name to... \n", - "40 Variations on the Transformer architecture wit... \n", - "41 Performance of the Transformer on WMT 2014 Eng... \n", - "42 English constituency parsing results on WSJ te... \n", - "43 \\caption{Illustration of synthetic datasets (r... \n", - "44 \\caption{\\namelong compared to baselines in id... \n", - "45 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "46 \\caption{Classification accuracies in percent.... \n", - "47 \\caption{Accuracy results of applying \\name to... \n", - "48 Variations on the Transformer architecture wit... \n", - "49 Performance of the Transformer on WMT 2014 Eng... \n", - "50 English constituency parsing results on WSJ te... \n", - "51 \\caption{Illustration of synthetic datasets (r... \n", - "52 \\caption{\\namelong compared to baselines in id... \n", - "53 \\caption{\\namelong compared to \\textsc{Grad} b... \n", - "54 \\caption{Classification accuracies in percent.... \n", - "55 \\caption{Accuracy results of applying \\name to... \n", - "56 Variations on the Transformer architecture wit... \n", - "57 Performance of the Transformer on WMT 2014 Eng... \n", - "58 English constituency parsing results on WSJ te... \n" - ] - } - ], - "source": [ - "tables_df = research_arcade.get_all_node_features(\"arxiv_tables\")\n", - "print(f\"Total tables: {len(tables_df)}\")\n", - "print(\"\\nAll tables:\")\n", - "print(tables_df[['label', 'caption']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-section", - "metadata": {}, - "source": [ - "## 8. ArXiv Sections \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `content` (TEXT)\n", - "- `title` (TEXT)\n", - "- `appendix` (BOOLEAN)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `section_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "d9890560", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "c1735fc6", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-insert", - "metadata": {}, - "source": [ - "### Insert Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "insert-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted section: Introduction\n", - "Inserted section: Background\n", - "Inserted section: Model Architecture\n", - "Inserted section: Training\n", - "Inserted section: Results\n", - "Inserted section: Conclusion\n" - ] - } - ], - "source": [ - "# Insert sections for the Transformer paper\n", - "sections = [\n", - " {\n", - " 'content': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder...',\n", - " 'title': 'Introduction',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 1\n", - " },\n", - " {\n", - " 'content': 'Most competitive neural sequence transduction models have an encoder-decoder structure. Here, the encoder maps an input sequence of symbol representations...',\n", - " 'title': 'Background',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 2\n", - " },\n", - " {\n", - " 'content': 'Most neural sequence transduction models have an encoder-decoder structure. The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers...',\n", - " 'title': 'Model Architecture',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 3\n", - " },\n", - " {\n", - " 'content': 'In this section we describe the training regime for our models...',\n", - " 'title': 'Training',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 4\n", - " },\n", - " {\n", - " 'content': 'On the WMT 2014 English-to-German translation task, the big transformer model outperforms the best previously reported models...',\n", - " 'title': 'Results',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 5\n", - " },\n", - " {\n", - " 'content': 'In this work, we presented the Transformer, the first sequence transduction model based entirely on attention, replacing the recurrent layers...',\n", - " 'title': 'Conclusion',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 6\n", - " }\n", - "]\n", - "\n", - "for section in sections:\n", - " research_arcade.insert_node(\"arxiv_sections\", node_features=section)\n", - " print(f\"Inserted section: {section['title']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-get-all", - "metadata": {}, - "source": [ - "### Get All Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "get-all-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total sections: id content \\\n", - "0 1 The dominant sequence transduction models are ... \n", - "1 2 Most competitive neural sequence transduction ... \n", - "2 3 Most neural sequence transduction models have ... \n", - "3 4 In this section we describe the training regim... \n", - "4 5 On the WMT 2014 English-to-German translation ... \n", - ".. ... ... \n", - "141 142 Most competitive neural sequence transduction ... \n", - "142 143 Most neural sequence transduction models have ... \n", - "143 144 In this section we describe the training regim... \n", - "144 145 On the WMT 2014 English-to-German translation ... \n", - "145 146 In this work, we presented the Transformer, th... \n", - "\n", - " title appendix paper_arxiv_id section_in_paper_id \n", - "0 Introduction False 1706.03762v7 1.0 \n", - "1 Background False 1706.03762v7 2.0 \n", - "2 Model Architecture False 1706.03762v7 3.0 \n", - "3 Training False 1706.03762v7 4.0 \n", - "4 Results False 1706.03762v7 5.0 \n", - ".. ... ... ... ... \n", - "141 Background False 1706.03762v7 2.0 \n", - "142 Model Architecture False 1706.03762v7 3.0 \n", - "143 Training False 1706.03762v7 4.0 \n", - "144 Results False 1706.03762v7 5.0 \n", - "145 Conclusion False 1706.03762v7 6.0 \n", - "\n", - "[146 rows x 6 columns]\n", - "\n", - "All sections:\n", - " title section_in_paper_id appendix\n", - "0 Introduction 1.0 False\n", - "1 Background 2.0 False\n", - "2 Model Architecture 3.0 False\n", - "3 Training 4.0 False\n", - "4 Results 5.0 False\n", - ".. ... ... ...\n", - "141 Background 2.0 False\n", - "142 Model Architecture 3.0 False\n", - "143 Training 4.0 False\n", - "144 Results 5.0 False\n", - "145 Conclusion 6.0 False\n", - "\n", - "[146 rows x 3 columns]\n" - ] - } - ], - "source": [ - "sections_df = research_arcade.get_all_node_features(\"arxiv_sections\")\n", - "print(f\"Total sections: {sections_df}\")\n", - "print(\"\\nAll sections:\")\n", - "print(sections_df[['title', 'section_in_paper_id', 'appendix']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-section", - "metadata": {}, - "source": [ - "## 9. ArXiv Paragraphs \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paragraph_id` (INT)\n", - "- `content` (TEXT)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `paper_section` (TEXT)\n", - "- `section_id` (INT)\n", - "- `paragraph_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "1b3fd1cc", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "480dabdf", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 430.47it/s]\n", - "100%|██████████| 2/2 [00:00<00:00, 146.23it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1903.03894v4\n", - "Key to References: {'fig:explainer-motivation': 'figures_3', 'fig:explainer-intro': 'figures_4', 'fig:definition-node-features': 'figures_5', 'fig:including-node-features': 'figures_7', 'fig:subgraph_node': 'figures_8', 'fig:subgraph_graph': 'figures_9', 'fig:prototype': 'figures_12', 'fig:my_label': 'figures_11', 'fig:synth_datasets': 'table_13', 'tab:results_pr': 'table_15'}\n", - "No paper found for cho2011friendship\n", - "No paper found for you2018graph\n", - "No paper found for zitnik2018decagon\n", - "No paper found for zhang_deep_2018\n", - "No paper found for zhou_graph_2018\n", - "No paper found for graphsage\n", - "No paper found for kipf2016semi\n", - "No paper found for ying2018hierarchical\n", - "No paper found for zhang2018link\n", - "No paper found for doshi-velez_towards_2017\n", - "No paper found for lakkaraju_interpretable_2017\n", - "No paper found for ribeiro_why_2016\n", - "No paper found for schmitz_ann-dt:_1999\n", - "No paper found for chen2018learning\n", - "No paper found for Erhan2009VisualizingHF\n", - "No paper found for lundberg_unified_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for koh_understanding_2017\n", - "No paper found for DBLP:journals/corr/abs-1811-09720\n", - "No paper found for ribeiro_why_2016\n", - "No paper found for augasta_reverse_2012\n", - "No paper found for lakkaraju_interpretable_2017\n", - "No paper found for calders_deepred_2016\n", - "No paper found for Erhan2009VisualizingHF\n", - "No paper found for fleet_visualizing_2014\n", - "No paper found for chen2018learning\n", - "No paper found for shrikumar_learning_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for Kang2019explaine\n", - "No paper found for fleet_visualizing_2014\n", - "No paper found for 2018sanity\n", - "No paper found for shrikumar_learning_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for adadi_peeking_2018\n", - "No paper found for fisher_all_2018\n", - "No paper found for guidotti_survey_2018\n", - "No paper found for hooker_discovering_2004\n", - "No paper found for koh_understanding_2017\n", - "No paper found for DBLP:journals/corr/abs-1811-09720\n", - "No paper found for mutag\n", - "No paper found for duvenaud_convolutional_2015\n", - "No paper found for neil2018interpretable\n", - "No paper found for velickovic2018graph\n", - "No paper found for PhysRevLett.120.145301\n", - "No paper found for battaglia\n", - "No paper found for zhang_deep_2018\n", - "No paper found for zhou_graph_2018\n", - "No paper found for graphsage\n", - "No paper found for xu2018powerful\n", - "No paper found for kipf2016semi\n", - "No paper found for xujumping\n", - "No paper found for chen2018supervised\n", - "No paper found for kipf2016semi\n", - "No paper found for xujumping\n", - "No paper found for chen2018supervised\n", - "No paper found for mutag\n", - "No paper found for yanardag2015deep\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "No paper found for velickovic2018graph\n", - "No paper found for velickovic2018graph\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "No paper found for kumar2018community\n", - "No paper found for kumar2018community\n", - "1806.08804v4\n", - "Key to References: {'fig:assignment_vis': 'figures_157', 'tab:results': 'table_158', 'tab:results2': 'table_159'}\n", - "tab:results\n", - "tab:results2\n", - "Paper count: 2\n", - "Total nodes: 254\n", - "Total edges: 474\n", - "Paper nodes: 2\n", - "Figure nodes: 0\n", - "Table nodes: 2\n", - "Text nodes: 250\n", - "0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-insert", - "metadata": {}, - "source": [ - "### Insert Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "insert-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted paragraph 1 from Introduction\n", - "Inserted paragraph 2 from Introduction\n", - "Inserted paragraph 3 from Introduction\n", - "Inserted paragraph 4 from Introduction\n", - "Inserted paragraph 5 from Introduction\n" - ] - } - ], - "source": [ - "# Insert paragraphs from the Introduction section\n", - "paragraphs = [\n", - " {\n", - " 'paragraph_id': 1,\n", - " 'content': 'Recurrent neural networks, long short-term memory and gated recurrent neural networks in particular, have been firmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 1\n", - " },\n", - " {\n", - " 'paragraph_id': 2,\n", - " 'content': 'Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures. Recurrent models typically factor computation along the symbol positions of the input and output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 2\n", - " },\n", - " {\n", - " 'paragraph_id': 3,\n", - " 'content': 'Aligning the positions to steps in computation time, they generate a sequence of hidden states h_t, as a function of the previous hidden state h_{t-1} and the input for position t. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 3\n", - " },\n", - " {\n", - " 'paragraph_id': 4,\n", - " 'content': 'Attention mechanisms have become an integral part of compelling sequence modeling and transduction models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 4\n", - " },\n", - " {\n", - " 'paragraph_id': 5,\n", - " 'content': 'In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for significantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 5\n", - " }\n", - "]\n", - "\n", - "for paragraph in paragraphs:\n", - " research_arcade.insert_node(\"arxiv_paragraphs\", node_features=paragraph)\n", - " print(f\"Inserted paragraph {paragraph['paragraph_id']} from {paragraph['paper_section']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-get-all", - "metadata": {}, - "source": [ - "### Get All Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "get-all-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total paragraphs: 255\n", - "\n", - "First 3 paragraphs:\n", - " paragraph_id paper_section \\\n", - "0 1 Introduction \n", - "1 2 Introduction \n", - "2 3 Introduction \n", - "\n", - " content \n", - "0 Recurrent neural networks, long short-term mem... \n", - "1 Numerous efforts have since continued to push ... \n", - "2 Aligning the positions to steps in computation... \n" - ] - } - ], - "source": [ - "paragraphs_df = research_arcade.get_all_node_features(\"arxiv_paragraphs\")\n", - "print(f\"Total paragraphs: {len(paragraphs_df)}\")\n", - "print(\"\\nFirst 3 paragraphs:\")\n", - "print(paragraphs_df[['paragraph_id', 'paper_section', 'content']].head(3))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 10. Relationships/Edges \n", - "\n", - "This section demonstrates how to create and manage relationships between different entities." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.2 ArXiv Citations (arxiv_citation)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Citation" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Citation created!\n" - ] - } - ], - "source": [ - "citation = {\n", - " 'citing_arxiv_id': '1810.04805v2',\n", - " 'cited_arxiv_id': '1706.03762v7',\n", - " 'bib_title': 'attention is all you need',\n", - " 'bib_key': 'something',\n", - " 'citing_sections': 'citing_sections',\n", - "}\n", - "research_arcade.insert_edge(\"arxiv_citation\", edge_features=citation)\n", - "print(\"Citation created!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Citations" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total citations: 1\n", - " id citing_arxiv_id cited_arxiv_id bib_title bib_key \\\n", - "0 1 1810.04805v2 1706.03762v7 attention is all you need something \n", - "\n", - " author_cited_paper citing_sections citing_paragraphs \n", - "0 NaN \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "all_citations = research_arcade.get_all_edge_features(\"arxiv_citation\")\n", - "print(f\"Total citations: {len(all_citations)}\")\n", - "print(all_citations.head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Cited Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers cited:\n", - " id citing_arxiv_id cited_arxiv_id bib_title bib_key \\\n", - "0 1 1810.04805v2 1706.03762v7 attention is all you need something \n", - "\n", - " author_cited_paper citing_sections citing_paragraphs \n", - "0 NaN \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "citing_paper = {'citing_paper_id': '1810.04805v2'}\n", - "cited_papers = research_arcade.get_neighborhood(\"arxiv_citation\", primary_key=citing_paper)\n", - "print(\"Papers cited:\")\n", - "print(cited_papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Citing Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers that cite:\n", - " id citing_arxiv_id cited_arxiv_id bib_title bib_key \\\n", - "0 1 1810.04805v2 1706.03762v7 attention is all you need something \n", - "\n", - " author_cited_paper citing_sections citing_paragraphs \n", - "0 NaN \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "cited_paper = {'cited_paper_id': '1706.03762v7'}\n", - "citing_papers = research_arcade.get_neighborhood(\"arxiv_citation\", primary_key=cited_paper)\n", - "print(\"Papers that cite:\")\n", - "print(citing_papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Citation" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted citation: 1810.04805v2 -> 1706.03762v7\n", - "Citation deleted!\n" - ] - } - ], - "source": [ - "citation_id = {\n", - " 'citing_paper_id': '1810.04805v2',\n", - " 'cited_paper_id': '1706.03762v7'\n", - "}\n", - "research_arcade.delete_edge_by_id(\"arxiv_citation\", primary_key=citation_id)\n", - "print(\"Citation deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.3 ArXiv Paper-Author (arxiv_paper_author)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Author Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked author ss_ashish_vaswani (position 1)\n", - "Linked author ss_noam_shazeer (position 2)\n", - "Linked author ss_niki_parmar (position 3)\n" - ] - } - ], - "source": [ - "paper_authors = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani', 'author_sequence': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_noam_shazeer', 'author_sequence': 2},\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_niki_parmar', 'author_sequence': 3}\n", - "]\n", - "for relation in paper_authors:\n", - " research_arcade.insert_edge(\"arxiv_paper_author\", edge_features=relation)\n", - " print(f\"Linked author {relation['author_id']} (position {relation['author_sequence']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Paper-Author Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total relationships: 3\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_ashish_vaswani 1\n", - "1 1706.03762v7 ss_noam_shazeer 2\n", - "2 1706.03762v7 ss_niki_parmar 3\n" - ] - } - ], - "source": [ - "all_relations = research_arcade.get_all_edge_features(\"arxiv_paper_author\")\n", - "print(f\"Total relationships: {len(all_relations)}\")\n", - "print(all_relations.head(10))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Authors for a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Authors:\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_ashish_vaswani 1\n", - "1 1706.03762v7 ss_noam_shazeer 2\n", - "2 1706.03762v7 ss_niki_parmar 3\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "authors = research_arcade.get_neighborhood(\"arxiv_paper_author\", primary_key=paper_id)\n", - "print(\"Authors:\")\n", - "print(authors.sort_values('author_sequence'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Papers by Author" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers by author:\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_ashish_vaswani 1\n" - ] - } - ], - "source": [ - "author_id = {'author_id': 'ss_ashish_vaswani'}\n", - "papers = research_arcade.get_neighborhood(\"arxiv_paper_author\", primary_key=author_id)\n", - "print(\"Papers by author:\")\n", - "print(papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Paper-Author Link" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Relationship deleted!\n" - ] - } - ], - "source": [ - "relation_id = {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani'}\n", - "research_arcade.delete_edge_by_id(\"arxiv_paper_author\", primary_key=relation_id)\n", - "print(\"Relationship deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.4 ArXiv Paper-Category (arxiv_paper_category)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Category Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked 1\n", - "Linked 1\n", - "Linked 2\n" - ] - } - ], - "source": [ - "paper_categories = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '1'},\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '1'},\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '2'}\n", - "]\n", - "for relation in paper_categories:\n", - " research_arcade.insert_edge(\"arxiv_paper_category\", edge_features=relation)\n", - " print(f\"Linked {relation['category_id']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Paper-Category Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total relationships: 15\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n", - "2 1810.04805v2 1\n", - "3 1810.04805v2 3\n", - "4 1706.03762v7 1\n" - ] - } - ], - "source": [ - "all_relations = research_arcade.get_all_edge_features(\"arxiv_paper_category\")\n", - "print(f\"Total relationships: {len(all_relations)}\")\n", - "print(all_relations.head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Categories for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Categories:\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n", - "2 1706.03762v7 1\n", - "3 1706.03762v7 1\n", - "4 1706.03762v7 1\n", - "5 1706.03762v7 1\n", - "6 1706.03762v7 2\n", - "7 1706.03762v7 1\n", - "8 1706.03762v7 1\n", - "9 1706.03762v7 2\n", - "10 1706.03762v7 1\n", - "11 1706.03762v7 1\n", - "12 1706.03762v7 2\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "categories = research_arcade.get_neighborhood(\"arxiv_paper_category\", primary_key=paper_id)\n", - "print(\"Categories:\")\n", - "print(categories)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Papers in Category" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers in category:\n", - "None\n" - ] - } - ], - "source": [ - "category_id = {'category_id': 'cs.LG'}\n", - "papers = research_arcade.get_neighborhood(\"arxiv_paper_category\", primary_key=category_id)\n", - "print(\"Papers in category:\")\n", - "print(papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Paper-Category Link" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Relationship deleted!\n" - ] - } - ], - "source": [ - "relation_id = {'paper_arxiv_id': '1706.03762v7', 'category_id': 'cs.AI'}\n", - "research_arcade.delete_edge_by_id(\"arxiv_paper_category\", primary_key=relation_id)\n", - "print(\"Relationship deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.5 ArXiv Paper-Figure (arxiv_paper_figure)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Figure Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked figure 1)\n", - "Linked figure 2)\n" - ] - } - ], - "source": [ - "paper_figures = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'figure_id': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'figure_id': 2}\n", - "]\n", - "for relation in paper_figures:\n", - " research_arcade.insert_edge(\"arxiv_paper_figure\", edge_features=relation)\n", - " print(f\"Linked figure {relation['figure_id']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Figures for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Figures:\n", - " paper_arxiv_id figure_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "figures = research_arcade.get_neighborhood(\"arxiv_paper_figure\", primary_key=paper_id)\n", - "print(\"Figures:\")\n", - "print(figures)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.6 ArXiv Paper-Table (arxiv_paper_table)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Table Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked table 1\n", - "Linked table 2\n" - ] - } - ], - "source": [ - "paper_tables = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'table_id': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'table_id': 2}\n", - "]\n", - "for relation in paper_tables:\n", - " research_arcade.insert_edge(\"arxiv_paper_table\", edge_features=relation)\n", - " print(f\"Linked table {relation['table_id']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Tables for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tables:\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "tables = research_arcade.get_neighborhood(\"arxiv_paper_table\", primary_key=paper_id)\n", - "print(\"Tables:\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.7 ArXiv Paragraph-Reference (arxiv_paragraph_reference)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paragraph-Reference Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [], - "source": [ - "paragraph_references = [\n", - " {'paragraph_id': 1, 'paper_section': 'established approaches', 'paper_arxiv_id': '1706.03762v7', 'reference_label': \"{something}\", 'reference_type': 'figure'}\n", - "]\n", - "\n", - "for relation in paragraph_references:\n", - " research_arcade.insert_edge(\"arxiv_paragraph_reference\", edge_features=relation)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get References in Paragraph" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "References:\n", - " id paragraph_id paper_section paper_arxiv_id reference_label \\\n", - "0 1 1 established approaches 1706.03762v7 {something} \n", - "1 2 1 established approaches 1706.03762v7 {something} \n", - "2 3 1 established approaches 1706.03762v7 {something} \n", - "3 4 1 established approaches 1706.03762v7 {something} \n", - "\n", - " reference_type \n", - "0 figure \n", - "1 figure \n", - "2 figure \n", - "3 figure \n" - ] - } - ], - "source": [ - "paragraph_id = {'paragraph_id': 1}\n", - "references = research_arcade.get_neighborhood(\"arxiv_paragraph_reference\", primary_key=paragraph_id)\n", - "print(\"References:\")\n", - "print(references)" - ] - }, - { - "cell_type": "markdown", - "id": "conclusion", - "metadata": {}, - "source": [ - "## Conclusion\n", - "\n", - "This tutorial has covered:\n", - "\n", - "1. Setting up the ResearchArcade database connection\n", - "2. Working with OpenReview data\n", - "3. CRUD operations for all ArXiv entity types:\n", - " - Papers\n", - " - Authors\n", - " - Categories\n", - " - Figures\n", - " - Tables\n", - " - Sections\n", - " - Paragraphs\n", - "4. Creating relationships between entities:\n", - " - Authorship\n", - " - Citations\n", - " - Paper-Category links\n", - " - Paper-Figure/Table links\n", - " - Paragraph-level references\n", - "5. Advanced querying patterns\n", - "6. Best practices for data validation\n", - "\n", - "For more information, refer to the ResearchArcade documentation." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "research_arcade", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/research_arcade_complete_tutorial_with_imports.ipynb b/examples/research_arcade_complete_tutorial_with_imports.ipynb deleted file mode 100644 index cc3b7c8..0000000 --- a/examples/research_arcade_complete_tutorial_with_imports.ipynb +++ /dev/null @@ -1,2968 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "header", - "metadata": {}, - "source": [ - "# ResearchArcade Complete Tutorial\n", - "\n", - "This tutorial demonstrates how to work with the ResearchArcade database, covering all node types and edge relationships.\n", - "\n", - "## Table of Contents\n", - "1. [Setup](#setup)\n", - "2. [OpenReview Data](#openreview)\n", - "3. [ArXiv Papers](#arxiv-papers)\n", - "4. [ArXiv Authors](#arxiv-authors)\n", - "5. [ArXiv Categories](#arxiv-categories)\n", - "6. [ArXiv Figures](#arxiv-figures)\n", - "7. [ArXiv Tables](#arxiv-tables)\n", - "8. [ArXiv Sections](#arxiv-sections)\n", - "9. [ArXiv Paragraphs](#arxiv-paragraphs)\n", - "10. [Relationships/Edges](#relationships)\n", - "11. [Advanced Queries](#advanced-queries)" - ] - }, - { - "cell_type": "markdown", - "id": "setup-section", - "metadata": {}, - "source": [ - "## 1. Setup " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "05b9b352", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "import os\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..')))\n", - "from research_arcade.research_arcade import ResearchArcade\n", - "import pandas as pd\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "markdown", - "id": "c4f1a1e4", - "metadata": {}, - "source": [ - "### Choose Database Backend" - ] - }, - { - "cell_type": "markdown", - "id": "c9672a27", - "metadata": {}, - "source": [ - "#### CSV Based" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "169f7a6d", - "metadata": {}, - "outputs": [], - "source": [ - "db_type = \"csv\"\n", - "config = {\n", - " \"csv_dir\": \"../data/my_research_arcade_data/\"\n", - "}\n", - "\n", - "research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-section", - "metadata": {}, - "source": [ - "## 3. ArXiv Papers \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `arxiv_id` (VARCHAR, unique) - e.g., 1802.08773v3\n", - "- `base_arxiv_id` (VARCHAR) - e.g., 1802.08773\n", - "- `version` (INT) - e.g., 3\n", - "- `title` (TEXT)\n", - "- `abstract` (TEXT)\n", - "- `submit_date` (DATE)\n", - "- `metadata` (JSONB)" - ] - }, - { - "cell_type": "markdown", - "id": "91b8e215", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "id": "ccaeefb3", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1806.08804v4\", \"1903.03894v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_papers_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_papers_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_papers_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_papers_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-insert", - "metadata": {}, - "source": [ - "### Insert a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "insert-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 1: Insert the famous \"Attention is All You Need\" paper\n", - "new_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'base_arxiv_id': '1706.03762',\n", - " 'version': 7,\n", - " 'title': 'Attention Is All You Need',\n", - " 'abstract': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.',\n", - " 'submit_date': '2017-06-12',\n", - " 'metadata': {'venue': 'NeurIPS 2017', 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf'}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)\n", - "print(\"Paper inserted successfully!\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "insert-paper-bert", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BERT paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 2: Insert BERT paper\n", - "bert_paper = {\n", - " 'arxiv_id': '1810.04805v2',\n", - " 'base_arxiv_id': '1810.04805',\n", - " 'version': 2,\n", - " 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding',\n", - " 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.',\n", - " 'submit_date': '2018-10-11',\n", - " 'metadata': {'venue': 'NAACL 2019', 'citations': 50000}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=bert_paper)\n", - "print(\"BERT paper inserted successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-all", - "metadata": {}, - "source": [ - "### Get All Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "get-all-papers", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total papers in database: 6\n", - "\n", - "First 5 papers:\n", - " id arxiv_id base_arxiv_id version \\\n", - "0 2 1810.04805v2 1810.04805 2 \n", - "1 3 1409.0473v7 1409.04730 7 \n", - "2 4 1512.03385v1 1512.03385 1 \n", - "3 5 1806.08804v4 1806.08804 4 \n", - "4 6 1903.03894v4 1903.03894 4 \n", - "\n", - " title \\\n", - "0 BERT: Pre-training of Deep Bidirectional Trans... \n", - "1 Neural Machine Translation by Jointly Learning... \n", - "2 Deep Residual Learning for Image Recognition \n", - "3 Hierarchical Graph Representation Learning wit... \n", - "4 GNNExplainer: Generating Explanations for Grap... \n", - "\n", - " abstract \\\n", - "0 We introduce a new language representation mod... \n", - "1 Neural machine translation is a recently propo... \n", - "2 Deeper neural networks are more difficult to t... \n", - "3 Recently, graph neural networks (GNNs) have re... \n", - "4 Graph Neural Networks (GNNs) are a powerful to... \n", - "\n", - " submit_date \\\n", - "0 2018-10-11 \n", - "1 2014-09-01 \n", - "2 2015-12-10 \n", - "3 2018-06-22 18:04:46+00:00 \n", - "4 2019-03-10 00:56:26+00:00 \n", - "\n", - " metadata \n", - "0 {\"venue\": \"NAACL 2019\", \"citations\": 50000} \n", - "1 {\"venue\": \"ICLR 2015\"} \n", - "2 {\"venue\": \"CVPR 2016\"} \n", - "3 {\"id\": \"1806.08804v4\", \"title\": \"Hierarchical ... \n", - "4 {\"id\": \"1903.03894v4\", \"title\": \"GNNExplainer:... \n" - ] - } - ], - "source": [ - "arxiv_papers_df = research_arcade.get_all_node_features(\"arxiv_papers\")\n", - "print(f\"Total papers in database: {len(arxiv_papers_df)}\")\n", - "print(\"\\nFirst 5 papers:\")\n", - "print(arxiv_papers_df.head())" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Paper by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "get-paper-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper details:\n", - "{'id': 2, 'arxiv_id': '1810.04805v2', 'base_arxiv_id': 1810.04805, 'version': 2, 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding', 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.', 'submit_date': '2018-10-11', 'metadata': '{\"venue\": \"NAACL 2019\", \"citations\": 50000}'}\n" - ] - } - ], - "source": [ - "paper_id = {\"arxiv_id\": \"1810.04805v2\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Paper details:\")\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-update", - "metadata": {}, - "source": [ - "### Update a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "update-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper updated successfully!\n" - ] - } - ], - "source": [ - "# Update metadata for a paper\n", - "updated_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'metadata': {\n", - " 'venue': 'NeurIPS 2017',\n", - " 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf',\n", - " 'citations': 75000,\n", - " 'influential': True\n", - " }\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_papers\", node_features=updated_paper)\n", - "print(\"Paper updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-delete", - "metadata": {}, - "source": [ - "### Delete a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "delete-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted paper:\n", - "True\n" - ] - } - ], - "source": [ - "# Delete a paper by ID\n", - "paper_id = {\"arxiv_id\": \"1706.03762v7\"}\n", - "deleted_paper = research_arcade.delete_node_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Deleted paper:\")\n", - "print(deleted_paper)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-section", - "metadata": {}, - "source": [ - "## 4. ArXiv Authors \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `semantic_scholar_id` (VARCHAR, unique)\n", - "- `name` (VARCHAR)\n", - "- `homepage` (VARCHAR)" - ] - }, - { - "cell_type": "markdown", - "id": "6e14ad06", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "c18c7737", - "metadata": {}, - "outputs": [], - "source": [ - "# config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "# research_arcade.construct_table_from_api(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_authors_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_authors_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_authors_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_authors_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-insert", - "metadata": {}, - "source": [ - "### Insert Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "insert-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted author: Ashish Vaswani\n", - "Inserted author: Noam Shazeer\n", - "Inserted author: Niki Parmar\n", - "Inserted author: Jakob Uszkoreit\n", - "Inserted author: Llion Jones\n" - ] - } - ], - "source": [ - "# Insert authors from the Transformer paper\n", - "authors = [\n", - " {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'name': 'Ashish Vaswani',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_noam_shazeer',\n", - " 'name': 'Noam Shazeer',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_niki_parmar',\n", - " 'name': 'Niki Parmar',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_jakob_uszkoreit',\n", - " 'name': 'Jakob Uszkoreit',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_llion_jones',\n", - " 'name': 'Llion Jones',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " }\n", - "]\n", - "\n", - "for author in authors:\n", - " research_arcade.insert_node(\"arxiv_authors\", node_features=author)\n", - " print(f\"Inserted author: {author['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-all", - "metadata": {}, - "source": [ - "### Get All Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "get-all-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total authors in database: 13\n", - "\n", - "All authors:\n", - " id semantic_scholar_id name \\\n", - "0 1 ss_ashish_vaswani Ashish Vaswani \n", - "1 2 ss_noam_shazeer Noam Shazeer \n", - "2 3 ss_niki_parmar Niki Parmar \n", - "3 4 ss_jakob_uszkoreit Jakob Uszkoreit \n", - "4 5 ss_llion_jones Llion Jones \n", - "5 6 83539859 Rex Ying \n", - "6 7 40974349 Dylan Bourgeois \n", - "7 8 145829303 Jiaxuan You \n", - "8 9 2095762 M. Zitnik \n", - "9 10 1702139 J. Leskovec \n", - "10 11 143622465 Christopher Morris \n", - "11 12 145201124 Xiang Ren \n", - "12 13 49437682 William L. Hamilton \n", - "\n", - " homepage \n", - "0 https://ashishvaswani.com \n", - "1 https://scholar.google.com/citations?user=oR9s... \n", - "2 https://scholar.google.com/citations?user=oR9s... \n", - "3 https://scholar.google.com/citations?user=oR9s... \n", - "4 https://scholar.google.com/citations?user=oR9s... \n", - "5 https://www.semanticscholar.org/author/83539859 \n", - "6 https://www.semanticscholar.org/author/40974349 \n", - "7 https://www.semanticscholar.org/author/145829303 \n", - "8 https://www.semanticscholar.org/author/2095762 \n", - "9 https://www.semanticscholar.org/author/1702139 \n", - "10 https://www.semanticscholar.org/author/143622465 \n", - "11 https://www.semanticscholar.org/author/145201124 \n", - "12 https://www.semanticscholar.org/author/49437682 \n" - ] - } - ], - "source": [ - "authors_df = research_arcade.get_all_node_features(\"arxiv_authors\")\n", - "print(f\"Total authors in database: {len(authors_df)}\")\n", - "print(\"\\nAll authors:\")\n", - "print(authors_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Author by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "get-author-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author details:\n", - "None\n" - ] - } - ], - "source": [ - "author_id = {\"semantic_scholar_id\": \"ss_ashish_vaswani\"}\n", - "author_features = research_arcade.get_node_features_by_id(\"arxiv_authors\", author_id)\n", - "print(\"Author details:\")\n", - "print(author_features)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-update", - "metadata": {}, - "source": [ - "### Update an Author" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "update-author", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author updated successfully!\n" - ] - } - ], - "source": [ - "updated_author = {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'homepage': 'https://ashishvaswani.com'\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_authors\", node_features=updated_author)\n", - "print(\"Author updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-section", - "metadata": {}, - "source": [ - "## 5. ArXiv Categories \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `name` (VARCHAR, unique)\n", - "- `description` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "3e9eeea6", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "168633f2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': '1903.03894v4', 'title': 'GNNExplainer: Generating Explanations for Graph Neural Networks', 'abstract': \"Graph Neural Networks (GNNs) are a powerful tool for machine learning on\\ngraphs.GNNs combine node feature information with the graph structure by\\nrecursively passing neural messages along edges of the input graph. However,\\nincorporating both graph structure and feature information leads to complex\\nmodels, and explaining predictions made by GNNs remains unsolved. Here we\\npropose GNNExplainer, the first general, model-agnostic approach for providing\\ninterpretable explanations for predictions of any GNN-based model on any\\ngraph-based machine learning task. Given an instance, GNNExplainer identifies a\\ncompact subgraph structure and a small subset of node features that have a\\ncrucial role in GNN's prediction. Further, GNNExplainer can generate consistent\\nand concise explanations for an entire class of instances. We formulate\\nGNNExplainer as an optimization task that maximizes the mutual information\\nbetween a GNN's prediction and distribution of possible subgraph structures.\\nExperiments on synthetic and real-world graphs show that our approach can\\nidentify important graph structures as well as node features, and outperforms\\nbaselines by 17.1% on average. GNNExplainer provides a variety of benefits,\\nfrom the ability to visualize semantically relevant structures to\\ninterpretability, to giving insights into errors of faulty GNNs.\", 'authors': ['Rex Ying', 'Dylan Bourgeois', 'Jiaxuan You', 'Marinka Zitnik', 'Jure Leskovec'], 'published': '2019-03-10 00:56:26+00:00', 'categories': ['cs.LG', 'stat.ML'], 'url': 'http://arxiv.org/abs/1903.03894v4'}\n", - "{'id': '1806.08804v4', 'title': 'Hierarchical Graph Representation Learning with Differentiable Pooling', 'abstract': 'Recently, graph neural networks (GNNs) have revolutionized the field of graph\\nrepresentation learning through effectively learned node embeddings, and\\nachieved state-of-the-art results in tasks such as node classification and link\\nprediction. However, current GNN methods are inherently flat and do not learn\\nhierarchical representations of graphs---a limitation that is especially\\nproblematic for the task of graph classification, where the goal is to predict\\nthe label associated with an entire graph. Here we propose DiffPool, a\\ndifferentiable graph pooling module that can generate hierarchical\\nrepresentations of graphs and can be combined with various graph neural network\\narchitectures in an end-to-end fashion. DiffPool learns a differentiable soft\\ncluster assignment for nodes at each layer of a deep GNN, mapping nodes to a\\nset of clusters, which then form the coarsened input for the next GNN layer.\\nOur experimental results show that combining existing GNN methods with DiffPool\\nyields an average improvement of 5-10% accuracy on graph classification\\nbenchmarks, compared to all existing pooling approaches, achieving a new\\nstate-of-the-art on four out of five benchmark data sets.', 'authors': ['Rex Ying', 'Jiaxuan You', 'Christopher Morris', 'Xiang Ren', 'William L. Hamilton', 'Jure Leskovec'], 'published': '2018-06-22 18:04:46+00:00', 'categories': ['cs.LG', 'cs.NE', 'cs.SI', 'stat.ML'], 'url': 'http://arxiv.org/abs/1806.08804v4'}\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_categories_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_categories_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_categories_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_categories_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-insert", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "insert-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-get-all", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "get-all-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 7\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.CL Computation and Language (Natural Language Pro...\n", - "1 2 cs.LG Machine Learning\n", - "2 3 cs.AI Artificial Intelligence\n", - "3 4 cs.CV Computer Vision and Pattern Recognition\n", - "4 5 stat.ML Machine Learning (Statistics)\n", - "5 6 cs.NE NaN\n", - "6 7 cs.SI NaN\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-section", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-insert", - "metadata": {}, - "source": [ - "### Insert Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "insert-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted Figure 1\n", - "Inserted Figure 2\n", - "Inserted Figure 3\n" - ] - } - ], - "source": [ - "# Insert figures for the Transformer paper\n", - "figures = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/transformer_architecture.png',\n", - " 'caption': 'The Transformer model architecture. The left side shows the encoder stack and the right side shows the decoder stack.',\n", - " 'label': 'fig:architecture',\n", - " 'name': 'Figure 1'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/scaled_dot_product_attention.png',\n", - " 'caption': 'Scaled Dot-Product Attention and Multi-Head Attention mechanisms.',\n", - " 'label': 'fig:attention',\n", - " 'name': 'Figure 2'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/positional_encoding.png',\n", - " 'caption': 'Positional encoding visualization showing sine and cosine functions of different frequencies.',\n", - " 'label': 'fig:positional',\n", - " 'name': 'Figure 3'\n", - " }\n", - "]\n", - "\n", - "for figure in figures:\n", - " research_arcade.insert_node(\"arxiv_figures\", node_features=figure)\n", - " print(f\"Inserted {figure['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-get-all", - "metadata": {}, - "source": [ - "### Get All Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "get-all-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total figures: 245\n", - "\n", - "All figures:\n", - " name caption \\\n", - "0 Figure 1 The Transformer model architecture. The left s... \n", - "1 Figure 2 Scaled Dot-Product Attention and Multi-Head At... \n", - "2 Figure 3 Positional encoding visualization showing sine... \n", - "3 NaN \\caption{\\gnn computation graph $G_c$ for maki... \n", - "4 NaN \\caption{\\name provides interpretable explanat... \n", - ".. ... ... \n", - "240 NaN \\caption{} \n", - "241 NaN \\caption{} \n", - "242 NaN \\caption{} \n", - "243 NaN \\caption{} \n", - "244 NaN \\caption{} \n", - "\n", - " label \n", - "0 fig:architecture \n", - "1 fig:attention \n", - "2 fig:positional \n", - "3 \\label{fig:explainer-motivation} \n", - "4 \\label{fig:explainer-intro} \n", - ".. ... \n", - "240 NaN \n", - "241 NaN \n", - "242 NaN \n", - "243 NaN \n", - "244 NaN \n", - "\n", - "[245 rows x 3 columns]\n" - ] - } - ], - "source": [ - "figures_df = research_arcade.get_all_node_features(\"arxiv_figures\")\n", - "print(f\"Total figures: {len(figures_df)}\")\n", - "print(\"\\nAll figures:\")\n", - "print(figures_df[['name', 'caption', 'label']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-section", - "metadata": {}, - "source": [ - "## 7. ArXiv Tables \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `table_text` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "c240602b", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "54a13d98", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_tables_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_tables_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_tables_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_tables_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "b7002162", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "b809fdd9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3771de83", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "1f1357fc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 7\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.CL Computation and Language (Natural Language Pro...\n", - "1 2 cs.LG Machine Learning\n", - "2 3 cs.AI Artificial Intelligence\n", - "3 4 cs.CV Computer Vision and Pattern Recognition\n", - "4 5 stat.ML Machine Learning (Statistics)\n", - "5 6 cs.NE NaN\n", - "6 7 cs.SI NaN\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "827d6714", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "1a7d33eb", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "195e218d", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_figures_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_figures_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_figures_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_figures_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-insert", - "metadata": {}, - "source": [ - "### Insert Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "insert-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted table: tab:variations\n", - "Inserted table: tab:wmt\n", - "Inserted table: tab:parsing\n" - ] - } - ], - "source": [ - "# Insert tables for the Transformer paper\n", - "tables = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/model_variations.tex',\n", - " 'caption': 'Variations on the Transformer architecture with different hyperparameters.',\n", - " 'label': 'tab:variations',\n", - " 'table_text': 'Model | N | d_model | d_ff | h | d_k | d_v | P_drop | train time\\nbase | 6 | 512 | 2048 | 8 | 64 | 64 | 0.1 | 12 hrs'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/wmt_results.tex',\n", - " 'caption': 'Performance of the Transformer on WMT 2014 English-German and English-French translation tasks.',\n", - " 'label': 'tab:wmt',\n", - " 'table_text': 'Model | EN-DE BLEU | EN-FR BLEU\\nTransformer (base) | 27.3 | 38.1\\nTransformer (big) | 28.4 | 41.8'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/parsing_results.tex',\n", - " 'caption': 'English constituency parsing results on WSJ test set.',\n", - " 'label': 'tab:parsing',\n", - " 'table_text': 'Model | WSJ 23 F1\\nTransformer | 91.3'\n", - " }\n", - "]\n", - "\n", - "for table in tables:\n", - " research_arcade.insert_node(\"arxiv_tables\", node_features=table)\n", - " print(f\"Inserted table: {table['label']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-get-all", - "metadata": {}, - "source": [ - "### Get All Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "get-all-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total tables: 99\n", - "\n", - "All tables:\n", - " label \\\n", - "0 tab:variations \n", - "1 tab:wmt \n", - "2 tab:parsing \n", - "3 \\label{fig:synth_datasets} \n", - "4 \\label{tab:results_pr} \n", - ".. ... \n", - "94 \\label{tab:results} \n", - "95 \\label{tab:results2} \n", - "96 tab:variations \n", - "97 tab:wmt \n", - "98 tab:parsing \n", - "\n", - " caption \n", - "0 Variations on the Transformer architecture wit... \n", - "1 Performance of the Transformer on WMT 2014 Eng... \n", - "2 English constituency parsing results on WSJ te... \n", - "3 \\caption{Illustration of synthetic datasets (r... \n", - "4 \\caption{\\namelong compared to baselines in id... \n", - ".. ... \n", - "94 \\caption{Classification accuracies in percent.... \n", - "95 \\caption{Accuracy results of applying \\name to... \n", - "96 Variations on the Transformer architecture wit... \n", - "97 Performance of the Transformer on WMT 2014 Eng... \n", - "98 English constituency parsing results on WSJ te... \n", - "\n", - "[99 rows x 2 columns]\n" - ] - } - ], - "source": [ - "tables_df = research_arcade.get_all_node_features(\"arxiv_tables\")\n", - "print(f\"Total tables: {len(tables_df)}\")\n", - "print(\"\\nAll tables:\")\n", - "print(tables_df[['label', 'caption']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-section", - "metadata": {}, - "source": [ - "## 8. ArXiv Sections \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `content` (TEXT)\n", - "- `title` (TEXT)\n", - "- `appendix` (BOOLEAN)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `section_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "d9890560", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "c1735fc6", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_sections_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_sections_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_sections_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_sections_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-insert", - "metadata": {}, - "source": [ - "### Insert Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "insert-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted section: Introduction\n", - "Inserted section: Background\n", - "Inserted section: Model Architecture\n", - "Inserted section: Training\n", - "Inserted section: Results\n", - "Inserted section: Conclusion\n" - ] - } - ], - "source": [ - "# Insert sections for the Transformer paper\n", - "sections = [\n", - " {\n", - " 'content': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder...',\n", - " 'title': 'Introduction',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 1\n", - " },\n", - " {\n", - " 'content': 'Most competitive neural sequence transduction models have an encoder-decoder structure. Here, the encoder maps an input sequence of symbol representations...',\n", - " 'title': 'Background',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 2\n", - " },\n", - " {\n", - " 'content': 'Most neural sequence transduction models have an encoder-decoder structure. The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers...',\n", - " 'title': 'Model Architecture',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 3\n", - " },\n", - " {\n", - " 'content': 'In this section we describe the training regime for our models...',\n", - " 'title': 'Training',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 4\n", - " },\n", - " {\n", - " 'content': 'On the WMT 2014 English-to-German translation task, the big transformer model outperforms the best previously reported models...',\n", - " 'title': 'Results',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 5\n", - " },\n", - " {\n", - " 'content': 'In this work, we presented the Transformer, the first sequence transduction model based entirely on attention, replacing the recurrent layers...',\n", - " 'title': 'Conclusion',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 6\n", - " }\n", - "]\n", - "\n", - "for section in sections:\n", - " research_arcade.insert_node(\"arxiv_sections\", node_features=section)\n", - " print(f\"Inserted section: {section['title']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-get-all", - "metadata": {}, - "source": [ - "### Get All Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "get-all-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total sections: id content \\\n", - "0 1 The dominant sequence transduction models are ... \n", - "1 2 Most competitive neural sequence transduction ... \n", - "2 3 Most neural sequence transduction models have ... \n", - "3 4 In this section we describe the training regim... \n", - "4 5 On the WMT 2014 English-to-German translation ... \n", - ".. ... ... \n", - "241 242 Most competitive neural sequence transduction ... \n", - "242 243 Most neural sequence transduction models have ... \n", - "243 244 In this section we describe the training regim... \n", - "244 245 On the WMT 2014 English-to-German translation ... \n", - "245 246 In this work, we presented the Transformer, th... \n", - "\n", - " title appendix paper_arxiv_id section_in_paper_id \n", - "0 Introduction False 1706.03762v7 1.0 \n", - "1 Background False 1706.03762v7 2.0 \n", - "2 Model Architecture False 1706.03762v7 3.0 \n", - "3 Training False 1706.03762v7 4.0 \n", - "4 Results False 1706.03762v7 5.0 \n", - ".. ... ... ... ... \n", - "241 Background False 1706.03762v7 2.0 \n", - "242 Model Architecture False 1706.03762v7 3.0 \n", - "243 Training False 1706.03762v7 4.0 \n", - "244 Results False 1706.03762v7 5.0 \n", - "245 Conclusion False 1706.03762v7 6.0 \n", - "\n", - "[246 rows x 6 columns]\n", - "\n", - "All sections:\n", - " title section_in_paper_id appendix\n", - "0 Introduction 1.0 False\n", - "1 Background 2.0 False\n", - "2 Model Architecture 3.0 False\n", - "3 Training 4.0 False\n", - "4 Results 5.0 False\n", - ".. ... ... ...\n", - "241 Background 2.0 False\n", - "242 Model Architecture 3.0 False\n", - "243 Training 4.0 False\n", - "244 Results 5.0 False\n", - "245 Conclusion 6.0 False\n", - "\n", - "[246 rows x 3 columns]\n" - ] - } - ], - "source": [ - "sections_df = research_arcade.get_all_node_features(\"arxiv_sections\")\n", - "print(f\"Total sections: {sections_df}\")\n", - "print(\"\\nAll sections:\")\n", - "print(sections_df[['title', 'section_in_paper_id', 'appendix']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-section", - "metadata": {}, - "source": [ - "## 9. ArXiv Paragraphs \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paragraph_id` (INT)\n", - "- `content` (TEXT)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `paper_section` (TEXT)\n", - "- `section_id` (INT)\n", - "- `paragraph_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "1b3fd1cc", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "480dabdf", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 811.04it/s]\n", - "100%|██████████| 2/2 [00:00<00:00, 191.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1903.03894v4\n", - "Key to References: {'fig:explainer-motivation': 'figures_3', 'fig:explainer-intro': 'figures_4', 'fig:definition-node-features': 'figures_5', 'fig:including-node-features': 'figures_7', 'fig:subgraph_node': 'figures_8', 'fig:subgraph_graph': 'figures_9', 'fig:prototype': 'figures_12', 'fig:my_label': 'figures_11', 'fig:synth_datasets': 'table_13', 'tab:results_pr': 'table_15'}\n", - "No paper found for cho2011friendship\n", - "No paper found for you2018graph\n", - "No paper found for zitnik2018decagon\n", - "No paper found for zhang_deep_2018\n", - "No paper found for zhou_graph_2018\n", - "No paper found for graphsage\n", - "No paper found for kipf2016semi\n", - "No paper found for ying2018hierarchical\n", - "No paper found for zhang2018link\n", - "No paper found for doshi-velez_towards_2017\n", - "No paper found for lakkaraju_interpretable_2017\n", - "No paper found for ribeiro_why_2016\n", - "No paper found for schmitz_ann-dt:_1999\n", - "No paper found for chen2018learning\n", - "No paper found for Erhan2009VisualizingHF\n", - "No paper found for lundberg_unified_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for koh_understanding_2017\n", - "No paper found for DBLP:journals/corr/abs-1811-09720\n", - "No paper found for ribeiro_why_2016\n", - "No paper found for augasta_reverse_2012\n", - "No paper found for lakkaraju_interpretable_2017\n", - "No paper found for calders_deepred_2016\n", - "No paper found for Erhan2009VisualizingHF\n", - "No paper found for fleet_visualizing_2014\n", - "No paper found for chen2018learning\n", - "No paper found for shrikumar_learning_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for Kang2019explaine\n", - "No paper found for fleet_visualizing_2014\n", - "No paper found for 2018sanity\n", - "No paper found for shrikumar_learning_2017\n", - "No paper found for sundararajan_axiomatic_nodate\n", - "No paper found for adadi_peeking_2018\n", - "No paper found for fisher_all_2018\n", - "No paper found for guidotti_survey_2018\n", - "No paper found for hooker_discovering_2004\n", - "No paper found for koh_understanding_2017\n", - "No paper found for DBLP:journals/corr/abs-1811-09720\n", - "No paper found for mutag\n", - "No paper found for duvenaud_convolutional_2015\n", - "No paper found for neil2018interpretable\n", - "No paper found for velickovic2018graph\n", - "No paper found for PhysRevLett.120.145301\n", - "No paper found for battaglia\n", - "No paper found for zhang_deep_2018\n", - "No paper found for zhou_graph_2018\n", - "No paper found for graphsage\n", - "No paper found for xu2018powerful\n", - "No paper found for kipf2016semi\n", - "No paper found for xujumping\n", - "No paper found for chen2018supervised\n", - "No paper found for kipf2016semi\n", - "No paper found for xujumping\n", - "No paper found for chen2018supervised\n", - "No paper found for mutag\n", - "No paper found for yanardag2015deep\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "No paper found for velickovic2018graph\n", - "No paper found for velickovic2018graph\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "fig:synth_datasets\n", - "No paper found for kumar2018community\n", - "No paper found for kumar2018community\n", - "1806.08804v4\n", - "Key to References: {'fig:assignment_vis': 'figures_157', 'tab:results': 'table_158', 'tab:results2': 'table_159'}\n", - "tab:results\n", - "tab:results2\n", - "Paper count: 2\n", - "Total nodes: 254\n", - "Total edges: 474\n", - "Paper nodes: 2\n", - "Figure nodes: 0\n", - "Table nodes: 2\n", - "Text nodes: 250\n", - "0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paragraphs_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paragraphs_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paragraphs_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paragraphs_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-insert", - "metadata": {}, - "source": [ - "### Insert Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "id": "insert-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted paragraph 1 from Introduction\n", - "Inserted paragraph 2 from Introduction\n", - "Inserted paragraph 3 from Introduction\n", - "Inserted paragraph 4 from Introduction\n", - "Inserted paragraph 5 from Introduction\n" - ] - } - ], - "source": [ - "# Insert paragraphs from the Introduction section\n", - "paragraphs = [\n", - " {\n", - " 'paragraph_id': 1,\n", - " 'content': 'Recurrent neural networks, long short-term memory and gated recurrent neural networks in particular, have been firmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 1\n", - " },\n", - " {\n", - " 'paragraph_id': 2,\n", - " 'content': 'Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures. Recurrent models typically factor computation along the symbol positions of the input and output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 2\n", - " },\n", - " {\n", - " 'paragraph_id': 3,\n", - " 'content': 'Aligning the positions to steps in computation time, they generate a sequence of hidden states h_t, as a function of the previous hidden state h_{t-1} and the input for position t. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 3\n", - " },\n", - " {\n", - " 'paragraph_id': 4,\n", - " 'content': 'Attention mechanisms have become an integral part of compelling sequence modeling and transduction models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 4\n", - " },\n", - " {\n", - " 'paragraph_id': 5,\n", - " 'content': 'In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for significantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 5\n", - " }\n", - "]\n", - "\n", - "for paragraph in paragraphs:\n", - " research_arcade.insert_node(\"arxiv_paragraphs\", node_features=paragraph)\n", - " print(f\"Inserted paragraph {paragraph['paragraph_id']} from {paragraph['paper_section']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-get-all", - "metadata": {}, - "source": [ - "### Get All Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "id": "get-all-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total paragraphs: 255\n", - "\n", - "First 3 paragraphs:\n", - " paragraph_id paper_section \\\n", - "0 1 Introduction \n", - "1 2 Introduction \n", - "2 3 Introduction \n", - "\n", - " content \n", - "0 Recurrent neural networks, long short-term mem... \n", - "1 Numerous efforts have since continued to push ... \n", - "2 Aligning the positions to steps in computation... \n" - ] - } - ], - "source": [ - "paragraphs_df = research_arcade.get_all_node_features(\"arxiv_paragraphs\")\n", - "print(f\"Total paragraphs: {len(paragraphs_df)}\")\n", - "print(\"\\nFirst 3 paragraphs:\")\n", - "print(paragraphs_df[['paragraph_id', 'paper_section', 'content']].head(3))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 10. Relationships/Edges \n", - "\n", - "This section demonstrates how to create and manage relationships between different entities." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.2 ArXiv Citations (arxiv_citation)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Citation" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Citation created!\n" - ] - } - ], - "source": [ - "citation = {\n", - " 'citing_arxiv_id': '1810.04805v2',\n", - " 'cited_arxiv_id': '1706.03762v7',\n", - " 'bib_title': 'attention is all you need',\n", - " 'bib_key': 'something',\n", - " 'citing_sections': 'citing_sections',\n", - "}\n", - "research_arcade.insert_edge(\"arxiv_citation\", edge_features=citation)\n", - "print(\"Citation created!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paper_citation_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_citation_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_citation\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paper_citation_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_citation_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_citation\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Citations" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total citations: 1\n", - " id citing_arxiv_id cited_arxiv_id bib_title bib_key \\\n", - "0 1 1810.04805v2 1706.03762v7 attention is all you need something \n", - "\n", - " author_cited_paper citing_sections citing_paragraphs \n", - "0 NaN \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "all_citations = research_arcade.get_all_edge_features(\"arxiv_citation\")\n", - "print(f\"Total citations: {len(all_citations)}\")\n", - "print(all_citations.head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Cited Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers cited:\n", - " id citing_arxiv_id cited_arxiv_id bib_title bib_key \\\n", - "0 1 1810.04805v2 1706.03762v7 attention is all you need something \n", - "\n", - " author_cited_paper citing_sections citing_paragraphs \n", - "0 NaN \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "citing_paper = {'citing_paper_id': '1810.04805v2'}\n", - "cited_papers = research_arcade.get_neighborhood(\"arxiv_citation\", primary_key=citing_paper)\n", - "print(\"Papers cited:\")\n", - "print(cited_papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Citing Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers that cite:\n", - " id citing_arxiv_id cited_arxiv_id bib_title bib_key \\\n", - "0 1 1810.04805v2 1706.03762v7 attention is all you need something \n", - "\n", - " author_cited_paper citing_sections citing_paragraphs \n", - "0 NaN \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "cited_paper = {'cited_paper_id': '1706.03762v7'}\n", - "citing_papers = research_arcade.get_neighborhood(\"arxiv_citation\", primary_key=cited_paper)\n", - "print(\"Papers that cite:\")\n", - "print(citing_papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Citation" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted citation: 1810.04805v2 -> 1706.03762v7\n", - "Citation deleted!\n" - ] - } - ], - "source": [ - "citation_id = {\n", - " 'citing_paper_id': '1810.04805v2',\n", - " 'cited_paper_id': '1706.03762v7'\n", - "}\n", - "research_arcade.delete_edge_by_id(\"arxiv_citation\", primary_key=citation_id)\n", - "print(\"Citation deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.3 ArXiv Paper-Author (arxiv_paper_author)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Author Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked author ss_ashish_vaswani (position 1)\n", - "Linked author ss_noam_shazeer (position 2)\n", - "Linked author ss_niki_parmar (position 3)\n" - ] - } - ], - "source": [ - "paper_authors = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani', 'author_sequence': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_noam_shazeer', 'author_sequence': 2},\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_niki_parmar', 'author_sequence': 3}\n", - "]\n", - "for relation in paper_authors:\n", - " research_arcade.insert_edge(\"arxiv_paper_author\", edge_features=relation)\n", - " print(f\"Linked author {relation['author_id']} (position {relation['author_sequence']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paper_author_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_author_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_author\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paper_author_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_author_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_author\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Paper-Author Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total relationships: 3\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_noam_shazeer 2\n", - "1 1706.03762v7 ss_niki_parmar 3\n", - "2 1706.03762v7 ss_ashish_vaswani 1\n" - ] - } - ], - "source": [ - "all_relations = research_arcade.get_all_edge_features(\"arxiv_paper_author\")\n", - "print(f\"Total relationships: {len(all_relations)}\")\n", - "print(all_relations.head(10))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Authors for a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Authors:\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_ashish_vaswani 1\n", - "1 1706.03762v7 ss_noam_shazeer 2\n", - "2 1706.03762v7 ss_niki_parmar 3\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "authors = research_arcade.get_neighborhood(\"arxiv_paper_author\", primary_key=paper_id)\n", - "print(\"Authors:\")\n", - "print(authors.sort_values('author_sequence'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Papers by Author" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers by author:\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_ashish_vaswani 1\n" - ] - } - ], - "source": [ - "author_id = {'author_id': 'ss_ashish_vaswani'}\n", - "papers = research_arcade.get_neighborhood(\"arxiv_paper_author\", primary_key=author_id)\n", - "print(\"Papers by author:\")\n", - "print(papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Paper-Author Link" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Relationship deleted!\n" - ] - } - ], - "source": [ - "relation_id = {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani'}\n", - "research_arcade.delete_edge_by_id(\"arxiv_paper_author\", primary_key=relation_id)\n", - "print(\"Relationship deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.4 ArXiv Paper-Category (arxiv_paper_category)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Category Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked 1\n", - "Linked 1\n", - "Linked 2\n" - ] - } - ], - "source": [ - "paper_categories = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '1'},\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '1'},\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '2'}\n", - "]\n", - "for relation in paper_categories:\n", - " research_arcade.insert_edge(\"arxiv_paper_category\", edge_features=relation)\n", - " print(f\"Linked {relation['category_id']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paper_category_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_category_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_category\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paper_category_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_category_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_category\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Paper-Category Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total relationships: 30\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n", - "2 1810.04805v2 1\n", - "3 1810.04805v2 3\n", - "4 1706.03762v7 1\n" - ] - } - ], - "source": [ - "all_relations = research_arcade.get_all_edge_features(\"arxiv_paper_category\")\n", - "print(f\"Total relationships: {len(all_relations)}\")\n", - "print(all_relations.head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Categories for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Categories:\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n", - "2 1706.03762v7 1\n", - "3 1706.03762v7 1\n", - "4 1706.03762v7 1\n", - "5 1706.03762v7 1\n", - "6 1706.03762v7 2\n", - "7 1706.03762v7 1\n", - "8 1706.03762v7 1\n", - "9 1706.03762v7 2\n", - "10 1706.03762v7 1\n", - "11 1706.03762v7 1\n", - "12 1706.03762v7 2\n", - "13 1706.03762v7 1\n", - "14 1706.03762v7 1\n", - "15 1706.03762v7 2\n", - "16 1706.03762v7 1\n", - "17 1706.03762v7 1\n", - "18 1706.03762v7 2\n", - "19 1706.03762v7 1\n", - "20 1706.03762v7 1\n", - "21 1706.03762v7 2\n", - "22 1706.03762v7 1\n", - "23 1706.03762v7 1\n", - "24 1706.03762v7 2\n", - "25 1706.03762v7 1\n", - "26 1706.03762v7 1\n", - "27 1706.03762v7 2\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "categories = research_arcade.get_neighborhood(\"arxiv_paper_category\", primary_key=paper_id)\n", - "print(\"Categories:\")\n", - "print(categories)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Papers in Category" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers in category:\n", - "None\n" - ] - } - ], - "source": [ - "category_id = {'category_id': 'cs.LG'}\n", - "papers = research_arcade.get_neighborhood(\"arxiv_paper_category\", primary_key=category_id)\n", - "print(\"Papers in category:\")\n", - "print(papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Paper-Category Link" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Relationship deleted!\n" - ] - } - ], - "source": [ - "relation_id = {'paper_arxiv_id': '1706.03762v7', 'category_id': 'cs.AI'}\n", - "research_arcade.delete_edge_by_id(\"arxiv_paper_category\", primary_key=relation_id)\n", - "print(\"Relationship deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.5 ArXiv Paper-Figure (arxiv_paper_figure)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Figure Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked figure 1)\n", - "Linked figure 2)\n" - ] - } - ], - "source": [ - "paper_figures = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'figure_id': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'figure_id': 2}\n", - "]\n", - "for relation in paper_figures:\n", - " research_arcade.insert_edge(\"arxiv_paper_figure\", edge_features=relation)\n", - " print(f\"Linked figure {relation['figure_id']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paper_figure_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_figure_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_figure\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paper_figure_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_figure_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_figure\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Figures for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Figures:\n", - " paper_arxiv_id figure_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "figures = research_arcade.get_neighborhood(\"arxiv_paper_figure\", primary_key=paper_id)\n", - "print(\"Figures:\")\n", - "print(figures)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.6 ArXiv Paper-Table (arxiv_paper_table)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Table Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked table 1\n", - "Linked table 2\n" - ] - } - ], - "source": [ - "paper_tables = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'table_id': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'table_id': 2}\n", - "]\n", - "for relation in paper_tables:\n", - " research_arcade.insert_edge(\"arxiv_paper_table\", edge_features=relation)\n", - " print(f\"Linked table {relation['table_id']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paper_table_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_table_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_table\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paper_table_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_table_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_table\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Tables for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tables:\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "tables = research_arcade.get_neighborhood(\"arxiv_paper_table\", primary_key=paper_id)\n", - "print(\"Tables:\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.7 ArXiv Paragraph-Reference (arxiv_paragraph_reference)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paragraph-Reference Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [], - "source": [ - "paragraph_references = [\n", - " {'paragraph_id': 1, 'paper_section': 'established approaches', 'paper_arxiv_id': '1706.03762v7', 'reference_label': \"{something}\", 'reference_type': 'figure'}\n", - "]\n", - "\n", - "for relation in paragraph_references:\n", - " research_arcade.insert_edge(\"arxiv_paragraph_reference\", edge_features=relation)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: CSV file ./examples/csv_data/csv_arxiv_paragraph_reference_example.csv does not exist.\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paragraph_reference_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paragraph_reference\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paragraph_reference_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paragraph_reference_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paragraph_reference\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get References in Paragraph" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "References:\n", - " id paragraph_id paper_section paper_arxiv_id reference_label \\\n", - "0 1 1 established approaches 1706.03762v7 {something} \n", - "1 2 1 established approaches 1706.03762v7 {something} \n", - "2 3 1 established approaches 1706.03762v7 {something} \n", - "3 4 1 established approaches 1706.03762v7 {something} \n", - "4 5 1 established approaches 1706.03762v7 {something} \n", - "5 6 1 established approaches 1706.03762v7 {something} \n", - "6 7 1 established approaches 1706.03762v7 {something} \n", - "7 8 1 established approaches 1706.03762v7 {something} \n", - "8 9 1 established approaches 1706.03762v7 {something} \n", - "\n", - " reference_type \n", - "0 figure \n", - "1 figure \n", - "2 figure \n", - "3 figure \n", - "4 figure \n", - "5 figure \n", - "6 figure \n", - "7 figure \n", - "8 figure \n" - ] - } - ], - "source": [ - "paragraph_id = {'paragraph_id': 1}\n", - "references = research_arcade.get_neighborhood(\"arxiv_paragraph_reference\", primary_key=paragraph_id)\n", - "print(\"References:\")\n", - "print(references)" - ] - }, - { - "cell_type": "markdown", - "id": "conclusion", - "metadata": {}, - "source": [ - "## Conclusion\n", - "\n", - "This tutorial has covered:\n", - "\n", - "1. Setting up the ResearchArcade database connection\n", - "2. Working with OpenReview data\n", - "3. CRUD operations for all ArXiv entity types:\n", - " - Papers\n", - " - Authors\n", - " - Categories\n", - " - Figures\n", - " - Tables\n", - " - Sections\n", - " - Paragraphs\n", - "4. Creating relationships between entities:\n", - " - Authorship\n", - " - Citations\n", - " - Paper-Category links\n", - " - Paper-Figure/Table links\n", - " - Paragraph-level references\n", - "\n", - "For more information, refer to the ResearchArcade documentation." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "research_arcade", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials.ipynb b/examples/tutorials.ipynb deleted file mode 100644 index 652e302..0000000 --- a/examples/tutorials.ipynb +++ /dev/null @@ -1,2316 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 10, - "id": "ce9f278c", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "import os\n", - "from tqdm import tqdm\n", - "\n", - "# Get current working directory instead of __file__\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..')))\n", - "from research_arcade.research_arcade import ResearchArcade" - ] - }, - { - "cell_type": "markdown", - "id": "622e1381", - "metadata": {}, - "source": [ - "## Choose Database Backend" - ] - }, - { - "cell_type": "markdown", - "id": "6779f1c3", - "metadata": {}, - "source": [ - "#### CSV Based" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "0af14435", - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "module 'openreview' has no attribute 'Client'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[22]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 1\u001b[39m db_type = \u001b[33m\"\u001b[39m\u001b[33mcsv\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 2\u001b[39m config = {\n\u001b[32m 3\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mcsv_dir\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33m../data/my_research_arcade_data/\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 4\u001b[39m }\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m research_arcade = \u001b[43mResearchArcade\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdb_type\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdb_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m=\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/Research/uiuc/research-arcade/research_arcade/research_arcade.py:63\u001b[39m, in \u001b[36mResearchArcade.__init__\u001b[39m\u001b[34m(self, db_type, config)\u001b[39m\n\u001b[32m 58\u001b[39m \u001b[38;5;28mself\u001b[39m.arxiv_paragraph_reference = CSVArxivParagraphReference(**config)\n\u001b[32m 60\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[33;03mBelow is the openreview dataset\u001b[39;00m\n\u001b[32m 62\u001b[39m \u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m63\u001b[39m \u001b[38;5;28mself\u001b[39m.openreview_arxiv = \u001b[43mCSVOpenReviewArxiv\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 64\u001b[39m \u001b[38;5;28mself\u001b[39m.openreview_authors = CSVOpenReviewAuthors(**config)\n\u001b[32m 65\u001b[39m \u001b[38;5;28mself\u001b[39m.openreview_papers_authors = CSVOpenReviewPapersAuthors(**config)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/Research/uiuc/research-arcade/research_arcade/csv_database/csv_openreview_arxiv.py:11\u001b[39m, in \u001b[36mCSVOpenReviewArxiv.__init__\u001b[39m\u001b[34m(self, csv_dir)\u001b[39m\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, csv_dir: \u001b[38;5;28mstr\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 10\u001b[39m \u001b[38;5;28mself\u001b[39m.csv_path = csv_dir + \u001b[33m\"\u001b[39m\u001b[33mopenreview_arxiv.csv\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m \u001b[38;5;28mself\u001b[39m.openreview_crawler = \u001b[43mOpenReviewCrawler\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 13\u001b[39m \u001b[38;5;66;03m# 如果CSV文件不存在，创建空的DataFrame\u001b[39;00m\n\u001b[32m 14\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os.path.exists(\u001b[38;5;28mself\u001b[39m.csv_path):\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/Research/uiuc/research-arcade/research_arcade/openreview_utils/openreview_crawler.py:14\u001b[39m, in \u001b[36mOpenReviewCrawler.__init__\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 13\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m14\u001b[39m \u001b[38;5;28mself\u001b[39m.client_v1 = \u001b[43mopenreview\u001b[49m\u001b[43m.\u001b[49m\u001b[43mClient\u001b[49m(\n\u001b[32m 15\u001b[39m baseurl=\u001b[33m'\u001b[39m\u001b[33mhttps://api.openreview.net\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 16\u001b[39m )\n\u001b[32m 18\u001b[39m \u001b[38;5;28mself\u001b[39m.client_v2 = openreview.api.OpenReviewClient(\n\u001b[32m 19\u001b[39m baseurl=\u001b[33m'\u001b[39m\u001b[33mhttps://api2.openreview.net\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 20\u001b[39m )\n", - "\u001b[31mAttributeError\u001b[39m: module 'openreview' has no attribute 'Client'" - ] - } - ], - "source": [ - "db_type = \"csv\"\n", - "config = {\n", - " \"csv_dir\": \"../data/my_research_arcade_data/\"\n", - "}\n", - "\n", - "research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "e6352298", - "metadata": {}, - "source": [ - "#### SQL Based" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2d04122a", - "metadata": {}, - "outputs": [], - "source": [ - "db_type = \"sql\"\n", - "config = {\n", - " \"host\": \"localhost\",\n", - " \"dbname\": \"iclr_openreview_database\",\n", - " \"user\": \"jingjunx\",\n", - " \"password\": \"\",\n", - " \"port\": \"5432\"\n", - "}\n", - "\n", - "research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "5bb4510c", - "metadata": {}, - "source": [ - "## openreview_authors" - ] - }, - { - "cell_type": "markdown", - "id": "fd8665d6", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "a7a6f6e6", - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'research_arcade' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m config = {\u001b[33m\"\u001b[39m\u001b[33mvenue\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mICLR.cc/2025/Conference\u001b[39m\u001b[33m\"\u001b[39m}\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[43mresearch_arcade\u001b[49m.construct_table_from_api(\u001b[33m\"\u001b[39m\u001b[33mopenreview_authors\u001b[39m\u001b[33m\"\u001b[39m, config)\n", - "\u001b[31mNameError\u001b[39m: name 'research_arcade' is not defined" - ] - } - ], - "source": [ - "config = {\"venue\": \"ICLR.cc/2025/Conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "98ea499f", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "00facdc0", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_author_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "1efcd951", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71e08827", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_author_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "2301cc72", - "metadata": {}, - "source": [ - "#### insert node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b48d61ae", - "metadata": {}, - "outputs": [], - "source": [ - "new_author = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'author_openreview_id': '~ishmam_zabir1', \n", - " 'author_full_name': 'ishmam zabir', \n", - " 'email': '****@microsoft.com', \n", - " 'affiliation': 'Microsoft', \n", - " 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', \n", - " 'dblp': ''}\n", - "research_arcade.insert_node(\"openreview_authors\", node_features=new_author)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d57bda64", - "metadata": {}, - "outputs": [], - "source": [ - "new_paper = {'arxiv_id': '1706.03762v7',\n", - " 'base_arxiv_id': '1706.03762',\n", - " 'version': '7',\n", - " 'title': 'Attention Is All You Need',\n", - " 'abstract': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.',\n", - " 'submit_date': '2017-06-12',\n", - " 'metadata': None}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)" - ] - }, - { - "cell_type": "markdown", - "id": "008ac40c", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ef4f7b17", - "metadata": {}, - "outputs": [], - "source": [ - "author_id = {\"author_openreview_id\": \"~ishmam_zabir1\"}\n", - "author_features = research_arcade.delete_node_by_id(\"openreview_authors\", author_id)\n", - "print(author_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "986ab856", - "metadata": {}, - "source": [ - "## openreview_authors" - ] - }, - { - "cell_type": "markdown", - "id": "a0a1535c", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6251f25", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2025/Conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "1b438cf1", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11f5f6c8", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_author_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "10552364", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1b1ff1b2", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_author_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "49ca7e19", - "metadata": {}, - "source": [ - "#### insert node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f37a47f", - "metadata": {}, - "outputs": [], - "source": [ - "new_author = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'author_openreview_id': '~ishmam_zabir1', \n", - " 'author_full_name': 'ishmam zabir', \n", - " 'email': '****@microsoft.com', \n", - " 'affiliation': 'Microsoft', \n", - " 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', \n", - " 'dblp': ''}\n", - "research_arcade.insert_node(\"openreview_authors\", node_features=new_author)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f5ee8e15", - "metadata": {}, - "outputs": [], - "source": [ - "new_paper = {'arxiv_id': '1706.03762v7',\n", - " 'base_arxiv_id': '1706.03762',\n", - " 'version': '7',\n", - " 'title': 'Attention Is All You Need',\n", - " 'abstract': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.',\n", - " 'submit_date': '2017-06-12',\n", - " 'metadata': None}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)" - ] - }, - { - "cell_type": "markdown", - "id": "59a9b808", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4893588b", - "metadata": {}, - "outputs": [], - "source": [ - "author_id = {\"author_openreview_id\": \"~ishmam_zabir1\"}\n", - "author_features = research_arcade.delete_node_by_id(\"openreview_authors\", author_id)\n", - "print(author_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "038c8122", - "metadata": {}, - "source": [ - "#### get all nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cb91c087", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_authors_df = research_arcade.get_all_node_features(\"openreview_authors\")\n", - "print(len(openreview_authors_df))\n", - "\n", - "\n", - "arxiv_papers_df = research_arcade.get_all_node_features(\"arxiv_papers\")\n", - "print(len(arxiv_papers_df))" - ] - }, - { - "cell_type": "markdown", - "id": "a4b48716", - "metadata": {}, - "source": [ - "#### get specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ca229da0", - "metadata": {}, - "outputs": [], - "source": [ - "author_id = {\"author_openreview_id\": \"~ishmam_zabir1\"}\n", - "author_features = research_arcade.get_node_features_by_id(\"openreview_authors\", author_id)\n", - "print(author_features.to_dict(orient=\"records\")[0])\n", - "\n", - "\n", - "paper_id = {\"arxiv_papers\": \"1706.03762v7\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"arxiv_papers\", author_id)\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "212d2bcc", - "metadata": {}, - "source": [ - "#### update specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "66eaf6be", - "metadata": {}, - "outputs": [], - "source": [ - "new_author = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'author_openreview_id': '~ishmam_zabir1', \n", - " 'author_full_name': 'ishmam zabir', \n", - " 'email': '****@microsoft.com', \n", - " 'affiliation': 'Microsoft', \n", - " 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', \n", - " 'dblp': ''}\n", - "\n", - "research_arcade.update_node(\"openreview_authors\", node_features=new_author)\n", - "author_id = {\"author_openreview_id\": \"~ishmam_zabir1\"}" - ] - }, - { - "cell_type": "markdown", - "id": "0badeb41", - "metadata": {}, - "source": [ - "#### get all nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "513f73b9", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_authors_df = research_arcade.get_all_node_features(\"openreview_authors\")\n", - "print(len(openreview_authors_df))\n", - "\n", - "\n", - "arxiv_papers_df = research_arcade.get_all_node_features(\"arxiv_papers\")\n", - "print(len(arxiv_papers_df))" - ] - }, - { - "cell_type": "markdown", - "id": "1edac08f", - "metadata": {}, - "source": [ - "#### get specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56c479c9", - "metadata": {}, - "outputs": [], - "source": [ - "author_id = {\"author_openreview_id\": \"~ishmam_zabir1\"}\n", - "author_features = research_arcade.get_node_features_by_id(\"openreview_authors\", author_id)\n", - "print(author_features.to_dict(orient=\"records\")[0])\n", - "\n", - "\n", - "paper_id = {\"arxiv_papers\": \"1706.03762v7\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"arxiv_papers\", author_id)\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "69af8be0", - "metadata": {}, - "source": [ - "#### update specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de6f6467", - "metadata": {}, - "outputs": [], - "source": [ - "new_author = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'author_openreview_id': '~ishmam_zabir1', \n", - " 'author_full_name': 'ishmam zabir', \n", - " 'email': '****@microsoft.com', \n", - " 'affiliation': 'Microsoft', \n", - " 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', \n", - " 'dblp': ''}\n", - "\n", - "research_arcade.update_node(\"openreview_authors\", node_features=new_author)\n", - "author_id = {\"author_openreview_id\": \"~ishmam_zabir1\"}" - ] - }, - { - "cell_type": "markdown", - "id": "953c20d1", - "metadata": {}, - "source": [ - "## openreview_papers" - ] - }, - { - "cell_type": "markdown", - "id": "2ddc6248", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12a9bdba", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2025/Conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "bf6cd198", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "981da97b", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_paper_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "c48f1b2f", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a0b8fcf1", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_paper_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "3808a63d", - "metadata": {}, - "source": [ - "#### insert node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6030ff2d", - "metadata": {}, - "outputs": [], - "source": [ - "paper_features = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'paper_openreview_id': 'zGej22CBnS', \n", - " 'title': 'Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles', \n", - " 'abstract': \"Tokenization is associated with many poorly understood shortcomings in language models (LMs), yet remains an important component for long sequence scaling purposes. This work studies how tokenization impacts model performance by analyzing and comparing the stochastic behavior of tokenized models with their byte-level, or token-free, counterparts. We discover that, even when the two models are statistically equivalent, their predictive distributions over the next byte can be substantially different, a phenomenon we term as ``tokenization bias''. To fully characterize this phenomenon, we introduce the Byte-Token Representation Lemma, a framework that establishes a mapping between the learned token distribution and its equivalent byte-level distribution. From this result, we develop a next-byte sampling algorithm that eliminates tokenization bias without requiring further training or optimization. In other words, this enables zero-shot conversion of tokenized LMs into statistically equivalent token-free ones. We demonstrate its broad applicability with two use cases: fill-in-the-middle (FIM) tasks and model ensembles. In FIM tasks where input prompts may terminate mid-token, leading to out-of-distribution tokenization, our method mitigates performance degradation and achieves 18\\\\% improvement in FIM coding benchmarks, while consistently outperforming the standard token healing fix. For model ensembles where each model employs a distinct vocabulary, our approach enables seamless integration, resulting in improved performance up to 3.7\\\\% over individual models across various standard baselines in reasoning, knowledge, and coding. Code is available at:https: //github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs.\", \n", - " 'paper_decision': 'ICLR 2025 Poster', \n", - " 'paper_pdf_link': '/pdf/cdd2212a20c4034029874cba11a05e081bfdb83e.pdf'}\n", - "research_arcade.insert_node(\"openreview_papers\", node_features=paper_features)" - ] - }, - { - "cell_type": "markdown", - "id": "4b6054d3", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "441e6582", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"zGej22CBnS\"}\n", - "paper_features = research_arcade.delete_node_by_id(\"openreview_papers\", paper_id)\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "b8d501d7", - "metadata": {}, - "source": [ - "#### get all nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4dc1600c", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_papers_df = research_arcade.get_all_node_features(\"openreview_papers\")\n", - "print(len(openreview_papers_df))" - ] - }, - { - "cell_type": "markdown", - "id": "8e2a5cc1", - "metadata": {}, - "source": [ - "#### get specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2e544ad", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"zGej22CBnS\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"openreview_papers\", paper_id)\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "debc39fc", - "metadata": {}, - "source": [ - "#### update specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c6f4e74", - "metadata": {}, - "outputs": [], - "source": [ - "new_paper_features = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'paper_openreview_id': 'zGej22CBnS', \n", - " 'title': 'Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles', \n", - " 'abstract': \"Tokenization is associated with many poorly understood shortcomings in language models (LMs), yet remains an important component for long sequence scaling purposes. This work studies how tokenization impacts model performance by analyzing and comparing the stochastic behavior of tokenized models with their byte-level, or token-free, counterparts. We discover that, even when the two models are statistically equivalent, their predictive distributions over the next byte can be substantially different, a phenomenon we term as ``tokenization bias''. To fully characterize this phenomenon, we introduce the Byte-Token Representation Lemma, a framework that establishes a mapping between the learned token distribution and its equivalent byte-level distribution. From this result, we develop a next-byte sampling algorithm that eliminates tokenization bias without requiring further training or optimization. In other words, this enables zero-shot conversion of tokenized LMs into statistically equivalent token-free ones. We demonstrate its broad applicability with two use cases: fill-in-the-middle (FIM) tasks and model ensembles. In FIM tasks where input prompts may terminate mid-token, leading to out-of-distribution tokenization, our method mitigates performance degradation and achieves 18\\\\% improvement in FIM coding benchmarks, while consistently outperforming the standard token healing fix. For model ensembles where each model employs a distinct vocabulary, our approach enables seamless integration, resulting in improved performance up to 3.7\\\\% over individual models across various standard baselines in reasoning, knowledge, and coding. Code is available at:https: //github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs.\", \n", - " 'paper_decision': 'ICLR 2025 Poster', \n", - " 'paper_pdf_link': '/pdf/cdd2212a20c4034029874cba11a05e081bfdb83e.pdf'}\n", - "research_arcade.update_node(\"openreview_papers\", node_features=new_paper_features)" - ] - }, - { - "cell_type": "markdown", - "id": "dff3dd00", - "metadata": {}, - "source": [ - "## openreview_reviews" - ] - }, - { - "cell_type": "markdown", - "id": "95dcd85d", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38379f60", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2013/conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "daf7c324", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "932abaab", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_review_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "bf1bddf9", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24323766", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_review_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "d462caaa", - "metadata": {}, - "source": [ - "#### insert node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "477aaf07", - "metadata": {}, - "outputs": [], - "source": [ - "review_features = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'review_openreview_id': 'DHwZxFryth', \n", - " 'replyto_openreview_id': 'Yqbllggrmw', \n", - " 'writer': 'Authors', \n", - " 'title': 'Response by Authors', \n", - " 'content': {'Title': 'Response to Reviewer 7i95 (1/2)', 'Comment': '> The method does not improve much in the AlpacaEval 2.0 Score. The author should give a detailed explanation. And why not use metrics like length-controlled win rate?**Response:** Thank you for your careful observation and question. We would like to clarify that we are already using the length-controlled (LC) AlpacaEval 2.0 win-rate metric in our evaluations. We will make this clearer in the table header of Table 3.Regarding the fact that the AlpacaEval 2.0 scores on LLama-3 (8B) do not improve compared to the baselines, we believe this is because our base model, the instruction-finetuned LLama-3 (8B), is already trained to perform exceptionally well in terms of helpfulness, which is the focus of the AlpacaEval benchmark. Additionally, the preference dataset we used, UltraFeedback, may not provide significant further enhancement in the helpfulness aspect. This is supported by the slight decrease observed in the AlpacaEval score for the standard DPO baseline as well (see Table 3, results on LLama-3). Therefore, we think these AlpacaEval 2.0 results on LLama-3 (8B) may not indicate that SAIL is ineffective; it may be simply caused by an ill-suited combination of base model, finetuning dataset, and evaluation benchmark.We also further conducted experiments on the Zephyr (7B) model as the backbone, whose AlpacaEval 2.0 win-rate is lower. We still train on the UltraFeedback preference dataset and the other experiment setups are unchanged. In this experiment, we see a larger improvement of the SAIL method compared to the standard DPO baseline (Zephyr-7B-Beta).| | AlpacaEval 2.0 (LC) Win-Rate ||--------------------|------------------------------|| Base (Zephyr-7B-SFT-Full) | 6.4 % || DPO (Zephyr-7B-Beta) | 13.2 % || SAIL-PP | 15.9 % |> Authors should compare more advanced preference optimization algorithms like ORPO and SimPO. And current results are not impressive for the alignment community.**Response:** Thank you for raising this insightful point. We see ORPO and SimPO are two recent work which propose a different objective than the standard RLHF, and achieve remarkable improvements in terms of alignment performance and efficiency.Our work focus more on bringing standard RLHF to a bilevel optimization framework and propose an effective and efficient approximate algorithm on top of it. We can see some new preference optimization methods including ORPO and SimPO have one fundamental difference from our approach: they do not explicitly incorporate the KL regularization term. The absence of the KL regularization term allows these methods to optimize more aggressively for the reward function by deviating significantly from the reference model. In contrast, our approach is specifically grounded in the standard RLHF, where the KL regularization term ensures that the model remains aligned with the reference distribution while optimizing for the reward function. This distinction makes direct comparisons with ORPO or SimPO less meaningful theoretically, as those methods omit the KL regularization and adopt a fundamentally different optimization objective design.However, we think our work, although developed adhering to the standard RLHF setup, can be compatible and combined with some recent advanced preference optimization algorithms, despite their differences in optimization setups and objectives. This is because we can reformulate their alignment problem as bilevel optimization, and go through the derivation as done in the paper. Taking SimPO as an example, we can treat their reward model definition (Equation (4) in the SimPO paper) as the solution of the upper level optimization (replacing Equation (4) in our manuscript), and adopt their modified Bradley-Terry objective with reward margin (Equation (5) in the SimPO paper) to replace the standard one (Equation (10) in our manuscript). By applying these changes and rederiving the extra gradient terms, we can formulate an adaptation of our method to the SimPO objective. We will implement this combined algorithm, which adapt our methodology to the SimPO objective, and compare with the SimPO as a baseline.Recently many different alignment objectives and algorithms have emerged; it is an interesting question to discuss the compatibility and combination of our method with each objective. We will add more relevant discussions to the appendices, but due to the fact that the compatibility problem with each design is a non-trivial question, this process may incur considerably more work, and we hope the reviewer understands that this effort cannot be fully reflected by the rebuttal period. But we will continue to expand the discussion as the wide compatibility to other designs also strengthens our contribution to the community. We thank the reviewer for raising this insightful point.'}, \n", - " 'time': '2024-11-26 15:27:26'\n", - "}\n", - "research_arcade.insert_node(\"openreview_reviews\", node_features=review_features)" - ] - }, - { - "cell_type": "markdown", - "id": "89f2d6dc", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b505f190", - "metadata": {}, - "outputs": [], - "source": [ - "review_id = {\"review_openreview_id\": \"DHwZxFryth\"}\n", - "review_features = research_arcade.delete_node_by_id(\"openreview_reviews\", review_id)\n", - "print(review_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "ef313c52", - "metadata": {}, - "source": [ - "#### get all nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c127d3b0", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_reviews_df = research_arcade.get_all_node_features(\"openreview_reviews\")\n", - "print(len(openreview_reviews_df))" - ] - }, - { - "cell_type": "markdown", - "id": "a79dff0d", - "metadata": {}, - "source": [ - "#### get specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a440d1c6", - "metadata": {}, - "outputs": [], - "source": [ - "review_id = {\"review_openreview_id\": \"DHwZxFryth\"}\n", - "review_features = research_arcade.get_node_features_by_id(\"openreview_reviews\", review_id)\n", - "print(review_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "c9b77630", - "metadata": {}, - "source": [ - "#### update specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "98a24c60", - "metadata": {}, - "outputs": [], - "source": [ - "new_review_features = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'review_openreview_id': 'DHwZxFryth', \n", - " 'replyto_openreview_id': 'Yqbllggrmw', \n", - " 'writer': 'Authors', \n", - " 'title': 'Response by Authors', \n", - " 'content': {'Title': 'Response to Reviewer 7i95 (1/2)', 'Comment': '> The method does not improve much in the AlpacaEval 2.0 Score. The author should give a detailed explanation. And why not use metrics like length-controlled win rate?**Response:** Thank you for your careful observation and question. We would like to clarify that we are already using the length-controlled (LC) AlpacaEval 2.0 win-rate metric in our evaluations. We will make this clearer in the table header of Table 3.Regarding the fact that the AlpacaEval 2.0 scores on LLama-3 (8B) do not improve compared to the baselines, we believe this is because our base model, the instruction-finetuned LLama-3 (8B), is already trained to perform exceptionally well in terms of helpfulness, which is the focus of the AlpacaEval benchmark. Additionally, the preference dataset we used, UltraFeedback, may not provide significant further enhancement in the helpfulness aspect. This is supported by the slight decrease observed in the AlpacaEval score for the standard DPO baseline as well (see Table 3, results on LLama-3). Therefore, we think these AlpacaEval 2.0 results on LLama-3 (8B) may not indicate that SAIL is ineffective; it may be simply caused by an ill-suited combination of base model, finetuning dataset, and evaluation benchmark.We also further conducted experiments on the Zephyr (7B) model as the backbone, whose AlpacaEval 2.0 win-rate is lower. We still train on the UltraFeedback preference dataset and the other experiment setups are unchanged. In this experiment, we see a larger improvement of the SAIL method compared to the standard DPO baseline (Zephyr-7B-Beta).| | AlpacaEval 2.0 (LC) Win-Rate ||--------------------|------------------------------|| Base (Zephyr-7B-SFT-Full) | 6.4 % || DPO (Zephyr-7B-Beta) | 13.2 % || SAIL-PP | 15.9 % |> Authors should compare more advanced preference optimization algorithms like ORPO and SimPO. And current results are not impressive for the alignment community.**Response:** Thank you for raising this insightful point. We see ORPO and SimPO are two recent work which propose a different objective than the standard RLHF, and achieve remarkable improvements in terms of alignment performance and efficiency.Our work focus more on bringing standard RLHF to a bilevel optimization framework and propose an effective and efficient approximate algorithm on top of it. We can see some new preference optimization methods including ORPO and SimPO have one fundamental difference from our approach: they do not explicitly incorporate the KL regularization term. The absence of the KL regularization term allows these methods to optimize more aggressively for the reward function by deviating significantly from the reference model. In contrast, our approach is specifically grounded in the standard RLHF, where the KL regularization term ensures that the model remains aligned with the reference distribution while optimizing for the reward function. This distinction makes direct comparisons with ORPO or SimPO less meaningful theoretically, as those methods omit the KL regularization and adopt a fundamentally different optimization objective design.However, we think our work, although developed adhering to the standard RLHF setup, can be compatible and combined with some recent advanced preference optimization algorithms, despite their differences in optimization setups and objectives. This is because we can reformulate their alignment problem as bilevel optimization, and go through the derivation as done in the paper. Taking SimPO as an example, we can treat their reward model definition (Equation (4) in the SimPO paper) as the solution of the upper level optimization (replacing Equation (4) in our manuscript), and adopt their modified Bradley-Terry objective with reward margin (Equation (5) in the SimPO paper) to replace the standard one (Equation (10) in our manuscript). By applying these changes and rederiving the extra gradient terms, we can formulate an adaptation of our method to the SimPO objective. We will implement this combined algorithm, which adapt our methodology to the SimPO objective, and compare with the SimPO as a baseline.Recently many different alignment objectives and algorithms have emerged; it is an interesting question to discuss the compatibility and combination of our method with each objective. We will add more relevant discussions to the appendices, but due to the fact that the compatibility problem with each design is a non-trivial question, this process may incur considerably more work, and we hope the reviewer understands that this effort cannot be fully reflected by the rebuttal period. But we will continue to expand the discussion as the wide compatibility to other designs also strengthens our contribution to the community. We thank the reviewer for raising this insightful point.'}, \n", - " 'time': '2024-11-26 15:27:26'\n", - "}\n", - "research_arcade.update_node(\"openreview_reviews\", node_features=new_review_features)" - ] - }, - { - "cell_type": "markdown", - "id": "1dd9a7f1", - "metadata": {}, - "source": [ - "## openreview_revisions" - ] - }, - { - "cell_type": "markdown", - "id": "703ea91b", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "markdown", - "id": "f4b6de92", - "metadata": {}, - "source": [ - "##### get pdfs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5295829c", - "metadata": {}, - "outputs": [], - "source": [ - "from .get_pdfs import get_paper_pdf, get_revision_pdf\n", - "import os\n", - "import openreview\n", - "import time\n", - "\n", - "client_v1 = openreview.Client(baseurl='https://api.openreview.net')\n", - "client_v2 = openreview.api.OpenReviewClient(baseurl='https://api2.openreview.net')\n", - "\n", - "venue = 'ICLR.cc/2017/conference'\n", - "pdf_dir = \"/data/jingjunx/openreview_pdfs_2017/\"\n", - "log_file = \"./download_failed_ids_revisions_2017.log\"\n", - "start_idx = 0\n", - "end_idx = 5" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f24ee9bd", - "metadata": {}, - "outputs": [], - "source": [ - "if \"2023\" in venue or \"2022\" in venue or \"2021\" in venue or \"2020\" in venue or \"2019\" in venue or \"2018\" in venue or \"2017\" in venue or \"2014\" in venue or \"2013\" in venue:\n", - " if \"2023\" in venue or \"2022\" in venue or \"2021\" in venue or \"2020\" in venue or \"2019\" in venue or \"2018\" in venue:\n", - " submissions = client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='revisions')\n", - " elif \"2017\" in venue or \"2014\" in venue or \"2013\" in venue:\n", - " submissions = client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='revisions')\n", - " \n", - " if submissions is None:\n", - " print(f\"No submissions found for venue: {venue}\")\n", - " else:\n", - " for submission in tqdm(submissions[start_idx:end_idx]):\n", - " # get paper openreview id\n", - " paper_id = submission.id\n", - " if \"pdf\" in submission.content:\n", - " pdf_link = submission.content[\"pdf\"]\n", - " pdf_path = str(pdf_dir)+str(paper_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " get_paper_pdf(pdf_link, pdf_path, log_file)\n", - " \n", - " revisions = client_v1.get_references(referent=paper_id, original=True)\n", - " time.sleep(1)\n", - " \n", - " pdf_revisions_ids = []\n", - " for revision in revisions:\n", - " if \"pdf\" in revision.content:\n", - " pdf_revisions_ids.append(revision.id)\n", - " \n", - " if len(pdf_revisions_ids) <= 1:\n", - " continue\n", - " else:\n", - " for pdf_revision_id in pdf_revisions_ids:\n", - " pdf_path = str(pdf_dir)+str(pdf_revision_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " get_revision_pdf(venue, pdf_revision_id, pdf_path, log_file)\n", - " time.sleep(1)\n", - "else:\n", - " submissions = client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='revisions')\n", - " if submissions is None:\n", - " print(f\"No submissions found for venue: {venue}\")\n", - " else:\n", - " for submission in tqdm(submissions[start_idx:end_idx]):\n", - " decision = submission.content[\"venueid\"][\"value\"].split('/')[-1]\n", - " if decision == \"Withdrawn_Submission\":\n", - " continue\n", - " else:\n", - " # get paper openreview id\n", - " paper_id = submission.id\n", - " if \"pdf\" in submission.content:\n", - " pdf_link = submission.content[\"pdf\"][\"value\"]\n", - " pdf_path = str(pdf_dir)+str(paper_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " get_paper_pdf(pdf_link, pdf_path, log_file)\n", - " \n", - " revisions = client_v2.get_note_edits(note_id=paper_id)\n", - " if len(revisions) <= 1:\n", - " continue\n", - " else:\n", - " for revision in revisions:\n", - " pdf_revision_id = revision.id\n", - " pdf_path = str(pdf_dir)+str(pdf_revision_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " time.sleep(1)\n", - " get_revision_pdf(venue, pdf_revision_id, pdf_path, log_file)\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "markdown", - "id": "1b480a76", - "metadata": {}, - "source": [ - "#### construct the table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9dc165be", - "metadata": {}, - "outputs": [], - "source": [ - "venue = \"ICLR.cc/2017/conference\"\n", - "filter_list = [\"Under review as a conference paper at ICLR 2017\", \"Published as a conference paper at ICLR 2017\"]\n", - "pdf_dir = \"/data/jingjunx/openreview_pdfs_2017/\"\n", - "log_file = \"./log/failed_ids_revisions_2017.log\"\n", - "config = {\"venue\": venue, \"filter_list\": filter_list, \"pdf_dir\": pdf_dir, \"log_file\": log_file}\n", - "research_arcade.construct_table_from_api(\"openreview_revisions\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "e09e38e7", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "da2d35f6", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_revision_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_revisions\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "a5464f3b", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59d90e06", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_revision_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_revisions\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "d72bcfad", - "metadata": {}, - "source": [ - "#### insert node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eaecbbea", - "metadata": {}, - "outputs": [], - "source": [ - "revision_feature = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'original_openreview_id': 'pbTVNlX8Ig', \n", - " 'revision_openreview_id': 'yfHQOp5zWc', \n", - " 'content': [{'section': '1 INTRODUCTION', \n", - " 'after_section': None, \n", - " 'context_after': '2 RELATED WORK ', \n", - " 'paragraph_idx': 9, \n", - " 'before_section': None, \n", - " 'context_before': 'Published as a conference paper at ICLR 2025 tograd system in PyTorch, specifically tailored for our experimental setup, which is available at ', \n", - " 'modified_lines': 'https://github.com/stephane-rivaud/PETRA. ', \n", - " 'original_lines': 'https://github.com/streethagore/PETRA. ', \n", - " 'after_paragraph_idx': None, \n", - " 'before_paragraph_idx': None}], \n", - " 'time': '2025-03-14 15:35:37'}\n", - "research_arcade.insert_node(\"openreview_revisions\", node_features=revision_feature)" - ] - }, - { - "cell_type": "markdown", - "id": "032e9934", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fbb02146", - "metadata": {}, - "outputs": [], - "source": [ - "revision_id = {\"revision_openreview_id\": \"yfHQOp5zWc\"}\n", - "revision_feature = research_arcade.delete_node_by_id(\"openreview_revisions\", revision_id)\n", - "print(revision_feature.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "7d88deaf", - "metadata": {}, - "source": [ - "#### get all nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "08d02462", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_revisions_df = research_arcade.get_all_node_features(\"openreview_revisions\")\n", - "print(len(openreview_revisions_df))" - ] - }, - { - "cell_type": "markdown", - "id": "3383ce74", - "metadata": {}, - "source": [ - "#### get specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8b748a79", - "metadata": {}, - "outputs": [], - "source": [ - "revision_id = {\"revision_openreview_id\": \"yfHQOp5zWc\"}\n", - "revision_feature = research_arcade.get_node_features_by_id(\"openreview_revisions\", revision_id)\n", - "print(revision_feature.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "b19c7c2d", - "metadata": {}, - "source": [ - "#### update specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a945848", - "metadata": {}, - "outputs": [], - "source": [ - "new_revision_features = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'original_openreview_id': 'pbTVNlX8Ig', \n", - " 'revision_openreview_id': 'yfHQOp5zWc', \n", - " 'content': [{'section': '1 INTRODUCTION', \n", - " 'after_section': None, \n", - " 'context_after': '2 RELATED WORK ', \n", - " 'paragraph_idx': 9, \n", - " 'before_section': None, \n", - " 'context_before': 'Published as a conference paper at ICLR 2025 tograd system in PyTorch, specifically tailored for our experimental setup, which is available at ', \n", - " 'modified_lines': 'https://github.com/stephane-rivaud/PETRA. ', \n", - " 'original_lines': 'https://github.com/streethagore/PETRA. ', \n", - " 'after_paragraph_idx': None, \n", - " 'before_paragraph_idx': None}], \n", - " 'time': '2025-03-14 15:35:37'}\n", - "research_arcade.update_node(\"openreview_revisions\", node_features=new_revision_features)" - ] - }, - { - "cell_type": "markdown", - "id": "c3ae2e8b", - "metadata": {}, - "source": [ - "## openreview_paragraphs" - ] - }, - { - "cell_type": "markdown", - "id": "a0e7f2be", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "markdown", - "id": "67a98817", - "metadata": {}, - "source": [ - "##### get pdfs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "760ba108", - "metadata": {}, - "outputs": [], - "source": [ - "from .get_pdfs import get_paper_pdf, get_revision_pdf\n", - "import os\n", - "import openreview\n", - "from tqdm import tqdm\n", - "import time\n", - "\n", - "client_v1 = openreview.Client(baseurl='https://api.openreview.net')\n", - "client_v2 = openreview.api.OpenReviewClient(baseurl='https://api2.openreview.net')\n", - "\n", - "venue = 'ICLR.cc/2025/Conference'\n", - "pdf_dir = \"/data/jingjunx/openreview_pdfs_2025/\"\n", - "log_file = \"./download_failed_ids_revisions_2025.log\"\n", - "start_idx = 0\n", - "end_idx = 5" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "788b4db2", - "metadata": {}, - "outputs": [], - "source": [ - "if \"2023\" in venue or \"2022\" in venue or \"2021\" in venue or \"2020\" in venue or \"2019\" in venue or \"2018\" in venue or \"2017\" in venue or \"2014\" in venue or \"2013\" in venue:\n", - " if \"2023\" in venue or \"2022\" in venue or \"2021\" in venue or \"2020\" in venue or \"2019\" in venue or \"2018\" in venue:\n", - " submissions = client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='revisions')\n", - " elif \"2017\" in venue or \"2014\" in venue or \"2013\" in venue:\n", - " submissions = client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='revisions')\n", - " \n", - " if submissions is None:\n", - " print(f\"No submissions found for venue: {venue}\")\n", - " else:\n", - " for submission in tqdm(submissions[start_idx:end_idx]):\n", - " # get paper openreview id\n", - " paper_id = submission.id\n", - " if \"pdf\" in submission.content:\n", - " pdf_link = submission.content[\"pdf\"]\n", - " pdf_path = str(pdf_dir)+str(paper_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " get_paper_pdf(pdf_link, pdf_path, log_file)\n", - " \n", - " revisions = client_v1.get_references(referent=paper_id, original=True)\n", - " time.sleep(1)\n", - " \n", - " pdf_revisions_ids = []\n", - " for revision in revisions:\n", - " if \"pdf\" in revision.content:\n", - " pdf_revisions_ids.append(revision.id)\n", - " \n", - " if len(pdf_revisions_ids) <= 1:\n", - " continue\n", - " else:\n", - " for pdf_revision_id in pdf_revisions_ids:\n", - " pdf_path = str(pdf_dir)+str(pdf_revision_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " get_revision_pdf(venue, pdf_revision_id, pdf_path, log_file)\n", - " time.sleep(1)\n", - "else:\n", - " submissions = client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='revisions')\n", - " if submissions is None:\n", - " print(f\"No submissions found for venue: {venue}\")\n", - " else:\n", - " for submission in tqdm(submissions[start_idx:end_idx]):\n", - " decision = submission.content[\"venueid\"][\"value\"].split('/')[-1]\n", - " if decision == \"Withdrawn_Submission\":\n", - " continue\n", - " else:\n", - " # get paper openreview id\n", - " paper_id = submission.id\n", - " if \"pdf\" in submission.content:\n", - " pdf_link = submission.content[\"pdf\"][\"value\"]\n", - " pdf_path = str(pdf_dir)+str(paper_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " get_paper_pdf(pdf_link, pdf_path, log_file)\n", - " \n", - " revisions = client_v2.get_note_edits(note_id=paper_id)\n", - " if len(revisions) <= 1:\n", - " continue\n", - " else:\n", - " for revision in revisions:\n", - " pdf_revision_id = revision.id\n", - " pdf_path = str(pdf_dir)+str(pdf_revision_id)+\".pdf\"\n", - " if os.path.isfile(pdf_path):\n", - " continue\n", - " else:\n", - " time.sleep(1)\n", - " get_revision_pdf(venue, pdf_revision_id, pdf_path, log_file)\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "markdown", - "id": "7e699b0a", - "metadata": {}, - "source": [ - "##### construct the table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de3a1a9e", - "metadata": {}, - "outputs": [], - "source": [ - "venue = \"ICLR.cc/2025/Conference\"\n", - "filter_list = [\"Under review as a conference paper at ICLR 2025\", \"Published as a conference paper at ICLR 2025\"]\n", - "pdf_dir = \"/data/jingjunx/openreview_pdfs_2025/\"\n", - "log_file = \"./log/failed_ids_revisions_2025.log\"\n", - "config = {\"venue\": venue, \"filter_list\": filter_list, \"pdf_dir\": pdf_dir, \"log_file\": log_file, \"is_paper\": True, \"is_revision\": True, \"is_pdf_delete\": False}\n", - "research_arcade.construct_table_from_api(\"openreview_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "aba41ea1", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bae71acc", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_paragraphs_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "d518ea2c", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7f2a0054", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_paragraphs_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "dda0fdb1", - "metadata": {}, - "source": [ - "#### insert node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bceb66f8", - "metadata": {}, - "outputs": [], - "source": [ - "paragraph_feature = {'venue': 'xujj_test', \n", - " 'paper_openreview_id': 'xujj_test', \n", - " 'paragraph_idx': 1, \n", - " 'section': \"xujj_test\", \n", - " 'content': \"xujj_test\"}\n", - "research_arcade.insert_node(\"openreview_paragraphs\", node_features=paragraph_feature)" - ] - }, - { - "cell_type": "markdown", - "id": "121ca021", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9467f07", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"xujj_test\"}\n", - "paragraph_feature = research_arcade.delete_node_by_id(\"openreview_paragraphs\", paper_id)\n", - "print(len(paragraph_feature))\n", - "print(paragraph_feature.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "34173e7d", - "metadata": {}, - "source": [ - "#### get all nodes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f82b7ece", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_paragraphs_df = research_arcade.get_all_node_features(\"openreview_paragraphs\")\n", - "print(len(openreview_paragraphs_df))" - ] - }, - { - "cell_type": "markdown", - "id": "e9e6973e", - "metadata": {}, - "source": [ - "#### get specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5208ab0a", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"ryxB0Rtxx\"}\n", - "paragraph_feature = research_arcade.get_node_features_by_id(\"openreview_paragraphs\", paper_id)\n", - "print(paragraph_feature.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "084e3495", - "metadata": {}, - "source": [ - "## openreview_arxiv" - ] - }, - { - "cell_type": "markdown", - "id": "d651b5e3", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "623fb599", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2017/conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_arxiv\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "6874601b", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "743590ab", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_arxiv_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_arxiv\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "0f4521ee", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fa1f89b8", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_arxiv_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_arxiv\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "0e577ec2", - "metadata": {}, - "source": [ - "#### insert edge" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16deb344", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_arxiv = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'paper_openreview_id': 'zkNCWtw2fd', \n", - " 'arxiv_id': 'http://arxiv.org/abs/2408.10536v1', \n", - " 'title': 'Synergistic Approach for Simultaneous Optimization of Monolingual, Cross-lingual, and Multilingual Information Retrieval'\n", - "}\n", - "research_arcade.insert_edge(\"openreview_arxiv\", openreview_arxiv)" - ] - }, - { - "cell_type": "markdown", - "id": "6948e86a", - "metadata": {}, - "source": [ - "#### delete specific edge by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dcc7f437", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_id = {\"paper_openreview_id\": \"zkNCWtw2fd\"}\n", - "openreview_arxiv_df = research_arcade.delete_edge_by_id(\"openreview_arxiv\", openreview_id)\n", - "print(openreview_arxiv_df.to_dict(orient=\"records\")[0])\n", - "\n", - "arxiv_id = {\"arxiv_id\": \"http://arxiv.org/abs/2408.10536v1\"}\n", - "openreview_arxiv_df = research_arcade.delete_edge_by_id(\"openreview_arxiv\", arxiv_id)\n", - "print(openreview_arxiv_df.to_dict(orient=\"records\")[0])\n", - "\n", - "openreview_arxiv_id = {\"paper_openreview_id\": \"zkNCWtw2fd\", \"arxiv_id\": \"http://arxiv.org/abs/2408.10536v1\"}\n", - "openreview_arxiv_df = research_arcade.delete_edge_by_id(\"openreview_arxiv\", openreview_arxiv_id)\n", - "print(openreview_arxiv_df.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "0e8a48b7", - "metadata": {}, - "source": [ - "#### get all edges" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c076926", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_arxiv_df = research_arcade.get_all_edge_features(\"openreview_arxiv\")\n", - "print(len(openreview_arxiv_df))" - ] - }, - { - "cell_type": "markdown", - "id": "94f938cc", - "metadata": {}, - "source": [ - "#### get neighborhood by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3d86236b", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_id = {\"paper_openreview_id\": \"zkNCWtw2fd\"}\n", - "openreview_arxiv_df = research_arcade.get_neighborhood(\"openreview_arxiv\", openreview_id)\n", - "print(openreview_arxiv_df.to_dict(orient=\"records\")[0])\n", - "\n", - "arxiv_id = {\"arxiv_id\": \"http://arxiv.org/abs/2408.10536v1\"}\n", - "openreview_arxiv_df = research_arcade.get_neighborhood(\"openreview_arxiv\", arxiv_id)\n", - "print(openreview_arxiv_df.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "03795c8d", - "metadata": {}, - "source": [ - "## openreview_papers_authors" - ] - }, - { - "cell_type": "markdown", - "id": "1e0e5be0", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7dbe3ae2", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2025/Conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_papers_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "6f24e934", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69886e20", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_authors_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_papers_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "b8ce80fd", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9525c3cc", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_authors_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_papers_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "f6e68945", - "metadata": {}, - "source": [ - "#### insert edge" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2609d33c", - "metadata": {}, - "outputs": [], - "source": [ - "paper_authors = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Zaid_Khan1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Jaemin_Cho1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Mohit_Bansal2'}]\n", - "for item in paper_authors:\n", - " research_arcade.insert_edge(\"openreview_papers_authors\", item)\n", - "\n", - "author_papers = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'Xbl6t6zxZs', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'fDcn3S8oAt', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'j9wBgcxa7N', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'zd0iX5xBhA', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2024/Conference', 'paper_openreview_id': 'L4nOxziGf9', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2024/Conference', 'paper_openreview_id': 'qL9gogRepu', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}]\n", - "for item in author_papers:\n", - " research_arcade.insert_edge(\"openreview_papers_authors\", item)\n", - "\n", - "paper_author = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}]\n", - "for item in paper_author:\n", - " research_arcade.insert_edge(\"openreview_papers_authors\", item)" - ] - }, - { - "cell_type": "markdown", - "id": "de351a73", - "metadata": {}, - "source": [ - "#### delete specific edge by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "00deb3c9", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"00SnKBGTsz\"}\n", - "openreview_papers_authors = research_arcade.delete_edge_by_id(\"openreview_papers_authors\", paper_id)\n", - "print(openreview_papers_authors.to_dict(orient=\"records\"))\n", - "\n", - "author_id = {'author_openreview_id': '~Elias_Stengel-Eskin1'}\n", - "openreview_papers_authors = research_arcade.delete_edge_by_id(\"openreview_papers_authors\", author_id)\n", - "print(openreview_papers_authors.to_dict(orient=\"records\"))\n", - "\n", - "paper_author = {\"paper_openreview_id\": \"00SnKBGTsz\", 'author_openreview_id': '~Elias_Stengel-Eskin1'}\n", - "openreview_papers_authors = research_arcade.delete_edge_by_id(\"openreview_papers_authors\", paper_author)\n", - "print(openreview_papers_authors.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "1b449550", - "metadata": {}, - "source": [ - "#### get all edges" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a997930", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_papers_authors = research_arcade.get_all_edge_features(\"openreview_papers_authors\")\n", - "print(len(openreview_papers_authors))" - ] - }, - { - "cell_type": "markdown", - "id": "dac86a39", - "metadata": {}, - "source": [ - "#### get neighborhood by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9c82adba", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"00SnKBGTsz\"}\n", - "openreview_papers_authors = research_arcade.get_neighborhood(\"openreview_papers_authors\", paper_id)\n", - "print(openreview_papers_authors.to_dict(orient=\"records\"))\n", - "\n", - "author_id = {'author_openreview_id': '~Elias_Stengel-Eskin1'}\n", - "openreview_papers_authors = research_arcade.get_neighborhood(\"openreview_papers_authors\", author_id)\n", - "print(openreview_papers_authors.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "ddcc5502", - "metadata": {}, - "source": [ - "## openreview_papers_reviews" - ] - }, - { - "cell_type": "markdown", - "id": "7601bf42", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "92ecfbf5", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2017/conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_papers_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "93a50568", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cb035544", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_reviews_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_papers_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "6dfaa92f", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "89ab8577", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_reviews_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_papers_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "e75e7088", - "metadata": {}, - "source": [ - "#### insert edge" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0e3596f6", - "metadata": {}, - "outputs": [], - "source": [ - "paper_review = {'venue': 'ICLR.cc/2025/Conference', \n", - " 'paper_openreview_id': '00SnKBGTsz', \n", - " 'review_openreview_id': '13mj0Rtn5W', \n", - " 'title': 'Response by Authors', \n", - " 'time': '2024-11-27 17:27:45'}\n", - "research_arcade.insert_edge(\"openreview_papers_reviews\", paper_review)\n", - "\n", - "paper_reviews = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '7XT4kLWV2f', 'title': 'Official Review by Reviewer_wuGW', 'time': '2024-11-01 14:52:22'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'i3QgWgrJff', 'title': 'Official Review by Reviewer_rVo8', 'time': '2024-11-04 02:37:10'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'GMsjHLXdOx', 'title': 'Official Review by Reviewer_c5nB', 'time': '2024-11-04 09:59:14'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'r8ZflFk3T7', 'title': 'Official Review by Reviewer_VQ9Y', 'time': '2024-11-06 00:15:47'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '4CnQpVCYkF', 'title': 'Response by Authors', 'time': '2024-11-20 22:48:42'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'h1qvpjhRP3', 'title': 'Response by Authors', 'time': '2024-11-20 22:51:07'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'pOR42YNLtU', 'title': 'Response by Authors', 'time': '2024-11-20 22:55:04'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'Aq2tBtB0lt', 'title': 'Response by Authors', 'time': '2024-11-20 22:57:18'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'm1iUqPHpwk', 'title': 'Response by Authors', 'time': '2024-11-20 22:58:29'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '66buacQmRe', 'title': 'Response by Authors', 'time': '2024-11-20 23:02:21'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'Bgr7Ol90m7', 'title': 'Response by Authors', 'time': '2024-11-22 23:11:06'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'H2h2K6a8x5', 'title': 'Response by Reviewer', 'time': '2024-11-23 10:04:58'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'la5jPwJU4g', 'title': 'Response by Authors', 'time': '2024-11-24 19:17:22'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'DjVKsUoFN2', 'title': 'Response by Reviewer', 'time': '2024-11-25 04:00:18'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'C3MhCuKhTf', 'title': 'Response by Authors', 'time': '2024-11-25 19:44:38'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'ZqwAYtcmhv', 'title': 'Response by Authors', 'time': '2024-11-25 19:45:43'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '9OQJoesINr', 'title': 'Response by Reviewer', 'time': '2024-11-25 20:07:51'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'wqTNtVDwef', 'title': 'Response by Authors', 'time': '2024-11-26 03:32:30'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'NEsxOTkkIV', 'title': 'Response by Reviewer', 'time': '2024-11-26 20:00:00'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '13mj0Rtn5W', 'title': 'Response by Authors', 'time': '2024-11-27 17:27:45'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'hWat8aFBRw', 'title': 'Response by Reviewer', 'time': '2024-11-27 11:34:03'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'wnsiUkDh00', 'title': 'Response by Authors', 'time': '2024-11-27 17:28:35'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'zpboemkkjR', 'title': 'Meta Review of Submission11063 by Area_Chair_eoLd', 'time': '2024-12-20 15:14:25'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'kokKFEn2fw', 'title': 'Paper Decision', 'time': '2025-01-22 05:35:00'}\n", - "]\n", - "for item in tqdm(paper_reviews):\n", - " research_arcade.insert_edge(\"openreview_papers_reviews\", item)" - ] - }, - { - "cell_type": "markdown", - "id": "c2d8f462", - "metadata": {}, - "source": [ - "#### delete specific edge by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "102f2179", - "metadata": {}, - "outputs": [], - "source": [ - "paper_review_id = {\"paper_openreview_id\": \"00SnKBGTsz\", \"review_openreview_id\": \"13mj0Rtn5W\"}\n", - "openreview_papers_reviews = research_arcade.delete_edge_by_id(\"openreview_papers_reviews\", paper_review_id)\n", - "print(openreview_papers_reviews.to_dict(orient=\"records\"))\n", - "\n", - "review_id = {\"review_openreview_id\": \"13mj0Rtn5W\"}\n", - "openreview_papers_reviews = research_arcade.delete_edge_by_id(\"openreview_papers_reviews\", review_id)\n", - "print(openreview_papers_reviews.to_dict(orient=\"records\"))\n", - "\n", - "paper_id = {\"paper_openreview_id\": \"00SnKBGTsz\"}\n", - "openreview_papers_reviews = research_arcade.delete_edge_by_id(\"openreview_papers_reviews\", paper_id)\n", - "print(openreview_papers_reviews.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "8c3fcb06", - "metadata": {}, - "source": [ - "#### get all edges" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2164207", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_papers_reviews = research_arcade.get_all_edge_features(\"openreview_papers_reviews\")\n", - "print(len(openreview_papers_reviews))" - ] - }, - { - "cell_type": "markdown", - "id": "8b8fa9b0", - "metadata": {}, - "source": [ - "#### get neighborhood by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "77c645be", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"00SnKBGTsz\"}\n", - "openreview_papers_reviews = research_arcade.get_neighborhood(\"openreview_papers_reviews\", paper_id)\n", - "print(openreview_papers_reviews.to_dict(orient=\"records\"))\n", - "\n", - "review_id = {\"review_openreview_id\": \"13mj0Rtn5W\"}\n", - "openreview_papers_reviews = research_arcade.get_neighborhood(\"openreview_papers_reviews\", review_id)\n", - "print(openreview_papers_reviews.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "c10236f8", - "metadata": {}, - "source": [ - "## openreview_papers_revisions" - ] - }, - { - "cell_type": "markdown", - "id": "a9a75850", - "metadata": {}, - "source": [ - "#### construct table from api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b9e98a93", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"venue\": \"ICLR.cc/2025/Conference\"}\n", - "research_arcade.construct_table_from_api(\"openreview_papers_revisions\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "c7095bc3", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2e858b2d", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_revisions_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_papers_revisions\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "e3f77782", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cac9628f", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_revisions_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_papers_revisions\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "aee6fe7c", - "metadata": {}, - "source": [ - "#### insert edge" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5bc96388", - "metadata": {}, - "outputs": [], - "source": [ - "paper_revision = {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'dzL3IRBnE4', 'title': 'Camera_Ready_Revision', 'time': '2025-03-01 03:36:55'}\n", - "research_arcade.insert_edge(\"openreview_papers_revisions\", paper_revision)\n", - "\n", - "paper_revisions = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'oT4N28siLO', 'title': 'Camera_Ready_Revision', 'time': '2025-03-02 01:35:16'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'dzL3IRBnE4', 'title': 'Camera_Ready_Revision', 'time': '2025-03-01 03:36:55'}]\n", - "for item in tqdm(paper_revisions):\n", - " research_arcade.insert_edge(\"openreview_papers_revisions\", item)" - ] - }, - { - "cell_type": "markdown", - "id": "5716690e", - "metadata": {}, - "source": [ - "#### delete specific node by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f4ae0e07", - "metadata": {}, - "outputs": [], - "source": [ - "paper_revision_id = {\"paper_openreview_id\": \"00SnKBGTsz\", \"revision_openreview_id\": \"dzL3IRBnE4\"}\n", - "paper_revision = research_arcade.delete_edge_by_id(\"openreview_papers_revisions\", paper_revision_id)\n", - "print(paper_revision.to_dict(orient=\"records\"))\n", - "\n", - "revision_id = {\"revision_openreview_id\": \"dzL3IRBnE4\"}\n", - "paper_revision = research_arcade.delete_edge_by_id(\"openreview_papers_revisions\", revision_id)\n", - "print(paper_revision.to_dict(orient=\"records\"))\n", - "\n", - "paper_id = {\"paper_openreview_id\": \"00SnKBGTsz\"}\n", - "paper_revision = research_arcade.delete_edge_by_id(\"openreview_papers_revisions\", paper_id)\n", - "print(paper_revision.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "4e733028", - "metadata": {}, - "source": [ - "#### get all edges" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce044128", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_papers_revisions = research_arcade.get_all_edge_features(\"openreview_papers_revisions\")\n", - "print(len(openreview_papers_revisions))" - ] - }, - { - "cell_type": "markdown", - "id": "7e6863ba", - "metadata": {}, - "source": [ - "#### get neighborhood by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e57a9a5", - "metadata": {}, - "outputs": [], - "source": [ - "paper_id = {\"paper_openreview_id\": \"00SnKBGTsz\"}\n", - "paper_revision = research_arcade.get_neighborhood(\"openreview_papers_revisions\", paper_id)\n", - "print(paper_revision.to_dict(orient=\"records\"))\n", - "\n", - "revision_id = {\"revision_openreview_id\": \"dzL3IRBnE4\"}\n", - "paper_revision = research_arcade.get_neighborhood(\"openreview_papers_revisions\", revision_id)\n", - "print(paper_revision.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "ef38635e", - "metadata": {}, - "source": [ - "## openreview_revisions_reviews" - ] - }, - { - "cell_type": "markdown", - "id": "64c2cc79", - "metadata": {}, - "source": [ - "#### construct table based on existing tables" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7200b32a", - "metadata": {}, - "outputs": [], - "source": [ - "papers_reviews_df = research_arcade.get_all_edge_features(\"openreview_papers_reviews\")\n", - "print(len(papers_reviews_df))\n", - "papers_revisions_df = research_arcade.get_all_edge_features(\"openreview_papers_revisions\")\n", - "print(len(papers_revisions_df))\n", - "config = {\"papers_reviews_df\": papers_reviews_df, \"papers_revisions_df\": papers_revisions_df}\n", - "research_arcade.construct_table_from_api(\"openreview_revisions_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "45986e7a", - "metadata": {}, - "source": [ - "#### construct table from csv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2139877b", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"csv_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_revisions_reviews_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"openreview_revisions_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "6122bba4", - "metadata": {}, - "source": [ - "#### construct table from json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1dfcb6b8", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"json_file\": \"/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_revisions_reviews_example.json\"}\n", - "research_arcade.construct_table_from_json(\"openreview_revisions_reviews\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "36040c82", - "metadata": {}, - "source": [ - "#### insert edge" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "53b9bf25", - "metadata": {}, - "outputs": [], - "source": [ - "revision_review = {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'}\n", - "research_arcade.insert_edge(\"openreview_revisions_reviews\", revision_review)\n", - "\n", - "revision_reviews = [{'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '138cOdBpgA'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'yKh1fQYnUZ'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'Pvt0OjNSp2'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'MUhlEYyBD9'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '2mqiS3J8wC'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'Er8QTorcyr'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'AvtD9uxRtX'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '2tgxTGynNm'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '5MKJE3sFsd'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wViZ0H4ErF'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '0c1It75dTb'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'PFwia9lcjP'}, \n", - " {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'ygCqaGNPee'}]\n", - "for item in tqdm(revision_reviews):\n", - " research_arcade.insert_edge(\"openreview_revisions_reviews\", item)" - ] - }, - { - "cell_type": "markdown", - "id": "3c97ba61", - "metadata": {}, - "source": [ - "#### delete edge by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c14cc2e7", - "metadata": {}, - "outputs": [], - "source": [ - "revision_review_id = {'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'}\n", - "revision_review = research_arcade.delete_edge_by_id(\"openreview_revisions_reviews\", revision_review_id)\n", - "print(revision_review.to_dict(orient=\"records\"))\n", - "\n", - "review_id = {'review_openreview_id': 'wumckDPIQ3'}\n", - "revision_review = research_arcade.delete_edge_by_id(\"openreview_revisions_reviews\", review_id)\n", - "print(revision_review.to_dict(orient=\"records\"))\n", - "\n", - "paper_id = {'revision_openreview_id': 'cX02yuzwWI'}\n", - "revision_review = research_arcade.delete_edge_by_id(\"openreview_revisions_reviews\", paper_id)\n", - "print(revision_review.to_dict(orient=\"records\"))" - ] - }, - { - "cell_type": "markdown", - "id": "2380217c", - "metadata": {}, - "source": [ - "#### get all edges" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73d269e4", - "metadata": {}, - "outputs": [], - "source": [ - "openreview_revisions_reviews = research_arcade.get_all_edge_features(\"openreview_revisions_reviews\")\n", - "print(len(openreview_revisions_reviews))" - ] - }, - { - "cell_type": "markdown", - "id": "ac02c01e", - "metadata": {}, - "source": [ - "#### get neighborhood by id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72a9434e", - "metadata": {}, - "outputs": [], - "source": [ - "revision_id = {'revision_openreview_id': 'cX02yuzwWI'}\n", - "revision_review = research_arcade.get_neighborhood(\"openreview_revisions_reviews\", revision_id)\n", - "print(revision_review.to_dict(orient=\"records\"))\n", - "\n", - "review_id = {'review_openreview_id': 'wumckDPIQ3'}\n", - "revision_review = research_arcade.get_neighborhood(\"openreview_revisions_reviews\", review_id)\n", - "print(revision_review.to_dict(orient=\"records\"))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "research_arcade", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3357af0..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -arxiv==2.2.0 -openreview==0.0.2 -openreview_py==1.51.2 -pandas==2.3.3 -pdfminer==20191125 -pdfminer_six==20250506 -psycopg2==2.9.10 -Requests==2.32.5 -tqdm==4.67.1 diff --git a/research_arcade/.DS_Store b/research_arcade/.DS_Store deleted file mode 100644 index 2626417..0000000 Binary files a/research_arcade/.DS_Store and /dev/null differ diff --git a/research_arcade/__init__.py b/research_arcade/__init__.py deleted file mode 100644 index 629a6c8..0000000 --- a/research_arcade/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .research_arcade import ResearchArcade - -__all__ = [ - 'ResearchArcade' -] \ No newline at end of file diff --git a/research_arcade/arxiv_utils/arxiv_crawler.py b/research_arcade/arxiv_utils/arxiv_crawler.py deleted file mode 100644 index 0d05353..0000000 --- a/research_arcade/arxiv_utils/arxiv_crawler.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys -import os - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -from ..arxiv_utils.paper_crawler.crawler_job import CrawlerJob - - -class ArxivCrawler: - def __init__(self, dest_dir): - self.cj = CrawlerJob(dest_dir=dest_dir) - - def crawl_paper_by_time(self, start_date, end_date, dest_path): - - arxiv_ids = self.cj.crawl_recent_arxiv_paper_new(start_date=start_date, end_date=end_date, path=dest_path) - - return arxiv_ids - - # Wait a second... is this method sql based or csv based? - def download_papers(self, arxiv_ids): - self.cj.download_papers(arxiv_ids=arxiv_ids) - - def process_paper_graphs(self, arxiv_ids): - self.cj.process_paper_graphs(arxiv_ids=arxiv_ids) - - def process_paper_paragraphs(self, arxiv_ids): - self.cj.process_paper_paragraphs(arxiv_ids=arxiv_ids) - - def crawl_paper_data_from_api(self, arxiv_ids): - pass - - def crawl_section_data_from_api(self, arxiv_ids): - pass - - def crawl_paragraph_data_from_api(self, arxiv_ids): - # Process first then tackle down the problems - pass - - - def crawl_category_data_from_api(self, arxiv_ids): - # Give the category of data - pass - - def crawl_paper_category_data_from_api(self, arxiv_ids): - # Process first then tackle down the problems - pass - - def crawl_author_data_from_api(self, arxiv_ids): - # Given the arxiv ids, search the authors on semantic scholar - # This also returns the edge information? Edge between papers and authors - pass - - def crawl_figure_data_from_api(self, arxiv_ids): - - pass - - def crawl_table_data_from_api(self, arxiv_ids): - pass - - def crawl_paragraph_references_data_from_api(self, arxiv_ids): - pass - - diff --git a/research_arcade/arxiv_utils/graph_constructor/README.md b/research_arcade/arxiv_utils/graph_constructor/README.md deleted file mode 100644 index a9aece4..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/README.md +++ /dev/null @@ -1,22 +0,0 @@ -This directory stores the hetero graph database of information extracted from paper, including paper, figures, author, and so on. The information is stored as nodes and connected with edges reflecting the corresponding relations, including citation authorship, etc. - -Current nodes and their attributes: -1. papers: id, arxiv_id, title, abstract, submit_data, authors (which will be included in the edge tables?), metadata(url, version, categories, etc.) -2. authors: id, semantic scholar id, name, orcid, pubs? (to be included in the edge tables) -3. categories? -4. figures: id, paper_id (also in tables?), figure index/path, caption (text) -5. tables: id, paper_id (tables?), path?, caption -6. institution? - - - -Current Edges/Relations: - -1. paper_authors: paper id to author id, sequence -2. paper_category: paper id to category id -3. citations: paper id to paper id -4. paper figure: paper id to table/figure id -5. author affiliation: author id to affilitation id - -The idea_extractor here is originally for extracting ideas from the papers. Since this step is not of high priority, ignore it first. - diff --git a/research_arcade/arxiv_utils/graph_constructor/citation_processor.py b/research_arcade/arxiv_utils/graph_constructor/citation_processor.py deleted file mode 100644 index 586928f..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/citation_processor.py +++ /dev/null @@ -1,77 +0,0 @@ -from semanticscholar import SemanticScholar -import os -import re -import psycopg2 - -sch = SemanticScholar(api_key=os.getenv("S2_API_KEY")) - -def normalize_arxiv(aid: str) -> str: - core = re.sub(r'(?i)^arxiv:', '', aid) - core = re.sub(r'v\d+$', '', core) - return f"ARXIV:{core}" - -def cited_abstracts(arxiv_id, limit = 100): - pid = normalize_arxiv(arxiv_id) - - # print(f"pid: {pid}") - - # just plain fields — no "paper." prefix - refs = sch.get_paper_references( - pid, fields=["paperId", "title", "abstract", "year", "externalIds"], limit=limit - ) - - results = [] - for ref in refs: - p = ref.paper # this is already a Paper object - results.append({ - "paperId": p.paperId, - "title": p.title, - "year": p.year, - "abstract": p.abstract, - "externalIds": p.externalIds, - }) - return results - - -def citation_processing(arxiv_ids, limit = 100): - - # First connect to database - conn = psycopg2.connect( - host="localhost", - port="5433", - dbname="postgres", - user="cl195" - ) - conn.autocommit = True - cur = conn.cursor() - sql = """ - INSERT INTO citation_sch (arxiv_id, paper_id, title, year, abstract, external_ids) - VALUES (%s, %s, %s, %s, %s, %s) - RETURNING id - """ - - for arxiv_id in arxiv_ids: - data = cited_abstracts(arxiv_id, limit=limit) - - for datum in data: - - paperId = datum["paperId"] - title = datum["title"] - year = datum["year"] - abstract = datum["abstract"] - externalIds = datum["externalIds"] - cur.execute(sql, (arxiv_id, paperId, title, year, abstract, str(externalIds))) - - - - # Inser the stuff into database -# # Example -# data = cited_abstracts("1706.03762") -# print(len(data), "references") - -# for datum in data: - -# print(datum["title"]) -# print(datum["abstract"]) - - diff --git a/research_arcade/arxiv_utils/graph_constructor/database.py b/research_arcade/arxiv_utils/graph_constructor/database.py deleted file mode 100644 index ab8d1eb..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/database.py +++ /dev/null @@ -1,721 +0,0 @@ -import psycopg2 -from psycopg2.extras import Json -import json - -# Store the pwd of db server in the env or here as a global variable -# PASSWORD = - -class Database: - - def __init__(self): - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host="localhost", - port="5433", - dbname="postgres", - user="cl195" - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - - def create_papers_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS papers ( - id SERIAL PRIMARY KEY, - arxiv_id VARCHAR(100) UNIQUE, - base_arxiv_id VARCHAR(100), - version VARCHAR(100), - title TEXT NOT NULL, - abstract TEXT, - submit_date DATE, - metadata JSONB - ) - """) - - def create_sections_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS sections ( - id SERIAL PRIMARY KEY, - content TEXT, - title TEXT, - appendix BOOLEAN, - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE - ) - """) - - def create_paragraphs_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paragraphs ( - id SERIAL PRIMARY KEY, - paragraph_id INT NOT NULL, - content TEXT, - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - paper_section TEXT, - UNIQUE (paragraph_id, paper_arxiv_id, paper_section) - ); - """) - - def create_authors_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS authors ( - id SERIAL PRIMARY KEY, - semantic_scholar_id VARCHAR(100) UNIQUE, - name VARCHAR(255) NOT NULL, - homepage VARCHAR(255) - ) - """) - - def create_categories_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS categories ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) UNIQUE NOT NULL, - description TEXT - ) - """) - - def create_institutions_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS institutions ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL, - location VARCHAR(255) - ) - """) - - def create_figures_table(self): - # Change paper_id to paper_arxiv_id VARCHAR referencing papers(arxiv_id) - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS figures ( - id SERIAL PRIMARY KEY, - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - path VARCHAR(500), - caption TEXT, - label TEXT, - name TEXT - ) - """) - - def create_tables_table(self): - # Note: 'tables' here is literal; change paper_id to paper_arxiv_id referencing papers(arxiv_id) - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS tables ( - id SERIAL PRIMARY KEY, - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - path VARCHAR(500), - caption TEXT, - label TEXT, - table_text TEXT - ) - """) - - def create_paper_authors_table(self): - # Link by paper_arxiv_id (string) and author_id (int) - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paper_authors ( - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - author_id VARCHAR(100) NOT NULL REFERENCES authors(semantic_scholar_id) ON DELETE CASCADE, - author_sequence INT NOT NULL, - PRIMARY KEY (paper_arxiv_id, author_id) - ) - """) - - def create_paper_category_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paper_category ( - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - category_id INT NOT NULL REFERENCES categories(id) ON DELETE CASCADE, - PRIMARY KEY (paper_arxiv_id, category_id) - ) - """) - - def create_citations_table(self): - # citation edges by arxiv_id strings - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS citations ( - id SERIAL PRIMARY KEY, - citing_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - cited_arxiv_id VARCHAR(100), - bib_title TEXT, - bib_key VARCHAR(255), - author_cited_paper VARCHAR(255), - citing_sections TEXT[] DEFAULT '{}', - citing_paragraphs INT[] DEFAULT '{}', - UNIQUE (citing_arxiv_id, cited_arxiv_id) - ) - """) - - - def create_paper_figures_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paper_figures ( - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - figure_id INT NOT NULL REFERENCES figures(id) ON DELETE CASCADE, - PRIMARY KEY (paper_arxiv_id, figure_id) - ) - """) - - def create_paper_tables_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paper_tables ( - paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - table_id INT NOT NULL REFERENCES tables(id) ON DELETE CASCADE, - PRIMARY KEY (paper_arxiv_id, table_id) - ) - """) - - def create_paragraph_citations_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paragraph_citations ( - id SERIAL PRIMARY KEY, - paragraph_id INT NOT NULL, - paper_section TEXT, - citing_arxiv_id VARCHAR(100) NOT NULL - REFERENCES papers(arxiv_id) - ON DELETE CASCADE, - bib_key VARCHAR(255) - ); - """) - - - def create_paragraph_references_table(self): - """ - Stores references that appear within paragraphs— - e.g. figures, tables, equations, etc. - """ - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paragraph_references ( - id SERIAL PRIMARY KEY, - paragraph_id INT NOT NULL, - paper_section TEXT, - paper_arxiv_id VARCHAR(100) NOT NULL - REFERENCES papers(arxiv_id) - ON DELETE CASCADE, - reference_label TEXT NOT NULL, - reference_type TEXT - ); - """) - - def create_author_affiliation_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS author_affiliation ( - author_id INT NOT NULL REFERENCES authors(id) ON DELETE CASCADE, - institution_id INT NOT NULL REFERENCES institutions(id) ON DELETE CASCADE, - PRIMARY KEY (author_id, institution_id) - ) - """) - - def create_citation_sch_table(self): - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS citation_sch( - id SERIAL PRIMARY KEY, - arxiv_id VARCHAR(100) NOT NULL, - paper_id VARCHAR(100), - title TEXT NOT NULL, - year VARCHAR(100), - abstract TEXT, - external_ids TEXT - ); - """) - - - def create_all(self): - # Create tables in order respecting dependencies - self.create_papers_table() - self.create_sections_table() - self.create_paragraphs_table() - self.create_authors_table() - self.create_categories_table() - self.create_institutions_table() - self.create_figures_table() - self.create_tables_table() - self.create_paper_authors_table() - self.create_paper_category_table() - self.create_citations_table() - self.create_paper_figures_table() - self.create_paper_tables_table() - self.create_author_affiliation_table() - self.create_paragraph_references_table() - self.create_paragraph_citations_table() - - # def create_papers_table(self): - # self.cur.execute(""" - # CREATE TABLE IF NOT EXISTS papers ( - # id SERIAL PRIMARY KEY, - # arxiv_id VARCHAR(100) UNIQUE, - # base_arxiv_id VARCHAR(100), - # version VARCHAR(100), - # title TEXT NOT NULL, - # abstract TEXT, - # submit_date DATE, - # metadata JSONB - # ) - # """) - - # Insert methods for papers/authors/etc remain the same for papers: - def insert_paper(self, arxiv_id, base_arxiv_id, version, title, abstract=None, submit_date=None, metadata=None): - """ - Insert a paper. Returns the generated paper id. - - arxiv_id: str (unique) or None - - title: str - - abstract: str or None - - submit_date: datetime.date or ISO-format str or None - - metadata: dict or None - """ - sql = """ - INSERT INTO papers (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (arxiv_id) DO NOTHING - RETURNING id - """ - meta_val = Json(metadata) if metadata is not None else None - self.cur.execute(sql, (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, meta_val)) - res = self.cur.fetchone() - return res[0] if res else None - - def insert_section(self, content, title, is_appendix, paper_arxiv_id): - """ - Insert a section. Returns the generated section id - - content: str - - title: str - - is_appendix: boolean - - paper_arxiv_id: str - """ - sql = """ - INSERT INTO sections (content, title, appendix, paper_arxiv_id) - VALUES(%s, %s, %s, %s) - RETURNING id - """ - self.cur.execute(sql, (content, title, is_appendix, paper_arxiv_id)) - res = self.cur.fetchone() - return res[0] if res else None - - # def create_paragraphs_table(self): - # self.cur.execute(""" - # CREATE TABLE IF NOT EXISTS paragraphs ( - # id SERIAL PRIMARY KEY, - # paragraph_id VARCHAR(100), - # content TEXT, - # paper_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - # paper_section TEXT NOT NULL REFERENCES sections(title) ON DELETE CASCADE - # ) - # """) - - # Here we don't have an extra paragraph-section or paragraph-paper table since the link information is already included here. - def insert_paragraph(self, paragraph_id, content, paper_arxiv_id, paper_section): - sql = """ - INSERT INTO paragraphs (paragraph_id, content, paper_arxiv_id, paper_section) - VALUES (%s, %s, %s, %s) - ON CONFLICT (paragraph_id, paper_arxiv_id, paper_section) DO NOTHING - RETURNING id - """ - self.cur.execute(sql, (paragraph_id, content, paper_arxiv_id, paper_section)) - res = self.cur.fetchone() - return res[0] if res else None - - - - def insert_author(self, semantic_scholar_id, name, homepage=None): - sql = """ - INSERT INTO authors (semantic_scholar_id, name, homepage) - VALUES (%s, %s, %s) - ON CONFLICT (semantic_scholar_id) DO NOTHING - RETURNING id - """ - self.cur.execute(sql, (semantic_scholar_id, name, homepage)) - res = self.cur.fetchone() - return res[0] if res else None - - def insert_category(self, name, description=None): - sql = """ - INSERT INTO categories (name, description) - VALUES (%s, %s) - ON CONFLICT (name) DO NOTHING - RETURNING id - """ - self.cur.execute(sql, (name, description)) - res = self.cur.fetchone() - if res: - return res[0] - # conflict: fetch existing - self.cur.execute("SELECT id FROM categories WHERE name = %s", (name,)) - return self.cur.fetchone()[0] - - def insert_institution(self, name, location=None): - sql = """ - INSERT INTO institutions (name, location) - VALUES (%s, %s) - ON CONFLICT (name) DO NOTHING - RETURNING id - """ - self.cur.execute(sql, (name, location)) - res = self.cur.fetchone() - return res[0] if res else None - - # Insert figure: now paper_arxiv_id instead of numeric paper_id - def insert_figure(self, paper_arxiv_id, path, caption=None, label=None, name=None): - """ - Insert a figure. Returns the generated figure id. - - paper_arxiv_id: str (the arxiv_id of the paper) - - path: str or None - - caption: str or None - - label: str or None - - name: str or None - """ - sql = """ - INSERT INTO figures (paper_arxiv_id, path, caption, label, name) - VALUES (%s, %s, %s, %s, %s) - RETURNING id - """ - # No ON CONFLICT here because we may allow multiple figures per paper. - self.cur.execute(sql, (paper_arxiv_id, path, caption, label, name)) - res = self.cur.fetchone() - return res[0] if res else None - - # Insert table record: now paper_arxiv_id - def insert_table(self, paper_arxiv_id, path=None, caption=None, label=None, table_text=None): - """ - Insert a table record. Returns the generated table id. - - paper_arxiv_id: str - - path: str or None - - caption: str or None - - label: str or None - - table_text: str or None - """ - sql = """ - INSERT INTO tables (paper_arxiv_id, path, caption, label, table_text) - VALUES (%s, %s, %s, %s, %s) - RETURNING id - """ - self.cur.execute(sql, (paper_arxiv_id, path, caption, label, table_text)) - res = self.cur.fetchone() - return res[0] if res else None - - # Link methods now take paper_arxiv_id (string) - def insert_paper_author(self, paper_arxiv_id, author_id, author_sequence): - """ - Link a paper and an author via arxiv_id. Returns True if inserted, False if already exists. - """ - sql = """ - INSERT INTO paper_authors (paper_arxiv_id, author_id, author_sequence) - VALUES (%s, %s, %s) - ON CONFLICT (paper_arxiv_id, author_id) DO NOTHING - """ - self.cur.execute(sql, (paper_arxiv_id, author_id, author_sequence)) - return self.cur.rowcount == 1 - - def insert_paper_category(self, paper_arxiv_id, category_id): - """ - Link a paper and a category via arxiv_id. Returns True if inserted, False if exists. - """ - sql = """ - INSERT INTO paper_category (paper_arxiv_id, category_id) - VALUES (%s, %s) - ON CONFLICT (paper_arxiv_id, category_id) DO NOTHING - """ - self.cur.execute(sql, (paper_arxiv_id, category_id)) - return self.cur.rowcount == 1 - - # def create_citations_table(self): - # # citation edges by arxiv_id strings - # self.cur.execute(""" - # CREATE TABLE IF NOT EXISTS citations ( - # citing_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - # cited_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - # citing_sections TEXT[] DEFAULT '{}', - # PRIMARY KEY (citing_arxiv_id, cited_arxiv_id), - # CHECK (citing_arxiv_id <> cited_arxiv_id) - # ) - # """) - - def insert_citation(self, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, author_cited_paper, citing_sections): - """ - Insert a citation edge by arxiv_id strings. Returns True if inserted, False if exists or invalid. - """ - - # self.cur.execute(""" - # CREATE TABLE IF NOT EXISTS citations ( - # id SERIAL PRIMARY KEY, - # citing_arxiv_id VARCHAR(100) NOT NULL REFERENCES papers(arxiv_id) ON DELETE CASCADE, - # cited_arxiv_id VARCHAR(100), - # bib_title TEXT, - # bib_key VARCHAR(255), - # author_cited_paper VARCHAR(255), - # citing_sections TEXT[] DEFAULT '{}', - # ) - # """) - - if citing_arxiv_id == cited_arxiv_id: - return False - sql = """ - INSERT INTO citations (citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, author_cited_paper, citing_sections) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (citing_arxiv_id, cited_arxiv_id) DO NOTHING - """ - self.cur.execute(sql, (citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, author_cited_paper, citing_sections)) - return self.cur.rowcount == 1 - - - def insert_citation_paragraph(self, paper_arxiv_id: str, paragraph_id: str, bib_key: str) -> None: - """ - Add a paragraph reference to the citing_paragraphs array for a given citation. - If the paragraph_id is already present, this will add a duplicate; if you - want to avoid duplicates, see the note below. - """ - sql = """ - UPDATE citations - SET citing_paragraphs = array_append( - COALESCE(citing_paragraphs, '{}'), %s) - WHERE citing_arxiv_id = %s - AND bib_key = %s - """ - try: - self.cur.execute(sql, (paragraph_id, paper_arxiv_id, bib_key)) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - # Optional: check rowcount to see if an update actually happened - if self.cur.rowcount == 0: - # no existing citation row to update - # you could choose to INSERT a new citation here if that makes sense: - # self.insert_citation(paper_arxiv_id, ..., citing_paragraphs=[paragraph_id]) - pass - - - - # CREATE TABLE IF NOT EXISTS citations ( - # id SERIAL PRIMARY KEY, - # paragraph_id INT NOT NULL, - # paper_section TEXT, - # citing_arxiv_id VARCHAR(100) NOT NULL - # REFERENCES papers(arxiv_id) - # ON DELETE CASCADE, - # bib_key VARCHAR(255), - # ) - - - def insert_paragraph_citations(self, paragraph_id, paper_section, citing_arxiv_id, bib_key): - sql = """ - INSERT INTO paragraph_citations - (paragraph_id, paper_section, citing_arxiv_id, bib_key) - VALUES (%s, %s, %s, %s) - RETURNING id - """ - self.cur.execute(sql, (paragraph_id, paper_section, citing_arxiv_id, bib_key)) - res = self.cur.fetchone() - return res[0] if res else None - - # self.cur.execute(""" - # CREATE TABLE IF NOT EXISTS paragraph_references ( - # id SERIAL PRIMARY KEY, - # paragraph_id INT NOT NULL, - # paper_section TEXT, - # paper_arxiv_id VARCHAR(100) NOT NULL - # REFERENCES papers(arxiv_id) - # ON DELETE CASCADE, - # reference_label TEXT NOT NULL, - # reference_type TEXT - # ); - # """) - - def insert_paragraph_reference(self, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type=None): - sql = """ - INSERT INTO paragraph_references - (paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - VALUES (%s, %s, %s, %s, %s) - RETURNING id - """ - self.cur.execute(sql, - (paragraph_id, - paper_section, - paper_arxiv_id, - reference_label, - reference_type)) - res = self.cur.fetchone() - return res[0] if res else None - - - - def insert_paper_figure(self, paper_arxiv_id, figure_id): - """ - Link paper to figure explicitly by arxiv_id. Returns True if inserted, False if exists. - """ - sql = """ - INSERT INTO paper_figures (paper_arxiv_id, figure_id) - VALUES (%s, %s) - ON CONFLICT (paper_arxiv_id, figure_id) DO NOTHING - """ - self.cur.execute(sql, (paper_arxiv_id, figure_id)) - return self.cur.rowcount == 1 - - def insert_paper_table(self, paper_arxiv_id, table_id): - """ - Link paper to table explicitly by arxiv_id. Returns True if inserted, False if exists. - """ - sql = """ - INSERT INTO paper_tables (paper_arxiv_id, table_id) - VALUES (%s, %s) - ON CONFLICT (paper_arxiv_id, table_id) DO NOTHING - """ - self.cur.execute(sql, (paper_arxiv_id, table_id)) - return self.cur.rowcount == 1 - - def insert_author_affiliation(self, author_id, institution_id): - """ - Link an author to an institution. Returns True if inserted, False if exists. - """ - sql = """ - INSERT INTO author_affiliation (author_id, institution_id) - VALUES (%s, %s) - ON CONFLICT (author_id, institution_id) DO NOTHING - """ - self.cur.execute(sql, (author_id, institution_id)) - return self.cur.rowcount == 1 - - def check_exist_figure(self, bib_key): - entry = f"\\label{{{bib_key}}}" - sql = """ - SELECT EXISTS( - SELECT 1 - FROM figures - WHERE label = %s - ); - """ - - self.cur.execute(sql, (entry,)) - exists, = self.cur.fetchone() - return exists - - def check_exist_table(self, bib_key): - entry = f"\\label{{{bib_key}}}" - print(entry) - sql = """ - SELECT EXISTS( - SELECT 1 - FROM tables - WHERE label = %s - ); - """ - - self.cur.execute(sql, (entry,)) - exists, = self.cur.fetchone() - return exists - - def check_exist(self, paper_arxiv_id): - """ - Check if the paper with given arxiv id exists in the database - Return True or False as boolean value - - paper_arxiv_id: str - """ - - # TODO: this should be removed later - - # return False - sql = """ - SELECT EXISTS( - SELECT 1 - FROM papers - WHERE arxiv_id = %s - ); - """ - # execute the query - self.cur.execute(sql, (paper_arxiv_id,)) - # fetchone returns a tuple like (True,) or (False,) - exists, = self.cur.fetchone() - return exists - - def paper_authors_exist(self, paper_arxiv_id): - """ - Check if the paper with given arxiv id exists in the author database. - If not, it means that the paper with arxiv id is not yet added into the semantic scholar or previous fetching failed. - - paper_arxiv_id: str - """ - - sql = """ - SELECT EXISTS( - SELECT 1 - FROM paper_authors - WHERE paper_arxiv_id = %s - ) - """ - - self.cur.execute(sql, (paper_arxiv_id,)) - - exists, = self.cur.fetchone() - - return exists - - # def _dict_from_cursor(self, cursor, parser= None): - # """Convert last SELECT into list of dicts, applying parser to string fields if given.""" - # cols = [col.name for col in cursor.description] - # result = [] - # for row in cursor.fetchall(): - # rd = {} - # for col, val in zip(cols, row): - # rd[col] = parser(val) if parser and isinstance(val, str) else val - # result.append(rd) - # return result - - # def serialize_table(self, table_name, schema, parser=None): - # """Fetch all rows from schema.table_name and return list of dicts.""" - # sql = f'SELECT * FROM "{schema}"."{table_name}"' - # with self.conn.cursor() as cur: - # cur.execute(sql) - # return self._dict_from_cursor(cur, parser) - - - # def serialize_all(self, tables, schema = 'public', parser = None): - # """Dump multiple tables into a dict, applying parser to string fields.""" - # if tables is None: - # tables = [ - # 'papers', 'sections', 'authors', 'categories', 'institutions', - # 'figures', 'tables', 'paper_authors', 'paper_category', - # 'citations', 'paper_figures', 'paper_tables', 'author_affiliation' - # ] - # out = {} - # for t in tables: - # out[t] = self.serialize_table(t, schema, parser) - # return out - - def export_to_json(self, path: str, tables = None, parser = None, **json_kwargs) -> None: - """Write out schema (or subset) to JSON file, parsing text if provided.""" - data = self.serialize_all(tables, parser=parser) - with open(path, 'w') as f: - json.dump(data, f, **json_kwargs) - - - def close(self): - self.cur.close() - self.conn.close() - - def drop_all(self): - """ - Drop all tables in reverse dependency order. - """ - # List tables in order so that dependent tables are dropped before the ones they reference - tables = [ - "paper_figures", - "paper_tables", - "paper_authors", - "paper_category", - "citations", - "author_affiliation", - "figures", - "tables", - "categories", - "authors", - "institutions", - "papers", - "sections", - "paragraphs", - "paragraph_citations", - "paragraph_references" - ] - for tbl in tables: - # Use IF EXISTS to avoid errors if a table is already gone - # Use CASCADE for safety in case there are lingering dependencies - self.cur.execute(f"DROP TABLE IF EXISTS {tbl} CASCADE") - \ No newline at end of file diff --git a/research_arcade/arxiv_utils/graph_constructor/database_loader.py b/research_arcade/arxiv_utils/graph_constructor/database_loader.py deleted file mode 100644 index 3c80893..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/database_loader.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -from typing import Callable, Optional - -class Database: - # … all your existing connection and table-creation methods … - - def __init__(self): - - - pass - - def serialize_table( - self, - table_name: str, - schema: str = 'public', - parser: Optional[Callable[[str], str]] = None - ) -> list[dict]: - """ - Fetch all rows from `schema.table_name` and return a list of dicts. - If `parser` is provided, apply it to each string field. - """ - sql = f'SELECT * FROM "{schema}"."{table_name}"' - with self.conn.cursor() as cur: - cur.execute(sql) - return self._dict_from_cursor(cur, parser) - - def serialize_all( - self, - tables: Optional[list[str]] = None, - schema: str = 'public', - parser: Optional[Callable[[str], str]] = None - ) -> dict: - """ - Dump multiple tables into a single dict, applying `parser` to string fields. - - tables: list of table names; if None, will use default list. - Returns { table_name: [ {col: val,…}, … ], … } - """ - if tables is None: - tables = [ - 'papers', 'sections', 'authors', 'categories', 'institutions', - 'figures', 'tables', 'paper_authors', 'paper_category', - 'citations', 'paper_figures', 'paper_tables', 'author_affiliation' - ] - export_data = {} - for t in tables: - export_data[t] = self.serialize_table(t, schema, parser) - return export_data - - def export_to_json( - self, - path: str, - tables: Optional[list[str]] = None, - parser: Optional[Callable[[str], str]] = None, - **json_kwargs - ) -> None: - """ - Write out your entire schema (or a subset) to JSON file. - Applies `parser` to string fields if provided. - """ - data = self.serialize_all(tables, parser=parser) - with open(path, 'w') as f: - json.dump(data, f, **json_kwargs) - -# Example usage: -if __name__ == '__main__': - def latex_parser(raw: str) -> str: - # e.g. integrate your LaTeX-to-text parser here - from pylatexenc.latex2text import LatexNodes2Text - return LatexNodes2Text().latex_to_text(raw) - - db = Database() - # Serialize 'sections' table, parsing LaTeX in content/title columns - sections = db.serialize_table('sections', parser=latex_parser) - print(f"Loaded {len(sections)} sections (LaTeX parsed).") - - # Full export with parsing - db.export_to_json('db_dump_parsed.json', parser=latex_parser, indent=2) - print("Exported database with parsed fields.") - db.close() diff --git a/research_arcade/arxiv_utils/graph_constructor/node_processor.py b/research_arcade/arxiv_utils/graph_constructor/node_processor.py deleted file mode 100644 index e492423..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/node_processor.py +++ /dev/null @@ -1,597 +0,0 @@ -from ..graph_constructor.database import Database -from semanticscholar import SemanticScholar -import arxiv -from ..paper_collector.latex_parser import clean_latex_format -import re -import json -import time -import os -from dotenv import load_dotenv -from typing import List, Tuple -import unicodedata -import difflib - -class NodeConstructor: - - """ - This class serves for converting entities such as authors, papers into nodes and insert them into the paper graph database - """ - - def __init__(self): - self.db = Database() - self.sch = None - load_dotenv() - api_key = os.getenv('SEMANTIC_SCHOLAR_API_KEY') - if not api_key: - # We may still proceed, but it takes longer - print("SEMANTIC_SCHOLAR_API_KEY not set in .env") - self.sch = SemanticScholar() - else: - # self.sch = SemanticScholar(api_key=api_key) - self.sch = SemanticScholar() - - # Construct the author node based on his or her semantic scholar id - def author_constructor(self, semantic_scholar_id): - author = self.sch.get_author(semantic_scholar_id) - name = author.name - url = author.url - self.db.insert_author(semantic_scholar_id=semantic_scholar_id, name=name, homepage=url) - - # Construct the paper node based on the full information provided - def paper_constructor(self, arxiv_id, title, abstract=None, submit_date=None, metadata=None): - base_arxiv_id, version = self.arxiv_id_processor(arxiv_id=arxiv_id) - self.db.insert_paper(arxiv_id=arxiv_id, base_arxiv_id=base_arxiv_id, version=version, title=title, abstract=abstract, submit_date=submit_date, metadata=metadata) - - # Construct the paper node based on the arxiv_id used for fetching the rest of information from SemanticScholar - def paper_constructor_arxiv_id(self, arxiv_id): - try: - search = arxiv.Search(id_list=[arxiv_id]) - paper = next(arxiv.Client().results(search)) - except Exception as e: - raise RuntimeError(f"Failed to fetch arXiv entry for {arxiv_id}: {e}") - metadata = { - 'id': arxiv_id, - 'title': paper.title, - 'summary': paper.summary, - 'authors': [a.name for a in paper.authors], - 'published': str(paper.published), - 'categories': paper.categories, - 'url': paper.entry_id, - } - - abstract = clean_latex_format(paper.summary) - title = paper.title - submit_date = str(paper.published) - - base_arxiv_id, version = self.arxiv_id_processor(arxiv_id=arxiv_id) - self.db.insert_paper(arxiv_id=arxiv_id, base_arxiv_id=base_arxiv_id, version=version, title=title, abstract=abstract, submit_date=submit_date, metadata=metadata) - - # Construct the paper node based on given json file with information provided - def paper_constructor_json(self, arxiv_id, json_file): - - # print(json_file) - title = json_file['title'] - abstract = clean_latex_format(json_file['abstract']) - submit_date = json_file['published'] - base_arxiv_id, version = self.arxiv_id_processor(arxiv_id=arxiv_id) - self.db.insert_paper(arxiv_id=arxiv_id, base_arxiv_id=base_arxiv_id, version=version, title=title, abstract=abstract, submit_date=submit_date, metadata=str(json_file)) - - - # Construct category node with given category name and category description - def category_constructor(self, name, description=None): - self.db.insert_category(name=name, description=description) - - # Construct institution node with institution name and location - def institution_constructor(self, name, location=None): - self.db.insert_institution(name=name, location=location) - - # Construct figure node given the paper id, index of figure (?), path to figure and caption/label - # def figure_constructor(self, paper_id, figure_index, path, caption=None): - # self.db.insert_figure(paper_arxiv_id=paper_id, figure_index=figure_index, path=path, caption=caption) - - # Construct tabel node given the paper id, index of figure (?), path to figure and caption/label - # def figure_constructor(self, paper_id, table_index, path, caption=None): - # self.db.insert_figure(paper_arxiv_id=paper_id, table_index=table_index, path=path, caption=caption) - - # Construct the paragraph given the paper id, index of paragraph, index of section, and the arxiv id that this paper belongs to - # def paragraph_constructor(self, paragraph, paragraph_index, section_index, arxiv_id): - - - - # pass - - ''' - Here, not that in the future when we process papers, we can authomatically add figures and tables into database and construct the papers into it - ''' - - - ''' - Also, we need to build methods that can automatically store figures into - ''' - - # Given the arxiv_id and the directory path that stores the extracted paper information, store the paper, author, figure and tables into the database - def process_paper(self, arxiv_id, dir_path): - """ - Given a paper: - 1. Store it as a node - 2. Build edge to paper authors. If the author does not exist, create one - - Assume that all the papers have been fully extracted. - """ - - # First thing first: check if the paper exists in the database - # If so, stop and return None - - paper_exists = self.db.check_exist(arxiv_id) - - if paper_exists: - print(f"The paper {arxiv_id} already exists in the database.") - print("Skipped") - return - - times = {} - - # Find the corresponding files - # json_path = f"{dir_path}/output/{arxiv_id}.json" - json_path = f"{dir_path}/output/{arxiv_id}.json" - print(f"File path: {json_path}") - # metadata_path = f"{dir_path}/{arxiv_id}/{arxiv_id}_metadata.json" - metadata_path = f"{dir_path}/{arxiv_id}/{arxiv_id}_metadata.json" - - if paper_exists: - print(f"The paper with arxiv_id {arxiv_id} already exists in the database") - return False - # Then check if we have the json file of paper and meta_data - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - except FileNotFoundError: - print(f"Error: The file '{file_json}' was not found.") - return False - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{file_json}'. Check if the file contains valid JSON.") - return False - except Exception as e: - print(f"An unexpected error occurred: {e}") - return False - - - metadata_json = None - - t0 = time.perf_counter() - - try: - with open(metadata_path, 'r') as file: - metadata_json = json.load(file) - except FileNotFoundError: - print(f"Error: The file '{metadata_path}' was not found.") - return False - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{metadata_path}'. Check if the file contains valid JSON.") - return False - except Exception as e: - print(f"An unexpected error occurred: {e}") - return False - - times['load_metadata'] = time.perf_counter() - t0 - - print(f"Time of loading metadata: {times['load_metadata']}") - - t0 = time.perf_counter() - self.paper_constructor_json(arxiv_id=arxiv_id, json_file=metadata_json) - times['paper_constructor'] = time.perf_counter() - t0 - print(f"Time of constructing paper json file: {times['paper_constructor']}") - - t0 = time.perf_counter() - authors = None - - # TODO: since a lot of latest papers are not yet on semantic scholar, we choose to move this process in later stages - # Add the author into paper directory if the paper is on semantic scholar - # base_arxiv_id, version = self.arxiv_id_processor(arxiv_id) - # try: - # paper_sch = self.sch.get_paper(f"ARXIV:{base_arxiv_id}") - # authors = paper_sch.authors - # except Exception as e: - # print(f"Paper with arxiv id {base_arxiv_id} not found on semantic scholar: {e}") - - - # Add authors into database if not exist - author_order = 0 - if authors: - for author in authors: - self.author_constructor(author.authorId) - author_order += 1 - # Add paper-author edge as follows - self.db.insert_paper_author(paper_arxiv_id=arxiv_id, author_id=author.authorId, author_sequence=author_order) - - times['author_adding'] = time.perf_counter() - t0 - print(f"Time of finding authors and adding authors to database: {times['author_adding']}") - - # Add figures to papers - # Here we store the path to figures/images instead of directly storing them inside of the database - # We don't really need the figure id LOL - # Here, we use the existing json file of paper to extract the figure information - - - t0 = time.perf_counter() - - - # Go through all the figure files following the extracted information in json - - section_jsons = file_json['sections'] - - for title, section_json in section_jsons.items(): - is_appendix = section_json['appendix'] == 'true' - content = section_json['content'] - self.db.insert_section(content=content, title=title, is_appendix=is_appendix, paper_arxiv_id=arxiv_id) - - figure_jsons = file_json['figure'] - - # TODO: recursively add figures - # Iterate through subfigures first - # Extremely important - # print(f"figure_jsons: {figure_jsons}") - for figure_json in figure_jsons: - # label = figure_json['label'] - # caption = figure_json['caption'] - # file_name = figure_json['figure_paths'][0] - # path = f"{dir_path}/output/figures/figures_{file_name}" - # path = file_name - # print(f"figure_json: {figure_json}") - - figures = self.figure_iteration_recursive(figure_json=figure_json) - - for figure in figures: - path, caption, label = figure - # print("Figure tuple:") - # print(path, caption, label) - figure_id = self.db.insert_figure(paper_arxiv_id=arxiv_id, path=path, caption=caption, label=label, name=None) - - self.db.insert_paper_figure(paper_arxiv_id=arxiv_id, figure_id=figure_id) - - - # We do the same thing to tables - # For tables, we currently do not have a good way to reconstruct the table. Instead, we store the raw textual data of tables in the database directly. - - table_jsons = file_json['table'] - for table_json in table_jsons: - - caption = table_json['caption'] - label = table_json['label'] - table = table_json['tabular'] - # We don't currently store the table anywhere as a file so the table path is empty - path = None - - table_id = self.db.insert_table(paper_arxiv_id=arxiv_id, path=path, caption=caption, label=label, table_text=table) - - self.db.insert_paper_table(paper_arxiv_id = arxiv_id, table_id=table_id) - - - # We now add the table to the category. - # We first insert the category into database. - categories = file_json['categories'] - - - - for category in categories: - category_id = self.db.insert_category(category) - self.db.insert_paper_category(category_id=category_id, paper_arxiv_id=arxiv_id) - - times['info_extraction'] = time.perf_counter() - t0 - print(f"Time of adding figures, tables, and sections to database: {times['info_extraction']}") - - # The last thing is to deal with citation maps. - # For now we simply loop throught the citations part of the paper and obtain the arxiv ids of cited papers. - # Comparing the semantic scholar, we choose to use extracted citations since they all provide arxiv ids. - # Recall that we have to make sure every paper can be traced to its arxiv id since only downloading from arxiv provides us full context. - - - t0 = time.perf_counter() - print("Now processing citations") - for citation in file_json['citations'].values(): - # print(f"Citation: {citation}") - cited_arxiv_id = citation.get('arxiv_id') - bib_key = citation.get('bib_key') - bib_title = citation.get('bib_title') - bib_author = citation.get('bib_author ') - contexts = citation.get('context') - citing_sections = set() - for context in contexts: - citing_section = context['section'] - citing_sections.add(citing_section) - # It seems that the cited paper sometimes does not provide arxiv id, or that column is null. How can I tackle this issue? - # if not cited_arxiv_id: - # # In that case, we need to use the tile for searching. - # # We first remove colon and plus sign in the title as they are prefix and relaitonal sign in the searching query - # title_cleaned = bib_title.replace('+', ' ').replace(':', '') - - # # and we only use the family name of author - # bib_author_surname = bib_author.split(',')[0].strip() - - # cited_arxiv_id = self.search_title_with_name(title=title_cleaned, name=bib_author_surname) - - self.db.insert_citation(citing_arxiv_id=arxiv_id, cited_arxiv_id=cited_arxiv_id, citing_sections=list(citing_sections),bib_title=bib_title, bib_key=bib_key, author_cited_paper=bib_author) - - times['citaion_extraction'] = time.perf_counter() - t0 - print(f"Time of searching arxiv id of cited paper (if not provided) and adding citation information to database: {times['citaion_extraction']}") - - return True - - def figure_iteration_recursive(self, figure_json): - - # Create a set of figures along with the - # list represents (path, caption, label) - path_to_info: List[Tuple[str, str, str]] = [] - - # First iterate through parent, then go into the children - - def figure_iteration(figure_json): - nonlocal path_to_info - - if not figure_json: - return - if figure_json['figure_paths']: - path = figure_json['figure_paths'][0] - caption = figure_json['caption'] - label = figure_json['label'] - path_to_info.append((path, caption, label)) - subfigures = figure_json['subfigures'] - - for subfigure in subfigures: - figure_iteration(subfigure) - - figure_iteration(figure_json=figure_json) - return path_to_info - - def process_paragraphs(self, dir_path): - """ - Process all the paragraphs after calling process_paper - """ - - paragraph_path = f"{dir_path}/output/paragraphs/text_nodes.jsonl" - with open(paragraph_path) as f: - data = [json.loads(line) for line in f] - - - # Use arxiv_id + section name as key - # Find the smallest paragraph_id generated by knowledge debugger - # Subtract all paragraph id of the same section (of the same paper) with the smallest one to ensure that order starts with zero - section_min_paragraph = {} - - for paragraph in data: - paragraph_id = paragraph.get('id') - # Extract paragraph_id - id_number = self.get_paragraph_num(paragraph_id) - paper_arxiv_id = paragraph.get('paper_id') - paper_section = paragraph.get('section') - if (paper_arxiv_id, paper_section) not in section_min_paragraph: - section_min_paragraph[(paper_arxiv_id, paper_section)] = int(id_number) - else: - section_min_paragraph[(paper_arxiv_id, paper_section)] = min(section_min_paragraph[(paper_arxiv_id, paper_section)], int(id_number)) - - - - for paragraph in data: - paragraph_id = paragraph.get('id') - content = paragraph.get('content') - paper_arxiv_id = paragraph.get('paper_id') - paper_section = paragraph.get('section') - id_number = self.get_paragraph_num(paragraph_id) - id_zero_based = id_number - section_min_paragraph[(paper_arxiv_id, paper_section)] - self.db.insert_paragraph(paragraph_id=id_zero_based, content=content, paper_arxiv_id=paper_arxiv_id, paper_section=paper_section) - - paragraph_cite_bib_keys = paragraph.get('cites') - for bib_key in paragraph_cite_bib_keys: - self.db.insert_paragraph_citations(paragraph_id=id_zero_based, paper_section=paper_section, citing_arxiv_id=paper_arxiv_id, bib_key=bib_key) - - - paragraph_ref_labels = paragraph.get('ref_labels') - - - # def insert_paragraph_reference(self, paragraph_id, paper_arxiv_id, reference_label, reference_type=None): - - for ref_label in paragraph_ref_labels: - - ref_type = None - # First search bib_key in databases. - # If presented in one of them, we can determine the type of reference - - is_figure = self.db.check_exist_figure(bib_key=ref_label) - is_table = self.db.check_exist_table(bib_key=ref_label) - if is_figure: - ref_type = 'figure' - elif is_table: - ref_type = 'table' - - - self.db.insert_paragraph_reference(paragraph_id=id_zero_based, paper_section=paper_section, paper_arxiv_id=paper_arxiv_id, reference_label=ref_label, reference_type=ref_type) - - - - # For here, we also need to call insert_paragraph_reference and insert paragraphs-refs(including tables, figures, and more in the future) into the database - - - - def create_tables(self): - self.db.create_all() - - def drop_tables(self): - self.db.drop_all() - - # def search_title_with_name(self, title, name, max_result=20): - # query = f"ti:{title} AND au:{name}" - # search = arxiv.Search( - # query=query, - # max_results=max_result, - # sort_by=arxiv.SortCriterion.Relevance, - # ) - - # print("Title of Cited Paper:") - # print(title) - # try: - # print("Result:") - # for result in search.results(): - # print(result.title) - # if (self.title_cleaner(result.title )== title): - # # and any(name.lower() in a.name.lower() for a in result.authors)): - # return result.entry_id - # except UnexpectedEmptyPageError: - # # no more pages—stop iterating - # pass - - def _norm(self, s): - s = unicodedata.normalize("NFKC", s).casefold() - s = re.sub(r"\s+", " ", s) - s = re.sub(r"[^\w\s]", "", s) - return s.strip() - - def search_title_with_name(self, title, name, max_result= 20, return_short_id= True): - def _q(s: str) -> str: - return '"' + s.replace('"', '\\"') + '"' - - query = f'ti:{_q(title)} AND au:{_q(name)}' - search = arxiv.Search( - query=query, - max_results=max_result, - sort_by=arxiv.SortCriterion.Relevance, - ) - - client = arxiv.Client(page_size=min(max_result, 100), delay_seconds=3.0, num_retries=3) - - want_title = self._norm(title) - want_author = self._norm(name) - want_author=want_author - - try: - for r in client.results(search): - # strict or near-exact title match - title_ok = (self._norm(r.title) == want_title) or ( - difflib.SequenceMatcher(a=self._norm(r.title), b=want_title).ratio() >= 0.97 - ) - if not title_ok: - continue - - # authors = [self._norm(getattr(a, "name", str(a))) for a in r.authors] - # if not any(want_author in a or a in want_author for a in authors): - # continue - - # prefer short arXiv ID; otherwise fall back to /abs/ tail - if return_short_id and hasattr(r, "get_short_id"): - return r.get_short_id() - return r.entry_id.rsplit("/", 1)[-1] - except (arxiv.UnexpectedEmptyPageError, arxiv.HTTPError, arxiv.ArxivError): - # nothing found or transient API issue - pass - - return None - - def arxiv_id_processor(self, arxiv_id): - """ - If the arxiv id is in the format a.bvc, we extract the a.b and c - If it is in the format a.b, we only extract the a.b - """ - ARXIV_RE = re.compile(r'^(?P(?:[a-z\-]+\/\d{7}|\d{4}\.\d{4,5}))(?:v(?P\d+))?$', re.IGNORECASE) - - m = ARXIV_RE.match(arxiv_id.strip()) - if not m: - raise ValueError(f"Not a valid arXiv id: {arxiv_id!r}") - base = m.group('id') - v = m.group('v') - return base, (int(v) if v is not None else None) - - - def author_processor(self, arxiv_id): - - """ - Motivation: sometimes the paper is not yet added into the semantic scholar. We can check if certain paper has author info in the db. If not, we search it on semantic scholar again. - """ - - # exists = self.db.paper_authors_exist(paper_arxiv_id=arxiv_id) - - base_arxiv_id, version = self.arxiv_id_processor(arxiv_id) - print(f"base_arxiv_id: {base_arxiv_id}") - try: - paper_sch = self.sch.get_paper(f"ARXIV:{base_arxiv_id}") - authors = paper_sch.authors - except Exception as e: - print(f"Paper with arxiv id {base_arxiv_id} not found on semantic scholar: {e}") - return False - - - # Add authors into database if not exist - author_order = 0 - if authors: - for author in authors: - self.author_constructor(author.authorId) - author_order += 1 - # Add paper-author edge as follows - self.db.insert_paper_author(paper_arxiv_id=arxiv_id, author_id=author.authorId, author_sequence=author_order) - - return True - - - def citation_processor(self, arxiv_id): - """ - For all citations where we know the citing paper but the cited_arxiv_id is NULL or empty, - try to look it up via title + author surname, and update the record in the DB. - """ - # 1. Pull the relevant citation rows - # TODO - select_sql = """ - SELECT id, bib_title, author_cited_paper - FROM citations - WHERE citing_arxiv_id = %s - AND (cited_arxiv_id IS NULL OR TRIM(cited_arxiv_id) = ''); - """ - self.db.cur.execute(select_sql, (arxiv_id,)) - rows = self.db.cur.fetchall() - print(arxiv_id) - print(rows) - # 2. For each, try to look up the arXiv ID and write it back - for row in rows: - citation_id, bib_title, bib_author_full = row - - # clean up the title to remove problematic characters - title_cleaned = self.title_cleaner(bib_title) - # assume bib_author_full is "Last, First" or similar - author_surname = bib_author_full.split(',')[0].strip() - - found_id = self.search_title_with_name(title=title_cleaned, name=author_surname) - if found_id: - update_sql = """ - UPDATE citations - SET cited_arxiv_id = %s - WHERE id = %s - """ - self.db.cur.execute(update_sql, (found_id, citation_id)) - print(f"Updated citation id {citation_id} → {found_id}") - else: - print(f"Could not find arXiv ID for citation {citation_id}: '{bib_title}' by {author_surname}") - - # 3. Commit once at the end (if your Database requires it) - # If you're using autocommit=True, this is optional. - try: - self.db.conn.commit() - except Exception: - pass - - return True - - def title_cleaner(self, title: str) -> str: - """ - Remove all symbols (non-alphanumeric, non-space characters) from the title. - Collapses multiple spaces down to one and trims ends. - """ - # Remove anything that isn't a letter, number, or whitespace - cleaned = re.sub(r'[^A-Za-z0-9\s]', '', title) - # Collapse multiple spaces and strip leading/trailing spaces - cleaned = re.sub(r'\s+', ' ', cleaned).strip() - return cleaned.strip().lower() - - def get_paragraph_num(self, pid): - pattern = re.compile(r'^text_(\d+)$') - m = pattern.match(pid) - if not m: - raise ValueError(f"Bad paragraph id format: {pid!r}") - return int(m.group(1)) - diff --git a/research_arcade/arxiv_utils/graph_constructor/paragraph_processor.py b/research_arcade/arxiv_utils/graph_constructor/paragraph_processor.py deleted file mode 100644 index 1f5dcb4..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/paragraph_processor.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -This program extracts the paragraphs from a paper and store them in the database -""" - - -from paper_collector.paper_graph_processor import PaperGraphProcessor - -class ParagraphProcessor: - - def __init__(self, data_dir, figures_dir, output_dir): - - self.paper_graph_processor = PaperGraphProcessor(data_dir=data_dir, figures_dir=figures_dir, output_dir=output_dir) - - - def extract_paragraphs(self): - - # First process all the json files - self.paper_graph_processor.process_all_papers() - # After that, we extract the corresponding paragraphs and store the paragraphs into database - - # Then we go through all the extracted information - - - - pass - diff --git a/research_arcade/arxiv_utils/graph_constructor/utils.py b/research_arcade/arxiv_utils/graph_constructor/utils.py deleted file mode 100644 index d7cc072..0000000 --- a/research_arcade/arxiv_utils/graph_constructor/utils.py +++ /dev/null @@ -1,8 +0,0 @@ -import re - -def figure_latex_path_to_path(path, arxiv_id, latex_path): - # 2. Replace any forward or backward slash with underscore - latex_path = re.sub(r'[/]', '_', latex_path) - # 3. Build the filename with the arXiv ID prefix - return f"{path}/output/figures/{arxiv_id}/{latex_path}" - diff --git a/research_arcade/arxiv_utils/multi_input/arxiv_crawler_new.py b/research_arcade/arxiv_utils/multi_input/arxiv_crawler_new.py deleted file mode 100644 index 7e1d89c..0000000 --- a/research_arcade/arxiv_utils/multi_input/arxiv_crawler_new.py +++ /dev/null @@ -1,69 +0,0 @@ -from paperscraper.get_dumps import arxiv -from .multi_input import MultiInput -import json -from typing import List - - -def download_with_time(start_date, end_date=None, save_path="./download"): - - save_path = f"{save_path}/arxiv_metadata_{start_date}_{end_date}.jsonl" - - # Create an empty file of the same path - - - - arxiv(start_date=start_date, end_date=end_date, save_path=save_path) # scrapes all metadata from 2024 until today. - return save_path - - - -def extract_arxiv_ids(file_path: str) -> List[str]: - """ - Reads either NDJSON (one JSON object per line) or a JSON array file, - pulls 'doi' (or common variants), converts to arXiv IDs via MultiInput.doi_to_id, - and returns a de-duplicated list preserving order. - """ - mi = MultiInput() - arxiv_ids: List[str] = [] - seen = set() - - def try_add(doi): - if not doi: - return - try: - arx_id = mi.doi_to_id(doi) - except Exception: - return - if arx_id not in seen: - seen.add(arx_id) - arxiv_ids.append(arx_id) - - with open(file_path, "r", encoding="utf-8") as f: - # Peek first non-whitespace char to detect JSON array vs NDJSON - start = f.read(1) - while start and start.isspace(): - start = f.read(1) - f.seek(0) - - if start == "[": # JSON array - data = json.load(f) - for obj in data: - if isinstance(obj, dict): - doi = obj.get("doi") or obj.get("DOI") or obj.get("paper_doi") - try_add(doi) - else: # NDJSON - for lineno, line in enumerate(f, 1): - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - # skip malformed line - continue - if isinstance(obj, dict): - doi = obj.get("doi") or obj.get("DOI") or obj.get("paper_doi") - try_add(doi) - - return arxiv_ids - diff --git a/research_arcade/arxiv_utils/multi_input/multi_download.py b/research_arcade/arxiv_utils/multi_input/multi_download.py deleted file mode 100644 index ca32948..0000000 --- a/research_arcade/arxiv_utils/multi_input/multi_download.py +++ /dev/null @@ -1,508 +0,0 @@ -import requests -import datetime -import json -import os -import arxiv -from filelock import FileLock - -from ..paper_collector.graph_construction import build_citation_graph_thread -from ..paper_collector.utils import None_constraint -from utils.error_handler import api_calling_error_exponential_backoff -from ..paper_collector.latex_parser import clean_latex_code -from semanticscholar import SemanticScholar - -from ..multi_input.multi_input import MultiInput -import time -from typing import Optional, List, Dict, Any -import sys - - -TIMEOUT = 10 -class MultiDownload: - """ - This class supports downloading arxiv latex code, pdf and html webpage using arxiv id, link or bib - """ - - @api_calling_error_exponential_backoff(retries=5, base_wait_time=1) - def download_arxiv(self, input: str, input_type: str, output_type: str, dest_dir: str = None): - - """ - Input type: - - id/arxiv_id -> arxiv id of the paper - - bib/arxiv_bib -> bib of the paper which has the arxiv id field - - url/link -> url of the link of paper on arxiv - """ - - mi = MultiInput() - arxiv_id = mi.extract_arxiv_id(input, input_type) - - dest_dir = f"{dest_dir}/{arxiv_id}" - try: - search = arxiv.Search(id_list=[arxiv_id]) - paper = next(arxiv.Client().results(search)) - except Exception as e: - raise RuntimeError(f"Failed to fetch arXiv entry for {arxiv_id}: {e}") - - filename_pdf = arxiv_id + ".pdf" - filename_latex = arxiv_id + ".tar.gz" - - # Save metadata - if dest_dir: - metadata = { - 'id': arxiv_id, - 'title': paper.title, - 'abstract': paper.summary, - 'authors': [a.name for a in paper.authors], - 'published': str(paper.published), - 'categories': paper.categories, - 'url': paper.entry_id, - } - os.makedirs(dest_dir, exist_ok=True) - with open(f"{dest_dir}/{arxiv_id}_metadata.json", "w", encoding="utf-8") as f: - json.dump(metadata, f, ensure_ascii=False, indent=2) - - if output_type == "pdf": - pdf_path = paper.download_pdf(filename = filename_pdf, dirpath = dest_dir) - - - if output_type == "latex": - latex_path = paper.download_source(filename = filename_latex, dirpath = dest_dir) - # Extract the real (with or without the version number) arxiv id - print(latex_path) - - if output_type == "both": - latex_path = paper.download_source(filename = filename_latex, dirpath = dest_dir) - pdf_path = paper.download_pdf(filename = filename_pdf, dirpath = dest_dir) - - - - def download_papers_by_field_and_date( - self, - field: str, - start_date: str, - output_type: str = "both", - max_results: Optional[int] = None, - dest_dir: str = "./download_by_field", - sort_order: str = "descending", - page_size: int = 100, - delay_seconds: float = 15.0, - ): - """ - Search arXiv for papers in a given subject category (e.g., "cs.AI") submitted from the start_date to today, - then download PDFs and/or LaTeX source along with metadata. - Note that due to the detection of arxiv API, the download might be incomplete, that the pdf or latex files might be missing. - - Parameters: - - field: str, arXiv subject category, e.g., "cs.AI", "stat.ML", "math.AG", etc. - - start_date: str, in "YYYY-MM-DD" format (inclusive). - - output_type: str, one of "pdf", "latex", or "both". - - max_results: Optional[int], maximum total number of papers to retrieve/download. If None, retrieves as many as possible up to API limits. - - dest_dir: str, directory to save downloads and metadata. - - sort_order: str, "ascending" or "descending" by submission date. - - page_size: int, number of items per page/request to arXiv API (controls `page_size` in Client). - - delay_seconds: float, seconds to wait between API calls (to be polite to arXiv servers). - """ - # Parse dates - try: - dt_start = datetime.datetime.strptime(start_date, "%Y-%m-%d").date() - # dt_end = datetime.datetime.strptime(end_date, "%Y-%m-%d") - except ValueError as e: - raise ValueError(f"start_date and end_date must be in YYYY-MM-DD format: {e}") - - # if dt_end < dt_start: - # raise ValueError("end_date must be the same or after start_date") - - # start_str = dt_start.strftime("%Y%m%d") + "0000" - # end_str = dt_end.strftime("%Y%m%d") + "2359" - - query = f"cat:{field}" - - sort_by = arxiv.SortCriterion.SubmittedDate - sort_order_enum = ( - arxiv.SortOrder.Ascending - if sort_order.lower() == "ascending" - else arxiv.SortOrder.Descending - ) - print("ArXiv query:", query) - - search = arxiv.Search( - query=query, - max_results=max_results, - sort_by=sort_by, - sort_order=sort_order_enum, - ) - - client = arxiv.Client(page_size=page_size, delay_seconds=delay_seconds) - - os.makedirs(dest_dir, exist_ok=True) - - downloaded_ids: List[str] = [] - count = 0 - for result in client.results(search): - # Stop if we reached max_results - if max_results is not None and count >= max_results: - break - - # The time when the paper was updated - paper_date = result.updated - - # Ensure that we are not downloading paper prior to the given date - if paper_date.date() < dt_start: - continue - - # Each result has attributes: entry_id (URL), pdf_url, title, summary, authors, published, updated, primary_category, categories, comment, journal_ref, doi, etc. - arxiv_id = result.entry_id.split('/')[-1] - paper_dir = os.path.join(dest_dir, arxiv_id) - os.makedirs(paper_dir, exist_ok=True) - # paper_date = result. - - # Save metadata to JSON - metadata = { - 'id': arxiv_id, - 'title': result.title, - 'abstract': result.summary, - 'authors': [author.name for author in result.authors], - 'published': result.published.isoformat() if hasattr(result.published, "isoformat") else str(result.published), - 'categories': result.categories, - 'url': result.entry_id, - } - metadata_path = os.path.join(paper_dir, f"{arxiv_id}_metadata.json") - - lock_path = metadata_path + ".lock" - lock = FileLock(lock_path) - with lock: - with open(metadata_path, "w", encoding="utf-8") as f: - json.dump(metadata, f, ensure_ascii=False, indent=2) - f.flush() - os.fsync(f.fileno()) # ensure it's flushed to disk - - # Download PDF and/or LaTeX source - if output_type in ("pdf", "both"): - try: - # filename defaults to {id}.pdf - result.download_pdf(filename=f"{arxiv_id}.pdf", dirpath=paper_dir) - except Exception as e: - print(f"[Warning] Failed to download PDF for {arxiv_id}: {e}") - - if output_type in ("latex", "both"): - try: - # The arxiv library may download a tar.gz of source; filename {id}.tar.gz - result.download_source(filename=f"{arxiv_id}.tar.gz", dirpath=paper_dir) - except Exception as e: - print(f"[Warning] Failed to download source for {arxiv_id}: {e}") - downloaded_ids.append(arxiv_id) - count += 1 - print(f"Downloaded {count}: {arxiv_id}") - - print(f"Finished: downloaded metadata for {count} papers in field '{field}' from {start_date}'") - return downloaded_ids - - @api_calling_error_exponential_backoff(retries=5, base_wait_time=1) - def download_semantic_scholar(self, input: str, input_type: str, dest_dir: str = None) -> None: - - mi = MultiInput() - - semantic_id = mi.extract_arxiv_id(input, input_type) - - sc = SemanticScholar(timeout=TIMEOUT) - try: - paper = sc.get_paper(paper_id=semantic_id) - except Exception as e: - raise RuntimeError(f"Failed to fetch arXiv entry for {semantic_id}: {e}") - - # print(paper) - - print(type(paper)) - # paper. - - pass - - @api_calling_error_exponential_backoff(retries=5, base_wait_time=1) - def build_paper_graph(self, input: str, input_type: str, dest_dir: str = None) -> None: - """ - Extract the paper graph of the paper using knowledge_debugger, provided the paper id/url/bib, the type of input and the directory the output is stored - - input: str - - input_type: str - - dest_dir: str - """ - mi = MultiInput() - - arxiv_id = mi.extract_arxiv_id(input, input_type) - - arxiv_list = [arxiv_id] - build_citation_graph_thread( - seed=arxiv_list, - source_path=f"{dest_dir}/{arxiv_id}", - working_path=f"{dest_dir}/working_folder", - output_path=f"{dest_dir}/output", - debug_path=None, - constraint=None_constraint, - num_threads=len(arxiv_list), - clear_source=False - ) - - @api_calling_error_exponential_backoff(retries=5, base_wait_time=1) - def build_paper_graphs(self, input: List[str], input_type: str, dest_dir: str = None) -> None: - """ - Extract the paper graph of the paper using knowledge_debugger, provided the paper id/url/bib, the type of input and the directory the output is stored - - input: str - - input_type: str - - dest_dir: str - """ - mi = MultiInput() - arxiv_list = [] - for id in input: - arxiv_id = mi.extract_arxiv_id(id, input_type) - arxiv_list.append(arxiv_id) - source_path = f"{dest_dir}/{arxiv_id}" - print(f"Source path: {source_path}") - build_citation_graph_thread( - seed=arxiv_list, - source_path=source_path, - working_path=f"{dest_dir}/working_folder", - output_path=f"{dest_dir}/output", - debug_path=None, - constraint=None_constraint, - num_threads=len(arxiv_list), - clear_source=True, - max_figure = sys.maxsize - ) - - def build_paragraphs(self, dest_dir: str): - """ - - dest_dir: str - """ - - - def get_abstract(self, input: str, input_type: str, dest_dir: str = None) -> str: - - mi = MultiInput() - - arxiv_id = mi.extract_arxiv_id(input, input_type) - - self.build_paper_graph(input, input_type, dest_dir) - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - - with open(json_path, 'r', encoding='utf-8') as f: - data = json.load(f) - - if 'abstract' not in data: - raise KeyError(f"'abstract' field not found in JSON at {json_path}") - - return clean_latex_code(data['abstract']) - - - def get_title(self, input: str, input_type: str, dest_dir: str = None) -> str: - - mi = MultiInput() - - arxiv_id = mi.extract_arxiv_id(input, input_type) - - - json_path = f"{dest_dir}/{arxiv_id}_metadata.json" - # json_path = f"{dest_dir}/{arxiv_id}_metadeta.json" - # print(json_path) - try: - with open(json_path, 'r', encoding='utf-8') as f: - data = json.load(f) - except Exception as e: - raise RuntimeError(f"The paper with arxiv id {arxiv_id} is not fould in the directory {dest_dir}: {e}") - - if 'title' not in data: - raise KeyError(f"'abstract' field not found in JSON at {json_path}") - - return data['title'] - - def get_references(self, input: str, input_type: str, max_retries: int = 8) -> List[Dict[str, Any]]: - - mi = MultiInput() - - arxiv_id = mi.extract_arxiv_id(input, input_type) - - SEMANTIC_SCHOLAR_API_URL = 'https://api.semanticscholar.org/graph/v1/paper/' - url = f'{SEMANTIC_SCHOLAR_API_URL}ARXIV:{arxiv_id}/references' - params = {'limit': 100, 'offset': 0, 'fields': 'title,abstract'} - headers = {'User-Agent': 'PaperProcessor/1.0'} - # print(f"max_retries: {max_retries}") - for attempt in range(max_retries): - # print(f"Attempt {attempt}") - response = requests.get(url, params=params, headers=headers) # type: ignore - if response.status_code == 200: - data = response.json() - print(f"Retrieved data: {data}") - references = [] - for ref in data.get('data', []): - cited_paper = ref.get('citedPaper', {}) - if cited_paper: - ref_info = { - 'title': cited_paper.get('title'), - 'abstract': cited_paper.get('abstract'), - 'paper_id': cited_paper.get('paperId') - } - references.append(ref_info) - return references - else: - wait_time = 2**attempt - print( - f'Error {response.status_code} fetching references for {arxiv_id}. Retrying in {wait_time}s...' - ) - time.sleep(wait_time) # Exponential backoff - print(f'Failed to fetch references for {arxiv_id} after {max_retries} attempts.') - return [] - - def get_reference_arxiv(self, input: str, input_type: str) -> str: - - - mi = MultiInput() - arxiv_id = mi.extract_arxiv_id(input, input_type) - - try: - search = arxiv.Search(id_list=[arxiv_id]) - paper = next(arxiv.Client().results(search)) - except Exception as e: - raise RuntimeError(f"Failed to fetch arXiv entry for {arxiv_id}: {e}") - - return paper.journal_ref - - - # @api_calling_error_exponential_backoff(retries=5, base_wait_time=1) - # def download_metadata_arxiv(self, input: str, input_type: str, dest_dir: str = None) -> dict: - # """ - # Retrieve metadata for an arXiv paper (given by ID, bib entry string, or URL/link), returning a dict - # and optionally saving it to dest_dir as JSON (and also a .bib file with a simple BibTeX entry). - - # Returns: - # metadata (dict): keys include: - # - 'arxiv_id' - # - 'title' - # - 'authors' (list of names) - # - 'summary' - # - 'published' (ISO 8601 string) - # - 'updated' (ISO 8601 string) if available - # - 'comment' - # - 'journal_ref' - # - 'doi' - # - 'primary_category' - # - 'categories' (list) - # - 'pdf_url' - # - 'arxiv_url' - # - 'bibtex' (a simple generated BibTeX entry) - # """ - # mi = MultiInput() - # arxiv_id = mi.extract_arxiv_id(input, input_type) - - - # try: - # search = arxiv.Search(id_list=[arxiv_id]) - # paper = next(arxiv.Client().results(search)) - # except Exception as e: - # raise RuntimeError(f"Failed to fetch arXiv entry for {arxiv_id}: {e}") - - - # metadata = {} - # metadata['arxiv_id'] = arxiv_id - # metadata['title'] = getattr(paper, "title", None) - - # authors = [] - # try: - # for a in paper.authors: - # name = getattr(a, "name", a) if a is not None else None - # if name: - # authors.append(name) - # except Exception: - # if isinstance(paper.authors, list): - # authors = paper.authors.copy() - # metadata['authors'] = authors - - # metadata['summary'] = getattr(paper, "summary", None) - # pub = getattr(paper, "published", None) - # if isinstance(pub, (datetime.datetime,)): - # metadata['published'] = pub.isoformat() - # else: - # metadata['published'] = str(pub) if pub is not None else None - # upd = getattr(paper, "updated", None) - # if isinstance(upd, (datetime.datetime,)): - # metadata['updated'] = upd.isoformat() - # else: - # metadata['updated'] = str(upd) if upd is not None else None - - # metadata['comment'] = getattr(paper, "comment", None) - # metadata['journal_ref'] = getattr(paper, "journal_ref", None) - # metadata['doi'] = getattr(paper, "doi", None) - - - # metadata['primary_category'] = getattr(paper, "primary_category", None) - - # cats = getattr(paper, "categories", None) - # if isinstance(cats, (list, tuple)): - # metadata['categories'] = list(cats) - # else: - # if isinstance(cats, str): - # metadata['categories'] = cats.split() - # else: - # metadata['categories'] = None - - # metadata['pdf_url'] = getattr(paper, "pdf_url", None) - # metadata['arxiv_url'] = getattr(paper, "entry_id", None) - - - # bib_id = arxiv_id.replace('/', '_') - - # title_bib = metadata['title'] or "" - - # title_bib = title_bib.replace("{", "\\{").replace("}", "\\}") - # authors_bib = " and ".join(authors) if authors else "" - # year = None - # try: - # if isinstance(pub, datetime.datetime): - # year = pub.year - # else: - # year = int(str(pub)[:4]) - # except Exception: - # year = None - # bibtex_lines = [f"@article{{{bib_id},"] - # if title_bib: - # bibtex_lines.append(f" title = {{{title_bib}}},") - # if authors_bib: - # bibtex_lines.append(f" author = {{{authors_bib}}},") - - # bibtex_lines.append(f" journal = {{arXiv preprint arXiv:{arxiv_id}}},") - # if year: - # bibtex_lines.append(f" year = {{{year}}},") - - # if metadata.get('doi'): - # bibtex_lines.append(f" doi = {{{metadata['doi']}}},") - # if metadata.get('journal_ref'): - - # jr = metadata['journal_ref'].replace("{", "\\{").replace("}", "\\}") - # bibtex_lines.append(f" note = {{{jr}}},") - - # if len(bibtex_lines) > 1: - # last = bibtex_lines[-1] - # if last.endswith(','): - # bibtex_lines[-1] = last[:-1] - # bibtex_lines.append("}") - # metadata['bibtex'] = "\n".join(bibtex_lines) - - # if dest_dir: - # os.makedirs(dest_dir, exist_ok=True) - # # JSON - # json_path = os.path.join(dest_dir, f"{arxiv_id.replace('/', '_')}_metadata.json") - # try: - # with open(json_path, "w", encoding="utf-8") as f: - # json.dump(metadata, f, indent=2, ensure_ascii=False) - # except Exception as e: - # raise IOError(f"Could not write metadata JSON to {json_path}: {e}") - - # return metadata - - -# id_string = "1806.08804" -# # dest_path = "./download" -# start_time = "2024-11-21" -# end_time = "2024-12-22" -# area = "cs.AI" -# mo = MultiDownload() - - diff --git a/research_arcade/arxiv_utils/multi_input/multi_input.py b/research_arcade/arxiv_utils/multi_input/multi_input.py deleted file mode 100644 index d155dd6..0000000 --- a/research_arcade/arxiv_utils/multi_input/multi_input.py +++ /dev/null @@ -1,271 +0,0 @@ -import re - -class MultiInput: - - """ - Download the compresses latex source code given the bib, arxiv id or arxiv link - """ - - - - def extract_bib_from_string(self, bib_str: str) -> dict: - """ - Parse a single-entry BibTeX string into a Python dict. - - Example input (bib_str): - @misc{1802.08773, - Author = {Jiaxuan You and Rex Ying and Xiang Ren and William L. Hamilton and Jure Leskovec}, - Title = {GraphRNN: Generating Realistic Graphs with Deep Auto-regressive Models}, - Year = {2018}, - Eprint = {arXiv:1802.08773}, - } - - Returns a dict like: - { - "ENTRYTYPE": "misc", - "ID": "1802.08773", - "author": "Jiaxuan You and Rex Ying and ...", - "title": "GraphRNN: Generating Realistic ...", - "year": "2018", - "eprint": "arXiv:1802.08773" - } - - Raises: - ValueError if the string does not look like a well‐formed single BibTeX entry. - """ - s = bib_str.strip() - - # 1) Match the “@type{key,” header: - m_header = re.match(r'@\s*(?P\w+)\s*{\s*(?P[^,]+)\s*,', s) - if not m_header: - raise ValueError( - "Invalid BibTeX entry header. " - "Expected something like: @misc{, … }" - ) - - entry_type = m_header.group("type").lower() - entry_key = m_header.group("key").strip() - - # 2) Find the very first “{” in the entire string (that opens the entry), - # then scan forward to locate its matching “}” by counting nested braces. - first_brace = s.find('{') - if first_brace == -1: - raise ValueError("Could not find the opening brace '{' for the BibTeX entry.") - - brace_level = 0 - end_brace = -1 - for i, ch in enumerate(s[first_brace:], start=first_brace): - if ch == '{': - brace_level += 1 - elif ch == '}': - brace_level -= 1 - if brace_level == 0: - end_brace = i - break - - if end_brace == -1: - raise ValueError("Could not find matching closing '}' for the BibTeX entry.") - - # Everything between the first “{” and its matching “}” is the block: - block = s[first_brace+1 : end_brace].strip() - - # 3) That block begins with “,” (we already extracted the ID). Remove up to the first comma: - comma_pos = block.find(',') - if comma_pos == -1: - raise ValueError("Expected a comma after the entry key inside the braces.") - fields_str = block[comma_pos+1 :].strip() - - # 4) Split fields_str on commas that occur at brace‐level zero. - fields = {} - current = "" - level = 0 - - for ch in fields_str: - if ch == '{': - level += 1 - current += ch - elif ch == '}': - level -= 1 - current += ch - elif ch == ',' and level == 0: - # We have a complete “key = value” chunk - chunk = current.strip() - current = "" - if chunk: - m_field = re.match(r'(?P\w+)\s*=\s*(?P.+)$', chunk) - if m_field: - key = m_field.group("k").strip().lower() - val = m_field.group("v").strip() - # Strip outer braces or quotes if present: - if val.startswith("{") and val.endswith("}"): - val = val[1:-1].strip() - elif val.startswith('"') and val.endswith('"'): - val = val[1:-1].strip() - fields[key] = val - # else: skip empty chunk - else: - current += ch - - # If anything remains in current after the loop, that's the last field: - last_chunk = current.strip() - if last_chunk: - m_field = re.match(r'(?P\w+)\s*=\s*(?P.+)$', last_chunk) - if m_field: - key = m_field.group("k").strip().lower() - val = m_field.group("v").strip() - if val.startswith("{") and val.endswith("}"): - val = val[1:-1].strip() - elif val.startswith('"') and val.endswith('"'): - val = val[1:-1].strip() - fields[key] = val - - # 5) Build the final dict: - bib_dict = { - "ENTRYTYPE": entry_type, - "ID": entry_key - } - bib_dict.update(fields) - return bib_dict - - - def arxiv_url_to_id(self, arxiv_url: str) -> str: - """ - Extract the arXiv ID (with optional version) from a full arXiv URL. - - Supported URL patterns include (but aren’t limited to): - - https://arxiv.org/abs/1802.08773 - - http://arxiv.org/abs/1802.08773v2 - - https://arxiv.org/pdf/1802.08773 - - https://arxiv.org/pdf/1802.08773.pdf - - http://arxiv.org/pdf/1802.08773v3.pdf - - https://arxiv.org/abs/math/0301234 - - https://arxiv.org/pdf/math/0301234 - - https://arxiv.org/pdf/math.GT/0301234v2.pdf - - Returns: - The “bare” arXiv identifier (e.g. "1802.08773", "1802.08773v2", - "math/0301234" or "math.GT/0301234v1"). - - Raises: - ValueError if the URL doesn’t match any known arXiv pattern. - """ - # 1) Strip off query strings or fragments: - url = arxiv_url.split('?')[0].split('#')[0] - - # 2) Regex patterns for “new” vs. “old” ID formats, - # allowing /pdf/ (with or without “.pdf”) and /abs/. - - patterns = [ - # — New‐style “abs” URL (e.g. /abs/1802.08773 or /abs/1802.08773v2) - r"""arxiv\.org/ - abs/ - (?P\d{4}\.\d{4,5}(?:v\d+)?)$ - """, - - # — New‐style “pdf” URL (e.g. /pdf/1802.08773 or /pdf/1802.08773.pdf or /pdf/1802.08773v2.pdf) - r"""arxiv\.org/ - pdf/ - (?P\d{4}\.\d{4,5}(?:v\d+)?) - (?:\.pdf)?$ - """, - - # — Old‐style “abs” URL (e.g. /abs/math/0301234 or /abs/math.GT/0301234v2) - r"""arxiv\.org/ - abs/ - (?P[A-Za-z\-]+(?:\.[A-Za-z\-]+)?/\d{7}(?:v\d+)?)$ - """, - - # — Old‐style “pdf” URL (e.g. /pdf/math/0301234 or /pdf/math/0301234.pdf or /pdf/math.GT/0301234v2.pdf) - r"""arxiv\.org/ - pdf/ - (?P[A-Za-z\-]+(?:\.[A-Za-z\-]+)?/\d{7}(?:v\d+)?) - (?:\.pdf)?$ - """ - ] - - for pat in patterns: - m = re.search(pat, url, re.VERBOSE) - if m: - return m.group('id') - - - raise ValueError(f"Could not extract arXiv ID from URL: {arxiv_url}") - - def doi_to_id(self, arxiv_doi: str) -> str: - - if not isinstance(arxiv_doi, str): - raise TypeError("arxiv_doi must be a string") - - s = arxiv_doi.strip() - - s = re.sub(r'^(https?://(?:dx\.)?doi\.org/)', '', s, flags=re.I) - - m = re.match(r'^10\.48550/ARXIV\.(?P[^?#]+)$', s, flags=re.I) - if not m: - raise ValueError(f"Not an arXiv DOI: {arxiv_doi!r}") - - arxiv_id = m.group('id') - - # remove optional version suffix, e.g., 'v3' - arxiv_id = re.sub(r'v\d+$', '', arxiv_id, flags=re.I) - - return arxiv_id - - - - def extract_arxiv_id_from_bib(self, bib_entry: dict) -> str: - """ - Given a single Bib entry (as a JSON‐style dict), return its arXiv ID (with version, if any). - If no Eprint/eprint field is found, raises a KeyError. - If the Eprint value isn’t in the form "arXiv:", returns the raw field as a fallback. - - Example input dict: - { - "ENTRYTYPE": "misc", - "ID": "1802.08773", - "author": "...", - "title": "...", - "year": "2018", - "Eprint": "arXiv:1802.08773" - } - - Returns: - The bare arXiv identifier, e.g. "1802.08773" or "1802.08773v2". - - Raises: - KeyError if neither "Eprint" nor "eprint" is present. - """ - # Try to find any key named "Eprint" or "eprint" (case‐insensitive would be more robust): - for key in ("Eprint", "eprint"): - if key in bib_entry: - raw = bib_entry[key].strip() - # If it starts with "arXiv:" (case‐insensitive), strip that off: - m = re.match(r'(?i)arxiv\s*:\s*(?P.+)', raw) - if m: - return m.group("id").strip() - else: - # Fallback: return whatever is in Eprint if it doesn’t match "arXiv:" - return raw - - # If we get here, neither "Eprint" nor "eprint" was in the dict: - raise KeyError("No Eprint/eprint field found in this Bib entry") - - - def extract_arxiv_id(self, input: str, input_type: str) -> str: - input_type = input_type.lower() - arxiv_id = "" - if input_type == "id" or input_type == "arxiv_id": - arxiv_id = input - - elif input_type == "bib" or input_type == "arxiv_bib": - bib_dict = self.extract_bib_from_string(input) - arxiv_id = self.extract_arxiv_id_from_bib(bib_dict) - - elif input_type == "url" or input_type == "link": - arxiv_id = self.arxiv_url_to_id(input) - - else: - # Raise error for unknown input_type - raise ValueError(f"Unknown input_type '{input_type}'. " - f"Expected one of: 'id', 'arxiv_id', 'bib', 'arxiv_bib', 'url', 'link'.") - return arxiv_id \ No newline at end of file diff --git a/research_arcade/arxiv_utils/paper_collector/__init__.py b/research_arcade/arxiv_utils/paper_collector/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research_arcade/arxiv_utils/paper_collector/error_handler.py b/research_arcade/arxiv_utils/paper_collector/error_handler.py deleted file mode 100644 index 954fd0e..0000000 --- a/research_arcade/arxiv_utils/paper_collector/error_handler.py +++ /dev/null @@ -1,87 +0,0 @@ -import math -import time -from functools import wraps - -from beartype.typing import Any, Callable, Optional, TypeVar, cast -from pydantic import BaseModel - -INF = float(math.inf) - -T = TypeVar("T", bound=Callable[..., Any]) - - -def api_calling_error_exponential_backoff( - retries: int = 5, base_wait_time: int = 1 -) -> Callable[[T], T]: - """ - Decorator for applying exponential backoff to a function. - :param retries: Maximum number of retries. - :param base_wait_time: Base wait time in seconds for the exponential backoff. - :return: The wrapped function with exponential backoff applied. - """ - - def decorator(func: T) -> T: - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - error_handler_mode = kwargs.get("mode", None) - if error_handler_mode == "TEST": - modified_retries = 1 - modified_base_wait_time = 1 - else: - modified_retries = retries - modified_base_wait_time = base_wait_time - - attempts = 0 - while attempts < modified_retries: - try: - return func(*args, **kwargs) - except Exception as e: - wait_time = modified_base_wait_time * (2**attempts) - print(f"Attempt {attempts + 1} failed: {e}") - print(f"Waiting {wait_time} seconds before retrying...") - time.sleep(wait_time) - attempts += 1 - print( - f"Failed to execute '{func.__name__}' after {modified_retries} retries." - ) - return None - - return cast(T, wrapper) - - return cast(Callable[[T], T], decorator) - - -TBaseModel = TypeVar("TBaseModel", bound=Callable[..., BaseModel]) - - -def parsing_error_exponential_backoff( - retries: int = 5, base_wait_time: int = 1 -) -> Callable[[TBaseModel], TBaseModel]: - """ - Decorator for retrying a function that returns a BaseModel with exponential backoff. - :param retries: Maximum number of retries. - :param base_wait_time: Base wait time in seconds for the exponential backoff. - :return: The wrapped function with retry logic applied. - """ - - def decorator(func: TBaseModel) -> TBaseModel: - @wraps(func) - def wrapper(self: Any, *args: Any, **kwargs: Any) -> Optional[BaseModel]: - attempts = 0 - while attempts < retries: - try: - return func(self, *args, **kwargs) - except Exception as e: - wait_time = base_wait_time * (2**attempts) - print(f"Attempt {attempts + 1} failed: {e}") - print(f"Waiting {wait_time} seconds before retrying...") - time.sleep(wait_time) - attempts += 1 - print( - f"Failed to get valid input from {func.__name__} after {retries} retries." - ) - return None - - return cast(TBaseModel, wrapper) - - return cast(Callable[[TBaseModel], TBaseModel], decorator) diff --git a/research_arcade/arxiv_utils/paper_collector/graph_construction.py b/research_arcade/arxiv_utils/paper_collector/graph_construction.py deleted file mode 100644 index 5cc08dd..0000000 --- a/research_arcade/arxiv_utils/paper_collector/graph_construction.py +++ /dev/null @@ -1,702 +0,0 @@ -import json -import os -import queue -import threading -import time -from datetime import datetime - -from beartype.typing import Any, Callable, Dict, List, Optional -from tqdm import tqdm - -from .latex_parser import ( - LatexEnvironmentNode, - build_ast, - extract_citations_from_ast, - extract_section_info_from_ast, - get_base_info, - get_bib_names, - load_bbl_info, - load_bib_info, - load_bib_key_to_arxiv_id, -) -from .utils import download_latex_source, search_arxiv_id - - -def build_citation_graph_node_info( - working_path: str, tex_name: str, current_paper: str, get_citation_info: bool = True -) -> Dict[str, Any]: - structured_data = None - with open(os.path.join(working_path, tex_name), "r", encoding="utf-8") as tex_file: - latex_code = tex_file.read() - ast = build_ast(latex_code) - res = {"title": None, "author": None, "doc_node": None} - get_base_info(ast, res) - title, author, doc_node = res["title"], res["author"], res["doc_node"] - result = search_arxiv_id(current_paper) - categories = [] - primary_category = None - summary = None - published = None - if len(result): - result = result[0] - title = result.title - author = result.authors[0].name - categories = result.categories - primary_category = result.primary_category - summary = result.summary - published = str(result.published) - bib_names, bbl_nodes = get_bib_names(ast) - # print(f"bib_names: {bib_names}") - # print(f"bbl_nodes: {bbl_nodes}") - structured_data = { - "title": title, - "author": author, - "abstract": None, - "citations": {}, - "refs": [], - "table": [], - "figure": [], - "equations": [], - "algorithm": [], - "sections": {}, - "categories": categories, - "published": published, - "primary_category": primary_category, - "summary": summary, - } - key2title = {} - key2author = {} - key2id = {} - if doc_node: - extract_section_info_from_ast( - structured_data, - [], - [], - [doc_node], - [], - (None, None, None), - key2title, - key2author, - tex_name, - False, - [], - "", - working_path, - ) - print(f"numbmer of citations in node info method: {len(structured_data['citations'])}") - if get_citation_info: - if len(bib_names) > 0: - for bib in bib_names: - bib = bib.replace(".bib", "").strip() - if os.path.exists(os.path.join(working_path, bib + ".bib")): - bib_path = os.path.join(working_path, bib + ".bib") - print(f"bib_path: {bib_path}") - load_bib_info(bib_path, key2title, key2author) - load_bib_key_to_arxiv_id(bib_path, key2id=key2id) - # TODO: store the extracted information into database - print(f"key2id: {key2id}") - else: - print(f"Cannot find the bib file {bib}.bib") - - # print(f"len(key2title): {len(key2title)}") - # Unlikely that the bib here is empty - if len(key2title) == 0: - bbl_files = [f for f in os.listdir(working_path) if f.endswith(".bbl")] - for file in bbl_files: - try: - with open( - os.path.join(working_path, file), "r", encoding="utf-8" - ) as bbl_file: - bbl_text = bbl_file.read() - bbl_ast = build_ast(bbl_text) - bbl_nodes.extend( - [ - node - for node in bbl_ast - if isinstance(node, LatexEnvironmentNode) - and node.environmentname == "thebibliography" - ] - ) - except Exception as e: - print(e) - continue - for bbl_node in bbl_nodes: - load_bbl_info(bbl_node, key2title, key2author) - - if len(key2title) == 0: - print("Cannot find the bib file") - return structured_data - - # It seems that even though we can find the path to bib/bbl file, we still might not extract the citation information? - # print(f"key2title: {key2title}") - # print(f"key2author: {key2author}") - - appendix = False - prev_citation_contexts = [] - prev_ref_contexts = [] - next_context = "" - next_ref_context = "" - flattened_data = [] - recent_nodes = [] - env_stack = [] - current_section_name = None - current_subsection_name = None - current_subsubsection_name = None - # The problem must lie in here! - - - if doc_node: - extract_citations_from_ast( - structured_data, - flattened_data, - recent_nodes, - [doc_node], - env_stack, - (current_section_name, current_subsection_name, current_subsubsection_name), - key2title, - key2author, - tex_name, - appendix, - prev_citation_contexts, - prev_ref_contexts, - next_context, - next_ref_context, - working_path, - ) - - # Add the extracted arxiv id to the database - if key2id: - citations = structured_data.get('citations', {}) - for bib_key, arxiv_id in key2id.items(): - entry = citations.get(bib_key) - if entry and not entry.get('short_id'): - entry['short_id'] = arxiv_id - - return structured_data - -def build_citation_graph( - seed: List[str], - source_path: str, - working_path: str, - output_path: str, - debug_path: Optional[str], - constraint: Callable[[str], bool], - scale: int = 999999999, - clear_source: bool = False, - max_figure: int = 99999999999, -): - # seed consists of a list of arxiv short_id - # using BFS search to build the citation graph - # download the latex source code into source_path - # unzip the source code into working directory - # save the structured data into output_path - # constrain the paper published time to construct the train/test set - # scale the number of papers to download - BFS_que = queue.Queue() - for seed_ in seed: - BFS_que.put(seed_) - visited = set() - history = [] - cnt = 0 - endpoints_path = os.path.join(output_path, "endpoints") - # create folders - if not os.path.exists(working_path): - os.makedirs(working_path) - if not os.path.exists(output_path): - os.makedirs(output_path) - if not os.path.exists(endpoints_path): - os.makedirs(endpoints_path) - if not os.path.exists(source_path): - os.makedirs(source_path) - if debug_path and not os.path.exists(debug_path): - os.makedirs(debug_path) - # print(os.path.join(output_path, 'history.json')) - # print(os.path.exists(os.path.join(output_path, 'history.json'))) - if os.path.exists(os.path.join(output_path, "history.json")): - pass - # with open(os.path.join(output_path, "history.json"), "r") as f: - # history = json.load(f) - # for item in history: - # for item_ in item["extended"]: - # BFS_que.put(item_) - # cnt += 1 - # visited = set([item["paper"] for item in history]) - # print(cnt) - # Create a tqdm progress bar - with tqdm(total=scale, desc="Processing") as pbar: - for _ in range(cnt): - pbar.update(1) - while BFS_que.qsize() > 0: - current_paper = BFS_que.get() - if current_paper in visited: - continue - get_citation_info = cnt < scale - visited.add(current_paper) - download_latex_source(current_paper, source_path) - - # clear the working directory - os.system(f"rm -rf {working_path}/*") - os.system( - f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {working_path} > /dev/null" - ) - if clear_source: - os.system(f"rm {os.path.join(source_path, current_paper+'.tar.gz')}") - # get all files end with figure extensions in working directory - fig_path_mapping = {} - for root, dirs, files in os.walk(working_path): - for file in files: - if ( - file.endswith(".pdf") - or file.endswith(".png") - or file.endswith(".jpg") - or file.endswith(".jpeg") - ): - fig_path_mapping[file.split("/")[-1]] = os.path.join(root, file) - fig_path_mapping[file.split("/")[-1].lower()] = os.path.join( - root, file - ) - # os.system(f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {working_path} > /dev/null") - # find the tex file with document environment - tex_files = [f for f in os.listdir(working_path) if f.endswith(".tex")] - doc_files = [] - if len(tex_files) == 0: - # os.system(f"cp {working_path} {os.path.join(debug_path, current_paper)}") - if debug_path: - if not os.path.exists(os.path.join(debug_path, current_paper)): - os.makedirs(os.path.join(debug_path, current_paper)) - os.system( - f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {os.path.join(debug_path, current_paper)} > /dev/null" - ) - print(f"Failed to process {current_paper}") - continue - for tex_file in tex_files: - try: - with open( - os.path.join(working_path, tex_file), "r", encoding="utf-8" - ) as tex_file_: - latex_code = tex_file_.read() - ast = build_ast(latex_code) - except Exception as e: - print(e) - continue - res = {"title": None, "author": None, "doc_node": None} - get_base_info(ast, res) - if res["doc_node"]: - doc_files.append(tex_file) - structured_data = None - print(f"doc_files: {doc_files}") - for tex_file in doc_files: - print(f"Processing file {tex_file}!") - try: - structured_data = build_citation_graph_node_info( - working_path, - tex_file, - current_paper, - get_citation_info=get_citation_info, - ) - except Exception as e: - print(e) - # move the figures to output_path/figures/current_paper/... - if structured_data: - if not os.path.exists( - os.path.join(output_path, "figures", current_paper) - ): - os.makedirs(os.path.join(output_path, "figures", current_paper)) - if cnt < max_figure: - for figure in structured_data["figure"]: - for figure_path in figure["figure_paths"]: - if not os.path.exists( - os.path.join(working_path, figure_path) - ): - flag = False - file_key = figure_path.split("/")[-1].lower() - if fig_path_mapping.get(file_key): - flag = True - figure_path = fig_path_mapping[file_key] - for ending in [".pdf", ".png", ".jpg", ".jpeg"]: - if os.path.exists( - os.path.join(working_path, figure_path + ending) - ): - flag = True - figure_path = figure_path + ending - break - if fig_path_mapping.get(file_key + ending): - flag = True - figure_path = fig_path_mapping[ - file_key + ending - ] - break - if not flag: - print(f"Cannot find the figure {figure_path}") - os.system( - f"cp {os.path.join(working_path, figure_path)} {os.path.join(output_path, 'figures', current_paper, '_'.join(figure_path.split('/')))}" - ) - - # os.system(f"cp {working_path} {os.path.join(debug_path, current_paper)}") - if debug_path: - if not os.path.exists(os.path.join(debug_path, current_paper)): - os.makedirs(os.path.join(debug_path, current_paper)) - os.system( - f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {os.path.join(debug_path, current_paper)} > /dev/null" - ) - print(f"Failed to process {current_paper}!") - continue - if not structured_data or len(structured_data["citations"]) == 0: - if structured_data: - with open( - os.path.join(endpoints_path, current_paper + ".json"), "w" - ) as f: - json.dump(structured_data, f) - # os.system(f"cp {working_path} {os.path.join(debug_path, current_paper)}") - if debug_path: - if not os.path.exists(os.path.join(debug_path, current_paper)): - os.makedirs(os.path.join(debug_path, current_paper)) - os.system( - f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {os.path.join(debug_path, current_paper)} > /dev/null" - ) - print(f"Failed to process {current_paper}!") - continue - if structured_data: - if get_citation_info: - with open( - os.path.join(output_path, current_paper + ".json"), "w" - ) as f: - json.dump(structured_data, f) - else: - with open( - os.path.join(endpoints_path, current_paper + ".json"), "w" - ) as f: - json.dump(structured_data, f) - cnt += 1 - pbar.update(1) - extended_list = [] - # apply constraint here. - if (structured_data["citations"]) and (cnt <= scale): - for citation in structured_data["citations"].values(): - if citation["short_id"]: - id = citation["short_id"] - published = datetime.fromisoformat(citation["published"]) - if (id not in visited) and constraint(published): - BFS_que.put(citation["short_id"]) - extended_list.append(citation["short_id"]) - print(f"Extended list: {extended_list}") - history.append( - { - "paper": current_paper, - "extended": extended_list, - "endpoint": cnt >= scale, - } - ) - pass - # with open(os.path.join(output_path, "history.json"), "w") as f: - # json.dump(history, f) - - print(f"Finished processing {cnt} papers") - - -# Multithreaded version of the BFS function -def build_citation_graph_thread( - seed: List[str], - source_path: str, - working_path: str, - output_path: str, - debug_path: Optional[str], - constraint: Callable[[str], bool], - scale: int = 999999999, - num_threads: int = 4, - clear_source: bool = False, - max_figure: int = 99999999999, -): - # Shared BFS queue and visited set - BFS_que = queue.Queue() - print(f"seed: {seed}") - for seed_ in seed: - BFS_que.put((seed_, "True")) - visited_lock = threading.Lock() - history_lock = threading.Lock() - cnt_lock = threading.Lock() - visited = set() - history = [] - cnt = 0 - start_time = time.time() - endpoints_path = os.path.join(output_path, "endpoints") - # create folders - if not os.path.exists(working_path): - os.makedirs(working_path) - if not os.path.exists(output_path): - os.makedirs(output_path) - if not os.path.exists(endpoints_path): - os.makedirs(endpoints_path) - if not os.path.exists(source_path): - os.makedirs(source_path) - if debug_path and not os.path.exists(debug_path): - os.makedirs(debug_path) - os.system(f"rm -rf {working_path}/*") - # Load history if exists - if os.path.exists(os.path.join(output_path, "history.json")): - pass - # with open(os.path.join(output_path, "history.json"), "r") as f: - # history = json.load(f) - # # print(history) - # for item in history: - # for item_ in item["extended"]: - # BFS_que.put(item_) - # cnt += 1 - # visited = set([item["paper"] for item in history]) - # print(f"cnt: {cnt}") - print(f"BFS_que.qsize(): {BFS_que.qsize()}") - - def process_papers(): - nonlocal cnt - nonlocal BFS_que - nonlocal visited - nonlocal history - - # TODO: uncomment the print statement below - # print(f"Thread {str(threading.get_ident())} Started processing") - - while True: - try: - try: - # print(f"BFS_que: {BFS_que}") - # print(f"Is BFS_que empty? {BFS_que.empty()}") - current_paper, published = BFS_que.get( - timeout=10 - ) # Timeout to avoid infinite waiting - print(f"current paper: {current_paper}") - if published != "True": - published = datetime.fromisoformat(published) - except queue.Empty: - # print("The queue is empty") - break - print(f"Thread {str(threading.get_ident())} Processing {current_paper}") - with cnt_lock: - cnt_ = cnt - # print(f"constraint(published): {constraint(published)}") - get_citation_info = (cnt_ < scale) and constraint(published) - with visited_lock: - if current_paper in visited: - continue - visited.add(current_paper) - - # TODO: see what would happen if we comment out this line - # Here, we separate the paper downloading and paper processing stages, which is different from what is was like. - # download_latex_source(current_paper, source_path) - - # Clear the working directory - thread_working_path = os.path.join( - working_path, str(threading.get_ident()) - ) - if not os.path.exists(thread_working_path): - os.makedirs(thread_working_path) - os.system(f"rm -rf {thread_working_path}/*") - os.system( - f"tar -xvf {os.path.join(source_path, current_paper + '.tar.gz')} -C {thread_working_path} > /dev/null" - ) - if clear_source: - os.system( - f"rm {os.path.join(source_path, current_paper+'.tar.gz')}" - ) - # get all files end with figure extensions in working directory - fig_path_mapping = {} - for root, dirs, files in os.walk(thread_working_path): - for file in files: - if ( - file.endswith(".pdf") - or file.endswith(".png") - or file.endswith(".jpg") - or file.endswith(".jpeg") - ): - fig_path_mapping[file.split("/")[-1]] = os.path.join( - root, file - ) - fig_path_mapping[ - file.split("/")[-1].lower() - ] = os.path.join(root, file) - # Identify the main .tex file - # print("Here!") - tex_files = [ - f for f in os.listdir(thread_working_path) if f.endswith(".tex") - ] - doc_files = [] - if len(tex_files) == 0: - # os.system(f"cp {working_path} {os.path.join(debug_path, current_paper)}") - if debug_path: - if not os.path.exists(os.path.join(debug_path, current_paper)): - os.makedirs(os.path.join(debug_path, current_paper)) - os.system( - f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {os.path.join(debug_path, current_paper)} > /dev/null" - ) - print( - f"Thread {str(threading.get_ident())} Failed to process {current_paper}" - ) - continue - for tex_file in tex_files: - try: - with open( - os.path.join(thread_working_path, tex_file), - "r", - encoding="utf-8", - ) as tex_file_: - latex_code = tex_file_.read() - except Exception as e: - print(e) - continue - ast = build_ast(latex_code) - res = {"title": None, "author": None, "doc_node": None} - get_base_info(ast, res) - if res["doc_node"]: - doc_files.append(tex_file) - - structured_data = None - for tex_file in doc_files: - print( - f"Thread {str(threading.get_ident())} Processing file {tex_file}" - ) - try: - structured_data = build_citation_graph_node_info( - thread_working_path, - tex_file, - current_paper, - get_citation_info=get_citation_info, - ) - - if structured_data.get("sections"): - break - except Exception as e: - print(e) - - - if structured_data: - if not os.path.exists( - os.path.join(output_path, "figures", current_paper) - ): - os.makedirs(os.path.join(output_path, "figures", current_paper)) - if cnt_ < max_figure: - for figure in structured_data["figure"]: - for figure_path in figure["figure_paths"]: - if not os.path.exists( - os.path.join(thread_working_path, figure_path) - ): - flag = False - file_key = figure_path.split("/")[-1].lower() - if fig_path_mapping.get(file_key): - flag = True - figure_path = fig_path_mapping[file_key] - for ending in [".pdf", ".png", ".jpg", ".jpeg"]: - if os.path.exists( - os.path.join( - thread_working_path, - figure_path + ending, - ) - ): - flag = True - figure_path = figure_path + ending - break - if fig_path_mapping.get(file_key + ending): - flag = True - figure_path = fig_path_mapping[ - file_key + ending - ] - break - if not flag: - print(f"Cannot find the figure {figure_path}") - os.system( - f"cp {os.path.join(thread_working_path, figure_path)} {os.path.join(output_path, 'figures', current_paper, '_'.join(figure_path.split('/')))}" - ) - if not structured_data or len(structured_data["citations"]) == 0: - if structured_data: - with open( - os.path.join(endpoints_path, current_paper + ".json"), "w" - ) as f: - json.dump(structured_data, f) - if debug_path: - if not os.path.exists(os.path.join(debug_path, current_paper)): - os.makedirs(os.path.join(debug_path, current_paper)) - os.system( - f"tar -xvf {os.path.join(source_path, current_paper+'.tar.gz')} -C {os.path.join(debug_path, current_paper)} > /dev/null" - ) - print( - f"Thread {str(threading.get_ident())} Failed to process {current_paper}!" - ) - continue - - if structured_data: - if get_citation_info: - with open( - os.path.join(output_path, current_paper + ".json"), "w" - ) as f: - json.dump(structured_data, f) - else: - with open( - os.path.join(endpoints_path, current_paper + ".json"), "w" - ) as f: - json.dump(structured_data, f) - with cnt_lock: - cnt += 1 - cnt__ = cnt - print( - f"Thread {str(threading.get_ident())} Finished processing {current_paper} ({cnt}/{scale}) Time elapsed: {time.time()-start_time:.2f}s" - ) - # Extend BFS queue based on citations and constraint - extended_list = [] - if (structured_data["citations"]) and (cnt_ < scale): - for citation in structured_data["citations"].values(): - if citation["short_id"]: - id = citation["short_id"] - published = datetime.fromisoformat(citation["published"]) - with visited_lock: - if id not in visited: - BFS_que.put((id, str(published))) - extended_list.append((id, str(published))) - published = str(published) - with history_lock: - history.append( - { - "paper": current_paper, - "cnt": cnt__, - "extended": extended_list, - "time_used": time.time() - start_time, - "endpoint": cnt_ >= scale, - "que_len": BFS_que.qsize(), - "published": published, - } - ) - pass - # with open(os.path.join(output_path, "history.json"), "w") as f: - # json.dump(history, f) - except Exception as e: - print(e) - print( - f"Thread {str(threading.get_ident())} Failed to process {current_paper}" - ) - with history_lock: - history.append( - { - "paper": current_paper, - "cnt": cnt_, - "extended": [], - "time_used": time.time() - start_time, - "endpoint": cnt_ >= scale, - "que_len": BFS_que.qsize(), - "published": str(published), - } - ) - pass - # with open(os.path.join(output_path, "history.json"), "w") as f: - # json.dump(history, f) - continue - - # Create and start threads - num_threads = num_threads - threads = [] - for _ in range(num_threads): - thread = threading.Thread(target=process_papers) - threads.append(thread) - thread.start() - - # Wait for all threads to finish - for thread in threads: - thread.join() - - print(f"Thread {str(threading.get_ident())} Finished processing {cnt} papers") diff --git a/research_arcade/arxiv_utils/paper_collector/latex_parser.py b/research_arcade/arxiv_utils/paper_collector/latex_parser.py deleted file mode 100644 index 63f6944..0000000 --- a/research_arcade/arxiv_utils/paper_collector/latex_parser.py +++ /dev/null @@ -1,1113 +0,0 @@ -import os -import re - -import bibtexparser -from beartype.typing import Any, Dict, List, Tuple, Union -from pylatexenc.latexwalker import ( - LatexCharsNode, - LatexCommentNode, - LatexEnvironmentNode, - LatexGroupNode, - LatexMacroNode, - LatexMathNode, - LatexWalker, - get_default_latex_context_db, -) -from pylatexenc.macrospec import MacroSpec -from .utils import query_and_match - -_SUBFIGURE_ENVS = { - "figure", - "figure*", - "minipage", - "subfigure", # from subcaption / subfigure packages - "wrapfigure", - "floatrow", - "sidewaysfigure", -} - -_SUBFIGURE_MACROS = {"subfloat"} - - -def clean_latex_code(latex_str: str) -> str: - # Remove LaTeX commands - cleaned_str = re.sub(r"\\[a-zA-Z]+(\[[^\]]*\])?(\{[^}]*\})?", "", latex_str) - return cleaned_str - - -def clean_latex_format(latex_text: str) -> str: - # Remove LaTeX commands, e.g., \em, \textbf, \textit, etc. - plain_text = re.sub(r"\\[a-zA-Z]+\s?", "", latex_text) - - # Remove curly braces but keep the text inside - plain_text = re.sub(r"[{}]", "", plain_text) - - # Strip any leading/trailing whitespace - plain_text = plain_text.strip() - - return plain_text - - -def is_stop_node(node: str) -> bool: - if ( - "appendix" in node - or "label" in node - or "caption" in node - or "paragraph" in node - ): - return True - if "input" in node or "include" in node: - return True - if "section" in node: - return True - return False - - -def get_last_paragraph(text: str) -> Tuple[str, bool]: - # Split text into paragraph and handle common punctuation - paragraph = re.split(r"\n\n|\\\\|\}\n", text.strip()) - if not paragraph: - return None, False - - last_paragraph = paragraph[-1] - is_complete = len(paragraph) > 1 - return last_paragraph, is_complete - - -def get_first_paragraph(text: str) -> Tuple[str, bool]: - # Split text into paragraph and handle common punctuation - paragraph = re.split(r"\n\n|\\\\|\}\n", text.strip()) - if not paragraph: - return None, False - - last_paragraph = paragraph[0] - is_complete = len(paragraph) > 1 - return last_paragraph, is_complete - - -def get_last_sentence(text: str) -> Tuple[str, bool]: - # Split text into sentences and handle common punctuation - sentences = re.split(r"(?<=[.!?])\s+|\n+|\\\\|\&", text.strip()) - if not sentences: - return None, False - - last_sentence = sentences[-1] - is_complete = len(sentences) > 1 - if len(last_sentence) > 0 and last_sentence[-1] in ".?!": - last_sentence = "" - is_complete = True - return last_sentence, is_complete - - -def get_first_sentence(text: str) -> Tuple[str, bool]: - # Split text into sentences and handle common punctuation - sentences = re.split(r"(?<=[.!?])\s+|\n+|\\\\|\&", text.strip()) - if not sentences: - return None, False - - last_sentence = sentences[0] - is_complete = len(sentences) > 1 - if len(last_sentence) > 0 and last_sentence[-1] in ".?!\n": - is_complete = True - return last_sentence, is_complete - - -# flat_data: for storing the text content of the current section, ignoring the node if it is a subnode of a environment -# recent_nodes: storing recent node texts for citation context -def extract_section_info_from_ast( - structured_data: Dict[str, Any], - flat_data: List[str], - recent_nodes: List[str], - nodes: List[Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode]], - env_stack: List[str], - section_names: Dict[str, str], - key2title: Dict[str, str], - key2author: Dict[str, str], - current_file: str, - appendix: bool, - prev_citation_contexts: List[Dict[str, Any]] = [], - next_context: str = "", - working_path: str = "", - flag: bool = False, -): - try: - in_environment = len(env_stack) > 1 - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ) = section_names - for node in nodes: - if isinstance(node, LatexCommentNode): - continue - scope_name = "" - node_text = node.latex_verbatim() - if (not in_environment) or (not isinstance(node, LatexEnvironmentNode)): - flat_data.append(node_text) - if isinstance(node, LatexEnvironmentNode): - scope_name = node.environmentname - if "table" in node.environmentname: - table_info = parse_tableEnv(node) - if table_info: - structured_data["table"].append(table_info) - - # flat_data.append(table_info) - # structured_data['table'].append(table_info) - elif "figure" in node.environmentname: - figure_info = parse_figureEnv(node) - if figure_info: - structured_data["figure"].append(figure_info) - - # flat_data.append(figure_info) - # structured_data['figure'].append(figure_info) - elif ( - "equation" in node.environmentname - or "align" in node.environmentname - ): - equation_info = parse_equationEnv(node) - if equation_info: - structured_data["equations"].append(equation_info) - if not in_environment: - flat_data.append(equation_info) - elif "algorithm" in node.environmentname: - algorithm_info = parse_algorithmEnv(node) - if algorithm_info: - structured_data["algorithm"].append(algorithm_info) - if not in_environment: - flat_data.append(algorithm_info) - elif ( - "itemize" in node.environmentname - or "enumerate" in node.environmentname - ): - itemize_info = parse_itemizeEnv(node) - if not in_environment: - flat_data.append(itemize_info) - elif node.environmentname == "minipage": - if "tabular" in node.latex_verbatim(): - table_info = parse_tableEnv(node) - if table_info: - structured_data["table"].append(table_info) - elif "figure" in node.latex_verbatim(): - figure_info = parse_figureEnv(node) - if figure_info: - structured_data["figure"].append(figure_info) - elif node.environmentname == "document": - pass - elif node.environmentname == "tabular": - pass - else: - unknown_info = parse_unknownEnv(node) - if not in_environment: - flat_data.append(unknown_info) - # if unknown_info: - # structured_data['unknown'].append(unknown_info) - - if node.environmentname == "abstract": - abstract = node.latex_verbatim() - if "\\input" in abstract: - if not in_environment: - flat_data.pop() - input = ( - re.search(r".*?\\input{(.*)}", abstract) - .group(1) - .replace(".tex", "") - ) - try: - with open( - os.path.join(working_path, f"{input}.tex"), - "r", - encoding="utf-8", - ) as tex_file: - latex_code = tex_file.read() - abstract = latex_code - except Exception as e: - print(e) - structured_data["abstract"] = abstract - if isinstance(node, LatexMacroNode): - if node.macroname == "appendix": - if not in_environment: - flat_data.pop() - appendix = True - if node.macroname == "input" or node.macroname == "include": - if not in_environment: - flat_data.pop() - try: - input = safe_extract_chars(node.nodeargd.argnlist[0]) - input = input.replace(".tex", "").strip() - with open( - os.path.join(working_path, f"{input}.tex"), - "r", - encoding="utf-8", - ) as tex_file: - latex_code = tex_file.read() - ast = build_ast(latex_code) - - if f"{input}.tex" != current_file: - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ) = extract_section_info_from_ast( - structured_data, - flat_data, - recent_nodes, - ast, - env_stack, - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ), - key2title, - key2author, - "{input}.tex", - appendix, - prev_citation_contexts, - next_context, - working_path, - True, - ) - - except Exception as e: - print(e) - if node.macroname == "section": - section_name = "" - for arg_node in node.nodeargd.argnlist: - if ( - isinstance(arg_node, LatexGroupNode) - and arg_node.delimiters[0] == "{" - ): - section_name = clean_latex_code( - arg_node.latex_verbatim() - ).strip("{}") - # assert section_name - if current_section_name: - structured_data["sections"][current_section_name] = { - "content": "".join(flat_data[:-1]), - "appendix": appendix, - } - flat_data.clear() - current_section_name = section_name - current_subsection_name = None - current_subsubsection_name = None - if node.macroname == "subsection": - subsection_name = "" - for arg_node in node.nodeargd.argnlist: - if ( - isinstance(arg_node, LatexGroupNode) - and arg_node.delimiters[0] == "{" - ): - subsection_name = clean_latex_code( - arg_node.latex_verbatim() - ).strip("{}") - # assert subsection_name - current_subsection_name = subsection_name - current_subsubsection_name = None - if node.macroname == "subsubsection": - subsubsection_name = "" - for arg_node in node.nodeargd.argnlist: - if ( - isinstance(arg_node, LatexGroupNode) - and arg_node.delimiters[0] == "{" - ): - subsubsection_name = clean_latex_code( - arg_node.latex_verbatim() - ).strip("{}") - # assert subsubsection_name - current_subsubsection_name = subsubsection_name - # Recursively check child nodes if the node is an environment (e.g., document or section) - if hasattr(node, "nodelist"): - env_stack.append(scope_name) - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ) = extract_section_info_from_ast( - structured_data, - flat_data, - recent_nodes, - node.nodelist, - env_stack, - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ), - key2title, - key2author, - current_file, - appendix, - prev_citation_contexts, - next_context, - working_path, - flag, - ) - env_stack.pop() - - if ( - nodes - and isinstance(nodes[0], LatexEnvironmentNode) - and (nodes[0].environmentname == "document") - and (current_section_name) - ): - structured_data["sections"][current_section_name] = { - "content": "".join(flat_data), - "appendix": appendix, - } - return ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ) - except Exception as e: - print(e) - print("Failed to process the node") - return ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ) - - -def extract_citations_from_ast( - structured_data: Dict[str, Any], - flat_data: List[str], - recent_nodes: List[str], - nodes: List[Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode]], - env_stack: List[str], - section_names: Tuple[str, str, str], - key2title: Dict[str, str], - key2author: Dict[str, str], - current_file: str, - appendix: bool, - prev_citation_contexts: List[Dict[str, Any]] = [], - prev_ref_contexts: List[Dict[str, Any]] = [], - next_context: str = "", - next_ref_context: str = "", - working_path: str = "", - flag: bool = False, -): - try: - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ) = section_names - in_environment = len(env_stack) > 1 - for node in nodes: - if isinstance(node, LatexCommentNode): - continue - scope_name = "" - node_text = node.latex_verbatim() - if isinstance(node, LatexEnvironmentNode): - scope_name = node.environmentname - if not in_environment: - flat_data.append(node_text) - is_node_group = hasattr(node, "nodelist") - if not is_node_group: - recent_nodes.append(node_text) - - """if (not is_node_group) and (len(prev_ref_contexts)>0): - next_paragraph, is_complete = get_first_paragraph(node_text) - next_ref_context += next_paragraph - if is_complete: - for ref_context in prev_ref_contexts: - ref_context['next_context'] = next_ref_context[ref_context['_pos']:] - ref_context.pop('_pos') - prev_ref_contexts.clear() - next_ref_context = '' - """ - - if (not is_node_group) and (len(prev_citation_contexts) > 0): - next_sentence, is_complete = get_first_sentence(node_text) - next_context += next_sentence - if is_complete: - for citation_context in prev_citation_contexts: - citation_context["next_context"] = next_context[ - citation_context["_pos"] : - ] - citation_context.pop("_pos") - prev_citation_contexts.clear() - next_context = "" - - if isinstance(node, LatexMacroNode): - if node.macroname == "appendix": - appendix = True - if not in_environment: - flat_data.pop() - if node.macroname == "input" or node.macroname == "include": - if not in_environment: - flat_data.pop() - try: - input = safe_extract_chars(node.nodeargd.argnlist[0]) - input = input.replace(".tex", "").strip() - with open( - os.path.join(working_path, f"{input}.tex"), - "r", - encoding="utf-8", - ) as tex_file: - latex_code = tex_file.read() - ast = build_ast(latex_code) - if f"{input}.tex" != current_file: - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - next_ref_context, - next_context, - ) = extract_citations_from_ast( - structured_data, - flat_data, - recent_nodes, - ast, - env_stack, - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ), - key2title, - key2author, - f"{input}.tex", - appendix, - prev_citation_contexts, - prev_ref_contexts, - next_context, - next_ref_context, - working_path, - flag, - ) - except Exception as e: - print(e) - - if node.macroname == "section": - section_name = "" - for arg_node in node.nodeargd.argnlist: - if ( - isinstance(arg_node, LatexGroupNode) - and arg_node.delimiters[0] == "{" - ): - section_name = clean_latex_code( - arg_node.latex_verbatim() - ).strip("{}") - flat_data.clear() - current_section_name = section_name - current_subsection_name = None - current_subsubsection_name = None - if node.macroname == "subsection": - subsection_name = "" - for arg_node in node.nodeargd.argnlist: - if ( - isinstance(arg_node, LatexGroupNode) - and arg_node.delimiters[0] == "{" - ): - subsection_name = clean_latex_code( - arg_node.latex_verbatim() - ).strip("{}") - # assert subsection_name - current_subsection_name = subsection_name - current_subsubsection_name = None - if node.macroname == "subsubsection": - subsubsection_name = "" - for arg_node in node.nodeargd.argnlist: - if ( - isinstance(arg_node, LatexGroupNode) - and arg_node.delimiters[0] == "{" - ): - subsubsection_name = clean_latex_code( - arg_node.latex_verbatim() - ).strip("{}") - # assert subsubsection_name - current_subsubsection_name = subsubsection_name - """ - if isinstance(node, LatexMacroNode) and 'ref' in node.macroname: - for spec, argn in zip(node.nodeargd.argspec, node.nodeargd.argnlist): - if spec=='{': - label = '' - if len(argn.nodelist)>0: - label = argn.nodelist[0].chars - prev_context = '' - for prev in reversed(recent_nodes): - if is_stop_node(prev): - break - last_paragraph, is_complete = get_last_paragraph(prev) - prev_context = last_paragraph + prev_context - if is_complete: - break - ref_info = {'label': label, 'section': current_section_name, 'subsection': current_subsection_name, 'subsubsection': current_subsubsection_name, 'prev_context': prev_context, '_pos': len(next_ref_context)} - structured_data['refs'].append(ref_info) - prev_ref_contexts.append(ref_info) - """ - - # Check for citation macro "\cite" - if ( - key2title - and isinstance(node, LatexMacroNode) - and "cite" in node.macroname - ): - for spec, citation in zip( - node.nodeargd.argspec, node.nodeargd.argnlist - ): - if spec == "{": - citation_key = "" - if len(citation.nodelist) > 0: - citation_key = citation.nodelist[0].chars - cite_importance = 1 / len(citation_key.split(",")) - for key in citation_key.split(","): - if key2title.get(key.strip()): - title_ = key2title[key.strip()] - author_ = key2author[key.strip()] - prev_context = "" - for prev in reversed(recent_nodes): - if is_stop_node(prev): - break - last_sentence, is_complete = get_last_sentence(prev) - prev_context = last_sentence + prev_context - if is_complete: - break - - - # TODO - # For short id, it corresponds to the arxiv id - # However, arxiv id is often stored in the journal part of paper - if structured_data["citations"].get(key.strip()): - citation_data = structured_data["citations"][ - key.strip() - ] - citation_context = { - "section": current_section_name, - "subsection": current_subsection_name, - "subsubsection": current_subsubsection_name, - "prev_context": prev_context, - "_pos": len(next_context), - } - prev_citation_contexts.append(citation_context) - citation_data["context"].append(citation_context) - citation_data["importance_score"] += cite_importance - else: - paper, score = query_and_match(title_, author_) - # Since the method query_and_match() returns None only, ignore what's inside of the if statement below. - if paper: - citation_context = { - "section": current_section_name, - "subsection": current_subsection_name, - "subsubsection": current_subsubsection_name, - "prev_context": prev_context, - "_pos": len(next_context), - } - prev_citation_contexts.append(citation_context) - # Also loook at the journal part and see if arxiv id is available - - structured_data["citations"][key.strip()] = { - "bib_key": key.strip(), - "bib_title": title_, - "bib_author ": author_, - "arxiv_id": paper.entry_id, - "short_id": paper.get_short_id(), - "title": paper.title, - "author": paper.authors[0].name, - "categories": paper.categories, - "published": str(paper.published), - "abstract": paper.summary, - "similar_score": score, - "context": [citation_context], - "importance_score": cite_importance, - } - else: - citation_context = { - "section": current_section_name, - "subsection": current_subsection_name, - "subsubsection": current_subsubsection_name, - "prev_context": prev_context, - "_pos": len(next_context), - } - prev_citation_contexts.append(citation_context) - structured_data["citations"][key.strip()] = { - "bib_key": key.strip(), - "bib_title": title_, - "bib_author ": author_, - "arxiv_id": None, - "short_id": None, - "title": None, - "author": None, - "published": None, - "similar_score": None, - "context": [citation_context], - "importance_score": cite_importance, - } - # print("Result of Citation:") - # print(structured_data["citations"][key.strip()]) - - # Recursively check child nodes if the node is an environment (e.g., document or section) - if hasattr(node, "nodelist"): - env_stack.append(scope_name) - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - next_ref_context, - next_context, - ) = extract_citations_from_ast( - structured_data, - flat_data, - recent_nodes, - node.nodelist, - env_stack, - ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - ), - key2title, - key2author, - current_file, - appendix, - prev_citation_contexts, - prev_ref_contexts, - next_context, - next_ref_context, - working_path, - flag, - ) - env_stack.pop() - return ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - next_ref_context, - next_context, - ) - except Exception as e: - print(e) - print("Failed to process the node") - return ( - current_section_name, - current_subsection_name, - current_subsubsection_name, - next_ref_context, - next_context, - ) - - -def build_ast( - latex_code: str, -) -> List[Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode]]: - # Create a custom LaTeX context and add the new macro - latex_context = get_default_latex_context_db() - latex_context.add_context_category( - "custom", - macros=[ - MacroSpec("addbibliograph", args_parser="{"), - MacroSpec("addbibresource", args_parser="{"), - MacroSpec("newcommand", args_parser="{"), - MacroSpec("caption", args_parser="{"), - ], - prepend=True, - ) - - # Create a LatexWalker instance with the custom LaTeX context - walker = LatexWalker(latex_code, latex_context=latex_context) - nodes, pos, _ = walker.get_latex_nodes() - return nodes - - -class ErrorHandlerBibTexParser(bibtexparser.bparser.BibTexParser): - def _clean_val(self, val: str) -> str: - try: - return super()._clean_val(val) # Use the original cleaning logic - except KeyError as e: - print(f"Warning: Undefined string '{e}'. Ignoring.") - return "" # Or a default value - - -def load_bib_info( - path: str, key2title: Dict[str, Any], key2author: Dict[str, Any] -) -> Tuple[Dict[str, Any], Dict[str, Any]]: - # Load the BibTeX file - with open(path, "r") as bib_file: - parser = ErrorHandlerBibTexParser() - bib_database = bibtexparser.load(bib_file, parser=parser) - - # Iterate through all entries - for entry in bib_database.entries: - # print(f"entry: {entry}") - if entry.get("ID"): - id = entry.get("ID").strip() - key2title[id] = entry.get("title") - author = entry.get("author") - if author: - key2author[id] = author.split("and")[0].strip() - else: - key2author[id] = "" - return key2title, key2author - - -# We can use this to extract arxiv id written in the journal part of citations -def load_bib_key_to_arxiv_id( - path: str, key2id: Dict[str, Any] -) -> Dict[str, Any]: - - pattern = re.compile( - r'arxiv:(?P(?:\d{4}\.\d{4,5}(?:v\d+)?)|(?:[a-z\-]+/\d{7}(?:v\d+)?))', - re.IGNORECASE - ) - # print(f"pattern: {pattern}") - - with open(path, "r") as bib_file: - parser = ErrorHandlerBibTexParser() - bib_database = bibtexparser.load(bib_file, parser=parser) - for entry in bib_database.entries: - # print(f"entry: {entry}") - if entry.get("ID"): - bib_key = entry.get("ID", "").strip() - journal_field = entry.get("journal", "") - # print(f"journal: {journal}") - if not journal_field: - continue - journal_text = journal_field.strip().lower() - match = pattern.search(journal_text) - if match: - key2id[bib_key] = match.group('id') - else: - # Optional: try other fields or at least log it - # e.g. check entry.get("eprint"), entry.get("archiveprefix")... - # print(f"[!] No arXiv ID found in journal for bib key {bib_key!r}: {journal_text!r}") - pass - return key2id - - -def parse_MathNode(node: LatexMathNode) -> str: - try: - return node.latex_verbatim() - except Exception as e: - print(e) - return None - - -def parse_algorithmEnv(node: LatexEnvironmentNode) -> str: - try: - return node.latex_verbatim() - except Exception as e: - print(e) - return None - - -def extract_table_info( - nodes: Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode], - res: Dict[str, Any], -): - for node in nodes: - try: - if isinstance(node, LatexMacroNode): - if node.macroname == "caption": - res["caption"] = node.latex_verbatim() - if node.macroname == "label": - res["label"] = node.latex_verbatim() - if isinstance(node, LatexEnvironmentNode): - if "tabular" in node.environmentname: - res["tabular"] = node.latex_verbatim() - continue - if "subtable" in node.environmentname: - subtable = { - "caption": "", - "label": None, - "tabular": "", - "subtables": [], - } - extract_table_info(node.nodelist, subtable) - res["subtables"].append(subtable) - except Exception as e: - print(e) - if hasattr(node, "nodelist"): - extract_table_info(node.nodelist, res) - - -def parse_tableEnv(node: LatexEnvironmentNode) -> str: - try: - res = { - "original": node.latex_verbatim(), - "caption": "", - "label": None, - "tabular": "", - "subtables": [], - } - extract_table_info(node.nodelist, res) - return res - except Exception as e: - print(e) - return None - - -def extract_figure_info( - nodes: Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode], - res: Dict[str, Any], -): - for node in nodes: - try: - if isinstance(node, LatexMacroNode): - # print(f"node: {node}") - - if node.macroname == "caption": - res["caption"] = node.latex_verbatim() - if node.macroname == "label": - # It seems that here, there may be multiple labels in nodes, but it only takes the last one. - - res["label"] = node.latex_verbatim() - if node.macroname == "includegraphics": - # We can use this part to extract label-path pairs - # print(f"node of figure: {node}") - # print(f"node of figure verbatim: {node.latex_verbatim()}") - for argd, arg_node in zip( - node.nodeargd.argspec, node.nodeargd.argnlist - ): - # print(f"argd: {argd}") - # print(f"arg_node: {arg_node}") - if argd == "{": - res["figure_paths"].append( - arg_node.latex_verbatim().strip("{}") - ) - if isinstance(node, LatexEnvironmentNode): - if "figure" in node.environmentname: - subfigure = { - "caption": "", - "label": None, - "figure_paths": [], - "subfigures": [], - } - extract_figure_info(node.nodelist, subfigure) - res["subfigures"].append(subfigure) - except Exception as e: - print(e) - if hasattr(node, "nodelist"): - extract_figure_info(node.nodelist, res) - -def is_implicit_subfigure(node): - has_graphic = False - has_caption_or_label = False - - def walk(n): - nonlocal has_graphic, has_caption_or_label - if isinstance(n, LatexMacroNode): - if n.macroname == "includegraphics": - has_graphic = True - if n.macroname in ("caption", "label"): - has_caption_or_label = True - if hasattr(n, "nodelist") and n.nodelist: - for child in n.nodelist: - walk(child) - - walk(node) - return has_graphic and has_caption_or_label - - -def extract_figure_info_new( - nodes: Union[list, LatexCharsNode, LatexMacroNode, LatexEnvironmentNode], - res: Dict[str, Any], -): - """ - Extract subfigures inside of the figure - """ - if not nodes: - return - - # Normalize to iterable - if not isinstance(nodes, (list, tuple)): - nodes = [nodes] - - for node in nodes: - try: - # If this node itself is a subfigure-like container, create a fresh context - if isinstance(node, LatexEnvironmentNode) and node.environmentname in _SUBFIGURE_ENVS: - # New subfigure dict - subfigure = { - "caption": "", - "label": None, - "figure_paths": [], - "subfigures": [], - } - # Recurse into its content - extract_figure_info(node.nodelist, subfigure) - # Append to current result's subfigures - res.setdefault("subfigures", []).append(subfigure) - # Also continue into its children in case of nested structures - continue - - # explicit subfloat-style macro - if isinstance(node, LatexMacroNode) and node.macroname in _SUBFIGURE_MACROS: - subfigure = {"caption": "", "label": None, "figure_paths": [], "subfigures": []} - # recurse into its arguments (often contains the inner content) - for arg in node.nodeargd.argnlist: - extract_figure_info(arg, subfigure) - res.setdefault("subfigures", []).append(subfigure) - continue - - # implicit grouping via heuristic - if (isinstance(node, LatexEnvironmentNode) or isinstance(node, LatexMacroNode)) and is_implicit_subfigure(node): - subfigure = {"caption": "", "label": None, "figure_paths": [], "subfigures": []} - extract_figure_info(getattr(node, "nodelist", []) or [], subfigure) - # also dive into macro args if any - if isinstance(node, LatexMacroNode): - for arg in node.nodeargd.argnlist: - extract_figure_info(arg, subfigure) - res.setdefault("subfigures", []).append(subfigure) - continue - - if isinstance(node, LatexMacroNode): - if node.macroname == "caption": - # If multiple captions appear, you can decide to keep first/last or concatenate - # Here we overwrite so last wins; change if needed. - res["caption"] = node.latex_verbatim() - elif node.macroname == "label": - res["label"] = node.latex_verbatim() - elif node.macroname == "includegraphics": - for argd, arg_node in zip( - node.nodeargd.argspec, node.nodeargd.argnlist - ): - if argd == "{": - path = arg_node.latex_verbatim().strip("{}") - res.setdefault("figure_paths", []).append(path) - - if hasattr(node, "nodelist"): - # Recurse into children for everything else - extract_figure_info(node.nodelist, res) - except Exception as e: - # You might want to replace prints with logging. - print(f"error processing node {node}: {e}") - - -def parse_figureEnv(node: LatexEnvironmentNode) -> Dict[str, Any]: - try: - res = { - "original": node.latex_verbatim(), - "caption": "", - "label": None, - "subfigures": [], - "figure_paths": [], - } - res2 = res - # print(f"res before parsing: {res}") - # extract_figure_info(node.nodelist, res) - # print(f"parsed figure info: {res}") - extract_figure_info_new(node.nodelist, res2) - # print(f"parsed figure info with consideration of subfigures: {res2}") - return res2 - except Exception as e: - print(e) - return None - - -def parse_equationEnv(node: LatexEnvironmentNode) -> str: - try: - return node.latex_verbatim() - except Exception as e: - print(e) - return None - - -def parse_itemizeEnv(node: LatexEnvironmentNode) -> str: - try: - return node.latex_verbatim() - except Exception as e: - print(e) - return None - - -def parse_unknownEnv(node: LatexEnvironmentNode) -> str: - try: - return node.latex_verbatim() - except Exception as e: - print(e) - return None - - -def safe_extract_chars( - node: Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode, LatexMathNode] -) -> str: - if isinstance(node, LatexCharsNode): - return node.chars - elif isinstance(node, LatexGroupNode): - res = [safe_extract_chars(node_) for node_ in node.nodelist] - return "".join(res) - return "" - - -def get_bib_names( - nodes: List[Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode]] -) -> Tuple[List[str], List[LatexEnvironmentNode]]: - bib_names = [] - bbl_node = [] - for node in nodes: - if isinstance(node, LatexMacroNode): - if node.macroname == "bibliography": - bib_name = safe_extract_chars(node.nodeargd.argnlist[0]) - bib_names.extend(bib_name.split(",")) - if node.macroname == "addbibresource": - bib_name = safe_extract_chars(node.nodeargd.argnlist[0]) - bib_names.extend(bib_name.split(",")) - if isinstance(node, LatexEnvironmentNode): - if node.environmentname == "thebibliography": - bbl_node.append(node) - if hasattr(node, "nodelist"): - names, nodes = get_bib_names(node.nodelist) - bib_names.extend(names) - bbl_node.extend(nodes) - return bib_names, bbl_node - - -def get_base_info( - ast: List[Union[LatexCharsNode, LatexMacroNode, LatexEnvironmentNode]], - res: Dict[str, Any], -): - for node in ast: - if isinstance(node, LatexMacroNode): - if node.macroname == "title": - res["title"] = safe_extract_chars(node.nodeargd.argnlist[0]) - if node.macroname == "author": - res["author"] = safe_extract_chars(node.nodeargd.argnlist[0]) - if isinstance(node, LatexEnvironmentNode): - if node.environmentname == "document": - res["doc_node"] = node - if hasattr(node, "nodelist"): - get_base_info(node.nodelist, res) - - -def load_bbl_info( - bbl_node: LatexEnvironmentNode, - key2title: Dict[str, str], - key2author: Dict[str, str], -): - bbl_text = bbl_node.latex_verbatim() - bbl_list = re.sub(r"\[.*\]", "", bbl_text).split("\\bibitem") - for bbl_item in bbl_list[1:]: - key = re.search(r".*?{(.*?)}", bbl_item).group(1) - bbl_item = bbl_item.replace(r"{" + key + "}", "").strip("\n") - divider_left = ["{", "\\newblock", "'"] - divider_right = ["}", "\\newblock", "'"] - pos = len(bbl_item) - divider = -1 - for i in range(len(divider_left)): - divider_ = divider_left[i] - pos_ = bbl_item.find(divider_) - if pos_ >= 0 and pos_ < pos: - pos = pos_ - divider = i - if divider >= 0: - authors = bbl_item[:pos] - title = bbl_item[pos + len(divider_left[divider]) :] - pos_ = title.find(divider_right[divider]) - title = title[:pos_] - authors = clean_latex_format(" ".join(authors.split("\n"))) - title = clean_latex_format(" ".join(title.split("\n"))) - key2title[key] = title - key2author[key] = authors - -def extract_arxiv_id(): - """ - Extract arxiv id from latex by recognizing possible patterns - Example: journal={arXiv preprint arXiv:(arxiv id)} - We need to map the arxiv id to the bib_key - Possibly store it somewhere - """ - - - pass \ No newline at end of file diff --git a/research_arcade/arxiv_utils/paper_collector/paper_graph_processing.py b/research_arcade/arxiv_utils/paper_collector/paper_graph_processing.py deleted file mode 100644 index 5a22409..0000000 --- a/research_arcade/arxiv_utils/paper_collector/paper_graph_processing.py +++ /dev/null @@ -1,13 +0,0 @@ -from paper_graph_processor import PaperGraphProcessor - - -data_dir_path = "/Users/chongshan0lin/Documents/Research/UIUC/paper-crawler/arxiv_papers_with_source/2501.01149v2" -figures_dir_path = "/Users/chongshan0lin/Documents/Research/UIUC/paper-crawler/arxiv_papers_with_source/2501.01149v2/images" -output_dir_path = "/Users/chongshan0lin/Documents/Research/UIUC/paper-crawler/arxiv_papers_extracted_json" - -# It seems that nothing is extracted. Why? -# process_all_papers reads json files only. I need to first convert the papers into json format from latex code. How can I do that? - -pgp = PaperGraphProcessor(data_dir=data_dir_path, figures_dir=figures_dir_path, output_dir=output_dir_path) - -pgp.process_all_papers() \ No newline at end of file diff --git a/research_arcade/arxiv_utils/paper_collector/paper_graph_processor.py b/research_arcade/arxiv_utils/paper_collector/paper_graph_processor.py deleted file mode 100644 index 2eabd80..0000000 --- a/research_arcade/arxiv_utils/paper_collector/paper_graph_processor.py +++ /dev/null @@ -1,550 +0,0 @@ -import glob -import json -import os -import re -import shutil -from typing import Any, Dict, List, Optional, Tuple - -from tqdm import tqdm - -global discarded -discarded = 0 - -ENV_RE = re.compile( - r""" - hline - | \\begin\{(?:figure\*?|subfigure|table\*?|subtable| - tabular|tabularx|tabulary|longtable|array)\} - | \\(?:multicolumn|multirow) - | \\(?:cline) - | \\(?:toprule|midrule|bottomrule|cmidrule) -""", - re.IGNORECASE | re.VERBOSE, -) - - -def clean_latex_code(latex_str: str) -> str: - # Remove LaTeX commands - cleaned_str = re.sub(r"\\[a-zA-Z]+(\[[^\]]*\])?(\{[^}]*\})?", "", latex_str) - return cleaned_str.strip() - - -class PaperGraphProcessor: - """Handles the processing of academic papers and their figures.""" - - def clear_latex_format(self, text: str) -> str: - """Clear LaTeX formatting from text.""" - # Remove LaTeX commands, e.g., \em, \textbf, \textit, etc. - plain_text = re.sub(r"\\[a-zA-Z]+\s?", "", text) - - # Remove curly braces but keep the text inside - plain_text = re.sub(r"[{}]", "", plain_text) - - # Strip any leading/trailing whitespace - plain_text = plain_text.strip() - - return plain_text - - def __init__( - self, data_dir: str, figures_dir: str, output_dir: str, threshold: float = 0.8 - ): - """Initialize with directory paths for data processing. - - Args: - data_dir: Directory containing paper JSON files - figures_dir: Directory containing figure files - output_dir: Directory for processed output - """ - self.data_dir = data_dir - self.figures_dir = figures_dir - self.output_dir = output_dir - self.output_figure_dir = os.path.join(output_dir, "figures") - self.output_metadata = os.path.join(output_dir, "metadata.jsonl") - self.output_text_nodes = os.path.join(output_dir, "text_nodes.jsonl") - self.output_table_nodes = os.path.join(output_dir, "table_nodes.jsonl") - self.output_figure_nodes = os.path.join(output_dir, "figure_nodes.jsonl") - self.output_paper_nodes = os.path.join(output_dir, "paper_nodes.jsonl") - self.edge_metadata = [] - self.figure_nodes = [] - self.table_nodes = [] - self.text_nodes = [] - self.paper_nodes = [] - self.convert_codes = [] - self.paper_id2node = {} - self.threshold = threshold - self.node_id_counter = 0 - if os.path.exists(output_dir): - shutil.rmtree(output_dir) - os.makedirs(output_dir, exist_ok=True) - os.makedirs(self.output_figure_dir, exist_ok=True) - - @staticmethod - def list_json_files(directory: str) -> List[str]: - """Get all JSON files in directory.""" - return glob.glob(os.path.join(directory, "*.json")) - - @staticmethod - def clean_filename(filename: str) -> str: - """Remove file extensions and standardize filename.""" - return filename.replace(".pdf", "").replace(".png", "").replace(".jpg", "") - - def load_paper(self, file_path: str) -> Tuple[Optional[dict], str]: - """Load paper JSON and get its figure paths.""" - try: - arxiv_id = os.path.basename(file_path).split(".json")[0] - with open(file_path, "r") as f: - paper_data = json.load(f) - return paper_data, arxiv_id - except Exception as e: - print(f"Error loading {file_path}: {e}") - return None, "" - - def get_node_id(self): - """Get a unique node ID.""" - self.node_id_counter += 1 - return self.node_id_counter - - @staticmethod - def _create_label_mapping(objects: List[Any]) -> Dict[str, List[Any]]: - """Create mapping from labels to objects.""" - label_to_objects = {} - for object in objects: - if object["label"] not in label_to_objects: - label_to_objects[object["label"]] = [] - label_to_objects[object["label"]].append(object) - return label_to_objects - - def extract_label(self, text: str) -> str: - """Extract the label from a LaTeX figure reference.""" - pattern = r"\\label\{([^}]+)\}" - match = re.search(pattern, text) - return match.group(1) if match else "" - - def find_references(self, text: str) -> List[str]: - """Find all figure references in text.""" - pattern = r"\\ref\{([^}]+)\}" - return re.findall(pattern, text) - - def find_cites(self, text: str) -> List[str]: - """ - Find all citation keys in text, for commands like - \citet{}, \citet*{}, \citep{}, \citep*{}, - \citeauthor{}, \citeyear{} - """ - pattern = r"\\cite(?:t|p|author|year)\*?\{([^}]+)\}" - return re.findall(pattern, text) - - def create_figure_node(self, figure: dict, paper_id: str) -> Dict: - """Create a figure node for the graph.""" - have_paths = False - if figure["figure_paths"]: - content = [ - "_".join(fig_path.split("_")) for fig_path in figure["figure_paths"] - ] - have_paths = True - else: - content = figure["original"] - return { - "id": "figures_" + str(self.get_node_id()), - "content": content, - "fig_files": have_paths, - "label": figure["label"], - "caption": self.clear_latex_format(figure["caption"]), - "parent_paper_id": paper_id, - "type": "figureNode", - } - - def create_table_node(self, table: dict, paper_id: str) -> Dict: - """Create a table node for the graph.""" - return { - "id": "table_" + str(self.get_node_id()), - "label": table["label"], - "caption": table["caption"], - "content": table["tabular"], - "parent_paper_id": paper_id, - "type": "tableNode", - } - - def create_section_node(self, section: str) -> Dict: - """Create a section node for the graph.""" - return { - "id": self.get_node_id(), - "content": section, - "type": "sectionNode", - } - - def create_paper_node(self, paper_id, abstract, title): - return { - "id": "paper_" + str(self.get_node_id()), - "paper_id": paper_id, - "abstract": abstract, - "title": title, - "type": "paperNode", - } - - def create_text_node( - self, text: str, key2citation: Dict[str, Any], label2id: Dict[str, int] - ) -> Dict: - """Create a text node for the graph. - Return the list of bib keys of cited papers. - """ - # print("Here!") - cites_ = self.find_cites(text) - cites = [] - for cite in cites_: - cites.extend(cite.split(",")) - cites = [cite.strip() for cite in cites] - # print("Raw Cite:") - # print(cites) - # The problem is, it looks for short id/arxiv id when searching up citations in the citation part - # In most cases, as is stated before, the arxiv id is missing - # We can use bib_key for reference, as it is unique when combinede with arxiv id of the citing paper - cites = [ - key2citation[cite]["bib_key"] for cite in cites if cite in key2citation - ] - # print("Cite after looking up:") - # print(cites) - refs = self.find_references(text) - # IMPORTANT - # TODO - # Here, we pick the label as key, since it (should be) unique - raw_refs = refs - refs = [label2id[ref] for ref in refs if ref in label2id] - - isolation = (len(refs) == 0) and (len(cites) == 0) - # print("key2citation") - # print(key2citation) - # print("cites") - # print(cites) - return { - "id": "text_" + str(self.get_node_id()), - "cites": cites, - "refs": refs, - "content": text, - "type": "textNode", - "isolation": isolation, - "ref_labels": raw_refs - } - - def process_paper(self, data: dict, paper_id: str) -> Optional[Dict]: - citations = {} - figures = data["figure"] - table = data["table"] - label2id = {} - neighbors = [] - nodes = {} - global discarded - temp_figure_nodes = [] - figure_nodes_set = {} - temp_table_nodes = [] - table_nodes_set = {} - touched_refs = set() - for figure in figures: - if figure["label"]: - figure["label"] = self.extract_label(figure["label"]) - figure_node = self.create_figure_node(figure, paper_id) - nodes[figure_node["id"]] = figure_node - figure_nodes_set[figure_node["id"]] = figure_node - label2id[figure["label"]] = figure_node["id"] - temp_figure_nodes.append(figure_node) - for table in table: - if table["label"]: - table["label"] = self.extract_label(table["label"]) - table_node = self.create_table_node(table, paper_id) - table_nodes_set[table_node["id"]] = table_node - nodes[table_node["id"]] = table_node - label2id[table["label"]] = table_node["id"] - temp_table_nodes.append(table_node) - - for key, citation in data["citations"].items(): - # TODO Not quite sure why we need the similarity score. We can drop it. - if citation["similar_score"] and ( - citation["similar_score"] > self.threshold - ): - citations[key] = citation - neighbors.append(citation["short_id"]) - elif not citation["similar_score"]: - citations[key] = citation - neighbors.append(citation["short_id"]) - # print("key to citations:") - # print(citations) - print(f"Key to References: {label2id}") - for section, content in data["sections"].items(): - # section_node = self.create_section_node(section) - # nodes[section_node['id']] = section_node - chunks = [ - chunk.strip() - for chunk in content["content"].split("\n\n") - if chunk.strip() - ] - pre_node = None - # print - for chunk in chunks: - # print("Chunk:") - # print(chunk) - ######################################## - if ENV_RE.search(chunk): - continue - ######################################## - text_node = self.create_text_node(chunk, citations, label2id) - # print("text_node") - # print(text_node) - text_node["paper_id"] = paper_id - text_node["section"] = section - if True: # not text_node['isolation']: - nodes[text_node["id"]] = text_node - self.text_nodes.append(text_node) - for cite in text_node["cites"]: - if cite in self.paper_id2node: - target = self.paper_id2node[cite] - source = self.paper_id2node[paper_id] - self.edge_metadata.append( - { - "source": source, - "target": target, - "type": "citing_edge", - } - ) - self.edge_metadata.append( - { - "source": text_node["id"], - "target": source, - "type": "current_paper_edge", - } - ) - else: - print("No paper found for ", cite) - for ref in text_node["refs"]: - if ref in figure_nodes_set: - figure_node = figure_nodes_set[ref] - if not figure_node["fig_files"]: - continue - if len(figure_node["content"]) != 1: - discarded += 1 - continue - fig_path = figure_node["content"][0] - fig_id = "_".join(fig_path.split("/")) - from_path = os.path.join(self.figures_dir, paper_id, fig_id) - if not os.path.isfile(from_path): - continue - # print(figure_node['content']) - target = figure_node["content"][0] - target = figure_node["id"] + "." + target.split(".")[-1] - if target.endswith("pdf"): - target = target.replace("pdf", "png") - if target.endswith("eps"): - target = target.replace("eps", "png") - type_ = "ref_figure" - elif ref in table_nodes_set: - target = ref - print(table_nodes_set[ref]["label"]) - if table_nodes_set[ref]["content"]: - type_ = "ref_table" - else: - continue - touched_refs.add(ref) - self.edge_metadata.append( - { - "source": text_node["id"], - "target": target, - "type": type_, - } - ) - # for cite in text_node['cites']: - # self.edge_metadata.append({ - # 'source': 'text_'+str(text_node['id']), - # 'target': cite, - # 'type': 'cites', - # }) - if pre_node: - self.edge_metadata.append( - { - "source": pre_node["id"], - "target": text_node["id"], - "type": "adjacent_chunk_edge", - } - ) - self.edge_metadata.append( - { - "source": text_node["id"], - "target": pre_node["id"], - "type": "adjacent_chunk_edge", - } - ) - pre_node = text_node - - for figure_node in temp_figure_nodes: - if figure_node["id"] in touched_refs: - fig_ids = [] - if figure_node["fig_files"]: - for fig_path in figure_node["content"]: - fig_id = "_".join(fig_path.split("/")) - new_fig_id = figure_node["id"] + "." + fig_id.split(".")[-1] - from_path = os.path.join(self.figures_dir, paper_id, fig_id) - to_path = os.path.join(self.output_figure_dir, new_fig_id) - # check if figure file exists - if os.path.isfile(from_path): - shutil.copy(from_path, to_path) - if from_path.endswith("png"): - shutil.copy(from_path, to_path) - fig_ids.append(new_fig_id) - elif from_path.endswith("pdf"): - to_path = to_path.replace("pdf", "png") - new_fig_id = new_fig_id.replace("pdf", "png") - fig_ids.append(new_fig_id) - self.convert_codes.append( - f"convert -density 300 {from_path} {to_path}" - ) - elif from_path.endswith("eps"): - to_path = to_path.replace("eps", "png") - new_fig_id = new_fig_id.replace("eps", "png") - fig_ids.append(new_fig_id) - self.convert_codes.append( - f"convert -density 300 {from_path} {to_path}" - ) - figure_node["content"] = fig_ids - if len(fig_ids) == 1: - self.figure_nodes.append(figure_node) - for table_node in temp_table_nodes: - if table_node["id"] in touched_refs: - self.table_nodes.append(table_node) - - def save_jsonl(self, data: List[Dict], output_path: str, save_mode: str = "w"): - """Save data to JSONL file.""" - with open(output_path, save_mode) as f: - for line in data: - f.write(json.dumps(line) + "\n") - - def save_processed_data(self, save_mode: str = "w"): - """Save processed paper data to JSON file.""" - self.save_jsonl(self.edge_metadata, self.output_metadata, save_mode) - print( - "Total nodes: ", - len(self.paper_nodes) - + len(self.figure_nodes) - + len(self.table_nodes) - + len(self.text_nodes), - ) - print("Total edges: ", len(self.edge_metadata)) - print("Paper nodes: ", len(self.paper_nodes)) - print("Figure nodes: ", len(self.figure_nodes)) - print("Table nodes: ", len(self.table_nodes)) - print("Text nodes: ", len(self.text_nodes)) - self.save_jsonl(self.paper_nodes, self.output_paper_nodes, save_mode) - self.save_jsonl(self.figure_nodes, self.output_figure_nodes, save_mode) - self.save_jsonl(self.table_nodes, self.output_table_nodes, save_mode) - self.save_jsonl(self.text_nodes, self.output_text_nodes, save_mode) - with open("./convert.sh", "w") as f: - for code in self.convert_codes: - f.write(code + "\n") - - def process_all_papers(self): - """Process all papers in the data directory.""" - cnt = 0 - for paper_path in tqdm(self.list_json_files(self.data_dir)): - if "history" in paper_path: - continue - paper_data, paper_id = self.load_paper(paper_path) - if paper_data: - abstract = paper_data["abstract"] - if not abstract: - abstract = "" - paper_node = self.create_paper_node( - paper_id, clean_latex_code(abstract), paper_data["title"] - ) - self.paper_nodes.append(paper_node) - self.paper_id2node[paper_id] = paper_node["id"] - # for key, cite in paper_data['citations'].items(): - # if cite['similar_score'] and (cite['similar_score'] > self.threshold): - # id = cite['short_id'] - # abstract = cite['abstract'] - # title = cite['title'] - # paper_node = self.create_paper_node(id, clean_latex_code(abstract), title) - # self.paper_nodes.append(paper_node) - # self.paper_id2node[id] = paper_node['id'] - - # self.save_processed_data('w') - for paper_path in tqdm(self.list_json_files(self.data_dir)): - if "history" in paper_path: - continue - paper_data, paper_id = self.load_paper(paper_path) - if paper_data: - cnt += 1 - print(paper_id) - self.process_paper(paper_data, paper_id) - citations = {} - for key, cite in paper_data["citations"].items(): - if cite["similar_score"] and ( - cite["similar_score"] > self.threshold - ): - citations[key] = cite - paper_data["citations"] = citations - os.makedirs(os.path.join(self.output_dir, "papers"), exist_ok=True) - # shutil.copy(paper_path, os.path.join(self.output_dir, 'papers', os.path.basename(paper_path))) - with open( - os.path.join( - self.output_dir, "papers", os.path.basename(paper_path) - ), - "w", - ) as f: - json.dump(paper_data, f) - # if cnt % 100 == 0: - # self.save_processed_data('w') - print("Paper count: ", cnt) - self.save_processed_data("w") - print(discarded) - - - def process_papers(self, paper_paths): - cnt = 0 - for paper_path in tqdm(paper_paths): - if "history" in paper_path: - continue - paper_data, paper_id = self.load_paper(paper_path) - if paper_data: - abstract = paper_data["abstract"] - if not abstract: - abstract = "" - paper_node = self.create_paper_node( - paper_id, clean_latex_code(abstract), paper_data["title"] - ) - self.paper_nodes.append(paper_node) - self.paper_id2node[paper_id] = paper_node["id"] - # for key, cite in paper_data['citations'].items(): - # if cite['similar_score'] and (cite['similar_score'] > self.threshold): - # id = cite['short_id'] - # abstract = cite['abstract'] - # title = cite['title'] - # paper_node = self.create_paper_node(id, clean_latex_code(abstract), title) - # self.paper_nodes.append(paper_node) - # self.paper_id2node[id] = paper_node['id'] - - # self.save_processed_data('w') - for paper_path in tqdm(paper_paths): - if "history" in paper_path: - continue - paper_data, paper_id = self.load_paper(paper_path) - if paper_data: - cnt += 1 - print(paper_id) - self.process_paper(paper_data, paper_id) - citations = {} - for key, cite in paper_data["citations"].items(): - if cite["similar_score"] and ( - cite["similar_score"] > self.threshold - ): - citations[key] = cite - paper_data["citations"] = citations - os.makedirs(os.path.join(self.output_dir, "papers"), exist_ok=True) - # shutil.copy(paper_path, os.path.join(self.output_dir, 'papers', os.path.basename(paper_path))) - with open( - os.path.join( - self.output_dir, "papers", os.path.basename(paper_path) - ), - "w", - ) as f: - json.dump(paper_data, f) - - print("Paper count: ", cnt) - self.save_processed_data("w") - print(discarded) \ No newline at end of file diff --git a/research_arcade/arxiv_utils/paper_collector/paper_processor.py b/research_arcade/arxiv_utils/paper_collector/paper_processor.py deleted file mode 100644 index 337a1dc..0000000 --- a/research_arcade/arxiv_utils/paper_collector/paper_processor.py +++ /dev/null @@ -1,188 +0,0 @@ -import glob -import json -import os -import re -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple - -from tqdm import tqdm - - -@dataclass -class Figure: - """Represents a figure in an academic paper.""" - - caption: str - path: str - name: str - label: str - - -class PaperProcessor: - """Handles the processing of academic papers and their figures.""" - - def __init__(self, data_dir: str, figures_dir: str, output_dir: str): - """Initialize with directory paths for data processing. - - Args: - data_dir: Directory containing paper JSON files - figures_dir: Directory containing figure files - output_dir: Directory for processed output - """ - self.data_dir = data_dir - self.figures_dir = figures_dir - self.output_dir = output_dir - os.makedirs(output_dir, exist_ok=True) - - @staticmethod - def list_json_files(directory: str) -> List[str]: - """Get all JSON files in directory.""" - return glob.glob(os.path.join(directory, "*.json")) - - @staticmethod - def clean_filename(filename: str) -> str: - """Remove file extensions and standardize filename.""" - return filename.replace(".pdf", "").replace(".png", "").replace(".jpg", "") - - @staticmethod - def extract_label(latex_label: str) -> Optional[str]: - """Extract clean label from LaTeX label string.""" - if not latex_label: - return None - matches = re.findall(r"\\label\{([^}]+)\}", latex_label) - return PaperProcessor.clean_filename(matches[0]) if matches else None - - def extract_caption(self, latex_caption: str) -> str: - """Extract clean caption from LaTeX caption string.""" - if not latex_caption: - return None - matches = re.findall(r"\\caption\{([^}]+)\}", latex_caption) - return PaperProcessor.clean_filename(matches[0].strip()) if matches else None - - def load_paper(self, file_path: str) -> Tuple[Optional[dict], Dict[str, str]]: - """Load paper JSON and get its figure paths.""" - try: - arxiv_id = os.path.basename(file_path).split(".json")[0] - with open(file_path, "r") as f: - paper_data = json.load(f) - figure_paths = self.get_figure_paths(arxiv_id) - return paper_data, figure_paths - except Exception as e: - print(f"Error loading {file_path}: {e}") - return None, {} - - def get_figure_paths(self, arxiv_id: str) -> Dict[str, str]: - """Get mapping of figure names to their full paths.""" - figure_pattern = os.path.join(self.figures_dir, arxiv_id, "*") - figure_paths = {} - - for path in glob.glob(figure_pattern): - name = self.clean_filename(os.path.basename(path)) - figure_paths[name] = path - - return figure_paths - - def create_figure_objects( - self, figures: List[dict], figure_paths: Dict[str, str] - ) -> List[Figure]: - """Create Figure objects with path, name, and label.""" - figure_objects = [] - - for fig in figures: - caption = self.extract_caption(fig.get("caption")) - label = self.extract_label(fig.get("label")) - if not label or "figure_paths" not in fig: - continue - - for fig_path in fig["figure_paths"]: - name = self.clean_filename(fig_path.replace("/", "_")) - full_path = figure_paths.get(name) - - if full_path: - figure_objects.append( - Figure(caption=caption, path=full_path, name=name, label=label) - ) - print("Latex begin end figure has {}".format(len(figures))) - print("Figure objects has {}".format(len(figure_objects))) - return figure_objects - - def find_figure_references(self, text: str) -> List[str]: - """Find all figure references in text.""" - pattern = r"\\ref\{([^}]+)\}" - return re.findall(pattern, text) - - def process_paper(self, data: dict, figure_paths: Dict[str, str]) -> Optional[Dict]: - """Process paper data and extract chunks with figure references.""" - if not all(key in data for key in ["sections", "figure"]): - return None - - figures = self.create_figure_objects(data["figure"], figure_paths) - label_to_figures = self._create_label_mapping(figures) - - all_chunks = [] - fig_chunks = [] - - for section in data["sections"].values(): - chunks = [ - chunk.strip() - for chunk in section["content"].split("\n\n") - if chunk.strip() - ] - - for chunk in chunks: - all_chunks.append({"content": chunk}) - if "\\ref{" in chunk: - referenced_figures = [] - for label in self.find_figure_references(chunk): - if label in label_to_figures: - referenced_figures.extend( - [ - { - "caption": fig.caption, - "path": fig.path, - "name": fig.name, - "label": fig.label, - } - for fig in label_to_figures[label] - ] - ) - - if referenced_figures: - fig_chunks.append( - {"content": chunk, "figures": referenced_figures} - ) - - print("I find {} chunks with figure references".format(len(fig_chunks))) - return { - "fig_connected_chunks": fig_chunks, - "all_chunks": all_chunks, - } - - @staticmethod - def _create_label_mapping(figures: List[Figure]) -> Dict[str, List[Figure]]: - """Create mapping from labels to figures.""" - label_to_figures = {} - for fig in figures: - if fig.label not in label_to_figures: - label_to_figures[fig.label] = [] - label_to_figures[fig.label].append(fig) - return label_to_figures - - def save_processed_data(self, data: dict, original_path: str): - """Save processed paper data to JSON file.""" - if not data: - return - - filename = os.path.basename(original_path).replace(".json", "-processed.json") - save_path = os.path.join(self.output_dir, filename) - - with open(save_path, "w") as f: - json.dump(data, f, indent=4) - - def process_all_papers(self): - """Process all papers in the data directory.""" - for paper_path in tqdm(self.list_json_files(self.data_dir)): - paper_data, figure_paths = self.load_paper(paper_path) - if paper_data: - processed_data = self.process_paper(paper_data, figure_paths) - self.save_processed_data(processed_data, paper_path) diff --git a/research_arcade/arxiv_utils/paper_collector/utils.py b/research_arcade/arxiv_utils/paper_collector/utils.py deleted file mode 100644 index 6f90f5d..0000000 --- a/research_arcade/arxiv_utils/paper_collector/utils.py +++ /dev/null @@ -1,164 +0,0 @@ -import os -import re -import time -from difflib import SequenceMatcher - -import arxiv -import requests -from beartype.typing import Any, Callable, Dict, List, Optional - -from .error_handler import api_calling_error_exponential_backoff - - -def get_references(arxiv_id: str, max_retries: int = 5) -> List[Dict[str, Any]]: - SEMANTIC_SCHOLAR_API_URL = "https://api.semanticscholar.org/graph/v1/paper/" - url = f"{SEMANTIC_SCHOLAR_API_URL}ARXIV:{arxiv_id}/references" - params = {"limit": 100, "offset": 0, "fields": "title,abstract"} - headers = {"User-Agent": "PaperProcessor/1.0"} - - for attempt in range(max_retries): - response = requests.get(url, params=params, headers=headers) # type: ignore - if response.status_code == 200: - data = response.json() - references = [] - for ref in data.get("data", []): - cited_paper = ref.get("citedPaper", {}) - if cited_paper: - ref_info = { - "title": cited_paper.get("title"), - "abstract": cited_paper.get("abstract"), - } - references.append(ref_info) - return references - else: - wait_time = 2**attempt - print( - f"Error {response.status_code} fetching references for {arxiv_id}. Retrying in {wait_time}s..." - ) - time.sleep(wait_time) # Exponential backoff - print(f"Failed to fetch references for {arxiv_id} after {max_retries} attempts.") - return [] - - -def None_constraint(published: str) -> bool: - return True - - -def year_constraint(start_year: int, end_year: int) -> Callable[[str], bool]: - def constraint(published: str) -> bool: - if published == "True": - return True - return start_year <= published.year <= end_year - - return constraint - - -def fetch_papers_cs( - query: str, max_results_per_call: int = 100 -) -> List[Dict[str, Any]]: - # Get the results from ArXiv API - search = arxiv.Search( - query=query, - max_results=max_results_per_call, - sort_by=arxiv.SortCriterion.Relevance, - sort_order=arxiv.SortOrder.Descending, - ) - client = arxiv.Client() - papers = [] - for paper in client.results(search): - papers.append( - { - "paper": paper, - "title": paper.title, - "authors": [author.name for author in paper.authors], - "published": paper.published, - "updated": paper.updated, - "summary": paper.summary, - "pdf_url": paper.pdf_url, - "short_id": paper.get_short_id(), - "entry_id": paper.entry_id, - } - ) - - return papers - - -def download_latex_source(arxiv_id: str, save_dir: str = "latex_sources"): - # Create directory if it doesn't exist - if not os.path.exists(save_dir): - os.makedirs(save_dir) - print("Downloading source") - # Construct the URL for downloading the LaTeX source - source_url = f"https://export.arxiv.org//e-print/{arxiv_id}" - - try: - # Download the source as a tar file - response = requests.get(source_url, stream=True) - if response.status_code == 200: - file_path = os.path.join(save_dir, f"{arxiv_id}.tar.gz") - with open(file_path, "wb") as file: - for chunk in response.iter_content(chunk_size=8192): - file.write(chunk) - print(f"Downloaded source for {arxiv_id} to {file_path}") - else: - print( - f"Failed to download source for {arxiv_id}, status code: {response.status_code}" - ) - except Exception as e: - print(f"An error occurred: {e}") - - -def similar(str1: str, str2: str) -> float: - return SequenceMatcher(None, str1, str2).ratio() - - -@api_calling_error_exponential_backoff(retries=5, base_wait_time=1) -def search_arxiv_id(arxiv_id: str) -> Optional[Any]: - search = arxiv.Search(id_list=[arxiv_id]) - client = arxiv.Client() - result = list(client.results(search)) - return result - - -@api_calling_error_exponential_backoff(retries=5, base_wait_time=1) -def search_arxiv_query(query: str, max_results: int = 10) -> Optional[Any]: - client = arxiv.Client() - # Get the results from ArXiv API - search = arxiv.Search( - query=query, - max_results=max_results, - sort_by=arxiv.SortCriterion.Relevance, - sort_order=arxiv.SortOrder.Descending, - ) - result = list(client.results(search)) - return result - - -def query_and_match(title: str, author: str, max_results: int = 10): - return None, 0 - title_ = re.sub(r"[:!{}()]", "", title) - title_ = re.sub(r"[-_,\\]", " ", title_) - author_ = re.sub(r"[:!{}(),]", "", author) - author_ = re.sub(r"[-_,\\]", " ", author_) - query = f"ti: {title_}" - papers = search_arxiv_query(query, max_results) - - similarity = 0 - similar_paper = None - for paper in papers: - # print(dir(paper)) - # ['Author', 'Link', 'MissingFieldError', 'authors', 'categories', 'comment', 'doi', 'download_pdf', 'download_source', 'entry_id', 'get_short_id', 'journal_ref', 'links', 'pdf_url', 'primary_category', 'published', 'summary', 'title', 'updated'] - s = similar(paper.title.lower(), title.lower()) - if s > similarity: - similarity = s - similar_paper = paper - if similarity < 0.8: - query = f"au: {author_} AND ti: {title_}" - papers = search_arxiv_query(query, max_results) - for paper in papers: - s = similar(paper.title.lower(), title.lower()) - if s > similarity: - similarity = s - similar_paper = paper - - return similar_paper, similarity diff --git a/research_arcade/arxiv_utils/paper_crawler/crawler_job.py b/research_arcade/arxiv_utils/paper_crawler/crawler_job.py deleted file mode 100644 index 81a737b..0000000 --- a/research_arcade/arxiv_utils/paper_crawler/crawler_job.py +++ /dev/null @@ -1,412 +0,0 @@ -from ..graph_constructor.node_processor import NodeConstructor -from ..multi_input.multi_download import MultiDownload -from ..paper_crawler.task_database import TaskDatabase -from ..graph_constructor.database import Database -from ..paper_collector.paper_graph_processor import PaperGraphProcessor -import pytz -from arxiv import UnexpectedEmptyPageError -import datetime -import arxiv -from ..multi_input.arxiv_crawler_new import download_with_time, extract_arxiv_ids -from pathlib import Path - - -nc = NodeConstructor() -md = MultiDownload() - - # def __init__( - # self, data_dir: str, figures_dir: str, output_dir: str, threshold: float = 0.8 - # ): - -# We first obtain a series of arxiv id of papers published recently - -class CrawlerJob: - - def __init__(self, dest_dir): - - """ - - dest_dir: str - """ - - data_dir_path = f"{dest_dir}/output" - figures_dir_path = f"{dest_dir}/output/images" - output_dir_path = f"{dest_dir}/output/paragraphs" - - self.tdb = TaskDatabase() - self.db = Database() - self.nc = NodeConstructor() - self.md = MultiDownload() - self.pgp = PaperGraphProcessor(data_dir=data_dir_path, figures_dir=figures_dir_path, output_dir=output_dir_path) - self.dest_dir = dest_dir - - print(self.dest_dir) - - def create_task_database(self): - self.tdb.create_paper_task_table() - - def drop_task_database(self): - self.tdb.drop_paper_task_table() - - def crawl_recent_arxiv_paper_new(self, start_date, end_date=None, path=None): - save_path = download_with_time(start_date=start_date, end_date=end_date, save_path=path) - arxiv_ids = extract_arxiv_ids(file_path=save_path) - return arxiv_ids - - def crawl_recent_arxiv_paper(self, year, month, day, max_result): - """ - Crawl arxiv paper ids of papers published after the given date. - Stops when either: - • the paper's published date is older than cutoff, or - • arXiv returns an empty page (no more results). - category: only crawl papers in certain categories listed below - """ - - cutoff = datetime.datetime(year, month, day, tzinfo=pytz.UTC) - search = arxiv.Search( - query = "cat:cs.AI", - sort_by = arxiv.SortCriterion.SubmittedDate, - sort_order = arxiv.SortOrder.Descending, - max_results = max_result - ) - - recent_ids = [] - iterator = search.results() - - try: - for result in iterator: - # stop early if we've gone past the cutoff - if result.published < cutoff: - break - recent_ids.append(result.get_short_id()) - # optional: break if we hit our max - if len(recent_ids) >= max_result: - break - - except UnexpectedEmptyPageError: - # arXiv had no entries on the next page—just stop - pass - - return recent_ids - - def ids_with_major_category(self, arxiv_ids, category): - client = arxiv.Client(page_size=1, num_retries=2) - hits = [] - for arxiv_id in arxiv_ids: - try: - search = arxiv.Search(id_list=[arxiv_id]) - paper = next(client.results(search)) - except StopIteration: - continue - except Exception as e: - print(f"Failed to fetch arXiv entry for {arxiv_id}: {e}") - continue - - for cat in (paper.categories or []): - major = cat.split('.', 1)[0] # works even if there's no dot, e.g., "math-ph" - if major == category: - hits.append(arxiv_id) - break - return hits - - - def initialize_paper_tasks(self, arxiv_ids, category = None): - """ - Initialize a list of paper tasks given their arxiv ids - - arxiv_ids: list[str] - """ - - # First select the paper of the category specified (if so) - if category: - id_category = self.ids_with_major_category(arxiv_ids=arxiv_ids, category=category) - else: - id_category = arxiv_ids - - cleaned_ids = [] - - seen = set() - - for id in id_category or []: - if not id: - continue - - id = id.strip() - - if not id or id in seen: - continue - - seen.add(id) - cleaned_ids.append(id) - added, skipped, failed = [], [], {} - - for id in cleaned_ids: - try: - # fetched = self.tdb.initialize_state(id) - # if fetched: - added.append(id) - print(f"[+] Task for {id} initialized.") - # else: - # skipped.append(id) - # print(f"[!]\tTask for {id} already exists — skipping.") - - except Exception as e: - failed[id] = repr(e) - print(f"[!]\tPaper {id} not added into database: {e}") - - return {"added": added, "skipped": skipped, "failed": failed} - - # added_arxiv_ids = [] - - # for arxiv_id in arxiv_ids: - # try: - # self.tdb.initialize_state(arxiv_id) - # added_arxiv_ids.append(arxiv_id) - # except psycopg2.IntegrityError as e: - # # roll back so that the next iteration can run cleanly - # self.conn.rollback() - - # if e.pgcode == errorcodes.UNIQUE_VIOLATION: - # print(f"[!]\tTask for {arxiv_id} already exists — skipping.") - # else: - # print(f"[!]\tPaper {arxiv_id} not added into database: {e}") - - # except Exception as e: - # # catch any other exception - # self.conn.rollback() - # print(f"[!]\tUnexpected error initializing task for {arxiv_id}: {e}") - - # else: - # print(f"[+] Task for {arxiv_id} initialized.") - # return added_arxiv_ids - - def download_papers(self, arxiv_ids): - """ - Download the paper with specified arxiv id from arxiv and save metadata to JSON - If papers are not of the category specified (as is above) - - If the paper is not of the category specified, skip it. - """ - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - try: - self.md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - self.tdb.set_states(downloaded=True, paper_arxiv_id=arxiv_id) - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - self.tdb.conn.rollback() - - self.tdb.set_states( - paper_arxiv_id=arxiv_id, - downloaded=False - ) - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - def add_existing_paper_graphs(self): - """ - Given current paper graph json files, add them into the database. - """ - - # First find all the json files inside of the corresponding path - # If the file name is not history.json, then we treat it as paper graph fils - # Then extract the id before .json - data_dir_path = f"{self.dest_dir}/output" - - arxiv_ids = [] - for p in Path(data_dir_path).iterdir(): - if p.is_file() and p.suffix == ".json": - arxiv_id = p.stem # e.g., "2508.00223v2" - if arxiv_id != "history": - arxiv_ids.append(arxiv_id) - - processed_paper_ids = [] - - for arxiv_id in arxiv_ids: - try: - # Store the processed data into database afterward - is_processed = nc.process_paper(arxiv_id=arxiv_id, dir_path=f"{self.dest_dir}") - - if is_processed: - processed_paper_ids.append(arxiv_id) - self.tdb.set_states(paper_arxiv_id=arxiv_id, paper_graph=True) - - except Exception as e: - print(f"[Warning] Failed to add paper with arxiv id {arxiv_id} to database: {e}") - continue - - return processed_paper_ids - - def process_not_add(self, arxiv_ids): - """ - Similarly, we can create but not add arxiv_ids - """ - # processed_paper_ids = [] - - try: - self.md.build_paper_graphs( - input=arxiv_ids, - input_type="id", - dest_dir=self.dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - - - def process_paper_graphs(self, arxiv_ids): - """ - Build paper graph using the knowledge debugger - """ - - processed_paper_ids = [] - - # Instead of processing all papers at one, we process each paper separately - # This avoids different papers using the same dir with some arxiv id (possible the id of the very first paper) - - for arxiv_id in arxiv_ids: - - try: - self.md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=self.dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - - # try: - # self.md.build_paper_graphs( - # input=arxiv_ids, - # input_type="id", - # dest_dir=self.dest_dir - # ) - # except Exception as e: - # print(f"[Warning] Failed to process papers: {e}") - - - for arxiv_id in arxiv_ids: - try: - # Store the processed data into database afterward - is_processed = nc.process_paper(arxiv_id=arxiv_id, dir_path=f"{self.dest_dir}") - - if is_processed: - processed_paper_ids.append(arxiv_id) - self.tdb.set_states(paper_arxiv_id=arxiv_id, paper_graph=True) - - except Exception as e: - print(f"[Warning] Failed to add paper with arxiv id {arxiv_id} to database: {e}") - continue - - return processed_paper_ids - - # Build the paragraphs json files later? - # Use the node processor function in knowledge debugger - - - def process_paper_paragraphs(self, arxiv_ids=None): - """ - Convert the existing paper's paper graph json file into a collection of multiple jsons, including paragraph files. - Then store the paragraph information into database - """ - - paper_paths = [] - # We first build paper node - if not arxiv_ids: - self.pgp.process_all_papers() - else: - # We loop through the provided arxiv ids of paper. - for arxiv_id in arxiv_ids: - paper_paths.append(f"{self.dest_dir}/output/{arxiv_id}.json") - print(paper_paths) - self.pgp.process_papers(paper_paths) - - self.nc.process_paragraphs(dir_path=self.dest_dir) - - for arxiv_id in arxiv_ids: - - self.tdb.set_states(paper_arxiv_id=arxiv_id, paragraph=True) - - - def select_proceeded_task(self, task_type, max_results=100000): - - valid_types = { - 'downloaded', - 'paper_graph', - 'paragraph', - 'citation', - 'semantic_scholar', - } - if task_type not in valid_types: - raise ValueError(f"Invalid task_type: {task_type!r}") - - - sql = f""" - SELECT paper_arxiv_id FROM paper_task WHERE {task_type} = TRUE ORDER BY id ASC LIMIT %s - """ - - self.tdb.cur.execute(sql, (max_results,)) - rows = self.tdb.cur.fetchall() - return [row[0] for row in rows] - - def select_unproceeded_task(self, task_type, max_results=100000): - - valid_types = { - 'downloaded', - 'paper_graph', - 'paragraph', - 'citation', - 'semantic_scholar', - } - if task_type not in valid_types: - raise ValueError(f"Invalid task_type: {task_type!r}") - - more_constraint = "" - if task_type == "paragraph": - more_constraint = "AND paper_graph = true " - - - - - sql = f""" - SELECT paper_arxiv_id FROM paper_task WHERE {task_type} = FALSE {more_constraint}ORDER BY id ASC LIMIT %s - """ - - self.tdb.cur.execute(sql, (max_results,)) - rows = self.tdb.cur.fetchall() - return [row[0] for row in rows] - - - - def process_paper_citations(self, arxiv_ids): - """ - For papers with arxiv ids, search in database and see if any paper does not have arxiv id/semantic scholar/... - If so, search paper title online and see if the cited papers have been uploaded - """ - - # Go into database, fetch papers that does not have citations and search paper names on arxiv - citation_added_paper_ids = [] - for arxiv_id in arxiv_ids: - # Go to citation table in the database - - added = self.nc.citation_processor(arxiv_id=arxiv_id) - - if added: - self.tdb.set_states(paper_arxiv_id=arxiv_id, citation=added) - citation_added_paper_ids.append(arxiv_id) - - return citation_added_paper_ids - - def process_paper_authors(self, arxiv_ids): - """ - For papers with arxiv ids, search papers on semantic scholar and fetch the author information. Store them into database - """ - author_added_paper_ids = [] - - for arxiv_id in arxiv_ids: - - added = self.nc.author_processor(arxiv_id) - if added: - self.tdb.set_states(paper_arxiv_id=arxiv_id, semantic_scholar=added) - author_added_paper_ids.append(arxiv_id) - - return author_added_paper_ids - diff --git a/research_arcade/arxiv_utils/paper_crawler/task_database.py b/research_arcade/arxiv_utils/paper_crawler/task_database.py deleted file mode 100644 index 61d7df3..0000000 --- a/research_arcade/arxiv_utils/paper_crawler/task_database.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Since the original crawler_job.py fails to extract the citations and figures (sometimes), here we follow the code in paper_processing_test.py. - -We need to first download a range of files, then store the downloaded files in the database by their arxiv id. - -After that, we take papers from the database and process them, following paper_processing_test.py. - -Below is the database for task management, which faciliates the tracking of paper extraction process -""" - -import psycopg2 - -PASSWORD = "Lcs20031121!" - -class TaskDatabase: - - def __init__(self): - self.conn = psycopg2.connect( - host="localhost", - port="5433", - dbname="postgres", - user="cl195" - ) - self.conn.autocommit = True - self.cur = self.conn.cursor() - - def create_paper_task_table(self): - """ - Records the tasks of papers. - processing elements: - - downloaded (whether the paper has been downloaded from arxiv before) - - paper_graph (including section, figure and citation extraction) - - Semantic Scholar author retrieval (as some latest papers are not yet uploaded to semantic scholar) - - paragraphs - """ - self.cur.execute(""" - CREATE TABLE IF NOT EXISTS paper_task( - id SERIAL PRIMARY KEY, - paper_arxiv_id VARCHAR(100) UNIQUE, - downloaded BOOLEAN, - paper_graph BOOLEAN, - citation BOOLEAN, - semantic_scholar BOOLEAN, - paragraph BOOLEAN - ) - """) - - # def create_paper_search_intervals_table(self): - # # 1) Create enum safely (works on old PG too) - # self.cur.execute(""" - # DO $$ - # BEGIN - # IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'search_status') THEN - # CREATE TYPE search_status AS ENUM ('pending','success','failed'); - # END IF; - # END$$; - # """) - - # # 2) (Optional) install btree_gist; ignore if missing privilege/already installed - # self.cur.execute(""" - # DO $$ - # BEGIN - # BEGIN - # CREATE EXTENSION btree_gist; - # EXCEPTION - # WHEN duplicate_object THEN NULL; - # WHEN insufficient_privilege THEN - # RAISE NOTICE 'btree_gist not installed (no privilege); continuing'; - # END; - # END$$; - # """) - - # # 3) Table + index + exclusion constraint (no trailing comma!) - # self.cur.execute(""" - # CREATE TABLE IF NOT EXISTS paper_search_intervals( - # id BIGSERIAL PRIMARY KEY, - # period DATERANGE NOT NULL, -- [start, end) - # status search_status NOT NULL DEFAULT 'pending', - # CHECK (lower(period) < upper(period)) - # ) - # """) - - # self.cur.execute(""" - # CREATE INDEX IF NOT EXISTS paper_search_intervals_period_gist - # ON paper_search_intervals USING GIST (period) - # """) - - # def drop_paper_search_intervals_table(self): - # self.cur.execute("DROP TABLE IF EXISTS paper_search_intervals") - - # def insert_paper_search_intervals(self, search_key, start_date, end_date, status='pending'): - # """ - # Returns the uncovered sub-intervals within [start_date, end_date) that do not - # intersect existing rows for this search_key. Inserts those intervals (status=...). - # """ - # # 1) compute uncovered - # self.cur.execute(""" - # SELECT * FROM get_uncovered_paper_search_periods(%s, %s::date, %s::date) - # """, (search_key, start_date, end_date)) - # gaps = [row[0] for row in self.cur.fetchall()] # each row is a DATERANGE - - # if not gaps: - # return [] # fully covered - - # # 2) insert each uncovered interval (race-safe with ON CONFLICT on the exclusion constraint) - # inserted = [] - # for g in gaps: - # self.cur.execute(""" - # INSERT INTO paper_search_intervals (search_key, period, status) - # VALUES (%s, %s, %s) - # ON CONFLICT ON CONSTRAINT paper_search_intervals_no_overlap DO NOTHING - # RETURNING id - # """, (search_key, g, status)) - # row = self.cur.fetchone() - # if row: - # inserted.append({'id': row[0], 'period': g}) - - # return inserted - - - - def drop_paper_task_table(self): - self.cur.execute(""" - DROP TABLE IF EXISTS paper_task - """) - - - def set_states(self, paper_arxiv_id, downloaded=None, paper_graph=None, semantic_scholar=None, citation=None, paragraph=None): - - """ - Change the task states of a paper. - - paper_arxiv_id: str - - downloaded: bool - - paper_graph: bool - - semantic_scholar: bool - - paragraph: bool - """ - - fields = [] - params = [] - if downloaded is not None: - fields.append("downloaded = %s") - params.append(downloaded) - if paper_graph is not None: - fields.append("paper_graph = %s") - params.append(paper_graph) - if semantic_scholar is not None: - fields.append("semantic_scholar = %s") - params.append(semantic_scholar) - if citation is not None: - fields.append("citation = %s") - params.append(citation) - if paragraph is not None: - fields.append("paragraph = %s") - params.append(paragraph) - if not fields: - return - params.append(paper_arxiv_id) - - sql = f""" - UPDATE paper_task - SET {','.join(fields)} - where paper_arxiv_id = %s - """ - - self.cur.execute(sql, params) - - def initialize_state(self, paper_arxiv_id): - """ - Add a paper into task db with all task states being false. - """ - sql = """ - INSERT INTO paper_task - (paper_arxiv_id, downloaded, paper_graph, citation, semantic_scholar, paragraph) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (paper_arxiv_id) DO NOTHING - RETURNING id - """ - - params = (paper_arxiv_id, False, False, False, False, False) - self.cur.execute(sql, params) - return self.cur.fetchone() - - \ No newline at end of file diff --git a/research_arcade/arxiv_utils/semantic_author_crawler.py b/research_arcade/arxiv_utils/semantic_author_crawler.py deleted file mode 100644 index e69de29..0000000 diff --git a/research_arcade/arxiv_utils/utils.py b/research_arcade/arxiv_utils/utils.py deleted file mode 100644 index cdb8bd1..0000000 --- a/research_arcade/arxiv_utils/utils.py +++ /dev/null @@ -1,48 +0,0 @@ -import re -from typing import List, Tuple - -def arxiv_id_processor(arxiv_id): - """ - If the arxiv id is in the format a.bvc, we extract the a.b and c - If it is in the format a.b, we only extract the a.b - """ - ARXIV_RE = re.compile(r'^(?P(?:[a-z\-]+\/\d{7}|\d{4}\.\d{4,5}))(?:v(?P\d+))?$', re.IGNORECASE) - - m = ARXIV_RE.match(arxiv_id.strip()) - if not m: - raise ValueError(f"Not a valid arXiv id: {arxiv_id!r}") - base = m.group('id') - v = m.group('v') - return base, (int(v) if v is not None else None) - -def figure_iteration_recursive(figure_json): - - # Create a set of figures along with the - # list represents (path, caption, label) - path_to_info: List[Tuple[str, str, str]] = [] - - # First iterate through parent, then go into the children - - def figure_iteration(figure_json): - nonlocal path_to_info - - if not figure_json: - return - if figure_json['figure_paths']: - path = figure_json['figure_paths'][0] - caption = figure_json['caption'] - label = figure_json['label'] - path_to_info.append((path, caption, label)) - subfigures = figure_json['subfigures'] - - for subfigure in subfigures: - figure_iteration(subfigure) - - figure_iteration(figure_json=figure_json) - return path_to_info -def get_paragraph_num(pid): - pattern = re.compile(r'^text_(\d+)$') - m = pattern.match(pid) - if not m: - raise ValueError(f"Bad paragraph id format: {pid!r}") - return int(m.group(1)) diff --git a/research_arcade/csv_database/__init__.py b/research_arcade/csv_database/__init__.py deleted file mode 100644 index 39cbaea..0000000 --- a/research_arcade/csv_database/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -# research_arcade/csv_database/__init__.py - -# --- OpenReview CSVs --- -from .csv_openreview_arxiv import CSVOpenReviewArxiv -from .csv_openreview_authors import CSVOpenReviewAuthors -from .csv_openreview_papers_authors import CSVOpenReviewPapersAuthors -from .csv_openreview_papers_reviews import CSVOpenReviewPapersReviews -from .csv_openreview_papers_revisions import CSVOpenReviewPapersRevisions -from .csv_openreview_papers import CSVOpenReviewPapers -from .csv_openreview_reviews import CSVOpenReviewReviews -from .csv_openreview_revisions_reviews import CSVOpenReviewRevisionsReviews -from .csv_openreview_revisions import CSVOpenReviewRevisions -from .csv_openreview_paragraphs import CSVOpenReviewParagraphs - -# --- Arxiv CSVs --- -from .csv_arxiv_authors import CSVArxivAuthors -from .csv_arxiv_categories import CSVArxivCategory -from .csv_arxiv_citations import CSVArxivCitation -from .csv_arxiv_figures import CSVArxivFigure -from .csv_arxiv_paper_authors import CSVArxivPaperAuthor -from .csv_arxiv_paper_categories import CSVArxivPaperCategory -from .csv_arxiv_paper_figures import CSVArxivPaperFigure -from .csv_arxiv_paper_tables import CSVArxivPaperTable -from .csv_arxiv_papers import CSVArxivPapers -from .csv_arxiv_paragraph_references import CSVArxivParagraphReference -from .csv_arxiv_paragraphs import CSVArxivParagraphs -from .csv_arxiv_sections import CSVArxivSections -from .csv_arxiv_tables import CSVArxivTable - -__all__ = [ - # --- OpenReview --- - 'CSVOpenReviewArxiv', - 'CSVOpenReviewAuthors', - 'CSVOpenReviewPapersAuthors', - 'CSVOpenReviewPapersReviews', - 'CSVOpenReviewPapersRevisions', - 'CSVOpenReviewPapers', - 'CSVOpenReviewReviews', - 'CSVOpenReviewRevisionsReviews', - 'CSVOpenReviewRevisions', - 'CSVOpenReviewParagraphs', - - # --- Arxiv --- - 'CSVArxivAuthors', - 'CSVArxivCategory', - 'CSVArxivCitation', - 'CSVArxivFigure', - 'CSVArxivPaperAuthor', - 'CSVArxivPaperCategory', - 'CSVArxivPaperFigure', - 'CSVArxivPaperTable', - 'CSVArxivPapers', - 'CSVArxivParagraphReference', - 'CSVArxivParagraphs', - 'CSVArxivSections', - 'CSVArxivTable' -] diff --git a/research_arcade/csv_database/csv_arxiv_authors.py b/research_arcade/csv_database/csv_arxiv_authors.py deleted file mode 100644 index 8c11e41..0000000 --- a/research_arcade/csv_database/csv_arxiv_authors.py +++ /dev/null @@ -1,327 +0,0 @@ -import pandas as pd -import os -from typing import Optional -from pathlib import Path -import sys -from semanticscholar import SemanticScholar - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - -class CSVArxivAuthors: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_authors.csv" - self.csv_path = csv_path - # Set up the target directory - # Automatically create the csv path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_authors_table() - - def create_authors_table(self): - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'semantic_scholar_id', 'name', 'homepage' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - def insert_author(self, semantic_scholar_id, name, homepage=None): - """Insert an author. Returns the generated author id.""" - df = self._load_data() - - if semantic_scholar_id in df['semantic_scholar_id'].values: - return None - - new_id = df['id'].max() + 1 if not df.empty else 1 - - new_row = pd.DataFrame([{ - 'id': new_id, - 'semantic_scholar_id': semantic_scholar_id, - 'name': name, - 'homepage': homepage - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_author_by_id(self, id): - """Delete an author by its id. Returns True if deleted, False if not found.""" - df = self._load_data() - - if id not in df['id'].values: - return False - - df = df[df['id'] != id] - self._save_data(df) - return True - - def update_author(self, semantic_scholar_id=None, name=None, homepage=None): - """Update an author by id. Returns True if updated, False if not found.""" - df = self._load_data() - - if semantic_scholar_id not in df['semantic_scholar_id'].values: - return False - - mask = df['semantic_scholar_id'] == semantic_scholar_id - - if semantic_scholar_id is not None: - df.loc[mask, 'semantic_scholar_id'] = semantic_scholar_id - if name is not None: - df.loc[mask, 'name'] = name - if homepage is not None: - df.loc[mask, 'homepage'] = homepage - - self._save_data(df) - return True - - def get_author_by_id(self, semantic_scholar_id: int) -> Optional[pd.DataFrame]: - """Get an author by its id. Returns a DataFrame with the author or None if not found.""" - df = self._load_data() - - if df.empty or semantic_scholar_id not in df['id'].values: - return None - - author = df[df['semantic_scholar_id'] == semantic_scholar_id] - return author - - def check_author_exists(self, semantic_scholar_id: int) -> bool: - """Check if an author exists by its id.""" - df = self._load_data() - - if df.empty: - return False - - return semantic_scholar_id in df['semantic_scholar_id'].values - - def construct_author_table_from_csv(self, csv_file: str): - """ - Construct the authors table from an external CSV file. - Assumes the CSV has compatible columns or can be mapped to the authors schema. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['semantic_scholar_id', 'name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'homepage' not in external_df.columns: - external_df['homepage'] = None - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out authors that already exist (based on semantic_scholar_id) - if not current_df.empty: - existing_ids = set(current_df['semantic_scholar_id'].values) - external_df = external_df[~external_df['semantic_scholar_id'].isin(existing_ids)] - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} authors from {csv_file}") - return True - - def get_all_authors(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def construct_authors_table_from_api(self, arxiv_ids, dest_dir): - """ - Given arxiv ids, find the semantic scholar ids and pages of the authors - """ - # Search for authors online - sch = SemanticScholar() - for arxiv_id in arxiv_ids: - base_arxiv_id, version = arxiv_id_processor(arxiv_id=arxiv_id) - print(f"base_arxiv_id: {base_arxiv_id}") - try: - paper_sch = sch.get_paper(f"ARXIV:{base_arxiv_id}") - authors = paper_sch.authors - for author in authors: - semantic_scholar_id = author.authorId - author_r = sch.get_author(semantic_scholar_id) - name = author_r.name - url = author_r.url - - self.insert_author(semantic_scholar_id=semantic_scholar_id, name=name, homepage=url) - except Exception as e: - print(f"Paper with arxiv id {base_arxiv_id} not found on semantic scholar: {e}") - # return False - continue - - def construct_table_from_csv(self, csv_file): - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['semantic_scholar_id', 'name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'homepage' not in external_df.columns: - external_df['homepage'] = None - - # Generate IDs for new authors - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out authors that already exist (based on semantic_scholar_id) - if not current_df.empty: - existing_ids = set(current_df['semantic_scholar_id'].values) - external_df = external_df[~external_df['semantic_scholar_id'].isin(existing_ids)] - - if external_df.empty: - print("No new authors to import (all authors already exist)") - return True - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} authors from {csv_file}") - return True - - except Exception as e: - print(f"Error importing authors from CSV: {e}") - return False - - def construct_table_from_json(self, json_file): - """ - Construct the authors table from an external JSON file. - - Args: - json_file: Path to the JSON file containing author data - - Expected JSON format (list of objects): - [ - { - "semantic_scholar_id": "123456", - "name": "John Doe", - "homepage": "https://example.com" // optional - }, - ... - ] - - Or (single object with authors array): - { - "authors": [ - { - "semantic_scholar_id": "123456", - "name": "John Doe", - "homepage": "https://example.com" - }, - ... - ] - } - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - import json - - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - # If it's a dict, look for an 'authors' key - if 'authors' in json_data: - authors_list = json_data['authors'] - else: - # Treat the dict as a single author record - authors_list = [json_data] - elif isinstance(json_data, list): - authors_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not authors_list: - print("Error: No author data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(authors_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['semantic_scholar_id', 'name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'homepage' not in external_df.columns: - external_df['homepage'] = None - - # Generate IDs for new authors - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out authors that already exist (based on semantic_scholar_id) - if not current_df.empty: - existing_ids = set(current_df['semantic_scholar_id'].values) - external_df = external_df[~external_df['semantic_scholar_id'].isin(existing_ids)] - - if external_df.empty: - print("No new authors to import (all authors already exist)") - return True - - # Ensure correct column order and data types - external_df = external_df[['id', 'semantic_scholar_id', 'name', 'homepage']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} authors from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing authors from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/csv_database/csv_arxiv_categories.py b/research_arcade/csv_database/csv_arxiv_categories.py deleted file mode 100644 index 5292026..0000000 --- a/research_arcade/csv_database/csv_arxiv_categories.py +++ /dev/null @@ -1,295 +0,0 @@ -import pandas as pd -import os -from typing import Optional -from pathlib import Path -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor - - -class CSVArxivCategory: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_categories.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_categories_table() - # self.arxiv_crawler = ArxivCrawler() - - - def create_categories_table(self): - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'name', 'description' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - - def insert_category(self, name, description=None): - df = self._load_data() - - if name in df['name'].values: - return None - - new_id = df['id'].max() + 1 if not df.empty else 1 - - new_row = pd.DataFrame([{ - 'id': new_id, - 'name': name, - 'description': description - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_category_by_id(self, id): - df = self._load_data() - - if id not in df['id'].values: - return False - - df = df[df['id'] != id] - self._save_data(df) - return True - - - def update_category(self, id, name, description=None): - df = self._load_data() - - if id not in df['id'].values: - return False - - mask = df['id'] == id - - if name is not None: - df.loc[mask, 'name'] = name - if description is not None: - df.loc[mask, 'description'] = description - - self._save_data(df) - return True - - def get_category_by_id(self, id: int) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty or id not in df['id'].values: - return None - - category = df[df['id'] == id] - return category - - def check_category_exists(self, id: int) -> bool: - df = self._load_data() - - if df.empty: - return False - - return id in df['id'].values - - - - def construct_category_table_from_csv(self, csv_file: str): - """ - Construct the categories table from an external CSV file. - Assumes the CSV has compatible columns or can be mapped to the categories schema. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'description' not in external_df.columns: - external_df['description'] = None - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - if not current_df.empty: - existing_names = set(current_df['name'].values) - external_df = external_df[~external_df['name'].isin(existing_names)] - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} categories from {csv_file}") - return True - - def get_all_categories(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def construct_category_table_from_api(self, arxiv_ids, dest_dir): - - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - try: - metadata_path = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - with open(metadata_path, 'r') as f: - metadata = json.load(f) - print(metadata) - - required_fields = ['categories'] - if not all(field in metadata for field in required_fields): - raise ValueError(f"Missing category for {arxiv_id}") - - categories = metadata['categories'] - for category in categories: - self.insert_category(name=category) - except Exception as e: - print(e) - print(f"Paper {arxiv_id} does not have category found") - - - - - def construct_table_from_csv(self, csv_file): - - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - if 'description' not in external_df.columns: - external_df['description'] = None - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - if not current_df.empty: - existing_names = set(current_df['name'].values) - external_df = external_df[~external_df['name'].isin(existing_names)] - - if external_df.empty: - print("No new categories to import (all categories already exist)") - return True - - external_df = external_df[['id', 'name', 'description']] - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} categories from {csv_file}") - return True - - except Exception as e: - print(f"Error importing categories from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - if isinstance(json_data, dict): - if 'categories' in json_data: - categories_list = json_data['categories'] - else: - categories_list = [json_data] - elif isinstance(json_data, list): - categories_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not categories_list: - print("Error: No category data found in JSON file") - return False - - external_df = pd.DataFrame(categories_list) - current_df = self._load_data() - - required_cols = ['name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - if 'description' not in external_df.columns: - external_df['description'] = None - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - if not current_df.empty: - existing_names = set(current_df['name'].values) - external_df = external_df[~external_df['name'].isin(existing_names)] - - if external_df.empty: - print("No new categories to import (all categories already exist)") - return True - - - external_df = external_df[['id', 'name', 'description']] - - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} categories from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing categories from JSON: {e}") - return False diff --git a/research_arcade/csv_database/csv_arxiv_citations.py b/research_arcade/csv_database/csv_arxiv_citations.py deleted file mode 100644 index 0fd817c..0000000 --- a/research_arcade/csv_database/csv_arxiv_citations.py +++ /dev/null @@ -1,350 +0,0 @@ -import pandas as pd -import os -from pathlib import Path -from typing import Optional -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor - - -class CSVArxivCitation: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_citations.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_citations_table() - - def create_citations_table(self): - df = pd.DataFrame(columns=[ - 'id', 'citing_arxiv_id', 'cited_arxiv_id', - 'bib_title', 'bib_key', - 'citing_sections', 'citing_paragraphs' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created citations CSV at {self.csv_path}") - - def _load_data(self): - return pd.read_csv(self.csv_path) if os.path.exists(self.csv_path) else pd.DataFrame() - def _save_data(self, df): - df.to_csv(self.csv_path, index=False) - - def insert_citation(self, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, citing_sections=None): - if citing_arxiv_id == cited_arxiv_id: - return False - df = self._load_data() - conflict = df[ - (df['citing_arxiv_id'] == citing_arxiv_id) & - (df['cited_arxiv_id'] == cited_arxiv_id) - ] - if not conflict.empty: - return False - new_id = df['id'].max() + 1 if not df.empty else 1 - citing_sections_str = json.dumps(citing_sections) if citing_sections else '[]' - new_row = pd.DataFrame([{ - 'id': new_id, 'citing_arxiv_id': citing_arxiv_id, - 'cited_arxiv_id': cited_arxiv_id, 'bib_title': bib_title, - 'bib_key': bib_key, - 'citing_sections': citing_sections_str, 'citing_paragraphs': '[]' - }]) - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return True - - def construct_tables_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - for citation in file_json['citations'].values(): - # print(f"Citation: {citation}") - cited_arxiv_id = citation.get('arxiv_id') - bib_key = citation.get('bib_key') - bib_title = citation.get('bib_title') - bib_author = citation.get('bib_author ') - contexts = citation.get('context') - citing_sections = set() - for context in contexts: - citing_section = context['section'] - citing_sections.add(citing_section) - - self.insert_citation(citing_arxiv_id=arxiv_id, cited_arxiv_id=cited_arxiv_id, citing_sections=list(citing_section), bib_title=bib_title, bib_key=bib_key, author_cited_paper=bib_author) - - except FileNotFoundError: - print(f"Error: The file '{file_json}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{file_json}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue - - - def delete_citation_by_id(self, citing_paper_id, cited_paper_id): - - df = self._load_data() - - if df.empty: - return False - - mask = (df['citing_arxiv_id'] == citing_paper_id) & (df['cited_arxiv_id'] == cited_paper_id) - - if not df[mask].empty: - - df = df[~mask] - self._save_data(df) - print(f"Deleted citation: {citing_paper_id} -> {cited_paper_id}") - return True - else: - print(f"Citation not found: {citing_paper_id} -> {cited_paper_id}") - return False - - def get_all_citations(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - - def get_citing_neighboring_cited(self, citing_paper_id): - - df = self._load_data() - - if df.empty: - return None - - citing_citations = df[df['citing_arxiv_id'] == citing_paper_id].copy() - - if citing_citations.empty: - return None - - return citing_citations - - def get_cited_neighboring_citing(self, cited_paper_id): - df = self._load_data() - - if df.empty: - return None - - cited_by = df[df['cited_arxiv_id'] == cited_paper_id].copy() - - if cited_by.empty: - return None - - return cited_by - - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the citations table from an external CSV file. - - Args: - csv_file: Path to the CSV file containing citation data - - Expected CSV format: - - Required columns: citing_arxiv_id, cited_arxiv_id, bib_title, bib_key - - Optional columns: citing_sections, citing_paragraphs - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['citing_arxiv_id', 'cited_arxiv_id', 'bib_title', 'bib_key'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'citing_sections' not in external_df.columns: - external_df['citing_sections'] = '[]' - if 'citing_paragraphs' not in external_df.columns: - external_df['citing_paragraphs'] = '[]' - - # Generate IDs for new citations - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out citations that already exist (based on citing and cited papers) - if not current_df.empty: - existing_pairs = set(zip(current_df['citing_arxiv_id'], current_df['cited_arxiv_id'])) - external_df['_pair'] = list(zip(external_df['citing_arxiv_id'], external_df['cited_arxiv_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new citations to import (all citations already exist)") - return True - - # Ensure correct column order - external_df = external_df[['id', 'citing_arxiv_id', 'cited_arxiv_id', 'bib_title', 'bib_key', 'citing_sections', 'citing_paragraphs']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} citations from {csv_file}") - return True - - except Exception as e: - print(f"Error importing citations from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the citations table from an external JSON file. - - Args: - json_file: Path to the JSON file containing citation data - - Expected JSON format: - [ - { - "citing_arxiv_id": "1706.03762v7", - "cited_arxiv_id": "1409.0473v7", - "bib_title": "Neural Machine Translation", - "bib_key": "bahdanau2014neural", - "citing_sections": ["introduction", "related_work"], - "citing_paragraphs": [] - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'citations' in json_data: - citations_list = json_data['citations'] - else: - citations_list = [json_data] - elif isinstance(json_data, list): - citations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not citations_list: - print("Error: No citation data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(citations_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['citing_arxiv_id', 'cited_arxiv_id', 'bib_title', 'bib_key'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Handle optional columns - if 'citing_sections' not in external_df.columns: - external_df['citing_sections'] = '[]' - else: - # Convert lists to JSON strings - external_df['citing_sections'] = external_df['citing_sections'].apply( - lambda x: json.dumps(x) if isinstance(x, list) else x - ) - - if 'citing_paragraphs' not in external_df.columns: - external_df['citing_paragraphs'] = '[]' - else: - external_df['citing_paragraphs'] = external_df['citing_paragraphs'].apply( - lambda x: json.dumps(x) if isinstance(x, list) else x - ) - - # Generate IDs for new citations - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out citations that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['citing_arxiv_id'], current_df['cited_arxiv_id'])) - external_df['_pair'] = list(zip(external_df['citing_arxiv_id'], external_df['cited_arxiv_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new citations to import (all citations already exist)") - return True - - # Ensure correct column order - external_df = external_df[['id', 'citing_arxiv_id', 'cited_arxiv_id', 'bib_title', 'bib_key', 'citing_sections', 'citing_paragraphs']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} citations from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing citations from JSON: {e}") - return False - - diff --git a/research_arcade/csv_database/csv_arxiv_figures.py b/research_arcade/csv_database/csv_arxiv_figures.py deleted file mode 100644 index a9716e0..0000000 --- a/research_arcade/csv_database/csv_arxiv_figures.py +++ /dev/null @@ -1,364 +0,0 @@ -import pandas as pd -import os -from typing import Optional -from pathlib import Path -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor, figure_iteration_recursive -class CSVArxivFigure: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_figures.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_figures_table() - # self.arxiv_crawler = ArxivCrawler() - - def create_figures_table(self): - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'paper_arxiv_id', 'path', 'caption', 'label', 'name' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - - def insert_figure(self, paper_arxiv_id, path, caption=None, label=None, name=None): - df = self._load_data() - - if name in df['name'].values: - return None - - new_id = df['id'].max() + 1 if not df.empty else 1 - - new_row = pd.DataFrame([{ - 'id': new_id, - 'paper_arxiv_id': paper_arxiv_id, - 'path': path, - 'caption': caption, - 'label': label, - 'name': name - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_figure_by_id(self, id): - df = self._load_data() - - if id not in df['id'].values: - return False - - df = df[df['id'] != id] - self._save_data(df) - return True - - - def update_figure(self, id, paper_arxiv_id, path, caption=None, label=None, name=None): - df = self._load_data() - - if id not in df['id'].values: - return False - - mask = df['id'] == id - - if paper_arxiv_id is not None: - df.loc[mask, 'paper_arxiv_id'] = paper_arxiv_id - if path is not None: - df.loc[mask, 'path'] = path - if caption is not None: - df.loc[mask, 'caption'] = caption - if label is not None: - df.loc[mask, 'label'] = label - if name is not None: - df.loc[mask, 'name'] = name - - self._save_data(df) - return True - - def get_figure_by_id(self, id: int) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty or id not in df['id'].values: - return None - - figure = df[df['id'] == id] - return figure - - def check_figure_exists(self, id: int) -> bool: - df = self._load_data() - - if df.empty: - return False - - return id in df['id'].values - - def construct_figure_table_from_csv(self, csv_file: str): - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'path','caption','label', 'name'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} figures from {csv_file}") - return True - - - def get_all_figures(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - - def construct_figures_table_from_api(self, arxiv_ids, dest_dir): - md = MultiDownload() - - # Check if papers already exists in the directory - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - figure_jsons = file_json['figure'] - for figure_json in figure_jsons: - - figures = figure_iteration_recursive(figure_json=figure_json) - for figure in figures: - path, caption, label = figure - self.insert_figure(paper_arxiv_id=arxiv_id, path=path, caption=caption,label=label) - - except FileNotFoundError: - print(f"Error: The file with path '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from path '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue - - - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the figures table from an external CSV file. - - Args: - csv_file: Path to the CSV file containing figure data - - Expected CSV format: - - Required columns: paper_arxiv_id, path - - Optional columns: caption, label, name - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'path'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['caption', 'label', 'name']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new figures - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out figures that already exist (based on name if provided) - if not current_df.empty and 'name' in external_df.columns: - existing_names = set(current_df['name'].dropna().values) - # Only filter if name is not null - mask = external_df['name'].notna() & external_df['name'].isin(existing_names) - external_df = external_df[~mask] - - if external_df.empty: - print("No new figures to import") - return True - - # Ensure correct column order - external_df = external_df[['id', 'paper_arxiv_id', 'path', 'caption', 'label', 'name']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} figures from {csv_file}") - return True - - except Exception as e: - print(f"Error importing figures from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the figures table from an external JSON file. - - Args: - json_file: Path to the JSON file containing figure data - - Expected JSON format: - [ - { - "paper_arxiv_id": "1706.03762v7", - "path": "/path/to/figure1.png", - "caption": "Architecture diagram", - "label": "fig:architecture", - "name": "figure1" - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'figures' in json_data: - figures_list = json_data['figures'] - else: - figures_list = [json_data] - elif isinstance(json_data, list): - figures_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not figures_list: - print("Error: No figure data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(figures_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paper_arxiv_id', 'path'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['caption', 'label', 'name']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new figures - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out figures that already exist (based on name if provided) - if not current_df.empty and 'name' in external_df.columns: - existing_names = set(current_df['name'].dropna().values) - mask = external_df['name'].notna() & external_df['name'].isin(existing_names) - external_df = external_df[~mask] - - if external_df.empty: - print("No new figures to import") - return True - - # Ensure correct column order - external_df = external_df[['id', 'paper_arxiv_id', 'path', 'caption', 'label', 'name']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} figures from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing figures from JSON: {e}") - return False - diff --git a/research_arcade/csv_database/csv_arxiv_paper_authors.py b/research_arcade/csv_database/csv_arxiv_paper_authors.py deleted file mode 100644 index 0ab3368..0000000 --- a/research_arcade/csv_database/csv_arxiv_paper_authors.py +++ /dev/null @@ -1,255 +0,0 @@ -import pandas as pd -import os -from pathlib import Path -from typing import Optional -import json - - -class CSVArxivPaperAuthor: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_paper_authors.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_paper_authors_table() - - def create_paper_authors_table(self): - df = pd.DataFrame(columns=['paper_arxiv_id', 'author_id', 'author_sequence']) - df.to_csv(self.csv_path, index=False) - print(f"Created paper_authors CSV at {self.csv_path}") - - def _load_data(self): - return pd.read_csv(self.csv_path) if os.path.exists(self.csv_path) else pd.DataFrame() - - def _save_data(self, df): - df.to_csv(self.csv_path, index=False) - - def insert_paper_author(self, paper_arxiv_id, author_id, author_sequence): - df = self._load_data() - conflict = df[ - (df['paper_arxiv_id'] == paper_arxiv_id) & - (df['author_id'] == author_id) - ] - if not conflict.empty: - return False - new_row = pd.DataFrame([{ - 'paper_arxiv_id': paper_arxiv_id, - 'author_id': author_id, - 'author_sequence': author_sequence - }]) - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return True - - - def get_all_paper_authors(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def get_paper_neighboring_authors(self, paper_arxiv_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - result = df[df['paper_arxiv_id'] == paper_arxiv_id].copy() - - if result.empty: - return None - - result = result.sort_values('author_sequence', ascending=True) - - return result.reset_index(drop=True) - - def get_author_neighboring_papers(self, author_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - result = df[df['author_id'] == author_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - def delete_paper_author_by_id(self, paper_arxiv_id: str, author_id: str) -> bool: - df = self._load_data() - - if df.empty: - return False - - mask = (df['paper_arxiv_id'] == paper_arxiv_id) & (df['author_id'] == author_id) - - if not mask.any(): - return False - - df = df[~mask] - self._save_data(df) - - return True - - - def delete_paper_author_by_paper_id(self, paper_arxiv_id: str) -> int: - df = self._load_data() - - if df.empty: - return 0 - - mask = df['paper_arxiv_id'] == paper_arxiv_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - def delete_paper_author_by_author_id(self, author_id: str) -> int: - df = self._load_data() - - if df.empty: - return 0 - - mask = df['author_id'] == author_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-author relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, author_id, author_sequence - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'author_id', 'author_sequence'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['author_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['author_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-author relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'author_id', 'author_sequence']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-author relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-author relationships from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_authors' in json_data: - relations_list = json_data['paper_authors'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-author data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(relations_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paper_arxiv_id', 'author_id', 'author_sequence'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['author_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['author_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-author relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'author_id', 'author_sequence']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-author relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-author relationships from JSON: {e}") - return False - - diff --git a/research_arcade/csv_database/csv_arxiv_paper_categories.py b/research_arcade/csv_database/csv_arxiv_paper_categories.py deleted file mode 100644 index f888ec0..0000000 --- a/research_arcade/csv_database/csv_arxiv_paper_categories.py +++ /dev/null @@ -1,275 +0,0 @@ -import pandas as pd -import os -from pathlib import Path -from typing import Optional -import json - - -class CSVArxivPaperCategory: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_paper_category.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_paper_category_table() - - def create_paper_category_table(self): - df = pd.DataFrame(columns=['paper_arxiv_id', 'category_id']) - df.to_csv(self.csv_path, index=False) - print(f"Created paper_category CSV at {self.csv_path}") - - def _load_data(self): - return pd.read_csv(self.csv_path) if os.path.exists(self.csv_path) else pd.DataFrame() - def _save_data(self, df): - df.to_csv(self.csv_path, index=False) - - def insert_paper_category(self, paper_arxiv_id, category_id): - df = self._load_data() - conflict = df[ - (df['paper_arxiv_id'] == paper_arxiv_id) & - (df['category_id'] == category_id) - ] - if not conflict.empty: - return False - new_row = pd.DataFrame([{ - 'paper_arxiv_id': paper_arxiv_id, - 'category_id': category_id - }]) - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return True - - def get_all_paper_categories(self): - - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - - def get_paper_neighboring_categories(self, paper_arxiv_id: str) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['paper_arxiv_id'] == paper_arxiv_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def get_category_neighboring_papers(self, category_id: str) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['category_id'] == category_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def delete_paper_category_by_id(self, paper_arxiv_id: str, category_id: str) -> bool: - - df = self._load_data() - - if df.empty: - return False - - mask = (df['paper_arxiv_id'] == paper_arxiv_id) & (df['category_id'] == category_id) - - if not mask.any(): - return False - - df = df[~mask] - self._save_data(df) - - return True - - - def delete_paper_category_by_paper_id(self, paper_arxiv_id: str) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['paper_arxiv_id'] == paper_arxiv_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - def delete_paper_category_by_category_id(self, category_id: str) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['category_id'] == category_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-category relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, category_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'category_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['category_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['category_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-category relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'category_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-category relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-category relationships from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the paper-category relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - {"paper_arxiv_id": "1706.03762v7", "category_id": "cs.AI"}, - {"paper_arxiv_id": "1706.03762v7", "category_id": "cs.LG"}, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_categories' in json_data: - relations_list = json_data['paper_categories'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-category data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(relations_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paper_arxiv_id', 'category_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['category_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['category_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-category relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'category_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-category relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-category relationships from JSON: {e}") - return False - - diff --git a/research_arcade/csv_database/csv_arxiv_paper_figures.py b/research_arcade/csv_database/csv_arxiv_paper_figures.py deleted file mode 100644 index afd6785..0000000 --- a/research_arcade/csv_database/csv_arxiv_paper_figures.py +++ /dev/null @@ -1,273 +0,0 @@ -import pandas as pd -import os -from pathlib import Path -from typing import Optional -import json - -class CSVArxivPaperFigure: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_paper_figures.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_paper_figures_table() - - def create_paper_figures_table(self): - df = pd.DataFrame(columns=['paper_arxiv_id', 'figure_id']) - df.to_csv(self.csv_path, index=False) - print(f"Created paper_figures CSV at {self.csv_path}") - - def _load_data(self): - return pd.read_csv(self.csv_path) if os.path.exists(self.csv_path) else pd.DataFrame() - def _save_data(self, df): - df.to_csv(self.csv_path, index=False) - - def insert_paper_figure(self, paper_arxiv_id, figure_id): - df = self._load_data() - conflict = df[ - (df['paper_arxiv_id'] == paper_arxiv_id) & - (df['figure_id'] == figure_id) - ] - if not conflict.empty: - return False - new_row = pd.DataFrame([{ - 'paper_arxiv_id': paper_arxiv_id, - 'figure_id': figure_id - }]) - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return True - - def get_all_paper_figures(self): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def get_paper_neighboring_figures(self, paper_arxiv_id: str) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['paper_arxiv_id'] == paper_arxiv_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def get_figure_neighboring_papers(self, figure_id: int) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['figure_id'] == figure_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def delete_paper_figure_by_id(self, paper_arxiv_id: str, figure_id: int) -> bool: - - df = self._load_data() - - if df.empty: - return False - - mask = (df['paper_arxiv_id'] == paper_arxiv_id) & (df['figure_id'] == figure_id) - - if not mask.any(): - return False - - df = df[~mask] - self._save_data(df) - - return True - - - def delete_paper_figure_by_paper_id(self, paper_arxiv_id: str) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['paper_arxiv_id'] == paper_arxiv_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - def delete_paper_figure_by_figure_id(self, figure_id: int) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['figure_id'] == figure_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-figure relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, figure_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'figure_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['figure_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['figure_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-figure relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'figure_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-figure relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-figure relationships from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the paper-figure relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - {"paper_arxiv_id": "1706.03762v7", "figure_id": 1}, - {"paper_arxiv_id": "1706.03762v7", "figure_id": 2}, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_figures' in json_data: - relations_list = json_data['paper_figures'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-figure data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(relations_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paper_arxiv_id', 'figure_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['figure_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['figure_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-figure relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'figure_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-figure relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-figure relationships from JSON: {e}") - return False - - diff --git a/research_arcade/csv_database/csv_arxiv_paper_tables.py b/research_arcade/csv_database/csv_arxiv_paper_tables.py deleted file mode 100644 index 7f5de6b..0000000 --- a/research_arcade/csv_database/csv_arxiv_paper_tables.py +++ /dev/null @@ -1,274 +0,0 @@ -import pandas as pd -import os -from pathlib import Path -from typing import Optional -import json - -class CSVArxivPaperTable: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_paper_tables.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_paper_tables_table() - - def create_paper_tables_table(self): - df = pd.DataFrame(columns=['paper_arxiv_id', 'table_id']) - df.to_csv(self.csv_path, index=False) - print(f"Created paper_tables CSV at {self.csv_path}") - - def _load_data(self): - return pd.read_csv(self.csv_path) if os.path.exists(self.csv_path) else pd.DataFrame() - def _save_data(self, df): - df.to_csv(self.csv_path, index=False) - - def insert_paper_table(self, paper_arxiv_id, table_id): - df = self._load_data() - conflict = df[ - (df['paper_arxiv_id'] == paper_arxiv_id) & - (df['table_id'] == table_id) - ] - if not conflict.empty: - return False - new_row = pd.DataFrame([{ - 'paper_arxiv_id': paper_arxiv_id, - 'table_id': table_id - }]) - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return True - - def get_all_paper_tables(self): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - - def get_paper_neighboring_tables(self, paper_arxiv_id: str) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - # Filter for the specific paper - result = df[df['paper_arxiv_id'] == paper_arxiv_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def get_table_neighboring_papers(self, table_id: int) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['table_id'] == table_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def delete_paper_table_by_id(self, paper_arxiv_id: str, table_id: int) -> bool: - - df = self._load_data() - - if df.empty: - return False - - mask = (df['paper_arxiv_id'] == paper_arxiv_id) & (df['table_id'] == table_id) - - if not mask.any(): - return False - - df = df[~mask] - self._save_data(df) - - return True - - - def delete_paper_table_by_paper_id(self, paper_arxiv_id: str) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['paper_arxiv_id'] == paper_arxiv_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - def delete_paper_table_by_table_id(self, table_id: int) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['table_id'] == table_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-table relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, table_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'table_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['table_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['table_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-table relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'table_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-table relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-table relationships from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the paper-table relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - {"paper_arxiv_id": "1706.03762v7", "table_id": 1}, - {"paper_arxiv_id": "1706.03762v7", "table_id": 2}, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_tables' in json_data: - relations_list = json_data['paper_tables'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-table data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(relations_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paper_arxiv_id', 'table_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Filter out relationships that already exist - if not current_df.empty: - existing_pairs = set(zip(current_df['paper_arxiv_id'], current_df['table_id'])) - external_df['_pair'] = list(zip(external_df['paper_arxiv_id'], external_df['table_id'])) - external_df = external_df[~external_df['_pair'].isin(existing_pairs)] - external_df = external_df.drop(columns=['_pair']) - - if external_df.empty: - print("No new paper-table relationships to import") - return True - - # Ensure correct column order - external_df = external_df[['paper_arxiv_id', 'table_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paper-table relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-table relationships from JSON: {e}") - return False - - diff --git a/research_arcade/csv_database/csv_arxiv_papers.py b/research_arcade/csv_database/csv_arxiv_papers.py deleted file mode 100644 index 2cce6bc..0000000 --- a/research_arcade/csv_database/csv_arxiv_papers.py +++ /dev/null @@ -1,428 +0,0 @@ -""" CSV version of the dataset """ - -import pandas as pd -import os -from typing import Optional -from pathlib import Path -import json -import sys -# from ..data import * -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - -# TODO: refactor the original ArxivCrawler into the ArxivCrawler file - -class CSVArxivPapers: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_papers.csv" - self.csv_path = csv_path - # Set up the target directory - # Automatically create the csv path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_papers_table() - # self.arxiv_crawler = ArxivCrawler() - - def create_papers_table(self): - """Create papers CSV with appropriate columns.""" - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'arxiv_id', 'base_arxiv_id', 'version', - 'title', 'abstract', 'submit_date', 'metadata' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - def insert_paper(self, arxiv_id, base_arxiv_id, version, title, abstract=None, submit_date=None, metadata=None): - """Insert a paper. Returns the generated paper id.""" - df = self._load_data() - - # Check for conflict - if arxiv_id in df['arxiv_id'].values: - return None - - new_id = df['id'].max() + 1 if not df.empty else 1 - meta_str = json.dumps(metadata) if metadata is not None else None - - new_row = pd.DataFrame([{ - 'id': new_id, - 'arxiv_id': arxiv_id, - 'base_arxiv_id': base_arxiv_id, - 'version': version, - 'title': title, - 'abstract': abstract, - 'submit_date': submit_date, - 'metadata': meta_str - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_paper_by_id(self, arxiv_id): - """Delete a paper by its arxiv_id. Returns True if deleted, False if not found.""" - df = self._load_data() - - if arxiv_id not in df['arxiv_id'].values: - return False - - df = df[df['arxiv_id'] != arxiv_id] - self._save_data(df) - return True - - def delete_paper_by_year(self, year): - """Delete all papers from a specific year. Returns the number of papers deleted.""" - df = self._load_data() - - if df.empty or 'submit_date' not in df.columns: - return 0 - - # Convert submit_date to datetime and extract year - df['submit_date'] = pd.to_datetime(df['submit_date'], errors='coerce') - initial_count = len(df) - - # Filter out papers from the specified year - df = df[df['submit_date'].dt.year != year] - deleted_count = initial_count - len(df) - - self._save_data(df) - return deleted_count - def get_all_papers(self, is_all_features=True) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def get_paper_by_id(self, arxiv_id) -> Optional[pd.DataFrame]: - - df = self._load_data() - - mask = df['arxiv_id'] == arxiv_id - result = df[mask].copy() - - if result.empty: - print(f"No author found with author_openreview_id {arxiv_id}.") - return None - - return result - - - def update_paper(self, arxiv_id, base_arxiv_id=None, version=None, title=None, abstract=None, submit_date=None, metadata=None): - """Update a paper by arxiv_id. Returns True if updated, False if not found.""" - df = self._load_data() - - # Search for paper by arxiv id - if arxiv_id not in df['arxiv_id'].values: - return False - - # Update the corresponding paper - mask = df['arxiv_id'] == arxiv_id - - if base_arxiv_id is not None: - df.loc[mask, 'base_arxiv_id'] = base_arxiv_id - if version is not None: - df.loc[mask, 'version'] = version - if title is not None: - df.loc[mask, 'title'] = title - if abstract is not None: - df.loc[mask, 'abstract'] = abstract - if submit_date is not None: - df.loc[mask, 'submit_date'] = submit_date - if metadata is not None: - df.loc[mask, 'metadata'] = json.dumps(metadata) - - self._save_data(df) - return True - - def get_paper_by_arxiv_id(self, arxiv_id: str) -> Optional[pd.DataFrame]: - """Get a paper by its arxiv_id. Returns a DataFrame with the paper or None if not found.""" - df = self._load_data() - - if df.empty or arxiv_id not in df['arxiv_id'].values: - return None - - paper = df[df['arxiv_id'] == arxiv_id] - return paper - - def check_paper_exists(self, arxiv_id: str) -> bool: - """Check if a paper exists by its arxiv_id.""" - df = self._load_data() - - if df.empty: - return False - - return arxiv_id in df['arxiv_id'].values - - def construct_papers_table_from_csv(self, csv_file: str): - """ - Construct the papers table from an external CSV file. - Assumes the CSV has compatible columns or can be mapped to the papers schema. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - # Load the external CSV - external_df = pd.read_csv(csv_file) - - # Load current data - current_df = self._load_data() - - # Ensure the external CSV has required columns - required_cols = ['arxiv_id', 'base_arxiv_id', 'version', 'title'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['abstract', 'submit_date', 'metadata']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new papers - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out papers that already exist (based on arxiv_id) - if not current_df.empty: - existing_ids = set(current_df['arxiv_id'].values) - external_df = external_df[~external_df['arxiv_id'].isin(existing_ids)] - - # Concatenate and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} papers from {csv_file}") - return True - - def construct_papers_table_from_api(self, arxiv_ids, dest_dir): - - # Check if papers already exists in the directory - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - # Then collect information into databases - for arxiv_id in arxiv_ids: - # add metadata - # read paper information - - base_arxiv_id, version = arxiv_id_processor(arxiv_id=arxiv_id) - # Read metadata from path specified - - try: - metadata_path = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - with open(metadata_path, 'r') as f: - metadata = json.load(f) # Use json.load(), not json.loads() - - # Validate required fields - required_fields = ['title', 'abstract', 'published'] - if not all(field in metadata for field in required_fields): - raise ValueError(f"Missing required fields in metadata for {arxiv_id}") - - self.insert_paper( - arxiv_id=arxiv_id, - base_arxiv_id=base_arxiv_id, - version=version, - title=metadata['title'], - abstract=metadata['abstract'], - submit_date=metadata['published'], - metadata=metadata - ) - except Exception: - print(f"Paper {arxiv_id} does not have metadata downloaded") - - - def construct_table_from_csv(self, csv_file): - """ - Construct the papers table from an external CSV file. - - Args: - csv_file: Path to the CSV file containing paper data - - Expected CSV format: - - Required columns: arxiv_id, base_arxiv_id, version, title - - Optional columns: abstract, submit_date, metadata - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['arxiv_id', 'base_arxiv_id', 'version', 'title'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['abstract', 'submit_date', 'metadata']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new papers - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out papers that already exist (based on arxiv_id) - if not current_df.empty: - existing_ids = set(current_df['arxiv_id'].values) - external_df = external_df[~external_df['arxiv_id'].isin(existing_ids)] - - if external_df.empty: - print("No new papers to import (all papers already exist)") - return True - - # Ensure correct column order - external_df = external_df[['id', 'arxiv_id', 'base_arxiv_id', 'version', 'title', 'abstract', 'submit_date', 'metadata']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} papers from {csv_file}") - return True - - except Exception as e: - print(f"Error importing papers from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the papers table from an external JSON file. - - Args: - json_file: Path to the JSON file containing paper data - - Expected JSON format: - [ - { - "arxiv_id": "1706.03762v7", - "base_arxiv_id": "1706.03762", - "version": 7, - "title": "Attention Is All You Need", - "abstract": "...", - "submit_date": "2017-06-12", - "metadata": {"venue": "NeurIPS 2017"} - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'papers' in json_data: - papers_list = json_data['papers'] - else: - papers_list = [json_data] - elif isinstance(json_data, list): - papers_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not papers_list: - print("Error: No paper data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(papers_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['arxiv_id', 'base_arxiv_id', 'version', 'title'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['abstract', 'submit_date']: - if col not in external_df.columns: - external_df[col] = None - - # Handle metadata field - convert dict to JSON string if needed - if 'metadata' not in external_df.columns: - external_df['metadata'] = None - else: - external_df['metadata'] = external_df['metadata'].apply( - lambda x: json.dumps(x) if isinstance(x, dict) else x - ) - - # Generate IDs for new papers - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out papers that already exist - if not current_df.empty: - existing_ids = set(current_df['arxiv_id'].values) - external_df = external_df[~external_df['arxiv_id'].isin(existing_ids)] - - if external_df.empty: - print("No new papers to import (all papers already exist)") - return True - - # Ensure correct column order - external_df = external_df[['id', 'arxiv_id', 'base_arxiv_id', 'version', 'title', 'abstract', 'submit_date', 'metadata']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} papers from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing papers from JSON: {e}") - return False diff --git a/research_arcade/csv_database/csv_arxiv_paragraph_references.py b/research_arcade/csv_database/csv_arxiv_paragraph_references.py deleted file mode 100644 index a58a068..0000000 --- a/research_arcade/csv_database/csv_arxiv_paragraph_references.py +++ /dev/null @@ -1,262 +0,0 @@ -import pandas as pd -import os -from pathlib import Path -from typing import Optional - -class CSVArxivParagraphReference: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_paragraph_references.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_paragraph_references_table() - - def create_paragraph_references_table(self): - df = pd.DataFrame(columns=[ - 'id', 'paragraph_id', 'paper_section', - 'paper_arxiv_id', 'reference_label', 'reference_type' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created paragraph_references CSV at {self.csv_path}") - - def _load_data(self): - return pd.read_csv(self.csv_path) if os.path.exists(self.csv_path) else pd.DataFrame() - def _save_data(self, df): - df.to_csv(self.csv_path, index=False) - - def insert_paragraph_reference(self, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type): - df = self._load_data() - new_id = df['id'].max() + 1 if not df.empty else 1 - new_row = pd.DataFrame([{ - 'id': new_id, 'paragraph_id': paragraph_id, - 'paper_section': paper_section, 'paper_arxiv_id': paper_arxiv_id, - 'reference_label': reference_label, 'reference_type': reference_type - }]) - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def get_all_paragraph_references(self): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - - def get_paragraph_neighboring_references(self, paragraph_id: int) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['paragraph_id'] == paragraph_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def get_reference_neighboring_paragraphs(self, reference_id: int) -> Optional[pd.DataFrame]: - - df = self._load_data() - - if df.empty: - return None - - result = df[df['id'] == reference_id].copy() - - if result.empty: - return None - - return result.reset_index(drop=True) - - - def delete_paragraph_reference_by_id(self, paragraph_id: int, reference_id: int) -> bool: - df = self._load_data() - - if df.empty: - return False - - mask = (df['paragraph_id'] == paragraph_id) & (df['id'] == reference_id) - - if not mask.any(): - return False - - df = df[~mask] - self._save_data(df) - - return True - - - def delete_paragraph_reference_by_paragraph_id(self, paragraph_id: int) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['paragraph_id'] == paragraph_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - def delete_paragraph_reference_by_reference_id(self, reference_id: int) -> int: - - df = self._load_data() - - if df.empty: - return 0 - - mask = df['id'] == reference_id - count = mask.sum() - - if count == 0: - return 0 - - df = df[~mask] - self._save_data(df) - - return count - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the paragraph-reference relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paragraph_id', 'paper_section', 'paper_arxiv_id', 'reference_label', 'reference_type'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Generate IDs for new references - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Note: Not filtering duplicates as this table allows multiple references per paragraph - - # Ensure correct column order - external_df = external_df[['id', 'paragraph_id', 'paper_section', 'paper_arxiv_id', 'reference_label', 'reference_type']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paragraph-reference relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paragraph-reference relationships from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the paragraph-reference relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - { - "paragraph_id": 1, - "paper_section": "introduction", - "paper_arxiv_id": "1706.03762v7", - "reference_label": "fig:1", - "reference_type": "figure" - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paragraph_references' in json_data: - relations_list = json_data['paragraph_references'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paragraph-reference data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(relations_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paragraph_id', 'paper_section', 'paper_arxiv_id', 'reference_label', 'reference_type'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Generate IDs for new references - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Ensure correct column order - external_df = external_df[['id', 'paragraph_id', 'paper_section', 'paper_arxiv_id', 'reference_label', 'reference_type']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paragraph-reference relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paragraph-reference relationships from JSON: {e}") - return False - - diff --git a/research_arcade/csv_database/csv_arxiv_paragraphs.py b/research_arcade/csv_database/csv_arxiv_paragraphs.py deleted file mode 100644 index 07bae77..0000000 --- a/research_arcade/csv_database/csv_arxiv_paragraphs.py +++ /dev/null @@ -1,500 +0,0 @@ -import pandas as pd -import os -from typing import Optional -from pathlib import Path -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.paper_collector.paper_graph_processor import PaperGraphProcessor -from ..arxiv_utils.utils import get_paragraph_num - -class CSVArxivParagraphs: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_paragraphs.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_paragraphs_table() - # self.arxiv_crawler = ArxivCrawler() - - - def create_paragraphs_table(self): - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'paragraph_id', 'content', 'paper_arxiv_id', 'paper_section' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - - def insert_paragraph(self, paragraph_id, content, paper_arxiv_id, paper_section, section_id=None, paragraph_in_paper_id=None): - df = self._load_data() - - conflict = df[ - (df['paragraph_id'] == paragraph_id) & - (df['paper_arxiv_id'] == paper_arxiv_id) & - (df['paper_section'] == paper_section) - ] - if not conflict.empty: - return None - - new_id = df['id'].max() + 1 if not df.empty else 1 - - new_row = pd.DataFrame([{ - 'id': new_id, - 'paragraph_id': paragraph_id, - 'content': content, - 'paper_arxiv_id': paper_arxiv_id, - 'paper_section': paper_section, - 'section_id': section_id, - 'paragraph_in_paper_id': paragraph_in_paper_id - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_paragraph_by_id(self, id): - df = self._load_data() - - if id not in df['id'].values: - return False - - df = df[df['id'] != id] - self._save_data(df) - return True - - def delete_paragraph_by_paper_arxiv_id(self, paper_arxiv_id): - df = self._load_data() - - if df.empty or 'paper_arxiv_id' not in df.columns: - return 0 - - initial_count = len(df) - - df = df[df['paper_arxiv_id'] != paper_arxiv_id] - deleted_count = initial_count - len(df) - - self._save_data(df) - return deleted_count - - def delete_paragraph_by_paper_section(self, paper_arxiv_id, paper_section): - df = self._load_data() - - if df.empty or 'paper_arxiv_id' not in df.columns or 'paper_section' not in df.columns: - return 0 - - initial_count = len(df) - - df = df[~((df['paper_arxiv_id'] == paper_arxiv_id) & (df['paper_section'] == paper_section))] - deleted_count = initial_count - len(df) - - self._save_data(df) - return deleted_count - - - def update_paragraph(self, id, paragraph_id=None, content=None, paper_arxiv_id=None, paper_section=None): - df = self._load_data() - - if id not in df['id'].values: - return False - - mask = df['id'] == id - - if paragraph_id is not None: - df.loc[mask, 'paragraph_id'] = paragraph_id - if content is not None: - df.loc[mask, 'content'] = content - if paper_arxiv_id is not None: - df.loc[mask, 'paper_arxiv_id'] = paper_arxiv_id - if paper_section is not None: - df.loc[mask, 'paper_section'] = paper_section - - self._save_data(df) - return True - - def get_paragraph_by_id(self, id: int) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty or id not in df['id'].values: - return None - - paragraph = df[df['id'] == id] - return paragraph - - - def get_paragraphs_by_arxiv_id(self, arxiv_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty or 'paper_arxiv_id' not in df.columns: - return None - - paragraphs = df[df['paper_arxiv_id'] == arxiv_id] - - if paragraphs.empty: - return None - - return paragraphs - - def get_paragraphs_by_paper_section(self, paper_arxiv_id, paper_section): - df = self._load_data() - - if df.empty or 'paper_arxiv_id' not in df.columns or 'paper_section' not in df.columns: - return None - - paragraphs = df[(df['paper_arxiv_id'] == paper_arxiv_id) & (df['paper_section'] == paper_section)] - - if paragraphs.empty: - return None - - return paragraphs - - - def check_paragraph_exists(self, id: int) -> bool: - df = self._load_data() - - if df.empty: - return False - - return id in df['id'].values - - def construct_paragraph_table_from_csv(self, csv_file: str): - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paragraph_id', 'content', 'paper_arxiv_id', 'paper_section'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paragraphs from {csv_file}") - return True - - def get_all_paragraphs(self, is_all_features=True): - - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def construct_paragraphs_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - """ - section id and paragraph order required further - Or maybe we can write a incremental method to process such information - TODO - """ - downloaded_paper_ids = [] - md = MultiDownload() - - data_dir_path = f"{dest_dir}/output" - figures_dir_path = f"{dest_dir}/output/images" - output_dir_path = f"{dest_dir}/output/paragraphs" - pgp = PaperGraphProcessor(data_dir=data_dir_path, figures_dir=figures_dir_path, output_dir=output_dir_path) - - papers = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - papers.append(arxiv_id) - - paper_paths = [] - # We first build paper node - # We loop through the provided arxiv ids of paper. - for arxiv_id in papers: - paper_paths.append(f"{dest_dir}/output/{arxiv_id}.json") - pgp.process_papers(paper_paths) - - # Build the paragraphs - - paragraph_path = f"{dest_dir}/output/paragraphs/text_nodes.jsonl" - with open(paragraph_path) as f: - data = [json.loads(line) for line in f] - - - # Use arxiv_id + section name as key - # Find the smallest paragraph_id generated by knowledge debugger - # Subtract all paragraph id of the same section (of the same paper) with the smallest one to ensure that order starts with zero - section_min_paragraph = {} - - for paragraph in data: - paragraph_id = paragraph.get('id') - # Extract paragraph_id - id_number = get_paragraph_num(paragraph_id) - paper_arxiv_id = paragraph.get('paper_id') - paper_section = paragraph.get('section') - if (paper_arxiv_id, paper_section) not in section_min_paragraph: - section_min_paragraph[(paper_arxiv_id, paper_section)] = int(id_number) - else: - section_min_paragraph[(paper_arxiv_id, paper_section)] = min(section_min_paragraph[(paper_arxiv_id, paper_section)], int(id_number)) - - for paragraph in data: - paragraph_id = paragraph.get('id') - content = paragraph.get('content') - paper_arxiv_id = paragraph.get('paper_id') - paper_section = paragraph.get('section') - id_number = get_paragraph_num(paragraph_id) - id_zero_based = id_number - section_min_paragraph[(paper_arxiv_id, paper_section)] - self.insert_paragraph(paragraph_id=id_zero_based, content=content, paper_arxiv_id=paper_arxiv_id, paper_section=paper_section) - - # paragraph_cite_bib_keys = paragraph.get('cites') - # for bib_key in paragraph_cite_bib_keys: - # self.db.insert_paragraph_citations(paragraph_id=id_zero_based, paper_section=paper_section, citing_arxiv_id=paper_arxiv_id, bib_key=bib_key) - - - # paragraph_ref_labels = paragraph.get('ref_labels') - - - # # def insert_paragraph_reference(self, paragraph_id, paper_arxiv_id, reference_label, reference_type=None): - - # for ref_label in paragraph_ref_labels: - - # ref_type = None - # # First search bib_key in databases. - # # If presented in one of them, we can determine the type of reference - - # is_figure = self.db.check_exist_figure(bib_key=ref_label) - # is_table = self.db.check_exist_table(bib_key=ref_label) - # if is_figure: - # ref_type = 'figure' - # elif is_table: - # ref_type = 'table' - - # self.insert_paragraph(paragraph_id=id_zero_based, paper_section=paper_section, paper_arxiv_id=paper_arxiv_id, paper_section=paper_section, refe) - - # self.db.insert_paragraph_reference(paragraph_id=id_zero_based, paper_section=paper_section, paper_arxiv_id=paper_arxiv_id, reference_label=ref_label, reference_type=ref_type) - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the paragraphs table from an external CSV file. - - Args: - csv_file: Path to the CSV file containing paragraph data - - Expected CSV format: - - Required columns: paragraph_id, content, paper_arxiv_id, paper_section - - Optional columns: section_id, paragraph_in_paper_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paragraph_id', 'content', 'paper_arxiv_id', 'paper_section'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['section_id', 'paragraph_in_paper_id']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new paragraphs - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out paragraphs that already exist (based on paragraph_id, paper_arxiv_id, and paper_section) - if not current_df.empty: - existing_tuples = set(zip( - current_df['paragraph_id'], - current_df['paper_arxiv_id'], - current_df['paper_section'] - )) - external_df['_tuple'] = list(zip( - external_df['paragraph_id'], - external_df['paper_arxiv_id'], - external_df['paper_section'] - )) - external_df = external_df[~external_df['_tuple'].isin(existing_tuples)] - external_df = external_df.drop(columns=['_tuple']) - - if external_df.empty: - print("No new paragraphs to import (all paragraphs already exist)") - return True - - # Ensure correct column order - external_df = external_df[['id', 'paragraph_id', 'content', 'paper_arxiv_id', 'paper_section', 'section_id', 'paragraph_in_paper_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paragraphs from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paragraphs from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the paragraphs table from an external JSON file. - - Args: - json_file: Path to the JSON file containing paragraph data - - Expected JSON format: - [ - { - "paragraph_id": 0, - "content": "This paper introduces the Transformer...", - "paper_arxiv_id": "1706.03762v7", - "paper_section": "introduction", - "section_id": 1, - "paragraph_in_paper_id": 0 - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paragraphs' in json_data: - paragraphs_list = json_data['paragraphs'] - else: - paragraphs_list = [json_data] - elif isinstance(json_data, list): - paragraphs_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not paragraphs_list: - print("Error: No paragraph data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(paragraphs_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paragraph_id', 'content', 'paper_arxiv_id', 'paper_section'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['section_id', 'paragraph_in_paper_id']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new paragraphs - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Filter out paragraphs that already exist - if not current_df.empty: - existing_tuples = set(zip( - current_df['paragraph_id'], - current_df['paper_arxiv_id'], - current_df['paper_section'] - )) - external_df['_tuple'] = list(zip( - external_df['paragraph_id'], - external_df['paper_arxiv_id'], - external_df['paper_section'] - )) - external_df = external_df[~external_df['_tuple'].isin(existing_tuples)] - external_df = external_df.drop(columns=['_tuple']) - - if external_df.empty: - print("No new paragraphs to import (all paragraphs already exist)") - return True - - # Ensure correct column order - external_df = external_df[['id', 'paragraph_id', 'content', 'paper_arxiv_id', 'paper_section', 'section_id', 'paragraph_in_paper_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} paragraphs from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paragraphs from JSON: {e}") - return False - - - diff --git a/research_arcade/csv_database/csv_arxiv_sections.py b/research_arcade/csv_database/csv_arxiv_sections.py deleted file mode 100644 index 8b377c6..0000000 --- a/research_arcade/csv_database/csv_arxiv_sections.py +++ /dev/null @@ -1,371 +0,0 @@ -import pandas as pd -import os -from typing import Optional -from pathlib import Path -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor - -class CSVArxivSections: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_sections.csv" - self.csv_path = csv_path - # Set up the target directory - # Automatically create the csv path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_sections_table() - - def create_sections_table(self): - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'content', 'title', 'appendix', 'paper_arxiv_id' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - - def insert_section(self, content, title, appendix, paper_arxiv_id, section_in_paper_id): - """Insert a section. Returns the generated section id.""" - df = self._load_data() - - new_id = df['id'].max() + 1 if not df.empty else 1 - - new_row = pd.DataFrame([{ - 'id': new_id, - 'content': content, - 'title': title, - 'appendix': appendix, - 'paper_arxiv_id': paper_arxiv_id, - 'section_in_paper_id': section_in_paper_id - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_section_by_id(self, id): - """Delete a section by its id. Returns True if deleted, False if not found.""" - df = self._load_data() - - if id not in df['id'].values: - return False - - df = df[df['id'] != id] - self._save_data(df) - return True - - def delete_section_by_paper_arxiv_id(self, paper_arxiv_id): - """Delete all sections for a specific paper arxiv_id. Returns the number of sections deleted.""" - df = self._load_data() - - if df.empty or 'paper_arxiv_id' not in df.columns: - return 0 - - initial_count = len(df) - - df = df[df['paper_arxiv_id'] != paper_arxiv_id] - deleted_count = initial_count - len(df) - - self._save_data(df) - return deleted_count - - def update_section(self, id, content=None, title=None, is_appendix=None, paper_arxiv_id=None): - """Update a section by id. Returns True if updated, False if not found.""" - df = self._load_data() - - if id not in df['id'].values: - return False - - mask = df['id'] == id - - if content is not None: - df.loc[mask, 'content'] = content - if title is not None: - df.loc[mask, 'title'] = title - if is_appendix is not None: - df.loc[mask, 'appendix'] = is_appendix - if paper_arxiv_id is not None: - df.loc[mask, 'paper_arxiv_id'] = paper_arxiv_id - - self._save_data(df) - return True - - def get_section_by_id(self, id: int) -> Optional[pd.DataFrame]: - """Get a section by its id. Returns a DataFrame with the section or None if not found.""" - df = self._load_data() - - if df.empty or id not in df['id'].values: - return None - - section = df[df['id'] == id] - return section - - def get_sections_by_arxiv_id(self, arxiv_id: str) -> Optional[pd.DataFrame]: - """Get all sections for a paper by its arxiv_id. Returns a DataFrame with sections or None if not found.""" - df = self._load_data() - - if df.empty or 'paper_arxiv_id' not in df.columns: - return None - - sections = df[df['paper_arxiv_id'] == arxiv_id] - - if sections.empty: - return None - - return sections - - def check_section_exists(self, id: int) -> bool: - """Check if a section exists by its id.""" - df = self._load_data() - - if df.empty: - return False - - return id in df['id'].values - - def construct_sections_table_from_csv(self, csv_file: str): - """ - Construct the sections table from an external CSV file. - Assumes the CSV has compatible columns or can be mapped to the sections schema. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['content', 'title', 'appendix', 'paper_arxiv_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} sections from {csv_file}") - return True - - def get_all_sections(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - - def construct_sections_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - md = MultiDownload() - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - section_jsons = file_json['sections'] - - i = 0 - for title, section_json in section_jsons.items(): - i += 1 - is_appendix = section_json['appendix'] == 'true' - content = section_json['content'] - self.insert_section(content=content, title=title, appendix=is_appendix, paper_arxiv_id=arxiv_id, section_in_paper_id=i) - - except FileNotFoundError: - print(f"Error: The file at path '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON at path '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue - - - def construct_table_from_csv(self, csv_file): - """ - Construct the sections table from an external CSV file. - - Args: - csv_file: Path to the CSV file containing section data - - Expected CSV format: - - Required columns: content, title, appendix, paper_arxiv_id - - Optional columns: section_in_paper_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['content', 'title', 'appendix', 'paper_arxiv_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'section_in_paper_id' not in external_df.columns: - external_df['section_in_paper_id'] = None - - # Generate IDs for new sections - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Note: Not filtering for duplicates as sections can be re-imported - - # Ensure correct column order - external_df = external_df[['id', 'content', 'title', 'appendix', 'paper_arxiv_id', 'section_in_paper_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} sections from {csv_file}") - return True - - except Exception as e: - print(f"Error importing sections from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the sections table from an external JSON file. - - Args: - json_file: Path to the JSON file containing section data - - Expected JSON format: - [ - { - "content": "Section content...", - "title": "Introduction", - "appendix": false, - "paper_arxiv_id": "1706.03762v7", - "section_in_paper_id": 1 - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'sections' in json_data: - sections_list = json_data['sections'] - else: - sections_list = [json_data] - elif isinstance(json_data, list): - sections_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not sections_list: - print("Error: No section data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(sections_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['content', 'title', 'appendix', 'paper_arxiv_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'section_in_paper_id' not in external_df.columns: - external_df['section_in_paper_id'] = None - - # Generate IDs for new sections - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Ensure correct column order - external_df = external_df[['id', 'content', 'title', 'appendix', 'paper_arxiv_id', 'section_in_paper_id']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} sections from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing sections from JSON: {e}") - return False - diff --git a/research_arcade/csv_database/csv_arxiv_tables.py b/research_arcade/csv_database/csv_arxiv_tables.py deleted file mode 100644 index 2bc9266..0000000 --- a/research_arcade/csv_database/csv_arxiv_tables.py +++ /dev/null @@ -1,335 +0,0 @@ -import pandas as pd -import os -from typing import Optional -from pathlib import Path -from ..arxiv_utils.multi_input.multi_download import MultiDownload -import json - -class CSVArxivTable: - def __init__(self, csv_dir: str): - csv_path = f"{csv_dir}/arxiv_tables.csv" - self.csv_path = csv_path - Path(csv_path).parent.mkdir(parents=True, exist_ok=True) - if not os.path.exists(csv_path): - self.create_tables_table() - # self.arxiv_crawler = ArxivCrawler() - - def create_tables_table(self): - if not os.path.exists(self.csv_path): - df = pd.DataFrame(columns=[ - 'id', 'paper_arxiv_id', 'path', 'caption', 'label', 'table_text' - ]) - df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> pd.DataFrame: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return pd.DataFrame() - - def _save_data(self, df: pd.DataFrame): - df.to_csv(self.csv_path, index=False) - - - def insert_table(self, paper_arxiv_id, path=None, caption=None, label=None, table_text=None): - - df = self._load_data() - - new_id = df['id'].max() + 1 if not df.empty else 1 - - new_row = pd.DataFrame([{ - 'id': new_id, - 'paper_arxiv_id': paper_arxiv_id, - 'path': path, - 'caption': caption, - 'label': label, - 'table_text': table_text - }]) - - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - return new_id - - def delete_table_by_id(self, id): - df = self._load_data() - - if id not in df['id'].values: - return False - - df = df[df['id'] != id] - self._save_data(df) - return True - - - def update_table(self, id, paper_arxiv_id, path=None, caption=None, label=None, table_text=None): - df = self._load_data() - - if id not in df['id'].values: - return False - - mask = df['id'] == id - - if paper_arxiv_id is not None: - df.loc[mask, 'paper_arxiv_id'] = paper_arxiv_id - if path is not None: - df.loc[mask, 'path'] = path - if caption is not None: - df.loc[mask, 'caption'] = caption - if label is not None: - df.loc[mask, 'label'] = label - if table_text is not None: - df.loc[mask, 'table_text'] = table_text - - self._save_data(df) - return True - - def get_table_by_id(self, id: int) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty or id not in df['id'].values: - return None - - table = df[df['id'] == id] - return table - - def check_table_exists(self, id: int) -> bool: - df = self._load_data() - - if df.empty: - return False - - return id in df['id'].values - - def construct_tables_table_from_csv(self, csv_file: str): - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id', 'path','caption','label', 'table_text'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} tables from {csv_file}") - return True - - def get_all_tables(self, is_all_features=True): - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def construct_tables_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - md = MultiDownload() - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=self.dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - table_jsons = file_json['table'] - for table_json in table_jsons: - - caption = table_json['caption'] - label = table_json['label'] - table = table_json['tabular'] - self.insert_table(paper_arxiv_id=arxiv_id, path=None, caption=caption, label=label, table_text=table) - - except FileNotFoundError: - print(f"Error: The file at path '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue - - - - def construct_table_from_csv(self, csv_file): - """ - Construct the tables table from an external CSV file. - - Args: - csv_file: Path to the CSV file containing table data - - Expected CSV format: - - Required columns: paper_arxiv_id - - Optional columns: path, caption, label, table_text - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - external_df = pd.read_csv(csv_file) - current_df = self._load_data() - - required_cols = ['paper_arxiv_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['path', 'caption', 'label', 'table_text']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new tables - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Note: Not filtering duplicates as tables can be re-imported - - # Ensure correct column order - external_df = external_df[['id', 'paper_arxiv_id', 'path', 'caption', 'label', 'table_text']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} tables from {csv_file}") - return True - - except Exception as e: - print(f"Error importing tables from CSV: {e}") - return False - - - def construct_table_from_json(self, json_file): - """ - Construct the tables table from an external JSON file. - - Args: - json_file: Path to the JSON file containing table data - - Expected JSON format: - [ - { - "paper_arxiv_id": "1706.03762v7", - "path": "/path/to/table1.tex", - "caption": "Model performance comparison", - "label": "tab:performance", - "table_text": "Table content..." - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'tables' in json_data: - tables_list = json_data['tables'] - else: - tables_list = [json_data] - elif isinstance(json_data, list): - tables_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not tables_list: - print("Error: No table data found in JSON file") - return False - - # Convert to DataFrame - external_df = pd.DataFrame(tables_list) - current_df = self._load_data() - - # Check for required columns - required_cols = ['paper_arxiv_id'] - missing_cols = [col for col in required_cols if col not in external_df.columns] - - if missing_cols: - print(f"Error: JSON data is missing required fields: {missing_cols}") - return False - - # Add optional columns if they don't exist - for col in ['path', 'caption', 'label', 'table_text']: - if col not in external_df.columns: - external_df[col] = None - - # Generate IDs for new tables - start_id = current_df['id'].max() + 1 if not current_df.empty else 1 - external_df['id'] = range(start_id, start_id + len(external_df)) - - # Ensure correct column order - external_df = external_df[['id', 'paper_arxiv_id', 'path', 'caption', 'label', 'table_text']] - - # Combine and save - combined_df = pd.concat([current_df, external_df], ignore_index=True) - self._save_data(combined_df) - - print(f"Successfully imported {len(external_df)} tables from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing tables from JSON: {e}") - return False - diff --git a/research_arcade/csv_database/csv_openreview_arxiv.py b/research_arcade/csv_database/csv_openreview_arxiv.py deleted file mode 100644 index d507662..0000000 --- a/research_arcade/csv_database/csv_openreview_arxiv.py +++ /dev/null @@ -1,229 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewArxiv: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_arxiv.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_openreview_arxiv_table() - - def create_openreview_arxiv_table(self) -> None: - columns = ['venue', 'paper_openreview_id', 'arxiv_id', 'title'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_openreview_arxiv(self, venue: str, paper_openreview_id: str, - arxiv_id: str, title: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于venue和paper_openreview_id的组合键） - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'paper_openreview_id': self._clean_string(paper_openreview_id), - 'arxiv_id': self._clean_string(arxiv_id), - 'title': self._clean_string(title) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, paper_openreview_id) - - def delete_openreview_arxiv_by_id(self, paper_openreview_id: str, arxiv_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = (df['paper_openreview_id'] == paper_openreview_id) & (df['arxiv_id'] == arxiv_id) - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No records found in 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id} and arxiv_id = {arxiv_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Deleted {len(deleted_rows)} records from 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id} and arxiv_id = {arxiv_id}.") - return deleted_rows - - def delete_openreview_arxiv_by_openreview_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['paper_openreview_id'] == paper_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No records found in 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Deleted {len(deleted_rows)} records from 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id}.") - return deleted_rows - - def delete_openreview_arxiv_by_arxiv_id(self, arxiv_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['arxiv_id'] == arxiv_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No records found in 'openreview_arxiv' with arxiv_id = {arxiv_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Deleted {len(deleted_rows)} records from 'arxiv_id' with arxiv_id = {arxiv_id}.") - return deleted_rows - - def delete_openreview_arxiv_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No records found in 'openreview_arxiv' for venue = {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Deleted {len(deleted_rows)} records from 'openreview_arxiv' where venue = {venue}.") - return deleted_rows - - def get_openreview_neighboring_arxivs(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['paper_openreview_id'] == paper_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_arxiv_neighboring_openreviews(self, arxiv_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['arxiv_id'] == arxiv_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_openreview_arxiv_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_all_openreview_arxiv(self) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def check_openreview_arxiv_exists(self, venue: str, paper_openreview_id: str) -> bool: - df = self._load_data() - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id)).any() - return exists - - def construct_openreview_arxiv_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print(f"Crawling openreview arxiv data for venue: {venue}...") - openreview_arxiv_data = self.openreview_crawler.crawl_openreview_arxiv_data_from_api(venue) - - # 插入数据 - if len(openreview_arxiv_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(openreview_arxiv_data): - self.insert_openreview_arxiv(**data) - return True - else: - print("No new openreview arxiv data to insert.") - return False - - def construct_openreview_arxiv_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading openreview arxiv data from {csv_file}...") - import_df = pd.read_csv(csv_file) - openreview_arxiv_data = import_df.to_dict(orient='records') - - if len(openreview_arxiv_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(openreview_arxiv_data): - self.insert_openreview_arxiv(**data) - return True - else: - print("No new openreview arxiv data to insert.") - return False - - def construct_openreview_arxiv_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading openreview arxiv data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - openreview_arxiv_data = json.load(f) - - if len(openreview_arxiv_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(openreview_arxiv_data): - self.insert_openreview_arxiv(**data) - return True - else: - print("No new openreview arxiv data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_authors.py b/research_arcade/csv_database/csv_openreview_authors.py deleted file mode 100644 index 2f0a079..0000000 --- a/research_arcade/csv_database/csv_openreview_authors.py +++ /dev/null @@ -1,222 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewAuthors: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_authors.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_author_table() - - def create_author_table(self) -> None: - columns = ['venue', 'author_openreview_id', 'author_full_name', 'email', - 'affiliation', 'homepage', 'dblp'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - else: - None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_author(self, venue: str, author_openreview_id: str, - author_full_name: str, email: str, affiliation: str, - homepage: str, dblp: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于venue和author_openreview_id的组合键） - exists = ((df['venue'] == venue) & - (df['author_openreview_id'] == author_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'author_openreview_id': self._clean_string(author_openreview_id), - 'author_full_name': self._clean_string(author_full_name), - 'email': self._clean_string(email), - 'affiliation': self._clean_string(affiliation), - 'homepage': self._clean_string(homepage), - 'dblp': self._clean_string(dblp) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, author_openreview_id) - - def delete_author_by_id(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['author_openreview_id'] == author_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No author found with author_openreview_id {author_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Author with author_openreview_id {author_openreview_id} deleted successfully.") - return deleted_rows - - def delete_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No authors found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All authors in venue {venue} deleted successfully.") - return deleted_rows - - def update_author(self, venue: str, author_openreview_id: str, - author_full_name: str, email: str, affiliation: str, - homepage: str, dblp: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要更新的行 - mask = ((df['venue'] == venue) & - (df['author_openreview_id'] == author_openreview_id)) - - if not mask.any(): - print(f"No author found with author_openreview_id {author_openreview_id}.") - return None - - # 保存原始记录 - original_record = df[mask].copy() - - # 更新记录 - df.loc[mask, 'author_full_name'] = self._clean_string(author_full_name) - df.loc[mask, 'email'] = self._clean_string(email) - df.loc[mask, 'affiliation'] = self._clean_string(affiliation) - df.loc[mask, 'homepage'] = self._clean_string(homepage) - df.loc[mask, 'dblp'] = self._clean_string(dblp) - - self._save_data(df) - - print(f"Author with author_openreview_id {author_openreview_id} updated successfully.") - return original_record - - def get_author_by_id(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['author_openreview_id'] == author_openreview_id - result = df[mask].copy() - - if result.empty: - print(f"No author found with author_openreview_id {author_openreview_id}.") - return None - - return result - - def get_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - print(f"No authors found in venue {venue}.") - return None - - return result - - def get_all_authors(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - # 按author_openreview_id排序 - df = df.sort_values('author_openreview_id') - - if is_all_features: - return df[['venue', 'author_openreview_id', 'author_full_name', - 'email', 'affiliation', 'homepage', 'dblp']].copy() - else: - return df[['venue', 'author_openreview_id', 'author_full_name']].copy() - - def check_author_exists(self, author_openreview_id: str) -> bool: - df = self._load_data() - return (df['author_openreview_id'] == author_openreview_id).any() - - def construct_authors_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print("Crawling author data from OpenReview API...") - author_data = self.openreview_crawler.crawl_author_data_from_api(venue) - - # 插入数据 - if len(author_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(author_data): - self.insert_author(**data) - return True - else: - print("No new author data to insert.") - return False - - def construct_authors_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading authors data from {csv_file}...") - import_df = pd.read_csv(csv_file) - author_data = import_df.to_dict(orient='records') - - if len(author_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(author_data): - self.insert_author(**data) - return True - else: - print("No new author data to insert.") - return False - - def construct_authors_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading authors data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - author_data = json.load(f) - - if len(author_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(author_data): - self.insert_author(**data) - return True - else: - print("No new author data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_papers.py b/research_arcade/csv_database/csv_openreview_papers.py deleted file mode 100644 index 56fedff..0000000 --- a/research_arcade/csv_database/csv_openreview_papers.py +++ /dev/null @@ -1,221 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewPapers: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_papers.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_papers_table() - - def create_papers_table(self) -> None: - columns = ['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_paper(self, venue: str, paper_openreview_id: str, title: str, - abstract: str, paper_decision: str, - paper_pdf_link: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于venue和paper_openreview_id的组合键） - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'paper_openreview_id': self._clean_string(paper_openreview_id), - 'title': self._clean_string(title), - 'abstract': self._clean_string(abstract), - 'paper_decision': self._clean_string(paper_decision), - 'paper_pdf_link': self._clean_string(paper_pdf_link) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, paper_openreview_id) - - def delete_paper_by_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['paper_openreview_id'] == paper_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No paper found with paper_openreview_id {paper_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Paper with paper_openreview_id {paper_openreview_id} deleted successfully.") - return deleted_rows - - def delete_papers_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No papers found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All papers in venue {venue} deleted successfully.") - return deleted_rows - - def update_paper(self, venue: str, paper_openreview_id: str, title: str, - abstract: str, paper_decision: str, - paper_pdf_link: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要更新的行 - mask = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id)) - - if not mask.any(): - print(f"No paper found with paper_openreview_id {paper_openreview_id}.") - return None - - # 保存原始记录 - original_record = df[mask].copy() - - # 更新记录 - df.loc[mask, 'title'] = self._clean_string(title) - df.loc[mask, 'abstract'] = self._clean_string(abstract) - df.loc[mask, 'paper_decision'] = self._clean_string(paper_decision) - df.loc[mask, 'paper_pdf_link'] = self._clean_string(paper_pdf_link) - - self._save_data(df) - - print(f"Paper with paper_openreview_id {paper_openreview_id} updated successfully.") - return original_record - - def get_paper_by_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - """根据paper_openreview_id获取论文""" - df = self._load_data() - - mask = df['paper_openreview_id'] == paper_openreview_id - result = df[mask].copy() - - if result.empty: - print(f"No paper found with paper_openreview_id {paper_openreview_id}.") - return None - - return result - - def get_papers_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - """根据venue获取所有论文""" - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - print(f"No papers found in venue {venue}.") - return None - - return result - - def get_all_papers(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - """获取所有论文""" - df = self._load_data() - - if df.empty: - return None - - if is_all_features: - return df[['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link']].copy() - else: - return df[['venue', 'paper_openreview_id', 'title']].copy() - - def check_paper_exists(self, paper_openreview_id: str) -> bool: - """检查论文是否存在""" - df = self._load_data() - return (df['paper_openreview_id'] == paper_openreview_id).any() - - def construct_papers_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print("Crawling paper data from OpenReview API...") - paper_data = self.openreview_crawler.crawl_paper_data_from_api(venue) - - # 插入数据 - if len(paper_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(paper_data): - self.insert_paper(**data) - return True - else: - print("No new paper data to insert.") - return False - - def construct_papers_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading paper data from {csv_file}...") - import_df = pd.read_csv(csv_file) - paper_data = import_df.to_dict(orient='records') - - if len(paper_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(paper_data): - self.insert_paper(**data) - return True - else: - print("No new paper data to insert.") - return False - - def construct_papers_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading paper data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - paper_data = json.load(f) - - if len(paper_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(paper_data): - self.insert_paper(**data) - return True - else: - print("No new paper data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_papers_authors.py b/research_arcade/csv_database/csv_openreview_papers_authors.py deleted file mode 100644 index 3b9b287..0000000 --- a/research_arcade/csv_database/csv_openreview_papers_authors.py +++ /dev/null @@ -1,233 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewPapersAuthors: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_papers_authors.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_papers_authors_table() - - def create_papers_authors_table(self) -> None: - columns = ['venue', 'paper_openreview_id', 'author_openreview_id'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_paper_authors(self, venue: str, paper_openreview_id: str, - author_openreview_id: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于三个字段的组合键） - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id) & - (df['author_openreview_id'] == author_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'paper_openreview_id': self._clean_string(paper_openreview_id), - 'author_openreview_id': self._clean_string(author_openreview_id) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, paper_openreview_id, author_openreview_id) - - def delete_paper_author_by_id(self, paper_openreview_id: str, - author_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = ((df['paper_openreview_id'] == paper_openreview_id) & - (df['author_openreview_id'] == author_openreview_id)) - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found between paper {paper_openreview_id} and author {author_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection between paper {paper_openreview_id} and author {author_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_paper_author_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['paper_openreview_id'] == paper_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found for paper {paper_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection for paper {paper_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_paper_author_by_author_id(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['author_openreview_id'] == author_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found for author {author_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection for author {author_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_papers_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connections found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All connections in venue {venue} deleted successfully.") - return deleted_rows - - def get_paper_neighboring_authors(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['paper_openreview_id'] == paper_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_author_neighboring_papers(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['author_openreview_id'] == author_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_papers_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_all_papers_authors(self) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - # 按paper_openreview_id排序 - df = df.sort_values('paper_openreview_id') - return df.copy() - - def check_paper_author_exists(self, paper_openreview_id: str, - author_openreview_id: str) -> bool: - df = self._load_data() - exists = ((df['paper_openreview_id'] == paper_openreview_id) & - (df['author_openreview_id'] == author_openreview_id)).any() - return exists - - def construct_papers_authors_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print(f"Crawling papers-authors data for venue {venue} from OpenReview API...") - papers_authors_data = self.openreview_crawler.crawl_papers_authors_data_from_api(venue) - - if len(papers_authors_data) > 0: - print(f"Inserting papers-authors data for venue {venue}...") - for data in tqdm(papers_authors_data): - self.insert_paper_authors(**data) - return True - else: - print(f"No papers-authors data found for venue {venue}.") - return False - - def construct_papers_authors_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading papers-authors data from {csv_file}...") - import_df = pd.read_csv(csv_file) - papers_authors_data = import_df.to_dict(orient='records') - - if len(papers_authors_data) > 0: - print(f"Inserting papers-authors data from {csv_file}...") - for data in tqdm(papers_authors_data): - self.insert_paper_authors(**data) - return True - else: - print(f"No papers-authors data found in {csv_file}.") - return False - - def construct_papers_authors_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading papers-authors data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - papers_authors_data = json.load(f) - - if len(papers_authors_data) > 0: - print(f"Inserting papers-authors data from {json_file}...") - for data in tqdm(papers_authors_data): - self.insert_paper_authors(**data) - return True - else: - print(f"No papers-authors data found in {json_file}.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_papers_reviews.py b/research_arcade/csv_database/csv_openreview_papers_reviews.py deleted file mode 100644 index 75cf392..0000000 --- a/research_arcade/csv_database/csv_openreview_papers_reviews.py +++ /dev/null @@ -1,236 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewPapersReviews: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_papers_reviews.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_papers_reviews_table() - - def create_papers_reviews_table(self) -> None: - columns = ['venue', 'paper_openreview_id', 'review_openreview_id', - 'title', 'time'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_paper_reviews(self, venue: str, paper_openreview_id: str, - review_openreview_id: str, title: str, - time: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于三个字段的组合键） - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'paper_openreview_id': self._clean_string(paper_openreview_id), - 'review_openreview_id': self._clean_string(review_openreview_id), - 'title': self._clean_string(title), - 'time': self._clean_string(time) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, paper_openreview_id, review_openreview_id) - - def delete_paper_review_by_id(self, paper_openreview_id: str, - review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = ((df['paper_openreview_id'] == paper_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)) - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found between paper {paper_openreview_id} and review {review_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection between paper {paper_openreview_id} and review {review_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_paper_review_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['paper_openreview_id'] == paper_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found between paper {paper_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection between paper {paper_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_paper_review_by_review_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['review_openreview_id'] == review_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found between paper review {review_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection between review {review_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_papers_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connections found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All connections in venue {venue} deleted successfully.") - return deleted_rows - - def get_papers_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_all_papers_reviews(self) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - # 按paper_openreview_id排序 - df = df.sort_values('paper_openreview_id') - return df.copy() - - def check_paper_review_exists(self, paper_openreview_id: str, - review_openreview_id: str) -> bool: - df = self._load_data() - exists = ((df['paper_openreview_id'] == paper_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)).any() - return exists - - def get_paper_neighboring_reviews(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['paper_openreview_id'] == paper_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_review_neighboring_papers(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['review_openreview_id'] == review_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def construct_papers_reviews_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print(f"Crawling paper-review connections for venue: {venue}...") - papers_reviews_data = self.openreview_crawler.crawl_papers_reviews_from_api(venue) - - if len(papers_reviews_data) > 0: - print(f"Inserting paper-review connections into CSV file for venue: {venue}...") - for data in tqdm(papers_reviews_data): - self.insert_paper_reviews(**data) - return True - else: - print(f"No paper-review connections found for venue: {venue}.") - return False - - def construct_papers_reviews_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading paper-review connection data from {csv_file}...") - import_df = pd.read_csv(csv_file) - papers_reviews_data = import_df.to_dict(orient='records') - - if len(papers_reviews_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(papers_reviews_data): - self.insert_paper_reviews(**data) - return True - else: - print("No new paper-review connection data to insert.") - return False - - def construct_papers_reviews_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading paper-review connection data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - papers_reviews_data = json.load(f) - - if len(papers_reviews_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(papers_reviews_data): - self.insert_paper_reviews(**data) - return True - else: - print("No new paper-review connection data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_papers_revisions.py b/research_arcade/csv_database/csv_openreview_papers_revisions.py deleted file mode 100644 index 3326cff..0000000 --- a/research_arcade/csv_database/csv_openreview_papers_revisions.py +++ /dev/null @@ -1,238 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewPapersRevisions: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_papers_revisions.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_papers_revisions_table() - - def create_papers_revisions_table(self) -> None: - columns = ['venue', 'paper_openreview_id', 'revision_openreview_id', - 'title', 'time'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - """将DataFrame保存到CSV文件""" - df.to_csv(self.csv_path, index=False) - - def insert_paper_revisions(self, venue: str, paper_openreview_id: str, - revision_openreview_id: str, title: str, - time: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于三个字段的组合键） - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id) & - (df['revision_openreview_id'] == revision_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'paper_openreview_id': self._clean_string(paper_openreview_id), - 'revision_openreview_id': self._clean_string(revision_openreview_id), - 'title': self._clean_string(title), - 'time': self._clean_string(time) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, paper_openreview_id, revision_openreview_id) - - def delete_paper_revision_by_id(self, paper_openreview_id: str, - revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = ((df['paper_openreview_id'] == paper_openreview_id) & - (df['revision_openreview_id'] == revision_openreview_id)) - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found between paper {paper_openreview_id} and revision {revision_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection between paper {paper_openreview_id} and revision {revision_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_paper_revision_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['paper_openreview_id'] == paper_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found for paper {paper_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection for paper {paper_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_paper_revision_by_revision_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['revision_openreview_id'] == revision_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found for revision {revision_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection for revision {revision_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_papers_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connections found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All connections in venue {venue} deleted successfully.") - return deleted_rows - - def get_papers_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_all_papers_revisions(self) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - # 按paper_openreview_id排序 - df = df.sort_values('paper_openreview_id') - return df.copy() - - def check_paper_revision_exists(self, paper_openreview_id: str, - revision_openreview_id: str) -> bool: - df = self._load_data() - exists = ((df['paper_openreview_id'] == paper_openreview_id) & - (df['revision_openreview_id'] == revision_openreview_id)).any() - return exists - - def get_paper_neighboring_revisions(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['paper_openreview_id'] == paper_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_revision_neighboring_papers(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['revision_openreview_id'] == revision_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def construct_papers_revisions_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print(f"Crawling paper-revision data from OpenReview API for venue: {venue}...") - paper_revision_data = self.openreview_crawler.crawl_papers_revisions_data_from_api(venue) - - if len(paper_revision_data) > 0: - print(f"Inserting paper-revision data into CSV file for venue: {venue}...") - for data in tqdm(paper_revision_data): - self.insert_paper_revisions(**data) - return True - else: - print(f"No paper-revision data found for venue: {venue}.") - return False - - def construct_papers_revisions_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading paper-revision data from {csv_file}...") - import_df = pd.read_csv(csv_file) - revision_data = import_df.to_dict(orient='records') - - if len(revision_data) > 0: - print(f"Inserting paper-revision data from {csv_file} into CSV file...") - for data in tqdm(revision_data): - self.insert_paper_revisions(**data) - return True - else: - print(f"No paper-revision data found in {csv_file}.") - return False - - def construct_papers_revisions_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading revisions data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - revision_data = json.load(f) - - if len(revision_data) > 0: - print(f"Inserting paper-revision data from {json_file} into CSV file...") - for data in tqdm(revision_data): - self.insert_paper_revisions(**data) - return True - else: - print(f"No paper-revision data found in {json_file}.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_paragraphs.py b/research_arcade/csv_database/csv_openreview_paragraphs.py deleted file mode 100644 index 74f4b7d..0000000 --- a/research_arcade/csv_database/csv_openreview_paragraphs.py +++ /dev/null @@ -1,187 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewParagraphs: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_paragraphs.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_paragraphs_table() - - def create_paragraphs_table(self) -> None: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section', 'content'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - return None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_paragraph(self, venue: str, paper_openreview_id: str, paragraph_idx: int, section: str, content: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于venue和paper_openreview_id的组合键） - exists = ((df['venue'] == venue) & - (df['paper_openreview_id'] == paper_openreview_id) & - (df['paragraph_idx'] == paragraph_idx)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'paper_openreview_id': self._clean_string(paper_openreview_id), - 'paragraph_idx': paragraph_idx, - 'section': self._clean_string(section), - 'content': self._clean_string(content) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, paper_openreview_id, paragraph_idx) - - def get_all_paragraphs(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - """获取所有论文""" - df = self._load_data() - - if df.empty: - return None - - if is_all_features: - return df[['venue', 'paper_openreview_id', 'paragraph_idx', - 'section', 'content']].copy() - else: - return df[['venue', 'paper_openreview_id', 'paragraph_idx', 'section']].copy() - - def get_paragraphs_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['paper_openreview_id'] == paper_openreview_id - result = df[mask].copy() - - if result.empty: - print(f"No paragraph found with paper_openreview_id {paper_openreview_id}.") - return None - - return result - - def get_paragraphs_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - """根据venue获取所有论文""" - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - print(f"No paragraph found in venue {venue}.") - return None - - return result - - def delete_paragraphs_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No paragraphs found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All paragraphs in venue {venue} deleted successfully.") - return deleted_rows - - def delete_paragraphs_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['paper_openreview_id'] == paper_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No paragraph found with paper_openreview_id {paper_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Paragraphs with paper_openreview_id {paper_openreview_id} deleted successfully.") - return deleted_rows - - def construct_paragraphs_table_from_api(self, venue: str, pdf_dir: str, filter_list: list, log_file: str, - is_paper = True, is_revision = True, is_pdf_delete: bool = True) -> bool: - # 从API爬取数据 - print("Crawling paragraph data from OpenReview API...") - paragraph_data = self.openreview_crawler.crawl_paragraph_data_from_api(venue, pdf_dir, filter_list, log_file, is_paper, is_revision, is_pdf_delete) - - # 插入数据 - if len(paragraph_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(paragraph_data): - self.insert_paragraph(**data) - return True - else: - print("No new paragraph data to insert.") - return False - - def construct_paragraphs_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - print(f"File {csv_file} not exists") - return False - else: - print(f"Reading paragraph data from {csv_file}...") - import_df = pd.read_csv(csv_file) - paper_data = import_df.to_dict(orient='records') - - if len(paper_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(paper_data): - self.insert_paragraph(**data) - return True - else: - print("No new paragraph data to insert.") - return False - - def construct_paragraphs_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - print(f"File {json_file} not exists") - return False - else: - print(f"Reading paper data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - paper_data = json.load(f) - - if len(paper_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(paper_data): - self.insert_paragraph(**data) - return True - else: - print("No new paper data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_reviews.py b/research_arcade/csv_database/csv_openreview_reviews.py deleted file mode 100644 index 1cda2e6..0000000 --- a/research_arcade/csv_database/csv_openreview_reviews.py +++ /dev/null @@ -1,249 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import ast -import os -from typing import Optional, Union - -class CSVOpenReviewReviews: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_reviews.csv" - self.crawler = OpenReviewCrawler() - - if not os.path.exists(self.csv_path): - self.create_reviews_table() - - def create_reviews_table(self) -> None: - columns = ['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - # 将content列从JSON字符串转换为字典 - if not df.empty: - df['content'] = df['content'].apply( - lambda x: json.loads(x) if pd.notna(x) and x != '' else {} - ) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - # 创建副本以避免修改原始数据 - df_to_save = df.copy() - - # 将content列从字典转换为JSON字符串 - if 'content' in df_to_save.columns: - df_to_save['content'] = df_to_save['content'].apply( - lambda x: json.dumps(x) if isinstance(x, dict) else x - ) - - df_to_save.to_csv(self.csv_path, index=False) - - def insert_review(self, venue: str, review_openreview_id: str, - replyto_openreview_id: str, title: str, writer: str, - content: dict, time: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于venue和review_openreview_id的组合键） - exists = ((df['venue'] == venue) & - (df['review_openreview_id'] == review_openreview_id)).any() - - if exists: - return None - - # 清理content - cleaned_content = self._clean_json_content(content) - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'review_openreview_id': self._clean_string(review_openreview_id), - 'replyto_openreview_id': self._clean_string(replyto_openreview_id), - 'writer': self._clean_string(writer), - 'title': self._clean_string(title), - 'content': cleaned_content, - 'time': self._clean_string(time) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, review_openreview_id) - - def delete_review_by_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['review_openreview_id'] == review_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No review found with review_openreview_id {review_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Review with review_openreview_id {review_openreview_id} deleted successfully.") - return deleted_rows - - def delete_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No reviews found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All reviews in venue {venue} deleted successfully.") - return deleted_rows - - def update_review(self, venue: str, review_openreview_id: str, - replyto_openreview_id: str, writer: str, title: str, - content: dict, time: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要更新的行 - mask = ((df['venue'] == venue) & - (df['review_openreview_id'] == review_openreview_id)) - - if not mask.any(): - print(f"No review found with review_openreview_id {review_openreview_id}.") - return None - - # 保存原始记录 - original_record = df[mask].copy() - - # 更新记录 - cleaned_content = json.dumps(self._clean_json_content(content)) - df.loc[mask, 'replyto_openreview_id'] = self._clean_string(replyto_openreview_id) - df.loc[mask, 'writer'] = self._clean_string(writer) - df.loc[mask, 'title'] = self._clean_string(title) - df.loc[mask, 'content'] = cleaned_content - df.loc[mask, 'time'] = self._clean_string(time) - - self._save_data(df) - - print(f"Review with review_openreview_id {review_openreview_id} updated successfully.") - return original_record - - def get_review_by_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['review_openreview_id'] == review_openreview_id - result = df[mask].copy() - - if result.empty: - print(f"No review found with review_openreview_id {review_openreview_id}.") - return None - - return result - - def get_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - print(f"No reviews found in venue {venue}.") - return None - - return result - - def get_all_reviews(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - if is_all_features: - return df[['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time']].copy() - else: - return df[['venue', 'review_openreview_id', 'replyto_openreview_id', - 'title', 'time']].copy() - - def check_review_exists(self, review_openreview_id: str) -> bool: - df = self._load_data() - return (df['review_openreview_id'] == review_openreview_id).any() - - def construct_reviews_table_from_api(self, venue: str) -> bool: - # 从API爬取数据 - print("Crawling review data from OpenReview API...") - review_data = self.crawler.crawl_review_data_from_api(venue) - - # 插入数据 - if len(review_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(review_data): - self.insert_review(**data) - else: - print("No new review data to insert.") - - def construct_reviews_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading review data from {csv_file}...") - import_df = pd.read_csv(csv_file) - review_data = import_df.to_dict(orient='records') - - if len(review_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(review_data): - # 确保content是字典类型 - if isinstance(data.get('content'), str): - data['content'] = ast.literal_eval(data['content']) - self.insert_review(**data) - return False - else: - print("No new review data to insert.") - return True - - def construct_reviews_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading review data from {json_file}...") - with open(json_file, 'r') as f: - review_data = json.load(f) - - if len(review_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(review_data): - self.insert_review(**data) - return True - else: - print("No new review data to insert.") - return False - - def _clean_json_content(self, content: Union[str, dict, list, int, float, bool, None]) -> Union[str, dict, list, int, float, bool, None]: - if isinstance(content, str): - return ''.join(char for char in content if char.isprintable()) # 返回 str - elif isinstance(content, dict): - return {key: self._clean_json_content(value) for key, value in content.items()} # 返回 dict - elif isinstance(content, list): - return [self._clean_json_content(item) for item in content] # 返回 list - else: - return content # 返回原始类型 - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_revisions.py b/research_arcade/csv_database/csv_openreview_revisions.py deleted file mode 100644 index 48e17ab..0000000 --- a/research_arcade/csv_database/csv_openreview_revisions.py +++ /dev/null @@ -1,247 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -import ast -from typing import Optional, Union - -class CSVOpenReviewRevisions: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_revisions.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_revisions_table() - - def create_revisions_table(self) -> None: - columns = ['venue', 'original_openreview_id', 'revision_openreview_id', - 'content', 'time'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - # 将content列从JSON字符串转换为字典 - if not df.empty: - df['content'] = df['content'].apply( - lambda x: ast.literal_eval(str(x)) if pd.notna(x) and x != '' else [] - ) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - # 创建副本以避免修改原始数据 - df_to_save = df.copy() - # 将content列从字典转换为JSON字符串 - if 'content' in df_to_save.columns: - df_to_save['content'] = df_to_save['content'].apply( - lambda x: json.dumps(x) if isinstance(x, dict) else x - ) - df_to_save.to_csv(self.csv_path, index=False) - - def insert_revision(self, venue: str, original_openreview_id: str, - revision_openreview_id: str, content: dict, - time: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于venue和revision_openreview_id的组合键） - exists = ((df['venue'] == venue) & - (df['revision_openreview_id'] == revision_openreview_id)).any() - - if exists: - return None - - # 清理content - cleaned_content = self._clean_json_content(content) - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'original_openreview_id': self._clean_string(original_openreview_id), - 'revision_openreview_id': self._clean_string(revision_openreview_id), - 'content': cleaned_content, - 'time': self._clean_string(time) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, revision_openreview_id) - - def delete_revision_by_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['revision_openreview_id'] == revision_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No revision found with revision_openreview_id {revision_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"Revision with revision_openreview_id {revision_openreview_id} deleted successfully.") - return deleted_rows - - def delete_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No revisions found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All revisions in venue {venue} deleted successfully.") - return deleted_rows - - def update_revision(self, venue: str, original_openreview_id: str, - revision_openreview_id: str, content: dict, - time: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要更新的行 - mask = ((df['venue'] == venue) & - (df['revision_openreview_id'] == revision_openreview_id)) - - if not mask.any(): - print(f"No revision found with revision_openreview_id {revision_openreview_id}.") - return None - - # 保存原始记录 - original_record = df[mask].copy() - - # 更新记录 - cleaned_content = self._clean_json_content(content) - df.loc[mask, 'original_openreview_id'] = self._clean_string(original_openreview_id) - df.loc[mask, 'content'] = str(cleaned_content) - df.loc[mask, 'time'] = self._clean_string(time) - - self._save_data(df) - - print(f"Revision with revision_openreview_id {revision_openreview_id} updated successfully.") - return original_record - - def get_revision_by_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['revision_openreview_id'] == revision_openreview_id - result = df[mask].copy() - - if result.empty: - print(f"No revision found with revision_openreview_id {revision_openreview_id}.") - return None - - return result - - def get_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - print(f"No revisions found in venue {venue}.") - return None - - return result - - def get_all_revisions(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - if is_all_features: - return df[['venue', 'original_openreview_id', 'revision_openreview_id', - 'content', 'time']].copy() - else: - return df[['venue', 'original_openreview_id', 'revision_openreview_id', - 'time']].copy() - - def check_revision_exists(self, revision_openreview_id: str) -> bool: - df = self._load_data() - return (df['revision_openreview_id'] == revision_openreview_id).any() - - def construct_revisions_table_from_api(self, venue: str, filter_list: list, - pdf_dir: str, log_file: str) -> bool: - # 从API爬取数据 - print("Crawling revision data from OpenReview API...") - revision_data = self.openreview_crawler.crawl_revision_data_from_api( - venue, filter_list, pdf_dir, log_file - ) - - # 插入数据 - if len(revision_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(revision_data): - self.insert_revision(**data) - else: - print("No new revision data to insert.") - - def construct_revisions_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading revisions data from {csv_file}...") - import_df = pd.read_csv(csv_file) - revision_data = import_df.to_dict(orient='records') - - if len(revision_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(revision_data): - # 确保content是字典类型 - if isinstance(data.get('content'), str): - data['content'] = ast.literal_eval(data['content']) - self.insert_revision(**data) - return True - else: - print("No new revision data to insert.") - return False - - def construct_revisions_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading revisions data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - revision_data = json.load(f) - - if len(revision_data) > 0: - print("Inserting data into CSV file...") - for data in tqdm(revision_data): - self.insert_revision(**data) - return True - else: - print("No new revision data to insert.") - return False - - def _clean_json_content(self, content: Union[str, dict, list, int, float, bool, None]) -> Union[str, dict, list, int, float, bool, None]: - if isinstance(content, str): - return ''.join(char for char in content if char.isprintable()) # 返回 str - elif isinstance(content, dict): - return {key: self._clean_json_content(value) for key, value in content.items()} # 返回 dict - elif isinstance(content, list): - return [self._clean_json_content(item) for item in content] # 返回 list - else: - return content # 返回原始类型 - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/csv_database/csv_openreview_revisions_reviews.py b/research_arcade/csv_database/csv_openreview_revisions_reviews.py deleted file mode 100644 index 6170d02..0000000 --- a/research_arcade/csv_database/csv_openreview_revisions_reviews.py +++ /dev/null @@ -1,276 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -from typing import Optional - -class CSVOpenReviewRevisionsReviews: - def __init__(self, csv_dir: str) -> None: - self.csv_path = csv_dir + "openreview_revisions_reviews.csv" - self.openreview_crawler = OpenReviewCrawler() - - # 如果CSV文件不存在，创建空的DataFrame - if not os.path.exists(self.csv_path): - self.create_revisions_reviews_table() - - def create_revisions_reviews_table(self) -> None: - columns = ['venue', 'revision_openreview_id', 'review_openreview_id'] - empty_df = pd.DataFrame(columns=columns) - empty_df.to_csv(self.csv_path, index=False) - print(f"Created empty CSV file at {self.csv_path}") - - def _load_data(self) -> Optional[pd.DataFrame]: - if os.path.exists(self.csv_path): - df = pd.read_csv(self.csv_path) - return df - else: - return None - - def _save_data(self, df: pd.DataFrame) -> None: - df.to_csv(self.csv_path, index=False) - - def insert_revision_reviews(self, venue: str, revision_openreview_id: str, - review_openreview_id: str) -> Optional[tuple]: - df = self._load_data() - - # 检查是否已存在（基于三个字段的组合键） - exists = ((df['venue'] == venue) & - (df['revision_openreview_id'] == revision_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)).any() - - if exists: - return None - - # 创建新行 - new_row = pd.DataFrame([{ - 'venue': self._clean_string(venue), - 'revision_openreview_id': self._clean_string(revision_openreview_id), - 'review_openreview_id': self._clean_string(review_openreview_id) - }]) - - # 添加到DataFrame并保存 - df = pd.concat([df, new_row], ignore_index=True) - self._save_data(df) - - return (venue, revision_openreview_id, review_openreview_id) - - def get_revision_review_by_id(self, revision_openreview_id: str, - review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = ((df['revision_openreview_id'] == revision_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)) - result = df[mask].copy() - - if result.empty: - print(f"The revision {revision_openreview_id} and the review {review_openreview_id} are not connected in this database.") - return None - - return result - - def get_revisions_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['venue'] == venue - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_all_revisions_reviews(self) -> Optional[pd.DataFrame]: - df = self._load_data() - - if df.empty: - return None - - return df.copy() - - def delete_revision_review_by_id(self, revision_openreview_id: str, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = ((df['revision_openreview_id'] == revision_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)) - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found between revision {revision_openreview_id} and review {review_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection between revision {revision_openreview_id} and review {review_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_revision_review_by_revision_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['revision_openreview_id'] == revision_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found for revision {revision_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection for revision {revision_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_revision_review_by_review_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['review_openreview_id'] == review_openreview_id - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connection found for review {review_openreview_id}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"The connection for review {review_openreview_id} is deleted successfully.") - return deleted_rows - - def delete_revisions_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - # 查找要删除的行 - mask = df['venue'] == venue - deleted_rows = df[mask].copy() - - if deleted_rows.empty: - print(f"No connections found in venue {venue}.") - return None - - # 删除行 - df = df[~mask] - self._save_data(df) - - print(f"All connections in venue {venue} deleted successfully.") - return deleted_rows - - def check_revision_review_exists(self, revision_openreview_id: str, - review_openreview_id: str) -> bool: - df = self._load_data() - exists = ((df['revision_openreview_id'] == revision_openreview_id) & - (df['review_openreview_id'] == review_openreview_id)).any() - return exists - - def get_revision_neighboring_reviews(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['revision_openreview_id'] == revision_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def get_review_neighboring_revisions(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - df = self._load_data() - - mask = df['review_openreview_id'] == review_openreview_id - result = df[mask].copy() - - if result.empty: - return None - - return result - - def construct_revisions_reviews_table(self, papers_reviews_df: pd.DataFrame, - papers_revisions_df: pd.DataFrame) -> bool: - # 获取唯一的论文ID - unique_paper_ids = papers_revisions_df['paper_openreview_id'].unique() - - print(f"Constructing revisions-reviews connections for {len(unique_paper_ids)} papers...") - - for paper_id in tqdm(unique_paper_ids): - # 获取该论文的所有修订（按时间排序） - paper_revision_edges = papers_revisions_df[ - papers_revisions_df['paper_openreview_id'] == paper_id - ].sort_values(by='time', ascending=True) - - # 获取该论文的所有评审（按时间排序） - paper_review_edges = papers_reviews_df[ - papers_reviews_df['paper_openreview_id'] == paper_id - ].sort_values(by='time', ascending=True) - - if paper_review_edges.empty: - continue - - # 为每个修订关联在其之前的所有评审 - start_idx = 0 - for _, revision in paper_revision_edges.iterrows(): - revision_time = revision['time'] - revision_id = revision['revision_openreview_id'] - venue = revision['venue'] - - # 遍历所有在当前修订时间之前的评审 - for idx, review in paper_review_edges.iloc[start_idx:].iterrows(): - review_time = review['time'] - - # 如果评审时间晚于修订时间，停止 - if review_time > revision_time: - break - - review_id = review['review_openreview_id'] - - # 插入修订-评审关联 - self.insert_revision_reviews(venue, revision_id, review_id) - - start_idx += 1 - - print("Revisions-reviews table construction completed.") - return True - - def construct_revisions_reviews_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - print(f"Reading revisions-reviews data from {csv_file}...") - import_df = pd.read_csv(csv_file) - revisions_reviews_data = import_df.to_dict(orient='records') - - if len(revisions_reviews_data) > 0: - print(f"Inserting revisions-reviews data from {csv_file}...") - for data in tqdm(revisions_reviews_data): - self.insert_revision_reviews(**data) - return True - else: - print(f"No revisions-reviews data found in {csv_file}.") - return False - - def construct_revisions_reviews_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - print(f"Reading revisions-reviews data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - revisions_reviews_data = json.load(f) - - if len(revisions_reviews_data) > 0: - print(f"Inserting revisions-reviews data from {json_file}...") - for data in tqdm(revisions_reviews_data): - self.insert_revision_reviews(**data) - return True - else: - print(f"No revisions-reviews data found in {json_file}.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/openreview_utils/__init__.py b/research_arcade/openreview_utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research_arcade/openreview_utils/openreview_crawler.py b/research_arcade/openreview_utils/openreview_crawler.py deleted file mode 100644 index e00c9d3..0000000 --- a/research_arcade/openreview_utils/openreview_crawler.py +++ /dev/null @@ -1,1064 +0,0 @@ -import openreview -import arxiv -from arxiv import UnexpectedEmptyPageError -import os -import re -import time -from tqdm import tqdm -from datetime import datetime -from typing import Optional -from .pdf_utils import extract_paragraphs_from_pdf_new, connect_diffs_and_paragraphs - -class OpenReviewCrawler: - def __init__(self) -> None: - self.client_v1 = openreview.Client( - baseurl='https://api.openreview.net' - ) - - self.client_v2 = openreview.api.OpenReviewClient( - baseurl='https://api2.openreview.net' - ) - - def crawl_paper_data_from_api(self, venue: str) -> list[dict]: - paper_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper decision - decision = None - if "2019" in venue: - reviews = submission.details["replies"] - for review in reviews: - if "recommendation" in review['content']: - decision = review['content']['recommendation'] - break - elif "2018" in venue or "2020" in venue or "2021" or "2022" in venue or "2023" in venue: - reviews = submission.details["replies"] - for review in reviews: - if "decision" in review['content']: - decision = review['content']['decision'] - break - # get paper openreview id - paper_id = submission.id - # get title - title = submission.content["title"] - # get abstract - abstract = submission.content["abstract"] - # get paper's pdf - pdf = submission.content["pdf"] - paper_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "title": title, - "abstract": abstract, - "paper_pdf_link": pdf, - "paper_decision": decision - }) - return paper_data - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper decision - decision = None - if "2017" in venue: - reviews = submission.details["replies"] - for review in reviews: - if "decision" in review['content']: - decision = review['content']['decision'] - break - # get paper openreview id - paper_id = submission.id - # get title - title = submission.content["title"] - # get abstract - abstract = submission.content["abstract"] - # get paper's pdf - pdf = submission.content["pdf"] - paper_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "title": title, - "abstract": abstract, - "paper_pdf_link": pdf, - "paper_decision": decision - }) - return paper_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission') - if submissions is not None: - for submission in tqdm(submissions): - # get paper decision and remove withdrawn papers - decision = submission.content["venueid"]["value"].split('/')[-1] - if decision == "Withdrawn_Submission": - continue - else: - if decision == "Conference": - decision = submission.content["venue"]["value"] - # get paper openreview id - paper_id = submission.id - # get title - title = submission.content["title"]["value"] - # get abstract - abstract = submission.content["abstract"]["value"] - # get paper's pdf - if "pdf" in submission.content: - pdf = submission.content["pdf"]["value"] - else: - pdf = "" - paper_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "title": title, - "abstract": abstract, - "paper_pdf_link": pdf, - "paper_decision": decision - }) - return paper_data - else: - print(f"No submissions found for venue: {venue}") - return [] - - def crawl_author_data_from_api(self, venue: str) -> list[dict]: - author_data = [] - author_set = set() - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - author_ids = submission.content["authorids"] - author_set.update(author_ids) - # get profiles - author_profiles = openreview.tools.get_profiles(self.client_v1, author_set) - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - if "authorids" in submission.content: - author_ids = submission.content["authorids"] - author_set.update(author_ids) - # get profiles - author_profiles = openreview.tools.get_profiles(self.client_v1, author_set) - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - author_ids = submission.content["authorids"]["value"] - author_set.update(author_ids) - # get profiles - author_profiles = openreview.tools.get_profiles(self.client_v2, author_set) - for profile in tqdm(author_profiles): - # get author fullname and author openreview id - all_names = profile.content["names"] - author_id = "" - fullname = "" - for name in all_names: - if name.get("preferred") is not None: - if name["preferred"]: - author_id = name["username"] - fullname = name["fullname"] - break - else: - if name.get("username") is not None: - author_id = name["username"] - try: - fullname = name["fullname"] - break - except Exception: - fullname = author_id - else: - pass - if author_id == "": # remove the author with no username - author_id = profile.id - # get email - try: - email = profile.content["preferredEmail"] - except Exception: - try: - email = profile.content["emailsConfirmed"][0] - except Exception: - try: - email = profile.content["emails"][0] - except Exception: - email = "" - # get affiliation - try: - affiliation = profile.content["history"][0]["institution"]["name"] - except Exception: - affiliation = "" - # get homepage - if profile.content.get("homepage") is not None: - homepage = profile.content["homepage"] - else: - homepage = "" - # get dblp - if profile.content.get("dblp") is not None: - dblp = profile.content["dblp"] - else: - dblp = "" - - author_data.append({ - "venue": venue, - "author_openreview_id": author_id, - "author_full_name": fullname, - "email": email, - "affiliation": affiliation, - "homepage": homepage, - "dblp": dblp - }) - return author_data - - def crawl_review_data_from_api(self, venue: str) -> list[dict]: - review_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - reviews = submission.details["replies"] - for review in reviews: - # get review openreview id - reply_id = review["id"] - # get replyto openreview id - replyto_id = review["replyto"] - # get time - time = datetime.fromtimestamp(review['tmdate'] / 1000).strftime("%Y-%m-%d %H:%M:%S") - # get writer id - writer = review["signatures"][0].split('/')[-1] - # get title - if "summary_of_the_paper" in review["content"] or "rating" in review["content"]: - title = "Official Review by " + writer - elif "decision" in review["content"]: - title = "Paper Decision" - else: - if "reviewer" in review["signatures"][0].split('/')[-1].lower(): - title = "Response by " + writer - else: - title = "Response by Authors" - # get content - content = review["content"] - review_data.append({ - "venue": venue, - "review_openreview_id": reply_id, - "replyto_openreview_id": replyto_id, - "title": title, - "writer": writer, - "content": content, - "time": time - }) - return review_data - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - reviews = submission.details["replies"] - for review in reviews: - # get review openreview id - reply_id = review["id"] - # get replyto openreview id - replyto_id = review["replyto"] - # get time - time = datetime.fromtimestamp(review['tmdate'] / 1000).strftime("%Y-%m-%d %H:%M:%S") - # get writer id - writer = review["signatures"][0].split('/')[-1] - # get title - if "rating" in review["content"]: - title = "Official Review by " + writer - elif "decision" in review["content"]: - title = "Paper Decision" - else: - if "reviewer" in review["signatures"][0].split('/')[-1].lower(): - title = "Response by " + writer - else: - title = "Response by Authors" - # get content - content = review["content"] - review_data.append({ - "venue": venue, - "review_openreview_id": reply_id, - "replyto_openreview_id": replyto_id, - "title": title, - "writer": writer, - "content": content, - "time": time - }) - return review_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - if submission.content["venueid"]["value"].split('/')[-1] == "Withdrawn_Submission": - continue - else: - reviews = submission.details["replies"] - for review in reviews: - # get review openreview id - reply_id = review["id"] - # get replyto openreview id - replyto_id = review["replyto"] - # get time - time = datetime.fromtimestamp(review['tmdate'] / 1000).strftime("%Y-%m-%d %H:%M:%S") - # get writer id - writer = review["signatures"][0].split('/')[-1].lower() - # get title - if "summary" in review["content"]: - title = "Official Review by " + writer - elif "metareview" in review["content"]: - title = "Meta Review by " + writer - elif "decision" in review["content"]: - title = "Paper Decision" - else: - if "reviewer" in review["signatures"][0].split('/')[-1].lower(): - title = "Response by " + writer - else: - title = "Response by Authors" - # get content - content = review["content"] - review_data.append({ - "venue": venue, - "review_openreview_id": reply_id, - "replyto_openreview_id": replyto_id, - "title": title, - "writer": writer, - "content": content, - "time": time - }) - return review_data - - def crawl_revision_data_from_api(self, venue: str, filter_list: list, pdf_dir: str, log_file: str) -> list[dict]: - import time - revision_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='revisions') - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper openreview id - paper_id = submission.id - # get revisions and their time - revisions = {} - # get revisions and their time - note_edits = self.client_v1.get_references(referent=paper_id, original=True) - time.sleep(1) - - filtered_notes = [] - for note in note_edits: - if "pdf" in note.content: - filtered_notes.append(note) - - if len(filtered_notes) <= 1: - continue - else: - for note in filtered_notes: - revisions[note.id] = { - "Time": datetime.fromtimestamp(note.tmdate / 1000).strftime("%Y-%m-%d %H:%M:%S"), - "Title": "Paper Revision" - } - # sorted by time - sorted_revisions = sorted(revisions.items(), key=lambda x: datetime.strptime(x[1]["Time"], "%Y-%m-%d %H:%M:%S")) - num_revision = len(sorted_revisions) - - if num_revision <= 1: - continue - else: - original_id = None - modified_id = None - - for idx, revision in enumerate(sorted_revisions): - original_id = modified_id - modified_id = revision[0] - original_pdf = str(pdf_dir)+str(original_id)+".pdf" - modified_pdf = str(pdf_dir)+str(modified_id)+".pdf" - - if not os.path.exists(original_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {original_id} Failed\n") - if not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - elif not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - - if idx < 1: - continue - else: - date = revision[1]["Time"] - try: - content = connect_diffs_and_paragraphs(original_pdf, modified_pdf, filter_list) - revision_data.append({ - "venue": venue, - "original_openreview_id": original_id, - "revision_openreview_id": modified_id, - "content": content, - "time": date - }) - except Exception: - continue - return revision_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper decision and remove withdrawn papers - decision = submission.content["venueid"]["value"].split('/')[-1] - if decision == "Withdrawn_Submission": - continue - else: - # get paper openreview id - paper_id = submission.id - # get revisions and their time - revisions = {} - # all_diffs = [] - note_edits = self.client_v2.get_note_edits(note_id=paper_id) - time.sleep(1) - - if len(note_edits) <= 1: - continue - else: - for note in note_edits: - revisions[note.id] = { - "Time": datetime.fromtimestamp(note.tmdate / 1000).strftime("%Y-%m-%d %H:%M:%S"), - "Title": note.invitation.split('/')[-1] - } - # sorted by time - sorted_revisions = sorted(revisions.items(), key=lambda x: datetime.strptime(x[1]["Time"], "%Y-%m-%d %H:%M:%S")) - num_revision = len(sorted_revisions) - - if num_revision <= 1: - continue - else: - original_id = None - modified_id = None - - for idx, revision in enumerate(sorted_revisions): - original_id = modified_id - modified_id = revision[0] - original_pdf = str(pdf_dir)+str(original_id)+".pdf" - modified_pdf = str(pdf_dir)+str(modified_id)+".pdf" - - if not os.path.exists(original_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {original_id} Failed\n") - if not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - elif not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - - if idx > 1: - time = revision[1]["Time"] - try: - content = connect_diffs_and_paragraphs(original_pdf, modified_pdf, filter_list) - except Exception: - continue - revision_data.append({ - "venue": venue, - "original_openreview_id": original_id, - "revision_openreview_id": modified_id, - "content": content, - "time": time - }) - return revision_data - - def crawl_paragraph_data_from_api(self, venue: str, pdf_dir: str, filter_list: list, log_file: str, - is_paper = True, is_revision = True, is_pdf_delete: bool = True) -> list[dict]: - # TODO: pdf_path is undefined - pdf_path = None - # TODO: original_id is undefined - original_id = None - paragraph_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='revisions') - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions[0:5]): - # get paper openreview id - paper_id = submission.id - if is_paper: - paper_pdf = str(pdf_dir)+str(paper_id)+".pdf" - if not os.path.exists(paper_pdf): - with open(log_file, "a") as log: - log.write(f"Paper {original_id} Failed\n") - else: - try: - structured_content = extract_paragraphs_from_pdf_new(paper_pdf, filter_list) - paragraph_counter = 1 - for section, paragraphs in structured_content.items(): - for paragraph in paragraphs: - paragraph_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "paragraph_idx": paragraph_counter, - "section": section, - "content": paragraph - }) - paragraph_counter += 1 - if is_pdf_delete: - os.remove(pdf_path) - print(f"Deleted PDF file: {pdf_path}") - except Exception: - with open(log_file, "a") as log: - log.write(f"PDF {pdf_path} Failed\n") - if is_revision: - # get revisions and their time - revisions = {} - # get revisions and their time - note_edits = self.client_v1.get_references(referent=paper_id, original=True) - time.sleep(1) - - filtered_notes = [] - for note in note_edits: - if "pdf" in note.content: - filtered_notes.append(note) - - if len(filtered_notes) <= 1: - continue - else: - for note in filtered_notes: - revisions[note.id] = { - "Time": datetime.fromtimestamp(note.tmdate / 1000).strftime("%Y-%m-%d %H:%M:%S"), - "Title": "Paper Revision" - } - # sorted by time - sorted_revisions = sorted(revisions.items(), key=lambda x: datetime.strptime(x[1]["Time"], "%Y-%m-%d %H:%M:%S")) - num_revision = len(sorted_revisions) - - if num_revision <= 1: - continue - else: - original_id = None - modified_id = None - - for idx, revision in enumerate(sorted_revisions): - original_id = modified_id - modified_id = revision[0] - original_pdf = str(pdf_dir)+str(original_id)+".pdf" - modified_pdf = str(pdf_dir)+str(modified_id)+".pdf" - - if not os.path.exists(original_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {original_id} Failed\n") - if not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - elif not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - - if idx < 1: - continue - else: - try: - structured_content = extract_paragraphs_from_pdf_new(original_pdf, filter_list) - paragraph_counter = 1 - for section, paragraphs in structured_content.items(): - for paragraph in paragraphs: - paragraph_data.append({ - "venue": venue, - "paper_openreview_id": original_id, - "paragraph_idx": paragraph_counter, - "section": section, - "content": paragraph - }) - paragraph_counter += 1 - if is_pdf_delete: - os.remove(pdf_path) - print(f"Deleted PDF file: {pdf_path}") - except Exception: - with open(log_file, "a") as log: - log.write(f"PDF {pdf_path} Failed\n") - if idx == num_revision - 1: - try: - structured_content = extract_paragraphs_from_pdf_new(modified_pdf, filter_list) - paragraph_counter = 1 - for section, paragraphs in structured_content.items(): - for paragraph in paragraphs: - paragraph_data.append({ - "venue": venue, - "paper_openreview_id": modified_id, - "paragraph_idx": paragraph_counter, - "section": section, - "content": paragraph - }) - paragraph_counter += 1 - if is_pdf_delete: - os.remove(pdf_path) - print(f"Deleted PDF file: {pdf_path}") - except Exception: - with open(log_file, "a") as log: - log.write(f"PDF {pdf_path} Failed\n") - return paragraph_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions[0:5]): - # get paper decision and remove withdrawn papers - decision = submission.content["venueid"]["value"].split('/')[-1] - if decision == "Withdrawn_Submission": - continue - else: - # get paper openreview id - paper_id = submission.id - if is_paper: - paper_pdf = str(pdf_dir)+str(paper_id)+".pdf" - if not os.path.exists(paper_pdf): - with open(log_file, "a") as log: - log.write(f"Paper {original_id} Failed\n") - else: - try: - structured_content = extract_paragraphs_from_pdf_new(paper_pdf, filter_list) - paragraph_counter = 1 - for section, paragraphs in structured_content.items(): - for paragraph in paragraphs: - paragraph_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "paragraph_idx": paragraph_counter, - "section": section, - "content": paragraph - }) - paragraph_counter += 1 - if is_pdf_delete: - os.remove(pdf_path) - print(f"Deleted PDF file: {pdf_path}") - except Exception: - with open(log_file, "a") as log: - log.write(f"PDF {pdf_path} Failed\n") - - if is_revision: - # get revisions and their time - revisions = {} - # all_diffs = [] - note_edits = self.client_v2.get_note_edits(note_id=paper_id) - time.sleep(1) - - if len(note_edits) <= 1: - continue - else: - for note in note_edits: - revisions[note.id] = { - "Time": datetime.fromtimestamp(note.tmdate / 1000).strftime("%Y-%m-%d %H:%M:%S"), - "Title": note.invitation.split('/')[-1] - } - # sorted by time - sorted_revisions = sorted(revisions.items(), key=lambda x: datetime.strptime(x[1]["Time"], "%Y-%m-%d %H:%M:%S")) - num_revision = len(sorted_revisions) - - if num_revision <= 1: - continue - else: - original_id = None - modified_id = None - - for idx, revision in enumerate(sorted_revisions): - original_id = modified_id - modified_id = revision[0] - original_pdf = str(pdf_dir)+str(original_id)+".pdf" - modified_pdf = str(pdf_dir)+str(modified_id)+".pdf" - - if not os.path.exists(original_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {original_id} Failed\n") - if not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - elif not os.path.exists(modified_pdf): - with open(log_file, "a") as log: - log.write(f"Revision {modified_id} Failed\n") - continue - - if idx > 1: - try: - structured_content = extract_paragraphs_from_pdf_new(original_pdf, filter_list) - paragraph_counter = 1 - for section, paragraphs in structured_content.items(): - for paragraph in paragraphs: - paragraph_data.append({ - "venue": venue, - "paper_openreview_id": original_id, - "paragraph_idx": paragraph_counter, - "section": section, - "content": paragraph - }) - paragraph_counter += 1 - if is_pdf_delete: - os.remove(pdf_path) - print(f"Deleted PDF file: {pdf_path}") - except Exception: - with open(log_file, "a") as log: - log.write(f"PDF {pdf_path} Failed\n") - if idx == num_revision - 1: - try: - structured_content = extract_paragraphs_from_pdf_new(modified_pdf, filter_list) - paragraph_counter = 1 - for section, paragraphs in structured_content.items(): - for paragraph in paragraphs: - paragraph_data.append({ - "venue": venue, - "paper_openreview_id": modified_id, - "paragraph_idx": paragraph_counter, - "section": section, - "content": paragraph - }) - paragraph_counter += 1 - if is_pdf_delete: - os.remove(pdf_path) - print(f"Deleted PDF file: {pdf_path}") - except Exception: - with open(log_file, "a") as log: - log.write(f"PDF {pdf_path} Failed\n") - return paragraph_data - - def crawl_papers_authors_data_from_api(self, venue: str) -> list[dict]: - papers_authors_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='replies') - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper openreview id - paper_id = submission.id - # get author openreview ids - if "authorids" in submission.content: - author_ids = set(submission.content["authorids"]) - for author_id in author_ids: - papers_authors_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "author_openreview_id": author_id - }) - return papers_authors_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission') - if submissions is not None: - for submission in tqdm(submissions): - # get paper openreview id - paper_id = submission.id - # get author openreview ids - author_ids = set(submission.content["authorids"]["value"]) - for author_id in author_ids: - papers_authors_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "author_openreview_id": author_id - }) - return papers_authors_data - else: - print(f"No submissions found for venue: {venue}") - return [] - - def crawl_papers_revisions_data_from_api(self, venue: str) -> list[dict]: - import time - papers_revisions_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='revisions') - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper openreview id - paper_id = submission.id - # get revisions and their time - # revisions = {} - # get revisions and their time - note_edits = self.client_v1.get_references(referent=paper_id, original=True) - time.sleep(1) - - filtered_notes = [] - for note in note_edits: - if "pdf" in note.content: - filtered_notes.append(note) - - if len(filtered_notes) <= 1: - continue - else: - for revision in filtered_notes: - revision_openreview_id = revision.id - title = "Paper Revision" - date = datetime.fromtimestamp(revision.tmdate / 1000).strftime("%Y-%m-%d %H:%M:%S") - papers_revisions_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "revision_openreview_id": revision_openreview_id, - "title": title, - "time": date - }) - return papers_revisions_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='revisions') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper decision and remove withdrawn papers - decision = submission.content["venueid"]["value"].split('/')[-1] - if decision == "Withdrawn_Submission": - continue - else: - # get paper openreview id - paper_id = submission.id - # get revisions and their time - # revisions = {} - # all_diffs = [] - note_edits = self.client_v2.get_note_edits(note_id=paper_id) - time.sleep(1) - - if len(note_edits) <= 1: - continue - else: - for note in note_edits: - revision_openreview_id = note.id - title = note.invitation.split('/')[-1] - date = datetime.fromtimestamp(note.tmdate / 1000).strftime("%Y-%m-%d %H:%M:%S") - papers_revisions_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "revision_openreview_id": revision_openreview_id, - "title": title, - "time": date - }) - return papers_revisions_data - - def crawl_papers_reviews_from_api(self, venue: str) -> list[dict]: - papers_reviews_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - reviews = submission.details["replies"] - paper_id = submission.id - for review in reviews: - # get review openreview id - reply_id = review["id"] - # get time - time = datetime.fromtimestamp(review['tmdate'] / 1000).strftime("%Y-%m-%d %H:%M:%S") - # get writer id - writer = review["signatures"][0].split('/')[-1] - # get title - if "summary_of_the_paper" in review["content"] or "rating" in review["content"]: - title = "Official Review by " + writer - elif "decision" in review["content"]: - title = "Paper Decision" - else: - if "reviewer" in review["signatures"][0].split('/')[-1].lower(): - title = "Response by " + writer - else: - title = "Response by Authors" - papers_reviews_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "review_openreview_id": reply_id, - "title": title, - "time": time - }) - return papers_reviews_data - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - reviews = submission.details["replies"] - paper_id = submission.id - for review in reviews: - # get review openreview id - reply_id = review["id"] - # get time - time = datetime.fromtimestamp(review['tmdate'] / 1000).strftime("%Y-%m-%d %H:%M:%S") - # get writer id - writer = review["signatures"][0].split('/')[-1] - # get title - if "rating" in review["content"]: - title = "Official Review by " + writer - elif "decision" in review["content"]: - title = "Paper Decision" - else: - if "reviewer" in review["signatures"][0].split('/')[-1].lower(): - title = "Response by " + writer - else: - title = "Response by Authors" - papers_reviews_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "review_openreview_id": reply_id, - "title": title, - "time": time - }) - return papers_reviews_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission', details='replies') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - if submission.content["venueid"]["value"].split('/')[-1] == "Withdrawn_Submission": - continue - else: - reviews = submission.details["replies"] - paper_id = submission.id - for review in reviews: - # get review openreview id - reply_id = review["id"] - # get time - time = datetime.fromtimestamp(review['tmdate'] / 1000).strftime("%Y-%m-%d %H:%M:%S") - # get writer id - writer = review["signatures"][0].split('/')[-1].lower() - # get title - if "summary" in review["content"]: - title = "Official Review by " + writer - elif "metareview" in review["content"]: - title = "Meta Review by " + writer - elif "decision" in review["content"]: - title = "Paper Decision" - else: - if "reviewer" in review["signatures"][0].split('/')[-1].lower(): - title = "Response by " + writer - else: - title = "Response by Authors" - papers_reviews_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "review_openreview_id": reply_id, - "title": title, - "time": time - }) - return papers_reviews_data - - def crawl_openreview_arxiv_data_from_api(self, venue: str) -> list[dict]: - openreview_arxiv_data = [] - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue or "2017" in venue or "2014" in venue or "2013" in venue: - if "2023" in venue or "2022" in venue or "2021" in venue or "2020" in venue or "2019" in venue or "2018" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/Blind_Submission') - elif "2017" in venue or "2014" in venue or "2013" in venue: - submissions = self.client_v1.get_all_notes(invitation=f'{venue}/-/submission') - if submissions is None: - print(f"No submissions found for venue: {venue}") - return [] - else: - for submission in tqdm(submissions): - # get paper openreview id - openreview_id = submission.id - # get title - title = submission.content["title"] - # get arxiv id if exists - arxiv_id = self._search_title_with_name(title) - openreview_arxiv_data.append({ - "venue": venue, - "paper_openreview_id": openreview_id, - "arxiv_id": arxiv_id, - "title": title - }) - return openreview_arxiv_data - else: - submissions = self.client_v2.get_all_notes(invitation=f'{venue}/-/Submission') - if submissions is not None: - for submission in tqdm(submissions): - # get paper decision and remove withdrawn papers - decision = submission.content["venueid"]["value"].split('/')[-1] - if decision == "Withdrawn_Submission": - continue - else: - if decision == "Conference": - decision = submission.content["venue"]["value"] - # get paper openreview id - paper_id = submission.id - # get title - title = submission.content["title"]["value"] - # get arxiv id if exists - arxiv_id = self._search_title_with_name(title) - openreview_arxiv_data.append({ - "venue": venue, - "paper_openreview_id": paper_id, - "arxiv_id": arxiv_id, - "title": title - }) - return openreview_arxiv_data - else: - print(f"No submissions found for venue: {venue}") - return [] - - def _title_cleaner(self, title: str) -> str: - # Remove anything that isn't a letter, number, or whitespace - cleaned = re.sub(r'[^A-Za-z0-9\s]', '', title) - # Collapse multiple spaces and strip leading/trailing spaces - cleaned = re.sub(r'\s+', ' ', cleaned).strip() - return cleaned.strip().lower() - - def _search_title_with_name(self, title, max_result=5) -> Optional[str]: - query = f"ti:{title}" - search = arxiv.Search( - query=query, - max_results=max_result, - sort_by=arxiv.SortCriterion.Relevance, - ) - - try: - for result in search.results(): - if (self._title_cleaner(result.title) == self._title_cleaner(title)): - return result.entry_id - except UnexpectedEmptyPageError: - return None \ No newline at end of file diff --git a/research_arcade/openreview_utils/pdf_utils.py b/research_arcade/openreview_utils/pdf_utils.py deleted file mode 100644 index 0e5b0db..0000000 --- a/research_arcade/openreview_utils/pdf_utils.py +++ /dev/null @@ -1,401 +0,0 @@ -import re -import os -import difflib -import requests -from tqdm import tqdm -from typing import List, Optional -from pathlib import Path -from pdfminer.high_level import extract_text - -# get pdf based on openreview_id -def get_pdf_by_id(id, pdf_name: str) -> bool: - # pdf url - pdf_url = "https://openreview.net/notes/edits/attachment?id="+id+"&name=pdf" - - headers = { - "User-Agent": "Mozilla/5.0" - } - response = requests.get(pdf_url, headers=headers) - if response.status_code == 200: - with open(pdf_name, "wb") as f: - f.write(response.content) - print("✅ PDF is downloaded as "+pdf_name) - return True - else: - print("❌ Failure, Status Code: ", response.status_code) - return False - -# extract text from pdf -def extract_text_from_pdf(pdf_path) -> Optional[str]: - if os.path.isfile(pdf_path): - return extract_text(pdf_path) - else: - return None - -# compare the differences between two pdfs -def compare_texts(text1, text2) -> str: - diff = difflib.unified_diff( - text1.splitlines(), - text2.splitlines(), - fromfile='Original', - tofile='Modified', - lineterm='' - ) - return '\n'.join(diff) - -# format the differences -def parse_diff(diff_text) -> list[dict]: - lines = diff_text.splitlines() - - all_diff = [] - current_diff = None - for line in tqdm(lines[2:]): - # Check for diff change markers - if line.startswith('@@'): - if current_diff is not None: - # Add the previous diff to the corresponding list - all_diff.append(current_diff) - # Start a new diff block - current_diff = { - 'context_before': "", - 'context_after': "", - 'original_lines': "", - 'modified_lines': "", - } - elif line.startswith('-'): - current_diff['original_lines'] = current_diff['original_lines'] + line[1:].strip() + " " - elif line.startswith('+'): - current_diff['modified_lines'] = current_diff['modified_lines'] + line[1:].strip() + " " - elif line.strip() != "" and (current_diff['original_lines'] == "" and current_diff['modified_lines'] == ""): - current_diff['context_before'] = current_diff['context_before'] + line.strip() + " " - elif line.strip() != "" and (current_diff['original_lines'] != "" or current_diff['modified_lines'] != ""): - current_diff['context_after'] = current_diff['context_after'] + line.strip() + " " - - return all_diff - -# more than 3 math-related symbols and less than 10 characters -def check_str_regex(s: str) -> bool: - math_symbol_pattern = r'[0-9+\-*/=]' - math_count = len(re.findall(math_symbol_pattern, s)) - letter_count = len(re.findall(r'[A-Za-z]', s)) - return (math_count >= 3) and (letter_count < 10) - -# primarily format lines into paragraphs -def preprocess_lines_in_paragraphs(lines: list) -> list: - formatted_lines = [] - buffer = [] - for line in lines: - if line.strip(): # Non-empty -> the same paragraph - if line[-1] == '-': - buffer.append(line[:-1]) - else: - buffer.append(line) - else: # Empty -> Next paragraph - if buffer: # Combine the content in buffer - formatted_lines.append("".join(buffer)) - buffer = [] # Clean buffer - if buffer: - formatted_lines.append("".join(buffer)) - - return formatted_lines - -# finally format pdf into paragraphs -def extract_paragraphs_from_pdf_new(pdf_path: Path, filter_list: Optional[List[str]] = None) -> dict: - # extract all the text from pdf - full_text = extract_text(pdf_path) - - # split the text into lines - lines = full_text.splitlines() - - # construct paragraphs based on the empty lines - formatted_lines = preprocess_lines_in_paragraphs(lines) - - # only extract paragraphs between abstract and appendix - start = 0 - try: - start = formatted_lines.index("Abstract") - except Exception: - try: - start = formatted_lines.index("ABSTRACT") - except Exception: - print("can not find abstract") - end = len(formatted_lines) - try: - end = formatted_lines.index("References") - except Exception: - try: - end = formatted_lines.index("REFERENCES") - except Exception: - print("can not find reference") - - # the structured content and insert the title - structured_content = { - "Title": [formatted_lines[1]], - } - - # start constructing the structured content - before_context = "" - current_section_idx = 0 - current_subsection_idx = 0 - # before_section = "" - current_section = "" - current_image_table = [] - is_chapter = False - num_paragraph = 0 - num_image_table = 0 - for line in formatted_lines[start:end]: - if line == "": - continue - if check_str_regex(line): # no more than 3 digits or at least 10 characters - continue - - # check if before_context contains invalid content - is_filter = False - if filter_list is not None: - for text in filter_list: - if text in before_context: - is_filter = True - break - - # before_context add into formatted context - if is_filter: - pass - elif is_chapter: - num_image_table = 0 - num_paragraph = -1 - is_chapter = False - elif before_context.startswith("Figure") or before_context.startswith("Table"): - current_image_table.append(before_context) - elif before_context != "": - if not before_context.isdigit() and before_context != "": # get rid of pure digit - if len(structured_content[current_section]) == 0: - num_image_table = 0 - num_paragraph += 1 - structured_content[current_section].append(before_context) - else: - char_end = structured_content[current_section][num_paragraph-num_image_table][-1] - - is_append = True - if char_end != ".": - is_append = False - elif char_end == "." and (before_context[0].isdigit() and before_context[1] == "."): # 1. 2. - is_append = False - elif char_end == "." and (before_context[0] == "•"): # • - is_append = False - elif char_end == "." and (before_context[0].isdigit() and before_context[1] == ")"): # 1) 2) - is_append = False - elif char_end == "." and before_context[0] == "(": # (1), (information) - is_append = False - - if is_append: - num_image_table = 0 - num_paragraph += 1 - structured_content[current_section].append(before_context) - if len(current_image_table) != 0: - num_image_table = len(current_image_table) - structured_content[current_section].extend(current_image_table) - current_image_table = [] - else: - structured_content[current_section][num_paragraph-num_image_table] = structured_content[current_section][num_paragraph-num_image_table] + " " + before_context - - # abstract - if line == "Abstract" or line == "ABSTRACT": - is_chapter = True - current_section = "Abstract" - structured_content[current_section] = [] - # reference - # if line == "REFERENCES" or line == "References": - # is_chapter = True - # # before_section = current_section - # current_section = "References" - # structured_content[current_section] = [] - # chapter - if before_context.isdigit() and len(line) <= 20: - is_chapter = True - # before_section = current_section - current_section = before_context+" "+line - structured_content[current_section] = [] - current_section_idx = current_section_idx + 1 - current_subsection_idx = 0 - if not line.isdigit() and line[0].isdigit() and line[1] == " ": - is_chapter = True - # before_section = current_section - current_section = line - structured_content[current_section] = [] - current_section_idx = current_section_idx + 1 - current_subsection_idx = 0 - # sub-chapter - if not line.isdigit() and line[0] == str(current_section_idx) and line[1] == "." and line[2] == str(current_subsection_idx+1): - is_chapter = True - # before_section = current_section - current_section = line - structured_content[current_section] = [] - current_subsection_idx = current_subsection_idx + 1 - - before_context = line - - return structured_content - -# connect the differences with the paragraphs -def connect_diffs_and_paragraphs(original_pdf_path: Path, modified_pdf_path: Path, filter_list: Optional[List[str]] = None) -> list[list[dict]]: - # extract text - original_text = extract_text_from_pdf(original_pdf_path) - if original_text is not None: - print("Successfully extract text from the original pdf") - else: - print("File "+str(original_pdf_path)+" not existed") - modified_text = extract_text_from_pdf(modified_pdf_path) - if modified_text is not None: - print("Successfully extract text from the modified pdf") - else: - print("File "+str(modified_pdf_path)+" not existed") - - - # get the differences - all_diff_result = compare_texts(original_text, modified_text) - print("Successfully extract differences between original pdf and modified pdf") - - # get formatted differences - formatted_diff_result = parse_diff(all_diff_result) - print("Successfully get formatted differences") - - # get the structured paragraphs from modified paper - structured_paragraphs_from_modified = extract_paragraphs_from_pdf_new(modified_pdf_path, filter_list) - print("Successfully extract paragraphs from the modified pdf") - - # connect differences with paragraphs - all_diff_loc = [] - for diff in formatted_diff_result: - diff_context = diff["modified_lines"] - diff_before_context = diff["context_before"] - diff_after_context = diff["context_after"] - idx = 0 - paragraph_list = { - "paragraph_before": { - "section": "", - "paragraph_idx": 1, - "paragraph_content": "" - }, - "paragraph_current": { - "section": "", - "paragraph_idx": 2, - "paragraph_content": "" - }, - "paragraph_after": { - "section": "", - "paragraph_idx": 3, - "paragraph_content": "" - }, - } - total_diff_examples = [] - for key, val in zip(structured_paragraphs_from_modified.keys(), structured_paragraphs_from_modified.values()): - for paragraph in val: - idx = idx + 1 - if idx == 1: - paragraph_list["paragraph_before"]["section"] = key - paragraph_list["paragraph_before"]["paragraph_idx"] = idx - paragraph_list["paragraph_before"]["paragraph_content"] = paragraph - elif idx == 2: - paragraph_list["paragraph_current"]["section"] = key - paragraph_list["paragraph_current"]["paragraph_idx"] = idx - paragraph_list["paragraph_current"]["paragraph_content"] = paragraph - elif idx == 3: - paragraph_list["paragraph_after"]["section"] = key - paragraph_list["paragraph_after"]["paragraph_idx"] = idx - paragraph_list["paragraph_after"]["paragraph_content"] = paragraph - if diff_context[:10] in paragraph_list["paragraph_current"]["paragraph_content"]: - diff_sample = {} - diff_sample["context_before"] = diff["context_before"] - diff_sample["context_after"] = diff["context_after"] - diff_sample["original_lines"] = diff["original_lines"] - diff_sample["modified_lines"] = diff["modified_lines"] - diff_sample["section"] = paragraph_list["paragraph_current"]["section"] - diff_sample["paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - # check diff before context - if diff_before_context[:10] in paragraph_list["paragraph_current"]["paragraph_content"]: - diff_sample["before_section"] = paragraph_list["paragraph_current"]["section"] - diff_sample["before_paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - elif diff_before_context[:10] in paragraph_list["paragraph_before"]["paragraph_content"]: - diff_sample["before_section"] = paragraph_list["paragraph_before"]["section"] - diff_sample["before_paragraph_idx"] = paragraph_list["paragraph_before"]["paragraph_idx"] - else: - diff_sample["before_section"] = None - diff_sample["before_paragraph_idx"] = None - # check diff after context - if diff_after_context[:10] in paragraph_list["paragraph_current"]["paragraph_content"]: - diff_sample["after_section"] = paragraph_list["paragraph_current"]["section"] - diff_sample["after_paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - elif diff_after_context[:10] in paragraph_list["paragraph_after"]["paragraph_content"]: - diff_sample["after_section"] = paragraph_list["paragraph_after"]["section"] - diff_sample["after_paragraph_idx"] = paragraph_list["paragraph_after"]["paragraph_idx"] - else: - diff_sample["after_section"] = None - diff_sample["after_paragraph_idx"] = None - total_diff_examples.append(diff_sample) - elif idx > 3: - # move the list - paragraph_list["paragraph_before"]["section"] = paragraph_list["paragraph_current"]["section"] - paragraph_list["paragraph_before"]["paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - paragraph_list["paragraph_before"]["paragraph_content"] = paragraph_list["paragraph_current"]["paragraph_content"] - - paragraph_list["paragraph_current"]["section"] = paragraph_list["paragraph_after"]["section"] - paragraph_list["paragraph_current"]["paragraph_idx"] = paragraph_list["paragraph_after"]["paragraph_idx"] - paragraph_list["paragraph_current"]["paragraph_content"] = paragraph_list["paragraph_after"]["paragraph_content"] - - paragraph_list["paragraph_after"]["section"] = key - paragraph_list["paragraph_after"]["paragraph_idx"] = idx - paragraph_list["paragraph_after"]["paragraph_content"] = paragraph - - # start match the differences - if diff_context[:10] in paragraph_list["paragraph_current"]["paragraph_content"]: - diff_sample = {} - diff_sample["context_before"] = diff["context_before"] - diff_sample["context_after"] = diff["context_after"] - diff_sample["original_lines"] = diff["original_lines"] - diff_sample["modified_lines"] = diff["modified_lines"] - diff_sample["section"] = paragraph_list["paragraph_current"]["section"] - diff_sample["paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - # check diff before context - if diff_before_context[:10] in paragraph_list["paragraph_current"]["paragraph_content"]: - diff_sample["before_section"] = paragraph_list["paragraph_current"]["section"] - diff_sample["before_paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - elif diff_before_context[:10] in paragraph_list["paragraph_before"]["paragraph_content"]: - diff_sample["before_section"] = paragraph_list["paragraph_before"]["section"] - diff_sample["before_paragraph_idx"] = paragraph_list["paragraph_before"]["paragraph_idx"] - else: - diff_sample["before_section"] = None - diff_sample["before_paragraph_idx"] = None - # check diff after context - if diff_after_context[:10] in paragraph_list["paragraph_current"]["paragraph_content"]: - diff_sample["after_section"] = paragraph_list["paragraph_current"]["section"] - diff_sample["after_paragraph_idx"] = paragraph_list["paragraph_current"]["paragraph_idx"] - elif diff_after_context[:10] in paragraph_list["paragraph_after"]["paragraph_content"]: - diff_sample["after_section"] = paragraph_list["paragraph_after"]["section"] - diff_sample["after_paragraph_idx"] = paragraph_list["paragraph_after"]["paragraph_idx"] - else: - diff_sample["after_section"] = None - diff_sample["after_paragraph_idx"] = None - total_diff_examples.append(diff_sample) - if len(total_diff_examples) == 1: - all_diff_loc.append(total_diff_examples[0]) - elif len(total_diff_examples) > 1: - is_find = False - # before and after match - for diff_example in total_diff_examples: - if diff_sample["before_section"] is not None and diff_sample["after_section"] is not None: - all_diff_loc.append(diff_example) - is_find = True - break - if not is_find: - # before or after match - for diff_example in total_diff_examples: - if diff_sample["before_section"] is not None or diff_sample["after_section"] is not None: - all_diff_loc.append(diff_example) - is_find = True - break - if not is_find: - all_diff_loc.append(total_diff_examples[0]) - - print("Successfully connect differences with paragraphs") - - return all_diff_loc \ No newline at end of file diff --git a/research_arcade/research_arcade.py b/research_arcade/research_arcade.py deleted file mode 100644 index 482fc03..0000000 --- a/research_arcade/research_arcade.py +++ /dev/null @@ -1,710 +0,0 @@ -# OpenReview CSV -from .csv_database import ( - CSVOpenReviewArxiv, CSVOpenReviewAuthors, CSVOpenReviewPapersAuthors, - CSVOpenReviewPapersReviews, CSVOpenReviewPapersRevisions, CSVOpenReviewPapers, - CSVOpenReviewReviews, CSVOpenReviewRevisionsReviews, CSVOpenReviewRevisions, - CSVOpenReviewParagraphs -) -# OpenReview SQL -from .sql_database import ( - SQLOpenReviewArxiv, SQLOpenReviewAuthors, SQLOpenReviewPapersAuthors, - SQLOpenReviewPapersReviews, SQLOpenReviewPapersRevisions, SQLOpenReviewPapers, - SQLOpenReviewReviews, SQLOpenReviewRevisionsReviews, SQLOpenReviewRevisions, - SQLOpenReviewParagraphs -) - -# Arxiv CSV -from .csv_database import ( - CSVArxivAuthors, CSVArxivCategory, CSVArxivCitation, CSVArxivFigure, - CSVArxivPaperAuthor, CSVArxivPaperCategory, CSVArxivPaperFigure, - CSVArxivPaperTable, CSVArxivPapers, CSVArxivParagraphReference, - CSVArxivParagraphs, CSVArxivSections, CSVArxivTable -) - -# Arxiv SQL -from .sql_database import ( - SQLArxivAuthors, SQLArxivCategory, SQLArxivCitation, SQLArxivFigure, - SQLArxivPaperAuthor, SQLArxivPaperCategory, SQLArxivPaperFigure, - SQLArxivPaperTable, SQLArxivPapers, SQLArxivParagraphReference, - SQLArxivParagraphs, SQLArxivSections, SQLArxivTable -) -import os -from dotenv import load_dotenv -from typing import Optional -import pandas as pd - -class ResearchArcade: - def __init__(self, db_type: str, config: dict) -> None: - load_dotenv() - if db_type == 'csv': - if config["csv_dir"] is None: - config["csv_dir"] = os.getenv('CSV_DATASET_FOLDER_PATH') - - """ - Below is the arxiv dataset - """ - self.arxiv_authors = CSVArxivAuthors(**config) - self.arxiv_categories = CSVArxivCategory(**config) - self.arxiv_figures = CSVArxivFigure(**config) - self.arxiv_tables = CSVArxivTable(**config) - self.arxiv_papers = CSVArxivPapers(**config) - self.arxiv_paragraphs = CSVArxivParagraphs(**config) - self.arxiv_sections = CSVArxivSections(**config) - self.arxiv_citation = CSVArxivCitation(**config) - self.arxiv_paper_author = CSVArxivPaperAuthor(**config) - self.arxiv_paper_category = CSVArxivPaperCategory(**config) - self.arxiv_paper_figure = CSVArxivPaperFigure(**config) - self.arxiv_paper_table = CSVArxivPaperTable(**config) - self.arxiv_paragraph_reference = CSVArxivParagraphReference(**config) - - """ - Below is the openreview dataset - """ - self.openreview_arxiv = CSVOpenReviewArxiv(**config) - self.openreview_authors = CSVOpenReviewAuthors(**config) - self.openreview_papers_authors = CSVOpenReviewPapersAuthors(**config) - self.openreview_papers_reviews = CSVOpenReviewPapersReviews(**config) - self.openreview_papers_revisions = CSVOpenReviewPapersRevisions(**config) - self.openreview_papers = CSVOpenReviewPapers(**config) - self.openreview_reviews = CSVOpenReviewReviews(**config) - self.openreview_revisions_reviews = CSVOpenReviewRevisionsReviews(**config) - self.openreview_revisions = CSVOpenReviewRevisions(**config) - self.openreview_paragraphs = CSVOpenReviewParagraphs(**config) - elif db_type == 'sql': - if config["host"] is None: - config["csv_dir"] = os.getenv('CSV_DATASET_FOLDER_PATH') - - """ - Below is the arxiv dataset - """ - self.arxiv_authors = SQLArxivAuthors(**config) - self.arxiv_categories = SQLArxivCategory(**config) - self.arxiv_figures = SQLArxivFigure(**config) - self.arxiv_tables = SQLArxivTable(**config) - self.arxiv_papers = SQLArxivPapers(**config) - self.arxiv_paragraphs = SQLArxivParagraphs(**config) - self.arxiv_sections = SQLArxivSections(**config) - self.arxiv_citation = SQLArxivCitation(**config) - self.arxiv_paper_author = SQLArxivPaperAuthor(**config) - self.arxiv_paper_category = SQLArxivPaperCategory(**config) - self.arxiv_paper_figure = SQLArxivPaperFigure(**config) - self.arxiv_paper_table = SQLArxivPaperTable(**config) - self.arxiv_paragraph_reference = SQLArxivParagraphReference(**config) - - - """ - Below is the openreview dataset - """ - self.openreview_arxiv = SQLOpenReviewArxiv(**config) - self.openreview_authors = SQLOpenReviewAuthors(**config) - self.openreview_papers_authors = SQLOpenReviewPapersAuthors(**config) - self.openreview_papers_reviews = SQLOpenReviewPapersReviews(**config) - self.openreview_papers_revisions = SQLOpenReviewPapersRevisions(**config) - self.openreview_papers = SQLOpenReviewPapers(**config) - self.openreview_reviews = SQLOpenReviewReviews(**config) - self.openreview_revisions_reviews = SQLOpenReviewRevisionsReviews(**config) - self.openreview_revisions = SQLOpenReviewRevisions(**config) - self.openreview_paragraphs = SQLOpenReviewParagraphs(**config) - - - def insert_node(self, table: str, node_features: dict) -> Optional[tuple]: - # Tables in openreview dataset - if table == 'openreview_authors': - return self.openreview_authors.insert_author(**node_features) - elif table == 'openreview_papers': - return self.openreview_papers.insert_paper(**node_features) - elif table == 'openreview_reviews': - return self.openreview_reviews.insert_review(**node_features) - elif table == 'openreview_revisions': - return self.openreview_revisions.insert_revision(**node_features) - elif table == 'openreview_paragraphs': - return self.openreview_paragraphs.insert_paragraph(**node_features) - # Tables in arxiv dataset - elif table == 'arxiv_authors': - return self.arxiv_authors.insert_author(**node_features) - elif table == 'arxiv_categories': - return self.arxiv_categories.insert_category(**node_features) - elif table == 'arxiv_figures': - return self.arxiv_figures.insert_figure(**node_features) - elif table == 'arxiv_tables': - return self.arxiv_tables.insert_table(**node_features) - elif table == 'arxiv_papers': - return self.arxiv_papers.insert_paper(**node_features) - elif table == 'arxiv_paragraphs': - return self.arxiv_paragraphs.insert_paragraph(**node_features) - elif table == 'arxiv_sections': - return self.arxiv_sections.insert_section(**node_features) - else: - print(f"Table {table} not found.") - return None - - def delete_node_by_id(self, table: str, primary_key: dict) -> Optional[pd.DataFrame]: - # Tables in openreview dataset - if table == 'openreview_authors': - return self.openreview_authors.delete_author_by_id(**primary_key) - elif table == 'openreview_papers': - return self.openreview_papers.delete_paper_by_id(**primary_key) - elif table == 'openreview_reviews': - return self.openreview_reviews.delete_review_by_id(**primary_key) - elif table == 'openreview_revisions': - return self.openreview_revisions.delete_revision_by_id(**primary_key) - elif table == 'openreview_paragraphs': - return self.openreview_paragraphs.delete_paragraphs_by_paper_id(**primary_key) - # Tables in arxiv dataset - elif table == 'arxiv_authors': - return self.arxiv_authors.delete_author_by_id(**primary_key) - elif table == 'arxiv_categories': - return self.arxiv_categories.delete_category_by_id(**primary_key) - elif table == 'arxiv_figures': - return self.arxiv_figures.delete_figure_by_id(**primary_key) - elif table == 'arxiv_tables': - return self.arxiv_tables.delete_table_by_id(**primary_key) - elif table == 'arxiv_papers': - return self.arxiv_papers.delete_paper_by_id(**primary_key) - elif table == 'arxiv_paragraphs': - return self.arxiv_paragraphs.delete_paragraph_by_id(**primary_key) - elif table == 'arxiv_sections': - return self.arxiv_sections.delete_section_by_id(**primary_key) - - else: - print(f"Table {table} not found.") - return None - - def update_node(self, table: str, node_features: dict) -> Optional[pd.DataFrame]: - # Tables in openreview dataset - if table == 'openreview_authors': - return self.openreview_authors.update_author(**node_features) - elif table == 'openreview_papers': - return self.openreview_papers.update_paper(**node_features) - elif table == 'openreview_reviews': - return self.openreview_reviews.update_review(**node_features) - elif table == 'openreview_revisions': - return self.openreview_revisions.update_revision(**node_features) - # Tables in arxiv dataset - elif table == 'arxiv_authors': - return self.arxiv_authors.update_author(**node_features) - elif table == 'arxiv_categories': - return self.arxiv_categories.update_category(**node_features) - elif table == 'arxiv_figures': - return self.arxiv_figures.update_figure(**node_features) - elif table == 'arxiv_tables': - return self.arxiv_tables.update_table(**node_features) - elif table == 'arxiv_papers': - return self.arxiv_papers.update_paper(**node_features) - elif table == 'arxiv_paragraphs': - return self.arxiv_paragraphs.update_paragraph(**node_features) - elif table == 'arxiv_sections': - return self.arxiv_sections.update_section(**node_features) - else: - print(f"Table {table} not found.") - return None - - def get_node_features_by_id(self, table: str, primary_key: dict) -> Optional[pd.DataFrame]: - # Tables in openreview dataset - if table == 'openreview_authors': - return self.openreview_authors.get_author_by_id(**primary_key) - elif table == 'openreview_papers': - return self.openreview_papers.get_paper_by_id(**primary_key) - elif table == 'openreview_reviews': - return self.openreview_reviews.get_review_by_id(**primary_key) - elif table == 'openreview_revisions': - return self.openreview_revisions.get_revision_by_id(**primary_key) - elif table == 'openreview_paragraphs': - return self.openreview_paragraphs.get_paragraphs_by_paper_id(**primary_key) - # Tables in arxiv dataset - elif table == 'arxiv_authors': - return self.arxiv_authors.get_author_by_id(**primary_key) - elif table == 'arxiv_categories': - return self.arxiv_categories.get_category_by_id(**primary_key) - elif table == 'arxiv_figures': - return self.arxiv_figures.get_figure_by_id(**primary_key) - elif table == 'arxiv_tables': - return self.arxiv_tables.get_table_by_id(**primary_key) - elif table == 'arxiv_papers': - return self.arxiv_papers.get_paper_by_id(**primary_key) - elif table == 'arxiv_paragraphs': - return self.arxiv_paragraphs.get_paragraph_by_id(**primary_key) - elif table == 'arxiv_sections': - return self.arxiv_sections.get_section_by_id(**primary_key) - - else: - print(f"Table {table} not found.") - return None - - def get_all_node_features(self, table: str) -> Optional[pd.DataFrame]: - # Openreview tables - if table == 'openreview_authors': - return self.openreview_authors.get_all_authors(is_all_features=True) - elif table == 'openreview_papers': - return self.openreview_papers.get_all_papers(is_all_features=True) - elif table == 'openreview_reviews': - return self.openreview_reviews.get_all_reviews(is_all_features=True) - elif table == 'openreview_revisions': - return self.openreview_revisions.get_all_revisions(is_all_features=True) - elif table == 'openreview_paragraphs': - return self.openreview_paragraphs.get_all_paragraphs(is_all_features=True) - - # Arxiv tables - elif table == 'arxiv_authors': - return self.arxiv_authors.get_all_authors(is_all_features=True) - elif table == 'arxiv_categories': - return self.arxiv_categories.get_all_categories(is_all_features=True) - elif table == 'arxiv_figures': - return self.arxiv_figures.get_all_figures(is_all_features=True) - elif table == 'arxiv_tables': - return self.arxiv_tables.get_all_tables(is_all_features=True) - elif table == 'arxiv_papers': - return self.arxiv_papers.get_all_papers(is_all_features=True) - elif table == 'arxiv_paragraphs': - return self.arxiv_paragraphs.get_all_paragraphs(is_all_features=True) - elif table == 'arxiv_sections': - return self.arxiv_sections.get_all_sections(is_all_features=True) - else: - print(f"Table {table} not found.") - return None - - def insert_edge(self, table: str, edge_features: dict) -> Optional[pd.DataFrame]: - # openreview - if table == 'openreview_arxiv': - return self.openreview_arxiv.insert_openreview_arxiv(**edge_features) - elif table == "openreview_papers_authors": - return self.openreview_papers_authors.insert_paper_authors(**edge_features) - elif table == "openreview_papers_reviews": - return self.openreview_papers_reviews.insert_paper_reviews(**edge_features) - elif table == "openreview_papers_revisions": - return self.openreview_papers_revisions.insert_paper_revisions(**edge_features) - elif table == "openreview_revisions_reviews": - return self.openreview_revisions_reviews.insert_revision_reviews(**edge_features) - - # arxiv - elif table == 'arxiv_citation': - return self.arxiv_citation.insert_citation(**edge_features) - elif table == 'arxiv_paper_author': - return self.arxiv_paper_author.insert_paper_author(**edge_features) - elif table == 'arxiv_paper_category': - return self.arxiv_paper_category.insert_paper_category(**edge_features) - elif table == 'arxiv_paper_figure': - return self.arxiv_paper_figure.insert_paper_figure(**edge_features) - elif table == 'arxiv_paper_table': - return self.arxiv_paper_table.insert_paper_table(**edge_features) - elif table == 'arxiv_paragraph_reference': - return self.arxiv_paragraph_reference.insert_paragraph_reference(**edge_features) - else: - print(f"Table {table} not found.") - return None - - def delete_edge_by_id(self, table: str, primary_key: dict) -> Optional[pd.DataFrame]: - # Openreview - if table == 'openreview_arxiv': - if "paper_openreview_id" in primary_key and "arxiv_id" in primary_key: - return self.openreview_arxiv.delete_openreview_arxiv_by_id(**primary_key) - elif "arxiv_id" in primary_key: - return self.openreview_arxiv.delete_openreview_arxiv_by_arxiv_id(**primary_key) - elif "paper_openreview_id" in primary_key: - return self.openreview_arxiv.delete_openreview_arxiv_by_openreview_id(**primary_key) - else: - print("For openreview_arxiv table, the primary key should be 'paper_openreview_id' or 'arxiv_id'.") - return None - elif table == "openreview_papers_authors": - if "paper_openreview_id" in primary_key and "author_openreview_id" in primary_key: - return self.openreview_papers_authors.delete_paper_author_by_id(**primary_key) - elif "paper_openreview_id" in primary_key: - return self.openreview_papers_authors.delete_paper_author_by_paper_id(**primary_key) - elif "author_openreview_id" in primary_key: - return self.openreview_papers_authors.delete_paper_author_by_author_id(**primary_key) - else: - print("For openreview_papers_authors table, the primary key should be 'paper_openreview_id' or 'author_openreview_id'.") - return None - elif table == "openreview_papers_reviews": - if "paper_openreview_id" in primary_key and "review_openreview_id" in primary_key: - return self.openreview_papers_reviews.delete_paper_review_by_id(**primary_key) - elif "paper_openreview_id" in primary_key: - return self.openreview_papers_reviews.delete_paper_review_by_paper_id(**primary_key) - elif "review_openreview_id" in primary_key: - return self.openreview_papers_reviews.delete_paper_review_by_review_id(**primary_key) - else: - print("For openreview_papers_reviews table, the primary key should be 'paper_openreview_id' or 'review_openreview_id'.") - return None - elif table == "openreview_papers_revisions": - if "paper_openreview_id" in primary_key and "revision_openreview_id" in primary_key: - return self.openreview_papers_revisions.delete_paper_revision_by_id(**primary_key) - elif "paper_openreview_id" in primary_key: - return self.openreview_papers_revisions.delete_paper_revision_by_paper_id(**primary_key) - elif "revision_openreview_id" in primary_key: - return self.openreview_papers_revisions.delete_paper_revision_by_revision_id(**primary_key) - else: - print("For openreview_papers_revisions table, the primary key should be 'paper_openreview_id' or 'revision_openreview_id'.") - return None - elif table == "openreview_revisions_reviews": - if "revision_openreview_id" in primary_key and "review_openreview_id" in primary_key: - return self.openreview_revisions_reviews.delete_revision_review_by_id(**primary_key) - elif "revision_openreview_id" in primary_key: - return self.openreview_revisions_reviews.delete_revision_review_by_revision_id(**primary_key) - elif "review_openreview_id" in primary_key: - return self.openreview_revisions_reviews.delete_revision_review_by_review_id(**primary_key) - else: - print("For openreview_revisions_reviews table, the primary key should be 'revision_openreview_id' or 'review_openreview_id'.") - return None - # Arxiv - elif table == 'arxiv_citation': - # Expect keys: 'citing_paper_id' and 'cited_paper_id' - if "citing_paper_id" in primary_key and "cited_paper_id" in primary_key: - return self.arxiv_citation.delete_citation_by_id(**primary_key) - elif "citing_paper_id" in primary_key: - return self.arxiv_citation.delete_citation_by_citing_id(**primary_key) - elif "cited_paper_id" in primary_key: - return self.arxiv_citation.delete_citation_by_cited_id(**primary_key) - else: - print("For arxiv_citation, primary key should include 'citing_paper_id' and/or 'cited_paper_id'.") - return None - elif table == 'arxiv_paper_author': - # Expect keys: 'paper_id' and/or 'author_id' - if "paper_arxiv_id" in primary_key and "author_id" in primary_key: - return self.arxiv_paper_author.delete_paper_author_by_id(**primary_key) - elif "paper_arxiv_id" in primary_key: - return self.arxiv_paper_author.delete_paper_author_by_paper_id(**primary_key) - elif "author_id" in primary_key: - return self.arxiv_paper_author.delete_paper_author_by_author_id(**primary_key) - else: - print("For arxiv_paper_author, primary key should include 'paper_id' and/or 'author_id'.") - return None - elif table == 'arxiv_paper_category': - # Expect keys: 'paper_id' and/or 'category_id' - if "paper_arxiv_id" in primary_key and "category_id" in primary_key: - return self.arxiv_paper_category.delete_paper_category_by_id(**primary_key) - elif "paper_arxiv_id" in primary_key: - return self.arxiv_paper_category.delete_paper_category_by_paper_id(**primary_key) - elif "category_id" in primary_key: - return self.arxiv_paper_category.delete_paper_category_by_category_id(**primary_key) - else: - print("For arxiv_paper_category, primary key should include 'paper_id' and/or 'category_id'.") - return None - elif table == 'arxiv_paper_figure': - # Expect keys: 'paper_id' and/or 'figure_id' - if "paper_arxiv_id" in primary_key and "figure_id" in primary_key: - return self.arxiv_paper_figure.delete_paper_figure_by_id(**primary_key) - elif "paper_arxiv_id" in primary_key: - return self.arxiv_paper_figure.delete_paper_figure_by_paper_id(**primary_key) - elif "figure_id" in primary_key: - return self.arxiv_paper_figure.delete_paper_figure_by_figure_id(**primary_key) - else: - print("For arxiv_paper_figure, primary key should include 'paper_id' and/or 'figure_id'.") - return None - elif table == 'arxiv_paper_table': - # Expect keys: 'paper_id' and/or 'table_id' - if "paper_arxiv_id" in primary_key and "table_id" in primary_key: - return self.arxiv_paper_table.delete_paper_table_by_id(**primary_key) - elif "paper_arxiv_id" in primary_key: - return self.arxiv_paper_table.delete_paper_table_by_paper_id(**primary_key) - elif "table_id" in primary_key: - return self.arxiv_paper_table.delete_paper_table_by_table_id(**primary_key) - else: - print("For arxiv_paper_table, primary key should include 'paper_id' and/or 'table_id'.") - return None - elif table == 'arxiv_paragraph_reference': - # Expect keys: 'paragraph_id' and/or 'reference_id' - if "paragraph_id" in primary_key and "reference_id" in primary_key: - return self.arxiv_paragraph_reference.delete_paragraph_reference_by_id(**primary_key) - elif "paragraph_id" in primary_key: - return self.arxiv_paragraph_reference.delete_paragraph_reference_by_paragraph_id(**primary_key) - elif "reference_id" in primary_key: - return self.arxiv_paragraph_reference.delete_paragraph_reference_by_reference_id(**primary_key) - else: - print("For arxiv_paragraph_reference, primary key should include 'paragraph_id' and/or 'reference_id'.") - return None - - else: - print(f"Table {table} not found.") - return None - - def get_all_edge_features(self, table: str) -> Optional[pd.DataFrame]: - # Openreview - if table == 'openreview_arxiv': - return self.openreview_arxiv.get_all_openreview_arxiv() - elif table == 'openreview_papers_authors': - return self.openreview_papers_authors.get_all_papers_authors() - elif table == 'openreview_papers_reviews': - return self.openreview_papers_reviews.get_all_papers_reviews() - elif table == 'openreview_papers_revisions': - return self.openreview_papers_revisions.get_all_papers_revisions() - elif table == 'openreview_revisions_reviews': - return self.openreview_revisions_reviews.get_all_revisions_reviews() - - # Arxiv - elif table == 'arxiv_citation': - return self.arxiv_citation.get_all_citations() - elif table == 'arxiv_paper_author': - return self.arxiv_paper_author.get_all_paper_authors() - elif table == 'arxiv_paper_category': - return self.arxiv_paper_category.get_all_paper_categories() - elif table == 'arxiv_paper_figure': - return self.arxiv_paper_figure.get_all_paper_figures() - elif table == 'arxiv_paper_table': - return self.arxiv_paper_table.get_all_paper_tables() - elif table == 'arxiv_paragraph_reference': - return self.arxiv_paragraph_reference.get_all_paragraph_references() - else: - print(f"Table {table} not found.") - return None - - def get_neighborhood(self, table: str, primary_key: dict) -> Optional[pd.DataFrame]: - # Openreview - if table == 'openreview_arxiv': - if "paper_openreview_id" in primary_key: - return self.openreview_arxiv.get_openreview_neighboring_arxivs(**primary_key) - elif "arxiv_id" in primary_key: - return self.openreview_arxiv.get_arxiv_neighboring_openreviews(**primary_key) - else: - print("For openreview_arxiv table, the primary key should be either 'paper_openreview_id' or 'arxiv_id'.") - return None - elif table == 'openreview_papers_authors': - if "paper_openreview_id" in primary_key: - return self.openreview_papers_authors.get_paper_neighboring_authors(**primary_key) - elif "author_openreview_id" in primary_key: - return self.openreview_papers_authors.get_author_neighboring_papers(**primary_key) - else: - print("For openreview_papers_authors table, the primary key should be either 'paper_openreview_id' or 'author_openreview_id'.") - return None - elif table == 'openreview_papers_reviews': - if "paper_openreview_id" in primary_key: - return self.openreview_papers_reviews.get_paper_neighboring_reviews(**primary_key) - elif "review_openreview_id" in primary_key: - return self.openreview_papers_reviews.get_review_neighboring_papers(**primary_key) - else: - print("For openreview_papers_reviews table, the primary key should be either 'paper_openreview_id' or 'review_openreview_id'.") - return None - elif table == 'openreview_papers_revisions': - if "paper_openreview_id" in primary_key: - return self.openreview_papers_revisions.get_paper_neighboring_revisions(**primary_key) - elif "revision_openreview_id" in primary_key: - return self.openreview_papers_revisions.get_revision_neighboring_papers(**primary_key) - else: - print("For openreview_papers_revisions table, the primary key should be either 'paper_openreview_id' or 'revision_openreview_id'.") - return None - elif table == 'openreview_revisions_reviews': - if "revision_openreview_id" in primary_key: - return self.openreview_revisions_reviews.get_revision_neighboring_reviews(**primary_key) - elif "review_openreview_id" in primary_key: - return self.openreview_revisions_reviews.get_review_neighboring_revisions(**primary_key) - else: - print("For openreview_revisions_reviews table, the primary key should be either 'revision_openreview_id' or 'review_openreview_id'.") - return None - # Arxiv - elif table == 'arxiv_citation': - # Expect either 'citing_paper_id' or 'cited_paper_id' - if "citing_paper_id" in primary_key: - return self.arxiv_citation.get_citing_neighboring_cited(**primary_key) - elif "cited_paper_id" in primary_key: - return self.arxiv_citation.get_cited_neighboring_citing(**primary_key) - else: - print("For arxiv_citation, provide 'citing_paper_id' or 'cited_paper_id'.") - return None - elif table == 'arxiv_paper_author': - if "paper_arxiv_id" in primary_key: - return self.arxiv_paper_author.get_paper_neighboring_authors(**primary_key) - elif "author_id" in primary_key: - return self.arxiv_paper_author.get_author_neighboring_papers(**primary_key) - else: - print("For arxiv_paper_author, provide 'paper_id' or 'author_id'.") - return None - elif table == 'arxiv_paper_category': - if "paper_arxiv_id" in primary_key: - return self.arxiv_paper_category.get_paper_neighboring_categories(**primary_key) - elif "category_id" in primary_key: - return self.arxiv_paper_category.get_category_neighboring_papers(**primary_key) - else: - print("For arxiv_paper_category, provide 'paper_id' or 'category_id'.") - return None - elif table == 'arxiv_paper_figure': - if "paper_arxiv_id" in primary_key: - return self.arxiv_paper_figure.get_paper_neighboring_figures(**primary_key) - elif "figure_id" in primary_key: - return self.arxiv_paper_figure.get_figure_neighboring_papers(**primary_key) - else: - print("For arxiv_paper_figure, provide 'paper_id' or 'figure_id'.") - return None - elif table == 'arxiv_paper_table': - if "paper_arxiv_id" in primary_key: - return self.arxiv_paper_table.get_paper_neighboring_tables(**primary_key) - elif "table_id" in primary_key: - return self.arxiv_paper_table.get_table_neighboring_papers(**primary_key) - else: - print("For arxiv_paper_table, provide 'paper_id' or 'table_id'.") - return None - elif table == 'arxiv_paragraph_reference': - if "paragraph_id" in primary_key: - return self.arxiv_paragraph_reference.get_paragraph_neighboring_references(**primary_key) - elif "reference_id" in primary_key: - return self.arxiv_paragraph_reference.get_reference_neighboring_paragraphs(**primary_key) - else: - print("For arxiv_paragraph_reference, provide 'paragraph_id' or 'reference_id'.") - return None - - else: - print(f"Table {table} not found.") - return None - - def construct_table_from_api(self, table: str, config: dict) -> Optional[pd.DataFrame]: - if table == "openreview_papers": - self.openreview_papers.construct_papers_table_from_api(**config) - elif table == "openreview_authors": - self.openreview_authors.construct_authors_table_from_api(**config) - elif table == "openreview_reviews": - self.openreview_reviews.construct_reviews_table_from_api(**config) - elif table == "openreview_revisions": - if config["pdf_dir"] is None: - config["pdf_dir"] = os.getenv("PDF_FOLDER_PATH") - self.openreview_revisions.construct_revisions_table_from_api(**config) - elif table == "openreview_papers_authors": - self.openreview_papers_authors.construct_papers_authors_table_from_api(**config) - elif table == "openreview_papers_reviews": - self.openreview_papers_reviews.construct_papers_reviews_table_from_api(**config) - elif table == "openreview_papers_revisions": - self.openreview_papers_revisions.construct_papers_revisions_table_from_api(**config) - elif table == "openreview_revisions_reviews": - self.openreview_revisions_reviews.construct_revisions_reviews_table(**config) - elif table == "openreview_arxiv": - self.openreview_arxiv.construct_openreview_arxiv_table_from_api(**config) - elif table == "openreview_paragraphs": - if config["pdf_dir"] is None: - config["pdf_dir"] = os.getenv("PDF_FOLDER_PATH") - self.openreview_paragraphs.construct_paragraphs_table_from_api(**config) - elif table == "arxiv_papers": - self.arxiv_papers.construct_papers_table_from_api(**config) - elif table == "arxiv_authors": - self.arxiv_authors.construct_authors_table_from_api(**config) - elif table == "arxiv_categories": - self.arxiv_categories.construct_category_table_from_api(**config) - elif table == "arxiv_figures": - self.arxiv_figures.construct_figures_table_from_api(**config) - elif table == "arxiv_tables": - self.arxiv_tables.construct_tables_table_from_api(**config) - elif table == "arxiv_sections": - self.arxiv_sections.construct_sections_table_from_api(**config) - elif table == "arxiv_paragraphs": - self.arxiv_paragraphs.construct_paragraphs_table_from_api(**config) - elif table == "arxiv_categories": - self.arxiv_categories.construct_category_table_from_api(**config) - elif table == "arxiv_paper_authors": - self.arxiv_paper_authors.construct_papers_table_from_api(**config) - elif table == "arxiv_paper_figures": - self.arxiv_paper_figures.construct_papers_table_from_api(**config) - elif table == "arxiv_paper_tables": - self.arxiv_paper_tables.construct_papers_table_from_api(**config) - elif table == "arxiv_paper_categories": - self.arxiv_paper_tables.construct_papers_table_from_api(**config) - elif table == "arxiv_citations": - self.arxiv_citations.construct_papers_table_from_api(**config) - elif table == "arxiv_paragraph_references": - self.arxiv_paragraph_references.construct_papers_table_from_api(**config) - else: - print(f"Table {table} does not support construction from API") - - - - - def construct_table_from_csv(self, table: str, config: dict) -> Optional[pd.DataFrame]: - # OpenReview tables - if table == "openreview_papers": - self.openreview_papers.construct_papers_table_from_csv(**config) - elif table == "openreview_authors": - self.openreview_authors.construct_authors_table_from_csv(**config) - elif table == "openreview_reviews": - self.openreview_reviews.construct_reviews_table_from_csv(**config) - elif table == "openreview_revisions": - self.openreview_revisions.construct_revisions_table_from_csv(**config) - elif table == "openreview_papers_authors": - self.openreview_papers_authors.construct_papers_authors_table_from_csv(**config) - elif table == "openreview_papers_reviews": - self.openreview_papers_reviews.construct_papers_reviews_table_from_csv(**config) - elif table == "openreview_papers_revisions": - self.openreview_papers_revisions.construct_papers_revisions_table_from_csv(**config) - elif table == "openreview_revisions_reviews": - self.openreview_revisions_reviews.construct_revisions_reviews_table_from_csv(**config) - elif table == "openreview_arxiv": - self.openreview_arxiv.construct_openreview_arxiv_table_from_csv(**config) - elif table == "openreview_paragraphs": - self.openreview_paragraphs.construct_paragraphs_table_from_csv(**config) - # ArXiv tables - NODES - elif table == "arxiv_papers": - self.arxiv_papers.construct_table_from_csv(**config) - elif table == "arxiv_authors": - self.arxiv_authors.construct_table_from_csv(**config) - elif table == "arxiv_categories": - self.arxiv_categories.construct_table_from_csv(**config) - elif table == "arxiv_figures": - self.arxiv_figures.construct_table_from_csv(**config) - elif table == "arxiv_tables": - self.arxiv_tables.construct_table_from_csv(**config) - elif table == "arxiv_sections": - self.arxiv_sections.construct_table_from_csv(**config) - elif table == "arxiv_paragraphs": - self.arxiv_paragraphs.construct_table_from_csv(**config) - # ArXiv tables - EDGES - elif table == "arxiv_paper_citation": - self.arxiv_citation.construct_table_from_csv(**config) - elif table == "arxiv_paper_author": - self.arxiv_paper_author.construct_table_from_csv(**config) - elif table == "arxiv_paper_category": - self.arxiv_paper_category.construct_table_from_csv(**config) - elif table == "arxiv_paper_figure": - self.arxiv_paper_figure.construct_table_from_csv(**config) - elif table == "arxiv_paper_table": - self.arxiv_paper_table.construct_table_from_csv(**config) - elif table == "arxiv_paragraph_reference": - self.arxiv_paragraph_reference.construct_table_from_csv(**config) - else: - print(f"Table {table} does not support construction from CSV") - - def construct_table_from_json(self, table: str, config: dict) -> Optional[pd.DataFrame]: - # OpenReview tables - if table == "openreview_papers": - self.openreview_papers.construct_papers_table_from_json(**config) - elif table == "openreview_authors": - self.openreview_authors.construct_authors_table_from_json(**config) - elif table == "openreview_reviews": - self.openreview_reviews.construct_reviews_table_from_json(**config) - elif table == "openreview_revisions": - self.openreview_revisions.construct_revisions_table_from_json(**config) - elif table == "openreview_papers_authors": - self.openreview_papers_authors.construct_papers_authors_table_from_json(**config) - elif table == "openreview_papers_reviews": - self.openreview_papers_reviews.construct_papers_reviews_table_from_json(**config) - elif table == "openreview_papers_revisions": - self.openreview_papers_revisions.construct_papers_revisions_table_from_json(**config) - elif table == "openreview_revisions_reviews": - self.openreview_revisions_reviews.construct_revisions_reviews_table_from_json(**config) - elif table == "openreview_arxiv": - self.openreview_arxiv.construct_openreview_arxiv_table_from_json(**config) - elif table == "openreview_paragraphs": - self.openreview_paragraphs.construct_paragraphs_table_from_json(**config) - # ArXiv tables - NODES - elif table == "arxiv_papers": - self.arxiv_papers.construct_table_from_json(**config) - elif table == "arxiv_authors": - self.arxiv_authors.construct_table_from_json(**config) - elif table == "arxiv_categories": - self.arxiv_categories.construct_table_from_json(**config) - elif table == "arxiv_figures": - self.arxiv_figures.construct_table_from_json(**config) - elif table == "arxiv_tables": - self.arxiv_tables.construct_table_from_json(**config) - elif table == "arxiv_sections": - self.arxiv_sections.construct_table_from_json(**config) - elif table == "arxiv_paragraphs": - self.arxiv_paragraphs.construct_table_from_json(**config) - # ArXiv tables - EDGES - elif table == "arxiv_paper_citation": - self.arxiv_citation.construct_table_from_json(**config) - elif table == "arxiv_paper_author": - self.arxiv_paper_author.construct_table_from_json(**config) - elif table == "arxiv_paper_category": - self.arxiv_paper_category.construct_table_from_json(**config) - elif table == "arxiv_paper_figure": - self.arxiv_paper_figure.construct_table_from_json(**config) - elif table == "arxiv_paper_table": - self.arxiv_paper_table.construct_table_from_json(**config) - elif table == "arxiv_paragraph_reference": - self.arxiv_paragraph_reference.construct_table_from_json(**config) - else: - print(f"Table {table} does not support construction from JSON") diff --git a/research_arcade/sql_database/__init__.py b/research_arcade/sql_database/__init__.py deleted file mode 100644 index c48f398..0000000 --- a/research_arcade/sql_database/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -# research_arcade/sql_database/__init__.py - -# --- OpenReview SQLs --- -from .sql_openreview_arxiv import SQLOpenReviewArxiv -from .sql_openreview_authors import SQLOpenReviewAuthors -from .sql_openreview_papers_authors import SQLOpenReviewPapersAuthors -from .sql_openreview_papers_reviews import SQLOpenReviewPapersReviews -from .sql_openreview_papers_revisions import SQLOpenReviewPapersRevisions -from .sql_openreview_papers import SQLOpenReviewPapers -from .sql_openreview_reviews import SQLOpenReviewReviews -from .sql_openreview_revisions_reviews import SQLOpenReviewRevisionsReviews -from .sql_openreview_revisions import SQLOpenReviewRevisions -from .sql_openreview_paragraphs import SQLOpenReviewParagraphs - -# --- Arxiv SQLs --- -from .sql_arxiv_authors import SQLArxivAuthors -from .sql_arxiv_categories import SQLArxivCategory -from .sql_arxiv_citations import SQLArxivCitation -from .sql_arxiv_figures import SQLArxivFigure -from .sql_arxiv_paper_authors import SQLArxivPaperAuthor -from .sql_arxiv_paper_categories import SQLArxivPaperCategory -from .sql_arxiv_paper_figures import SQLArxivPaperFigure -from .sql_arxiv_paper_tables import SQLArxivPaperTable -from .sql_arxiv_papers import SQLArxivPapers -from .sql_arxiv_paragraph_references import SQLArxivParagraphReference -from .sql_arxiv_paragraphs import SQLArxivParagraphs -from .sql_arxiv_sections import SQLArxivSections -from .sql_arxiv_tables import SQLArxivTable - -__all__ = [ - # --- OpenReview --- - 'SQLOpenReviewArxiv', - 'SQLOpenReviewAuthors', - 'SQLOpenReviewPapersAuthors', - 'SQLOpenReviewPapersReviews', - 'SQLOpenReviewPapersRevisions', - 'SQLOpenReviewPapers', - 'SQLOpenReviewReviews', - 'SQLOpenReviewRevisionsReviews', - 'SQLOpenReviewRevisions', - 'SQLOpenReviewParagraphs', - - # --- Arxiv --- - 'SQLArxivAuthors', - 'SQLArxivCategory', - 'SQLArxivCitation', - 'SQLArxivFigure', - 'SQLArxivPaperAuthor', - 'SQLArxivPaperCategory', - 'SQLArxivPaperFigure', - 'SQLArxivPaperTable', - 'SQLArxivPapers', - 'SQLArxivParagraphReference', - 'SQLArxivParagraphs', - 'SQLArxivSections', - 'SQLArxivTable' -] diff --git a/research_arcade/sql_database/sql_arxiv_authors.py b/research_arcade/sql_database/sql_arxiv_authors.py deleted file mode 100644 index 1239e1b..0000000 --- a/research_arcade/sql_database/sql_arxiv_authors.py +++ /dev/null @@ -1,343 +0,0 @@ -import psycopg2 -import psycopg2.extras -from semanticscholar import SemanticScholar - -import os -import json -import pandas as pd - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - -class SQLArxivAuthors: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password - ) - conn.autocommit = self.autocommit - return conn - - def create_authors_table(self): - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS authors ( - id SERIAL PRIMARY KEY, - semantic_scholar_id VARCHAR(100) UNIQUE, - name VARCHAR(255) NOT NULL, - homepage VARCHAR(255) - ) - """) - cur.close() - finally: - conn.close() - - def insert_author(self, semantic_scholar_id, name, homepage=None): - """Insert an author. Returns the generated author id (or None if conflict).""" - conn = self._get_connection() - try: - cur = conn.cursor() - sql = """ - INSERT INTO authors (semantic_scholar_id, name, homepage) - VALUES (%s, %s, %s) - ON CONFLICT (semantic_scholar_id) DO NOTHING - RETURNING id - """ - cur.execute(sql, (semantic_scholar_id, name, homepage)) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_author_by_id(self, id: int) -> bool: - """Delete an author by its id. Returns True if deleted, False if not found.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM authors WHERE id = %s RETURNING id", (id,)) - deleted = cur.fetchone() is not None - cur.close() - return deleted - finally: - conn.close() - - def update_author(self, id: int, semantic_scholar_id=None, name=None, homepage=None) -> bool: - """ - Update an author by id. Returns True if updated, False if not found or no fields provided. - """ - fields = [] - values = [] - if semantic_scholar_id is not None: - fields.append("semantic_scholar_id = %s") - values.append(semantic_scholar_id) - if name is not None: - fields.append("name = %s") - values.append(name) - if homepage is not None: - fields.append("homepage = %s") - values.append(homepage) - - if not fields: - return False # Nothing to update - - sql = f"UPDATE authors SET {', '.join(fields)} WHERE id = %s RETURNING id" - values.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(values)) - updated = cur.fetchone() is not None - cur.close() - return updated - finally: - conn.close() - - def get_author_by_id(self, id: int, return_all=False): - """ - Get an author by its id. - - If return_all=False, returns a single row tuple (id, semantic_scholar_id, name, homepage). - - If return_all=True, returns all matching rows as a list of tuples. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, semantic_scholar_id, name, homepage FROM authors WHERE id = %s", - (id,), - ) - res = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return res if res else None - finally: - conn.close() - - def check_author_exists(self, id: int) -> bool: - """Check if an author exists by its id.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM authors WHERE id = %s LIMIT 1", (id,)) - exists = cur.fetchone() is not None - cur.close() - return exists - finally: - conn.close() - - def get_all_authors(self, is_all_features=True): - """Get all authors from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT id, semantic_scholar_id, name, homepage FROM authors") - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def construct_authors_table_from_api(self, arxiv_ids, dest_dir): - """ - Given arxiv ids, find the semantic scholar ids and pages of the authors - """ - # Search for authors online - sch = SemanticScholar() - for arxiv_id in arxiv_ids: - base_arxiv_id, version = arxiv_id_processor(arxiv_id=arxiv_id) - print(f"base_arxiv_id: {base_arxiv_id}") - try: - paper_sch = sch.get_paper(f"ARXIV:{base_arxiv_id}") - authors = paper_sch.authors - for author in authors: - semantic_scholar_id = author.authorId - author_r = sch.get_author(semantic_scholar_id) - name = author_r.name - url = author_r.url - - self.insert_author(semantic_scholar_id=semantic_scholar_id, name=name, homepage=url) - except Exception as e: - print(f"Paper with arxiv id {base_arxiv_id} not found on semantic scholar: {e}") - continue - - def construct_table_from_csv(self, csv_file): - """ - Construct the authors table from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: semantic_scholar_id, name - - Optional columns: homepage - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - df = pd.read_csv(csv_file) - - required_cols = ['semantic_scholar_id', 'name'] - missing_cols = [col for col in required_cols if col not in df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - # Add optional columns if they don't exist - if 'homepage' not in df.columns: - df['homepage'] = None - - # Prepare rows for bulk insert - rows = list(df[['semantic_scholar_id', 'name', 'homepage']].itertuples(index=False, name=None)) - - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO authors (semantic_scholar_id, name, homepage) - VALUES %s - ON CONFLICT (semantic_scholar_id) DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} authors from {csv_file}") - return True - - except Exception as e: - print(f"Error importing authors from CSV: {e}") - return False - - def construct_table_from_json(self, json_file): - """ - Construct the authors table from an external JSON file. - - Args: - json_file: Path to the JSON file containing author data - - Expected JSON format (list of objects): - [ - { - "semantic_scholar_id": "123456", - "name": "John Doe", - "homepage": "https://example.com" // optional - }, - ... - ] - - Or (single object with authors array): - { - "authors": [ - { - "semantic_scholar_id": "123456", - "name": "John Doe", - "homepage": "https://example.com" - }, - ... - ] - } - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - # If it's a dict, look for an 'authors' key - if 'authors' in json_data: - authors_list = json_data['authors'] - else: - # Treat the dict as a single author record - authors_list = [json_data] - elif isinstance(json_data, list): - authors_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not authors_list: - print("Error: No author data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for author in authors_list: - if 'semantic_scholar_id' not in author or 'name' not in author: - print(f"Warning: Skipping author record missing required fields: {author}") - continue - - rows.append(( - author['semantic_scholar_id'], - author['name'], - author.get('homepage', None) - )) - - if not rows: - print("No valid author records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO authors (semantic_scholar_id, name, homepage) - VALUES %s - ON CONFLICT (semantic_scholar_id) DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} authors from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing authors from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_categories.py b/research_arcade/sql_database/sql_arxiv_categories.py deleted file mode 100644 index 15c4032..0000000 --- a/research_arcade/sql_database/sql_arxiv_categories.py +++ /dev/null @@ -1,350 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivCategory: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_categories_table(self): - """ - Creates arxiv_categories with UNIQUE(name) to mirror CSV conflict rule. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_categories ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) UNIQUE NOT NULL, - description TEXT - ) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_category(self, name: str, description: Optional[str] = None) -> Optional[int]: - """ - Insert a category; returns generated id or None if name conflicts. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_categories (name, description) - VALUES (%s, %s) - ON CONFLICT (name) DO NOTHING - RETURNING id - """, - (name, description) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_category_by_id(self, id: int) -> bool: - """ - Delete by id; returns True if a row was deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_categories WHERE id = %s RETURNING id", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def update_category(self, id: int, name: Optional[str] = None, description: Optional[str] = None) -> bool: - """ - Partial update by id. Only non-None fields are updated. - Returns True if a row was updated. - """ - sets: List[str] = [] - vals: List = [] - if name is not None: - sets.append("name = %s") - vals.append(name) - if description is not None: - sets.append("description = %s") - vals.append(description) - - if not sets: - return False - - sql = f"UPDATE arxiv_categories SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_category_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple (id, name, description) - If return_all=True: returns a list of such tuples. - Returns None if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, name, description FROM arxiv_categories WHERE id = %s", - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def check_category_exists(self, id: int) -> bool: - """ - True if a category with the given id exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_categories WHERE id = %s LIMIT 1", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_all_categories(self, is_all_features=True): - """Get all categories from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT id, name, description FROM arxiv_categories") - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def construct_category_table_from_api(self, arxiv_ids, dest_dir): - """ - Construct categories from arXiv API by downloading papers and extracting their categories. - """ - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type="id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - try: - metadata_path = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - with open(metadata_path, 'r') as f: - metadata = json.load(f) - print(metadata) - - required_fields = ['categories'] - if not all(field in metadata for field in required_fields): - raise ValueError(f"Missing category for {arxiv_id}") - - categories = metadata['categories'] - for category in categories: - self.insert_category(name=category) - except Exception as e: - print(e) - print(f"Paper {arxiv_id} does not have category found") - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_category_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with required column: ['name'] - Optional column: ['description'] - Uses ON CONFLICT (name) DO NOTHING to skip duplicates. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['name'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - if 'description' not in df.columns: - df['description'] = None - - rows: List[Tuple] = list(df[['name', 'description']].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_categories (name, description) - VALUES %s - ON CONFLICT (name) DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} categories from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_category_table_from_csv for consistency.""" - return self.construct_category_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the categories table from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - {"name": "cs.AI", "description": "Artificial Intelligence"}, - {"name": "cs.LG", "description": "Machine Learning"}, - ... - ] - - Or: - { - "categories": [ - {"name": "cs.AI", "description": "Artificial Intelligence"}, - ... - ] - } - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - if isinstance(json_data, dict): - if 'categories' in json_data: - categories_list = json_data['categories'] - else: - categories_list = [json_data] - elif isinstance(json_data, list): - categories_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not categories_list: - print("Error: No category data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for category in categories_list: - if 'name' not in category: - print(f"Warning: Skipping category record missing required field 'name': {category}") - continue - - rows.append(( - category['name'], - category.get('description', None) - )) - - if not rows: - print("No valid category records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_categories (name, description) - VALUES %s - ON CONFLICT (name) DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} categories from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing categories from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_citations.py b/research_arcade/sql_database/sql_arxiv_citations.py deleted file mode 100644 index cad96c1..0000000 --- a/research_arcade/sql_database/sql_arxiv_citations.py +++ /dev/null @@ -1,593 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - - - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivCitation: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_citations_table(self): - """ - Creates the arxiv_citations table. - Enforces no self-citations and unique (citing_arxiv_id, cited_arxiv_id) pairs. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_citations ( - id SERIAL PRIMARY KEY, - citing_arxiv_id VARCHAR(100) NOT NULL, - cited_arxiv_id VARCHAR(100) NOT NULL, - bib_title TEXT, - bib_key VARCHAR(255), - author_cited_paper TEXT, - citing_sections TEXT[], - citing_paragraphs TEXT[], - CONSTRAINT no_self_citation CHECK (citing_arxiv_id != cited_arxiv_id), - CONSTRAINT unique_citation UNIQUE (citing_arxiv_id, cited_arxiv_id) - ) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_citation( - self, - citing_arxiv_id: str, - cited_arxiv_id: str, - bib_title: str = None, - bib_key: str = None, - author_cited_paper: str = None, - citing_sections: List[str] = None, - citing_paragraphs: List[str] = None - ) -> Optional[int]: - """ - Insert a citation row; returns generated id or None on conflict. - Automatically prevents self-citations and duplicate citations. - """ - if citing_arxiv_id == cited_arxiv_id: - return None - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_citations - (citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs) - VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT ON CONSTRAINT unique_citation DO NOTHING - RETURNING id - """, - ( - citing_arxiv_id, - cited_arxiv_id, - bib_title, - bib_key, - author_cited_paper, - citing_sections or [], - citing_paragraphs or [] - ) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_citation_by_id(self, citing_paper_id: str, cited_paper_id: str) -> bool: - """ - Delete a citation by citing and cited paper ids. - Returns True if deleted, False if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - DELETE FROM arxiv_citations - WHERE citing_arxiv_id = %s AND cited_arxiv_id = %s - RETURNING id - """, - (citing_paper_id, cited_paper_id) - ) - ok = cur.fetchone() is not None - cur.close() - if ok: - print(f"Deleted citation: {citing_paper_id} -> {cited_paper_id}") - else: - print(f"Citation not found: {citing_paper_id} -> {cited_paper_id}") - return ok - finally: - conn.close() - - def update_citation( - self, - id: int, - citing_arxiv_id: str = None, - cited_arxiv_id: str = None, - bib_title: str = None, - bib_key: str = None, - author_cited_paper: str = None, - citing_sections: List[str] = None, - citing_paragraphs: List[str] = None - ) -> bool: - """ - Partial update by id. Only non-None fields are updated. - Returns True if a row was updated. - """ - sets: List[str] = [] - vals: List = [] - - if citing_arxiv_id is not None: - sets.append("citing_arxiv_id = %s") - vals.append(citing_arxiv_id) - if cited_arxiv_id is not None: - sets.append("cited_arxiv_id = %s") - vals.append(cited_arxiv_id) - if bib_title is not None: - sets.append("bib_title = %s") - vals.append(bib_title) - if bib_key is not None: - sets.append("bib_key = %s") - vals.append(bib_key) - if author_cited_paper is not None: - sets.append("author_cited_paper = %s") - vals.append(author_cited_paper) - if citing_sections is not None: - sets.append("citing_sections = %s") - vals.append(citing_sections) - if citing_paragraphs is not None: - sets.append("citing_paragraphs = %s") - vals.append(citing_paragraphs) - - if not sets: - return False - - sql = f"UPDATE arxiv_citations SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_citation_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs) - If return_all=True: returns a list of such tuples. - Returns None if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs - FROM arxiv_citations WHERE id = %s - """, - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_all_citations(self, is_all_features=True): - """Get all citations from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs - FROM arxiv_citations - """ - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_citing_neighboring_cited(self, citing_paper_id: str): - """ - Get all papers cited by the given paper. - Returns citations where this paper is the citing paper. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs - FROM arxiv_citations WHERE citing_arxiv_id = %s - """, - (citing_paper_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_cited_neighboring_citing(self, cited_paper_id: str): - """ - Get all papers that cite the given paper. - Returns citations where this paper is the cited paper. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs - FROM arxiv_citations WHERE cited_arxiv_id = %s - """, - (cited_paper_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_citations_by_paper(self, arxiv_id: str, as_citing: bool = True): - """ - Get all citations for a paper. - If as_citing=True: returns citations where this paper cites others - If as_citing=False: returns citations where others cite this paper - """ - conn = self._get_connection() - try: - cur = conn.cursor() - if as_citing: - cur.execute( - """ - SELECT id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs - FROM arxiv_citations WHERE citing_arxiv_id = %s - """, - (arxiv_id,) - ) - else: - cur.execute( - """ - SELECT id, citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs - FROM arxiv_citations WHERE cited_arxiv_id = %s - """, - (arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else [] - finally: - conn.close() - - def check_citation_exists(self, citing_arxiv_id: str, cited_arxiv_id: str) -> bool: - """ - Returns True if a citation already exists between these two papers. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT 1 FROM arxiv_citations - WHERE citing_arxiv_id = %s AND cited_arxiv_id = %s - LIMIT 1 - """, - (citing_arxiv_id, cited_arxiv_id) - ) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_citation_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with columns: - ['citing_arxiv_id', 'cited_arxiv_id', 'bib_title', 'bib_key', - 'author_cited_paper', 'citing_sections', 'citing_paragraphs'] - Ignores any 'id' column; DB assigns SERIAL ids. - Skips conflicts and self-citations. - - Note: citing_sections and citing_paragraphs should be JSON arrays in CSV, - which will be parsed and stored as PostgreSQL arrays. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['citing_arxiv_id', 'cited_arxiv_id', 'bib_title', - 'bib_key'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: CSV is missing required columns: {missing}") - return False - - # Add optional columns if missing - if 'author_cited_paper' not in df.columns: - df['author_cited_paper'] = None - if 'citing_sections' not in df.columns: - df['citing_sections'] = '[]' - if 'citing_paragraphs' not in df.columns: - df['citing_paragraphs'] = '[]' - - # Parse JSON arrays from CSV - rows = [] - for _, row in df.iterrows(): - # Skip self-citations - if row['citing_arxiv_id'] == row['cited_arxiv_id']: - continue - - citing_sections = json.loads(row['citing_sections']) if pd.notna(row['citing_sections']) else [] - citing_paragraphs = json.loads(row['citing_paragraphs']) if pd.notna(row['citing_paragraphs']) else [] - - rows.append(( - row['citing_arxiv_id'], - row['cited_arxiv_id'], - row['bib_title'] if pd.notna(row['bib_title']) else None, - row['bib_key'] if pd.notna(row['bib_key']) else None, - row['author_cited_paper'] if pd.notna(row['author_cited_paper']) else None, - citing_sections, - citing_paragraphs - )) - - if not rows: - print("No valid rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_citations - (citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs) - VALUES %s - ON CONFLICT ON CONSTRAINT unique_citation DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} citations from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_citation_table_from_csv for consistency.""" - return self.construct_citation_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the citations table from an external JSON file. - - Args: - json_file: Path to the JSON file containing citation data - - Expected JSON format: - [ - { - "citing_arxiv_id": "1706.03762v7", - "cited_arxiv_id": "1409.0473v7", - "bib_title": "Neural Machine Translation", - "bib_key": "bahdanau2014neural", - "citing_sections": ["introduction", "related_work"], - "citing_paragraphs": [] - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'citations' in json_data: - citations_list = json_data['citations'] - else: - citations_list = [json_data] - elif isinstance(json_data, list): - citations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not citations_list: - print("Error: No citation data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for citation in citations_list: - if 'citing_arxiv_id' not in citation or 'cited_arxiv_id' not in citation: - print(f"Warning: Skipping citation missing required fields: {citation}") - continue - - # Skip self-citations - if citation['citing_arxiv_id'] == citation['cited_arxiv_id']: - continue - - citing_sections = citation.get('citing_sections', []) - if isinstance(citing_sections, str): - citing_sections = json.loads(citing_sections) - - citing_paragraphs = citation.get('citing_paragraphs', []) - if isinstance(citing_paragraphs, str): - citing_paragraphs = json.loads(citing_paragraphs) - - rows.append(( - citation['citing_arxiv_id'], - citation['cited_arxiv_id'], - citation.get('bib_title', None), - citation.get('bib_key', None), - citation.get('author_cited_paper', None), - citing_sections, - citing_paragraphs - )) - - if not rows: - print("No valid citation records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_citations - (citing_arxiv_id, cited_arxiv_id, bib_title, bib_key, - author_cited_paper, citing_sections, citing_paragraphs) - VALUES %s - ON CONFLICT ON CONSTRAINT unique_citation DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} citations from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing citations from JSON: {e}") - return False - - def construct_tables_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md = MultiDownload() - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - for citation in file_json['citations'].values(): - # print(f"Citation: {citation}") - cited_arxiv_id = citation.get('arxiv_id') - bib_key = citation.get('bib_key') - bib_title = citation.get('bib_title') - bib_author = citation.get('bib_author ') - contexts = citation.get('context') - citing_sections = set() - for context in contexts: - citing_section = context['section'] - citing_sections.add(citing_section) - - self.insert_citation(citing_arxiv_id=arxiv_id, cited_arxiv_id=cited_arxiv_id, citing_sections=list(citing_sections), bib_title=bib_title, bib_key=bib_key, author_cited_paper=bib_author) - - except FileNotFoundError: - print(f"Error: The file '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_figures.py b/research_arcade/sql_database/sql_arxiv_figures.py deleted file mode 100644 index 9fe19fb..0000000 --- a/research_arcade/sql_database/sql_arxiv_figures.py +++ /dev/null @@ -1,403 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd - - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor -from ..arxiv_utils.utils import arxiv_id_processor, figure_iteration_recursive -import json - - -class SQLArxivFigure: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_figures_table(self): - """ - Creates the arxiv_figures table. - We add a partial UNIQUE index on name (WHERE name IS NOT NULL) to match the CSV - behavior where we disallow duplicate non-null names. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_figures ( - id SERIAL PRIMARY KEY, - paper_arxiv_id VARCHAR(100) NOT NULL, - path VARCHAR(1024), - caption TEXT, - label VARCHAR(255), - name VARCHAR(255) - ) - """) - # Partial unique: only enforce uniqueness when name IS NOT NULL - cur.execute(""" - CREATE UNIQUE INDEX IF NOT EXISTS ux_arxiv_figures_name_notnull - ON arxiv_figures (name) - WHERE name IS NOT NULL - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_figure(self, paper_arxiv_id, path, caption=None, label=None, name=None) -> Optional[int]: - """ - Insert a figure row; returns generated id or None on (name) conflict (when name is not NULL). - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_figures (paper_arxiv_id, path, caption, label, name) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT ON CONSTRAINT ux_arxiv_figures_name_notnull DO NOTHING - RETURNING id - """, - (paper_arxiv_id, path, caption, label, name) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_figure_by_id(self, id: int) -> bool: - """ - Delete by id; returns True if a row was deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_figures WHERE id = %s RETURNING id", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def update_figure(self, id: int, paper_arxiv_id=None, path=None, caption=None, label=None, name=None) -> bool: - """ - Partial update by id. Only non-None fields are updated. - Returns True if a row was updated. - """ - sets: List[str] = [] - vals: List = [] - - if paper_arxiv_id is not None: - sets.append("paper_arxiv_id = %s") - vals.append(paper_arxiv_id) - if path is not None: - sets.append("path = %s") - vals.append(path) - if caption is not None: - sets.append("caption = %s") - vals.append(caption) - if label is not None: - sets.append("label = %s") - vals.append(label) - if name is not None: - sets.append("name = %s") - vals.append(name) - - if not sets: - return False - - sql = f"UPDATE arxiv_figures SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_figure_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, paper_arxiv_id, path, caption, label, name) - If return_all=True: returns a list of such tuples. - Returns None if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paper_arxiv_id, path, caption, label, name " - "FROM arxiv_figures WHERE id = %s", - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def check_figure_exists(self, id: int) -> bool: - """ - Returns True if a figure with the given id exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_figures WHERE id = %s LIMIT 1", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_all_figures(self, is_all_features=True): - """Get all figures from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paper_arxiv_id, path, caption, label, name FROM arxiv_figures" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_figure_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with columns: - ['paper_arxiv_id', 'path', 'caption', 'label', 'name'] - Ignores any 'id' column; DB assigns SERIAL ids. - Skips conflicts on non-null 'name'. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['paper_arxiv_id', 'path'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - # Add optional columns if missing - for col in ['caption', 'label', 'name']: - if col not in df.columns: - df[col] = None - - rows: List[Tuple] = list(df[['paper_arxiv_id', 'path', 'caption', 'label', 'name']].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - # Use upsert-like bulk insert; conflict only triggers if name IS NOT NULL and duplicates. - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_figures (paper_arxiv_id, path, caption, label, name) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_figures_name_notnull DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} figures from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_figure_table_from_csv for consistency.""" - return self.construct_figure_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the figures table from an external JSON file. - - Args: - json_file: Path to the JSON file containing figure data - - Expected JSON format: - [ - { - "paper_arxiv_id": "1706.03762v7", - "path": "/path/to/figure1.png", - "caption": "Architecture diagram", - "label": "fig:architecture", - "name": "figure1" - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'figures' in json_data: - figures_list = json_data['figures'] - else: - figures_list = [json_data] - elif isinstance(json_data, list): - figures_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not figures_list: - print("Error: No figure data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for figure in figures_list: - if 'paper_arxiv_id' not in figure or 'path' not in figure: - print(f"Warning: Skipping figure missing required fields: {figure}") - continue - - rows.append(( - figure['paper_arxiv_id'], - figure['path'], - figure.get('caption', None), - figure.get('label', None), - figure.get('name', None) - )) - - if not rows: - print("No valid figure records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_figures (paper_arxiv_id, path, caption, label, name) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_figures_name_notnull DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} figures from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing figures from JSON: {e}") - return False - - def construct_figures_table_from_api(self, arxiv_ids, dest_dir): - - # Check if papers already exists in the directory - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md = MultiDownload() - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - figure_jsons = file_json['figure'] - for figure_json in figure_jsons: - - figures = figure_iteration_recursive(figure_json=figure_json) - for figure in figures: - path, caption, label = figure - self.insert_figure(paper_arxiv_id=arxiv_id, path=path, caption=caption,label=label) - - except FileNotFoundError: - print(f"Error: The file '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_paper_authors.py b/research_arcade/sql_database/sql_arxiv_paper_authors.py deleted file mode 100644 index 50de38f..0000000 --- a/research_arcade/sql_database/sql_arxiv_paper_authors.py +++ /dev/null @@ -1,354 +0,0 @@ -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivPaperAuthor: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_paper_authors_table(self): - """ - Create the arxiv_paper_authors table with composite uniqueness on (paper_arxiv_id, author_id). - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_paper_authors ( - paper_arxiv_id VARCHAR(100) NOT NULL, - author_id VARCHAR(100) NOT NULL, - author_sequence INTEGER NOT NULL - ) - """) - # Create composite unique index to prevent duplicates - cur.execute(""" - CREATE UNIQUE INDEX IF NOT EXISTS ux_arxiv_paper_authors_unique - ON arxiv_paper_authors (paper_arxiv_id, author_id) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # Insert - # ------------------------- - def insert_paper_author(self, paper_arxiv_id, author_id, author_sequence) -> bool: - """ - Insert a (paper_arxiv_id, author_id, author_sequence) record. - Returns True if inserted, False if already exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_paper_authors (paper_arxiv_id, author_id, author_sequence) - VALUES (%s, %s, %s) - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_authors_unique DO NOTHING - """, - (paper_arxiv_id, author_id, author_sequence) - ) - inserted = cur.rowcount > 0 # 1 if inserted, 0 if conflict - cur.close() - return inserted - finally: - conn.close() - - def get_all_paper_authors(self, is_all_features=True): - """Get all paper-author relationships from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT paper_arxiv_id, author_id, author_sequence FROM arxiv_paper_authors" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paper_neighboring_authors(self, paper_arxiv_id: str): - """ - Get all authors for a given paper, ordered by author_sequence. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT paper_arxiv_id, author_id, author_sequence - FROM arxiv_paper_authors - WHERE paper_arxiv_id = %s - ORDER BY author_sequence ASC - """, - (paper_arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_author_neighboring_papers(self, author_id: str): - """ - Get all papers for a given author. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT paper_arxiv_id, author_id, author_sequence - FROM arxiv_paper_authors - WHERE author_id = %s - """, - (author_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def delete_paper_author_by_id(self, paper_arxiv_id: str, author_id: str) -> bool: - """ - Delete a specific paper-author relationship. - Returns True if deleted, False if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - DELETE FROM arxiv_paper_authors - WHERE paper_arxiv_id = %s AND author_id = %s - """, - (paper_arxiv_id, author_id) - ) - deleted = cur.rowcount > 0 - cur.close() - return deleted - finally: - conn.close() - - def delete_paper_author_by_paper_id(self, paper_arxiv_id: str) -> int: - """ - Delete all author relationships for a given paper. - Returns the count of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paper_authors WHERE paper_arxiv_id = %s", - (paper_arxiv_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def delete_paper_author_by_author_id(self, author_id: str) -> int: - """ - Delete all paper relationships for a given author. - Returns the count of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paper_authors WHERE author_id = %s", - (author_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-author relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, author_id, author_sequence - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - df = pd.read_csv(csv_file) - - required_cols = ['paper_arxiv_id', 'author_id', 'author_sequence'] - missing_cols = [col for col in required_cols if col not in df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - rows = list(df[required_cols].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paper_authors (paper_arxiv_id, author_id, author_sequence) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_authors_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paper-author relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-author relationships from CSV: {e}") - return False - - def construct_table_from_json(self, json_file): - """ - Construct the paper-author relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - { - "paper_arxiv_id": "1706.03762v7", - "author_id": "12345", - "author_sequence": 1 - }, - ... - ] - - Or: - { - "paper_authors": [ - { - "paper_arxiv_id": "1706.03762v7", - "author_id": "12345", - "author_sequence": 1 - }, - ... - ] - } - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_authors' in json_data: - relations_list = json_data['paper_authors'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-author data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for relation in relations_list: - if 'paper_arxiv_id' not in relation or 'author_id' not in relation or 'author_sequence' not in relation: - print(f"Warning: Skipping relation missing required fields: {relation}") - continue - - rows.append(( - relation['paper_arxiv_id'], - relation['author_id'], - relation['author_sequence'] - )) - - if not rows: - print("No valid paper-author records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paper_authors (paper_arxiv_id, author_id, author_sequence) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_authors_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paper-author relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-author relationships from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_paper_categories.py b/research_arcade/sql_database/sql_arxiv_paper_categories.py deleted file mode 100644 index 33b1bac..0000000 --- a/research_arcade/sql_database/sql_arxiv_paper_categories.py +++ /dev/null @@ -1,344 +0,0 @@ -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivPaperCategory: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_paper_category_table(self): - """ - Create the arxiv_paper_category table with a composite uniqueness constraint. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_paper_category ( - paper_arxiv_id VARCHAR(100) NOT NULL, - category_id VARCHAR(100) NOT NULL - ) - """) - # Composite unique index for preventing duplicates - cur.execute(""" - CREATE UNIQUE INDEX IF NOT EXISTS ux_arxiv_paper_category_unique - ON arxiv_paper_category (paper_arxiv_id, category_id) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # Insert - # ------------------------- - def insert_paper_category(self, paper_arxiv_id, category_id) -> bool: - """ - Insert a (paper_arxiv_id, category_id) mapping. - Returns True if inserted, False if it already exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_paper_category (paper_arxiv_id, category_id) - VALUES (%s, %s) - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_category_unique DO NOTHING - """, - (paper_arxiv_id, category_id) - ) - inserted = cur.rowcount > 0 # rowcount == 1 if inserted, 0 if skipped due to conflict - cur.close() - return inserted - finally: - conn.close() - - def get_all_paper_categories(self): - """Get all paper-category relationships from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT paper_arxiv_id, category_id FROM arxiv_paper_category" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paper_neighboring_categories(self, paper_arxiv_id: str): - """ - Get all categories for a given paper. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT paper_arxiv_id, category_id - FROM arxiv_paper_category - WHERE paper_arxiv_id = %s - """, - (paper_arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_category_neighboring_papers(self, category_id: str): - """ - Get all papers for a given category. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT paper_arxiv_id, category_id - FROM arxiv_paper_category - WHERE category_id = %s - """, - (category_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def delete_paper_category_by_id(self, paper_arxiv_id: str, category_id: str) -> bool: - """ - Delete a specific paper-category relationship. - Returns True if deleted, False if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - DELETE FROM arxiv_paper_category - WHERE paper_arxiv_id = %s AND category_id = %s - """, - (paper_arxiv_id, category_id) - ) - deleted = cur.rowcount > 0 - cur.close() - return deleted - finally: - conn.close() - - def delete_paper_category_by_paper_id(self, paper_arxiv_id: str) -> int: - """ - Delete all category relationships for a given paper. - Returns the count of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paper_category WHERE paper_arxiv_id = %s", - (paper_arxiv_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def delete_paper_category_by_category_id(self, category_id: str) -> int: - """ - Delete all paper relationships for a given category. - Returns the count of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paper_category WHERE category_id = %s", - (category_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-category relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, category_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - df = pd.read_csv(csv_file) - - required_cols = ['paper_arxiv_id', 'category_id'] - missing_cols = [col for col in required_cols if col not in df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - rows = list(df[required_cols].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paper_category (paper_arxiv_id, category_id) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_category_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paper-category relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-category relationships from CSV: {e}") - return False - - def construct_table_from_json(self, json_file): - """ - Construct the paper-category relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - {"paper_arxiv_id": "1706.03762v7", "category_id": "cs.AI"}, - {"paper_arxiv_id": "1706.03762v7", "category_id": "cs.LG"}, - ... - ] - - Or: - { - "paper_categories": [ - {"paper_arxiv_id": "1706.03762v7", "category_id": "cs.AI"}, - ... - ] - } - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_categories' in json_data: - relations_list = json_data['paper_categories'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-category data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for relation in relations_list: - if 'paper_arxiv_id' not in relation or 'category_id' not in relation: - print(f"Warning: Skipping relation missing required fields: {relation}") - continue - - rows.append(( - relation['paper_arxiv_id'], - relation['category_id'] - )) - - if not rows: - print("No valid paper-category records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paper_category (paper_arxiv_id, category_id) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_category_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paper-category relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-category relationships from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_paper_figures.py b/research_arcade/sql_database/sql_arxiv_paper_figures.py deleted file mode 100644 index 2fd5388..0000000 --- a/research_arcade/sql_database/sql_arxiv_paper_figures.py +++ /dev/null @@ -1,344 +0,0 @@ -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivPaperFigure: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_paper_figures_table(self): - """ - Create the arxiv_paper_figures table with a composite uniqueness constraint. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_paper_figures ( - paper_arxiv_id VARCHAR(100) NOT NULL, - figure_id INTEGER NOT NULL - ) - """) - # Composite unique index for conflict prevention - cur.execute(""" - CREATE UNIQUE INDEX IF NOT EXISTS ux_arxiv_paper_figures_unique - ON arxiv_paper_figures (paper_arxiv_id, figure_id) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # Insert - # ------------------------- - def insert_paper_figure(self, paper_arxiv_id, figure_id) -> bool: - """ - Insert a (paper_arxiv_id, figure_id) record. - Returns True if inserted, False if the pair already exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_paper_figures (paper_arxiv_id, figure_id) - VALUES (%s, %s) - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_figures_unique DO NOTHING - """, - (paper_arxiv_id, figure_id) - ) - inserted = cur.rowcount > 0 - cur.close() - return inserted - finally: - conn.close() - - def get_all_paper_figures(self): - """Get all paper-figure relationships from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT paper_arxiv_id, figure_id FROM arxiv_paper_figures" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paper_neighboring_figures(self, paper_arxiv_id: str): - """ - Get all figures for a given paper. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT paper_arxiv_id, figure_id - FROM arxiv_paper_figures - WHERE paper_arxiv_id = %s - """, - (paper_arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_figure_neighboring_papers(self, figure_id: int): - """ - Get all papers that contain a given figure. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - SELECT paper_arxiv_id, figure_id - FROM arxiv_paper_figures - WHERE figure_id = %s - """, - (figure_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def delete_paper_figure_by_id(self, paper_arxiv_id: str, figure_id: int) -> bool: - """ - Delete a specific paper-figure relationship. - Returns True if deleted, False if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - DELETE FROM arxiv_paper_figures - WHERE paper_arxiv_id = %s AND figure_id = %s - """, - (paper_arxiv_id, figure_id) - ) - deleted = cur.rowcount > 0 - cur.close() - return deleted - finally: - conn.close() - - def delete_paper_figure_by_paper_id(self, paper_arxiv_id: str) -> int: - """ - Delete all figure relationships for a given paper. - Returns the count of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paper_figures WHERE paper_arxiv_id = %s", - (paper_arxiv_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def delete_paper_figure_by_figure_id(self, figure_id: int) -> int: - """ - Delete all paper relationships for a given figure. - Returns the count of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paper_figures WHERE figure_id = %s", - (figure_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def construct_table_from_csv(self, csv_file): - """ - Construct the paper-figure relationships from an external CSV file. - - Args: - csv_file: Path to the CSV file - - Expected CSV format: - - Required columns: paper_arxiv_id, figure_id - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - try: - df = pd.read_csv(csv_file) - - required_cols = ['paper_arxiv_id', 'figure_id'] - missing_cols = [col for col in required_cols if col not in df.columns] - - if missing_cols: - print(f"Error: External CSV is missing required columns: {missing_cols}") - return False - - rows = list(df[required_cols].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paper_figures (paper_arxiv_id, figure_id) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_figures_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paper-figure relationships from {csv_file}") - return True - - except Exception as e: - print(f"Error importing paper-figure relationships from CSV: {e}") - return False - - def construct_table_from_json(self, json_file): - """ - Construct the paper-figure relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - {"paper_arxiv_id": "1706.03762v7", "figure_id": 1}, - {"paper_arxiv_id": "1706.03762v7", "figure_id": 2}, - ... - ] - - Or: - { - "paper_figures": [ - {"paper_arxiv_id": "1706.03762v7", "figure_id": 1}, - ... - ] - } - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paper_figures' in json_data: - relations_list = json_data['paper_figures'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paper-figure data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for relation in relations_list: - if 'paper_arxiv_id' not in relation or 'figure_id' not in relation: - print(f"Warning: Skipping relation missing required fields: {relation}") - continue - - rows.append(( - relation['paper_arxiv_id'], - relation['figure_id'] - )) - - if not rows: - print("No valid paper-figure records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paper_figures (paper_arxiv_id, figure_id) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_figures_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paper-figure relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paper-figure relationships from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_paper_tables.py b/research_arcade/sql_database/sql_arxiv_paper_tables.py deleted file mode 100644 index 0deeaec..0000000 --- a/research_arcade/sql_database/sql_arxiv_paper_tables.py +++ /dev/null @@ -1,71 +0,0 @@ -import psycopg2 - -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor -class SQLArxivPaperTable: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.autocommit = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - def create_paper_tables_table(self): - """ - Create the arxiv_paper_tables table with a composite uniqueness constraint. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_paper_tables ( - paper_arxiv_id VARCHAR(100) NOT NULL, - table_id INTEGER NOT NULL - ) - """) - # Composite unique index mirrors CSV conflict check - cur.execute(""" - CREATE UNIQUE INDEX IF NOT EXISTS ux_arxiv_paper_tables_unique - ON arxiv_paper_tables (paper_arxiv_id, table_id) - """) - cur.close() - finally: - conn.close() - - def insert_paper_table(self, paper_arxiv_id, table_id): - """ - Insert a (paper_arxiv_id, table_id) edge. - Returns True if inserted, False if the pair already exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_paper_tables (paper_arxiv_id, table_id) - VALUES (%s, %s) - ON CONFLICT ON CONSTRAINT ux_arxiv_paper_tables_unique DO NOTHING - """, - (paper_arxiv_id, table_id) - ) - inserted = cur.rowcount > 0 # rowcount==1 if inserted, 0 if conflict - cur.close() - return inserted - finally: - conn.close() diff --git a/research_arcade/sql_database/sql_arxiv_papers.py b/research_arcade/sql_database/sql_arxiv_papers.py deleted file mode 100644 index 0943296..0000000 --- a/research_arcade/sql_database/sql_arxiv_papers.py +++ /dev/null @@ -1,483 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd # only used for CSV import -import json - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivPapers: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_papers_table(self): - """ - Creates the arxiv_papers table. - arxiv_id is UNIQUE to mirror your CSV conflict rule. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_papers ( - id SERIAL PRIMARY KEY, - arxiv_id VARCHAR(64) NOT NULL UNIQUE, - base_arxiv_id VARCHAR(64) NOT NULL, - version VARCHAR(32) NOT NULL, - title TEXT NOT NULL, - abstract TEXT, - submit_date TIMESTAMP NULL, - metadata JSONB - ) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_paper(self, arxiv_id, base_arxiv_id, version, title, - abstract=None, submit_date=None, metadata=None) -> Optional[int]: - """ - Insert a paper; returns generated id or None if arxiv_id already exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_papers - (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (arxiv_id) DO NOTHING - RETURNING id - """, - ( - arxiv_id, - base_arxiv_id, - version, - title, - abstract, - submit_date, # can be a string parsable by PG or a datetime - psycopg2.extras.Json(metadata) if metadata is not None else None - ) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_paper_by_arxiv_id(self, arxiv_id: str) -> bool: - """ - Delete by arxiv_id; returns True if a row was deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_papers WHERE arxiv_id = %s RETURNING id", (arxiv_id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def delete_paper_by_year(self, year: int) -> int: - """ - Delete all papers whose submit_date falls in the given year. - Returns number of rows deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - DELETE FROM arxiv_papers - WHERE submit_date IS NOT NULL - AND EXTRACT(YEAR FROM submit_date)::int = %s - RETURNING id - """, - (year,) - ) - rows = cur.fetchall() - cur.close() - return len(rows) if rows else 0 - finally: - conn.close() - - def update_paper(self, arxiv_id: str, base_arxiv_id=None, version=None, title=None, - abstract=None, submit_date=None, metadata=None) -> bool: - """ - Partial update by arxiv_id. Only non-None fields are updated. - Returns True if a row was updated. - """ - sets: List[str] = [] - vals: List = [] - - if base_arxiv_id is not None: - sets.append("base_arxiv_id = %s") - vals.append(base_arxiv_id) - if version is not None: - sets.append("version = %s") - vals.append(version) - if title is not None: - sets.append("title = %s") - vals.append(title) - if abstract is not None: - sets.append("abstract = %s") - vals.append(abstract) - if submit_date is not None: - sets.append("submit_date = %s") - vals.append(submit_date) - if metadata is not None: - sets.append("metadata = %s") - vals.append(psycopg2.extras.Json(metadata)) - - if not sets: - return False - - sql = f"UPDATE arxiv_papers SET {', '.join(sets)} WHERE arxiv_id = %s RETURNING id" - vals.append(arxiv_id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_paper_by_arxiv_id(self, arxiv_id: str, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - If return_all=True: returns a list of such tuples. - Returns None if not found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata " - "FROM arxiv_papers WHERE arxiv_id = %s", - (arxiv_id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def check_paper_exists(self, arxiv_id: str) -> bool: - """ - Returns True if a paper with the given arxiv_id exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_papers WHERE arxiv_id = %s LIMIT 1", (arxiv_id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_all_papers(self, is_all_features=True): - """Get all papers from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata " - "FROM arxiv_papers" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_papers_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with required columns: - ['arxiv_id', 'base_arxiv_id', 'version', 'title'] - Optional columns (auto-filled to NULL if missing): - ['abstract', 'submit_date', 'metadata'] - Ignores any 'id' in the CSV; DB assigns SERIAL ids. - Skips conflicts on arxiv_id (ON CONFLICT DO NOTHING). - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - - required = ['arxiv_id', 'base_arxiv_id', 'version', 'title'] - missing = [c for c in required if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - # Ensure optional columns exist - for c in ['abstract', 'submit_date', 'metadata']: - if c not in df.columns: - df[c] = None - - # If metadata is present as strings/dicts, keep it as-is; psycopg2.Json handles serialization. - rows: List[Tuple] = [] - for _, r in df.iterrows(): - meta_val = r['metadata'] - # If metadata column is a string that looks like JSON, try to keep it raw; psycopg2.Json can take dicts as well. - if isinstance(meta_val, str): - try: - # Best-effort parse to dict for proper JSONB storage - meta_val = json.loads(meta_val) - except Exception: - # Leave as string; PG will accept it as text → JSONB cast may fail if not valid JSON. - # Safer approach: leave it None if invalid JSON strings are expected. - pass - - rows.append(( - r['arxiv_id'], - r['base_arxiv_id'], - r['version'], - r['title'], - r['abstract'], - r['submit_date'], - meta_val - )) - - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_papers - (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - VALUES %s - ON CONFLICT (arxiv_id) DO NOTHING - """, - [ - ( - arxiv_id, - base_arxiv_id, - version, - title, - abstract, - submit_date, - psycopg2.extras.Json(metadata) if metadata is not None else None - ) - for (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - in rows - ], - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} papers from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_papers_table_from_csv for consistency.""" - return self.construct_papers_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the papers table from an external JSON file. - - Args: - json_file: Path to the JSON file containing paper data - - Expected JSON format: - [ - { - "arxiv_id": "1706.03762v7", - "base_arxiv_id": "1706.03762", - "version": 7, - "title": "Attention Is All You Need", - "abstract": "...", - "submit_date": "2017-06-12", - "metadata": {"venue": "NeurIPS 2017"} - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'papers' in json_data: - papers_list = json_data['papers'] - else: - papers_list = [json_data] - elif isinstance(json_data, list): - papers_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not papers_list: - print("Error: No paper data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for paper in papers_list: - if 'arxiv_id' not in paper or 'base_arxiv_id' not in paper or 'version' not in paper or 'title' not in paper: - print(f"Warning: Skipping paper missing required fields: {paper}") - continue - - # Handle metadata - keep as dict for psycopg2.Json - metadata = paper.get('metadata', None) - - rows.append(( - paper['arxiv_id'], - paper['base_arxiv_id'], - paper['version'], - paper['title'], - paper.get('abstract', None), - paper.get('submit_date', None), - metadata - )) - - if not rows: - print("No valid paper records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_papers - (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - VALUES %s - ON CONFLICT (arxiv_id) DO NOTHING - """, - [ - ( - arxiv_id, - base_arxiv_id, - version, - title, - abstract, - submit_date, - psycopg2.extras.Json(metadata) if metadata is not None else None - ) - for (arxiv_id, base_arxiv_id, version, title, abstract, submit_date, metadata) - in rows - ], - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} papers from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing papers from JSON: {e}") - return False - - def construct_papers_table_from_api(self, arxiv_ids, dest_dir): - - # Check if papers already exists in the directory - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - md = MultiDownload() - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - # Then collect information into databases - for arxiv_id in arxiv_ids: - # add metadata - # read paper information - - base_arxiv_id, version = arxiv_id_processor(arxiv_id=arxiv_id) - # Read metadata from path specified - - try: - metadata_path = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - with open(metadata_path, 'r') as f: - metadata = json.load(f) # Use json.load(), not json.loads() - - # Validate required fields - required_fields = ['title', 'abstract', 'published'] - if not all(field in metadata for field in required_fields): - raise ValueError(f"Missing required fields in metadata for {arxiv_id}") - - self.insert_paper( - arxiv_id=arxiv_id, - base_arxiv_id=base_arxiv_id, - version=version, - title=metadata['title'], - abstract=metadata['abstract'], - submit_date=metadata['published'], - metadata=metadata - ) - except Exception: - print(f"Paper {arxiv_id} does not have metadata downloaded") \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_paragraph_references.py b/research_arcade/sql_database/sql_arxiv_paragraph_references.py deleted file mode 100644 index 0ee3530..0000000 --- a/research_arcade/sql_database/sql_arxiv_paragraph_references.py +++ /dev/null @@ -1,409 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivParagraphReference: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_paragraph_references_table(self): - """ - Creates the paragraph_references table. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_paragraph_references ( - id SERIAL PRIMARY KEY, - paragraph_id VARCHAR(255) NOT NULL, - paper_section VARCHAR(255) NOT NULL, - paper_arxiv_id VARCHAR(100) NOT NULL, - reference_label VARCHAR(255), - reference_type VARCHAR(100) - ) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_paragraph_reference(self, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) -> Optional[int]: - """ - Insert a paragraph reference. Returns generated id. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_paragraph_references - (paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - VALUES (%s, %s, %s, %s, %s) - RETURNING id - """, - (paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_paragraph_reference_by_id(self, id: int) -> bool: - """ - Delete by id. Returns True if deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_paragraph_references WHERE id = %s RETURNING id", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def update_paragraph_reference(self, id: int, paragraph_id=None, paper_section=None, paper_arxiv_id=None, reference_label=None, reference_type=None) -> bool: - """ - Partial update by id. Only non-None fields are updated. - Returns True if updated. - """ - sets, vals = [], [] - - if paragraph_id is not None: - sets.append("paragraph_id = %s") - vals.append(paragraph_id) - if paper_section is not None: - sets.append("paper_section = %s") - vals.append(paper_section) - if paper_arxiv_id is not None: - sets.append("paper_arxiv_id = %s") - vals.append(paper_arxiv_id) - if reference_label is not None: - sets.append("reference_label = %s") - vals.append(reference_label) - if reference_type is not None: - sets.append("reference_type = %s") - vals.append(reference_type) - - if not sets: - return False - - sql = f"UPDATE arxiv_paragraph_references SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_paragraph_reference_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - If return_all=True: returns a list of such tuples. - Returns None if no row found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type " - "FROM arxiv_paragraph_references WHERE id = %s", - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paragraph_references_by_paragraph_id(self, paragraph_id: str): - """ - Returns a list of tuples for a given paragraph_id. - Each tuple: (id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type " - "FROM arxiv_paragraph_references WHERE paragraph_id = %s ORDER BY id", - (paragraph_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_all_paragraph_references(self): - """Get all paragraph references from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type " - "FROM arxiv_paragraph_references" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paragraph_neighboring_references(self, paragraph_id: int): - """Get all references for a given paragraph.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type " - "FROM arxiv_paragraph_references WHERE paragraph_id = %s", - (paragraph_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_reference_neighboring_paragraphs(self, reference_id: int): - """Get all paragraphs that reference a given reference_id.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type " - "FROM arxiv_paragraph_references WHERE id = %s", - (reference_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def delete_paragraph_reference_by_paragraph_id(self, paragraph_id: int) -> int: - """Delete all references for a given paragraph. Returns count of deleted rows.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paragraph_references WHERE paragraph_id = %s", - (paragraph_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def delete_paragraph_reference_by_reference_id(self, reference_id: int) -> int: - """Delete a reference by its id. Returns count of deleted rows.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paragraph_references WHERE id = %s", - (reference_id,) - ) - count = cur.rowcount - cur.close() - return count - finally: - conn.close() - - def check_paragraph_reference_exists(self, id: int) -> bool: - """ - Returns True if a paragraph reference with the given id exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_paragraph_references WHERE id = %s LIMIT 1", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_paragraph_references_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with columns: - ['paragraph_id', 'paper_section', 'paper_arxiv_id', 'reference_label', 'reference_type'] - Ignores any 'id' column; DB assigns SERIAL ids. - Returns True on success, False on validation error or missing file. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['paragraph_id', 'paper_section', 'paper_arxiv_id', 'reference_label', 'reference_type'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - rows: List[Tuple] = list(df[required_cols].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paragraph_references - (paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - VALUES %s - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paragraph references from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_paragraph_references_table_from_csv for consistency.""" - return self.construct_paragraph_references_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the paragraph-reference relationships from an external JSON file. - - Args: - json_file: Path to the JSON file - - Expected JSON format: - [ - { - "paragraph_id": 1, - "paper_section": "introduction", - "paper_arxiv_id": "1706.03762v7", - "reference_label": "fig:1", - "reference_type": "figure" - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paragraph_references' in json_data: - relations_list = json_data['paragraph_references'] - else: - relations_list = [json_data] - elif isinstance(json_data, list): - relations_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not relations_list: - print("Error: No paragraph-reference data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for relation in relations_list: - if 'paragraph_id' not in relation or 'paper_section' not in relation or 'paper_arxiv_id' not in relation or 'reference_label' not in relation or 'reference_type' not in relation: - print(f"Warning: Skipping relation missing required fields: {relation}") - continue - - rows.append(( - relation['paragraph_id'], - relation['paper_section'], - relation['paper_arxiv_id'], - relation['reference_label'], - relation['reference_type'] - )) - - if not rows: - print("No valid paragraph-reference records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paragraph_references - (paragraph_id, paper_section, paper_arxiv_id, reference_label, reference_type) - VALUES %s - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paragraph-reference relationships from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paragraph-reference relationships from JSON: {e}") - return False \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_paragraphs.py b/research_arcade/sql_database/sql_arxiv_paragraphs.py deleted file mode 100644 index 24c924e..0000000 --- a/research_arcade/sql_database/sql_arxiv_paragraphs.py +++ /dev/null @@ -1,527 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd -import json - -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor -from ..arxiv_utils.utils import get_paragraph_num -from ..arxiv_utils.paper_collector.paper_graph_processor import PaperGraphProcessor - - -class SQLArxivParagraphs: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password, - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_paragraphs_table(self): - """ - Creates table with a composite uniqueness on (paragraph_id, paper_arxiv_id, paper_section) - to mirror the CSV conflict check. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_paragraphs ( - id SERIAL PRIMARY KEY, - paragraph_id VARCHAR(255) NOT NULL, - content TEXT, - paper_arxiv_id VARCHAR(100) NOT NULL, - paper_section VARCHAR(255) NOT NULL - ) - """) - # Composite unique index for conflict prevention - cur.execute(""" - CREATE UNIQUE INDEX IF NOT EXISTS ux_arxiv_paragraphs_unique - ON arxiv_paragraphs (paragraph_id, paper_arxiv_id, paper_section) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_paragraph(self, paragraph_id, content, paper_arxiv_id, paper_section, section_id=None, paragraph_in_paper_id=None) -> Optional[int]: - """ - Insert a paragraph. Returns generated id or None if conflicts with the composite unique constraint. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_paragraphs (paragraph_id, content, paper_arxiv_id, paper_section) - VALUES (%s, %s, %s, %s) - ON CONFLICT ON CONSTRAINT ux_arxiv_paragraphs_unique DO NOTHING - RETURNING id - """, - (paragraph_id, content, paper_arxiv_id, paper_section) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_paragraph_by_id(self, id: int) -> bool: - """ - Delete a paragraph by id. Returns True if a row was deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_paragraphs WHERE id = %s RETURNING id", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def delete_paragraph_by_paper_arxiv_id(self, paper_arxiv_id: str) -> int: - """ - Delete all paragraphs for a given paper_arxiv_id. Returns number of rows deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "DELETE FROM arxiv_paragraphs WHERE paper_arxiv_id = %s RETURNING id", - (paper_arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return len(rows) if rows else 0 - finally: - conn.close() - - def delete_paragraph_by_paper_section(self, paper_arxiv_id: str, paper_section: str) -> int: - """ - Delete all paragraphs for a given (paper_arxiv_id, paper_section). - Returns number of rows deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - DELETE FROM arxiv_paragraphs - WHERE paper_arxiv_id = %s AND paper_section = %s - RETURNING id - """, - (paper_arxiv_id, paper_section) - ) - rows = cur.fetchall() - cur.close() - return len(rows) if rows else 0 - finally: - conn.close() - - def update_paragraph(self, id: int, paragraph_id=None, content=None, paper_arxiv_id=None, paper_section=None) -> bool: - """ - Partial update by id. Only non-None fields are updated. - Returns True if a row was updated. - """ - sets: List[str] = [] - vals: List = [] - - if paragraph_id is not None: - sets.append("paragraph_id = %s") - vals.append(paragraph_id) - if content is not None: - sets.append("content = %s") - vals.append(content) - if paper_arxiv_id is not None: - sets.append("paper_arxiv_id = %s") - vals.append(paper_arxiv_id) - if paper_section is not None: - sets.append("paper_section = %s") - vals.append(paper_section) - - if not sets: - return False - - sql = f"UPDATE arxiv_paragraphs SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_paragraph_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, paragraph_id, content, paper_arxiv_id, paper_section) - If return_all=True: returns a list of such tuples. - Returns None if no row found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, content, paper_arxiv_id, paper_section " - "FROM arxiv_paragraphs WHERE id = %s", - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paragraphs_by_arxiv_id(self, arxiv_id: str): - """ - Returns a list of tuples - (id, paragraph_id, content, paper_arxiv_id, paper_section) - or None if no rows exist. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, content, paper_arxiv_id, paper_section " - "FROM arxiv_paragraphs WHERE paper_arxiv_id = %s ORDER BY id", - (arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_paragraphs_by_paper_section(self, paper_arxiv_id: str, paper_section: str): - """ - Returns a list of tuples - (id, paragraph_id, content, paper_arxiv_id, paper_section) - filtered by (paper_arxiv_id, paper_section), or None if no rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, content, paper_arxiv_id, paper_section " - "FROM arxiv_paragraphs WHERE paper_arxiv_id = %s AND paper_section = %s " - "ORDER BY id", - (paper_arxiv_id, paper_section) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def check_paragraph_exists(self, id: int) -> bool: - """ - Returns True if a paragraph with the given id exists. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_paragraphs WHERE id = %s LIMIT 1", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_all_paragraphs(self, is_all_features=True): - """Get all paragraphs from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paragraph_id, content, paper_arxiv_id, paper_section FROM arxiv_paragraphs" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_paragraph_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with columns: - ['paragraph_id', 'content', 'paper_arxiv_id', 'paper_section'] - Ignores any 'id' column; DB assigns SERIAL ids. - Skips rows that violate the composite uniqueness (via ON CONFLICT DO NOTHING). - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['paragraph_id', 'content', 'paper_arxiv_id', 'paper_section'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - rows: List[Tuple] = list(df[required_cols].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - # Use execute_values with ON CONFLICT to bulk-insert - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paragraphs (paragraph_id, content, paper_arxiv_id, paper_section) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paragraphs_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paragraphs from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_paragraph_table_from_csv for consistency.""" - return self.construct_paragraph_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the paragraphs table from an external JSON file. - - Args: - json_file: Path to the JSON file containing paragraph data - - Expected JSON format: - [ - { - "paragraph_id": 0, - "content": "This paper introduces the Transformer...", - "paper_arxiv_id": "1706.03762v7", - "paper_section": "introduction" - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'paragraphs' in json_data: - paragraphs_list = json_data['paragraphs'] - else: - paragraphs_list = [json_data] - elif isinstance(json_data, list): - paragraphs_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not paragraphs_list: - print("Error: No paragraph data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for paragraph in paragraphs_list: - if 'paragraph_id' not in paragraph or 'content' not in paragraph or 'paper_arxiv_id' not in paragraph or 'paper_section' not in paragraph: - print(f"Warning: Skipping paragraph missing required fields: {paragraph}") - continue - - rows.append(( - paragraph['paragraph_id'], - paragraph['content'], - paragraph['paper_arxiv_id'], - paragraph['paper_section'] - )) - - if not rows: - print("No valid paragraph records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_paragraphs (paragraph_id, content, paper_arxiv_id, paper_section) - VALUES %s - ON CONFLICT ON CONSTRAINT ux_arxiv_paragraphs_unique DO NOTHING - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} paragraphs from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing paragraphs from JSON: {e}") - return False - - def construct_paragraphs_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - """ - section id and paragraph order required further - Or maybe we can write a incremental method to process such information - TODO - """ - downloaded_paper_ids = [] - md = MultiDownload() - - data_dir_path = f"{dest_dir}/output" - figures_dir_path = f"{dest_dir}/output/images" - output_dir_path = f"{dest_dir}/output/paragraphs" - pgp = PaperGraphProcessor(data_dir=data_dir_path, figures_dir=figures_dir_path, output_dir=output_dir_path) - - papers = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - papers.append(arxiv_id) - - paper_paths = [] - # We first build paper node - # We loop through the provided arxiv ids of paper. - for arxiv_id in papers: - paper_paths.append(f"{dest_dir}/output/{arxiv_id}.json") - pgp.process_papers(paper_paths) - - # Build the paragraphs - - paragraph_path = f"{dest_dir}/output/paragraphs/text_nodes.jsonl" - with open(paragraph_path) as f: - data = [json.loads(line) for line in f] - - - # Use arxiv_id + section name as key - # Find the smallest paragraph_id generated by knowledge debugger - # Subtract all paragraph id of the same section (of the same paper) with the smallest one to ensure that order starts with zero - section_min_paragraph = {} - - for paragraph in data: - paragraph_id = paragraph.get('id') - # Extract paragraph_id - id_number = get_paragraph_num(paragraph_id) - paper_arxiv_id = paragraph.get('paper_id') - paper_section = paragraph.get('section') - if (paper_arxiv_id, paper_section) not in section_min_paragraph: - section_min_paragraph[(paper_arxiv_id, paper_section)] = int(id_number) - else: - section_min_paragraph[(paper_arxiv_id, paper_section)] = min(section_min_paragraph[(paper_arxiv_id, paper_section)], int(id_number)) - - for paragraph in data: - paragraph_id = paragraph.get('id') - content = paragraph.get('content') - paper_arxiv_id = paragraph.get('paper_id') - paper_section = paragraph.get('section') - id_number = get_paragraph_num(paragraph_id) - id_zero_based = id_number - section_min_paragraph[(paper_arxiv_id, paper_section)] - self.insert_paragraph(paragraph_id=id_zero_based, content=content, paper_arxiv_id=paper_arxiv_id, paper_section=paper_section) - - # paragraph_cite_bib_keys = paragraph.get('cites') - # for bib_key in paragraph_cite_bib_keys: - # self.db.insert_paragraph_citations(paragraph_id=id_zero_based, paper_section=paper_section, citing_arxiv_id=paper_arxiv_id, bib_key=bib_key) - - - # paragraph_ref_labels = paragraph.get('ref_labels') - - - # # def insert_paragraph_reference(self, paragraph_id, paper_arxiv_id, reference_label, reference_type=None): - - # for ref_label in paragraph_ref_labels: - - # ref_type = None - # # First search bib_key in databases. - # # If presented in one of them, we can determine the type of reference - - # is_figure = self.db.check_exist_figure(bib_key=ref_label) - # is_table = self.db.check_exist_table(bib_key=ref_label) - # if is_figure: - # ref_type = 'figure' - # elif is_table: - # ref_type = 'table' - - # self.insert_paragraph(paragraph_id=id_zero_based, paper_section=paper_section, paper_arxiv_id=paper_arxiv_id, paper_section=paper_section, refe) - - # self.db.insert_paragraph_reference(paragraph_id=id_zero_based, paper_section=paper_section, paper_arxiv_id=paper_arxiv_id, reference_label=ref_label, reference_type=ref_type) \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_sections.py b/research_arcade/sql_database/sql_arxiv_sections.py deleted file mode 100644 index 81f8b5c..0000000 --- a/research_arcade/sql_database/sql_arxiv_sections.py +++ /dev/null @@ -1,407 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivSections: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_sections_table(self): - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_sections ( - id SERIAL PRIMARY KEY, - content TEXT, - title VARCHAR(512), - appendix BOOLEAN, - paper_arxiv_id VARCHAR(100) NOT NULL - ) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_section(self, content, title, is_appendix, paper_arxiv_id, section_in_paper_id=None) -> Optional[int]: - """ - Insert a section; returns generated id or None. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_sections (content, title, appendix, paper_arxiv_id) - VALUES (%s, %s, %s, %s) - RETURNING id - """, - (content, title, is_appendix, paper_arxiv_id) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_section_by_id(self, id: int) -> bool: - """ - Delete by id; returns True if a row was deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_sections WHERE id = %s RETURNING id", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def delete_section_by_paper_arxiv_id(self, paper_arxiv_id: str) -> int: - """ - Delete all sections for a paper; returns number of deleted rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_sections WHERE paper_arxiv_id = %s RETURNING id", (paper_arxiv_id,)) - rows = cur.fetchall() - cur.close() - return len(rows) if rows else 0 - finally: - conn.close() - - def update_section(self, id: int, content=None, title=None, is_appendix=None, paper_arxiv_id=None) -> bool: - """ - Partial update; only non-None fields are updated. - Returns True if a row was updated. - """ - sets = [] - vals: List = [] - if content is not None: - sets.append("content = %s") - vals.append(content) - if title is not None: - sets.append("title = %s") - vals.append(title) - if is_appendix is not None: - sets.append("appendix = %s") - vals.append(is_appendix) - if paper_arxiv_id is not None: - sets.append("paper_arxiv_id = %s") - vals.append(paper_arxiv_id) - - if not sets: - return False - - sql = f"UPDATE arxiv_sections SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - def get_section_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, content, title, appendix, paper_arxiv_id) - If return_all=True: returns a list of such tuples. - Returns None if no row found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, content, title, appendix, paper_arxiv_id FROM arxiv_sections WHERE id = %s", - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_sections_by_arxiv_id(self, arxiv_id: str): - """ - Returns a list of tuples: - (id, content, title, appendix, paper_arxiv_id) - or None if no rows. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, content, title, appendix, paper_arxiv_id " - "FROM arxiv_sections WHERE paper_arxiv_id = %s ORDER BY id", - (arxiv_id,) - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_all_sections(self, is_all_features=True): - """Get all sections from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, content, title, appendix, paper_arxiv_id FROM arxiv_sections" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def check_section_exists(self, id: int) -> bool: - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_sections WHERE id = %s LIMIT 1", (id,)) - ok = cur.fetchone() is not None - cur.close() - return ok - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_sections_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with columns: - ['content', 'title', 'appendix', 'paper_arxiv_id'] - Ignores any 'id' in the CSV; DB assigns SERIAL ids. - Returns True on success, False on validation error or missing file. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['content', 'title', 'appendix', 'paper_arxiv_id'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - # Normalize boolean-like 'appendix' column if needed (optional) - if df['appendix'].dtype == object: - true_vals = {'true', '1', 't', 'yes', 'y'} - df['appendix'] = df['appendix'].astype(str).str.lower().isin(true_vals) - - rows: List[Tuple] = list(df[required_cols].itertuples(index=False, name=None)) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_sections (content, title, appendix, paper_arxiv_id) - VALUES %s - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} sections from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_sections_table_from_csv for consistency.""" - return self.construct_sections_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the sections table from an external JSON file. - - Args: - json_file: Path to the JSON file containing section data - - Expected JSON format: - [ - { - "content": "Section content...", - "title": "Introduction", - "appendix": false, - "paper_arxiv_id": "1706.03762v7" - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'sections' in json_data: - sections_list = json_data['sections'] - else: - sections_list = [json_data] - elif isinstance(json_data, list): - sections_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not sections_list: - print("Error: No section data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for section in sections_list: - if 'content' not in section or 'title' not in section or 'appendix' not in section or 'paper_arxiv_id' not in section: - print(f"Warning: Skipping section missing required fields: {section}") - continue - - rows.append(( - section['content'], - section['title'], - section['appendix'], - section['paper_arxiv_id'] - )) - - if not rows: - print("No valid section records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_sections (content, title, appendix, paper_arxiv_id) - VALUES %s - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} sections from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing sections from JSON: {e}") - return False - - def construct_sections_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - md = MultiDownload() - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - section_jsons = file_json['sections'] - - i = 0 - for title, section_json in section_jsons.items(): - i += 1 - is_appendix = section_json['appendix'] == 'true' - content = section_json['content'] - self.insert_section(content=content, title=title, is_appendix=is_appendix, paper_arxiv_id=arxiv_id, section_in_paper_id=i) - - except FileNotFoundError: - print(f"Error: The file '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue \ No newline at end of file diff --git a/research_arcade/sql_database/sql_arxiv_tables.py b/research_arcade/sql_database/sql_arxiv_tables.py deleted file mode 100644 index 6c64f34..0000000 --- a/research_arcade/sql_database/sql_arxiv_tables.py +++ /dev/null @@ -1,380 +0,0 @@ -import os -from typing import Optional, List, Tuple -import psycopg2 -import psycopg2.extras -import pandas as pd -import json -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from ..arxiv_utils.multi_input.multi_download import MultiDownload -from ..arxiv_utils.graph_constructor.node_processor import NodeConstructor -from ..arxiv_utils.utils import arxiv_id_processor - - -class SQLArxivTable: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str): - self.host = host - self.dbname = dbname - self.user = user - self.password = password - self.port = port - self.autocommit = True - - def _get_connection(self): - conn = psycopg2.connect( - host=self.host, - port=self.port, - dbname=self.dbname, - user=self.user, - password=self.password - ) - conn.autocommit = self.autocommit - return conn - - # ------------------------- - # DDL - # ------------------------- - def create_tables_table(self): - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(""" - CREATE TABLE IF NOT EXISTS arxiv_tables ( - id SERIAL PRIMARY KEY, - paper_arxiv_id VARCHAR(100) NOT NULL, - path VARCHAR(1024), - caption TEXT, - label VARCHAR(255), - table_text TEXT - ) - """) - cur.close() - finally: - conn.close() - - # ------------------------- - # CRUD - # ------------------------- - def insert_table(self, paper_arxiv_id, path=None, caption=None, label=None, table_text=None) -> Optional[int]: - """ - Insert a table row; returns generated id (int) or None if not inserted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - """ - INSERT INTO arxiv_tables (paper_arxiv_id, path, caption, label, table_text) - VALUES (%s, %s, %s, %s, %s) - RETURNING id - """, - (paper_arxiv_id, path, caption, label, table_text) - ) - res = cur.fetchone() - cur.close() - return res[0] if res else None - finally: - conn.close() - - def delete_table_by_id(self, id: int) -> bool: - """ - Delete by id; returns True if a row was deleted. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("DELETE FROM arxiv_tables WHERE id = %s RETURNING id", (id,)) - deleted = cur.fetchone() is not None - cur.close() - return deleted - finally: - conn.close() - - def update_table(self, id: int, paper_arxiv_id=None, path=None, caption=None, label=None, table_text=None) -> bool: - """ - Partial update. Only non-None fields are updated. - Returns True if a row was updated. - """ - sets = [] - vals = [] - if paper_arxiv_id is not None: - sets.append("paper_arxiv_id = %s") - vals.append(paper_arxiv_id) - if path is not None: - sets.append("path = %s") - vals.append(path) - if caption is not None: - sets.append("caption = %s") - vals.append(caption) - if label is not None: - sets.append("label = %s") - vals.append(label) - if table_text is not None: - sets.append("table_text = %s") - vals.append(table_text) - - if not sets: - return False # nothing to update - - sql = f"UPDATE arxiv_tables SET {', '.join(sets)} WHERE id = %s RETURNING id" - vals.append(id) - - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute(sql, tuple(vals)) - updated = cur.fetchone() is not None - cur.close() - return updated - finally: - conn.close() - - def get_table_by_id(self, id: int, return_all: bool = False): - """ - If return_all=False: returns a single tuple - (id, paper_arxiv_id, path, caption, label, table_text) - If return_all=True: returns a list of such tuples. - Returns None if no row found. - """ - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paper_arxiv_id, path, caption, label, table_text FROM arxiv_tables WHERE id = %s", - (id,) - ) - rows = cur.fetchall() if return_all else cur.fetchone() - cur.close() - return rows if rows else None - finally: - conn.close() - - def get_all_tables(self, is_all_features=True): - """Get all tables from the database.""" - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute( - "SELECT id, paper_arxiv_id, path, caption, label, table_text FROM arxiv_tables" - ) - rows = cur.fetchall() - cur.close() - return rows if rows else None - finally: - conn.close() - - def check_table_exists(self, id: int) -> bool: - conn = self._get_connection() - try: - cur = conn.cursor() - cur.execute("SELECT 1 FROM arxiv_tables WHERE id = %s LIMIT 1", (id,)) - exists = cur.fetchone() is not None - cur.close() - return exists - finally: - conn.close() - - # ------------------------- - # Bulk import from CSV - # ------------------------- - def construct_tables_table_from_csv(self, csv_file: str) -> bool: - """ - Imports rows from a CSV with columns: - ['paper_arxiv_id', 'path', 'caption', 'label', 'table_text'] - Ignores any 'id' from the CSV and lets DB assign SERIAL ids. - Returns True on success, False on validation error or missing file. - """ - if not os.path.exists(csv_file): - print(f"Error: CSV file {csv_file} does not exist.") - return False - - df = pd.read_csv(csv_file) - required_cols = ['paper_arxiv_id'] - missing = [c for c in required_cols if c not in df.columns] - if missing: - print(f"Error: External CSV is missing required columns: {missing}") - return False - - # Add optional columns if missing - for col in ['path', 'caption', 'label', 'table_text']: - if col not in df.columns: - df[col] = None - - rows: List[Tuple] = list( - df[['paper_arxiv_id', 'path', 'caption', 'label', 'table_text']].itertuples(index=False, name=None) - ) - if not rows: - print("No rows to import.") - return True - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_tables (paper_arxiv_id, path, caption, label, table_text) - VALUES %s - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} tables from {csv_file}") - return True - - def construct_table_from_csv(self, csv_file: str) -> bool: - """Alias for construct_tables_table_from_csv for consistency.""" - return self.construct_tables_table_from_csv(csv_file) - - def construct_table_from_json(self, json_file): - """ - Construct the tables table from an external JSON file. - - Args: - json_file: Path to the JSON file containing table data - - Expected JSON format: - [ - { - "paper_arxiv_id": "1706.03762v7", - "path": "/path/to/table1.tex", - "caption": "Model performance comparison", - "label": "tab:performance", - "table_text": "Table content..." - }, - ... - ] - - Returns: - bool: True if successful, False otherwise - """ - if not os.path.exists(json_file): - print(f"Error: JSON file {json_file} does not exist.") - return False - - try: - # Load JSON data - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - - # Handle different JSON structures - if isinstance(json_data, dict): - if 'tables' in json_data: - tables_list = json_data['tables'] - else: - tables_list = [json_data] - elif isinstance(json_data, list): - tables_list = json_data - else: - print("Error: JSON file must contain either a list or a dictionary") - return False - - if not tables_list: - print("Error: No table data found in JSON file") - return False - - # Convert to list of tuples for bulk insert - rows = [] - for table in tables_list: - if 'paper_arxiv_id' not in table: - print(f"Warning: Skipping table missing required field 'paper_arxiv_id': {table}") - continue - - rows.append(( - table['paper_arxiv_id'], - table.get('path', None), - table.get('caption', None), - table.get('label', None), - table.get('table_text', None) - )) - - if not rows: - print("No valid table records to import") - return False - - conn = self._get_connection() - try: - cur = conn.cursor() - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO arxiv_tables (paper_arxiv_id, path, caption, label, table_text) - VALUES %s - """, - rows, - page_size=1000 - ) - cur.close() - finally: - conn.close() - - print(f"Successfully imported {len(rows)} tables from {json_file}") - return True - - except json.JSONDecodeError as e: - print(f"Error: Invalid JSON file - {e}") - return False - except Exception as e: - print(f"Error importing tables from JSON: {e}") - return False - - def construct_tables_table_from_api(self, arxiv_ids, dest_dir): - # Check if papers already exists in the directory - md = MultiDownload() - downloaded_paper_ids = [] - for arxiv_id in arxiv_ids: - paper_dir = f"{dest_dir}/{arxiv_id}/{arxiv_id}_metadata.json" - - if not os.path.exists(paper_dir): - downloaded_paper_ids.append(arxiv_id) - - for arxiv_id in downloaded_paper_ids: - try: - md.download_arxiv(input=arxiv_id, input_type = "id", output_type="latex", dest_dir=dest_dir) - print(f"paper with id {arxiv_id} downloaded") - downloaded_paper_ids.append(arxiv_id) - except RuntimeError as e: - print(f"[ERROR] Failed to download {arxiv_id}: {e}") - continue - - for arxiv_id in arxiv_ids: - # Search if the corresponding paper graph exists - - json_path = f"{dest_dir}/output/{arxiv_id}.json" - if not os.path.exists(json_path): - # arxiv_id_graph.append(arxiv_id) - try: - # Build corresponding graph - md.build_paper_graph( - input=arxiv_id, - input_type="id", - dest_dir=dest_dir - ) - except Exception as e: - print(f"[Warning] Failed to process papers: {e}") - continue - - try: - with open(json_path, 'r') as file: - file_json = json.load(file) - table_jsons = file_json['table'] - for table_json in table_jsons: - - caption = table_json['caption'] - label = table_json['label'] - table = table_json['tabular'] - self.insert_table(paper_arxiv_id=arxiv_id, path=None, caption=caption, label=label, table_text=table) - - except FileNotFoundError: - print(f"Error: The file '{json_path}' was not found.") - continue - except json.JSONDecodeError: - print(f"Error: Could not decode JSON from '{json_path}'. Check if the file contains valid JSON.") - continue - except Exception as e: - print(f"An unexpected error occurred: {e}") - continue \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_arxiv.py b/research_arcade/sql_database/sql_openreview_arxiv.py deleted file mode 100644 index fe9ac43..0000000 --- a/research_arcade/sql_database/sql_openreview_arxiv.py +++ /dev/null @@ -1,269 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewArxiv: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host=host, dbname=dbname, - user=user, password=password, port=port - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_openreview_arxiv_table() - - def create_openreview_arxiv_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_arxiv ( - venue TEXT, - paper_openreview_id VARCHAR(255), - arxiv_id VARCHAR(255), - title TEXT, - PRIMARY KEY (venue, paper_openreview_id) - ); - """ - # Execute the SQL to create the table - self.cur.execute(create_table_sql) - - def insert_openreview_arxiv(self, venue, paper_openreview_id, arxiv_id, title) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_arxiv (venue, paper_openreview_id, arxiv_id, title) - VALUES (%s, %s, %s, %s) - ON CONFLICT (venue, paper_openreview_id) DO NOTHING - RETURNING (venue, paper_openreview_id); - """ - - venue = self._clean_string(venue) - paper_openreview_id = self._clean_string(paper_openreview_id) - arxiv_id = self._clean_string(arxiv_id) - title = self._clean_string(title) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, paper_openreview_id, arxiv_id, title)) - - # Get the inserted paper's openreview id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_openreview_arxiv_by_openreview_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given paper_openreview_id and delete them - select_sql = """ - SELECT * FROM openreview_arxiv WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "arxiv_id", "title"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_arxiv WHERE paper_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id}.") - return None - - def delete_openreview_arxiv_by_id(self, paper_openreview_id: str, arxiv_id: str) -> Optional[pd.DataFrame]: - # search for records with the given paper_openreview_id and delete them - select_sql = """ - SELECT * FROM openreview_arxiv WHERE paper_openreview_id = %s AND arxiv_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id, arxiv_id)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "arxiv_id", "title"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_arxiv WHERE paper_openreview_id = %s AND arxiv_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id, arxiv_id)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id} and arxiv_id = {arxiv_id}.") - return records_df - else: - print(f"No records found in 'openreview_arxiv' with paper_openreview_id = {paper_openreview_id} and arxiv_id = {arxiv_id}.") - return None - - def delete_openreview_arxiv_by_arxiv_id(self, arxiv_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_arxiv WHERE arxiv_id = %s; - """ - self.cur.execute(select_sql, (arxiv_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "arxiv_id", "title"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_arxiv WHERE arxiv_id = %s; - """ - self.cur.execute(delete_sql, (arxiv_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_arxiv' with arxiv_id = {arxiv_id}.") - return records_df - else: - print(f"No records found in 'openreview_arxiv' with arxiv_id = {arxiv_id}.") - return None - - def delete_openreview_arxiv_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search for records with the given venue and delete them - select_sql = """ - SELECT * FROM openreview_arxiv WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "arxiv_id", "title"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_arxiv WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_arxiv' where venue = {venue}.") - return records_df - else: - print(f"No records found in 'openreview_arxiv' for venue = {venue}.") - return None - - def get_openreview_neighboring_arxivs(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, arxiv_id, title FROM openreview_arxiv - WHERE paper_openreview_id = %s; - """, (paper_openreview_id,)) - - openreview_arxiv = self.cur.fetchall() - - if openreview_arxiv is not None: - openreview_arxiv_df = pd.DataFrame(openreview_arxiv, columns=["venue", "paper_openreview_id", "arxiv_id", "title"]) - return openreview_arxiv_df - else: - return None - - def get_arxiv_neighboring_openreviews(self, arxiv_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, arxiv_id, title FROM openreview_arxiv - WHERE arxiv_id = %s; - """, (arxiv_id,)) - - openreview_arxiv = self.cur.fetchall() - - if openreview_arxiv is not None: - openreview_arxiv_df = pd.DataFrame(openreview_arxiv, columns=["venue", "paper_openreview_id", "arxiv_id", "title"]) - return openreview_arxiv_df - else: - return None - - def get_openreview_arxiv_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, arxiv_id, title FROM openreview_arxiv - WHERE venue = %s; - """, (venue,)) - - openreview_arxiv = self.cur.fetchall() - - if openreview_arxiv is not None: - openreview_arxiv_df = pd.DataFrame(openreview_arxiv, columns=["venue", "paper_openreview_id", "arxiv_id", "title"]) - return openreview_arxiv_df - else: - return None - - def get_all_openreview_arxiv(self) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, arxiv_id, title FROM openreview_arxiv; - """) - - openreview_arxiv = self.cur.fetchall() - - if openreview_arxiv is not None: - openreview_arxiv_df = pd.DataFrame(openreview_arxiv, columns=["venue", "paper_openreview_id", "arxiv_id", "title"]) - return openreview_arxiv_df - else: - return None - - def check_openreview_arxiv_exists(self, venue: str, paper_openreview_id: str) -> bool: - self.cur.execute(""" - SELECT 1 FROM openreview_arxiv - WHERE venue = %s AND paper_openreview_id = %s; - """, (venue, paper_openreview_id)) - - return self.cur.fetchone() is not None - - def construct_openreview_arxiv_table_from_api(self, venue: str) -> bool: - # crawl openreview arxiv data from openreview.net - print(f"Crawling openreview arxiv data for venue: {venue}...") - openreview_arxiv_data = self.openreview_crawler.crawl_openreview_arxiv_data_from_api(venue) - - # insert data into openreview_arxiv table - if len(openreview_arxiv_data) > 0: - print("Inserting data into 'openreview_arxiv' table...") - for data in tqdm(openreview_arxiv_data): - self.insert_openreview_arxiv(**data) - return True - else: - print("No new openreview arxiv data to insert.") - return False - - def construct_openreview_arxiv_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - # read openreview arxiv data from csv file - print(f"Reading openreview arxiv data from {csv_file}...") - openreview_arxiv_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into openreview_arxiv table - if len(openreview_arxiv_data) > 0: - print("Inserting data into 'openreview_arxiv' table...") - for data in tqdm(openreview_arxiv_data): - self.insert_openreview_arxiv(**data) - return True - else: - print("No new openreview arxiv data to insert.") - return False - - def construct_openreview_arxiv_table_from_json(self, json_file: str) -> None: - if not os.path.exists(json_file): - return False - else: - # read openreview arxiv data from json file - print(f"Reading openreview arxiv data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - openreview_arxiv_data = json.load(f) - - # insert data into openreview_arxiv table - if len(openreview_arxiv_data) > 0: - print("Inserting data into 'openreview_arxiv' table...") - for data in tqdm(openreview_arxiv_data): - self.insert_openreview_arxiv(**data) - return True - else: - print("No new openreview arxiv data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_authors.py b/research_arcade/sql_database/sql_openreview_authors.py deleted file mode 100644 index 9ebc365..0000000 --- a/research_arcade/sql_database/sql_openreview_authors.py +++ /dev/null @@ -1,281 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewAuthors: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - self.conn = psycopg2.connect( - host=host, - dbname=dbname, - user=user, - password=password, - port=port - ) - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_author_table() - - def create_author_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_authors ( - venue TEXT, - author_openreview_id VARCHAR(255), - author_full_name TEXT, - email TEXT, - affiliation TEXT, - homepage TEXT, - dblp TEXT, - PRIMARY KEY (venue, author_openreview_id) - ); - """ - # Execute the SQL to create the table - self.cur.execute(create_table_sql) - - def insert_author(self, venue: str, author_openreview_id: str, author_full_name: str, email: str, - affiliation: str, homepage: str, dblp: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_authors (venue, author_openreview_id, author_full_name, email, affiliation, homepage, dblp) - VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (venue, author_openreview_id) DO NOTHING - RETURNING (venue, author_openreview_id); - """ - - venue = self._clean_string(venue) - author_openreview_id = self._clean_string(author_openreview_id) - author_full_name = self._clean_string(author_full_name) - email = self._clean_string(email) - affiliation = self._clean_string(affiliation) - homepage = self._clean_string(homepage) - dblp = self._clean_string(dblp) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, author_openreview_id, author_full_name, email, affiliation, homepage, dblp)) - - # Get the inserted author id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_author_by_id(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - # search the row based on primary key - select_sql = """ - SELECT * FROM openreview_authors WHERE author_openreview_id = %s; - """ - self.cur.execute(select_sql, (author_openreview_id,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'author_openreview_id', 'author_full_name', 'email', - 'affiliation', 'homepage', 'dblp'] - # author_dict = dict(zip(columns, row)) - author_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_authors WHERE author_openreview_id = %s; - """ - self.cur.execute(delete_sql, (author_openreview_id,)) - self.conn.commit() - - print(f"Author with author_openreview_id {author_openreview_id} deleted successfully.") - return author_df - else: - print(f"No author found with author_openreview_id {author_openreview_id}.") - return None - - def delete_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search the row based on primary key - select_sql = """ - SELECT * FROM openreview_authors WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'author_openreview_id', 'author_full_name', 'email', - 'affiliation', 'homepage', 'dblp'] - # author_dict = dict(zip(columns, row)) - author_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_authors WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All openreview_authors in venue {venue} deleted successfully.") - return author_df - else: - print(f"No openreview_authors found in venue {venue}.") - return None - - def update_author(self, venue: str, author_openreview_id: str, author_full_name: str, email: str, - affiliation: str, homepage: str, dblp: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT * FROM openreview_authors WHERE author_openreview_id = %s AND venue = %s; - """ - - self.cur.execute(select_sql, (author_openreview_id, venue,)) - row = self.cur.fetchone() - - if not row: - print(f"No author found with author_openreview_id {author_openreview_id}.") - return None - else: - columns = ['venue', 'author_openreview_id', 'author_full_name', - 'email', 'affiliation', 'homepage', 'dblp'] - # original_record = dict(zip(columns, row)) - author_df = pd.DataFrame([row], columns=columns) - - update_sql = """ - UPDATE openreview_authors - SET venue = %s, - author_full_name = %s, - email = %s, - affiliation = %s, - homepage = %s, - dblp = %s - WHERE author_openreview_id = %s; - """ - - venue = self._clean_string(venue) - author_openreview_id = self._clean_string(author_openreview_id) - author_full_name = self._clean_string(author_full_name) - email = self._clean_string(email) - affiliation = self._clean_string(affiliation) - homepage = self._clean_string(homepage) - dblp = self._clean_string(dblp) - - self.cur.execute(update_sql, (venue, author_full_name, email, affiliation, homepage, dblp, author_openreview_id)) - self.conn.commit() - - print(f"Author with author_openreview_id {author_openreview_id} updated successfully.") - return author_df - - def get_author_by_id(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT * FROM openreview_authors WHERE author_openreview_id = %s; - """ - self.cur.execute(select_sql, (author_openreview_id,)) - row = self.cur.fetchall() - - if not row: - print(f"No author found with author_openreview_id {author_openreview_id}.") - return None - else: - columns = ['venue', 'author_openreview_id', 'author_full_name', - 'email', 'affiliation', 'homepage', 'dblp'] - # original_record = dict(zip(columns, row)) - author_df = pd.DataFrame(row, columns=columns) - return author_df - - def get_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # Query to select all records for a specific venue - select_sql = """ - SELECT * FROM openreview_authors WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if not rows: - print(f"No openreview_authors found in venue {venue}.") - return None - else: - columns = ['venue', 'author_openreview_id', 'author_full_name', - 'email', 'affiliation', 'homepage', 'dblp'] - # original_record = dict(zip(columns, row)) - authors_df = pd.DataFrame(rows, columns=columns) - return authors_df - - def get_all_authors(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - if is_all_features: - select_query = """ - SELECT venue, author_openreview_id, author_full_name, email, affiliation, homepage, dblp - FROM openreview_authors ORDER BY author_openreview_id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - authors= self.cur.fetchall() - - # Return the result as a list of tuples (venue, author_openreview_id, author_full_name) - authors_df = pd.DataFrame(authors, columns=["venue", "author_openreview_id", "author_full_name", "email", "affiliation", "homepage", "dblp"]) - return authors_df - else: - select_query = """ - SELECT venue, author_openreview_id, author_full_name - FROM openreview_authors ORDER BY author_openreview_id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - authors= self.cur.fetchall() - - # Return the result as a list of tuples (venue, author_openreview_id, author_full_name) - authors_df = pd.DataFrame(authors, columns=["venue", "author_openreview_id", "author_full_name"]) - return authors_df - - def check_author_exists(self, author_openreview_id: str) -> bool: - self.cur.execute("SELECT 1 FROM openreview_authors WHERE author_openreview_id = %s LIMIT 1;", (author_openreview_id,)) - result = self.cur.fetchone() - - return result is not None - - def construct_authors_table_from_api(self, venue: str) -> bool: - # crawl author data from openreview API - print("Crawling author data from OpenReview API...") - author_data = self.openreview_crawler.crawl_author_data_from_api(venue) - - # insert data into openreview_authors table - if len(author_data) > 0: - print("Inserting data into 'openreview_authors' table...") - for data in tqdm(author_data): - self.insert_author(**data) - return True - else: - print("No new author data to insert.") - return False - - def construct_authors_table_from_csv(self, csv_file: str) -> bool: - # read author data from csv file - print(f"Reading openreview_authors data from {csv_file}...") - author_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into openreview_authors table - if len(author_data) > 0: - print("Inserting data into 'openreview_authors' table...") - for data in tqdm(author_data): - self.insert_author(**data) - return True - else: - print("No new author data to insert.") - return False - - def construct_authors_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read author data from json file - print(f"Reading openreview_authors data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - author_data = json.load(f) - - # insert data into openreview_authors table - if len(author_data) > 0: - print("Inserting data into 'openreview_authors' table...") - for data in tqdm(author_data): - self.insert_author(**data) - return True - else: - print("No new author data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_papers.py b/research_arcade/sql_database/sql_openreview_papers.py deleted file mode 100644 index 09d0bb8..0000000 --- a/research_arcade/sql_database/sql_openreview_papers.py +++ /dev/null @@ -1,289 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewPapers: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - self.conn = psycopg2.connect( - host=host, - dbname=dbname, - user=user, - password=password, - port=port - ) - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_papers_table() - - def create_papers_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_papers ( - venue TEXT, - paper_openreview_id VARCHAR(255), - title TEXT, - abstract TEXT, - paper_decision TEXT, - paper_pdf_link TEXT, - PRIMARY KEY (venue, paper_openreview_id) - ); - """ - self.cur.execute(create_table_sql) - - def insert_paper(self, venue: str, paper_openreview_id: str, title: str, abstract: str, - paper_decision: str, paper_pdf_link: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_papers (venue, paper_openreview_id, title, abstract, paper_decision, paper_pdf_link) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (venue, paper_openreview_id) DO NOTHING - RETURNING (venue, paper_openreview_id); - """ - - venue = self._clean_string(venue) - paper_openreview_id = self._clean_string(paper_openreview_id) - title = self._clean_string(title) - abstract = self._clean_string(abstract) - paper_decision = self._clean_string(paper_decision) - paper_pdf_link = self._clean_string(paper_pdf_link) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, paper_openreview_id, title, abstract, paper_decision, paper_pdf_link)) - - # Get the inserted paper id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_paper_by_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # search for the row based on primary key - select_sql = """ - SELECT * FROM openreview_papers WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link'] - # paper_dict = dict(zip(columns, row)) - paper_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers WHERE paper_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id,)) - self.conn.commit() - - print(f"Paper with paper_openreview_id {paper_openreview_id} deleted successfully.") - return paper_df - else: - print(f"No paper found with paper_openreview_id {paper_openreview_id}.") - return None - - def delete_papers_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search for the row based on primary key - select_sql = """ - SELECT * FROM openreview_papers WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link'] - # paper_dict = dict(zip(columns, row)) - paper_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All openreview_papers in venue {venue} deleted successfully.") - return paper_df - else: - print(f"No openreview_papers found in venue {venue}.") - return None - - def update_paper(self, venue: str, paper_openreview_id: str, title: str, abstract: str, - paper_decision: str, paper_pdf_link: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT * FROM openreview_papers WHERE paper_openreview_id = %s AND venue = %s; - """ - # find the row in the table - self.cur.execute(select_sql, (paper_openreview_id, venue,)) - row = self.cur.fetchone() - - if not row: - print(f"No paper found with paper_openreview_id {paper_openreview_id}.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link'] - # original_record = dict(zip(columns, row)) - paper_df = pd.DataFrame([row], columns=columns) - - update_sql = """ - UPDATE openreview_papers - SET venue = %s, - title = %s, - abstract = %s, - paper_decision = %s, - paper_pdf_link = %s - WHERE paper_openreview_id = %s; - """ - - venue = self._clean_string(venue) - paper_openreview_id = self._clean_string(paper_openreview_id) - title = self._clean_string(title) - abstract = self._clean_string(abstract) - paper_decision = self._clean_string(paper_decision) - paper_pdf_link = self._clean_string(paper_pdf_link) - - self.cur.execute(update_sql, (venue, title, abstract, paper_decision, - paper_pdf_link, paper_openreview_id)) - - self.conn.commit() - - print(f"Paper with paper_openreview_id {paper_openreview_id} updated successfully.") - return paper_df - - def get_paper_by_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT * FROM openreview_papers WHERE paper_openreview_id = %s; - """ - # find the row in the table - self.cur.execute(select_sql, (paper_openreview_id,)) - row = self.cur.fetchall() - - if not row: - print(f"No paper found with paper_openreview_id {paper_openreview_id}.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link'] - # original_record = dict(zip(columns, row)) - paper_df = pd.DataFrame(row, columns=columns) - return paper_df - - def get_papers_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # Query to select all records for a specific venue - select_sql = """ - SELECT * FROM openreview_papers WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if not rows: - print(f"No openreview_papers found in venue {venue}.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'title', 'abstract', - 'paper_decision', 'paper_pdf_link'] - # original_record = dict(zip(columns, row)) - papers_df = pd.DataFrame(rows, columns=columns) - return papers_df - - def get_all_papers(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - if is_all_features: - # Select query to get paper_openreview_id, title, and author_full_names - select_query = """ - SELECT venue, paper_openreview_id, title, abstract, paper_decision, paper_pdf_link - FROM openreview_papers; - """ - self.cur.execute(select_query) - - # Fetch all the results - papers = self.cur.fetchall() - - # Return the result as a list of tuples (paper_openreview_id, title, author_full_names) - papers_df = pd.DataFrame(papers, columns=["venue", "paper_openreview_id", "title", "abstract", "paper_decision", "paper_pdf_link"]) - return papers_df - else: - # Select query to get paper_openreview_id, title, and author_full_names - select_query = """ - SELECT venue, paper_openreview_id, title - FROM openreview_papers; - """ - self.cur.execute(select_query) - - # Fetch all the results - papers = self.cur.fetchall() - - # Return the result as a list of tuples (paper_openreview_id, title, author_full_names) - papers_df = pd.DataFrame(papers, columns=["venue", "paper_openreview_id", "title"]) - return papers_df - - def check_paper_exists(self, paper_openreview_id: str) -> bool: - self.cur.execute("SELECT 1 FROM openreview_papers WHERE paper_openreview_id = %s LIMIT 1;", (paper_openreview_id,)) - result = self.cur.fetchone() - - return result is not None - - def construct_papers_table_from_api(self, venue: str) -> bool: - # crawl paper data from openreview API - print("Crawling paper data from OpenReview API...") - paper_data = self.openreview_crawler.crawl_paper_data_from_api(venue) - - # insert data into openreview_papers table - if len(paper_data) > 0: - print("Inserting data into 'openreview_papers' table...") - for data in tqdm(paper_data): - self.insert_paper(**data) - return True - else: - print("No new paper data to insert.") - return False - - def construct_papers_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - print(f"File {csv_file} not exists") - return False - else: - # read paper data from csv file - print(f"Reading paper data from {csv_file}...") - paper_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into openreview_papers table - if len(paper_data) > 0: - print("Inserting data into 'openreview_papers' table...") - for data in tqdm(paper_data): - self.insert_paper(**data) - return True - else: - print("No new paper data to insert.") - return False - - def construct_papers_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - if not os.path.exists(json_file): - print(f"File {json_file} not exists") - return False - # read paper data from json file - print(f"Reading paper data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - paper_data = json.load(f) - - # insert data into openreview_papers table - if len(paper_data) > 0: - print("Inserting data into 'openreview_papers' table...") - for data in tqdm(paper_data): - self.insert_paper(**data) - return True - else: - print("No new paper data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_papers_authors.py b/research_arcade/sql_database/sql_openreview_papers_authors.py deleted file mode 100644 index 5bc4d66..0000000 --- a/research_arcade/sql_database/sql_openreview_papers_authors.py +++ /dev/null @@ -1,263 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewPapersAuthors: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host=host, dbname=dbname, - user=user, password=password, port=port - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_papers_authors_table() - - def create_papers_authors_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_papers_authors ( - venue TEXT, - paper_openreview_id VARCHAR(255), - author_openreview_id VARCHAR(255), - PRIMARY KEY (venue, paper_openreview_id, author_openreview_id) - ); - """ - self.cur.execute(create_table_sql) - - def insert_paper_authors(self, venue: str, paper_openreview_id: str, author_openreview_id: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_papers_authors (venue, paper_openreview_id, author_openreview_id) - VALUES (%s, %s, %s) - ON CONFLICT (venue, paper_openreview_id, author_openreview_id) DO NOTHING - RETURNING (venue, paper_openreview_id, author_openreview_id); - """ - - venue = self._clean_string(venue) - paper_openreview_id = self._clean_string(paper_openreview_id) - author_openreview_id = self._clean_string(author_openreview_id) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, paper_openreview_id, author_openreview_id)) - - # Get the inserted paper's openreview id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_paper_author_by_id(self, paper_openreview_id: str, author_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_authors WHERE paper_openreview_id = %s AND author_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id, author_openreview_id)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "author_openreview_id"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_authors WHERE paper_openreview_id = %s AND author_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id, author_openreview_id)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_authors' with paper_openreview_id = {paper_openreview_id} and author_openreview_id = {author_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_authors' with paper_openreview_id = {paper_openreview_id} and author_openreview_id = {author_openreview_id}.") - return None - - def delete_paper_author_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_authors WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "author_openreview_id"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_authors WHERE paper_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_authors' with paper_openreview_id = {paper_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_authors' with paper_openreview_id = {paper_openreview_id}.") - return None - - def delete_paper_author_by_author_id(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_authors WHERE author_openreview_id = %s; - """ - self.cur.execute(select_sql, (author_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "author_openreview_id"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_authors WHERE author_openreview_id = %s; - """ - self.cur.execute(delete_sql, (author_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_authors' with author_openreview_id = {author_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_authors' with author_openreview_id = {author_openreview_id}.") - return None - - def delete_papers_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search the row based on primary key - select_sql = """ - SELECT venue, paper_openreview_id, author_openreview_id FROM openreview_papers_authors WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if rows: - columns = ['venue', 'paper_openreview_id', 'author_openreview_id'] - # papers_authors_dict = dict(zip(columns, rows)) - papers_authors_df = pd.DataFrame(rows, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_authors WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All connections in venue {venue} deleted successfully.") - return papers_authors_df - else: - print(f"No connections found in venue {venue}.") - return None - - def get_paper_neighboring_authors(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, author_openreview_id FROM openreview_papers_authors - WHERE paper_openreview_id = %s; - """, (paper_openreview_id,)) - - paper_neighboring_authors = self.cur.fetchall() - - if paper_neighboring_authors is not None: - paper_neighboring_authors_df = pd.DataFrame(paper_neighboring_authors, columns=["venue", "paper_openreview_id", "author_openreview_id"]) - return paper_neighboring_authors_df - else: - return None - - def get_author_neighboring_papers(self, author_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, author_openreview_id FROM openreview_papers_authors - WHERE author_openreview_id = %s; - """, (author_openreview_id,)) - - author_neighboring_papers = self.cur.fetchall() - - if author_neighboring_papers is not None: - author_neighboring_papers_df = pd.DataFrame(author_neighboring_papers, columns=["venue", "paper_openreview_id", "author_openreview_id"]) - return author_neighboring_papers_df - else: - return None - - def get_papers_authors_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, author_openreview_id FROM openreview_papers_authors - WHERE venue = %s; - """, (venue,)) - - openreview_papers_authors = self.cur.fetchall() - - if openreview_papers_authors is not None: - papers_authors_df = pd.DataFrame(openreview_papers_authors, columns=["venue", "paper_openreview_id", "author_openreview_id"]) - return papers_authors_df - else: - return None - - def get_all_papers_authors(self) -> Optional[pd.DataFrame]: - select_query = """ - SELECT venue, paper_openreview_id, author_openreview_id - FROM openreview_papers_authors ORDER BY paper_openreview_id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - openreview_papers_authors = self.cur.fetchall() - - # Return the result as a list of tuples (venue, author_openreview_id, author_full_name) - authors_df = pd.DataFrame(openreview_papers_authors, columns=["venue", "paper_openreview_id", "author_openreview_id"]) - return authors_df - - def check_paper_author_exists(self, paper_openreview_id: str, author_openreview_id: str) -> bool: - self.cur.execute(""" - SELECT 1 FROM openreview_papers_authors - WHERE paper_openreview_id = %s AND author_openreview_id = %s - LIMIT 1; - """, (paper_openreview_id, author_openreview_id)) - - result = self.cur.fetchone() - - return result is not None - - def construct_papers_authors_table_from_api(self, venue: str) -> bool: - # crawl the data from openreview API - print(f"Crawling papers-authors data for venue {venue} from OpenReview API...") - papers_authors_data = self.openreview_crawler.crawl_papers_authors_data_from_api(venue) - - if len(papers_authors_data) > 0: - print(f"Inserting papers-authors data for venue {venue}...") - for data in tqdm(papers_authors_data, desc=f"Inserting papers-authors data for venue {venue}"): - self.insert_paper_authors(**data) - return True - else: - print(f"No papers-authors data found for venue {venue}.") - return False - - def construct_papers_authors_table_from_csv(self, csv_file: str) -> bool: - # read the data from csv file - papers_authors_data = pd.read_csv(csv_file).to_dict(orient='records') - - if len(papers_authors_data) > 0: - print(f"Inserting papers-authors data from {csv_file}...") - for data in tqdm(papers_authors_data, desc=f"Inserting papers-authors data from {csv_file}"): - self.insert_paper_authors(**data) - return True - else: - print(f"No papers-authors data found in {csv_file}.") - return False - - def construct_papers_authors_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read the data from json file - with open(json_file, 'r', encoding='utf-8') as f: - papers_authors_data = json.load(f) - - if len(papers_authors_data) > 0: - print(f"Inserting papers-authors data from {json_file}...") - for data in tqdm(papers_authors_data, desc=f"Inserting papers-authors data from {json_file}"): - self.insert_paper_authors(**data) - return True - else: - print(f"No papers-authors data found in {json_file}.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_papers_reviews.py b/research_arcade/sql_database/sql_openreview_papers_reviews.py deleted file mode 100644 index 28b91a9..0000000 --- a/research_arcade/sql_database/sql_openreview_papers_reviews.py +++ /dev/null @@ -1,273 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewPapersReviews: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host=host, dbname=dbname, - user=user, password=password, port=port - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_papers_reviews_table() - - def create_papers_reviews_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_papers_reviews ( - venue TEXT, - paper_openreview_id VARCHAR(255), - review_openreview_id VARCHAR(255), - title TEXT, - time TEXT, - PRIMARY KEY (venue, paper_openreview_id, review_openreview_id) - ); - """ - self.cur.execute(create_table_sql) - - def insert_paper_reviews(self, venue: str, paper_openreview_id: str, review_openreview_id: str, title: str, time: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_papers_reviews (venue, paper_openreview_id, review_openreview_id, title, time) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT (venue, paper_openreview_id, review_openreview_id) DO NOTHING - RETURNING (venue, paper_openreview_id, review_openreview_id); - """ - - venue = self._clean_string(venue) - paper_openreview_id = self._clean_string(paper_openreview_id) - review_openreview_id = self._clean_string(review_openreview_id) - title = self._clean_string(title) - time = self._clean_string(time) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, paper_openreview_id, review_openreview_id, title, time)) - - # Get the inserted paper's openreview id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_paper_review_by_id(self, paper_openreview_id: str, review_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_reviews WHERE paper_openreview_id = %s AND review_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id, review_openreview_id)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "review_openreview_id", 'title', 'time'] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_reviews WHERE paper_openreview_id = %s AND review_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id, review_openreview_id)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_reviews' with paper_openreview_id = {paper_openreview_id} and review_openreview_id = {review_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_reviews' with paper_openreview_id = {paper_openreview_id} and review_openreview_id = {review_openreview_id}.") - return None - - def delete_paper_review_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_reviews WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "review_openreview_id", 'title', 'time'] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_reviews WHERE paper_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_reviews' with paper_openreview_id = {paper_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_reviews' with paper_openreview_id = {paper_openreview_id}.") - return None - - def delete_paper_review_by_review_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(select_sql, (review_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "review_openreview_id", 'title', 'time'] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(delete_sql, (review_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_reviews' with review_openreview_id = {review_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_reviews' with review_openreview_id = {review_openreview_id}.") - return None - - def delete_papers_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search the row based on primary key - select_sql = """ - SELECT venue, paper_openreview_id, review_openreview_id FROM openreview_papers_reviews WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if rows: - columns = ['venue', 'paper_openreview_id', 'review_openreview_id'] - # papers_reviews_dict = dict(zip(columns, rows)) - papers_reviews_df = pd.DataFrame(rows, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_reviews WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All connections in venue {venue} deleted successfully.") - return papers_reviews_df - else: - print(f"No connections found in venue {venue}.") - return None - - def get_papers_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, review_openreview_id, title, time FROM openreview_papers_reviews - WHERE venue = %s; - """, (venue,)) - - openreview_papers_reviews = self.cur.fetchall() - - if openreview_papers_reviews is not None: - papers_reviews_df = pd.DataFrame(openreview_papers_reviews, columns=["venue", "paper_openreview_id", "review_openreview_id", "title", "time"]) - return papers_reviews_df - else: - return None - - def get_all_papers_reviews(self) -> Optional[pd.DataFrame]: - select_query = """ - SELECT venue, paper_openreview_id, review_openreview_id, title, time - FROM openreview_papers_reviews ORDER BY paper_openreview_id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - openreview_papers_reviews = self.cur.fetchall() - - papers_reviews_df = pd.DataFrame(openreview_papers_reviews, columns=["venue", "paper_openreview_id", "review_openreview_id", "title", "time"]) - return papers_reviews_df - - def check_paper_review_exists(self, paper_openreview_id: str, review_openreview_id: str) -> bool: - self.cur.execute(""" - SELECT 1 FROM openreview_papers_reviews - WHERE paper_openreview_id = %s AND review_openreview_id = %s - LIMIT 1; - """, (paper_openreview_id, review_openreview_id)) - - result = self.cur.fetchone() - - return result is not None - - def get_paper_neighboring_reviews(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, review_openreview_id, title, time FROM openreview_papers_reviews - WHERE paper_openreview_id = %s; - """, (paper_openreview_id,)) - - paper_neighboring_reviews = self.cur.fetchall() - - if paper_neighboring_reviews is not None: - paper_neighboring_reviews_df = pd.DataFrame(paper_neighboring_reviews, columns=["venue", "paper_openreview_id", "review_openreview_id", "title", "time"]) - return paper_neighboring_reviews_df - else: - return None - - def get_review_neighboring_papers(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, review_openreview_id, title, time FROM openreview_papers_reviews - WHERE review_openreview_id = %s; - """, (review_openreview_id,)) - - review_neighboring_papers = self.cur.fetchall() - - if review_neighboring_papers is not None: - review_neighboring_papers_df = pd.DataFrame(review_neighboring_papers, columns=["venue", "paper_openreview_id", "review_openreview_id", "title", "time"]) - return review_neighboring_papers_df - else: - return None - - def construct_papers_reviews_table_from_api(self, venue: str) -> bool: - # crawl the data from openreview API - print(f"Crawling paper-review connections for venue: {venue}...") - papers_reviews_data = self.openreview_crawler.crawl_papers_reviews_from_api(venue) - - if len(papers_reviews_data) > 0: - print(f"Inserting paper-review connections into the database for venue: {venue}...") - for data in tqdm(papers_reviews_data): - self.insert_paper_reviews(**data) - return True - else: - print(f"No paper-review connections found for venue: {venue}.") - return False - - def construct_papers_reviews_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - # read paper-review connection data from csv file - print(f"Reading paper-review connection data from {csv_file}...") - papers_reviews_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into openreview_papers_reviewstable - if len(papers_reviews_data) > 0: - print("Inserting data into 'openreview_papers_reviews' table...") - for data in tqdm(papers_reviews_data): - self.insert_paper_reviews(**data) - return True - else: - print("No new paper-review connection data to insert.") - return False - - def construct_papers_reviews_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read paper-review connection data from json file - print(f"Reading paper-review connection data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - papers_reviews_data = json.load(f) - - # insert data into openreview_papers_reviews table - if len(papers_reviews_data) > 0: - print("Inserting data into 'openreview_papers_reviews' table...") - for data in tqdm(papers_reviews_data): - self.insert_paper_reviews(**data) - return True - else: - print("No new paper-review connection data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_papers_revisions.py b/research_arcade/sql_database/sql_openreview_papers_revisions.py deleted file mode 100644 index 9429692..0000000 --- a/research_arcade/sql_database/sql_openreview_papers_revisions.py +++ /dev/null @@ -1,276 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewPapersRevisions: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host=host, dbname=dbname, - user=user, password=password, port=port - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_papers_revisions_table() - - def create_papers_revisions_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_papers_revisions ( - venue TEXT, - paper_openreview_id VARCHAR(255), - revision_openreview_id VARCHAR(255), - title TEXT, - time TEXT, - PRIMARY KEY (venue, paper_openreview_id, revision_openreview_id) - ); - """ - self.cur.execute(create_table_sql) - - def insert_paper_revisions(self, venue: str, paper_openreview_id: str, revision_openreview_id: str, title: str, time: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_papers_revisions (venue, paper_openreview_id, revision_openreview_id, title, time) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT (venue, paper_openreview_id, revision_openreview_id) DO NOTHING - RETURNING (venue, paper_openreview_id, revision_openreview_id); - """ - - venue = self._clean_string(venue) - paper_openreview_id = self._clean_string(paper_openreview_id) - revision_openreview_id = self._clean_string(revision_openreview_id) - title = self._clean_string(title) - time = self._clean_string(time) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, paper_openreview_id, revision_openreview_id, title, time)) - - # Get the inserted paper's openreview id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_paper_revision_by_id(self, paper_openreview_id: str, revision_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_revisions WHERE paper_openreview_id = %s AND revision_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id, revision_openreview_id)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "revision_openreview_id", 'title', 'time'] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_revisions WHERE paper_openreview_id = %s AND revision_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id, revision_openreview_id)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_revisions' with paper_openreview_id = {paper_openreview_id} and revision_openreview_id = {revision_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_revisions' with paper_openreview_id = {paper_openreview_id} and revision_openreview_id = {revision_openreview_id}.") - return None - - def delete_paper_revision_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_revisions WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "revision_openreview_id", 'title', 'time'] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_revisions WHERE paper_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_revisions' with paper_openreview_id = {paper_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_revisions' with paper_openreview_id = {paper_openreview_id}.") - return None - - def delete_paper_revision_by_revision_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_papers_revisions WHERE revision_openreview_id = %s; - """ - self.cur.execute(select_sql, (revision_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "paper_openreview_id", "revision_openreview_id", 'title', 'time'] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_revisions WHERE revision_openreview_id = %s; - """ - self.cur.execute(delete_sql, (revision_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_papers_revisions' with revision_openreview_id = {revision_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_papers_revisions' with revision_openreview_id = {revision_openreview_id}.") - return None - - def delete_papers_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search the row based on primary key - select_sql = """ - SELECT venue, paper_openreview_id, revision_openreview_id, title, time FROM openreview_papers_revisions WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if rows: - columns = ['venue', 'paper_openreview_id', 'revision_openreview_id', 'title', 'time'] - # papers_revisions_dict = dict(zip(columns, rows)) - papers_revisions_df = pd.DataFrame(rows, columns=columns) - - delete_sql = """ - DELETE FROM openreview_papers_revisions WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All connections in venue {venue} deleted successfully.") - return papers_revisions_df - else: - print(f"No connections found in venue {venue}.") - return None - - def get_papers_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # TODO: papers_revisions is undefined - papers_revisions = None - self.cur.execute(""" - SELECT venue, paper_openreview_id, revision_openreview_id, title, time FROM openreview_papers_revisions - WHERE venue = %s; - """, (venue,)) - - openreview_papers_revisions = self.cur.fetchall() - - if openreview_papers_revisions is not None: - papers_revisions_df = pd.DataFrame(papers_revisions, columns=["venue", "paper_openreview_id", "revision_openreview_id", "title", "time"]) - return papers_revisions_df - else: - return None - - def get_all_papers_revisions(self) -> Optional[pd.DataFrame]: - select_query = """ - SELECT venue, paper_openreview_id, revision_openreview_id, title, time - FROM openreview_papers_revisions ORDER BY paper_openreview_id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - papers_authors = self.cur.fetchall() - - # Return the result as a list of tuples (venue, author_openreview_id, author_full_name) - authors_df = pd.DataFrame(papers_authors, columns=["venue", "paper_openreview_id", "revision_openreview_id", "title", "time"]) - return authors_df - - def check_paper_revision_exists(self, paper_openreview_id: str, revision_openreview_id: str) -> bool: - self.cur.execute(""" - SELECT 1 FROM openreview_papers_revisions - WHERE paper_openreview_id = %s AND revision_openreview_id = %s - LIMIT 1; - """, (paper_openreview_id, revision_openreview_id)) - - result = self.cur.fetchone() - - return result is not None - - def get_paper_neighboring_revisions(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, revision_openreview_id, title, time FROM openreview_papers_revisions - WHERE paper_openreview_id = %s; - """, (paper_openreview_id,)) - - paper_neighboring_revisions = self.cur.fetchall() - - if paper_neighboring_revisions is not None: - paper_neighboring_revisions_df = pd.DataFrame(paper_neighboring_revisions, columns=["venue", "paper_openreview_id", "revision_openreview_id", "title", "time"]) - return paper_neighboring_revisions_df - else: - return None - - def get_revision_neighboring_papers(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, paper_openreview_id, revision_openreview_id, title, time FROM openreview_papers_revisions - WHERE revision_openreview_id = %s; - """, (revision_openreview_id,)) - - revision_neighboring_papers = self.cur.fetchall() - - if revision_neighboring_papers is not None: - revision_neighboring_papers_df = pd.DataFrame(revision_neighboring_papers, columns=["venue", "paper_openreview_id", "revision_openreview_id", "title", "time"]) - return revision_neighboring_papers_df - else: - return None - - def construct_papers_revisions_table_from_api(self, venue: str) -> bool: - # fetch paper-revision data from openreview API - print(f"Crawling paper-revision data from OpenReview API for venue: {venue}...") - paper_revision_data = self.openreview_crawler.crawl_papers_revisions_data_from_api(venue) - - # insert data into openreview_papers_revisions table - if len(paper_revision_data) > 0: - print(f"Inserting paper-revision data into the database for venue: {venue}...") - for data in tqdm(paper_revision_data): - self.insert_paper_revisions(**data) - return True - else: - print(f"No paper-revision data found for venue: {venue}.") - return False - - def construct_papers_revisions_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - # read revision data from csv file - revision_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into papers table - if len(revision_data) > 0: - print(f"Inserting paper-revision data from {csv_file} into the database...") - for data in tqdm(revision_data): - self.insert_paper_revisions(**data) - return True - else: - print(f"No paper-revision data found in {csv_file}.") - return False - - def construct_papers_revisions_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read revision data from json file - print(f"Reading revisions data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - revision_data = json.load(f) - - # insert data into papers table - if len(revision_data) > 0: - print(f"Inserting paper-revision data from {json_file} into the database...") - for data in tqdm(revision_data): - self.insert_paper_revisions(**data) - return True - else: - print(f"No paper-revision data found in {json_file}.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_paragraphs.py b/research_arcade/sql_database/sql_openreview_paragraphs.py deleted file mode 100644 index 657203b..0000000 --- a/research_arcade/sql_database/sql_openreview_paragraphs.py +++ /dev/null @@ -1,225 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import os -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewParagraphs: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - self.conn = psycopg2.connect( - host=host, - dbname=dbname, - user=user, - password=password, - port=port - ) - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_paragraphs_table() - - def create_paragraphs_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_paragraphs ( - venue TEXT, - paper_openreview_id VARCHAR(255), - paragraph_idx INTEGER CHECK (paragraph_idx >= 0), - section TEXT, - content TEXT, - PRIMARY KEY (venue, paper_openreview_id, paragraph_idx) - ); - """ - # Execute the SQL to create the table - self.cur.execute(create_table_sql) - - def insert_paragraph(self, venue: str, paper_openreview_id: str, paragraph_idx: int, section: str, content: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_paragraphs (venue, paper_openreview_id, paragraph_idx, section, content) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT (venue, paper_openreview_id, paragraph_idx) DO NOTHING - RETURNING (venue, paper_openreview_id, paragraph_idx); - """ - - # Execute the insertion query - self.cur.execute(insert_sql, (self._clean_string(venue), self._clean_string(paper_openreview_id), paragraph_idx, self._clean_string(section), self._clean_string(content))) - - # Get the inserted review id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def get_all_paragraphs(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - if is_all_features: - select_sql = """ - SELECT * FROM openreview_paragraphs; - """ - self.cur.execute(select_sql) - row = self.cur.fetchall() - - if not row: - print("No paragraph found in openreview_paragraphs table.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section', 'content'] - # original_record = dict(zip(columns, row)) - paragraph_df = pd.DataFrame(row, columns=columns) - return paragraph_df - else: - select_sql = """ - SELECT venue, paper_openreview_id, paragraph_idx, section FROM openreview_paragraphs; - """ - self.cur.execute(select_sql) - row = self.cur.fetchall() - - if not row: - print("No paragraph found in openreview_paragraphs table.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section'] - # original_record = dict(zip(columns, row)) - paragraph_df = pd.DataFrame(row, columns=columns) - return paragraph_df - - def get_paragraphs_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT * FROM openreview_paragraphs WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - row = self.cur.fetchall() - - if not row: - print(f"No paragraph found with paper_openreview_id {paper_openreview_id}.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section', 'content'] - # original_record = dict(zip(columns, row)) - paragraph_df = pd.DataFrame(row, columns=columns) - return paragraph_df - - def get_paragraphs_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # Query to select all records for a specific venue - select_sql = """ - SELECT * FROM openreview_paragraphs WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if not rows: - print(f"No paragraphs found in venue {venue}.") - return None - else: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section', 'content'] - # original_record = dict(zip(columns, row)) - paragraphs_df = pd.DataFrame(rows, columns=columns) - return paragraphs_df - - def delete_paragraphs_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - select_sql = """ - SELECT * FROM openreview_paragraphs WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if rows: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section', 'content'] - # original_record = dict(zip(columns, row)) - paragraphs_df = pd.DataFrame(rows, columns=columns) - - delete_sql = """ - DELETE FROM openreview_paragraphs WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All paragraphs in venue {venue} deleted successfully.") - return paragraphs_df - else: - print(f"No paragraphs found in venue {venue}.") - return None - - def delete_paragraphs_by_paper_id(self, paper_openreview_id: str) -> Optional[pd.DataFrame]: - select_sql = """ - SELECT * FROM openreview_paragraphs WHERE paper_openreview_id = %s; - """ - self.cur.execute(select_sql, (paper_openreview_id,)) - rows = self.cur.fetchall() - - if rows: - columns = ['venue', 'paper_openreview_id', 'paragraph_idx', 'section', 'content'] - # original_record = dict(zip(columns, row)) - paragraphs_df = pd.DataFrame(rows, columns=columns) - - delete_sql = """ - DELETE FROM openreview_paragraphs WHERE paper_openreview_id = %s; - """ - self.cur.execute(delete_sql, (paper_openreview_id,)) - self.conn.commit() - - print(f"All paragraphs in paper {paper_openreview_id} deleted successfully.") - return paragraphs_df - else: - print(f"No paragraphs found in paper {paper_openreview_id}.") - return None - - def construct_paragraphs_table_from_api(self, venue: str, pdf_dir: str, filter_list: list, log_file: str, - is_paper = True, is_revision = True, is_pdf_delete: bool = True) -> bool: - # crawl paragraph data from openreview API - print("Crawling paragraph data from OpenReview API...") - paragraph_data = self.openreview_crawler.crawl_paragraph_data_from_api(venue, pdf_dir, filter_list, log_file, - is_paper, is_revision, is_pdf_delete) - # insert data into openreview_paragraphs table - if len(paragraph_data) > 0: - print("Inserting data into 'openreview_paragraphs' table...") - for data in tqdm(paragraph_data): - self.insert_paragraph(**data) - return True - else: - print("No new paragraph data to insert.") - return False - - def construct_paragraphs_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - print(f"File {csv_file} not exists") - return False - else: - # read paragraph data from csv file - print(f"Reading paragraph data from {csv_file}...") - paragraph_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into openreview_paragraphs table - if len(paragraph_data) > 0: - print("Inserting data into 'openreview_paragraphs' table...") - for data in tqdm(paragraph_data): - self.insert_paragraph(**data) - return True - else: - print("No new paragraph data to insert.") - return False - - def construct_paragraphs_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - print(f"File {json_file} not exists") - return False - else: - # read insert_paragraph data from json file - print(f"Reading paragraph data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - paragraph_data = json.load(f) - - # insert data into openreview_paragraphs table - if len(paragraph_data) > 0: - print("Inserting data into 'openreview_paragraphs' table...") - for data in tqdm(paragraph_data): - self.insert_paragraph(**data) - return True - else: - print("No new insert_paragraph data to insert.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_reviews.py b/research_arcade/sql_database/sql_openreview_reviews.py deleted file mode 100644 index 87b7d67..0000000 --- a/research_arcade/sql_database/sql_openreview_reviews.py +++ /dev/null @@ -1,331 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import ast -import psycopg2 -from psycopg2.extras import Json -import os -from typing import Optional, Union - -class SQLOpenReviewReviews: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - self.conn = psycopg2.connect( - host=host, - dbname=dbname, - user=user, - password=password, - port=port - ) - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.crawler = OpenReviewCrawler() - self.create_reviews_table() - - def create_reviews_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_reviews ( - id SERIAL UNIQUE, - venue TEXT, - review_openreview_id VARCHAR(255), - replyto_openreview_id VARCHAR(255), - writer TEXT, - title TEXT, - content JSONB, - time TEXT, - PRIMARY KEY (venue, review_openreview_id) - ); - """ - # Execute the SQL to create the table - self.cur.execute(create_table_sql) - - def insert_review(self, venue: str, review_openreview_id: str, replyto_openreview_id: str, - title: str, writer: str, content: dict, time: str) -> Optional[tuple]: - """ - Insert a review into the openreview_reviews table. Returns the inserted review id or None if it fails. - - venue: str, the venue where the paper was submitted. - - review_openreview_id: str, unique ID for the review (primary key). - - replyto_openreview_id: str or None, ID for a reply (optional). - - writer: str, the name or identity of the reviewer. - - title: str, the title of the review. - - content: JSON object, the content of the review. - - time - """ - insert_sql = """ - INSERT INTO openreview_reviews (venue, review_openreview_id, replyto_openreview_id, writer, title, content, time) - VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (venue, review_openreview_id) DO NOTHING - RETURNING (venue, review_openreview_id); - """ - - # clean content - cleaned_content = self._clean_json_content(content) - venue = self._clean_string(venue) - review_openreview_id = self._clean_string(review_openreview_id) - replyto_openreview_id = self._clean_string(replyto_openreview_id) - writer = self._clean_string(writer) - title = self._clean_string(title) - time = self._clean_string(time) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, review_openreview_id, replyto_openreview_id, writer, title, Json(cleaned_content), time)) - - # Get the inserted review id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_review_by_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - # search for the row based on primary key - select_sql = """ - SELECT venue, review_openreview_id, replyto_openreview_id, writer, title, content, time - FROM openreview_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(select_sql, (review_openreview_id,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time'] - # review_dict = dict(zip(columns, row)) - review_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(delete_sql, (review_openreview_id,)) - self.conn.commit() - - print(f"Review with review_openreview_id {review_openreview_id} deleted successfully.") - return review_df - else: - print(f"No review found with review_openreview_id {review_openreview_id}.") - return None - - def delete_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search for the row based on primary key - select_sql = """ - SELECT venue, review_openreview_id, replyto_openreview_id, writer, title, content, time - FROM openreview_reviews WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time'] - # review_dict = dict(zip(columns, row)) - review_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_reviews WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All openreview_reviews in venue {venue} deleted successfully.") - return review_df - else: - print(f"No openreview_reviews found in venue {venue}.") - return None - - def update_review(self, venue: str, review_openreview_id: str, replyto_openreview_id: str, - writer: str, title: str, content: dict, time: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT venue, review_openreview_id, replyto_openreview_id, writer, title, content, time - FROM openreview_reviews WHERE review_openreview_id = %s AND venue = %s; - """ - self.cur.execute(select_sql, (review_openreview_id, venue,)) - row = self.cur.fetchone() - - if not row: - print(f"No review found with review_openreview_id {review_openreview_id}.") - return None - else: - columns = ['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time'] - # original_record = dict(zip(columns, row)) - review_df = pd.DataFrame([row], columns=columns) - - # SQL query to update the record - update_sql = """ - UPDATE openreview_reviews - SET venue = %s, - replyto_openreview_id = %s, - writer = %s, - title = %s, - content = %s, - time = %s - WHERE review_openreview_id = %s; - """ - - venue = self._clean_string(venue) - review_openreview_id = self._clean_string(review_openreview_id) - replyto_openreview_id = self._clean_string(replyto_openreview_id) - writer = self._clean_string(writer) - title = self._clean_string(title) - time = self._clean_string(time) - - cleaned_content = self._clean_json_content(content) - - self.cur.execute(update_sql, (venue, replyto_openreview_id, writer, title, - Json(cleaned_content), time, review_openreview_id)) - self.conn.commit() - - print(f"Review with review_openreview_id {review_openreview_id} updated successfully.") - return review_df - - def get_review_by_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT venue, review_openreview_id, replyto_openreview_id, writer, title, content, time - FROM openreview_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(select_sql, (review_openreview_id,)) - row = self.cur.fetchone() - - if not row: - print(f"No review found with review_openreview_id {review_openreview_id}.") - return None - else: - columns = ['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time'] - - row_list = list(row) - if isinstance(row_list[5], dict): - row_list[5] = json.dumps(row_list[5]) - # original_record = dict(zip(columns, row)) - review_df = pd.DataFrame([row_list], columns=columns) - - return review_df - - def get_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # Query to select all records for a specific venue - select_sql = """ - SELECT venue, review_openreview_id, replyto_openreview_id, writer, title, content, time - FROM openreview_reviews WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if not rows: - print(f"No openreview_reviews found in venue {venue}.") - return None - else: - columns = ['venue', 'review_openreview_id', 'replyto_openreview_id', - 'writer', 'title', 'content', 'time'] - # original_record = dict(zip(columns, row)) - processed_rows = [] - for row in rows: - row_list = list(row) - if isinstance(row_list[5], dict): - row_list[3] = json.dumps(row_list[5]) - processed_rows.append(row_list) - reviews_df = pd.DataFrame(processed_rows, columns=columns) - return reviews_df - - def get_all_reviews(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - if is_all_features: - select_query = """ - SELECT venue, review_openreview_id, replyto_openreview_id, writer, title, content, time - FROM openreview_reviews ORDER BY id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - reviews= self.cur.fetchall() - processed_reviews= [] - for row in reviews: - row_list = list(row) - if isinstance(row_list[5], dict): - row_list[5] = json.dumps(row_list[5]) - processed_reviews.append(row_list) - - # Return the result as a list of tuples (paper_openreview_id, title, author_full_names) - reviews_df = pd.DataFrame(processed_reviews, columns=["venue", "review_openreview_id", "replyto_openreview_id", "writer", "title", "content", "time"]) - return reviews_df - else: - select_query = """ - SELECT venue, review_openreview_id, replyto_openreview_id, title, time - FROM openreview_reviews ORDER BY id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - reviews= self.cur.fetchall() - - # Return the result as a list of tuples (paper_openreview_id, title, author_full_names) - reviews_df = pd.DataFrame(reviews, columns=["venue", "review_openreview_id", "replyto_openreview_id", "title", "time"]) - return reviews_df - - def check_review_exists(self, review_openreview_id: str) -> bool: - self.cur.execute("SELECT 1 FROM openreview_reviews WHERE review_openreview_id = %s LIMIT 1;", (review_openreview_id,)) - result = self.cur.fetchone() - - return result is not None - - def construct_reviews_table_from_api(self, venue: str) -> None: - # crawl review data from openreview API - print("Crawling review data from OpenReview API...") - review_data = self.crawler.crawl_review_data_from_api(venue) - - # insert data into openreview_reviews table - if len(review_data) > 0: - print("Inserting data into 'openreview_reviews' table...") - for data in tqdm(review_data): - self.insert_review(**data) - else: - print("No new review data to insert.") - - def construct_reviews_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - # read review data from csv - print(f"Reading review data from {csv_file}...") - review_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into openreview_reviews table - if len(review_data) > 0: - print("Inserting data into 'openreview_reviews' table...") - for data in tqdm(review_data): - data["content"] = ast.literal_eval(data["content"]) - self.insert_review(**data) - return True - else: - print("No new review data to insert.") - return False - - def construct_reviews_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read review data from json - print(f"Reading review data from {json_file}...") - with open(json_file, 'r') as f: - review_data = json.load(f) - - # insert data into openreview_reviews table - if len(review_data) > 0: - print("Inserting data into 'openreview_reviews' table...") - for data in tqdm(review_data): - self.insert_review(**data) - return True - else: - print("No new review data to insert.") - return False - - def _clean_json_content(self, content: Union[str, dict, list, int, float, bool, None]) -> Union[str, dict, list, int, float, bool, None]: - if isinstance(content, str): - return ''.join(char for char in content if char.isprintable()) # 返回 str - elif isinstance(content, dict): - return {key: self._clean_json_content(value) for key, value in content.items()} # 返回 dict - elif isinstance(content, list): - return [self._clean_json_content(item) for item in content] # 返回 list - else: - return content # 返回原始类型 - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_revisions.py b/research_arcade/sql_database/sql_openreview_revisions.py deleted file mode 100644 index 67f1f2a..0000000 --- a/research_arcade/sql_database/sql_openreview_revisions.py +++ /dev/null @@ -1,314 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import ast -import psycopg2 -from psycopg2.extras import Json -import os -from typing import Optional, Union - -class SQLOpenReviewRevisions: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host=host, dbname=dbname, - user=user, password=password, port=port - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_revisions_table() - - def create_revisions_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_revisions ( - id SERIAL UNIQUE, - venue TEXT, - original_openreview_id VARCHAR(255), - revision_openreview_id VARCHAR(255), - content JSONB, - time TEXT, - PRIMARY KEY (venue, revision_openreview_id) - ); - """ - self.cur.execute(create_table_sql) - - def insert_revision(self, venue: str, original_openreview_id: str, - revision_openreview_id: str, content: dict, time: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_revisions (venue, original_openreview_id, revision_openreview_id, content, time) - VALUES (%s, %s, %s, %s, %s) - ON CONFLICT (venue, revision_openreview_id) DO NOTHING - RETURNING (venue, revision_openreview_id); - """ - # clean revisions - cleaned_revision_content = self._clean_json_content(content) - venue = self._clean_string(venue) - original_openreview_id = self._clean_string(original_openreview_id) - revision_openreview_id = self._clean_string(revision_openreview_id) - time = self._clean_string(time) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, original_openreview_id, revision_openreview_id, Json(cleaned_revision_content), time)) - - # Get the inserted paper's openreview id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def delete_revision_by_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - # search for the row based on primary key - select_sql = """ - SELECT venue, original_openreview_id, revision_openreview_id, content, time - FROM openreview_revisions WHERE revision_openreview_id = %s; - """ - self.cur.execute(select_sql, (revision_openreview_id,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'original_openreview_id', - 'revision_openreview_id', 'content', 'time'] - # revision_dict = dict(zip(columns, row)) - revision_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_revisions WHERE revision_openreview_id = %s; - """ - self.cur.execute(delete_sql, (revision_openreview_id,)) - self.conn.commit() - - print(f"Revision with revision_openreview_id {revision_openreview_id} deleted successfully.") - return revision_df - else: - print(f"No revision found with revision_openreview_id {revision_openreview_id}.") - return None - - def delete_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search for the row based on primary key - select_sql = """ - SELECT venue, original_openreview_id, revision_openreview_id, content, time - FROM openreview_revisions WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - row = self.cur.fetchall() - - if row: - columns = ['venue', 'original_openreview_id', - 'revision_openreview_id', 'content', 'time'] - # revision_dict = dict(zip(columns, row)) - revision_df = pd.DataFrame(row, columns=columns) - - delete_sql = """ - DELETE FROM openreview_revisions WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All openreview_revisions in venue {venue} deleted successfully.") - return revision_df - else: - print(f"No openreview_revisions found in venue {venue}.") - return None - - def update_revision(self, venue: str, original_openreview_id: str, - revision_openreview_id: str, content: dict, time: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT venue, original_openreview_id, revision_openreview_id, content, time - FROM openreview_revisions WHERE revision_openreview_id = %s AND venue = %s; - """ - self.cur.execute(select_sql, (revision_openreview_id, venue,)) - row = self.cur.fetchone() - - if not row: - print(f"No revision found with revision_openreview_id {revision_openreview_id}.") - return None - else: - # If record exists, return the original record as a dictionary - columns = ['venue', 'original_openreview_id', - 'revision_openreview_id', 'content', 'time'] - # original_record = dict(zip(columns, row)) - revision_df = pd.DataFrame([row], columns=columns) - - # SQL query to update the record - update_sql = """ - UPDATE openreview_revisions - SET venue = %s, - original_openreview_id = %s, - content = %s, - time = %s - WHERE revision_openreview_id = %s; - """ - - cleaned_content = self._clean_json_content(content) - venue = self._clean_string(venue) - original_openreview_id = self._clean_string(original_openreview_id) - revision_openreview_id = self._clean_string(revision_openreview_id) - time = self._clean_string(time) - - self.cur.execute(update_sql, (venue, original_openreview_id, - Json(cleaned_content), time, revision_openreview_id)) - self.conn.commit() - - print(f"Revision with revision_openreview_id {revision_openreview_id} updated successfully.") - return revision_df - - def get_revision_by_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - # Query to select the current record using primary key - select_sql = """ - SELECT * FROM openreview_revisions WHERE revision_openreview_id = %s; - """ - self.cur.execute(select_sql, (revision_openreview_id,)) - row = self.cur.fetchone() - if not row: - print(f"No revision found with revision_openreview_id {revision_openreview_id}.") - return None - else: - # If record exists, return the original record as a dictionary - columns = ['venue', 'original_openreview_id', - 'revision_openreview_id', 'content', 'time'] - - # 处理content字段，如果是dict则转换为JSON字符串 - row_list = list(row[1:]) - if isinstance(row_list[3], dict): - row_list[3] = json.dumps(row_list[3]) - revision_df = pd.DataFrame([row_list], columns=columns) - return revision_df - - def get_revisions_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # Query to select all records for a specific venue - select_sql = """ - SELECT venue, original_openreview_id, revision_openreview_id, content, time - FROM openreview_revisions WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if not rows: - print(f"No openreview_revisions found in venue {venue}.") - return None - else: - columns = ['venue', 'original_openreview_id', - 'revision_openreview_id', 'content', 'time'] - - # 处理所有行的content字段，如果是dict则转换为JSON字符串 - processed_rows = [] - for row in rows: - row_list = list(row[1:]) - if isinstance(row_list[3], dict): - row_list[3] = json.dumps(row_list[3]) - processed_rows.append(row_list) - - revisions_df = pd.DataFrame(processed_rows, columns=columns) - return revisions_df - - def get_all_revisions(self, is_all_features: bool = False) -> Optional[pd.DataFrame]: - if is_all_features: - select_query = """ - SELECT venue, original_openreview_id, revision_openreview_id, content, time - FROM openreview_revisions ORDER BY id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - revisions = self.cur.fetchall() - - # 处理所有行的content字段，如果是dict则转换为JSON字符串 - processed_revisions = [] - for row in revisions: - row_list = list(row) - if isinstance(row_list[3], dict): - row_list[3] = json.dumps(row_list[3]) - processed_revisions.append(row_list) - - # Return the result as a DataFrame - revisions_df = pd.DataFrame(processed_revisions, columns=["venue", "original_openreview_id", "revision_openreview_id", "content", "time"]) - return revisions_df - else: - select_query = """ - SELECT venue, original_openreview_id, revision_openreview_id, time - FROM openreview_revisions ORDER BY id ASC - """ - self.cur.execute(select_query) - - # Fetch all the results - revisions = self.cur.fetchall() - - # Return the result as a list of tuples (paper_openreview_id, title, author_full_names) - revisions_df = pd.DataFrame(revisions, columns=["venue", "original_openreview_id", "revision_openreview_id", "time"]) - return revisions_df - - def check_revision_exists(self, revision_openreview_id: str) -> bool: - self.cur.execute("SELECT 1 FROM openreview_revisions WHERE revision_openreview_id = %s LIMIT 1;", (revision_openreview_id,)) - result = self.cur.fetchone() - - return result is not None - - def construct_revisions_table_from_api(self, venue: str, filter_list: list, pdf_dir: str, log_file: str) -> bool: - # crawl revision data from openreview API - print("Crawling revision data from OpenReview API...") - revision_data = self.openreview_crawler.crawl_revision_data_from_api(venue, filter_list, pdf_dir, log_file) - - # insert data into openreview_revisions table - if len(revision_data) > 0: - print("Inserting data into 'papers' table...") - for data in tqdm(revision_data): - self.insert_revision(**data) - return True - else: - print("No new revision data to insert.") - return False - - def construct_revisions_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - # read revision data from csv file - print(f"Reading openreview_revisions data from {csv_file}...") - revision_data = pd.read_csv(csv_file).to_dict(orient='records') - - # insert data into papers table - if len(revision_data) > 0: - print("Inserting data into 'papers' table...") - for data in tqdm(revision_data): - data["content"] = ast.literal_eval(data["content"]) - self.insert_revision(**data) - return True - else: - print("No new revision data to insert.") - return False - - def construct_revisions_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read revision data from json file - print(f"Reading openreview_revisions data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - revision_data = json.load(f) - - # insert data into papers table - if len(revision_data) > 0: - print("Inserting data into 'papers' table...") - for data in tqdm(revision_data): - self.insert_revision(**data) - return True - else: - print("No new revision data to insert.") - return False - - def _clean_json_content(self, content: Union[str, dict, list, int, float, bool, None]) -> Union[str, dict, list, int, float, bool, None]: - if isinstance(content, str): - return ''.join(char for char in content if char.isprintable()) # 返回 str - elif isinstance(content, dict): - return {key: self._clean_json_content(value) for key, value in content.items()} # 返回 dict - elif isinstance(content, list): - return [self._clean_json_content(item) for item in content] # 返回 list - else: - return content # 返回原始类型 - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/sql_database/sql_openreview_revisions_reviews.py b/research_arcade/sql_database/sql_openreview_revisions_reviews.py deleted file mode 100644 index a864900..0000000 --- a/research_arcade/sql_database/sql_openreview_revisions_reviews.py +++ /dev/null @@ -1,308 +0,0 @@ -from ..openreview_utils.openreview_crawler import OpenReviewCrawler -from tqdm import tqdm -import pandas as pd -import json -import psycopg2 -import os -from typing import Optional - -class SQLOpenReviewRevisionsReviews: - def __init__(self, host: str, dbname: str, user: str, password: str, port: str) -> None: - # Store connection and cursor for reuse - self.conn = psycopg2.connect( - host=host, dbname=dbname, - user=user, password=password, port=port - ) - # Enable autocommit - self.conn.autocommit = True - self.cur = self.conn.cursor() - self.openreview_crawler = OpenReviewCrawler() - self.create_revisions_reviews_table() - - def create_revisions_reviews_table(self) -> None: - create_table_sql = """ - CREATE TABLE IF NOT EXISTS openreview_revisions_reviews ( - venue TEXT, - revision_openreview_id VARCHAR(255), - review_openreview_id VARCHAR(255), - PRIMARY KEY (venue, revision_openreview_id, review_openreview_id) - ); - """ - # Execute the SQL to create the table - self.cur.execute(create_table_sql) - - def insert_revision_reviews(self, venue: str, revision_openreview_id: str, review_openreview_id: str) -> Optional[tuple]: - insert_sql = """ - INSERT INTO openreview_revisions_reviews (venue, revision_openreview_id, review_openreview_id) - VALUES (%s, %s, %s) - ON CONFLICT (venue, revision_openreview_id, review_openreview_id) DO NOTHING - RETURNING (venue, revision_openreview_id, review_openreview_id); - """ - - venue = self._clean_string(venue) - revision_openreview_id = self._clean_string(revision_openreview_id) - review_openreview_id = self._clean_string(review_openreview_id) - - # Execute the insertion query - self.cur.execute(insert_sql, (venue, revision_openreview_id, review_openreview_id)) - - # Get the inserted paper's openreview id (if any) - res = self.cur.fetchone() - return res[0] if res else None - - def get_revision_review_by_id(self, revision_openreview_id: str, review_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, revision_openreview_id, review_openreview_id FROM openreview_revisions_reviews - WHERE revision_openreview_id = %s AND review_openreview_id = %s; - """, (revision_openreview_id, review_openreview_id)) - - result = self.cur.fetchone() - - if result is None: - print(f''' - The revision {revision_openreview_id} and the review {review_openreview_id} are not connect in this database. - ''') - return None - else: - columns = ['venue', 'revision_openreview_id', 'review_openreview_id'] - # result = dict(zip(columns, result)) - result_df = pd.DataFrame(result, columns=columns) - return result_df - - def get_revisions_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, revision_openreview_id, review_openreview_id FROM openreview_revisions_reviews - WHERE venue = %s; - """, (venue,)) - - openreview_revisions_reviews = self.cur.fetchall() - - if openreview_revisions_reviews is not None: - revisions_reviews_df = pd.DataFrame(openreview_revisions_reviews, columns=["venue", "revision_openreview_id", "review_openreview_id"]) - return revisions_reviews_df - else: - return None - - def get_all_revisions_reviews(self) -> Optional[pd.DataFrame]: - select_query = """ - SELECT venue, revision_openreview_id, review_openreview_id - FROM openreview_revisions_reviews; - """ - self.cur.execute(select_query) - - # Fetch all the results - openreview_revisions_reviews = self.cur.fetchall() - - # Return the result as a list of tuples (venue, author_openreview_id, author_full_name) - revisions_reviews_df = pd.DataFrame(openreview_revisions_reviews, columns=["venue", "revision_openreview_id", "review_openreview_id"]) - return revisions_reviews_df - - def delete_revision_review_by_id(self, revision_openreview_id: str, review_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_revisions_reviews WHERE revision_openreview_id = %s AND review_openreview_id = %s; - """ - self.cur.execute(select_sql, (revision_openreview_id, review_openreview_id)) - records = self.cur.fetchall() - - if records: - columns=["venue", "revision_openreview_id", "review_openreview_id"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_revisions_reviews WHERE revision_openreview_id = %s AND review_openreview_id = %s; - """ - self.cur.execute(delete_sql, (revision_openreview_id, review_openreview_id)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_revisions_reviews' with revision_openreview_id = {revision_openreview_id} and review_openreview_id = {review_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_revisions_reviews' with revision_openreview_id = {revision_openreview_id} and review_openreview_id = {review_openreview_id}.") - return None - - def delete_revision_review_by_revision_id(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_revisions_reviews WHERE revision_openreview_id = %s; - """ - self.cur.execute(select_sql, (revision_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "revision_openreview_id", "review_openreview_id"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_revisions_reviews WHERE revision_openreview_id = %s; - """ - self.cur.execute(delete_sql, (revision_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_revisions_reviews' with revision_openreview_id = {revision_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_revisions_reviews' with revision_openreview_id = {revision_openreview_id}.") - return None - - def delete_revision_review_by_review_id(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - # search for records with the given arxiv_id and delete them - select_sql = """ - SELECT * FROM openreview_revisions_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(select_sql, (review_openreview_id,)) - records = self.cur.fetchall() - - if records: - columns=["venue", "revision_openreview_id", "review_openreview_id"] - records_df = pd.DataFrame(records, columns=columns) - - delete_sql = """ - DELETE FROM openreview_revisions_reviews WHERE review_openreview_id = %s; - """ - self.cur.execute(delete_sql, (review_openreview_id,)) - self.conn.commit() - - print(f"Deleted {len(records)} records from 'openreview_revisions_reviews' with review_openreview_id = {review_openreview_id}.") - return records_df - else: - print(f"No records found in 'openreview_revisions_reviews' with review_openreview_id = {review_openreview_id}.") - return None - - def delete_revisions_reviews_by_venue(self, venue: str) -> Optional[pd.DataFrame]: - # search the row based on primary key - select_sql = """ - SELECT venue, revision_openreview_id, review_openreview_id FROM openreview_revisions_reviews WHERE venue = %s; - """ - self.cur.execute(select_sql, (venue,)) - rows = self.cur.fetchall() - - if rows: - columns = ['venue', 'revision_openreview_id', 'review_openreview_id'] - # papers_reviews_dict = dict(zip(columns, rows)) - revisions_reviews_df = pd.DataFrame(rows, columns=columns) - - delete_sql = """ - DELETE FROM openreview_revisions_reviews WHERE venue = %s; - """ - self.cur.execute(delete_sql, (venue,)) - self.conn.commit() - - print(f"All connections in venue {venue} deleted successfully.") - return revisions_reviews_df - else: - print(f"No connections found in venue {venue}.") - return None - - def check_revision_review_exists(self, revision_openreview_id: str, review_openreview_id: str) -> bool: - self.cur.execute(""" - SELECT 1 FROM openreview_revisions_reviews - WHERE revision_openreview_id = %s AND review_openreview_id = %s - LIMIT 1; - """, (revision_openreview_id, review_openreview_id)) - - result = self.cur.fetchone() - - return result is not None - - def get_revision_neighboring_reviews(self, revision_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, revision_openreview_id, review_openreview_id FROM openreview_revisions_reviews - WHERE revision_openreview_id = %s; - """, (revision_openreview_id,)) - - revision_neighboring_reviews = self.cur.fetchall() - - if revision_neighboring_reviews is not None: - revision_neighboring_reviews_df = pd.DataFrame(revision_neighboring_reviews, columns=["venue", "revision_openreview_id", "review_openreview_id"]) - return revision_neighboring_reviews_df - else: - return None - - def get_review_neighboring_revisions(self, review_openreview_id: str) -> Optional[pd.DataFrame]: - self.cur.execute(""" - SELECT venue, revision_openreview_id, review_openreview_id FROM openreview_revisions_reviews - WHERE review_openreview_id = %s; - """, (review_openreview_id,)) - - review_neighboring_revisions = self.cur.fetchall() - - if review_neighboring_revisions is not None: - review_neighboring_revisions_df = pd.DataFrame(review_neighboring_revisions, columns=["venue", "revision_openreview_id", "review_openreview_id"]) - return review_neighboring_revisions_df - else: - return None - - def construct_revisions_reviews_table(self, papers_reviews_df: pd.DataFrame, papers_revisions_df: pd.DataFrame) -> bool: - # get unique paper ids - unique_paper_ids = papers_revisions_df['paper_openreview_id'].unique() - - for paper_id in tqdm(unique_paper_ids): - # get the revision ids - paper_revision_edges = papers_revisions_df[papers_revisions_df['paper_openreview_id'] == paper_id].sort_values(by='time', ascending=True) - - # get the review ids - paper_review_edges = papers_reviews_df[papers_reviews_df['paper_openreview_id'] == paper_id].sort_values(by='time', ascending=True) - - start_idx = 0 - for revision in paper_revision_edges.itertuples(): - venue = revision.venue - # get the revision time - revision_time = revision.time - # get the revision id - revision_id = revision.revision_openreview_id - - # get the review ids - for review in paper_review_edges.iloc[start_idx:].itertuples(): - # get the review time - review_time = review.time - if review_time > revision_time: - break - - # get the review id - review_id = review.review_openreview_id - - # insert the edge - self.insert_revision_reviews(venue, revision_id, review_id) - - start_idx += 1 - - def construct_revisions_reviews_table_from_csv(self, csv_file: str) -> bool: - if not os.path.exists(csv_file): - return False - else: - # read data from csv file - print(f"Reading revisions-reviews data from {csv_file}...") - revisions_reviews_data = pd.read_csv(csv_file).to_dict(orient='records') - - if len(revisions_reviews_data) > 0: - print(f"Inserting revisions-reviews data from {csv_file}...") - for data in tqdm(revisions_reviews_data): - self.insert_revision_reviews(**data) - return True - else: - print(f"No revisions-reviews data found in {csv_file}.") - return False - - def construct_revisions_reviews_table_from_json(self, json_file: str) -> bool: - if not os.path.exists(json_file): - return False - else: - # read revision data from json file - print(f"Reading revisions-reviews data from {json_file}...") - with open(json_file, 'r', encoding='utf-8') as f: - revisions_reviews_data = json.load(f) - - if len(revisions_reviews_data) > 0: - print(f"Inserting revisions-reviews data from {json_file}...") - for data in tqdm(revisions_reviews_data): - self.insert_revision_reviews(**data) - return True - else: - print(f"No revisions-reviews data found in {json_file}.") - return False - - def _clean_string(self, s: str) -> str: - if isinstance(s, str): - return s.replace('\x00', '') - return s \ No newline at end of file diff --git a/research_arcade/utils/error_handler.py b/research_arcade/utils/error_handler.py deleted file mode 100644 index 235291a..0000000 --- a/research_arcade/utils/error_handler.py +++ /dev/null @@ -1,87 +0,0 @@ -import math -import time -from functools import wraps - -from beartype.typing import Any, Callable, Optional, TypeVar, cast -from pydantic import BaseModel - -INF = float(math.inf) - -T = TypeVar('T', bound=Callable[..., Any]) - - -def api_calling_error_exponential_backoff( - retries: int = 5, base_wait_time: int = 1 -) -> Callable[[T], T]: - """ - Decorator for applying exponential backoff to a function. - :param retries: Maximum number of retries. - :param base_wait_time: Base wait time in seconds for the exponential backoff. - :return: The wrapped function with exponential backoff applied. - """ - - def decorator(func: T) -> T: - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - error_handler_mode = kwargs.get('mode', None) - if error_handler_mode == 'TEST': - modified_retries = 1 - modified_base_wait_time = 1 - else: - modified_retries = retries - modified_base_wait_time = base_wait_time - - attempts = 0 - while attempts < modified_retries: - try: - return func(*args, **kwargs) - except Exception as e: - wait_time = modified_base_wait_time * (2**attempts) - print(f'Attempt {attempts + 1} failed: {e}') - print(f'Waiting {wait_time} seconds before retrying...') - time.sleep(wait_time) - attempts += 1 - print( - f"Failed to execute '{func.__name__}' after {modified_retries} retries." - ) - return None - - return cast(T, wrapper) - - return cast(Callable[[T], T], decorator) - - -TBaseModel = TypeVar('TBaseModel', bound=Callable[..., BaseModel]) - - -def parsing_error_exponential_backoff( - retries: int = 5, base_wait_time: int = 1 -) -> Callable[[TBaseModel], TBaseModel]: - """ - Decorator for retrying a function that returns a BaseModel with exponential backoff. - :param retries: Maximum number of retries. - :param base_wait_time: Base wait time in seconds for the exponential backoff. - :return: The wrapped function with retry logic applied. - """ - - def decorator(func: TBaseModel) -> TBaseModel: - @wraps(func) - def wrapper(self: Any, *args: Any, **kwargs: Any) -> Optional[BaseModel]: - attempts = 0 - while attempts < retries: - try: - return func(self, *args, **kwargs) - except Exception as e: - wait_time = base_wait_time * (2**attempts) - print(f'Attempt {attempts + 1} failed: {e}') - print(f'Waiting {wait_time} seconds before retrying...') - time.sleep(wait_time) - attempts += 1 - print( - f'Failed to get valid input from {func.__name__} after {retries} retries.' - ) - return None - - return cast(TBaseModel, wrapper) - - return cast(Callable[[TBaseModel], TBaseModel], decorator) \ No newline at end of file diff --git a/research_arcade_complete_tutorial_with_imports.ipynb b/research_arcade_complete_tutorial_with_imports.ipynb deleted file mode 100644 index d27e125..0000000 --- a/research_arcade_complete_tutorial_with_imports.ipynb +++ /dev/null @@ -1,3389 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "header", - "metadata": {}, - "source": [ - "# ResearchArcade Complete Tutorial\n", - "\n", - "This tutorial demonstrates how to work with the ResearchArcade database, covering all node types and edge relationships.\n", - "\n", - "## Table of Contents\n", - "1. [Setup](#setup)\n", - "2. [OpenReview Data](#openreview)\n", - "3. [ArXiv Papers](#arxiv-papers)\n", - "4. [ArXiv Authors](#arxiv-authors)\n", - "5. [ArXiv Categories](#arxiv-categories)\n", - "6. [ArXiv Figures](#arxiv-figures)\n", - "7. [ArXiv Tables](#arxiv-tables)\n", - "8. [ArXiv Sections](#arxiv-sections)\n", - "9. [ArXiv Paragraphs](#arxiv-paragraphs)\n", - "10. [Relationships/Edges](#relationships)\n", - "11. [Advanced Queries](#advanced-queries)" - ] - }, - { - "cell_type": "markdown", - "id": "setup-section", - "metadata": {}, - "source": [ - "## 1. Setup " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "05b9b352", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "import os\n", - "sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..')))\n", - "from research_arcade.research_arcade import ResearchArcade\n", - "import pandas as pd\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "markdown", - "id": "c4f1a1e4", - "metadata": {}, - "source": [ - "### Choose Database Backend" - ] - }, - { - "cell_type": "markdown", - "id": "c9672a27", - "metadata": {}, - "source": [ - "#### CSV Based" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "169f7a6d", - "metadata": {}, - "outputs": [], - "source": [ - "db_type = \"csv\"\n", - "config = {\n", - " \"csv_dir\": \"../data/my_research_arcade_data/\"\n", - "}\n", - "\n", - "research_arcade = ResearchArcade(db_type=db_type, config=config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-section", - "metadata": {}, - "source": [ - "## 3. ArXiv Papers \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `arxiv_id` (VARCHAR, unique) - e.g., 1802.08773v3\n", - "- `base_arxiv_id` (VARCHAR) - e.g., 1802.08773\n", - "- `version` (INT) - e.g., 3\n", - "- `title` (TEXT)\n", - "- `abstract` (TEXT)\n", - "- `submit_date` (DATE)\n", - "- `metadata` (JSONB)" - ] - }, - { - "cell_type": "markdown", - "id": "91b8e215", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "ccaeefb3", - "metadata": {}, - "outputs": [], - "source": [ - "config = {\"arxiv_ids\": [\"1806.08804v4\", \"1903.03894v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 1 papers from ./examples/csv_data/csv_arxiv_papers_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_papers_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new papers to import (all papers already exist)\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_papers_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_papers\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-insert", - "metadata": {}, - "source": [ - "### Insert a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "insert-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 1: Insert the famous \"Attention is All You Need\" paper\n", - "new_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'base_arxiv_id': '1706.03762',\n", - " 'version': 7,\n", - " 'title': 'Attention Is All You Need',\n", - " 'abstract': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing both with large and limited training data.',\n", - " 'submit_date': '2017-06-12',\n", - " 'metadata': {'venue': 'NeurIPS 2017', 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf'}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=new_paper)\n", - "print(\"Paper inserted successfully!\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "insert-paper-bert", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BERT paper inserted successfully!\n" - ] - } - ], - "source": [ - "# Example 2: Insert BERT paper\n", - "bert_paper = {\n", - " 'arxiv_id': '1810.04805v2',\n", - " 'base_arxiv_id': '1810.04805',\n", - " 'version': 2,\n", - " 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding',\n", - " 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications.',\n", - " 'submit_date': '2018-10-11',\n", - " 'metadata': {'venue': 'NAACL 2019', 'citations': 50000}\n", - "}\n", - "\n", - "research_arcade.insert_node(\"arxiv_papers\", node_features=bert_paper)\n", - "print(\"BERT paper inserted successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-all", - "metadata": {}, - "source": [ - "### Get All Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "get-all-papers", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total papers in database: 7\n", - "\n", - "First 5 papers:\n", - " id arxiv_id base_arxiv_id version \\\n", - "0 2 1810.04805v2 1810.04805 2 \n", - "1 3 1409.0473v7 1409.04730 7 \n", - "2 4 1512.03385v1 1512.03385 1 \n", - "3 5 2010.11929v2 2010.11929 2 \n", - "4 6 1806.08804v4 1806.08804 4 \n", - "\n", - " title \\\n", - "0 BERT: Pre-training of Deep Bidirectional Trans... \n", - "1 Neural Machine Translation by Jointly Learning... \n", - "2 Deep Residual Learning for Image Recognition \n", - "3 An Image is Worth 16x16 Words: Transformers fo... \n", - "4 Hierarchical Graph Representation Learning wit... \n", - "\n", - " abstract \\\n", - "0 We introduce a new language representation mod... \n", - "1 Neural machine translation is a recently propo... \n", - "2 Deeper neural networks are more difficult to t... \n", - "3 We show that a pure transformer applied direct... \n", - "4 Recently, graph neural networks (GNNs) have re... \n", - "\n", - " submit_date \\\n", - "0 2018-10-11 \n", - "1 2014-09-01 \n", - "2 2015-12-10 \n", - "3 2020-10-22 \n", - "4 2018-06-22 18:04:46+00:00 \n", - "\n", - " metadata \n", - "0 {\"venue\": \"NAACL 2019\", \"citations\": 60000} \n", - "1 {\"venue\": \"ICLR 2015\", \"citations\": 30000} \n", - "2 {\"venue\": \"CVPR 2016\", \"citations\": 70000} \n", - "3 {\"venue\": \"ICLR 2021\", \"citations\": 15000} \n", - "4 {\"id\": \"1806.08804v4\", \"title\": \"Hierarchical ... \n" - ] - } - ], - "source": [ - "arxiv_papers_df = research_arcade.get_all_node_features(\"arxiv_papers\")\n", - "print(f\"Total papers in database: {len(arxiv_papers_df)}\")\n", - "print(\"\\nFirst 5 papers:\")\n", - "print(arxiv_papers_df.head())" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Paper by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "get-paper-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper details:\n", - "{'id': 2, 'arxiv_id': '1810.04805v2', 'base_arxiv_id': 1810.04805, 'version': 2, 'title': 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding', 'abstract': 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers.', 'submit_date': '2018-10-11', 'metadata': '{\"venue\": \"NAACL 2019\", \"citations\": 60000}'}\n" - ] - } - ], - "source": [ - "paper_id = {\"arxiv_id\": \"1810.04805v2\"}\n", - "paper_features = research_arcade.get_node_features_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Paper details:\")\n", - "print(paper_features.to_dict(orient=\"records\")[0])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-update", - "metadata": {}, - "source": [ - "### Update a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "update-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paper updated successfully!\n" - ] - } - ], - "source": [ - "# Update metadata for a paper\n", - "updated_paper = {\n", - " 'arxiv_id': '1706.03762v7',\n", - " 'metadata': {\n", - " 'venue': 'NeurIPS 2017',\n", - " 'pdf_url': 'https://arxiv.org/pdf/1706.03762.pdf',\n", - " 'citations': 75000,\n", - " 'influential': True\n", - " }\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_papers\", node_features=updated_paper)\n", - "print(\"Paper updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-papers-delete", - "metadata": {}, - "source": [ - "### Delete a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "delete-paper", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted paper:\n", - "True\n" - ] - } - ], - "source": [ - "# Delete a paper by ID\n", - "paper_id = {\"arxiv_id\": \"1706.03762v7\"}\n", - "deleted_paper = research_arcade.delete_node_by_id(\"arxiv_papers\", paper_id)\n", - "print(\"Deleted paper:\")\n", - "print(deleted_paper)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-section", - "metadata": {}, - "source": [ - "## 4. ArXiv Authors \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `semantic_scholar_id` (VARCHAR, unique)\n", - "- `name` (VARCHAR)\n", - "- `homepage` (VARCHAR)" - ] - }, - { - "cell_type": "markdown", - "id": "6e14ad06", - "metadata": {}, - "source": [ - "### Construct Table from API" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "c18c7737", - "metadata": {}, - "outputs": [], - "source": [ - "# config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "# research_arcade.construct_table_from_api(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 10 authors from ./examples/csv_data/csv_arxiv_authors_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_authors_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new authors to import (all authors already exist)\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_authors_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_authors\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-insert", - "metadata": {}, - "source": [ - "### Insert Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "insert-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted author: Ashish Vaswani\n", - "Inserted author: Noam Shazeer\n", - "Inserted author: Niki Parmar\n", - "Inserted author: Jakob Uszkoreit\n", - "Inserted author: Llion Jones\n" - ] - } - ], - "source": [ - "# Insert authors from the Transformer paper\n", - "authors = [\n", - " {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'name': 'Ashish Vaswani',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_noam_shazeer',\n", - " 'name': 'Noam Shazeer',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_niki_parmar',\n", - " 'name': 'Niki Parmar',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_jakob_uszkoreit',\n", - " 'name': 'Jakob Uszkoreit',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " },\n", - " {\n", - " 'semantic_scholar_id': 'ss_llion_jones',\n", - " 'name': 'Llion Jones',\n", - " 'homepage': 'https://scholar.google.com/citations?user=oR9sCGYAAAAJ'\n", - " }\n", - "]\n", - "\n", - "for author in authors:\n", - " research_arcade.insert_node(\"arxiv_authors\", node_features=author)\n", - " print(f\"Inserted author: {author['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-all", - "metadata": {}, - "source": [ - "### Get All Authors" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "get-all-authors", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total authors in database: 55\n", - "\n", - "All authors:\n", - " id semantic_scholar_id name \\\n", - "0 1 1234567 Ashish Vaswani \n", - "1 2 2345678 Noam Shazeer \n", - "2 3 3456789 Niki Parmar \n", - "3 4 4567890 Jakob Uszkoreit \n", - "4 5 5678901 Llion Jones \n", - "5 6 6789012 Aidan N. Gomez \n", - "6 7 7890123 Lukasz Kaiser \n", - "7 8 8901234 Illia Polosukhin \n", - "8 9 9012345 Jacob Devlin \n", - "9 10 1234098 Ming-Wei Chang \n", - "10 11 1234567 Ashish Vaswani \n", - "11 12 2345678 Noam Shazeer \n", - "12 13 3456789 Niki Parmar \n", - "13 14 4567890 Jakob Uszkoreit \n", - "14 15 5678901 Llion Jones \n", - "15 16 6789012 Aidan N. Gomez \n", - "16 17 7890123 Lukasz Kaiser \n", - "17 18 8901234 Illia Polosukhin \n", - "18 19 9012345 Jacob Devlin \n", - "19 20 1234098 Ming-Wei Chang \n", - "20 21 1234567 Ashish Vaswani \n", - "21 22 2345678 Noam Shazeer \n", - "22 23 3456789 Niki Parmar \n", - "23 24 4567890 Jakob Uszkoreit \n", - "24 25 5678901 Llion Jones \n", - "25 26 6789012 Aidan N. Gomez \n", - "26 27 7890123 Lukasz Kaiser \n", - "27 28 8901234 Illia Polosukhin \n", - "28 29 9012345 Jacob Devlin \n", - "29 30 1234098 Ming-Wei Chang \n", - "30 31 ss_ashish_vaswani Ashish Vaswani \n", - "31 32 ss_noam_shazeer Noam Shazeer \n", - "32 33 ss_niki_parmar Niki Parmar \n", - "33 34 ss_jakob_uszkoreit Jakob Uszkoreit \n", - "34 35 ss_llion_jones Llion Jones \n", - "35 36 1234567 Ashish Vaswani \n", - "36 37 2345678 Noam Shazeer \n", - "37 38 3456789 Niki Parmar \n", - "38 39 4567890 Jakob Uszkoreit \n", - "39 40 5678901 Llion Jones \n", - "40 41 6789012 Aidan N. Gomez \n", - "41 42 7890123 Lukasz Kaiser \n", - "42 43 8901234 Illia Polosukhin \n", - "43 44 9012345 Jacob Devlin \n", - "44 45 1234098 Ming-Wei Chang \n", - "45 46 1234567 Ashish Vaswani \n", - "46 47 2345678 Noam Shazeer \n", - "47 48 3456789 Niki Parmar \n", - "48 49 4567890 Jakob Uszkoreit \n", - "49 50 5678901 Llion Jones \n", - "50 51 6789012 Aidan N. Gomez \n", - "51 52 7890123 Lukasz Kaiser \n", - "52 53 8901234 Illia Polosukhin \n", - "53 54 9012345 Jacob Devlin \n", - "54 55 1234098 Ming-Wei Chang \n", - "\n", - " homepage \n", - "0 https://scholar.google.com/citations?user=Pu55... \n", - "1 https://scholar.google.com/citations?user=8VY4... \n", - "2 https://scholar.google.com/citations?user=rK3M... \n", - "3 https://scholar.google.com/citations?user=UYZj... \n", - "4 https://scholar.google.com/citations?user=exam... \n", - "5 https://scholar.google.com/citations?user=exam... \n", - "6 https://scholar.google.com/citations?user=ZmBQ... \n", - "7 https://scholar.google.com/citations?user=exam... \n", - "8 https://scholar.google.com/citations?user=exam... \n", - "9 https://scholar.google.com/citations?user=exam... \n", - "10 https://scholar.google.com/citations?user=Pu55... \n", - "11 https://scholar.google.com/citations?user=8VY4... \n", - "12 https://scholar.google.com/citations?user=rK3M... \n", - "13 https://scholar.google.com/citations?user=UYZj... \n", - "14 https://scholar.google.com/citations?user=exam... \n", - "15 https://scholar.google.com/citations?user=exam... \n", - "16 https://scholar.google.com/citations?user=ZmBQ... \n", - "17 https://scholar.google.com/citations?user=exam... \n", - "18 https://scholar.google.com/citations?user=exam... \n", - "19 https://scholar.google.com/citations?user=exam... \n", - "20 https://scholar.google.com/citations?user=Pu55... \n", - "21 https://scholar.google.com/citations?user=8VY4... \n", - "22 https://scholar.google.com/citations?user=rK3M... \n", - "23 https://scholar.google.com/citations?user=UYZj... \n", - "24 https://scholar.google.com/citations?user=exam... \n", - "25 https://scholar.google.com/citations?user=exam... \n", - "26 https://scholar.google.com/citations?user=ZmBQ... \n", - "27 https://scholar.google.com/citations?user=exam... \n", - "28 https://scholar.google.com/citations?user=exam... \n", - "29 https://scholar.google.com/citations?user=exam... \n", - "30 https://ashishvaswani.com \n", - "31 https://scholar.google.com/citations?user=oR9s... \n", - "32 https://scholar.google.com/citations?user=oR9s... \n", - "33 https://scholar.google.com/citations?user=oR9s... \n", - "34 https://scholar.google.com/citations?user=oR9s... \n", - "35 https://scholar.google.com/citations?user=Pu55... \n", - "36 https://scholar.google.com/citations?user=8VY4... \n", - "37 https://scholar.google.com/citations?user=rK3M... \n", - "38 https://scholar.google.com/citations?user=UYZj... \n", - "39 https://scholar.google.com/citations?user=exam... \n", - "40 https://scholar.google.com/citations?user=exam... \n", - "41 https://scholar.google.com/citations?user=ZmBQ... \n", - "42 https://scholar.google.com/citations?user=exam... \n", - "43 https://scholar.google.com/citations?user=exam... \n", - "44 https://scholar.google.com/citations?user=exam... \n", - "45 https://scholar.google.com/citations?user=Pu55... \n", - "46 https://scholar.google.com/citations?user=8VY4... \n", - "47 https://scholar.google.com/citations?user=rK3M... \n", - "48 https://scholar.google.com/citations?user=UYZj... \n", - "49 https://scholar.google.com/citations?user=exam... \n", - "50 https://scholar.google.com/citations?user=exam... \n", - "51 https://scholar.google.com/citations?user=ZmBQ... \n", - "52 https://scholar.google.com/citations?user=exam... \n", - "53 https://scholar.google.com/citations?user=exam... \n", - "54 https://scholar.google.com/citations?user=exam... \n" - ] - } - ], - "source": [ - "authors_df = research_arcade.get_all_node_features(\"arxiv_authors\")\n", - "print(f\"Total authors in database: {len(authors_df)}\")\n", - "print(\"\\nAll authors:\")\n", - "print(authors_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-get-by-id", - "metadata": {}, - "source": [ - "### Get Specific Author by ID" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "get-author-by-id", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author details:\n", - "None\n" - ] - } - ], - "source": [ - "author_id = {\"semantic_scholar_id\": 8901234}\n", - "author_features = research_arcade.get_node_features_by_id(\"arxiv_authors\", author_id)\n", - "print(\"Author details:\")\n", - "print(author_features)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-authors-update", - "metadata": {}, - "source": [ - "### Update an Author" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "update-author", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Author updated successfully!\n" - ] - } - ], - "source": [ - "updated_author = {\n", - " 'semantic_scholar_id': 'ss_ashish_vaswani',\n", - " 'homepage': 'https://ashishvaswani.com'\n", - "}\n", - "\n", - "research_arcade.update_node(\"arxiv_authors\", node_features=updated_author)\n", - "print(\"Author updated successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-section", - "metadata": {}, - "source": [ - "## 5. ArXiv Categories \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `name` (VARCHAR, unique)\n", - "- `description` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "3e9eeea6", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "168633f2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': '1903.03894v4', 'title': 'GNNExplainer: Generating Explanations for Graph Neural Networks', 'abstract': \"Graph Neural Networks (GNNs) are a powerful tool for machine learning on\\ngraphs.GNNs combine node feature information with the graph structure by\\nrecursively passing neural messages along edges of the input graph. However,\\nincorporating both graph structure and feature information leads to complex\\nmodels, and explaining predictions made by GNNs remains unsolved. Here we\\npropose GNNExplainer, the first general, model-agnostic approach for providing\\ninterpretable explanations for predictions of any GNN-based model on any\\ngraph-based machine learning task. Given an instance, GNNExplainer identifies a\\ncompact subgraph structure and a small subset of node features that have a\\ncrucial role in GNN's prediction. Further, GNNExplainer can generate consistent\\nand concise explanations for an entire class of instances. We formulate\\nGNNExplainer as an optimization task that maximizes the mutual information\\nbetween a GNN's prediction and distribution of possible subgraph structures.\\nExperiments on synthetic and real-world graphs show that our approach can\\nidentify important graph structures as well as node features, and outperforms\\nbaselines by 17.1% on average. GNNExplainer provides a variety of benefits,\\nfrom the ability to visualize semantically relevant structures to\\ninterpretability, to giving insights into errors of faulty GNNs.\", 'authors': ['Rex Ying', 'Dylan Bourgeois', 'Jiaxuan You', 'Marinka Zitnik', 'Jure Leskovec'], 'published': '2019-03-10 00:56:26+00:00', 'categories': ['cs.LG', 'stat.ML'], 'url': 'http://arxiv.org/abs/1903.03894v4'}\n", - "{'id': '1806.08804v4', 'title': 'Hierarchical Graph Representation Learning with Differentiable Pooling', 'abstract': 'Recently, graph neural networks (GNNs) have revolutionized the field of graph\\nrepresentation learning through effectively learned node embeddings, and\\nachieved state-of-the-art results in tasks such as node classification and link\\nprediction. However, current GNN methods are inherently flat and do not learn\\nhierarchical representations of graphs---a limitation that is especially\\nproblematic for the task of graph classification, where the goal is to predict\\nthe label associated with an entire graph. Here we propose DiffPool, a\\ndifferentiable graph pooling module that can generate hierarchical\\nrepresentations of graphs and can be combined with various graph neural network\\narchitectures in an end-to-end fashion. DiffPool learns a differentiable soft\\ncluster assignment for nodes at each layer of a deep GNN, mapping nodes to a\\nset of clusters, which then form the coarsened input for the next GNN layer.\\nOur experimental results show that combining existing GNN methods with DiffPool\\nyields an average improvement of 5-10% accuracy on graph classification\\nbenchmarks, compared to all existing pooling approaches, achieving a new\\nstate-of-the-art on four out of five benchmark data sets.', 'authors': ['Rex Ying', 'Jiaxuan You', 'Christopher Morris', 'Xiang Ren', 'William L. Hamilton', 'Jure Leskovec'], 'published': '2018-06-22 18:04:46+00:00', 'categories': ['cs.LG', 'cs.NE', 'cs.SI', 'stat.ML'], 'url': 'http://arxiv.org/abs/1806.08804v4'}\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new categories to import (all categories already exist)\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_categories_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new categories to import (all categories already exist)\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_categories_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_categories\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-insert", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "insert-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-categories-get-all", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "get-all-categories", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 11\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.LG NaN\n", - "1 2 stat.ML NaN\n", - "2 3 cs.NE NaN\n", - "3 4 cs.SI NaN\n", - "4 5 cs.AI Artificial Intelligence\n", - "5 7 cs.CL Computation and Language\n", - "6 8 cs.CV Computer Vision and Pattern Recognition\n", - "7 11 cs.CR Cryptography and Security\n", - "8 12 cs.DS Data Structures and Algorithms\n", - "9 13 cs.IT Information Theory\n", - "10 14 math.IT Information Theory (Math)\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-section", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-insert", - "metadata": {}, - "source": [ - "### Insert Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "insert-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted Figure 1\n", - "Inserted Figure 2\n", - "Inserted Figure 3\n" - ] - } - ], - "source": [ - "# Insert figures for the Transformer paper\n", - "figures = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/transformer_architecture.png',\n", - " 'caption': 'The Transformer model architecture. The left side shows the encoder stack and the right side shows the decoder stack.',\n", - " 'label': 'fig:architecture',\n", - " 'name': 'Figure 1'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/scaled_dot_product_attention.png',\n", - " 'caption': 'Scaled Dot-Product Attention and Multi-Head Attention mechanisms.',\n", - " 'label': 'fig:attention',\n", - " 'name': 'Figure 2'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/figures/positional_encoding.png',\n", - " 'caption': 'Positional encoding visualization showing sine and cosine functions of different frequencies.',\n", - " 'label': 'fig:positional',\n", - " 'name': 'Figure 3'\n", - " }\n", - "]\n", - "\n", - "for figure in figures:\n", - " research_arcade.insert_node(\"arxiv_figures\", node_features=figure)\n", - " print(f\"Inserted {figure['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-figures-get-all", - "metadata": {}, - "source": [ - "### Get All Figures" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "get-all-figures", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total figures: 11\n", - "\n", - "All figures:\n", - " name caption \\\n", - "0 Figure 1 The Transformer model architecture. The left s... \n", - "1 Figure 2 Scaled Dot-Product Attention and Multi-Head At... \n", - "2 Figure 3 Positional encoding visualization showing sine... \n", - "3 figure1 The Transformer model architecture \n", - "4 figure2 Multi-head attention mechanism visualization \n", - "5 figure3 Variations on the Transformer architecture \n", - "6 figure4 BERT model architecture and pre-training tasks \n", - "7 figure5 Fine-tuning BERT for different tasks \n", - "8 figure6 Residual learning: a building block \n", - "9 figure7 ResNet architectures for ImageNet \n", - "10 figure8 Vision Transformer (ViT) model overview \n", - "\n", - " label \n", - "0 fig:architecture \n", - "1 fig:attention \n", - "2 fig:positional \n", - "3 fig:architecture \n", - "4 fig:attention \n", - "5 fig:variations \n", - "6 fig:bert_arch \n", - "7 fig:fine_tune \n", - "8 fig:residual \n", - "9 fig:resnet_arch \n", - "10 fig:vit \n" - ] - } - ], - "source": [ - "figures_df = research_arcade.get_all_node_features(\"arxiv_figures\")\n", - "print(f\"Total figures: {len(figures_df)}\")\n", - "print(\"\\nAll figures:\")\n", - "print(figures_df[['name', 'caption', 'label']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-section", - "metadata": {}, - "source": [ - "## 7. ArXiv Tables \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `table_text` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "c240602b", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "54a13d98", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed: ['1903.03894v4']\n", - "BFS_que.qsize(): 1\n", - "current paper: 1903.03894v4\n", - "Thread 13079867392 Processing 1903.03894v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "x 000abstract.tex\n", - "x 010intro.tex\n", - "x 020related.tex\n", - "x 030background.tex\n", - "x 030formulation.tex\n", - "x 030proposed.tex\n", - "x 040experiments.tex\n", - "x 050conclusion.tex\n", - "x 060supplement.tex\n", - "x acmart.bib\n", - "x acmart.cls\n", - "x acmart.dtx\n", - "x acmart.ins\n", - "x ACM-Reference-Format.bbx\n", - "x ACM-Reference-Format.bst\n", - "x ACM-Reference-Format.cbx\n", - "x ACM-Reference-Format.dbx\n", - "x figs/\n", - "x figs/explainer-introduction_v2.pdf\n", - "x figs/explainer-motivation.pdf\n", - "x figs/explainer.pdf\n", - "x figs/feature_importance_v2.pdf\n", - "x figs/fig3-graph-cls-v2.pdf\n", - "x figs/fig3-graph-cls.pdf\n", - "x figs/fig3-node-cls-v3.pdf\n", - "x figs/fig3-node-cls.pdf\n", - "x figs/fig3-v4.pdf\n", - "x figs/fig3-v5.pdf\n", - "x figs/including-node-features.pdf\n", - "x figs/local_subgraph.png\n", - "x figs/motivation-node-features.pdf\n", - "x figs/prototype.png\n", - "x figs/prototype1.png\n", - "x figs/single-instance-explanation-final.pdf\n", - "x figs/single-instance-explanation2.pdf\n", - "x figs/single-instance-explanations.pdf: truncated gzip input\n", - "tar: Error exit delayed from previous errors.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 13079867392 Finished processing 1903.03894v4 (1/999999999) Time elapsed: 0.70s\n", - "'NoneType' object is not subscriptable\n", - "Thread 13079867392 Failed to process 1903.03894v4\n", - "Thread 8614781504 Finished processing 1 papers\n", - "Error: The file at path './download/output/1903.03894v4.json' was not found.\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 6 tables from ./examples/csv_data/csv_arxiv_tables_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_tables_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_tables\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "b7002162", - "metadata": {}, - "source": [ - "### Insert Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "b809fdd9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted category: cs.CL\n", - "Inserted category: cs.LG\n", - "Inserted category: cs.AI\n", - "Inserted category: cs.CV\n", - "Inserted category: stat.ML\n" - ] - } - ], - "source": [ - "categories = [\n", - " {\n", - " 'name': 'cs.CL',\n", - " 'description': 'Computation and Language (Natural Language Processing)'\n", - " },\n", - " {\n", - " 'name': 'cs.LG',\n", - " 'description': 'Machine Learning'\n", - " },\n", - " {\n", - " 'name': 'cs.AI',\n", - " 'description': 'Artificial Intelligence'\n", - " },\n", - " {\n", - " 'name': 'cs.CV',\n", - " 'description': 'Computer Vision and Pattern Recognition'\n", - " },\n", - " {\n", - " 'name': 'stat.ML',\n", - " 'description': 'Machine Learning (Statistics)'\n", - " }\n", - "]\n", - "\n", - "for category in categories:\n", - " research_arcade.insert_node(\"arxiv_categories\", node_features=category)\n", - " print(f\"Inserted category: {category['name']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3771de83", - "metadata": {}, - "source": [ - "### Get All Categories" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "1f1357fc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total categories: 11\n", - "\n", - "All categories:\n", - " id name description\n", - "0 1 cs.LG NaN\n", - "1 2 stat.ML NaN\n", - "2 3 cs.NE NaN\n", - "3 4 cs.SI NaN\n", - "4 5 cs.AI Artificial Intelligence\n", - "5 7 cs.CL Computation and Language\n", - "6 8 cs.CV Computer Vision and Pattern Recognition\n", - "7 11 cs.CR Cryptography and Security\n", - "8 12 cs.DS Data Structures and Algorithms\n", - "9 13 cs.IT Information Theory\n", - "10 14 math.IT Information Theory (Math)\n" - ] - } - ], - "source": [ - "categories_df = research_arcade.get_all_node_features(\"arxiv_categories\")\n", - "print(f\"Total categories: {len(categories_df)}\")\n", - "print(\"\\nAll categories:\")\n", - "print(categories_df)" - ] - }, - { - "cell_type": "markdown", - "id": "827d6714", - "metadata": {}, - "source": [ - "## 6. ArXiv Figures \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `path` (VARCHAR)\n", - "- `caption` (TEXT)\n", - "- `label` (TEXT)\n", - "- `name` (TEXT)" - ] - }, - { - "cell_type": "markdown", - "id": "1a7d33eb", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "195e218d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed: ['1903.03894v4']\n", - "BFS_que.qsize(): 1\n", - "current paper: 1903.03894v4\n", - "Thread 13079867392 Processing 1903.03894v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "x 000abstract.tex\n", - "x 010intro.tex\n", - "x 020related.tex\n", - "x 030background.tex\n", - "x 030formulation.tex\n", - "x 030proposed.tex\n", - "x 040experiments.tex\n", - "x 050conclusion.tex\n", - "x 060supplement.tex\n", - "x acmart.bib\n", - "x acmart.cls\n", - "x acmart.dtx\n", - "x acmart.ins\n", - "x ACM-Reference-Format.bbx\n", - "x ACM-Reference-Format.bst\n", - "x ACM-Reference-Format.cbx\n", - "x ACM-Reference-Format.dbx\n", - "x figs/\n", - "x figs/explainer-introduction_v2.pdf\n", - "x figs/explainer-motivation.pdf\n", - "x figs/explainer.pdf\n", - "x figs/feature_importance_v2.pdf\n", - "x figs/fig3-graph-cls-v2.pdf\n", - "x figs/fig3-graph-cls.pdf\n", - "x figs/fig3-node-cls-v3.pdf\n", - "x figs/fig3-node-cls.pdf\n", - "x figs/fig3-v4.pdf\n", - "x figs/fig3-v5.pdf\n", - "x figs/including-node-features.pdf\n", - "x figs/local_subgraph.png\n", - "x figs/motivation-node-features.pdf\n", - "x figs/prototype.png\n", - "x figs/prototype1.png\n", - "x figs/single-instance-explanation-final.pdf\n", - "x figs/single-instance-explanation2.pdf\n", - "x figs/single-instance-explanations.pdf: truncated gzip input\n", - "tar: Error exit delayed from previous errors.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 13079867392 Finished processing 1903.03894v4 (1/999999999) Time elapsed: 0.71s\n", - "'NoneType' object is not subscriptable\n", - "Thread 13079867392 Failed to process 1903.03894v4\n", - "Thread 8614781504 Finished processing 1 papers\n", - "Error: The file with path './download/output/1903.03894v4.json' was not found.\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new figures to import\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_figures_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new figures to import\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_figures_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_figures\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-insert", - "metadata": {}, - "source": [ - "### Insert Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "insert-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted table: tab:variations\n", - "Inserted table: tab:wmt\n", - "Inserted table: tab:parsing\n" - ] - } - ], - "source": [ - "# Insert tables for the Transformer paper\n", - "tables = [\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/model_variations.tex',\n", - " 'caption': 'Variations on the Transformer architecture with different hyperparameters.',\n", - " 'label': 'tab:variations',\n", - " 'table_text': 'Model | N | d_model | d_ff | h | d_k | d_v | P_drop | train time\\nbase | 6 | 512 | 2048 | 8 | 64 | 64 | 0.1 | 12 hrs'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/wmt_results.tex',\n", - " 'caption': 'Performance of the Transformer on WMT 2014 English-German and English-French translation tasks.',\n", - " 'label': 'tab:wmt',\n", - " 'table_text': 'Model | EN-DE BLEU | EN-FR BLEU\\nTransformer (base) | 27.3 | 38.1\\nTransformer (big) | 28.4 | 41.8'\n", - " },\n", - " {\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'path': '/tables/parsing_results.tex',\n", - " 'caption': 'English constituency parsing results on WSJ test set.',\n", - " 'label': 'tab:parsing',\n", - " 'table_text': 'Model | WSJ 23 F1\\nTransformer | 91.3'\n", - " }\n", - "]\n", - "\n", - "for table in tables:\n", - " research_arcade.insert_node(\"arxiv_tables\", node_features=table)\n", - " print(f\"Inserted table: {table['label']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-tables-get-all", - "metadata": {}, - "source": [ - "### Get All Tables" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "get-all-tables", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total tables: 34\n", - "\n", - "All tables:\n", - " label caption\n", - "0 tab:wmt_results Machine translation performance on WMT datasets\n", - "1 tab:variations Variations on the Transformer architecture\n", - "2 tab:glue BERT performance on GLUE benchmark tasks\n", - "3 tab:squad Results on SQuAD question answering\n", - "4 tab:imagenet Classification error on ImageNet validation set\n", - "5 tab:vit_perf Vision Transformer performance comparison\n", - "6 tab:wmt_results Machine translation performance on WMT datasets\n", - "7 tab:variations Variations on the Transformer architecture\n", - "8 tab:glue BERT performance on GLUE benchmark tasks\n", - "9 tab:squad Results on SQuAD question answering\n", - "10 tab:imagenet Classification error on ImageNet validation set\n", - "11 tab:vit_perf Vision Transformer performance comparison\n", - "12 tab:variations Variations on the Transformer architecture wit...\n", - "13 tab:wmt Performance of the Transformer on WMT 2014 Eng...\n", - "14 tab:parsing English constituency parsing results on WSJ te...\n", - "15 \\label{tab:results} \\caption{Classification accuracies in percent....\n", - "16 \\label{tab:results2} \\caption{Accuracy results of applying \\name to...\n", - "17 tab:wmt_results Machine translation performance on WMT datasets\n", - "18 tab:variations Variations on the Transformer architecture\n", - "19 tab:glue BERT performance on GLUE benchmark tasks\n", - "20 tab:squad Results on SQuAD question answering\n", - "21 tab:imagenet Classification error on ImageNet validation set\n", - "22 tab:vit_perf Vision Transformer performance comparison\n", - "23 \\label{tab:results} \\caption{Classification accuracies in percent....\n", - "24 \\label{tab:results2} \\caption{Accuracy results of applying \\name to...\n", - "25 tab:wmt_results Machine translation performance on WMT datasets\n", - "26 tab:variations Variations on the Transformer architecture\n", - "27 tab:glue BERT performance on GLUE benchmark tasks\n", - "28 tab:squad Results on SQuAD question answering\n", - "29 tab:imagenet Classification error on ImageNet validation set\n", - "30 tab:vit_perf Vision Transformer performance comparison\n", - "31 tab:variations Variations on the Transformer architecture wit...\n", - "32 tab:wmt Performance of the Transformer on WMT 2014 Eng...\n", - "33 tab:parsing English constituency parsing results on WSJ te...\n" - ] - } - ], - "source": [ - "tables_df = research_arcade.get_all_node_features(\"arxiv_tables\")\n", - "print(f\"Total tables: {len(tables_df)}\")\n", - "print(\"\\nAll tables:\")\n", - "print(tables_df[['label', 'caption']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-section", - "metadata": {}, - "source": [ - "## 8. ArXiv Sections \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `content` (TEXT)\n", - "- `title` (TEXT)\n", - "- `appendix` (BOOLEAN)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `section_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "d9890560", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "c1735fc6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed: ['1903.03894v4']\n", - "BFS_que.qsize(): 1\n", - "current paper: 1903.03894v4\n", - "Thread 13079867392 Processing 1903.03894v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "x 000abstract.tex\n", - "x 010intro.tex\n", - "x 020related.tex\n", - "x 030background.tex\n", - "x 030formulation.tex\n", - "x 030proposed.tex\n", - "x 040experiments.tex\n", - "x 050conclusion.tex\n", - "x 060supplement.tex\n", - "x acmart.bib\n", - "x acmart.cls\n", - "x acmart.dtx\n", - "x acmart.ins\n", - "x ACM-Reference-Format.bbx\n", - "x ACM-Reference-Format.bst\n", - "x ACM-Reference-Format.cbx\n", - "x ACM-Reference-Format.dbx\n", - "x figs/\n", - "x figs/explainer-introduction_v2.pdf\n", - "x figs/explainer-motivation.pdf\n", - "x figs/explainer.pdf\n", - "x figs/feature_importance_v2.pdf\n", - "x figs/fig3-graph-cls-v2.pdf\n", - "x figs/fig3-graph-cls.pdf\n", - "x figs/fig3-node-cls-v3.pdf\n", - "x figs/fig3-node-cls.pdf\n", - "x figs/fig3-v4.pdf\n", - "x figs/fig3-v5.pdf\n", - "x figs/including-node-features.pdf\n", - "x figs/local_subgraph.png\n", - "x figs/motivation-node-features.pdf\n", - "x figs/prototype.png\n", - "x figs/prototype1.png\n", - "x figs/single-instance-explanation-final.pdf\n", - "x figs/single-instance-explanation2.pdf\n", - "x figs/single-instance-explanations.pdf: truncated gzip input\n", - "tar: Error exit delayed from previous errors.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 13079867392 Finished processing 1903.03894v4 (1/999999999) Time elapsed: 0.65s\n", - "'NoneType' object is not subscriptable\n", - "Thread 13079867392 Failed to process 1903.03894v4\n", - "Thread 8614781504 Finished processing 1 papers\n", - "Error: The file at path './download/output/1903.03894v4.json' was not found.\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 8 sections from ./examples/csv_data/csv_arxiv_sections_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_sections_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 8 sections from ./examples/json_data/json_arxiv_sections_example.json\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_sections_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_sections\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-insert", - "metadata": {}, - "source": [ - "### Insert Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "insert-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted section: Introduction\n", - "Inserted section: Background\n", - "Inserted section: Model Architecture\n", - "Inserted section: Training\n", - "Inserted section: Results\n", - "Inserted section: Conclusion\n" - ] - } - ], - "source": [ - "# Insert sections for the Transformer paper\n", - "sections = [\n", - " {\n", - " 'content': 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder...',\n", - " 'title': 'Introduction',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 1\n", - " },\n", - " {\n", - " 'content': 'Most competitive neural sequence transduction models have an encoder-decoder structure. Here, the encoder maps an input sequence of symbol representations...',\n", - " 'title': 'Background',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 2\n", - " },\n", - " {\n", - " 'content': 'Most neural sequence transduction models have an encoder-decoder structure. The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers...',\n", - " 'title': 'Model Architecture',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 3\n", - " },\n", - " {\n", - " 'content': 'In this section we describe the training regime for our models...',\n", - " 'title': 'Training',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 4\n", - " },\n", - " {\n", - " 'content': 'On the WMT 2014 English-to-German translation task, the big transformer model outperforms the best previously reported models...',\n", - " 'title': 'Results',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 5\n", - " },\n", - " {\n", - " 'content': 'In this work, we presented the Transformer, the first sequence transduction model based entirely on attention, replacing the recurrent layers...',\n", - " 'title': 'Conclusion',\n", - " 'appendix': False,\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'section_in_paper_id': 6\n", - " }\n", - "]\n", - "\n", - "for section in sections:\n", - " research_arcade.insert_node(\"arxiv_sections\", node_features=section)\n", - " print(f\"Inserted section: {section['title']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-sections-get-all", - "metadata": {}, - "source": [ - "### Get All Sections" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "get-all-sections", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total sections: id content title \\\n", - "0 1 The dominant sequence transduction models are ... Introduction \n", - "1 2 The goal of reducing sequential computation al... Background \n", - "2 3 The Transformer follows this overall architect... Model Architecture \n", - "3 4 We trained on the standard WMT 2014 English-Ge... Training \n", - "4 5 In this work we presented the Transformer, the... Conclusion \n", - "5 6 We introduce a new language representation mod... Introduction \n", - "6 7 Unlike recent language representation models, ... Related Work \n", - "7 8 BERT uses a multi-layer bidirectional Transfor... Model Architecture \n", - "8 9 The dominant sequence transduction models are ... Introduction \n", - "9 10 The goal of reducing sequential computation al... Background \n", - "10 11 The Transformer follows this overall architect... Model Architecture \n", - "11 12 We trained on the standard WMT 2014 English-Ge... Training \n", - "12 13 In this work we presented the Transformer, the... Conclusion \n", - "13 14 We introduce a new language representation mod... Introduction \n", - "14 15 Unlike recent language representation models, ... Related Work \n", - "15 16 BERT uses a multi-layer bidirectional Transfor... Model Architecture \n", - "16 17 The dominant sequence transduction models are ... Introduction \n", - "17 18 Most competitive neural sequence transduction ... Background \n", - "18 19 Most neural sequence transduction models have ... Model Architecture \n", - "19 20 In this section we describe the training regim... Training \n", - "20 21 On the WMT 2014 English-to-German translation ... Results \n", - "21 22 In this work, we presented the Transformer, th... Conclusion \n", - "22 23 \\n\\label{sec:intro}\\nIn recent years there has... Introduction \n", - "23 24 \\n\\nOur work builds upon a rich line of recent... Related Work \n", - "24 25 \\n\\label{sec:proposed}\\n\\nThe key idea of \\nam... Proposed Method \n", - "25 26 \\n\\label{sec:ex}\\n\\nWe evaluate the benefits o... Experiments \n", - "26 27 \\n\\nWe introduced a differentiable pooling met... Conclusion \n", - "27 28 \\nThis research has been supported in part by ... Acknowledgement \n", - "28 29 The dominant sequence transduction models are ... Introduction \n", - "29 30 The goal of reducing sequential computation al... Background \n", - "30 31 The Transformer follows this overall architect... Model Architecture \n", - "31 32 We trained on the standard WMT 2014 English-Ge... Training \n", - "32 33 In this work we presented the Transformer, the... Conclusion \n", - "33 34 We introduce a new language representation mod... Introduction \n", - "34 35 Unlike recent language representation models, ... Related Work \n", - "35 36 BERT uses a multi-layer bidirectional Transfor... Model Architecture \n", - "36 37 The dominant sequence transduction models are ... Introduction \n", - "37 38 The goal of reducing sequential computation al... Background \n", - "38 39 The Transformer follows this overall architect... Model Architecture \n", - "39 40 We trained on the standard WMT 2014 English-Ge... Training \n", - "40 41 In this work we presented the Transformer, the... Conclusion \n", - "41 42 We introduce a new language representation mod... Introduction \n", - "42 43 Unlike recent language representation models, ... Related Work \n", - "43 44 BERT uses a multi-layer bidirectional Transfor... Model Architecture \n", - "44 45 The dominant sequence transduction models are ... Introduction \n", - "45 46 Most competitive neural sequence transduction ... Background \n", - "46 47 Most neural sequence transduction models have ... Model Architecture \n", - "47 48 In this section we describe the training regim... Training \n", - "48 49 On the WMT 2014 English-to-German translation ... Results \n", - "49 50 In this work, we presented the Transformer, th... Conclusion \n", - "\n", - " appendix paper_arxiv_id section_in_paper_id \n", - "0 False 1706.03762v7 1.0 \n", - "1 False 1706.03762v7 2.0 \n", - "2 False 1706.03762v7 3.0 \n", - "3 False 1706.03762v7 4.0 \n", - "4 False 1706.03762v7 5.0 \n", - "5 False 1810.04805v2 1.0 \n", - "6 False 1810.04805v2 2.0 \n", - "7 False 1810.04805v2 3.0 \n", - "8 False 1706.03762v7 1.0 \n", - "9 False 1706.03762v7 2.0 \n", - "10 False 1706.03762v7 3.0 \n", - "11 False 1706.03762v7 4.0 \n", - "12 False 1706.03762v7 5.0 \n", - "13 False 1810.04805v2 1.0 \n", - "14 False 1810.04805v2 2.0 \n", - "15 False 1810.04805v2 3.0 \n", - "16 False 1706.03762v7 1.0 \n", - "17 False 1706.03762v7 2.0 \n", - "18 False 1706.03762v7 3.0 \n", - "19 False 1706.03762v7 4.0 \n", - "20 False 1706.03762v7 5.0 \n", - "21 False 1706.03762v7 6.0 \n", - "22 False 1806.08804v4 1.0 \n", - "23 False 1806.08804v4 2.0 \n", - "24 False 1806.08804v4 3.0 \n", - "25 False 1806.08804v4 4.0 \n", - "26 False 1806.08804v4 5.0 \n", - "27 False 1806.08804v4 6.0 \n", - "28 False 1706.03762v7 1.0 \n", - "29 False 1706.03762v7 2.0 \n", - "30 False 1706.03762v7 3.0 \n", - "31 False 1706.03762v7 4.0 \n", - "32 False 1706.03762v7 5.0 \n", - "33 False 1810.04805v2 1.0 \n", - "34 False 1810.04805v2 2.0 \n", - "35 False 1810.04805v2 3.0 \n", - "36 False 1706.03762v7 1.0 \n", - "37 False 1706.03762v7 2.0 \n", - "38 False 1706.03762v7 3.0 \n", - "39 False 1706.03762v7 4.0 \n", - "40 False 1706.03762v7 5.0 \n", - "41 False 1810.04805v2 1.0 \n", - "42 False 1810.04805v2 2.0 \n", - "43 False 1810.04805v2 3.0 \n", - "44 False 1706.03762v7 1.0 \n", - "45 False 1706.03762v7 2.0 \n", - "46 False 1706.03762v7 3.0 \n", - "47 False 1706.03762v7 4.0 \n", - "48 False 1706.03762v7 5.0 \n", - "49 False 1706.03762v7 6.0 \n", - "\n", - "All sections:\n", - " title section_in_paper_id appendix\n", - "0 Introduction 1.0 False\n", - "1 Background 2.0 False\n", - "2 Model Architecture 3.0 False\n", - "3 Training 4.0 False\n", - "4 Conclusion 5.0 False\n", - "5 Introduction 1.0 False\n", - "6 Related Work 2.0 False\n", - "7 Model Architecture 3.0 False\n", - "8 Introduction 1.0 False\n", - "9 Background 2.0 False\n", - "10 Model Architecture 3.0 False\n", - "11 Training 4.0 False\n", - "12 Conclusion 5.0 False\n", - "13 Introduction 1.0 False\n", - "14 Related Work 2.0 False\n", - "15 Model Architecture 3.0 False\n", - "16 Introduction 1.0 False\n", - "17 Background 2.0 False\n", - "18 Model Architecture 3.0 False\n", - "19 Training 4.0 False\n", - "20 Results 5.0 False\n", - "21 Conclusion 6.0 False\n", - "22 Introduction 1.0 False\n", - "23 Related Work 2.0 False\n", - "24 Proposed Method 3.0 False\n", - "25 Experiments 4.0 False\n", - "26 Conclusion 5.0 False\n", - "27 Acknowledgement 6.0 False\n", - "28 Introduction 1.0 False\n", - "29 Background 2.0 False\n", - "30 Model Architecture 3.0 False\n", - "31 Training 4.0 False\n", - "32 Conclusion 5.0 False\n", - "33 Introduction 1.0 False\n", - "34 Related Work 2.0 False\n", - "35 Model Architecture 3.0 False\n", - "36 Introduction 1.0 False\n", - "37 Background 2.0 False\n", - "38 Model Architecture 3.0 False\n", - "39 Training 4.0 False\n", - "40 Conclusion 5.0 False\n", - "41 Introduction 1.0 False\n", - "42 Related Work 2.0 False\n", - "43 Model Architecture 3.0 False\n", - "44 Introduction 1.0 False\n", - "45 Background 2.0 False\n", - "46 Model Architecture 3.0 False\n", - "47 Training 4.0 False\n", - "48 Results 5.0 False\n", - "49 Conclusion 6.0 False\n" - ] - } - ], - "source": [ - "sections_df = research_arcade.get_all_node_features(\"arxiv_sections\")\n", - "print(f\"Total sections: {sections_df}\")\n", - "print(\"\\nAll sections:\")\n", - "print(sections_df[['title', 'section_in_paper_id', 'appendix']])" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-section", - "metadata": {}, - "source": [ - "## 9. ArXiv Paragraphs \n", - "\n", - "### Table Schema\n", - "- `id` (SERIAL PK)\n", - "- `paragraph_id` (INT)\n", - "- `content` (TEXT)\n", - "- `paper_arxiv_id` (VARCHAR FK → papers.arxiv_id)\n", - "- `paper_section` (TEXT)\n", - "- `section_id` (INT)\n", - "- `paragraph_in_paper_id` (INT)" - ] - }, - { - "cell_type": "markdown", - "id": "1b3fd1cc", - "metadata": {}, - "source": [ - "### Insert From API" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "480dabdf", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seed: ['1903.03894v4']\n", - "BFS_que.qsize(): 1\n", - "current paper: 1903.03894v4\n", - "Thread 13079867392 Processing 1903.03894v4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "x 000abstract.tex\n", - "x 010intro.tex\n", - "x 020related.tex\n", - "x 030background.tex\n", - "x 030formulation.tex\n", - "x 030proposed.tex\n", - "x 040experiments.tex\n", - "x 050conclusion.tex\n", - "x 060supplement.tex\n", - "x acmart.bib\n", - "x acmart.cls\n", - "x acmart.dtx\n", - "x acmart.ins\n", - "x ACM-Reference-Format.bbx\n", - "x ACM-Reference-Format.bst\n", - "x ACM-Reference-Format.cbx\n", - "x ACM-Reference-Format.dbx\n", - "x figs/\n", - "x figs/explainer-introduction_v2.pdf\n", - "x figs/explainer-motivation.pdf\n", - "x figs/explainer.pdf\n", - "x figs/feature_importance_v2.pdf\n", - "x figs/fig3-graph-cls-v2.pdf\n", - "x figs/fig3-graph-cls.pdf\n", - "x figs/fig3-node-cls-v3.pdf\n", - "x figs/fig3-node-cls.pdf\n", - "x figs/fig3-v4.pdf\n", - "x figs/fig3-v5.pdf\n", - "x figs/including-node-features.pdf\n", - "x figs/local_subgraph.png\n", - "x figs/motivation-node-features.pdf\n", - "x figs/prototype.png\n", - "x figs/prototype1.png\n", - "x figs/single-instance-explanation-final.pdf\n", - "x figs/single-instance-explanation2.pdf\n", - "x figs/single-instance-explanations.pdf: truncated gzip input\n", - "tar: Error exit delayed from previous errors.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Thread 13079867392 Finished processing 1903.03894v4 (1/999999999) Time elapsed: 0.65s\n", - "'NoneType' object is not subscriptable\n", - "Thread 13079867392 Failed to process 1903.03894v4\n", - "Thread 8614781504 Finished processing 1 papers\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 922.94it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error loading ./download/output/1903.03894v4.json: [Errno 2] No such file or directory: './download/output/1903.03894v4.json'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2/2 [00:00<00:00, 281.94it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error loading ./download/output/1903.03894v4.json: [Errno 2] No such file or directory: './download/output/1903.03894v4.json'\n", - "1806.08804v4\n", - "Key to References: {'fig:assignment_vis': 'figures_3', 'tab:results': 'table_4', 'tab:results2': 'table_5'}\n", - "tab:results\n", - "tab:results2\n", - "Paper count: 1\n", - "Total nodes: 113\n", - "Total edges: 210\n", - "Paper nodes: 1\n", - "Figure nodes: 0\n", - "Table nodes: 2\n", - "Text nodes: 110\n", - "0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "config = {\"arxiv_ids\": [\"1903.03894v4\", \"1806.08804v4\"], \"dest_dir\": \"./download\"}\n", - "research_arcade.construct_table_from_api(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paragraphs to import (all paragraphs already exist)\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paragraphs_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paragraphs to import (all paragraphs already exist)\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paragraphs_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paragraphs\", config)" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-insert", - "metadata": {}, - "source": [ - "### Insert Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "insert-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Inserted paragraph 1 from Introduction\n", - "Inserted paragraph 2 from Introduction\n", - "Inserted paragraph 3 from Introduction\n", - "Inserted paragraph 4 from Introduction\n", - "Inserted paragraph 5 from Introduction\n" - ] - } - ], - "source": [ - "# Insert paragraphs from the Introduction section\n", - "paragraphs = [\n", - " {\n", - " 'paragraph_id': 1,\n", - " 'content': 'Recurrent neural networks, long short-term memory and gated recurrent neural networks in particular, have been firmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 1\n", - " },\n", - " {\n", - " 'paragraph_id': 2,\n", - " 'content': 'Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures. Recurrent models typically factor computation along the symbol positions of the input and output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 2\n", - " },\n", - " {\n", - " 'paragraph_id': 3,\n", - " 'content': 'Aligning the positions to steps in computation time, they generate a sequence of hidden states h_t, as a function of the previous hidden state h_{t-1} and the input for position t. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 3\n", - " },\n", - " {\n", - " 'paragraph_id': 4,\n", - " 'content': 'Attention mechanisms have become an integral part of compelling sequence modeling and transduction models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 4\n", - " },\n", - " {\n", - " 'paragraph_id': 5,\n", - " 'content': 'In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for significantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs.',\n", - " 'paper_arxiv_id': '1706.03762v7',\n", - " 'paper_section': 'Introduction',\n", - " 'section_id': 1,\n", - " 'paragraph_in_paper_id': 5\n", - " }\n", - "]\n", - "\n", - "for paragraph in paragraphs:\n", - " research_arcade.insert_node(\"arxiv_paragraphs\", node_features=paragraph)\n", - " print(f\"Inserted paragraph {paragraph['paragraph_id']} from {paragraph['paper_section']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "arxiv-paragraphs-get-all", - "metadata": {}, - "source": [ - "### Get All Paragraphs" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "id": "get-all-paragraphs", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total paragraphs: 123\n", - "\n", - "First 3 paragraphs:\n", - " paragraph_id paper_section \\\n", - "0 0 Introduction \n", - "1 1 Introduction \n", - "2 2 Introduction \n", - "\n", - " content \n", - "0 \\label{sec:intro}\\nIn recent years there has b... \n", - "1 However, a major limitation of current GNN arc... \n", - "2 Here we propose \\name, a differentiable graph ... \n" - ] - } - ], - "source": [ - "paragraphs_df = research_arcade.get_all_node_features(\"arxiv_paragraphs\")\n", - "print(f\"Total paragraphs: {len(paragraphs_df)}\")\n", - "print(\"\\nFirst 3 paragraphs:\")\n", - "print(paragraphs_df[['paragraph_id', 'paper_section', 'content']].head(3))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 10. Relationships/Edges \n", - "\n", - "This section demonstrates how to create and manage relationships between different entities." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.2 ArXiv Citations (arxiv_citation)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Citation" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Citation created!\n" - ] - } - ], - "source": [ - "citation = {\n", - " 'citing_arxiv_id': '1810.04805v2',\n", - " 'cited_arxiv_id': '1706.03762v7',\n", - " 'bib_title': 'attention is all you need',\n", - " 'bib_key': 'something',\n", - " 'citing_sections': 'citing_sections',\n", - "}\n", - "research_arcade.insert_edge(\"arxiv_citation\", edge_features=citation)\n", - "print(\"Citation created!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new citations to import (all citations already exist)\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_citation_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_citation\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new citations to import (all citations already exist)\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_citation_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_citation\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Citations" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total citations: 8\n", - " id citing_arxiv_id cited_arxiv_id \\\n", - "0 2 1706.03762v7 1409.0473v7 \n", - "1 3 1706.03762v7 1508.04025v5 \n", - "2 5 1810.04805v2 1802.05365v2 \n", - "3 6 2010.11929v2 1706.03762v7 \n", - "4 7 2010.11929v2 1810.04805v2 \n", - "\n", - " bib_title bib_key \\\n", - "0 Neural Machine Translation by Jointly Learning... bahdanau2014neural \n", - "1 Effective Approaches to Attention-based Neural... luong2015effective \n", - "2 Deep contextualized word representations peters2018deep \n", - "3 Attention Is All You Need vaswani2017attention \n", - "4 BERT: Pre-training of Deep Bidirectional Trans... devlin2018bert \n", - "\n", - " citing_sections citing_paragraphs \n", - "0 [\"introduction\", \"related_work\"] [] \n", - "1 [\"related_work\"] [] \n", - "2 [\"related_work\"] [] \n", - "3 [\"introduction\", \"model\"] [] \n", - "4 [\"related_work\"] [] \n" - ] - } - ], - "source": [ - "all_citations = research_arcade.get_all_edge_features(\"arxiv_citation\")\n", - "print(f\"Total citations: {len(all_citations)}\")\n", - "print(all_citations.head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Cited Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers cited:\n", - " id citing_arxiv_id cited_arxiv_id \\\n", - "2 5 1810.04805v2 1802.05365v2 \n", - "7 10 1810.04805v2 1706.03762v7 \n", - "\n", - " bib_title bib_key \\\n", - "2 Deep contextualized word representations peters2018deep \n", - "7 attention is all you need something \n", - "\n", - " citing_sections citing_paragraphs \n", - "2 [\"related_work\"] [] \n", - "7 \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "citing_paper = {'citing_paper_id': '1810.04805v2'}\n", - "cited_papers = research_arcade.get_neighborhood(\"arxiv_citation\", primary_key=citing_paper)\n", - "print(\"Papers cited:\")\n", - "print(cited_papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Citing Papers" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers that cite:\n", - " id citing_arxiv_id cited_arxiv_id bib_title \\\n", - "3 6 2010.11929v2 1706.03762v7 Attention Is All You Need \n", - "7 10 1810.04805v2 1706.03762v7 attention is all you need \n", - "\n", - " bib_key citing_sections citing_paragraphs \n", - "3 vaswani2017attention [\"introduction\", \"model\"] [] \n", - "7 something \"citing_sections\" [] \n" - ] - } - ], - "source": [ - "cited_paper = {'cited_paper_id': '1706.03762v7'}\n", - "citing_papers = research_arcade.get_neighborhood(\"arxiv_citation\", primary_key=cited_paper)\n", - "print(\"Papers that cite:\")\n", - "print(citing_papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Citation" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted citation: 1810.04805v2 -> 1706.03762v7\n", - "Citation deleted!\n" - ] - } - ], - "source": [ - "citation_id = {\n", - " 'citing_paper_id': '1810.04805v2',\n", - " 'cited_paper_id': '1706.03762v7'\n", - "}\n", - "research_arcade.delete_edge_by_id(\"arxiv_citation\", primary_key=citation_id)\n", - "print(\"Citation deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.3 ArXiv Paper-Author (arxiv_paper_author)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Author Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked author ss_ashish_vaswani (position 1)\n", - "Linked author ss_noam_shazeer (position 2)\n", - "Linked author ss_niki_parmar (position 3)\n" - ] - } - ], - "source": [ - "paper_authors = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani', 'author_sequence': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_noam_shazeer', 'author_sequence': 2},\n", - " {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_niki_parmar', 'author_sequence': 3}\n", - "]\n", - "for relation in paper_authors:\n", - " research_arcade.insert_edge(\"arxiv_paper_author\", edge_features=relation)\n", - " print(f\"Linked author {relation['author_id']} (position {relation['author_sequence']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 16 paper-author relationships from ./examples/csv_data/csv_arxiv_paper_author_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_author_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_author\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paper-author relationships to import\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_author_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_author\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Paper-Author Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total relationships: 35\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_noam_shazeer 2\n", - "1 1706.03762v7 ss_niki_parmar 3\n", - "2 1706.03762v7 1234567 1\n", - "3 1706.03762v7 2345678 2\n", - "4 1706.03762v7 3456789 3\n", - "5 1706.03762v7 4567890 4\n", - "6 1706.03762v7 5678901 5\n", - "7 1706.03762v7 6789012 6\n", - "8 1706.03762v7 7890123 7\n", - "9 1706.03762v7 8901234 8\n" - ] - } - ], - "source": [ - "all_relations = research_arcade.get_all_edge_features(\"arxiv_paper_author\")\n", - "print(f\"Total relationships: {len(all_relations)}\")\n", - "print(all_relations.head(10))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Authors for a Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Authors:\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 1234567 1\n", - "1 1706.03762v7 ss_ashish_vaswani 1\n", - "2 1706.03762v7 1234567 1\n", - "3 1706.03762v7 ss_noam_shazeer 2\n", - "4 1706.03762v7 2345678 2\n", - "5 1706.03762v7 2345678 2\n", - "6 1706.03762v7 ss_niki_parmar 3\n", - "7 1706.03762v7 3456789 3\n", - "8 1706.03762v7 3456789 3\n", - "10 1706.03762v7 4567890 4\n", - "9 1706.03762v7 4567890 4\n", - "11 1706.03762v7 5678901 5\n", - "12 1706.03762v7 5678901 5\n", - "13 1706.03762v7 6789012 6\n", - "14 1706.03762v7 6789012 6\n", - "15 1706.03762v7 7890123 7\n", - "16 1706.03762v7 7890123 7\n", - "17 1706.03762v7 8901234 8\n", - "18 1706.03762v7 8901234 8\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "authors = research_arcade.get_neighborhood(\"arxiv_paper_author\", primary_key=paper_id)\n", - "print(\"Authors:\")\n", - "print(authors.sort_values('author_sequence'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Papers by Author" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers by author:\n", - " paper_arxiv_id author_id author_sequence\n", - "0 1706.03762v7 ss_ashish_vaswani 1\n" - ] - } - ], - "source": [ - "author_id = {'author_id': 'ss_ashish_vaswani'}\n", - "papers = research_arcade.get_neighborhood(\"arxiv_paper_author\", primary_key=author_id)\n", - "print(\"Papers by author:\")\n", - "print(papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Paper-Author Link" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Relationship deleted!\n" - ] - } - ], - "source": [ - "relation_id = {'paper_arxiv_id': '1706.03762v7', 'author_id': 'ss_ashish_vaswani'}\n", - "research_arcade.delete_edge_by_id(\"arxiv_paper_author\", primary_key=relation_id)\n", - "print(\"Relationship deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.4 ArXiv Paper-Category (arxiv_paper_category)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Category Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked 1\n", - "Linked 1\n", - "Linked 2\n" - ] - } - ], - "source": [ - "paper_categories = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '1'},\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '1'},\n", - " {'paper_arxiv_id': '1706.03762v7', 'category_id': '2'}\n", - "]\n", - "for relation in paper_categories:\n", - " research_arcade.insert_edge(\"arxiv_paper_category\", edge_features=relation)\n", - " print(f\"Linked {relation['category_id']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 1 paper-category relationships from ./examples/csv_data/csv_arxiv_paper_category_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_category_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_category\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paper-category relationships to import\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_category_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_category\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get All Paper-Category Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total relationships: 17\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 1\n", - "2 1706.03762v7 2\n", - "3 1706.03762v7 cs.CL\n", - "4 1706.03762v7 cs.LG\n" - ] - } - ], - "source": [ - "all_relations = research_arcade.get_all_edge_features(\"arxiv_paper_category\")\n", - "print(f\"Total relationships: {len(all_relations)}\")\n", - "print(all_relations.head())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Categories for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Categories:\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 1\n", - "2 1706.03762v7 2\n", - "3 1706.03762v7 cs.CL\n", - "4 1706.03762v7 cs.LG\n", - "5 1706.03762v7 cs.AI\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "categories = research_arcade.get_neighborhood(\"arxiv_paper_category\", primary_key=paper_id)\n", - "print(\"Categories:\")\n", - "print(categories)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Papers in Category" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Papers in category:\n", - " paper_arxiv_id category_id\n", - "0 1706.03762v7 cs.LG\n", - "1 1810.04805v2 cs.LG\n", - "2 1409.0473v7 cs.LG\n", - "3 1512.03385v1 cs.LG\n", - "4 2010.11929v2 cs.LG\n" - ] - } - ], - "source": [ - "category_id = {'category_id': 'cs.LG'}\n", - "papers = research_arcade.get_neighborhood(\"arxiv_paper_category\", primary_key=category_id)\n", - "print(\"Papers in category:\")\n", - "print(papers)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Delete Paper-Category Link" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Relationship deleted!\n" - ] - } - ], - "source": [ - "relation_id = {'paper_arxiv_id': '1706.03762v7', 'category_id': 'cs.AI'}\n", - "research_arcade.delete_edge_by_id(\"arxiv_paper_category\", primary_key=relation_id)\n", - "print(\"Relationship deleted!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.5 ArXiv Paper-Figure (arxiv_paper_figure)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Figure Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked figure 1)\n", - "Linked figure 2)\n" - ] - } - ], - "source": [ - "paper_figures = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'figure_id': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'figure_id': 2}\n", - "]\n", - "for relation in paper_figures:\n", - " research_arcade.insert_edge(\"arxiv_paper_figure\", edge_features=relation)\n", - " print(f\"Linked figure {relation['figure_id']})\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paper-figure relationships to import\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_figure_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_figure\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paper-figure relationships to import\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_figure_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_figure\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Figures for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Figures:\n", - " paper_arxiv_id figure_id\n", - "0 1706.03762v7 1\n", - "1 1706.03762v7 2\n", - "2 1706.03762v7 3\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "figures = research_arcade.get_neighborhood(\"arxiv_paper_figure\", primary_key=paper_id)\n", - "print(\"Figures:\")\n", - "print(figures)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.6 ArXiv Paper-Table (arxiv_paper_table)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paper-Table Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Linked table 1\n", - "Linked table 2\n" - ] - } - ], - "source": [ - "paper_tables = [\n", - " {'paper_arxiv_id': '1706.03762v7', 'table_id': 1},\n", - " {'paper_arxiv_id': '1706.03762v7', 'table_id': 2}\n", - "]\n", - "for relation in paper_tables:\n", - " research_arcade.insert_edge(\"arxiv_paper_table\", edge_features=relation)\n", - " print(f\"Linked table {relation['table_id']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paper-table relationships to import\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paper_table_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paper_table\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No new paper-table relationships to import\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paper_table_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paper_table\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get Tables for Paper" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tables:\n" - ] - } - ], - "source": [ - "paper_id = {'paper_arxiv_id': '1706.03762v7'}\n", - "tables = research_arcade.get_neighborhood(\"arxiv_paper_table\", primary_key=paper_id)\n", - "print(\"Tables:\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 10.7 ArXiv Paragraph-Reference (arxiv_paragraph_reference)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Insert Paragraph-Reference Relationships" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [], - "source": [ - "paragraph_references = [\n", - " {'paragraph_id': 1, 'paper_section': 'established approaches', 'paper_arxiv_id': '1706.03762v7', 'reference_label': \"{something}\", 'reference_type': 'figure'}\n", - "]\n", - "\n", - "for relation in paragraph_references:\n", - " research_arcade.insert_edge(\"arxiv_paragraph_reference\", edge_features=relation)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from CSV" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successfully imported 10 paragraph-reference relationships from ./examples/csv_data/csv_arxiv_paragraph_reference_example.csv\n" - ] - } - ], - "source": [ - "config = {\"csv_file\": \"./examples/csv_data/csv_arxiv_paragraph_reference_example.csv\"}\n", - "research_arcade.construct_table_from_csv(\"arxiv_paragraph_reference\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Construct Table from JSON" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error: JSON file ./examples/json_data/json_arxiv_paragraph_reference_example.json does not exist.\n" - ] - } - ], - "source": [ - "config = {\"json_file\": \"./examples/json_data/json_arxiv_paragraph_reference_example.json\"}\n", - "research_arcade.construct_table_from_json(\"arxiv_paragraph_reference\", config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Get References in Paragraph" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "References:\n", - " id paragraph_id paper_section paper_arxiv_id \\\n", - "0 1 1 established approaches 1706.03762v7 \n", - "1 3 1 introduction 1706.03762v7 \n", - "2 6 1 background 1706.03762v7 \n", - "3 8 1 introduction 1810.04805v2 \n", - "4 11 1 model 1810.04805v2 \n", - "5 12 1 established approaches 1706.03762v7 \n", - "6 14 1 introduction 1706.03762v7 \n", - "7 17 1 background 1706.03762v7 \n", - "8 19 1 introduction 1810.04805v2 \n", - "9 22 1 model 1810.04805v2 \n", - "\n", - " reference_label reference_type \n", - "0 {something} figure \n", - "1 bahdanau2014neural citation \n", - "2 fig:attention figure \n", - "3 fig:bert_arch figure \n", - "4 peters2018deep citation \n", - "5 {something} figure \n", - "6 bahdanau2014neural citation \n", - "7 fig:attention figure \n", - "8 fig:bert_arch figure \n", - "9 peters2018deep citation \n" - ] - } - ], - "source": [ - "paragraph_id = {'paragraph_id': 1}\n", - "references = research_arcade.get_neighborhood(\"arxiv_paragraph_reference\", primary_key=paragraph_id)\n", - "print(\"References:\")\n", - "print(references)" - ] - }, - { - "cell_type": "markdown", - "id": "conclusion", - "metadata": {}, - "source": [ - "## Conclusion\n", - "\n", - "This tutorial has covered:\n", - "\n", - "1. Setting up the ResearchArcade database connection\n", - "2. Working with OpenReview data\n", - "3. CRUD operations for all ArXiv entity types:\n", - " - Papers\n", - " - Authors\n", - " - Categories\n", - " - Figures\n", - " - Tables\n", - " - Sections\n", - " - Paragraphs\n", - "4. Creating relationships between entities:\n", - " - Authorship\n", - " - Citations\n", - " - Paper-Category links\n", - " - Paper-Figure/Table links\n", - " - Paragraph-level references\n", - "\n", - "For more information, refer to the ResearchArcade documentation." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "research_arcade", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/test/arxiv_csvbased_test.py b/test/arxiv_csvbased_test.py deleted file mode 100644 index 31af89a..0000000 --- a/test/arxiv_csvbased_test.py +++ /dev/null @@ -1,626 +0,0 @@ -import sys -import os - -import unittest -from unittest.mock import Mock, patch - -from dotenv import load_dotenv - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -from research_arcade.research_arcade import ResearchArcade -load_dotenv() - - -class TestArxivArcadeNodeCRUD(unittest.TestCase): - """Test suite for Node CRUD operations""" - - def setUp(self): - """Set up test fixtures before each test method""" - csv_dir = os.getenv('CSV_DATASET_FOLDER_PATH') - # host = os.getenv('HOST') - # port = os.getenv('PORT') - # dbname = os.getenv('DBNAME') - # user = os.getenv('USER') - paper_dir_path = os.getenv('PAPER_FOLDER_PATH') - self.config = {'csv_dir': csv_dir} - self.db_type = "csv" - - # Mock all CSV database classes - with patch('research_arcade.csv_database.csv_arxiv_authors.CSVArxivAuthors'), \ - patch('research_arcade.csv_database.csv_arxiv_categories.CSVArxivCategory'), \ - patch('research_arcade.csv_database.csv_arxiv_citations.CSVArxivCitation'), \ - patch('research_arcade.csv_database.csv_arxiv_figures.CSVArxivFigure'), \ - patch('research_arcade.csv_database.csv_arxiv_tables.CSVArxivTable'), \ - patch('research_arcade.csv_database.csv_arxiv_papers.CSVArxivPapers'), \ - patch('research_arcade.csv_database.csv_arxiv_paragraphs.CSVArxivParagraphs'), \ - patch('research_arcade.csv_database.csv_arxiv_sections.CSVArxivSections'), \ - patch('research_arcade.csv_database.csv_arxiv_paper_authors.CSVArxivPaperAuthor'), \ - patch('research_arcade.csv_database.csv_arxiv_paper_categories.CSVArxivPaperCategory'), \ - patch('research_arcade.csv_database.csv_arxiv_paper_figures.CSVArxivPaperFigure'), \ - patch('research_arcade.csv_database.csv_arxiv_paper_tables.CSVArxivPaperTable'), \ - patch('research_arcade.csv_database.csv_arxiv_paragraph_references.CSVArxivParagraphReference'): - self.arcade = ResearchArcade(db_type=self.db_type, config=self.config) - - # ------------------------- - # insert_node tests - # ------------------------- - def test_insert_node_authors(self): - """Test inserting an author node""" - node_features = {'author_id': '1', 'name': 'John Doe'} - self.arcade.arxiv_authors.insert_author = Mock(return_value='author_1') - - result = self.arcade.insert_node('arxiv_authors', node_features) - - self.arcade.arxiv_authors.insert_author.assert_called_once_with(**node_features) - self.assertEqual(result, 'author_1') - - def test_insert_node_categories(self): - """Test inserting a category node""" - node_features = {'category_id': 'cs.AI', 'description': 'Artificial Intelligence'} - self.arcade.arxiv_categories.insert_category = Mock(return_value='cat_1') - - result = self.arcade.insert_node('arxiv_categories', node_features) - - self.arcade.arxiv_categories.insert_category.assert_called_once_with(**node_features) - self.assertEqual(result, 'cat_1') - - def test_insert_node_figures(self): - """Test inserting a figure node""" - node_features = {'figure_id': 'fig1', 'caption': 'Test figure'} - self.arcade.arxiv_figures.insert_figure = Mock(return_value='fig_1') - - result = self.arcade.insert_node('arxiv_figures', node_features) - - self.arcade.arxiv_figures.insert_figure.assert_called_once_with(**node_features) - self.assertEqual(result, 'fig_1') - - def test_insert_node_tables(self): - """Test inserting a table node""" - node_features = {'table_id': 'tab1', 'caption': 'Test table'} - self.arcade.arxiv_tables.insert_table = Mock(return_value='tab_1') - - result = self.arcade.insert_node('arxiv_tables', node_features) - - self.arcade.arxiv_tables.insert_table.assert_called_once_with(**node_features) - self.assertEqual(result, 'tab_1') - - def test_insert_node_papers(self): - """Test inserting a paper node""" - node_features = {'paper_id': 'arxiv123', 'title': 'Test Paper'} - self.arcade.arxiv_papers.insert_paper = Mock(return_value='paper_1') - - result = self.arcade.insert_node('arxiv_papers', node_features) - - self.arcade.arxiv_papers.insert_paper.assert_called_once_with(**node_features) - self.assertEqual(result, 'paper_1') - - def test_insert_node_paragraphs(self): - """Test inserting a paragraph node""" - node_features = {'paragraph_id': 'para1', 'text': 'Sample text'} - self.arcade.arxiv_paragraphs.insert_paragraph = Mock(return_value='para_1') - - result = self.arcade.insert_node('arxiv_paragraphs', node_features) - - self.arcade.arxiv_paragraphs.insert_paragraph.assert_called_once_with(**node_features) - self.assertEqual(result, 'para_1') - - def test_insert_node_sections(self): - """Test inserting a section node""" - node_features = {'section_id': 'sec1', 'title': 'Introduction'} - self.arcade.arxiv_sections.insert_section = Mock(return_value='sec_1') - - result = self.arcade.insert_node('arxiv_sections', node_features) - - self.arcade.arxiv_sections.insert_section.assert_called_once_with(**node_features) - self.assertEqual(result, 'sec_1') - - def test_insert_node_invalid_table(self): - """Test inserting node with invalid table name""" - node_features = {'id': '1'} - result = self.arcade.insert_node('invalid_table', node_features) - - self.assertIsNone(result) - - # ------------------------- - # delete_node_by_id tests - # ------------------------- - def test_delete_node_authors(self): - """Test deleting an author node""" - primary_key = {'author_id': '1'} - self.arcade.arxiv_authors.delete_author_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_authors', primary_key) - - self.arcade.arxiv_authors.delete_author_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_categories(self): - """Test deleting a category node""" - primary_key = {'category_id': 'cs.AI'} - self.arcade.arxiv_categories.delete_category_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_categories', primary_key) - - self.arcade.arxiv_categories.delete_category_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_figures(self): - """Test deleting a figure node""" - primary_key = {'figure_id': 'fig1'} - self.arcade.arxiv_figures.delete_figure_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_figures', primary_key) - - self.arcade.arxiv_figures.delete_figure_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_tables(self): - """Test deleting a table node""" - primary_key = {'table_id': 'tab1'} - self.arcade.arxiv_tables.delete_table_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_tables', primary_key) - - self.arcade.arxiv_tables.delete_table_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_papers(self): - """Test deleting a paper node""" - primary_key = {'paper_id': 'arxiv123'} - self.arcade.arxiv_papers.delete_paper_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_papers', primary_key) - - self.arcade.arxiv_papers.delete_paper_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_paragraphs(self): - """Test deleting a paragraph node""" - primary_key = {'paragraph_id': 'para1'} - self.arcade.arxiv_paragraphs.delete_paragraph_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_paragraphs', primary_key) - - self.arcade.arxiv_paragraphs.delete_paragraph_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_sections(self): - """Test deleting a section node""" - primary_key = {'section_id': 'sec1'} - self.arcade.arxiv_sections.delete_section_by_id = Mock(return_value=True) - - result = self.arcade.delete_node_by_id('arxiv_sections', primary_key) - - self.arcade.arxiv_sections.delete_section_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_node_invalid_table(self): - """Test deleting node with invalid table name""" - primary_key = {'id': '1'} - result = self.arcade.delete_node_by_id('invalid_table', primary_key) - - self.assertIsNone(result) - - # ------------------------- - # update_node tests - # ------------------------- - def test_update_node_authors(self): - """Test updating an author node""" - node_features = {'author_id': '1', 'name': 'Jane Doe'} - self.arcade.arxiv_authors.update_author = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_authors', node_features) - - self.arcade.arxiv_authors.update_author.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_categories(self): - """Test updating a category node""" - node_features = {'category_id': 'cs.AI', 'description': 'Updated AI'} - self.arcade.arxiv_categories.update_category = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_categories', node_features) - - self.arcade.arxiv_categories.update_category.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_figures(self): - """Test updating a figure node""" - node_features = {'figure_id': 'fig1', 'caption': 'Updated caption'} - self.arcade.arxiv_figures.update_figure = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_figures', node_features) - - self.arcade.arxiv_figures.update_figure.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_tables(self): - """Test updating a table node""" - node_features = {'table_id': 'tab1', 'caption': 'Updated table'} - self.arcade.arxiv_tables.update_table = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_tables', node_features) - - self.arcade.arxiv_tables.update_table.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_papers(self): - """Test updating a paper node""" - node_features = {'paper_id': 'arxiv123', 'title': 'Updated Title'} - self.arcade.arxiv_papers.update_paper = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_papers', node_features) - - self.arcade.arxiv_papers.update_paper.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_paragraphs(self): - """Test updating a paragraph node""" - node_features = {'paragraph_id': 'para1', 'text': 'Updated text'} - self.arcade.arxiv_paragraphs.update_paragraph = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_paragraphs', node_features) - - self.arcade.arxiv_paragraphs.update_paragraph.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_sections(self): - """Test updating a section node""" - node_features = {'section_id': 'sec1', 'title': 'Updated Section'} - self.arcade.arxiv_sections.update_section = Mock(return_value=True) - - result = self.arcade.update_node('arxiv_sections', node_features) - - self.arcade.arxiv_sections.update_section.assert_called_once_with(**node_features) - self.assertTrue(result) - - def test_update_node_invalid_table(self): - """Test updating node with invalid table name""" - node_features = {'id': '1'} - result = self.arcade.update_node('invalid_table', node_features) - - self.assertIsNone(result) - - # ------------------------- - # get_node_features_by_id tests - # ------------------------- - def test_get_node_features_authors(self): - """Test getting author features by ID""" - primary_key = {'author_id': '1'} - expected = {'author_id': '1', 'name': 'John Doe'} - self.arcade.arxiv_authors.get_author_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_authors', primary_key) - - self.arcade.arxiv_authors.get_author_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_categories(self): - """Test getting category features by ID""" - primary_key = {'category_id': 'cs.AI'} - expected = {'category_id': 'cs.AI', 'description': 'AI'} - self.arcade.arxiv_categories.get_category_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_categories', primary_key) - - self.arcade.arxiv_categories.get_category_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_figures(self): - """Test getting figure features by ID""" - primary_key = {'figure_id': 'fig1'} - expected = {'figure_id': 'fig1', 'caption': 'Test'} - self.arcade.arxiv_figures.get_figure_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_figures', primary_key) - - self.arcade.arxiv_figures.get_figure_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_tables(self): - """Test getting table features by ID""" - primary_key = {'table_id': 'tab1'} - expected = {'table_id': 'tab1', 'caption': 'Test'} - self.arcade.arxiv_tables.get_table_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_tables', primary_key) - - self.arcade.arxiv_tables.get_table_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_papers(self): - """Test getting paper features by ID""" - primary_key = {'paper_id': 'arxiv123'} - expected = {'paper_id': 'arxiv123', 'title': 'Test'} - self.arcade.arxiv_papers.get_paper_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_papers', primary_key) - - self.arcade.arxiv_papers.get_paper_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_paragraphs(self): - """Test getting paragraph features by ID""" - primary_key = {'paragraph_id': 'para1'} - expected = {'paragraph_id': 'para1', 'text': 'Test'} - self.arcade.arxiv_paragraphs.get_paragraph_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_paragraphs', primary_key) - - self.arcade.arxiv_paragraphs.get_paragraph_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_sections(self): - """Test getting section features by ID""" - primary_key = {'section_id': 'sec1'} - expected = {'section_id': 'sec1', 'title': 'Intro'} - self.arcade.arxiv_sections.get_section_by_id = Mock(return_value=expected) - - result = self.arcade.get_node_features_by_id('arxiv_sections', primary_key) - - self.arcade.arxiv_sections.get_section_by_id.assert_called_once_with(**primary_key) - self.assertEqual(result, expected) - - def test_get_node_features_invalid_table(self): - """Test getting features with invalid table name""" - primary_key = {'id': '1'} - result = self.arcade.get_node_features_by_id('invalid_table', primary_key) - - self.assertIsNone(result) - - # ------------------------- - # get_all_node_features tests - # ------------------------- - def test_get_all_node_features_authors(self): - """Test getting all author features""" - expected = [{'author_id': '1'}, {'author_id': '2'}] - self.arcade.arxiv_authors.get_all_authors = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_authors') - - self.arcade.arxiv_authors.get_all_authors.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_categories(self): - """Test getting all category features""" - expected = [{'category_id': 'cs.AI'}, {'category_id': 'cs.ML'}] - self.arcade.arxiv_categories.get_all_categories = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_categories') - - self.arcade.arxiv_categories.get_all_categories.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_figures(self): - """Test getting all figure features""" - expected = [{'figure_id': 'fig1'}, {'figure_id': 'fig2'}] - self.arcade.arxiv_figures.get_all_figures = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_figures') - - self.arcade.arxiv_figures.get_all_figures.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_tables(self): - """Test getting all table features""" - expected = [{'table_id': 'tab1'}, {'table_id': 'tab2'}] - self.arcade.arxiv_tables.get_all_tables = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_tables') - - self.arcade.arxiv_tables.get_all_tables.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_papers(self): - """Test getting all paper features""" - expected = [{'paper_id': 'p1'}, {'paper_id': 'p2'}] - self.arcade.arxiv_papers.get_all_papers = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_papers') - - self.arcade.arxiv_papers.get_all_papers.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_paragraphs(self): - """Test getting all paragraph features""" - expected = [{'paragraph_id': 'para1'}, {'paragraph_id': 'para2'}] - self.arcade.arxiv_paragraphs.get_all_paragraphs = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_paragraphs') - - self.arcade.arxiv_paragraphs.get_all_paragraphs.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_sections(self): - """Test getting all section features""" - expected = [{'section_id': 'sec1'}, {'section_id': 'sec2'}] - self.arcade.arxiv_sections.get_all_sections = Mock(return_value=expected) - - result = self.arcade.get_all_node_features('arxiv_sections') - - self.arcade.arxiv_sections.get_all_sections.assert_called_once_with(is_all_features=True) - self.assertEqual(result, expected) - - def test_get_all_node_features_invalid_table(self): - """Test getting all features with invalid table name""" - result = self.arcade.get_all_node_features('invalid_table') - - self.assertIsNone(result) - - -class TestArxivArcadeEdgeCRUD(unittest.TestCase): - """Test suite for Edge CRUD operations""" - - def setUp(self): - """Set up test fixtures before each test method""" - self.config = {'test_config': 'value'} - - with patch('arxiv_arcade.CSVArxivAuthors'), \ - patch('arxiv_arcade.CSVArxivCategory'), \ - patch('arxiv_arcade.CSVArxivFigure'), \ - patch('arxiv_arcade.CSVArxivTable'), \ - patch('arxiv_arcade.CSVArxivPapers'), \ - patch('arxiv_arcade.CSVArxivParagraphs'), \ - patch('arxiv_arcade.CSVArxivSections'), \ - patch('arxiv_arcade.CSVArxivCitation'), \ - patch('arxiv_arcade.CSVArxivPaperAuthor'), \ - patch('arxiv_arcade.CSVArxivPaperCategory'), \ - patch('arxiv_arcade.CSVArxivPaperFigure'), \ - patch('arxiv_arcade.CSVArxivPaperTable'), \ - patch('arxiv_arcade.CSVArxivParagraphReference'): - self.arcade = ResearchArcade(self.config) - - # ------------------------- - # insert_edge tests - # ------------------------- - def test_insert_edge_citation(self): - """Test inserting a citation edge""" - edge_features = {'citing_paper_id': 'p1', 'cited_paper_id': 'p2'} - self.arcade.arxiv_citation.insert_citation = Mock(return_value=True) - - result = self.arcade.insert_edge('arxiv_citation', edge_features) - - self.arcade.arxiv_citation.insert_citation.assert_called_once_with(**edge_features) - self.assertTrue(result) - - def test_insert_edge_paper_author(self): - """Test inserting a paper-author edge""" - edge_features = {'paper_id': 'p1', 'author_id': 'a1'} - self.arcade.arxiv_paper_author.insert_paper_author = Mock(return_value=True) - - result = self.arcade.insert_edge('arxiv_paper_author', edge_features) - - self.arcade.arxiv_paper_author.insert_paper_author.assert_called_once_with(**edge_features) - self.assertTrue(result) - - def test_insert_edge_paper_category(self): - """Test inserting a paper-category edge""" - edge_features = {'paper_id': 'p1', 'category_id': 'cs.AI'} - self.arcade.arxiv_paper_category.insert_paper_category = Mock(return_value=True) - - result = self.arcade.insert_edge('arxiv_paper_category', edge_features) - - self.arcade.arxiv_paper_category.insert_paper_category.assert_called_once_with(**edge_features) - self.assertTrue(result) - - def test_insert_edge_paper_figure(self): - """Test inserting a paper-figure edge""" - edge_features = {'paper_id': 'p1', 'figure_id': 'fig1'} - self.arcade.arxiv_paper_figure.insert_paper_figure = Mock(return_value=True) - - result = self.arcade.insert_edge('arxiv_paper_figure', edge_features) - - self.arcade.arxiv_paper_figure.insert_paper_figure.assert_called_once_with(**edge_features) - self.assertTrue(result) - - def test_insert_edge_paper_table(self): - """Test inserting a paper-table edge""" - edge_features = {'paper_id': 'p1', 'table_id': 'tab1'} - self.arcade.arxiv_paper_table.insert_paper_table = Mock(return_value=True) - - result = self.arcade.insert_edge('arxiv_paper_table', edge_features) - - self.arcade.arxiv_paper_table.insert_paper_table.assert_called_once_with(**edge_features) - self.assertTrue(result) - - def test_insert_edge_paragraph_reference(self): - """Test inserting a paragraph-reference edge""" - edge_features = {'paragraph_id': 'para1', 'reference_id': 'ref1'} - self.arcade.arxiv_paragraph_reference.insert_paragraph_reference = Mock(return_value=True) - - result = self.arcade.insert_edge('arxiv_paragraph_reference', edge_features) - - self.arcade.arxiv_paragraph_reference.insert_paragraph_reference.assert_called_once_with(**edge_features) - self.assertTrue(result) - - def test_insert_edge_invalid_table(self): - """Test inserting edge with invalid table name""" - edge_features = {'id': '1'} - result = self.arcade.insert_edge('invalid_table', edge_features) - - self.assertIsNone(result) - - # ------------------------- - # delete_edge_by_id tests - citation - # ------------------------- - def test_delete_edge_citation_both_keys(self): - """Test deleting citation with both citing and cited paper IDs""" - primary_key = {'citing_paper_id': 'p1', 'cited_paper_id': 'p2'} - self.arcade.arxiv_citation.delete_citation_by_id = Mock(return_value=True) - - result = self.arcade.delete_edge_by_id('arxiv_citation', primary_key) - - self.arcade.arxiv_citation.delete_citation_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_edge_citation_citing_only(self): - """Test deleting citation with only citing paper ID""" - primary_key = {'citing_paper_id': 'p1'} - self.arcade.arxiv_citation.delete_citation_by_citing_id = Mock(return_value=True) - - result = self.arcade.delete_edge_by_id('arxiv_citation', primary_key) - - self.arcade.arxiv_citation.delete_citation_by_citing_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_edge_citation_cited_only(self): - """Test deleting citation with only cited paper ID""" - primary_key = {'cited_paper_id': 'p2'} - self.arcade.arxiv_citation.delete_citation_by_cited_id = Mock(return_value=True) - - result = self.arcade.delete_edge_by_id('arxiv_citation', primary_key) - - self.arcade.arxiv_citation.delete_citation_by_cited_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_edge_citation_no_keys(self): - """Test deleting citation with no valid keys""" - primary_key = {'invalid_key': 'value'} - result = self.arcade.delete_edge_by_id('arxiv_citation', primary_key) - - self.assertIsNone(result) - - # ------------------------- - # delete_edge_by_id tests - paper_author - # ------------------------- - def test_delete_edge_paper_author_both_keys(self): - primary_key = {'paper_id': 'p1', 'author_id': 'a1'} - self.arcade.arxiv_paper_author.delete_paper_author_by_id = Mock(return_value=True) - - result = self.arcade.delete_edge_by_id('arxiv_paper_author', primary_key) - - self.arcade.arxiv_paper_author.delete_paper_author_by_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_edge_paper_author_paper_only(self): - primary_key = {'paper_id': 'p1'} - self.arcade.arxiv_paper_author.delete_paper_author_by_paper_id = Mock(return_value=True) - - result = self.arcade.delete_edge_by_id('arxiv_paper_author', primary_key) - - self.arcade.arxiv_paper_author.delete_paper_author_by_paper_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_edge_paper_author_author_only(self): - """Test deleting paper-author with only author ID""" - primary_key = {'author_id': 'a1'} - self.arcade.arxiv_paper_author.delete_paper_author_by_author_id = Mock(return_value=True) - - result = self.arcade.delete_edge_by_id('arxiv_paper_author', primary_key) - - self.arcade.arxiv_paper_author.delete_paper_author_by_author_id.assert_called_once_with(**primary_key) - self.assertTrue(result) - - def test_delete_edge_paper_author_invalid_keys(self): - """Test deleting paper-author with invalid keys""" - primary_key = {'invalid_key': 'val'} - result = self.arcade.delete_edge_by_id('arxiv_paper_author', primary_key) - - self.assertIsNone(result) - - -taadc = TestArxivArcadeNodeCRUD() - -taadc.setUp() -taadc.test_insert_node_authors() \ No newline at end of file diff --git a/test/openreview_csvbased_test.py b/test/openreview_csvbased_test.py deleted file mode 100644 index c36b1f6..0000000 --- a/test/openreview_csvbased_test.py +++ /dev/null @@ -1,579 +0,0 @@ -import sys -from pathlib import Path -from tqdm import tqdm -project_root = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(project_root)) -from research_arcade import ResearchArcade - -db_type = "csv" -config = { - "csv_dir": "/data/jingjunx/my_research_arcade_data/" -} - -research_arcade = ResearchArcade(db_type=db_type, config=config) - -########## openreview_authors ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_authors", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_author_example.csv"} -# research_arcade.construct_table_from_csv("openreview_authors", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_author_example.json"} -# research_arcade.construct_table_from_json("openreview_authors", config) - -# get_all_node_features -# openreview_authors_df = research_arcade.get_all_node_features("openreview_authors") -# print(len(openreview_authors_df)) - -# get_node_features_by_id -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.get_node_features_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) - -# delete_node -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.delete_node_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) - -# insert_node -# new_author = {'venue': 'ICLR.cc/2025/Conference', -# 'author_openreview_id': '~ishmam_zabir1', -# 'author_full_name': 'ishmam zabir', -# 'email': '****@microsoft.com', -# 'affiliation': 'Microsoft', -# 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', -# 'dblp': ''} -# research_arcade.insert_node("openreview_authors", node_features=new_author) - -# update_node -# new_author = {'venue': 'ICLR.cc/2025/Conference', -# 'author_openreview_id': '~ishmam_zabir1', -# 'author_full_name': 'ishmam zabir', -# 'email': '****@microsoft.com', -# 'affiliation': 'Microsoft', -# 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', -# 'dblp': ''} -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.get_node_features_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_authors", node_features=new_author) -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.get_node_features_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) - -########## openreview_papers ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_papers", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_paper_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_paper_example.json"} -# research_arcade.construct_table_from_json("openreview_papers", config) - -# get_all_node_features -# openreview_papers_df = research_arcade.get_all_node_features("openreview_papers") -# print(len(openreview_papers_df)) - -# get_node_features_by_id -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.get_node_features_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) - -# delete_node -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.delete_node_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) - -# insert_node -# paper_features = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': 'zGej22CBnS', -# 'title': 'Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles', -# 'abstract': "Tokenization is associated with many poorly understood shortcomings in language models (LMs), yet remains an important component for long sequence scaling purposes. This work studies how tokenization impacts model performance by analyzing and comparing the stochastic behavior of tokenized models with their byte-level, or token-free, counterparts. We discover that, even when the two models are statistically equivalent, their predictive distributions over the next byte can be substantially different, a phenomenon we term as ``tokenization bias''. To fully characterize this phenomenon, we introduce the Byte-Token Representation Lemma, a framework that establishes a mapping between the learned token distribution and its equivalent byte-level distribution. From this result, we develop a next-byte sampling algorithm that eliminates tokenization bias without requiring further training or optimization. In other words, this enables zero-shot conversion of tokenized LMs into statistically equivalent token-free ones. We demonstrate its broad applicability with two use cases: fill-in-the-middle (FIM) tasks and model ensembles. In FIM tasks where input prompts may terminate mid-token, leading to out-of-distribution tokenization, our method mitigates performance degradation and achieves 18\\% improvement in FIM coding benchmarks, while consistently outperforming the standard token healing fix. For model ensembles where each model employs a distinct vocabulary, our approach enables seamless integration, resulting in improved performance up to 3.7\\% over individual models across various standard baselines in reasoning, knowledge, and coding. Code is available at:https: //github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs.", -# 'paper_decision': 'ICLR 2025 Poster', -# 'paper_pdf_link': '/pdf/cdd2212a20c4034029874cba11a05e081bfdb83e.pdf'} -# research_arcade.insert_node("openreview_papers", node_features=paper_features) - -# update_node -# new_paper_features = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': 'zGej22CBnS', -# 'title': 'Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles', -# 'abstract': "Tokenization is associated with many poorly understood shortcomings in language models (LMs), yet remains an important component for long sequence scaling purposes. This work studies how tokenization impacts model performance by analyzing and comparing the stochastic behavior of tokenized models with their byte-level, or token-free, counterparts. We discover that, even when the two models are statistically equivalent, their predictive distributions over the next byte can be substantially different, a phenomenon we term as ``tokenization bias''. To fully characterize this phenomenon, we introduce the Byte-Token Representation Lemma, a framework that establishes a mapping between the learned token distribution and its equivalent byte-level distribution. From this result, we develop a next-byte sampling algorithm that eliminates tokenization bias without requiring further training or optimization. In other words, this enables zero-shot conversion of tokenized LMs into statistically equivalent token-free ones. We demonstrate its broad applicability with two use cases: fill-in-the-middle (FIM) tasks and model ensembles. In FIM tasks where input prompts may terminate mid-token, leading to out-of-distribution tokenization, our method mitigates performance degradation and achieves 18\\% improvement in FIM coding benchmarks, while consistently outperforming the standard token healing fix. For model ensembles where each model employs a distinct vocabulary, our approach enables seamless integration, resulting in improved performance up to 3.7\\% over individual models across various standard baselines in reasoning, knowledge, and coding. Code is available at:https: //github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs.", -# 'paper_decision': 'ICLR 2025 Poster', -# 'paper_pdf_link': '/pdf/cdd2212a20c4034029874cba11a05e081bfdb83e.pdf'} -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.get_node_features_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_papers", node_features=new_paper_features) -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.get_node_features_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) - -########## openreview_reviews ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2013/conference"} -# research_arcade.construct_table_from_api("openreview_reviews", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_review_example.csv"} -# research_arcade.construct_table_from_csv("openreview_reviews", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_review_example.json"} -# research_arcade.construct_table_from_json("openreview_reviews", config) - -# get_all_node_features -# openreview_reviews_df = research_arcade.get_all_node_features("openreview_reviews") -# print(len(openreview_reviews_df)) - -# get_node_features_by_id -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.get_node_features_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) - -# delete_node -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.delete_node_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) - -# insert_node -# review_features = {'venue': 'ICLR.cc/2025/Conference', -# 'review_openreview_id': 'DHwZxFryth', -# 'replyto_openreview_id': 'Yqbllggrmw', -# 'writer': 'Authors', -# 'title': 'Response by Authors', -# 'content': {'Title': 'Response to Reviewer 7i95 (1/2)', 'Comment': '> The method does not improve much in the AlpacaEval 2.0 Score. The author should give a detailed explanation. And why not use metrics like length-controlled win rate?**Response:** Thank you for your careful observation and question. We would like to clarify that we are already using the length-controlled (LC) AlpacaEval 2.0 win-rate metric in our evaluations. We will make this clearer in the table header of Table 3.Regarding the fact that the AlpacaEval 2.0 scores on LLama-3 (8B) do not improve compared to the baselines, we believe this is because our base model, the instruction-finetuned LLama-3 (8B), is already trained to perform exceptionally well in terms of helpfulness, which is the focus of the AlpacaEval benchmark. Additionally, the preference dataset we used, UltraFeedback, may not provide significant further enhancement in the helpfulness aspect. This is supported by the slight decrease observed in the AlpacaEval score for the standard DPO baseline as well (see Table 3, results on LLama-3). Therefore, we think these AlpacaEval 2.0 results on LLama-3 (8B) may not indicate that SAIL is ineffective; it may be simply caused by an ill-suited combination of base model, finetuning dataset, and evaluation benchmark.We also further conducted experiments on the Zephyr (7B) model as the backbone, whose AlpacaEval 2.0 win-rate is lower. We still train on the UltraFeedback preference dataset and the other experiment setups are unchanged. In this experiment, we see a larger improvement of the SAIL method compared to the standard DPO baseline (Zephyr-7B-Beta).| | AlpacaEval 2.0 (LC) Win-Rate ||--------------------|------------------------------|| Base (Zephyr-7B-SFT-Full) | 6.4 % || DPO (Zephyr-7B-Beta) | 13.2 % || SAIL-PP | 15.9 % |> Authors should compare more advanced preference optimization algorithms like ORPO and SimPO. And current results are not impressive for the alignment community.**Response:** Thank you for raising this insightful point. We see ORPO and SimPO are two recent work which propose a different objective than the standard RLHF, and achieve remarkable improvements in terms of alignment performance and efficiency.Our work focus more on bringing standard RLHF to a bilevel optimization framework and propose an effective and efficient approximate algorithm on top of it. We can see some new preference optimization methods including ORPO and SimPO have one fundamental difference from our approach: they do not explicitly incorporate the KL regularization term. The absence of the KL regularization term allows these methods to optimize more aggressively for the reward function by deviating significantly from the reference model. In contrast, our approach is specifically grounded in the standard RLHF, where the KL regularization term ensures that the model remains aligned with the reference distribution while optimizing for the reward function. This distinction makes direct comparisons with ORPO or SimPO less meaningful theoretically, as those methods omit the KL regularization and adopt a fundamentally different optimization objective design.However, we think our work, although developed adhering to the standard RLHF setup, can be compatible and combined with some recent advanced preference optimization algorithms, despite their differences in optimization setups and objectives. This is because we can reformulate their alignment problem as bilevel optimization, and go through the derivation as done in the paper. Taking SimPO as an example, we can treat their reward model definition (Equation (4) in the SimPO paper) as the solution of the upper level optimization (replacing Equation (4) in our manuscript), and adopt their modified Bradley-Terry objective with reward margin (Equation (5) in the SimPO paper) to replace the standard one (Equation (10) in our manuscript). By applying these changes and rederiving the extra gradient terms, we can formulate an adaptation of our method to the SimPO objective. We will implement this combined algorithm, which adapt our methodology to the SimPO objective, and compare with the SimPO as a baseline.Recently many different alignment objectives and algorithms have emerged; it is an interesting question to discuss the compatibility and combination of our method with each objective. We will add more relevant discussions to the appendices, but due to the fact that the compatibility problem with each design is a non-trivial question, this process may incur considerably more work, and we hope the reviewer understands that this effort cannot be fully reflected by the rebuttal period. But we will continue to expand the discussion as the wide compatibility to other designs also strengthens our contribution to the community. We thank the reviewer for raising this insightful point.'}, -# 'time': '2024-11-26 15:27:26' -# } -# research_arcade.insert_node("openreview_reviews", node_features=review_features) - -# update_node -# new_review_features = {'venue': 'ICLR.cc/2025/Conference', -# 'review_openreview_id': 'DHwZxFryth', -# 'replyto_openreview_id': 'Yqbllggrmw', -# 'writer': 'Authors', -# 'title': 'Response by Authors', -# 'content': {'Title': 'Response to Reviewer 7i95 (1/2)', 'Comment': '> The method does not improve much in the AlpacaEval 2.0 Score. The author should give a detailed explanation. And why not use metrics like length-controlled win rate?**Response:** Thank you for your careful observation and question. We would like to clarify that we are already using the length-controlled (LC) AlpacaEval 2.0 win-rate metric in our evaluations. We will make this clearer in the table header of Table 3.Regarding the fact that the AlpacaEval 2.0 scores on LLama-3 (8B) do not improve compared to the baselines, we believe this is because our base model, the instruction-finetuned LLama-3 (8B), is already trained to perform exceptionally well in terms of helpfulness, which is the focus of the AlpacaEval benchmark. Additionally, the preference dataset we used, UltraFeedback, may not provide significant further enhancement in the helpfulness aspect. This is supported by the slight decrease observed in the AlpacaEval score for the standard DPO baseline as well (see Table 3, results on LLama-3). Therefore, we think these AlpacaEval 2.0 results on LLama-3 (8B) may not indicate that SAIL is ineffective; it may be simply caused by an ill-suited combination of base model, finetuning dataset, and evaluation benchmark.We also further conducted experiments on the Zephyr (7B) model as the backbone, whose AlpacaEval 2.0 win-rate is lower. We still train on the UltraFeedback preference dataset and the other experiment setups are unchanged. In this experiment, we see a larger improvement of the SAIL method compared to the standard DPO baseline (Zephyr-7B-Beta).| | AlpacaEval 2.0 (LC) Win-Rate ||--------------------|------------------------------|| Base (Zephyr-7B-SFT-Full) | 6.4 % || DPO (Zephyr-7B-Beta) | 13.2 % || SAIL-PP | 15.9 % |> Authors should compare more advanced preference optimization algorithms like ORPO and SimPO. And current results are not impressive for the alignment community.**Response:** Thank you for raising this insightful point. We see ORPO and SimPO are two recent work which propose a different objective than the standard RLHF, and achieve remarkable improvements in terms of alignment performance and efficiency.Our work focus more on bringing standard RLHF to a bilevel optimization framework and propose an effective and efficient approximate algorithm on top of it. We can see some new preference optimization methods including ORPO and SimPO have one fundamental difference from our approach: they do not explicitly incorporate the KL regularization term. The absence of the KL regularization term allows these methods to optimize more aggressively for the reward function by deviating significantly from the reference model. In contrast, our approach is specifically grounded in the standard RLHF, where the KL regularization term ensures that the model remains aligned with the reference distribution while optimizing for the reward function. This distinction makes direct comparisons with ORPO or SimPO less meaningful theoretically, as those methods omit the KL regularization and adopt a fundamentally different optimization objective design.However, we think our work, although developed adhering to the standard RLHF setup, can be compatible and combined with some recent advanced preference optimization algorithms, despite their differences in optimization setups and objectives. This is because we can reformulate their alignment problem as bilevel optimization, and go through the derivation as done in the paper. Taking SimPO as an example, we can treat their reward model definition (Equation (4) in the SimPO paper) as the solution of the upper level optimization (replacing Equation (4) in our manuscript), and adopt their modified Bradley-Terry objective with reward margin (Equation (5) in the SimPO paper) to replace the standard one (Equation (10) in our manuscript). By applying these changes and rederiving the extra gradient terms, we can formulate an adaptation of our method to the SimPO objective. We will implement this combined algorithm, which adapt our methodology to the SimPO objective, and compare with the SimPO as a baseline.Recently many different alignment objectives and algorithms have emerged; it is an interesting question to discuss the compatibility and combination of our method with each objective. We will add more relevant discussions to the appendices, but due to the fact that the compatibility problem with each design is a non-trivial question, this process may incur considerably more work, and we hope the reviewer understands that this effort cannot be fully reflected by the rebuttal period. But we will continue to expand the discussion as the wide compatibility to other designs also strengthens our contribution to the community. We thank the reviewer for raising this insightful point.'}, -# 'time': '2024-11-26 15:27:26' -# } -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.get_node_features_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_reviews", node_features=new_review_features) -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.get_node_features_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) - -########## openreview_revisions ########## -# construct_from_api -# venue = "ICLR.cc/2025/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2025", "Published as a conference paper at ICLR 2025"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2025/" -# log_file = "./log/failed_ids_revisions_2025.log" -# venue = "ICLR.cc/2023/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2023", "Published as a conference paper at ICLR 2023"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2023/" -# log_file = "./log/failed_ids_revisions_2023.log" -# venue = "ICLR.cc/2017/conference" -# filter_list = ["Under review as a conference paper at ICLR 2017", "Published as a conference paper at ICLR 2017"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2017/" -# log_file = "./log/failed_ids_revisions_2017.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file} -# research_arcade.construct_table_from_api("openreview_revisions", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_revision_example.csv"} -# research_arcade.construct_table_from_csv("openreview_revisions", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_revision_example.json"} -# research_arcade.construct_table_from_json("openreview_revisions", config) - -# get_all_node_features -# openreview_revisions_df = research_arcade.get_all_node_features("openreview_revisions") -# print(len(openreview_revisions_df)) - -# get_node_features_by_id -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.get_node_features_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) - -# delete_node -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.delete_node_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) - -# insert_node -# revision_feature = {'venue': 'ICLR.cc/2025/Conference', -# 'original_openreview_id': 'pbTVNlX8Ig', -# 'revision_openreview_id': 'yfHQOp5zWc', -# 'content': [{'section': '1 INTRODUCTION', -# 'after_section': None, -# 'context_after': '2 RELATED WORK ', -# 'paragraph_idx': 9, -# 'before_section': None, -# 'context_before': 'Published as a conference paper at ICLR 2025 tograd system in PyTorch, specifically tailored for our experimental setup, which is available at ', -# 'modified_lines': 'https://github.com/stephane-rivaud/PETRA. ', -# 'original_lines': 'https://github.com/streethagore/PETRA. ', -# 'after_paragraph_idx': None, -# 'before_paragraph_idx': None}], -# 'time': '2025-03-14 15:35:37'} -# research_arcade.insert_node("openreview_revisions", node_features=revision_feature) - -# update_node -# new_revision_features = {'venue': 'ICLR.cc/2025/Conference', -# 'original_openreview_id': 'pbTVNlX8Ig', -# 'revision_openreview_id': 'yfHQOp5zWc', -# 'content': [{'section': '1 INTRODUCTION', -# 'after_section': None, -# 'context_after': '2 RELATED WORK ', -# 'paragraph_idx': 9, -# 'before_section': None, -# 'context_before': 'Published as a conference paper at ICLR 2025 tograd system in PyTorch, specifically tailored for our experimental setup, which is available at ', -# 'modified_lines': 'https://github.com/stephane-rivaud/PETRA. ', -# 'original_lines': 'https://github.com/streethagore/PETRA. ', -# 'after_paragraph_idx': None, -# 'before_paragraph_idx': None}], -# 'time': '2025-03-14 15:35:37'} -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.get_node_features_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_revisions", node_features=new_revision_features) -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.get_node_features_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) - -########## openreview_paragraphs ########## -# construct_from_api -# venue = "ICLR.cc/2025/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2025", "Published as a conference paper at ICLR 2025"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2025/" -# log_file = "./log/failed_ids_revisions_2025.log" -# venue = "ICLR.cc/2023/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2023", "Published as a conference paper at ICLR 2023"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2023/" -# log_file = "./log/failed_ids_revisions_2023.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file} -# venue = "ICLR.cc/2017/conference" -# filter_list = ["Under review as a conference paper at ICLR 2017", "Published as a conference paper at ICLR 2017"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2017/" -# log_file = "./log/failed_ids_revisions_2017.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file, "is_paper": True, "is_revision": True, "is_pdf_delete": False} -# research_arcade.construct_table_from_api("openreview_paragraphs", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_paragraphs_example.csv"} -# research_arcade.construct_table_from_csv("openreview_paragraphs", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_paragraphs_example.json"} -# research_arcade.construct_table_from_json("openreview_paragraphs", config) - -# get_all_node_features -# openreview_paragraphs_df = research_arcade.get_all_node_features("openreview_paragraphs") -# print(len(openreview_paragraphs_df)) - -# get_node_features_by_id -# paper_id = {"paper_openreview_id": "ryxB0Rtxx"} -# paragraph_feature = research_arcade.get_node_features_by_id("openreview_paragraphs", paper_id) -# print(paragraph_feature.to_dict(orient="records")[0]) - -# insert_node -# paragraph_feature = {'venue': 'xujj_test', -# 'paper_openreview_id': 'xujj_test', -# 'paragraph_idx': 1, -# 'section': "xujj_test", -# 'content': "xujj_test"} -# research_arcade.insert_node("openreview_paragraphs", node_features=paragraph_feature) - -# delete_node -# paper_id = {"paper_openreview_id": "xujj_test"} -# paragraph_feature = research_arcade.delete_node_by_id("openreview_paragraphs", paper_id) -# print(len(paragraph_feature)) -# print(paragraph_feature.to_dict(orient="records")[0]) - -########## openreview_arxiv ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2017/conference"} -# research_arcade.construct_table_from_api("openreview_arxiv", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_arxiv_example.csv"} -# research_arcade.construct_table_from_csv("openreview_arxiv", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_arxiv_example.json"} -# research_arcade.construct_table_from_json("openreview_arxiv", config) - -# get_all_edge_features -# openreview_arxiv_df = research_arcade.get_all_edge_features("openreview_arxiv") -# print(len(openreview_arxiv_df)) - -# get_all_edge_features -# openreview_arxiv_df = research_arcade.get_all_edge_features("openreview_arxiv") -# print(len(openreview_arxiv_df)) - -# get_neighborhood -# openreview_id = {"paper_openreview_id": "zkNCWtw2fd"} -# openreview_arxiv_df = research_arcade.get_neighborhood("openreview_arxiv", openreview_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) -# arxiv_id = {"arxiv_id": "http://arxiv.org/abs/2408.10536v1"} -# openreview_arxiv_df = research_arcade.get_neighborhood("openreview_arxiv", arxiv_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) - -# delete_edge -# openreview_id = {"paper_openreview_id": "zkNCWtw2fd"} -# openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) -# arxiv_id = {"arxiv_id": "http://arxiv.org/abs/2408.10536v1"} -# openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", arxiv_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) -# openreview_arxiv_id = {"paper_openreview_id": "zkNCWtw2fd", "arxiv_id": "http://arxiv.org/abs/2408.10536v1"} -# openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_arxiv_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) - -# insert_edge -# openreview_arxiv = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': 'zkNCWtw2fd', -# 'arxiv_id': 'http://arxiv.org/abs/2408.10536v1', -# 'title': 'Synergistic Approach for Simultaneous Optimization of Monolingual, Cross-lingual, and Multilingual Information Retrieval' -# } -# research_arcade.insert_edge("openreview_arxiv", openreview_arxiv) - -########## openreview_papers_authors ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_papers_authors", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_authors_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers_authors", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_authors_example.json"} -# research_arcade.construct_table_from_json("openreview_papers_authors", config) - -# get_all_edge_features -# openreview_papers_authors = research_arcade.get_all_edge_features("openreview_papers_authors") -# print(len(openreview_papers_authors)) - -# get_neighborhood -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_authors = research_arcade.get_neighborhood("openreview_papers_authors", paper_id) -# print(openreview_papers_authors.to_dict(orient="records")) -# author_id = {'author_openreview_id': '~Elias_Stengel-Eskin1'} -# openreview_papers_authors = research_arcade.get_neighborhood("openreview_papers_authors", author_id) -# print(openreview_papers_authors.to_dict(orient="records")) - -# delete_edge -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", paper_id) -# print(openreview_papers_authors.to_dict(orient="records")) -# author_id = {'author_openreview_id': '~Elias_Stengel-Eskin1'} -# openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", author_id) -# print(openreview_papers_authors.to_dict(orient="records")) -# paper_author = {"paper_openreview_id": "00SnKBGTsz", 'author_openreview_id': '~Elias_Stengel-Eskin1'} -# openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", paper_author) -# print(openreview_papers_authors.to_dict(orient="records")) - -# insert_edge -# paper_authors = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Zaid_Khan1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Jaemin_Cho1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Mohit_Bansal2'}] -# for item in paper_authors: -# research_arcade.insert_edge("openreview_papers_authors", item) -# author_papers = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'Xbl6t6zxZs', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'fDcn3S8oAt', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'j9wBgcxa7N', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'zd0iX5xBhA', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2024/Conference', 'paper_openreview_id': 'L4nOxziGf9', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2024/Conference', 'paper_openreview_id': 'qL9gogRepu', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}] -# for item in author_papers: -# research_arcade.insert_edge("openreview_papers_authors", item) -# paper_author = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}] -# for item in paper_author: -# research_arcade.insert_edge("openreview_papers_authors", item) - -########## openreview_papers_reviews ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2017/conference"} -# research_arcade.construct_table_from_api("openreview_papers_reviews", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_reviews_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers_reviews", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_reviews_example.json"} -# research_arcade.construct_table_from_json("openreview_papers_reviews", config) - -# get_all_edge_features -# openreview_papers_reviews = research_arcade.get_all_edge_features("openreview_papers_reviews") -# print(len(openreview_papers_reviews)) - -# get_neighborhood -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_reviews = research_arcade.get_neighborhood("openreview_papers_reviews", paper_id) -# print(openreview_papers_reviews.to_dict(orient="records")) -# review_id = {"review_openreview_id": "13mj0Rtn5W"} -# openreview_papers_reviews = research_arcade.get_neighborhood("openreview_papers_reviews", review_id) -# print(openreview_papers_reviews.to_dict(orient="records")) - -# delete_edge -# paper_review_id = {"paper_openreview_id": "00SnKBGTsz", "review_openreview_id": "13mj0Rtn5W"} -# openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", paper_review_id) -# print(openreview_papers_reviews.to_dict(orient="records")) -# review_id = {"review_openreview_id": "13mj0Rtn5W"} -# openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", review_id) -# print(openreview_papers_reviews.to_dict(orient="records")) -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", paper_id) -# print(openreview_papers_reviews.to_dict(orient="records")) - -# insert_edge -# paper_review = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': '00SnKBGTsz', -# 'review_openreview_id': '13mj0Rtn5W', -# 'title': 'Response by Authors', -# 'time': '2024-11-27 17:27:45'} -# research_arcade.insert_edge("openreview_papers_reviews", paper_review) -# paper_reviews = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '7XT4kLWV2f', 'title': 'Official Review by Reviewer_wuGW', 'time': '2024-11-01 14:52:22'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'i3QgWgrJff', 'title': 'Official Review by Reviewer_rVo8', 'time': '2024-11-04 02:37:10'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'GMsjHLXdOx', 'title': 'Official Review by Reviewer_c5nB', 'time': '2024-11-04 09:59:14'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'r8ZflFk3T7', 'title': 'Official Review by Reviewer_VQ9Y', 'time': '2024-11-06 00:15:47'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '4CnQpVCYkF', 'title': 'Response by Authors', 'time': '2024-11-20 22:48:42'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'h1qvpjhRP3', 'title': 'Response by Authors', 'time': '2024-11-20 22:51:07'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'pOR42YNLtU', 'title': 'Response by Authors', 'time': '2024-11-20 22:55:04'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'Aq2tBtB0lt', 'title': 'Response by Authors', 'time': '2024-11-20 22:57:18'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'm1iUqPHpwk', 'title': 'Response by Authors', 'time': '2024-11-20 22:58:29'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '66buacQmRe', 'title': 'Response by Authors', 'time': '2024-11-20 23:02:21'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'Bgr7Ol90m7', 'title': 'Response by Authors', 'time': '2024-11-22 23:11:06'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'H2h2K6a8x5', 'title': 'Response by Reviewer', 'time': '2024-11-23 10:04:58'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'la5jPwJU4g', 'title': 'Response by Authors', 'time': '2024-11-24 19:17:22'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'DjVKsUoFN2', 'title': 'Response by Reviewer', 'time': '2024-11-25 04:00:18'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'C3MhCuKhTf', 'title': 'Response by Authors', 'time': '2024-11-25 19:44:38'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'ZqwAYtcmhv', 'title': 'Response by Authors', 'time': '2024-11-25 19:45:43'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '9OQJoesINr', 'title': 'Response by Reviewer', 'time': '2024-11-25 20:07:51'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'wqTNtVDwef', 'title': 'Response by Authors', 'time': '2024-11-26 03:32:30'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'NEsxOTkkIV', 'title': 'Response by Reviewer', 'time': '2024-11-26 20:00:00'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '13mj0Rtn5W', 'title': 'Response by Authors', 'time': '2024-11-27 17:27:45'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'hWat8aFBRw', 'title': 'Response by Reviewer', 'time': '2024-11-27 11:34:03'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'wnsiUkDh00', 'title': 'Response by Authors', 'time': '2024-11-27 17:28:35'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'zpboemkkjR', 'title': 'Meta Review of Submission11063 by Area_Chair_eoLd', 'time': '2024-12-20 15:14:25'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'kokKFEn2fw', 'title': 'Paper Decision', 'time': '2025-01-22 05:35:00'} -# ] -# for item in tqdm(paper_reviews): -# research_arcade.insert_edge("openreview_papers_reviews", item) - -########## openreview_papers_revisions ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_papers_revisions", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_revisions_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers_revisions", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_revisions_example.json"} -# research_arcade.construct_table_from_json("openreview_papers_revisions", config) - -# get_all_edge_features -# openreview_papers_revisions = research_arcade.get_all_edge_features("openreview_papers_revisions") -# print(len(openreview_papers_revisions)) - -# get_neighborhood -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# paper_revision = research_arcade.get_neighborhood("openreview_papers_revisions", paper_id) -# print(paper_revision.to_dict(orient="records")) -# revision_id = {"revision_openreview_id": "dzL3IRBnE4"} -# paper_revision = research_arcade.get_neighborhood("openreview_papers_revisions", revision_id) -# print(paper_revision.to_dict(orient="records")) - -# delete_edge -# paper_revision_id = {"paper_openreview_id": "00SnKBGTsz", "revision_openreview_id": "dzL3IRBnE4"} -# paper_revision = research_arcade.delete_edge_by_id("openreview_papers_revisions", paper_revision_id) -# print(paper_revision.to_dict(orient="records")) -# revision_id = {"revision_openreview_id": "dzL3IRBnE4"} -# paper_revision = research_arcade.delete_edge_by_id("openreview_papers_revisions", revision_id) -# print(paper_revision.to_dict(orient="records")) -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# paper_revision = research_arcade.delete_edge_by_id("openreview_papers_revisions", paper_id) -# print(paper_revision.to_dict(orient="records")) - -# insert_edge -# paper_revision = {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'dzL3IRBnE4', 'title': 'Camera_Ready_Revision', 'time': '2025-03-01 03:36:55'} -# research_arcade.insert_edge("openreview_papers_revisions", paper_revision) -# paper_revisions = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'oT4N28siLO', 'title': 'Camera_Ready_Revision', 'time': '2025-03-02 01:35:16'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'dzL3IRBnE4', 'title': 'Camera_Ready_Revision', 'time': '2025-03-01 03:36:55'}] -# for item in tqdm(paper_revisions): -# research_arcade.insert_edge("openreview_papers_revisions", item) - -########## openreview_revisions_reviews ########## -# construct_based_on_existing_tables -# papers_reviews_df = research_arcade.get_all_edge_features("openreview_papers_reviews") -# print(len(papers_reviews_df)) -# papers_revisions_df = research_arcade.get_all_edge_features("openreview_papers_revisions") -# print(len(papers_revisions_df)) -# config = {"papers_reviews_df": papers_reviews_df, "papers_revisions_df": papers_revisions_df} -# research_arcade.construct_table_from_api("openreview_revisions_reviews", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_revisions_reviews_example.csv"} -# research_arcade.construct_table_from_csv("openreview_revisions_reviews", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_revisions_reviews_example.json"} -# research_arcade.construct_table_from_json("openreview_revisions_reviews", config) - -# get_all_edge_features -# openreview_revisions_reviews = research_arcade.get_all_edge_features("openreview_revisions_reviews") -# print(len(openreview_revisions_reviews)) - -# get_neighborhood -# revision_id = {'revision_openreview_id': 'cX02yuzwWI'} -# revision_review = research_arcade.get_neighborhood("openreview_revisions_reviews", revision_id) -# print(revision_review.to_dict(orient="records")) -# review_id = {'review_openreview_id': 'wumckDPIQ3'} -# revision_review = research_arcade.get_neighborhood("openreview_revisions_reviews", review_id) -# print(revision_review.to_dict(orient="records")) - -# delete_edge -# revision_review_id = {'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'} -# revision_review = research_arcade.delete_edge_by_id("openreview_revisions_reviews", revision_review_id) -# print(revision_review.to_dict(orient="records")) -# review_id = {'review_openreview_id': 'wumckDPIQ3'} -# revision_review = research_arcade.delete_edge_by_id("openreview_revisions_reviews", review_id) -# print(revision_review.to_dict(orient="records")) -# paper_id = {'revision_openreview_id': 'cX02yuzwWI'} -# revision_review = research_arcade.delete_edge_by_id("openreview_revisions_reviews", paper_id) -# print(revision_review.to_dict(orient="records")) - -# insert_edge -# revision_review = {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'} -# research_arcade.insert_edge("openreview_revisions_reviews", revision_review) -# revision_reviews = [{'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '138cOdBpgA'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'yKh1fQYnUZ'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'Pvt0OjNSp2'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'MUhlEYyBD9'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '2mqiS3J8wC'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'Er8QTorcyr'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'AvtD9uxRtX'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '2tgxTGynNm'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '5MKJE3sFsd'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wViZ0H4ErF'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '0c1It75dTb'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'PFwia9lcjP'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'ygCqaGNPee'}] -# for item in tqdm(revision_reviews): -# research_arcade.insert_edge("openreview_revisions_reviews", item) \ No newline at end of file diff --git a/test/openreview_sqlbased_test.py b/test/openreview_sqlbased_test.py deleted file mode 100644 index e85fe6b..0000000 --- a/test/openreview_sqlbased_test.py +++ /dev/null @@ -1,573 +0,0 @@ -import sys -from pathlib import Path -project_root = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(project_root)) -from research_arcade import ResearchArcade - -db_type = "sql" -config = { - "host": "localhost", - "dbname": "iclr_openreview_database", - "user": "jingjunx", - "password": "", - "port": "5432" -} - -research_arcade = ResearchArcade(db_type=db_type, config=config) - -########## openreview_authors ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_authors", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_author_example.csv"} -# research_arcade.construct_table_from_csv("openreview_authors", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_author_example.json"} -# research_arcade.construct_table_from_json("openreview_authors", config) - -# get_all_node_features -# openreview_authors_df = research_arcade.get_all_node_features("openreview_authors") -# print(len(openreview_authors_df)) - -# get_node_features_by_id -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.get_node_features_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) - -# delete_node -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.delete_node_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) - -# insert_node -# new_author = {'venue': 'ICLR.cc/2025/Conference', -# 'author_openreview_id': '~ishmam_zabir1', -# 'author_full_name': 'ishmam zabir', -# 'email': '****@microsoft.com', -# 'affiliation': 'Microsoft', -# 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', -# 'dblp': ''} -# research_arcade.insert_node("openreview_authors", node_features=new_author) - -# update_node -# new_author = {'venue': 'ICLR.cc/2025/Conference', -# 'author_openreview_id': '~ishmam_zabir1', -# 'author_full_name': 'ishmam zabir', -# 'email': '****@microsoft.com', -# 'affiliation': 'Microsoft', -# 'homepage': 'https://scholar.google.com/citations?user=X7bjzrUAAAAJ&hl=en&oi=ao', -# 'dblp': ''} -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.get_node_features_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_authors", node_features=new_author) -# author_id = {"author_openreview_id": "~ishmam_zabir1"} -# author_features = research_arcade.get_node_features_by_id("openreview_authors", author_id) -# print(author_features.to_dict(orient="records")[0]) - -########## openreview_papers ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_papers", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_paper_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_paper_example.json"} -# research_arcade.construct_table_from_json("openreview_papers", config) - -# get_all_node_features -# openreview_papers_df = research_arcade.get_all_node_features("openreview_papers") -# print(len(openreview_papers_df)) - -# get_node_features_by_id -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.get_node_features_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) - -# delete_node -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.delete_node_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) - -# insert_node -# paper_features = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': 'zGej22CBnS', -# 'title': 'Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles', -# 'abstract': "Tokenization is associated with many poorly understood shortcomings in language models (LMs), yet remains an important component for long sequence scaling purposes. This work studies how tokenization impacts model performance by analyzing and comparing the stochastic behavior of tokenized models with their byte-level, or token-free, counterparts. We discover that, even when the two models are statistically equivalent, their predictive distributions over the next byte can be substantially different, a phenomenon we term as ``tokenization bias''. To fully characterize this phenomenon, we introduce the Byte-Token Representation Lemma, a framework that establishes a mapping between the learned token distribution and its equivalent byte-level distribution. From this result, we develop a next-byte sampling algorithm that eliminates tokenization bias without requiring further training or optimization. In other words, this enables zero-shot conversion of tokenized LMs into statistically equivalent token-free ones. We demonstrate its broad applicability with two use cases: fill-in-the-middle (FIM) tasks and model ensembles. In FIM tasks where input prompts may terminate mid-token, leading to out-of-distribution tokenization, our method mitigates performance degradation and achieves 18\\% improvement in FIM coding benchmarks, while consistently outperforming the standard token healing fix. For model ensembles where each model employs a distinct vocabulary, our approach enables seamless integration, resulting in improved performance up to 3.7\\% over individual models across various standard baselines in reasoning, knowledge, and coding. Code is available at:https: //github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs.", -# 'paper_decision': 'ICLR 2025 Poster', -# 'paper_pdf_link': '/pdf/cdd2212a20c4034029874cba11a05e081bfdb83e.pdf'} -# research_arcade.insert_node("openreview_papers", node_features=paper_features) - -# update_node -# new_paper_features = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': 'zGej22CBnS', -# 'title': 'Exact Byte-Level Probabilities from Tokenized Language Models for FIM-Tasks and Model Ensembles', -# 'abstract': "Tokenization is associated with many poorly understood shortcomings in language models (LMs), yet remains an important component for long sequence scaling purposes. This work studies how tokenization impacts model performance by analyzing and comparing the stochastic behavior of tokenized models with their byte-level, or token-free, counterparts. We discover that, even when the two models are statistically equivalent, their predictive distributions over the next byte can be substantially different, a phenomenon we term as ``tokenization bias''. To fully characterize this phenomenon, we introduce the Byte-Token Representation Lemma, a framework that establishes a mapping between the learned token distribution and its equivalent byte-level distribution. From this result, we develop a next-byte sampling algorithm that eliminates tokenization bias without requiring further training or optimization. In other words, this enables zero-shot conversion of tokenized LMs into statistically equivalent token-free ones. We demonstrate its broad applicability with two use cases: fill-in-the-middle (FIM) tasks and model ensembles. In FIM tasks where input prompts may terminate mid-token, leading to out-of-distribution tokenization, our method mitigates performance degradation and achieves 18\\% improvement in FIM coding benchmarks, while consistently outperforming the standard token healing fix. For model ensembles where each model employs a distinct vocabulary, our approach enables seamless integration, resulting in improved performance up to 3.7\\% over individual models across various standard baselines in reasoning, knowledge, and coding. Code is available at:https: //github.com/facebookresearch/Exact-Byte-Level-Probabilities-from-Tokenized-LMs.", -# 'paper_decision': 'ICLR 2025 Poster', -# 'paper_pdf_link': '/pdf/cdd2212a20c4034029874cba11a05e081bfdb83e.pdf'} -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.get_node_features_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_papers", node_features=new_paper_features) -# paper_id = {"paper_openreview_id": "zGej22CBnS"} -# paper_features = research_arcade.get_node_features_by_id("openreview_papers", paper_id) -# print(paper_features.to_dict(orient="records")[0]) - -########## openreview_reviews ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2013/conference"} -# research_arcade.construct_table_from_api("openreview_reviews", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_review_example.csv"} -# research_arcade.construct_table_from_csv("openreview_reviews", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_review_example.json"} -# research_arcade.construct_table_from_json("openreview_reviews", config) - -# get_all_node_features -# openreview_reviews_df = research_arcade.get_all_node_features("openreview_reviews") -# print(len(openreview_reviews_df)) - -# get_node_features_by_id -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.get_node_features_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) - -# delete_node -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.delete_node_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) - -# insert_node -# review_features = {'venue': 'ICLR.cc/2025/Conference', -# 'review_openreview_id': 'DHwZxFryth', -# 'replyto_openreview_id': 'Yqbllggrmw', -# 'writer': 'Authors', -# 'title': 'Response by Authors', -# 'content': {'Title': 'Response to Reviewer 7i95 (1/2)', 'Comment': '> The method does not improve much in the AlpacaEval 2.0 Score. The author should give a detailed explanation. And why not use metrics like length-controlled win rate?**Response:** Thank you for your careful observation and question. We would like to clarify that we are already using the length-controlled (LC) AlpacaEval 2.0 win-rate metric in our evaluations. We will make this clearer in the table header of Table 3.Regarding the fact that the AlpacaEval 2.0 scores on LLama-3 (8B) do not improve compared to the baselines, we believe this is because our base model, the instruction-finetuned LLama-3 (8B), is already trained to perform exceptionally well in terms of helpfulness, which is the focus of the AlpacaEval benchmark. Additionally, the preference dataset we used, UltraFeedback, may not provide significant further enhancement in the helpfulness aspect. This is supported by the slight decrease observed in the AlpacaEval score for the standard DPO baseline as well (see Table 3, results on LLama-3). Therefore, we think these AlpacaEval 2.0 results on LLama-3 (8B) may not indicate that SAIL is ineffective; it may be simply caused by an ill-suited combination of base model, finetuning dataset, and evaluation benchmark.We also further conducted experiments on the Zephyr (7B) model as the backbone, whose AlpacaEval 2.0 win-rate is lower. We still train on the UltraFeedback preference dataset and the other experiment setups are unchanged. In this experiment, we see a larger improvement of the SAIL method compared to the standard DPO baseline (Zephyr-7B-Beta).| | AlpacaEval 2.0 (LC) Win-Rate ||--------------------|------------------------------|| Base (Zephyr-7B-SFT-Full) | 6.4 % || DPO (Zephyr-7B-Beta) | 13.2 % || SAIL-PP | 15.9 % |> Authors should compare more advanced preference optimization algorithms like ORPO and SimPO. And current results are not impressive for the alignment community.**Response:** Thank you for raising this insightful point. We see ORPO and SimPO are two recent work which propose a different objective than the standard RLHF, and achieve remarkable improvements in terms of alignment performance and efficiency.Our work focus more on bringing standard RLHF to a bilevel optimization framework and propose an effective and efficient approximate algorithm on top of it. We can see some new preference optimization methods including ORPO and SimPO have one fundamental difference from our approach: they do not explicitly incorporate the KL regularization term. The absence of the KL regularization term allows these methods to optimize more aggressively for the reward function by deviating significantly from the reference model. In contrast, our approach is specifically grounded in the standard RLHF, where the KL regularization term ensures that the model remains aligned with the reference distribution while optimizing for the reward function. This distinction makes direct comparisons with ORPO or SimPO less meaningful theoretically, as those methods omit the KL regularization and adopt a fundamentally different optimization objective design.However, we think our work, although developed adhering to the standard RLHF setup, can be compatible and combined with some recent advanced preference optimization algorithms, despite their differences in optimization setups and objectives. This is because we can reformulate their alignment problem as bilevel optimization, and go through the derivation as done in the paper. Taking SimPO as an example, we can treat their reward model definition (Equation (4) in the SimPO paper) as the solution of the upper level optimization (replacing Equation (4) in our manuscript), and adopt their modified Bradley-Terry objective with reward margin (Equation (5) in the SimPO paper) to replace the standard one (Equation (10) in our manuscript). By applying these changes and rederiving the extra gradient terms, we can formulate an adaptation of our method to the SimPO objective. We will implement this combined algorithm, which adapt our methodology to the SimPO objective, and compare with the SimPO as a baseline.Recently many different alignment objectives and algorithms have emerged; it is an interesting question to discuss the compatibility and combination of our method with each objective. We will add more relevant discussions to the appendices, but due to the fact that the compatibility problem with each design is a non-trivial question, this process may incur considerably more work, and we hope the reviewer understands that this effort cannot be fully reflected by the rebuttal period. But we will continue to expand the discussion as the wide compatibility to other designs also strengthens our contribution to the community. We thank the reviewer for raising this insightful point.'}, -# 'time': '2024-11-26 15:27:26' -# } -# research_arcade.insert_node("openreview_reviews", node_features=review_features) - -# update_node -# new_review_features = {'venue': 'ICLR.cc/2025/Conference', -# 'review_openreview_id': 'DHwZxFryth', -# 'replyto_openreview_id': 'Yqbllggrmw', -# 'writer': 'Authors', -# 'title': 'Response by Authors', -# 'content': {'Title': 'Response to Reviewer 7i95 (1/2)', 'Comment': '> The method does not improve much in the AlpacaEval 2.0 Score. The author should give a detailed explanation. And why not use metrics like length-controlled win rate?**Response:** Thank you for your careful observation and question. We would like to clarify that we are already using the length-controlled (LC) AlpacaEval 2.0 win-rate metric in our evaluations. We will make this clearer in the table header of Table 3.Regarding the fact that the AlpacaEval 2.0 scores on LLama-3 (8B) do not improve compared to the baselines, we believe this is because our base model, the instruction-finetuned LLama-3 (8B), is already trained to perform exceptionally well in terms of helpfulness, which is the focus of the AlpacaEval benchmark. Additionally, the preference dataset we used, UltraFeedback, may not provide significant further enhancement in the helpfulness aspect. This is supported by the slight decrease observed in the AlpacaEval score for the standard DPO baseline as well (see Table 3, results on LLama-3). Therefore, we think these AlpacaEval 2.0 results on LLama-3 (8B) may not indicate that SAIL is ineffective; it may be simply caused by an ill-suited combination of base model, finetuning dataset, and evaluation benchmark.We also further conducted experiments on the Zephyr (7B) model as the backbone, whose AlpacaEval 2.0 win-rate is lower. We still train on the UltraFeedback preference dataset and the other experiment setups are unchanged. In this experiment, we see a larger improvement of the SAIL method compared to the standard DPO baseline (Zephyr-7B-Beta).| | AlpacaEval 2.0 (LC) Win-Rate ||--------------------|------------------------------|| Base (Zephyr-7B-SFT-Full) | 6.4 % || DPO (Zephyr-7B-Beta) | 13.2 % || SAIL-PP | 15.9 % |> Authors should compare more advanced preference optimization algorithms like ORPO and SimPO. And current results are not impressive for the alignment community.**Response:** Thank you for raising this insightful point. We see ORPO and SimPO are two recent work which propose a different objective than the standard RLHF, and achieve remarkable improvements in terms of alignment performance and efficiency.Our work focus more on bringing standard RLHF to a bilevel optimization framework and propose an effective and efficient approximate algorithm on top of it. We can see some new preference optimization methods including ORPO and SimPO have one fundamental difference from our approach: they do not explicitly incorporate the KL regularization term. The absence of the KL regularization term allows these methods to optimize more aggressively for the reward function by deviating significantly from the reference model. In contrast, our approach is specifically grounded in the standard RLHF, where the KL regularization term ensures that the model remains aligned with the reference distribution while optimizing for the reward function. This distinction makes direct comparisons with ORPO or SimPO less meaningful theoretically, as those methods omit the KL regularization and adopt a fundamentally different optimization objective design.However, we think our work, although developed adhering to the standard RLHF setup, can be compatible and combined with some recent advanced preference optimization algorithms, despite their differences in optimization setups and objectives. This is because we can reformulate their alignment problem as bilevel optimization, and go through the derivation as done in the paper. Taking SimPO as an example, we can treat their reward model definition (Equation (4) in the SimPO paper) as the solution of the upper level optimization (replacing Equation (4) in our manuscript), and adopt their modified Bradley-Terry objective with reward margin (Equation (5) in the SimPO paper) to replace the standard one (Equation (10) in our manuscript). By applying these changes and rederiving the extra gradient terms, we can formulate an adaptation of our method to the SimPO objective. We will implement this combined algorithm, which adapt our methodology to the SimPO objective, and compare with the SimPO as a baseline.Recently many different alignment objectives and algorithms have emerged; it is an interesting question to discuss the compatibility and combination of our method with each objective. We will add more relevant discussions to the appendices, but due to the fact that the compatibility problem with each design is a non-trivial question, this process may incur considerably more work, and we hope the reviewer understands that this effort cannot be fully reflected by the rebuttal period. But we will continue to expand the discussion as the wide compatibility to other designs also strengthens our contribution to the community. We thank the reviewer for raising this insightful point.'}, -# 'time': '2024-11-26 15:27:26' -# } -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.get_node_features_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_reviews", node_features=new_review_features) -# review_id = {"review_openreview_id": "DHwZxFryth"} -# review_features = research_arcade.get_node_features_by_id("openreview_reviews", review_id) -# print(review_features.to_dict(orient="records")[0]) - -########## openreview_revisions ########## -# construct_from_api -# venue = "ICLR.cc/2025/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2025", "Published as a conference paper at ICLR 2025"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2025/" -# log_file = "./log/failed_ids_revisions_2025.log" -# venue = "ICLR.cc/2023/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2023", "Published as a conference paper at ICLR 2023"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2023/" -# log_file = "./log/failed_ids_revisions_2023.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file} -# venue = "ICLR.cc/2017/conference" -# filter_list = ["Under review as a conference paper at ICLR 2017", "Published as a conference paper at ICLR 2017"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2017/" -# log_file = "./log/failed_ids_revisions_2017.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file} -# research_arcade.construct_table_from_api("openreview_revisions", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_revision_example.csv"} -# research_arcade.construct_table_from_csv("openreview_revisions", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_revision_example.json"} -# research_arcade.construct_table_from_json("openreview_revisions", config) - -# get_all_node_features -# openreview_revisions_df = research_arcade.get_all_node_features("openreview_revisions") -# print(len(openreview_revisions_df)) - -# get_node_features_by_id -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.get_node_features_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) - -# delete_node -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.delete_node_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) - -# insert_node -# revision_feature = {'venue': 'ICLR.cc/2025/Conference', -# 'original_openreview_id': 'pbTVNlX8Ig', -# 'revision_openreview_id': 'yfHQOp5zWc', -# 'content': [{'section': '1 INTRODUCTION', -# 'after_section': None, -# 'context_after': '2 RELATED WORK ', -# 'paragraph_idx': 9, -# 'before_section': None, -# 'context_before': 'Published as a conference paper at ICLR 2025 tograd system in PyTorch, specifically tailored for our experimental setup, which is available at ', -# 'modified_lines': 'https://github.com/stephane-rivaud/PETRA. ', -# 'original_lines': 'https://github.com/streethagore/PETRA. ', -# 'after_paragraph_idx': None, -# 'before_paragraph_idx': None}], -# 'time': '2025-03-14 15:35:37'} -# research_arcade.insert_node("openreview_revisions", node_features=revision_feature) - -# update_node -# new_revision_features = {'venue': 'ICLR.cc/2025/Conference', -# 'original_openreview_id': 'pbTVNlX8Ig', -# 'revision_openreview_id': 'yfHQOp5zWc', -# 'content': [{'section': '1 INTRODUCTION', -# 'after_section': None, -# 'context_after': '2 RELATED WORK ', -# 'paragraph_idx': 9, -# 'before_section': None, -# 'context_before': 'Published as a conference paper at ICLR 2025 tograd system in PyTorch, specifically tailored for our experimental setup, which is available at ', -# 'modified_lines': 'https://github.com/stephane-rivaud/PETRA. ', -# 'original_lines': 'https://github.com/streethagore/PETRA. ', -# 'after_paragraph_idx': None, -# 'before_paragraph_idx': None}], -# 'time': '2025-03-14 15:35:37'} -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.get_node_features_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) -# research_arcade.update_node("openreview_revisions", node_features=new_revision_features) -# revision_id = {"revision_openreview_id": "yfHQOp5zWc"} -# revision_feature = research_arcade.get_node_features_by_id("openreview_revisions", revision_id) -# print(revision_feature.to_dict(orient="records")[0]) - -########## openreview_paragraphs ########## -# construct_from_api -# venue = "ICLR.cc/2025/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2025", "Published as a conference paper at ICLR 2025"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2025/" -# log_file = "./log/failed_ids_revisions_2025.log" -# venue = "ICLR.cc/2023/Conference" -# filter_list = ["Under review as a conference paper at ICLR 2023", "Published as a conference paper at ICLR 2023"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2023/" -# log_file = "./log/failed_ids_revisions_2023.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file} -# venue = "ICLR.cc/2017/conference" -# filter_list = ["Under review as a conference paper at ICLR 2017", "Published as a conference paper at ICLR 2017"] -# pdf_dir = "/data/jingjunx/openreview_pdfs_2017/" -# log_file = "./log/failed_ids_revisions_2017.log" -# config = {"venue": venue, "filter_list": filter_list, "pdf_dir": pdf_dir, "log_file": log_file, "is_paper": True, "is_revision": True, "is_pdf_delete": False} -# research_arcade.construct_table_from_api("openreview_paragraphs", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_paragraphs_example.csv"} -# research_arcade.construct_table_from_csv("openreview_paragraphs", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_paragraphs_example.json"} -# research_arcade.construct_table_from_json("openreview_paragraphs", config) - -# get_all_node_features -# openreview_paragraphs_df = research_arcade.get_all_node_features("openreview_paragraphs") -# print(len(openreview_paragraphs_df)) - -# get_node_features_by_id -# paper_id = {"paper_openreview_id": "ryxB0Rtxx"} -# paragraph_feature = research_arcade.get_node_features_by_id("openreview_paragraphs", paper_id) -# print(paragraph_feature.to_dict(orient="records")[0]) - -# insert_node -# paragraph_feature = {'venue': 'xujj_test', -# 'paper_openreview_id': 'xujj_test', -# 'paragraph_idx': 1, -# 'section': "xujj_test", -# 'content': "xujj_test"} -# research_arcade.insert_node("openreview_paragraphs", node_features=paragraph_feature) - -# delete_node -# paper_id = {"paper_openreview_id": "xujj_test"} -# paragraph_feature = research_arcade.delete_node_by_id("openreview_paragraphs", paper_id) -# print(len(paragraph_feature)) -# print(paragraph_feature.to_dict(orient="records")[0]) - -########## openreview_arxiv ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2017/conference"} -# research_arcade.construct_table_from_api("openreview_arxiv", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_arxiv_example.csv"} -# research_arcade.construct_table_from_csv("openreview_arxiv", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_arxiv_example.json"} -# research_arcade.construct_table_from_json("openreview_arxiv", config) - -# get_all_edge_features -# openreview_arxiv_df = research_arcade.get_all_edge_features("openreview_arxiv") -# print(len(openreview_arxiv_df)) - -# get_neighborhood -# openreview_id = {"paper_openreview_id": "zkNCWtw2fd"} -# openreview_arxiv_df = research_arcade.get_neighborhood("openreview_arxiv", openreview_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) -# arxiv_id = {"arxiv_id": "http://arxiv.org/abs/2408.10536v1"} -# openreview_arxiv_df = research_arcade.get_neighborhood("openreview_arxiv", arxiv_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) - -# delete_edge -# openreview_id = {"paper_openreview_id": "zkNCWtw2fd"} -# openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) -# arxiv_id = {"arxiv_id": "http://arxiv.org/abs/2408.10536v1"} -# openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", arxiv_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) -# openreview_arxiv_id = {"paper_openreview_id": "zkNCWtw2fd", "arxiv_id": "http://arxiv.org/abs/2408.10536v1"} -# openreview_arxiv_df = research_arcade.delete_edge_by_id("openreview_arxiv", openreview_arxiv_id) -# print(openreview_arxiv_df.to_dict(orient="records")[0]) - -# insert_edge -# openreview_arxiv = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': 'zkNCWtw2fd', -# 'arxiv_id': 'http://arxiv.org/abs/2408.10536v1', -# 'title': 'Synergistic Approach for Simultaneous Optimization of Monolingual, Cross-lingual, and Multilingual Information Retrieval' -# } -# research_arcade.insert_edge("openreview_arxiv", openreview_arxiv) - -########## openreview_papers_authors ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_papers_authors", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_authors_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers_authors", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_authors_example.json"} -# research_arcade.construct_table_from_json("openreview_papers_authors", config) - -# get_all_edge_features -# openreview_papers_authors = research_arcade.get_all_edge_features("openreview_papers_authors") -# print(len(openreview_papers_authors)) - -# get_neighborhood -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_authors = research_arcade.get_neighborhood("openreview_papers_authors", paper_id) -# print(openreview_papers_authors.to_dict(orient="records")) -# author_id = {'author_openreview_id': '~Elias_Stengel-Eskin1'} -# openreview_papers_authors = research_arcade.get_neighborhood("openreview_papers_authors", author_id) -# print(openreview_papers_authors.to_dict(orient="records")) - -# delete_edge -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", paper_id) -# print(openreview_papers_authors.to_dict(orient="records")) -# author_id = {'author_openreview_id': '~Elias_Stengel-Eskin1'} -# openreview_papers_authors = research_arcade.delete_edge_by_id("openreview_papers_authors", author_id) -# print(openreview_papers_authors.to_dict(orient="records")) - -# insert_edge -# paper_authors = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Zaid_Khan1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Jaemin_Cho1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Mohit_Bansal2'}] -# for item in paper_authors: -# research_arcade.insert_edge("openreview_papers_authors", item) -# author_papers = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'Xbl6t6zxZs', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'fDcn3S8oAt', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'j9wBgcxa7N', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': 'zd0iX5xBhA', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2024/Conference', 'paper_openreview_id': 'L4nOxziGf9', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2024/Conference', 'paper_openreview_id': 'qL9gogRepu', 'author_openreview_id': '~Elias_Stengel-Eskin1'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'author_openreview_id': '~Elias_Stengel-Eskin1'}] -# for item in author_papers: -# research_arcade.insert_edge("openreview_papers_authors", item) - -########## openreview_papers_reviews ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2017/conference"} -# research_arcade.construct_table_from_api("openreview_papers_reviews", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_reviews_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers_reviews", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_reviews_example.json"} -# research_arcade.construct_table_from_json("openreview_papers_reviews", config) - -# get_all_edge_features -# openreview_papers_reviews = research_arcade.get_all_edge_features("openreview_papers_reviews") -# print(len(openreview_papers_reviews)) - -# get_neighborhood -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_reviews = research_arcade.get_neighborhood("openreview_papers_reviews", paper_id) -# print(openreview_papers_reviews.to_dict(orient="records")) -# review_id = {"review_openreview_id": "13mj0Rtn5W"} -# openreview_papers_reviews = research_arcade.get_neighborhood("openreview_papers_reviews", review_id) -# print(openreview_papers_reviews.to_dict(orient="records")) - -# delete_edge -# paper_review_id = {"paper_openreview_id": "00SnKBGTsz", "review_openreview_id": "13mj0Rtn5W"} -# openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", paper_review_id) -# print(openreview_papers_reviews.to_dict(orient="records")) -# review_id = {"review_openreview_id": "13mj0Rtn5W"} -# openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", review_id) -# print(openreview_papers_reviews.to_dict(orient="records")) -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# openreview_papers_reviews = research_arcade.delete_edge_by_id("openreview_papers_reviews", paper_id) -# print(openreview_papers_reviews.to_dict(orient="records")) - -# insert_edge -# paper_review = {'venue': 'ICLR.cc/2025/Conference', -# 'paper_openreview_id': '00SnKBGTsz', -# 'review_openreview_id': '13mj0Rtn5W', -# 'title': 'Response by Authors', -# 'time': '2024-11-27 17:27:45'} -# research_arcade.insert_edge("openreview_papers_reviews", paper_review) -# paper_reviews = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '7XT4kLWV2f', 'title': 'Official Review by Reviewer_wuGW', 'time': '2024-11-01 14:52:22'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'i3QgWgrJff', 'title': 'Official Review by Reviewer_rVo8', 'time': '2024-11-04 02:37:10'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'GMsjHLXdOx', 'title': 'Official Review by Reviewer_c5nB', 'time': '2024-11-04 09:59:14'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'r8ZflFk3T7', 'title': 'Official Review by Reviewer_VQ9Y', 'time': '2024-11-06 00:15:47'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '4CnQpVCYkF', 'title': 'Response by Authors', 'time': '2024-11-20 22:48:42'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'h1qvpjhRP3', 'title': 'Response by Authors', 'time': '2024-11-20 22:51:07'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'pOR42YNLtU', 'title': 'Response by Authors', 'time': '2024-11-20 22:55:04'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'Aq2tBtB0lt', 'title': 'Response by Authors', 'time': '2024-11-20 22:57:18'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'm1iUqPHpwk', 'title': 'Response by Authors', 'time': '2024-11-20 22:58:29'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '66buacQmRe', 'title': 'Response by Authors', 'time': '2024-11-20 23:02:21'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'Bgr7Ol90m7', 'title': 'Response by Authors', 'time': '2024-11-22 23:11:06'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'H2h2K6a8x5', 'title': 'Response by Reviewer', 'time': '2024-11-23 10:04:58'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'la5jPwJU4g', 'title': 'Response by Authors', 'time': '2024-11-24 19:17:22'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'DjVKsUoFN2', 'title': 'Response by Reviewer', 'time': '2024-11-25 04:00:18'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'C3MhCuKhTf', 'title': 'Response by Authors', 'time': '2024-11-25 19:44:38'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'ZqwAYtcmhv', 'title': 'Response by Authors', 'time': '2024-11-25 19:45:43'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '9OQJoesINr', 'title': 'Response by Reviewer', 'time': '2024-11-25 20:07:51'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'wqTNtVDwef', 'title': 'Response by Authors', 'time': '2024-11-26 03:32:30'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'NEsxOTkkIV', 'title': 'Response by Reviewer', 'time': '2024-11-26 20:00:00'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': '13mj0Rtn5W', 'title': 'Response by Authors', 'time': '2024-11-27 17:27:45'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'hWat8aFBRw', 'title': 'Response by Reviewer', 'time': '2024-11-27 11:34:03'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'wnsiUkDh00', 'title': 'Response by Authors', 'time': '2024-11-27 17:28:35'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'zpboemkkjR', 'title': 'Meta Review of Submission11063 by Area_Chair_eoLd', 'time': '2024-12-20 15:14:25'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'review_openreview_id': 'kokKFEn2fw', 'title': 'Paper Decision', 'time': '2025-01-22 05:35:00'} -# ] -# for item in paper_reviews: -# research_arcade.insert_edge("openreview_papers_reviews", item) - -########## openreview_papers_revisions ########## -# construct_from_api -# config = {"venue": "ICLR.cc/2025/Conference"} -# research_arcade.construct_table_from_api("openreview_papers_revisions", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_papers_revisions_example.csv"} -# research_arcade.construct_table_from_csv("openreview_papers_revisions", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_papers_revisions_example.json"} -# research_arcade.construct_table_from_json("openreview_papers_revisions", config) - -# get_all_edge_features -# openreview_papers_revisions = research_arcade.get_all_edge_features("openreview_papers_revisions") -# print(len(openreview_papers_revisions)) - -# get_neighborhood -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# paper_revision = research_arcade.get_neighborhood("openreview_papers_revisions", paper_id) -# print(paper_revision.to_dict(orient="records")) -# revision_id = {"revision_openreview_id": "dzL3IRBnE4"} -# paper_revision = research_arcade.get_neighborhood("openreview_papers_revisions", revision_id) -# print(paper_revision.to_dict(orient="records")) - -# delete_edge -# paper_revision_id = {"paper_openreview_id": "00SnKBGTsz", "revision_openreview_id": "dzL3IRBnE4"} -# paper_revision = research_arcade.delete_edge_by_id("openreview_papers_revisions", paper_revision_id) -# print(paper_revision.to_dict(orient="records")) -# revision_id = {"revision_openreview_id": "dzL3IRBnE4"} -# paper_revision = research_arcade.delete_edge_by_id("openreview_papers_revisions", revision_id) -# print(paper_revision.to_dict(orient="records")) -# paper_id = {"paper_openreview_id": "00SnKBGTsz"} -# paper_revision = research_arcade.delete_edge_by_id("openreview_papers_revisions", paper_id) -# print(paper_revision.to_dict(orient="records")) - -# insert_edge -# paper_revision = {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'dzL3IRBnE4', 'title': 'Camera_Ready_Revision', 'time': '2025-03-01 03:36:55'} -# research_arcade.insert_edge("openreview_papers_revisions", paper_revision) -# paper_revisions = [{'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'oT4N28siLO', 'title': 'Camera_Ready_Revision', 'time': '2025-03-02 01:35:16'}, -# {'venue': 'ICLR.cc/2025/Conference', 'paper_openreview_id': '00SnKBGTsz', 'revision_openreview_id': 'dzL3IRBnE4', 'title': 'Camera_Ready_Revision', 'time': '2025-03-01 03:36:55'}] -# for item in paper_revisions: -# research_arcade.insert_edge("openreview_papers_revisions", item) - -########## openreview_revisions_reviews ########## -# construct_based_on_existing_tables -# papers_reviews_df = research_arcade.get_all_edge_features("openreview_papers_reviews") -# print(len(papers_reviews_df)) -# papers_revisions_df = research_arcade.get_all_edge_features("openreview_papers_revisions") -# print(len(papers_revisions_df)) -# config = {"papers_reviews_df": papers_reviews_df, "papers_revisions_df": papers_revisions_df} -# research_arcade.construct_table_from_api("openreview_revisions_reviews", config) - -# construct_from_csv -# config = {"csv_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/csv_data/csv_openreview_revisions_reviews_example.csv"} -# research_arcade.construct_table_from_csv("openreview_revisions_reviews", config) - -# construct_from_json -# config = {"json_file": "/home/jingjunx/openreview_benchmark/Code/paper-crawler/examples/json_data/json_openreview_revisions_reviews_example.json"} -# research_arcade.construct_table_from_json("openreview_revisions_reviews", config) - -# get_all_edge_features -# openreview_revisions_reviews = research_arcade.get_all_edge_features("openreview_revisions_reviews") -# print(len(openreview_revisions_reviews)) - -# get_neighborhood -# revision_id = {'revision_openreview_id': 'cX02yuzwWI'} -# revision_review = research_arcade.get_neighborhood("openreview_revisions_reviews", revision_id) -# print(revision_review.to_dict(orient="records")) -# review_id = {'review_openreview_id': 'wumckDPIQ3'} -# revision_review = research_arcade.get_neighborhood("openreview_revisions_reviews", review_id) -# print(revision_review.to_dict(orient="records")) - -# delete_edge -# revision_review_id = {'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'} -# revision_review = research_arcade.delete_edge_by_id("openreview_revisions_reviews", revision_review_id) -# print(revision_review.to_dict(orient="records")) -# review_id = {'review_openreview_id': 'wumckDPIQ3'} -# revision_review = research_arcade.delete_edge_by_id("openreview_revisions_reviews", review_id) -# print(revision_review.to_dict(orient="records")) -# paper_id = {'revision_openreview_id': 'cX02yuzwWI'} -# revision_review = research_arcade.delete_edge_by_id("openreview_revisions_reviews", paper_id) -# print(revision_review.to_dict(orient="records")) - -# insert_edge -# revision_review = {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'} -# research_arcade.insert_edge("openreview_revisions_reviews", revision_review) -# revision_reviews = [{'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wumckDPIQ3'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '138cOdBpgA'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'yKh1fQYnUZ'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'Pvt0OjNSp2'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'MUhlEYyBD9'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '2mqiS3J8wC'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'Er8QTorcyr'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'AvtD9uxRtX'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '2tgxTGynNm'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '5MKJE3sFsd'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'wViZ0H4ErF'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': '0c1It75dTb'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'PFwia9lcjP'}, -# {'venue': 'ICLR.cc/2025/Conference', 'revision_openreview_id': 'cX02yuzwWI', 'review_openreview_id': 'ygCqaGNPee'}] -# for item in revision_reviews: -# research_arcade.insert_edge("openreview_revisions_reviews", item) \ No newline at end of file