github-issues/load.py at main · blackbearman/github-issues · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import re
import os
import sys
import json

import requests

def fetch_issue_list(owner, repo, list_dir, token=None):
    issues = []
    i = 0
    url = f"https://api.github.com/repos/{owner}/{repo}/issues?state=all"
    headers = {"Accept": "application/vnd.github.full+json"}
    if token:
        headers["Authorization"] = f"token {token}"

    r = requests.get(url, headers=headers)
    while r.status_code == 200:
        data = r.json()
        with open(list_dir + f"issues{i}.json", "w") as f:
            json.dump(data, f)

        issues += [a["url"] for a in data if not 'pull_request' in a]
        if 'next' in r.links:
            next_url = r.links['next']['url']
            print(f"Next page URL: {next_url}")
            r = requests.get(next_url)
            i+=1
        else:
            break

    return issues

def fetch_issue_comments(issue_url, comments_dir, token=None):
    i = 0
    id = issue_url.split('/')[-1]
    url = f"{issue_url}/comments"
    headers = {"Accept": "application/vnd.github.full+json"}
    if token:
        headers["Authorization"] = f"token {token}"

    r = requests.get(url, headers=headers)
    while r.status_code == 200:
        data = r.json()
        with open(comments_dir + f"issue{id}-c{i}.json", "w") as f:
            json.dump(data, f)

        if 'next' in r.links:
            next_url = r.links['next']['url']
            print(f"Next page URL: {next_url}")
            r = requests.get(next_url)
            i+=1
        else:
            break

    return i

if __name__ == "__main__":

    output_dir = 'output/'

    if len(sys.argv) < 3:
        print("Usage: python3 load.py <owner> <repo> [load] [convert] [github_token]")
        print("Example: python3 load.py blackbearman github-issues False True")
    else:
        owner = sys.argv[1]
        repo = sys.argv[2]
        load = sys.argv[3].lower() == 'true' if len(sys.argv) > 3 else False
        convert = sys.argv[4].lower() == 'true' if len(sys.argv) > 4 else True
        token = sys.argv[5] if len(sys.argv) > 5 else None

    lists_dir = output_dir + f'{repo}/lists/'
    comments_dir = output_dir + f'{repo}/comments/'

    if load:
        print(f"Load issues from {owner}/{repo}")
        issue_count = 0
        os.makedirs(lists_dir, exist_ok=True)
        issues = fetch_issue_list(owner, repo, lists_dir, token)

        issue_count = len(issues)
        print(issues)
        print(str(issue_count) + " issues found")

        os.makedirs(comments_dir, exist_ok=True)
        pages = 0
        for issue in issues:
            pages += fetch_issue_comments(issue, comments_dir, token)

        print(str(pages) + " pages loaded")

    if convert:
        print(f"Convert issues from {owner}/{repo} to Markdown")
        rows = []
        lists = [f for f in sorted(os.listdir(lists_dir)) if os.path.isfile(lists_dir + f)]
        comments = [f for f in sorted(os.listdir(comments_dir)) if os.path.isfile(comments_dir + f)]
        for l in lists:
            data = []
            with open(lists_dir + l, "r") as f:
                data = json.load(f)
            for s in data:
                # check issue or pull request
                if not "pull_request" in s:
                    rows.append(f"# {s['number']}  {s['title']}\n")
                    labels = ','.join([l['name'] for l in s['labels']])
                    rows.append(f"*State: {s['state']}. Labels:[{labels}]*\n\n")
                    rows.append(f"*{s['user']['login']} created at {s['created_at']}*\n\n")
                    body = s["body"]
                    if body:
                        rows.append(f"{body}\n\n")

                    # load comments
                    i = 0
                    cname = f"issue{s['number']}-c{i}.json"
                    while cname in comments:
                        comment = []
                        with open(comments_dir + cname, "r") as f:
                            comment = json.load(f)
                        for c in comment:
                            rows.append(f"*{c['user']['login']} commented at {c['created_at']}*\n\n")
                            body = c["body"]
                            if body:
                                rows.append(f"{body}\n\n")
                        i += 1
                        cname = f"issue{s['number']}-c{i}.json"
                    if s['state'] == "closed":
                        rows.append(f"*{s['closed_by']['login']} closed at {s['closed_at']}*\n\n")

        with open(output_dir + f"{repo}/issues.md", "w") as f:
            f.writelines(rows)