ParsToTg/scraper.py at master · Shaidyk/ParsToTg · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import json
import requests

from bs4 import BeautifulSoup

from config import login_data, user_agent


class ParserTesmania:  # noqa
    login_link = 'https://www.tesmanian.com/account'
    link = 'https://www.tesmanian.com/'
    headers = {
        'user_agent': user_agent
    }
    spacex_dict = {}
    tesla_dict = {}

    def __init__(self):
        self.session = requests.Session()
        self.login()
        self.div_id_spacex = 'shopify-section-1581705557561'
        self.div_id_tesla = 'shopify-section-1581706887820'
        self.response = self.get_response()
        self.soup = self.get_soup()

    def login(self):
        self.session.post(self.login_link, data=login_data, headers=self.headers)

    def get_response(self):
        self.response = self.session.get(self.link, headers=self.headers).text
        return self.response

    def get_soup(self):
        self.soup = BeautifulSoup(self.response, 'lxml')
        return self.soup

    def get_spacex(self):
        try:
            with open('spacex_article.json') as file:
                self.spacex_dict = json.load(file)
        except FileNotFoundError:
            with open('spacex_article.json', 'w') as file:
                json.dump(self.spacex_dict, file, indent=4, ensure_ascii=False)

        block = self.soup.find('div', id=self.div_id_spacex)
        articles = block.find_all('h3', class_='sub_title')

        new_articles = {}
        for article in articles:
            article_title = article.find('a').text
            article_url = 'https://www.tesmanian.com/' + article.find('a').get('href')
            article_id = article_url.split('/')[-1]

            if article_id in self.spacex_dict:
                continue
            else:
                self.spacex_dict[article_id] = {
                    'article_title': article_title,
                    'article_url': article_url
                }

                new_articles[article_id] = {
                    'article_title': article_title,
                    'article_url': article_url
                }

        with open('spacex_article.json', 'w') as file:
            json.dump(self.spacex_dict, file, indent=4, ensure_ascii=False)

        return new_articles

    def get_tesla(self):
        try:
            with open('tesla_article.json') as file:
                self.tesla_dict = json.load(file)
        except FileNotFoundError:
            with open('tesla_article.json', 'w') as file:
                json.dump(self.tesla_dict, file, indent=4, ensure_ascii=False)

        block = self.soup.find('div', id=self.div_id_tesla)
        articles = block.find_all('h3', class_='sub_title')

        new_articles = {}
        for article in articles:
            article_title = article.find('a').text
            article_url = 'https://www.tesmanian.com/' + article.find('a').get('href')
            article_id = article_url.split('/')[-1]

            if article_id in self.tesla_dict:
                continue
            else:
                self.tesla_dict[article_id] = {
                    'article_title': article_title,
                    'article_url': article_url
                }

                new_articles[article_id] = {
                    'article_title': article_title,
                    'article_url': article_url
                }

        with open('tesla_article.json', 'w') as file:
            json.dump(self.tesla_dict, file, indent=4, ensure_ascii=False)

        return new_articles