Structured-Web-Data-Crawler/spider.py at master · robin98sun/Structured-Web-Data-Crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
from urllib.request import urlopen
from urllib.request import Request
from urllib.error import HTTPError
from content_extractor.page_parser import PageParser
from content_extractor.storage import StorageWrapper
from content_extractor.content_target import ContentTarget
from domain import *
from general import *
import sys;
import random
import time

class Spider:

    project_name = ''
    base_url = ''
    domain_name = ''
    queue_file = ''
    crawled_file = ''
    queue = set()
    crawled = set()

    def __init__(self, project_name, base_url, domain_name, user_agent = None, white_list = None, target_definition = None, sleep = None):
        Spider.project_name = project_name
        Spider.base_url = base_url
        Spider.domain_name = domain_name
        Spider.queue_file = Spider.project_name + '/queue.txt'
        Spider.crawled_file = Spider.project_name + '/crawled.txt'
        Spider.failed_file = Spider.project_name + '/failed.txt'
        Spider.white_list = white_list;
        Spider.target_definition = target_definition;
        Spider.user_agent = user_agent;

        Spider.sleep = None;
        if sleep is not None:
            Spider.sleep = int(sleep);

        Spider.targets = [];
        if target_definition is not None:
            for target_def in target_definition:
                t = ContentTarget(base_url, target_def, storage_base_path = project_name);
                Spider.targets.append(t);

        self.boot()
        self.crawl_page('First spider', Spider.base_url)

    # Creates directory and files for project on first run and starts the spider
    @staticmethod
    def boot():
        create_project_dir(Spider.project_name)
        create_data_files(Spider.project_name, Spider.base_url)
        Spider.queue = file_to_set(Spider.queue_file)
        Spider.crawled = file_to_set(Spider.crawled_file)
        Spider.failed = file_to_set(Spider.failed_file)

    # Updates user display, fills queue and updates files
    @staticmethod
    def crawl_page(thread_name, page_url):
        if page_url not in Spider.crawled:
            print(thread_name + ' now crawling ' + page_url)
            print('Queue ' + str(len(Spider.queue)) + ' | Crawled  ' + str(len(Spider.crawled)))
            (code, links) = Spider.gather_links(page_url);
            if code == 200:
                Spider.add_links_to_queue(links);
                if page_url in Spider.queue:
                    Spider.queue.remove(page_url)
                    Spider.crawled.add(page_url)
                    Spider.update_files()
            else:
                if page_url in Spider.queue:
                    Spider.queue.remove(page_url);
                    Spider.failed.add(page_url)
                    Spider.update_files();

            if Spider.sleep is not None:
                time.sleep(random.random() * Spider.sleep + Spider.sleep);

    # Converts raw response data into readable information and checks for proper html formatting
    @staticmethod
    def gather_links(page_url):
        if type(page_url) != str or len(page_url) == 0:
            return set();

        html_string = ''
        response = None;
        try:
            response = None;
            if Spider.user_agent is not None and len(Spider.user_agent) > 0:
                user_agent = Spider.user_agent;
                if type(user_agent) == list:
                    user_agent = user_agent[round((len(user_agent)-1)*random.random())];

                req = Request(page_url,
                    data = None,
                    headers = {
                        'User-Agent': user_agent
                    });
                response = urlopen(req);
            else:
                response = urlopen(page_url);

            if 'text/html' in response.getheader('Content-Type'):
                html_bytes = response.read()
                html_string = html_bytes.decode("utf-8")
            parser = PageParser(Spider.base_url, page_url, Spider.white_list);
            parser.add_targets(Spider.targets);
            parser.feed(html_string)
        except HTTPError as e:
            # raise HTTPError(req.full_url, code, msg, hdrs, fp)
            print('ERROR when requesting url: [', page_url, '], error code:', e.code, ' error message: [', e.msg, '], http headers:', e.hdrs);
            # raise e;
            return (e.code, set())
        return (200, parser.page_links())
        # return set();

    # Saves queue data to project files
    @staticmethod
    def add_links_to_queue(links):
        for url in links:
            if (url in Spider.queue) or (url in Spider.crawled):
                continue
            if Spider.domain_name != get_domain_name(url):
                continue
            Spider.queue.add(url)

    @staticmethod
    def update_files():
        set_to_file(Spider.queue, Spider.queue_file)
        set_to_file(Spider.crawled, Spider.crawled_file)
        set_to_file(Spider.failed, Spider.failed_file)