HakuEngine/main.cpp at master · germinolegrand/HakuEngine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
#include <iostream>
#include <system_error>
#include <thread>

#include "DownloadResource.h"
#include "AnalyseResource.h"
#include "PersistResults.h"

int main(int argc, char* argv[])
{
    std::clog << "Hello Miyazaki's world!" << std::endl;

    DatabaseSession dbsession(argc >= 2 ? argv[1] : "user=postgres");

    Analyzer analyser;
    analyser.setAnalyser("ftp/file", analyse_ftp_file);

    TextHTMLAnalyser html_an;

    html_an.addAnalyser(generate_skip_tag(GUMBO_TAG_STYLE));
    html_an.addAnalyser(generate_skip_tag(GUMBO_TAG_SCRIPT));
    html_an.addAnalyser(search_for_text);
    html_an.addAnalyser(search_for_links);
    html_an.addAnalyser(search_for_words);
    html_an.addAnalyser(search_for_img);

    analyser.setAnalyser("text/html", html_an);

    while(true)
    {
        std::vector<ToBeCrawled> crontab = dbsession.getCrontab();

        if(crontab.empty())
        {
            std::this_thread::sleep_for(std::chrono::milliseconds(30000));
            continue;
        }

        for(auto& tobecrawled : crontab)
        {
            try
            {
                WebRessource webres = downloadFromURL(tobecrawled.url);

                AnalyseResults results = analyser(webres);

                std::cout << results.full_text << std::endl;

                if(tobecrawled.url == webres.url)
                {
                    dbsession.persist(tobecrawled, results);
                }
                else
                {
                    std::swap(webres.url, tobecrawled.url);
                    dbsession.persist_redirected(webres.url, tobecrawled, results);
                }
            }
            catch(UnknownProtocol const& e)
            {
                std::cerr << "\n" << tobecrawled.url << " : Unknown protocol " << e.protocol << std::endl;
                dbsession.persist_error(tobecrawled.url);
            }
            catch(ImpossibleAccess const& e)
            {
                std::cerr << "\n" << tobecrawled.url << " : Can't access " << e.uri << " with error code " << e.error_code << std::endl;
                dbsession.persist_error(tobecrawled.url);
            }
            catch(UnknownContentType const& e)
            {
                std::cerr << "\n" << tobecrawled.url << " : Unknown content type " << e.content_type << std::endl;
                dbsession.persist_error(tobecrawled.url);
            }
            catch(pqxx::sql_error const&e)
            {
                std::cerr << "\n" << tobecrawled.url << e.what() << std::endl;
            }
        }
    }

    return 0;
}