Python-wed-scraper/Blog_search.py at main · NomadCoder101/Python-wed-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
from bs4 import BeautifulSoup

import os
import functions

import requests


"""

main function
make http request  --> reuqests
use bs4            --> beautifulsoup
store cleaned data -- > funtions
read -data
remove data

"""


def main_scrapper(url,directory):

    functions.create_dir(directory)
    source_code=requests.get(url)
    source_text= source_code.text
    soup=BeautifulSoup(source_text,"html.parser")
    articles=(soup.find_all('article',{'class':'blog-post'}))


    for article in articles:
        print("url:" + article.a.get('href'))
        #print("title:" + article.a.text)
        print("title:" + article.a.get('title'))
        print()
        article_foramtted= "url:" + article.a.get('href') + "\n\n" + "title:" +  article.a.get('title') + "\n"
        if functions.does_file_exists(directory+"/articles.txt") is False:
            functions.create_new_file(directory+"/articles.txt")

        functions.write_to_file(directory+"/articles.txt", article_foramtted)
        get_details(article.a.get('href'))


def get_details(url):

    source_code=requests.get(url)
    source_text= source_code.text
    soup=BeautifulSoup(source_text,"html.parser")
    div_entry =soup.find('div',{'class':'entry'})
    soup=BeautifulSoup(str(div_entry),"html.parser")
    paragraphs=soup.find_all('p')
    print()
    print("Paragraphs")
    functions.write_to_file("DailyCoffeNews/articles.txt","Paragraphs :\n\n")

    for p in paragraphs:
        if p.string is not None:
            if "coffee" in p.string:
                print("Found it")
                print(p.string)
                functions.write_to_file("DailyCoffeNews/articles.txt", p.string)

    print("------------------------------------------------------------------")
    print("------------------------------------------------------------------")
    print("------------------------------------------------------------------")
    functions.write_to_file("DailyCoffeNews/articles.txt","---------------\n\n")


main_scrapper("https://dailycoffeenews.com/","DailyCoffeNews")

#functions.read_lines("DailyCoffeNews/articles.txt",14)