Python-wed-scraper/simple_test.py at main · NomadCoder101/Python-wed-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
from bs4 import BeautifulSoup

import os
import functions

import requests


"""

main function
make http request  --> reuqests
use bs4            --> beautifulsoup
store cleaned data -- > funtions
read -data
remove data

"""


def main_scrapper(url,directory):

    functions.create_dir(directory)
    source_code=requests.get(url)
    source_text= source_code.text
    soup=BeautifulSoup(source_text,"html.parser")
   # results=(soup.find_all('div',{'class':'course-listing'})[0].a)
    results=(soup.find_all('h4',{'class':'ql-align-center'}))


   #print(results)

    for result in results:
        #print(result)
        #print()
        #print( result.text)
        title_foramtted= result.text + "\n"
        if functions.does_file_exists(directory+"/articles.txt") is False:
            functions.create_new_file(directory+"/articles.txt")

        functions.write_to_file(directory+"/articles.txt",title_foramtted)


    #print(results.get('href'))

    #

    #print(results.text)


    #print(source_text)

#block__text-wrapper
#b-126170120-top

#main_scrapper("https://calmandcode.teachable.com/p/create-a-laravel-social-network","calmandcode")

#functions.read_file("calmandcode/articles.txt")

functions.read_lines("calmandcode/articles.txt",4)