scraping/scrape.py at master · chellarao-chowdary/scraping · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
import requests
from bs4 import BeautifulSoup
import pprint

res=requests.get("https://news.ycombinator.com/news")
res2=requests.get('https://news.ycombinator.com/news?p=2')
soup = BeautifulSoup(res.text,'html.parser')
soup2 = BeautifulSoup(res2.text,'html.parser')
links2=soup2.select('.storylink')
subtext2=soup2.select('.subtext')
links=soup.select('.storylink')
subtext=soup.select('.subtext')

mega_links=links + links2
mega_subtext=subtext + subtext2

def create_custom_hn(links,subtext):
    hn=[]
    for idx,item in enumerate(links):
        title=item.getText()
        href=item.get('href',None)
        vote=subtext[idx].select('.score')
        if len(vote):
            points=int(vote[0].getText().replace(' points',''))
            if points>99:
                hn.append({'title':title,'link':href,'votes':points})
    return sort_story_by_votes(hn)

def sort_story_by_votes(hnlist):
    return sorted(hnlist,key=lambda k:k['votes'],reverse=True)


data=create_custom_hn(mega_links,mega_subtext)

fh=open('E:\\self notes\\data.txt', mode='w')
for i in data:
    fh.write(str(i) + "\n")

fh.close()