selenium-xpath-python/main.py at master · fardinho21/selenium-xpath-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
from BaseScraper import BaseScraper
from XPathScraper import XPathScraper
from BeautifulSoupScraper import BeautifulSoupScraper
def main():

    # loadbybutton="https://webscraper.io/test-sites/e-commerce/more/phones/touch"
    # loadbyscrolling="https://webscraper.io/test-sites/e-commerce/scroll/phones/touch"
    loadbypagination="https://webscraper.io/test-sites/e-commerce/static/phones/touch"
    # print('Scraping Dynamic Content - Load Button')
    # xPathScraper = XPathScraper(url=loadbybutton)
    # xPathScraper.scrapeXPATH_DynamicLoadButton(\
    #     xPathOfPresenceElement="//div[contains(@class, 'thumbnail')]",\
    #         xPathOfLoadMoreElement="//div/a[text()='More']",\
    #             xPathOfElementToScrape=".//h4[@class='price float-end pull-right']")

    # print('Scraping Dynamic Content - Scroll to Load')
    # xPathScraper = XPathScraper(url=loadbyscrolling)
    # xPathScraper.scrapeXPATH_DynamicScroll(\
    #     xPathOfPresenceElement="//div[contains(@class, 'thumbnail')]",\
    #         xPathOfElementToScrape=".//h4[@class='price float-end pull-right']")

    # print('Scraping Dynamic Content - Pagination')
    # xPathScraper = XPathScraper(url=loadbypagination)
    # xPathScraper.scrapeXPATH_DynamicPagination(\
    #     xPathOfPresenceElement="//div[contains(@class, 'thumbnail')]",\
    #         xPathOfPaginationElement="//div[@id='static-pagination']/nav/ul[@class='pagination']/li/a[@rel='next']",\
    #             xPathOfElementToScrape="//div[contains(@class, 'thumbnail')]",\
    #                 xPathOther=["//h4[@class='price float-end card-title pull-right']"])

    # bbcShopPagination="https://shop.bbc.com/collections/merchandise"
    # print('Scraping Dynamic Content - Pagination')
    # xPathScraper=XPathScraper(url=bbcShopPagination)
    # xPathScraper.scrapeXPATH_DynamicPagination(\
    #     xPathOfPresenceElement="//div[@id='bc-sf-filter-products']", \
    #         xPathOfPaginationElement="//div[@id='bc-sf-filter-bottom-pagination']/ul/li/a[text()='→']", \
    #             xPathOfElementToScrape="//div[contains(@class,'bc-sf-filter-product-item-inner')]",\
    #                 xPathOther=["//a[@class='bc-sf-filter-product-item-title']","//p[@class='bc-sf-filter-product-item-price']/span[@class='bc-sf-filter-product-item-sale-price']"])
    # xPathScraper._quitDriver()

    # print('Scraping Dynamic Content - Load Button')
    # bsScraper = BeautifulSoupScraper(url=loadbybutton)

    # elementsToScrape:dict[str,str]={"h4":"price float-end card-title pull-right", "a":"title"}
    # bsScraper.scrapeBeautifulSoup_DynamicLoadButton(\
    #     presenceElementCSS="a.ecomerce-items-scroll-more",\
    #         elementsToScrape=elementsToScrape)

    # print('Scraping Dynamic Content - Load by Scroll')
    # bsScraper = BeautifulSoupScraper(url=loadbyscrolling)
    # elementsToScrape:dict[str,str]={"h4":"price float-end card-title pull-right", "a":"title"}

    # bsScraper.scrapeBeautifulSoup_DynamicScroll(presenceElementCSS="div.row.ecomerce-items.ecomerce-items-scroll",\
    #     elementsToScrape=elementsToScrape)
    print('Scraping Dynamic Content - Load by Pagination')
    bsScraper = BeautifulSoupScraper(url=loadbypagination)
    elementsToScrape:dict[str,str]={"h4":"price float-end card-title pull-right", "a":"title"}

    bsScraper.scrapeBeautifulSoup_DynamicPagination(presenceElementCSS="a[rel='next']",\
        elementsToScrape=elementsToScrape)

    bsScraper._quitDriver()


if __name__ == "__main__":
    main()