Batch_Scrapper Enhancement Sugestion

[rickkk856]

First, I'd like to say THANKS! this project is awesome!

But I want to make a recomendation for future feature enhancement. Imagine I want to do a Search, like "brazilian housing" or something like that, and download everything from the search... it will return me an URL like this https://www.archdaily.com.br/search/br/projects/categories/casas/country/brasil?page=2 , then I could get the project ID from html at data-insights-value="projects/880032" , now comes the part that I don't know how to do... get the 880032 project ID and make it into a list. It's important to make sure chromedriver scrolls down to get all the IDs . I could make it using crome's HTML inspector, and Notepad++

So I made a csv file like this

codes
880032
961250
869474
880757
886066

Turned the csv file into a list

from pandas import *
codes = []
data = read_csv("./project_code.csv")
codes = data['codes'].tolist()

print(codes)

Used the list to itinerate between all project IDs
While scrapping I discovered that it works better if you pause for a while
so I implemented a 2minute break for every 10 scrapped links.

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC 
from selenium.common.exceptions import TimeoutException
import os
import requests
import time
import re
from bs4 import BeautifulSoup
import json

chromedriver = "./driver/chromedriver.exe"
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
actions = ActionChains(driver)

base_url = 'https://www.archdaily.com.br/'
country = "br/"

#project_link = base_url + input('Project Code : ')

#every 10 links will make a stop of 2 minute
break_split = []
breaks = 0
for i in range(1000):
    breaks += 10
    break_split.append(breaks)

counter = 1
for x in codes:
    project_link = base_url + country + x
    
    if counter in break_split:
        time.sleep(120) #time to sleep
        if not counter:
              pass
   
    print("Downloading images from project number: ", counter , " " , project_link)
    def project_scraper(driver , project_link):
                response = requests.get(project_link)
                soup = BeautifulSoup(response.content , 'html.parser')
                paragraph = soup.find_all('p')
                list_of_paragraph = []
                for each_paragpraph in paragraph:
                    list_of_paragraph.append(each_paragpraph.text.strip())
                driver.get(project_link)

                timeout = 20
                try:
                    WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.XPATH, '//*[contains(concat( " ", @class, " " ), concat( " ", "js-char-list", " " ))]')))
                except TimeoutException:
                    print('Timed out waiting for page to load')
                    driver.quit()
                try:    
                    area =  driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__item", " " )) and (((count(preceding-sibling::*) + 1) = 2) and parent::*)]')[0].text.split(':')[1].strip()
                    year =  driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__item", " " )) and (((count(preceding-sibling::*) + 1) = 3) and parent::*)]')[0].text.split(':')[1].strip()
                    photographs = driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__item", " " )) and (((count(preceding-sibling::*) + 1) = 4) and parent::*)]')[0].text.split(':')[1].strip()
                except:
                    area = ''
                    year = ''
                    photographs = ''   
                project_dict =  {
                        'Project ID': project_link[26:32],
                        'Project Titel' :driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-title-big--bmargin-big", " " ))]')[0].text.split('/')[0].strip(),
                        'Project Type': driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__header-category", " " ))]//a')[0].text.strip(),
                        'City': driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__header-location", " " ))]')[0].text.split(',')[0].strip(),
                        'Country': driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__header-location", " " ))]//a')[0].text.strip(),
                        'Architects': driver.find_elements_by_xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "afd-specs__item", " " )) and (((count(preceding-sibling::*) + 1) = 1) and parent::*)]')[0].text.split(':')[1].strip(),
                        'Area': area,
                        'Year': year,
                        'Photographs': photographs,
                        'Project URL': project_link.strip(),
                        'Text Content': list_of_paragraph[7:-2]
                    
                } 
                project_js = json.dumps(project_dict, indent=4 , ensure_ascii=False).encode('utf8')
                with open('projct_details.json', 'w' , encoding='utf-8') as file:
                    file.write(project_js.decode() + ',')
                print('Project details are ready.')

    def project_image_downloader(driver ,project_link):
        response = requests.get(project_link)
        soup = BeautifulSoup(response.content , 'html.parser')
        a_tag = soup.find('a' , attrs={'class':'gallery-link afd-desktop-e'})
        link = base_url + a_tag.get('href')
        driver.get(link)

        timeout = 18
        try:
            WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.XPATH, '//*[contains(concat( " ", @class, " " ), concat( " ", "js-gal-mob-img-onview", " " ))]')))
        except TimeoutException:
            print('Timed out waiting for page to load')
            driver.quit()

        this_table = driver.find_elements_by_id('gallery-items')
        for this in this_table:
            temp = this.get_attribute('data-images')

        large_urls = []

        urls = re.findall(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', temp)
        for url in urls:
            if 'large_jpg' in (url):
                large_urls.append(url)
        i = 1
        for each in large_urls:
            var = f'{x}-ID-{i}'
            with open('{}.jpg'.format(var), 'wb') as handle:
                    response = requests.get(each, stream=True)
                    #print('Picture number {} downloaded'.format(i))
                    time.sleep(2)

                    if not response.ok:
                        print(response)

                    for block in response.iter_content(1024):
                        if not block:
                            break

                        handle.write(block)
            i += 1

    counter += 1
    #Select only what you want to scrape (text or image)
    if __name__ == "__main__":
        project_scraper(driver , project_link)
        project_image_downloader(driver ,project_link)

Terminal output

Downloading images from project number:  1   https://www.archdaily.com.br/br/880032
Downloading images from project number:  2   https://www.archdaily.com.br/br/961250
Downloading images from project number:  3   https://www.archdaily.com.br/br/869474
Downloading images from project number:  4   https://www.archdaily.com.br/br/880757
Downloading images from project number:  5   https://www.archdaily.com.br/br/886066

PS: Most of the images will come 2000x1333 and size will be bigger than 2...3...5....9mb
The solution I had to keep data-size low while still scrapping is using imagemagick

import os

directory = "/Users/Pichau/ArchdailyProjectDownloader/DOWNLOADS"

infinite = 0
while infinite < 1:

    for filename in os.listdir(directory):
        Size1 = os.path.getsize("./"+filename)        
        if Size1 > 800000:
            !magick convert {filename} -define jpeg:extent=600kb {filename}
            Size2 = os.path.getsize("./"+filename)
            print("Converted image:", filename , "of Size" , Size1 , "--->", Size2)

this will keep all images between 600 and 800kb each.