From 12d1e27dea08ec250623da881096e6c36aa44450 Mon Sep 17 00:00:00 2001 From: airwave1981 Date: Tue, 9 Nov 2021 12:25:49 +0100 Subject: [PATCH 1/6] Align to master and changes to captcha and delivery options --- kleinanzeigen.py | 126 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 35 deletions(-) diff --git a/kleinanzeigen.py b/kleinanzeigen.py index 50fc48f..58baea4 100755 --- a/kleinanzeigen.py +++ b/kleinanzeigen.py @@ -25,9 +25,9 @@ from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys -from selenium.webdriver.firefox.options import Options from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import Select, WebDriverWait +from selenium_stealth import stealth log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -59,15 +59,28 @@ def login(config): input_email = config['glob_username'] input_pw = config['glob_password'] log.info("Login with account email: " + input_email) - driver.get('https://www.ebay-kleinanzeigen.de/m-einloggen.html') - + + driver.get('https://www.ebay-kleinanzeigen.de') + # wait for the 'accept cookie' banner to appear WebDriverWait(driver, 6).until(EC.element_to_be_clickable((By.ID, 'gdpr-banner-accept'))).click() - + + fake_wait(2000) + + driver.get('https://www.ebay-kleinanzeigen.de/m-einloggen.html') + + fake_wait(2000) + text_area = WebDriverWait(driver, 1).until(EC.presence_of_element_located((By.ID, 'login-email'))) text_area.send_keys(input_email) fake_wait(200) + has_captcha = login_has_captcha(driver) + if has_captcha: + log.info("\t*** Manual captcha input needed! ***") + log.info("\tFill out captcha but DON'T submit. After that press Enter here to continue ...") + wait_key() + text_area = driver.find_element_by_id('login-password') text_area.send_keys(input_pw) fake_wait(200) @@ -95,26 +108,38 @@ def delete_ad(driver, ad): if "id" in ad: try: - ad_id_elem = driver.find_element_by_xpath("//a[@data-adid='%s']" % ad["id"]) + ad_id_elem = driver.find_element_by_xpath("//li[@data-adid='%s']" % ad["id"]) except NoSuchElementException: log.info("\tNot found by ID") + try: + next_page = driver.find_element_by_class_name("Pagination--next") + next_page.click() + fake_wait() + ad_id_elem = driver.find_element_by_xpath("//li[@data-adid='%s']" % ad["id"]) + except NoSuchElementException: + log.info("\tNot found by ID on second page") if ad_id_elem is None: try: - ad_id_elem = driver.find_element_by_xpath("//a[contains(text(), '%s')]/../../../../.." % ad["title"]) + ad_id_elem = driver.find_element_by_xpath("//article[.//a[contains(text(), '%s')]]" % ad["title"]) except NoSuchElementException: log.info("\tNot found by title") if ad_id_elem is not None: try: - btn_del = ad_id_elem.find_elements_by_class_name("managead-listitem-action-delete")[1] + btn_del = ad_id_elem.find_element_by_class_name("managead-listitem-action-delete") btn_del.click() fake_wait() - - btn_confirm_del = driver.find_element_by_id("modal-bulk-delete-ad-sbmt") + + toogle_delete_reason = driver.find_element_by_id("DeleteWithoutReason") + toogle_delete_reason.click() + + fake_wait() + + btn_confirm_del = driver.find_element_by_id("sold-celebration-sbmt") btn_confirm_del.click() - + log.info("\tAd deleted") fake_wait(randint(2000, 3000)) webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform() @@ -152,6 +177,19 @@ def wait_key(): return result +def login_has_captcha(driver): + has_captcha = False + + try: + captcha_field = driver.find_element_by_xpath('//*[@id="login-recaptcha"]') + if captcha_field: + has_captcha = True + except NoSuchElementException: + pass + + log.info(f"Captcha: {has_captcha}") + + return has_captcha def post_ad_has_captcha(driver): has_captcha = False @@ -186,7 +224,7 @@ def post_ad_is_allowed(driver): def post_ad(driver, ad, interactive): - log.info("\tPublishing ad '...") + log.info("\tPublishing ad ...") if config['glob_phone_number'] is None: config['glob_phone_number'] = '' @@ -265,7 +303,7 @@ def post_ad(driver, ad, interactive): if ad.get("description_file", None) is not None: description_file = ad.get("description_file") - with open(description_file, "r") as f: + with open(description_file, "r", encoding="utf-8") as f: description_lines = f.readlines() else: desc = ad.get("desc") @@ -277,7 +315,6 @@ def post_ad(driver, ad, interactive): for p in description_lines: text_area.send_keys(p) - text_area.send_keys(Keys.RETURN) fake_wait() @@ -297,7 +334,13 @@ def post_ad(driver, ad, interactive): text_area = driver.find_element_by_id('pstad-price') if ad["price_type"] != 'GIVE_AWAY': text_area.send_keys(ad["price"]) - price = driver.find_element_by_xpath("//input[@name='priceType' and @value='%s']" % ad["price_type"]) + try: + price = driver.find_element_by_xpath("//input[@name='priceType' and @value='%s']" % ad["price_type"]) + except NoSuchElementException: + try: + price = driver.find_element_by_xpath("//select[@name='priceType']/option[@value='%s']" % ad["price_type"]) + except NoSuchElementException: + raise Exception('Cannot find price type selection!') price.click() fake_wait() @@ -332,16 +375,16 @@ def post_ad(driver, ad, interactive): fileup = driver.find_element_by_xpath("//input[@type='file']") for path in ad["photofiles"]: path_abs = config["glob_photo_path"] + path - uploaded_count = len(driver.find_elements_by_class_name("imagebox-thumbnail")) + uploaded_count = len(driver.find_elements_by_class_name("imagebox-new-thumbnail")) log.debug("\tUploading image: %s" % path_abs) fileup.send_keys(os.path.abspath(path_abs)) total_upload_time = 0 - while uploaded_count == len(driver.find_elements_by_class_name("imagebox-thumbnail")) and \ + while uploaded_count == len(driver.find_elements_by_class_name("imagebox-new-thumbnail")) and \ total_upload_time < 30: fake_wait(500) total_upload_time += 0.5 - if uploaded_count == len(driver.find_elements_by_class_name("imagebox-thumbnail")): + if uploaded_count == len(driver.find_elements_by_class_name("imagebox-new-thumbnail")): log.warning("\tCould not upload image: %s within %s seconds" % (path_abs, total_upload_time)) else: log.debug("\tUploaded file in %s seconds" % total_upload_time) @@ -360,16 +403,16 @@ def post_ad(driver, ad, interactive): if not filename.lower().endswith((".jpg", ".jpeg", ".png", ".gif")): continue file_path_abs = path_abs + filename - uploaded_count = len(driver.find_elements_by_class_name("imagebox-thumbnail")) + uploaded_count = len(driver.find_elements_by_class_name("imagebox-new-thumbnail")) log.debug("\tUploading image: %s" % file_path_abs) fileup.send_keys(os.path.abspath(file_path_abs)) total_upload_time = 0 - while uploaded_count == len(driver.find_elements_by_class_name("imagebox-thumbnail")) and \ - total_upload_time < 30: - fake_wait(500) - total_upload_time += 0.5 + while uploaded_count == len(driver.find_elements_by_class_name("imagebox-new-thumbnail")) and \ + total_upload_time < 60: + fake_wait(1000) + total_upload_time += 1 - if uploaded_count == len(driver.find_elements_by_class_name("imagebox-thumbnail")): + if uploaded_count == len(driver.find_elements_by_class_name("imagebox-new-thumbnail")): log.warning("\tCould not upload image: %s within %s seconds" % (file_path_abs, total_upload_time)) else: log.debug("\tUploaded file in %s seconds" % total_upload_time) @@ -378,12 +421,6 @@ def post_ad(driver, ad, interactive): fake_wait() - submit_button = driver.find_element_by_id('pstad-frmprview') - if submit_button: - submit_button.click() - - fake_wait() - has_captcha = post_ad_has_captcha(driver) if has_captcha: if interactive: @@ -394,6 +431,12 @@ def post_ad(driver, ad, interactive): log.info("\tCaptcha input needed, but running in non-interactive mode! Skipping ...") fRc = False + submit_button = driver.find_element_by_id('pstad-frmprview') + if submit_button: + submit_button.click() + + fake_wait() + if fRc: try: submit_button = driver.find_element_by_id('prview-btn-post') @@ -424,17 +467,28 @@ def post_ad(driver, ad, interactive): def session_create(config): log.info("Creating session") - options = Options() + options = webdriver.ChromeOptions() if config.get('headless', False) is True: log.info("Headless mode") options.add_argument("--headless") - if config.get('webdriver_enabled') is False: - options.set_preference("dom.webdriver.enabled", False) + if os.path.isfile("./chrome-win/chrome.exe"): + log.info("Found ./chrome-win/chrome.exe") + options.binary_location = "./chrome-win/chrome.exe" + + #options.add_extension(r'E:\Google Drive\Verkauf\_auto\ebayKleinanzeigen\chrome-extensions\crobot.crx') - driver = webdriver.Firefox(options=options) + driver = webdriver.Chrome(options=options) + stealth(driver, + languages=["en-US", "en"], + vendor="Google Inc.", + platform="Win32", + webgl_vendor="Intel Inc.", + renderer="Intel Iris OpenGL Engine", + fix_hairline=True, + ) log.info("New session is: %s %s" % (driver.session_id, driver.command_executor._url)) return driver @@ -521,7 +575,9 @@ def signal_handler(sig, frame): if fNeedsUpdate or fForceUpdate: - delete_ad(driver, ad) + # delete ad if it was published already + if "id" in ad or "date_published" in ad: + delete_ad(driver, ad) fPosted = post_ad(driver, ad, True) if not fPosted: @@ -533,4 +589,4 @@ def signal_handler(sig, frame): profile_write(sProfile, config) driver.close() - log.info("Script done") + log.info("Script done") \ No newline at end of file From 12083577ab5f9ea4873296b75148499c0e6db0e7 Mon Sep 17 00:00:00 2001 From: airwave1981 Date: Tue, 9 Nov 2021 12:52:17 +0100 Subject: [PATCH 2/6] align --- kleinanzeigen.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kleinanzeigen.py b/kleinanzeigen.py index 741ad6e..ae45344 100755 --- a/kleinanzeigen.py +++ b/kleinanzeigen.py @@ -476,11 +476,6 @@ def session_create(config): if os.path.isfile("./chrome-win/chrome.exe"): log.info("Found ./chrome-win/chrome.exe") options.binary_location = "./chrome-win/chrome.exe" -<<<<<<< HEAD - - #options.add_extension(r'E:\Google Drive\Verkauf\_auto\ebayKleinanzeigen\chrome-extensions\crobot.crx') -======= ->>>>>>> upstream/master driver = webdriver.Chrome(options=options) From 3a1b364dd565f18c6b090cdde45c36a06bbbce81 Mon Sep 17 00:00:00 2001 From: airwave1981 Date: Mon, 22 Nov 2021 11:50:44 +0100 Subject: [PATCH 3/6] 20211122 --- kleinanzeigen.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/kleinanzeigen.py b/kleinanzeigen.py index ae45344..06dccd7 100755 --- a/kleinanzeigen.py +++ b/kleinanzeigen.py @@ -320,12 +320,12 @@ def post_ad(driver, ad, interactive): if (ad['shipping_type']) != 'NONE': try: - select_element = driver.find_element_by_css_selector('select[id$=".versand_s"]') - shipment_select = Select(select_element) - log.debug("\t shipping select found with id: %s" % select_element.get_attribute('id')) if (ad['shipping_type']) == 'PICKUP': - shipment_select.select_by_visible_text("Nur Abholung") + ship_button = driver.find_element_by_xpath("/html/body/div[1]/form/fieldset[2]/div[3]/div/div/div[1]/div[1]/div[2]/div/label[2]/input") + ship_button.click() if (ad['shipping_type']) == 'SHIPPING': + select_element = driver.find_element_by_css_selector('select[id$=".versand_s"]') + shipment_select = Select(select_element) shipment_select.select_by_visible_text("Versand möglich") fake_wait() except NoSuchElementException: @@ -431,20 +431,17 @@ def post_ad(driver, ad, interactive): log.info("\tCaptcha input needed, but running in non-interactive mode! Skipping ...") fRc = False - submit_button = driver.find_element_by_id('pstad-frmprview') - if submit_button: - submit_button.click() - - fake_wait() - if fRc: try: - submit_button = driver.find_element_by_id('prview-btn-post') + submit_button = driver.find_element_by_id('pstad-submit') if submit_button: submit_button.click() - except NoSuchElementException: + except NoSuchElementException as e_msg: + log.debug(e_msg) pass - + + WebDriverWait(driver, 6).until(EC.url_contains('adId=')) + try: parsed_q = urllib.parse.parse_qs(urllib.parse.urlparse(driver.current_url).query) add_id = parsed_q.get('adId', None)[0] @@ -477,6 +474,8 @@ def session_create(config): log.info("Found ./chrome-win/chrome.exe") options.binary_location = "./chrome-win/chrome.exe" + #options.add_extension(r'E:\Google Drive\Verkauf\_auto\ebayKleinanzeigen\chrome-extensions\crobot.crx') + driver = webdriver.Chrome(options=options) stealth(driver, From e88da07b2fe1429c5d208fea2f62d45831570799 Mon Sep 17 00:00:00 2001 From: airwave1981 Date: Mon, 22 Nov 2021 11:53:31 +0100 Subject: [PATCH 4/6] 20211122 --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d9fed2a..c2a8edd 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,7 @@ data # default configuration files config.json config.jsonc +kleinanzeigen.json +.gitignore +.gitignore +*.json From 9867b73e098ef45583ed70df3592854b48759230 Mon Sep 17 00:00:00 2001 From: airwave1981 Date: Mon, 22 Nov 2021 12:07:54 +0100 Subject: [PATCH 5/6] 20211122 --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index c2a8edd..2c01194 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,8 @@ kleinanzeigen.json .gitignore .gitignore *.json +.vscode/ + +#chrome +chrome-win/ +chromedriver.exe \ No newline at end of file From 850b2556394da319994b174adc77ae57acf39b6c Mon Sep 17 00:00:00 2001 From: airwave1981 Date: Mon, 22 Nov 2021 12:17:42 +0100 Subject: [PATCH 6/6] 20211122 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2c01194..724b2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,5 @@ kleinanzeigen.json #chrome chrome-win/ +chrome-extensions/ chromedriver.exe \ No newline at end of file