From 5dfdaa5903cd4a5e77e38c9f5bd9841f2d191b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=85=E6=AD=8C?= Date: Fri, 12 Oct 2018 10:01:45 +0800 Subject: [PATCH] Add retry when start_spider mission fail start_spider mission will always fail beacause of network problem of bug of Scrapyd, so I add a retry mechanism at function start_spider. --- SpiderKeeper/app/proxy/contrib/scrapy.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SpiderKeeper/app/proxy/contrib/scrapy.py b/SpiderKeeper/app/proxy/contrib/scrapy.py index 9acad39e..9aca2948 100644 --- a/SpiderKeeper/app/proxy/contrib/scrapy.py +++ b/SpiderKeeper/app/proxy/contrib/scrapy.py @@ -67,7 +67,12 @@ def start_spider(self, project_name, spider_name, arguments): post_data = dict(project=project_name, spider=spider_name) post_data.update(arguments) data = request("post", self._scrapyd_url() + "/schedule.json", data=post_data, return_type="json") - return data['jobid'] if data and data['status'] == 'ok' else None + if data and data['status'] == 'ok': + return data['jobid'] + else: + import time + time.sleep(3) + return self.start_spider(project_name, spider_name, arguments) def cancel_spider(self, project_name, job_id): post_data = dict(project=project_name, job=job_id)