diff --git a/SpiderKeeper/app/__init__.py b/SpiderKeeper/app/__init__.py index 8a2ead4d..e1427cf8 100644 --- a/SpiderKeeper/app/__init__.py +++ b/SpiderKeeper/app/__init__.py @@ -2,7 +2,6 @@ import logging import traceback -import apscheduler from apscheduler.schedulers.background import BackgroundScheduler from flask import Flask from flask import jsonify diff --git a/SpiderKeeper/app/proxy/spiderctrl.py b/SpiderKeeper/app/proxy/spiderctrl.py index de01ea65..5850b1cd 100644 --- a/SpiderKeeper/app/proxy/spiderctrl.py +++ b/SpiderKeeper/app/proxy/spiderctrl.py @@ -1,9 +1,11 @@ import datetime import random -from functools import reduce +import requests +import re from SpiderKeeper.app import db -from SpiderKeeper.app.spider.model import SpiderStatus, JobExecution, JobInstance, Project, JobPriority +from SpiderKeeper.app.spider.model import SpiderStatus, JobExecution, JobInstance, Project, \ + JobPriority class SpiderServiceProxy(object): @@ -115,6 +117,14 @@ def sync_job_status(self, project): job_execution.start_time = job_execution_info['start_time'] job_execution.end_time = job_execution_info['end_time'] job_execution.running_status = SpiderStatus.FINISHED + + res = requests.get(self.log_url(job_execution)) + res.encoding = 'utf8' + raw = res.text[-4096:] + match = re.findall(job_execution.RAW_STATS_REGEX, raw, re.DOTALL) + if match: + job_execution.raw_stats = match[0] + job_execution.process_raw_stats() # commit db.session.commit() diff --git a/SpiderKeeper/app/schedulers/common.py b/SpiderKeeper/app/schedulers/common.py index 595518a0..07bac9e0 100644 --- a/SpiderKeeper/app/schedulers/common.py +++ b/SpiderKeeper/app/schedulers/common.py @@ -1,4 +1,3 @@ -import threading import time from SpiderKeeper.app import scheduler, app, agent, db @@ -29,7 +28,7 @@ def sync_spiders(): def run_spider_job(job_instance_id): ''' run spider by scheduler - :param job_instance: + :param job_instance_id: :return: ''' try: diff --git a/SpiderKeeper/app/spider/model.py b/SpiderKeeper/app/spider/model.py index 5376602b..6169a0c7 100644 --- a/SpiderKeeper/app/spider/model.py +++ b/SpiderKeeper/app/spider/model.py @@ -1,4 +1,6 @@ import datetime +import demjson +import re from sqlalchemy import desc from SpiderKeeper.app import db, Base @@ -159,6 +161,29 @@ class JobExecution(Base): running_status = db.Column(db.INTEGER, default=SpiderStatus.PENDING) running_on = db.Column(db.Text) + raw_stats = db.Column(db.Text) + items_count = db.Column(db.Integer) + warnings_count = db.Column(db.Integer) + errors_count = db.Column(db.Integer) + + RAW_STATS_REGEX = '\[scrapy\.statscollectors\][^{]+({[^}]+})' + + def process_raw_stats(self): + if self.raw_stats is None: + return + datetime_regex = '(datetime\.datetime\([^)]+\))' + self.raw_stats = re.sub(datetime_regex, r"'\1'", self.raw_stats) + stats = demjson.decode(self.raw_stats) + self.items_count = stats.get('item_scraped_count') or 0 + self.warnings_count = stats.get('log_count/WARNING') or 0 + self.errors_count = stats.get('log_count/ERROR') or 0 + + def has_warnings(self): + return not self.raw_stats or not self.items_count or self.warnings_count + + def has_errors(self): + return bool(self.errors_count) + def to_dict(self): job_instance = JobInstance.query.filter_by(id=self.job_instance_id).first() return { @@ -171,7 +196,12 @@ def to_dict(self): 'end_time': self.end_time.strftime('%Y-%m-%d %H:%M:%S') if self.end_time else None, 'running_status': self.running_status, 'running_on': self.running_on, - 'job_instance': job_instance.to_dict() if job_instance else {} + 'job_instance': job_instance.to_dict() if job_instance else {}, + 'has_warnings': self.has_warnings(), + 'has_errors': self.has_errors(), + 'items_count': self.items_count if self.items_count is not None else '-', + 'warnings_count': self.warnings_count if self.warnings_count is not None else '-', + 'errors_count': self.errors_count if self.errors_count is not None else '-' } @classmethod diff --git a/SpiderKeeper/app/static/css/app.css b/SpiderKeeper/app/static/css/app.css index 67a42216..549a4798 100644 --- a/SpiderKeeper/app/static/css/app.css +++ b/SpiderKeeper/app/static/css/app.css @@ -6,7 +6,6 @@ .txt-args { font-size: 10px; - display: block; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; diff --git a/SpiderKeeper/app/templates/job_dashboard.html b/SpiderKeeper/app/templates/job_dashboard.html index 494b93d7..544a86bd 100644 --- a/SpiderKeeper/app/templates/job_dashboard.html +++ b/SpiderKeeper/app/templates/job_dashboard.html @@ -153,12 +153,15 @@

Completed Jobs

Priority Runtime Started + Items + Warnings + Errors Log Status {% for job in job_status.COMPLETED %} {% if job.job_instance %} - + {{ job.job_execution_id }} {{ job.job_instance_id }} {{ job.job_instance.spider_name }} @@ -184,17 +187,20 @@

Completed Jobs

{% endif %} {{ timedelta(job.end_time,job.start_time) }} {{ job.start_time }} + {{ job.items_count }} + {{ job.warnings_count }} + {{ job.errors_count }} Log {% if job.running_status == 2 %} - - FINISHED - - {% else %} - - CANCELED - + + FINISHED + + {% else %} + + CANCELED + {% endif %} {% endif %} diff --git a/requirements.txt b/requirements.txt index 57daba1c..3a1e3fc6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ aniso8601==1.2.0 APScheduler==3.3.1 click==6.7 +demjson==2.2.4 Flask==0.12.1 Flask-BasicAuth==0.2.0 Flask-RESTful==0.3.5 diff --git a/setup.py b/setup.py index 3d4c690b..c822317d 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'aniso8601==1.2.0', 'APScheduler==3.3.1', 'click==6.7', + 'demjson==2.2.4', 'Flask==0.12.1', 'Flask-BasicAuth==0.2.0', 'Flask-RESTful==0.3.5',