diff --git a/SpiderKeeper/app/spider/controller.py b/SpiderKeeper/app/spider/controller.py
index 296126b3..f7c55b20 100644
--- a/SpiderKeeper/app/spider/controller.py
+++ b/SpiderKeeper/app/spider/controller.py
@@ -13,8 +13,8 @@
from flask_restful_swagger import swagger
from werkzeug.utils import secure_filename
-from SpiderKeeper.app import db, api, agent, app
-from SpiderKeeper.app.spider.model import JobInstance, Project, JobExecution, SpiderInstance, JobRunType
+from SpiderKeeper.app import agent, api, app, db
+from SpiderKeeper.app.spider.model import JobExecution, JobInstance, JobRunType, Project, SpiderInstance
api_spider_bp = Blueprint('spider', __name__)
@@ -25,20 +25,20 @@
class ProjectCtrl(flask_restful.Resource):
@swagger.operation(
- summary='list projects',
- parameters=[])
+ summary='list projects',
+ parameters=[])
def get(self):
return [project.to_dict() for project in Project.query.all()]
@swagger.operation(
- summary='add project',
- parameters=[{
- "name": "project_name",
- "description": "project name",
- "required": True,
- "paramType": "form",
- "dataType": 'string'
- }])
+ summary='add project',
+ parameters=[{
+ "name": "project_name",
+ "description": "project name",
+ "required": True,
+ "paramType": "form",
+ "dataType": 'string'
+ }])
def post(self):
project_name = request.form['project_name']
project = Project()
@@ -50,14 +50,14 @@ def post(self):
class SpiderCtrl(flask_restful.Resource):
@swagger.operation(
- summary='list spiders',
- parameters=[{
- "name": "project_id",
- "description": "project id",
- "required": True,
- "paramType": "path",
- "dataType": 'int'
- }])
+ summary='list spiders',
+ parameters=[{
+ "name": "project_id",
+ "description": "project id",
+ "required": True,
+ "paramType": "path",
+ "dataType": 'int'
+ }])
def get(self, project_id):
project = Project.find_project_by_id(project_id)
return [spider_instance.to_dict() for spider_instance in
@@ -66,66 +66,67 @@ def get(self, project_id):
class SpiderDetailCtrl(flask_restful.Resource):
@swagger.operation(
- summary='spider detail',
- parameters=[{
- "name": "project_id",
- "description": "project id",
- "required": True,
- "paramType": "path",
- "dataType": 'int'
- }, {
- "name": "spider_id",
- "description": "spider instance id",
- "required": True,
- "paramType": "path",
- "dataType": 'int'
- }])
+ summary='spider detail',
+ parameters=[{
+ "name": "project_id",
+ "description": "project id",
+ "required": True,
+ "paramType": "path",
+ "dataType": 'int'
+ }, {
+ "name": "spider_id",
+ "description": "spider instance id",
+ "required": True,
+ "paramType": "path",
+ "dataType": 'int'
+ }])
def get(self, project_id, spider_id):
spider_instance = SpiderInstance.query.filter_by(project_id=project_id, id=spider_id).first()
return spider_instance.to_dict() if spider_instance else abort(404)
@swagger.operation(
- summary='run spider',
- parameters=[{
- "name": "project_id",
- "description": "project id",
- "required": True,
- "paramType": "path",
- "dataType": 'int'
- }, {
- "name": "spider_id",
- "description": "spider instance id",
- "required": True,
- "paramType": "path",
- "dataType": 'int'
- }, {
- "name": "spider_arguments",
- "description": "spider arguments",
- "required": False,
- "paramType": "form",
- "dataType": 'string'
- }, {
- "name": "priority",
- "description": "LOW: -1, NORMAL: 0, HIGH: 1, HIGHEST: 2",
- "required": False,
- "paramType": "form",
- "dataType": 'int'
- }, {
- "name": "tags",
- "description": "spider tags",
- "required": False,
- "paramType": "form",
- "dataType": 'string'
- }, {
- "name": "desc",
- "description": "spider desc",
- "required": False,
- "paramType": "form",
- "dataType": 'string'
- }])
+ summary='run spider',
+ parameters=[{
+ "name": "project_id",
+ "description": "project id",
+ "required": True,
+ "paramType": "path",
+ "dataType": 'int'
+ }, {
+ "name": "spider_id",
+ "description": "spider instance id",
+ "required": True,
+ "paramType": "path",
+ "dataType": 'int'
+ }, {
+ "name": "spider_arguments",
+ "description": "spider arguments",
+ "required": False,
+ "paramType": "form",
+ "dataType": 'string'
+ }, {
+ "name": "priority",
+ "description": "LOW: -1, NORMAL: 0, HIGH: 1, HIGHEST: 2",
+ "required": False,
+ "paramType": "form",
+ "dataType": 'int'
+ }, {
+ "name": "tags",
+ "description": "spider tags",
+ "required": False,
+ "paramType": "form",
+ "dataType": 'string'
+ }, {
+ "name": "desc",
+ "description": "spider desc",
+ "required": False,
+ "paramType": "form",
+ "dataType": 'string'
+ }])
def put(self, project_id, spider_id):
spider_instance = SpiderInstance.query.filter_by(project_id=project_id, id=spider_id).first()
- if not spider_instance: abort(404)
+ if not spider_instance:
+ abort(404)
job_instance = JobInstance()
job_instance.spider_name = spider_instance.spider_name
job_instance.project_id = project_id
@@ -149,94 +150,94 @@ def put(self, project_id, spider_id):
class JobCtrl(flask_restful.Resource):
@swagger.operation(
- summary='list job instance',
- parameters=[{
- "name": "project_id",
- "description": "project id",
- "required": True,
- "paramType": "path",
- "dataType": 'int'
- }])
+ summary='list job instance',
+ parameters=[{
+ "name": "project_id",
+ "description": "project id",
+ "required": True,
+ "paramType": "path",
+ "dataType": 'int'
+ }])
def get(self, project_id):
return [job_instance.to_dict() for job_instance in
JobInstance.query.filter_by(run_type="periodic", project_id=project_id).all()]
@swagger.operation(
- summary='add job instance',
- notes="json keys: /job/add", methods=['post'])
def job_add(project_id):
- project = Project.find_project_by_id(project_id)
job_instance = JobInstance()
job_instance.spider_name = request.form['spider_name']
job_instance.project_id = project_id
@@ -565,20 +566,55 @@ def job_add(project_id):
db.session.commit()
agent.start_spider(job_instance)
if job_instance.run_type == JobRunType.PERIODIC:
- job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
- job_instance.cron_hour = request.form.get('cron_hour') or '*'
- job_instance.cron_day_of_month = request.form.get('cron_day_of_month') or '*'
- job_instance.cron_day_of_week = request.form.get('cron_day_of_week') or '*'
- job_instance.cron_month = request.form.get('cron_month') or '*'
# set cron exp manually
if request.form.get('cron_exp'):
job_instance.cron_minutes, job_instance.cron_hour, job_instance.cron_day_of_month, job_instance.cron_day_of_week, job_instance.cron_month = \
request.form['cron_exp'].split(' ')
+ else:
+ job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
+ job_instance.cron_hour = request.form.get('cron_hour') or '*'
+ job_instance.cron_day_of_month = request.form.get('cron_day_of_month') or '*'
+ job_instance.cron_day_of_week = request.form.get('cron_day_of_week') or '*'
+ job_instance.cron_month = request.form.get('cron_month') or '*'
db.session.add(job_instance)
db.session.commit()
return redirect(request.referrer, code=302)
+@app.route("/project//job//edit", methods=['post'])
+def job_edit(project_id, job_instance_id):
+ job_instance = JobInstance.query.filter_by(project_id=project_id, id=job_instance_id).first()
+ job_instance.spider_name = request.form['spider_name']
+ job_instance.project_id = project_id
+ job_instance.spider_arguments = request.form['spider_arguments']
+ job_instance.priority = request.form.get('priority', 0)
+ job_instance.run_type = request.form['run_type']
+ # chose daemon manually
+ if request.form['daemon'] != 'auto':
+ spider_args = []
+ if request.form['spider_arguments']:
+ spider_args = request.form['spider_arguments'].split(",")
+ spider_args.append("daemon={}".format(request.form['daemon']))
+ job_instance.spider_arguments = ','.join(spider_args)
+ if job_instance.run_type == JobRunType.ONETIME:
+ job_instance.enabled = -1
+ db.session.commit()
+ agent.start_spider(job_instance)
+ if job_instance.run_type == JobRunType.PERIODIC:
+ # set cron exp manually
+ if request.form.get('cron_exp'):
+ job_instance.cron_minutes, job_instance.cron_hour, job_instance.cron_day_of_month, job_instance.cron_day_of_week, job_instance.cron_month = \
+ request.form['cron_exp'].split(' ')
+ else:
+ job_instance.cron_minutes = request.form.get('cron_minutes') or '0'
+ job_instance.cron_hour = request.form.get('cron_hour') or '*'
+ job_instance.cron_day_of_month = request.form.get('cron_day_of_month') or '*'
+ job_instance.cron_day_of_week = request.form.get('cron_day_of_week') or '*'
+ job_instance.cron_month = request.form.get('cron_month') or '*'
+ db.session.commit()
+ return redirect(request.referrer, code=302)
+
+
@app.route("/project//jobexecs//stop")
def job_stop(project_id, job_exec_id):
job_execution = JobExecution.query.filter_by(project_id=project_id, id=job_exec_id).first()
diff --git a/SpiderKeeper/app/spider/model.py b/SpiderKeeper/app/spider/model.py
index 5376602b..d05835d9 100644
--- a/SpiderKeeper/app/spider/model.py
+++ b/SpiderKeeper/app/spider/model.py
@@ -131,7 +131,6 @@ def to_dict(self):
cron_month=self.cron_month,
enabled=self.enabled == 0,
run_type=self.run_type
-
)
@classmethod
diff --git a/SpiderKeeper/app/templates/job_dashboard.html b/SpiderKeeper/app/templates/job_dashboard.html
index 494b93d7..1d4f1cc9 100644
--- a/SpiderKeeper/app/templates/job_dashboard.html
+++ b/SpiderKeeper/app/templates/job_dashboard.html
@@ -1,7 +1,7 @@
{% extends "base.html" %}
{% block content_header %}
-Job Dashboard
-Job Dashboard
position: absolute;
top: 15px;
right: 10px;">
- RunOnce
-
-
+ RunOnce
+
+
{% endblock %}
{% block content_body %}
-
-
-