From 0ca3475d5ef62f246b0f0f6bbd3e41b4a05c6bec Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 14:43:04 +0200 Subject: [PATCH 01/19] Adding job moving feature --- mrq/basetasks/utils.py | 23 +++++++++++++++++++++++ mrq/dashboard/static/js/views/jobs.js | 10 ++++++++++ mrq/exceptions.py | 4 ++++ mrq/job.py | 9 ++++++++- 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/mrq/basetasks/utils.py b/mrq/basetasks/utils.py index 16d3c142..24458cff 100644 --- a/mrq/basetasks/utils.py +++ b/mrq/basetasks/utils.py @@ -98,6 +98,29 @@ def perform_action(self, action, query, destination_queue): if list(query.keys()) == ["queue"]: Queue(query["queue"]).empty() + elif action == "move": + cursor = self.collection.find(query, projection=["_id", "queue"]) + fetched_jobs = list(cursor) + for jobs in group_iter(fetched_jobs, n=1000): + jobs_by_queue = defaultdict(list) + for job in jobs: + jobs_by_queue[job["queue"]].append(job["_id"]) + stats["requeued"] += 1 + + for queue in jobs_by_queue: + + updates = { + "status": "queued", + "datequeued": datetime.datetime.utcnow(), + "dateupdated": datetime.datetime.utcnow(), + "queue": destination_queue, + "retry_count": 0 + } + + self.collection.update({ + "_id": {"$in": jobs_by_queue[queue]} + }, {"$set": updates}, multi=True) + elif action in ("requeue", "requeue_retry"): # Requeue task by groups of maximum 1k items (if all in the same diff --git a/mrq/dashboard/static/js/views/jobs.js b/mrq/dashboard/static/js/views/jobs.js index db96de54..54ea5442 100644 --- a/mrq/dashboard/static/js/views/jobs.js +++ b/mrq/dashboard/static/js/views/jobs.js @@ -223,6 +223,15 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio }); self.refreshCallStack(job_id); + } else if (action == "move") { + + var queue = prompt("Test"); + self.jobaction(evt, { + "id": job_id, + "action": action, + "destination_queue": queue + }); + } else { self.jobaction(evt, { @@ -392,6 +401,7 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio "

"+ ""+ ""+ + ""+ ""; } return ""; diff --git a/mrq/exceptions.py b/mrq/exceptions.py index 87c3537c..cf609d1d 100644 --- a/mrq/exceptions.py +++ b/mrq/exceptions.py @@ -27,6 +27,10 @@ class AbortInterrupt(_MrqInterrupt): pass +class MovedInterrupt(_MrqInterrupt): + pass + + class RetryInterrupt(_MrqInterrupt): delay = None queue = None diff --git a/mrq/job.py b/mrq/job.py index c25d6da7..bd68685f 100644 --- a/mrq/job.py +++ b/mrq/job.py @@ -5,7 +5,7 @@ from bson import ObjectId from redis.exceptions import LockError import time -from .exceptions import RetryInterrupt, MaxRetriesInterrupt, AbortInterrupt, MaxConcurrencyInterrupt +from .exceptions import RetryInterrupt, MaxRetriesInterrupt, AbortInterrupt, MaxConcurrencyInterrupt, MovedInterrupt from .utils import load_class_by_path, group_iter import gevent import objgraph @@ -270,6 +270,13 @@ def requeue(self, queue=None, retry_count=0): "retry_count": retry_count }) + def move(self, queue=None, retry_count=0): + """ Cancel and requeues the current job in an other queue.""" + exc = MovedInterrupt() + self._attach_original_exception(exc) + self.requeue(queue, retry_count) + raise exc + def perform(self): """ Loads and starts the main task for this job, the saves the result. """ From 346e0efa60de54499c5dffbfa8323202059462e3 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 14:57:27 +0200 Subject: [PATCH 02/19] Checking destination_queue is not null or empty before moving a job (dashboard/static/js/views/jobs.js) --- mrq/dashboard/static/js/views/jobs.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mrq/dashboard/static/js/views/jobs.js b/mrq/dashboard/static/js/views/jobs.js index 54ea5442..6eed249e 100644 --- a/mrq/dashboard/static/js/views/jobs.js +++ b/mrq/dashboard/static/js/views/jobs.js @@ -226,11 +226,14 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio } else if (action == "move") { var queue = prompt("Test"); - self.jobaction(evt, { - "id": job_id, - "action": action, - "destination_queue": queue - }); + if (queue != null && queue != "") + { + self.jobaction(evt, { + "id": job_id, + "action": action, + "destination_queue": queue + }); + } } else { From 2c73b9b07853609a3a2eb9ef5252455d4e3246e4 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 15:46:51 +0200 Subject: [PATCH 03/19] Prompt message modified for move action (dashboard/static/js/views/jobs.js) --- mrq/dashboard/static/js/views/jobs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mrq/dashboard/static/js/views/jobs.js b/mrq/dashboard/static/js/views/jobs.js index 6eed249e..91f457a9 100644 --- a/mrq/dashboard/static/js/views/jobs.js +++ b/mrq/dashboard/static/js/views/jobs.js @@ -225,7 +225,7 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio } else if (action == "move") { - var queue = prompt("Test"); + var queue = prompt("Enter destination queue"); if (queue != null && queue != "") { self.jobaction(evt, { From a3fc2ecba20dfdf70a1366113b104dbcecf20c49 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 15:49:59 +0200 Subject: [PATCH 04/19] Factoring move with requeue (basetasks/utils.py) --- mrq/basetasks/utils.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/mrq/basetasks/utils.py b/mrq/basetasks/utils.py index 24458cff..4ecc19a3 100644 --- a/mrq/basetasks/utils.py +++ b/mrq/basetasks/utils.py @@ -98,30 +98,7 @@ def perform_action(self, action, query, destination_queue): if list(query.keys()) == ["queue"]: Queue(query["queue"]).empty() - elif action == "move": - cursor = self.collection.find(query, projection=["_id", "queue"]) - fetched_jobs = list(cursor) - for jobs in group_iter(fetched_jobs, n=1000): - jobs_by_queue = defaultdict(list) - for job in jobs: - jobs_by_queue[job["queue"]].append(job["_id"]) - stats["requeued"] += 1 - - for queue in jobs_by_queue: - - updates = { - "status": "queued", - "datequeued": datetime.datetime.utcnow(), - "dateupdated": datetime.datetime.utcnow(), - "queue": destination_queue, - "retry_count": 0 - } - - self.collection.update({ - "_id": {"$in": jobs_by_queue[queue]} - }, {"$set": updates}, multi=True) - - elif action in ("requeue", "requeue_retry"): + elif action in ("requeue", "requeue_retry", "move"): # Requeue task by groups of maximum 1k items (if all in the same # queue) @@ -150,7 +127,7 @@ def perform_action(self, action, query, destination_queue): if destination_queue is not None: updates["queue"] = destination_queue - if action == "requeue": + if action in ("requeue", "move"): updates["retry_count"] = 0 self.collection.update({ From d1ea41832a41947e5922e4c697c19b0dcfc42d51 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 16:09:55 +0200 Subject: [PATCH 05/19] Remove move member function in Job (mrq/job.py) --- mrq/exceptions.py | 4 ---- mrq/job.py | 7 ------- 2 files changed, 11 deletions(-) diff --git a/mrq/exceptions.py b/mrq/exceptions.py index cf609d1d..87c3537c 100644 --- a/mrq/exceptions.py +++ b/mrq/exceptions.py @@ -27,10 +27,6 @@ class AbortInterrupt(_MrqInterrupt): pass -class MovedInterrupt(_MrqInterrupt): - pass - - class RetryInterrupt(_MrqInterrupt): delay = None queue = None diff --git a/mrq/job.py b/mrq/job.py index bd68685f..7c3b2966 100644 --- a/mrq/job.py +++ b/mrq/job.py @@ -270,13 +270,6 @@ def requeue(self, queue=None, retry_count=0): "retry_count": retry_count }) - def move(self, queue=None, retry_count=0): - """ Cancel and requeues the current job in an other queue.""" - exc = MovedInterrupt() - self._attach_original_exception(exc) - self.requeue(queue, retry_count) - raise exc - def perform(self): """ Loads and starts the main task for this job, the saves the result. """ From d5086adf5a7f068d9f80e27377511e5501c42549 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 17:00:40 +0200 Subject: [PATCH 06/19] Retry count is now shown in the dashboard (dashboard/static/js/views/jobs.js) --- mrq/dashboard/static/js/views/jobs.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mrq/dashboard/static/js/views/jobs.js b/mrq/dashboard/static/js/views/jobs.js index 91f457a9..ceb15692 100644 --- a/mrq/dashboard/static/js/views/jobs.js +++ b/mrq/dashboard/static/js/views/jobs.js @@ -355,6 +355,10 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio display.push("cputime "+String(source.time).substring(0,6)+"s ("+source.switches+" switches)"); } + if (source.retry_count) { + display.push("retried " + String(source.retry_count) + " times"); + } + return "" + display.join("
") + ""; } else { From fff76c602c78b9527767cf04eb36eaea7fc5b44d Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 17:10:47 +0200 Subject: [PATCH 07/19] Remove move exception import in Job (mrq/job.py) --- mrq/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mrq/job.py b/mrq/job.py index 7c3b2966..c25d6da7 100644 --- a/mrq/job.py +++ b/mrq/job.py @@ -5,7 +5,7 @@ from bson import ObjectId from redis.exceptions import LockError import time -from .exceptions import RetryInterrupt, MaxRetriesInterrupt, AbortInterrupt, MaxConcurrencyInterrupt, MovedInterrupt +from .exceptions import RetryInterrupt, MaxRetriesInterrupt, AbortInterrupt, MaxConcurrencyInterrupt from .utils import load_class_by_path, group_iter import gevent import objgraph From efbfffe447c757370edf08a38ded9780b8d5320d Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 5 Jul 2017 17:50:05 +0200 Subject: [PATCH 08/19] Adding group job moving feature --- mrq/dashboard/static/js/views/jobs.js | 4 ++++ mrq/dashboard/templates/index.html | 2 ++ 2 files changed, 6 insertions(+) diff --git a/mrq/dashboard/static/js/views/jobs.js b/mrq/dashboard/static/js/views/jobs.js index ceb15692..8743bc4b 100644 --- a/mrq/dashboard/static/js/views/jobs.js +++ b/mrq/dashboard/static/js/views/jobs.js @@ -105,6 +105,10 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio var action = $(evt.target).data("action"); var data = _.clone(this.filters); + if (action == "move") { + data["destination_queue"] = prompt("Enter destination queue"); + } + data["action"] = action; self.jobaction(evt, data); diff --git a/mrq/dashboard/templates/index.html b/mrq/dashboard/templates/index.html index 4ae9c0bc..c6e45145 100644 --- a/mrq/dashboard/templates/index.html +++ b/mrq/dashboard/templates/index.html @@ -394,6 +394,8 @@


+ +     From 62cd69367016e66e49ee34875c07f82d7791be90 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Thu, 6 Jul 2017 11:19:30 +0200 Subject: [PATCH 09/19] Adding a supplementary for move action in groupactions (dashboard/static/js/views/jobs.js) --- mrq/dashboard/static/js/views/jobs.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mrq/dashboard/static/js/views/jobs.js b/mrq/dashboard/static/js/views/jobs.js index 8743bc4b..f6371c59 100644 --- a/mrq/dashboard/static/js/views/jobs.js +++ b/mrq/dashboard/static/js/views/jobs.js @@ -105,7 +105,10 @@ define(["jquery", "underscore", "views/generic/datatablepage", "models"],functio var action = $(evt.target).data("action"); var data = _.clone(this.filters); - if (action == "move") { + if (action == "move"){ + if (destination_queue == null || destination_queue == "") { + return ; + } data["destination_queue"] = prompt("Enter destination queue"); } From 99e88b635363fac63dce5e2d3e61cd013a49cb6e Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Fri, 7 Jul 2017 15:47:02 +0200 Subject: [PATCH 10/19] Adding a UI for worker groups configuration (not working for now) --- mrq/dashboard/static/js/config.js | 3 +- .../static/js/vendor/quicksettings.min.js | 1 + mrq/dashboard/static/js/views/workergroups.js | 38 ++++++++++++++++--- mrq/dashboard/templates/index.html | 4 -- 4 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 mrq/dashboard/static/js/vendor/quicksettings.min.js diff --git a/mrq/dashboard/static/js/config.js b/mrq/dashboard/static/js/config.js index e87ad65d..0111e605 100644 --- a/mrq/dashboard/static/js/config.js +++ b/mrq/dashboard/static/js/config.js @@ -16,7 +16,8 @@ require.config({ datatables: "/static/js/vendor/jquery.dataTables.min", datatablesbs3: "/static/js/vendor/datatables.bs3", moment: "/static/js/vendor/moment.min", - sparkline: "/static/js/vendor/jquery.sparkline.min" + sparkline: "/static/js/vendor/jquery.sparkline.min", + quicksettings: "/static/js/vendor/quicksettings.min" }, // urlArgs: "bust=" + (new Date()).getTime(), diff --git a/mrq/dashboard/static/js/vendor/quicksettings.min.js b/mrq/dashboard/static/js/vendor/quicksettings.min.js new file mode 100644 index 00000000..80d135ff --- /dev/null +++ b/mrq/dashboard/static/js/vendor/quicksettings.min.js @@ -0,0 +1 @@ +!function(){function a(a,b){var d=c("div",null,"qs_label",b);return d.innerHTML=a,d}function b(a,b,d,e){var f=c("input",b,d,e);return f.type=a,f}function c(a,b,c,d){var e=document.createElement(a);if(e)return e.id=b,c&&(e.className=c),d&&d.appendChild(e),e}function d(){return navigator.userAgent.indexOf("rv:11")!=-1||navigator.userAgent.indexOf("MSIE")!=-1}function e(){var a=navigator.userAgent.toLowerCase();return!(a.indexOf("chrome")>-1||a.indexOf("firefox")>-1||a.indexOf("epiphany")>-1)&&a.indexOf("safari/")>-1}function f(){var a=navigator.userAgent.toLowerCase();return a.indexOf("edge")>-1}function g(){var a=document.createElement("style");a.innerText=i,document.head.appendChild(a),h=!0}var h=!1,i=".qs_main{background-color:#dddddd;text-align:left;position:absolute;width:200px;font:12px sans-serif;box-shadow:5px 5px 8px rgba(0,0,0,0.35);user-select:none;-webkit-user-select:none;color:#000000;border:none}.qs_content{background-color:#cccccc;overflow-y:auto}.qs_title_bar{background-color:#eeeeee;user-select:none;-webkit-user-select:none;cursor:pointer;padding:5px;font-weight:bold;border:none;color:#000000}.qs_container{margin:5px;padding:5px;background-color:#eeeeee;border:none;position:relative}.qs_container_selected{border:none;background-color:#ffffff}.qs_range{-webkit-appearance:none;-moz-appearance:none;width:100%;height:17px;padding:0;margin:0;background-color:transparent;border:none;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.qs_range:focus{outline:none;border:none}.qs_range::-webkit-slider-runnable-track{width:100%;height:15px;cursor:pointer;background:#cccccc;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.qs_range:focus::-webkit-slider-runnable-track{background:#cccccc}.qs_range::-webkit-slider-thumb{-webkit-appearance:none;height:15px;width:15px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;background:#999999;cursor:pointer;margin-top:0}.qs_range::-moz-range-track{width:100%;height:15px;cursor:pointer;background:#cccccc;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.qs_range::-moz-range-thumb{height:15px;width:15px;border:none;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;background:#999999;cursor:pointer}.qs_range::-ms-track{width:100%;height:15px;cursor:pointer;visibility:hidden;background:transparent}.qs_range::-ms-thumb{height:15px;width:15px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;background:#999999;cursor:pointer;border:none}.qs_range::-ms-fill-lower{background:#cccccc;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.qs_range:focus::-ms-fill-lower{background:#cccccc}.qs_range::-ms-fill-upper{background:#cccccc;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.qs_range:focus::-ms-fill-upper{background:#cccccc}.qs_button{background-color:#f6f6f6;color:#000000;height:30px;border:1px solid #aaaaaa;font:12px sans-serif}.qs_button:active{background-color:#ffffff;border:1px solid #aaaaaa}.qs_button:focus{border:1px solid #aaaaaa;outline:none}.qs_checkbox{cursor:pointer;display:inline}.qs_checkbox input{position:absolute;left:-99999px}.qs_checkbox span{height:16px;width:100%;display:block;text-indent:20px;background:url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAALklEQVQ4T2OcOXPmfwYKACPIgLS0NLKMmDVrFsOoAaNhMJoOGBioFwZkZUWoJgApdFaxjUM1YwAAAABJRU5ErkJggg==') no-repeat}.qs_checkbox input:checked+span{background:url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAvElEQVQ4T63Tyw2EIBAA0OFKBxBL40wDRovAUACcKc1IB1zZDAkG18GYZTmSmafzgTnnMgwchoDWGlJKheGcP3JtnPceCqCUAmttSZznuYtgchsXQrgC+77DNE0kUpPbmBOoJaBOIVQylnqWgAAeKhDve/AN+EaklJBzhhgjWRoJVGTbNjiOowAIret6a+4jYIwpX8aDwLIs74C2D0IIYIyVP6Gm898m9kbVm85ljHUTf16k4VUefkwDrxk+zoUEwCt0GbUAAAAASUVORK5CYII=') no-repeat}.qs_checkbox_label{position:absolute;top:7px;left:30px}.qs_label{margin-bottom:3px;user-select:none;-webkit-user-select:none;cursor:default;font:12px sans-serif}.qs_text_input{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;width:100%;padding:0 0 0 5px;height:24px;border:1px inset #ffffff;background-color:#ffffff;color:#000000;font-size:12px}.qs_text_input:focus{outline:none;background:#ffffff;border:1px inset #ffffff}.qs_select{background:url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAAp0lEQVRIS+2SsQ3FIAwF7RVYhA5mgQFhFuhYhJKWL0eKxI8SGylKZ0p4+OBsHGNM+HChAiS7qkgyBKrovaLeOxhjbgtxZ+cFtgelFMg5QwgBvPd/EO5sDbKAlBLUWo/8CjmL075zDmKMj6rEKbpCqBL9aqc4ZUQAhVbInBMQUXz5Vg/WfxOktXZsWWtZLds9uIqlqaH1NFV3jdhSJA47E1CAaE8ViYp+wGiWMZ/T+cgAAAAASUVORK5CYII=') no-repeat right #f6f6f6;-webkit-appearance:none;-moz-appearance:none;appearance:none;color:#000000;width:100%;height:24px;border:1px solid #aaaaaa;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;padding:0 5px;-moz-outline:none;font-size:14px}.qs_select option{font-size:14px}.qs_select::-ms-expand{display:none}.qs_select:focus{outline:none}.qs_number{height:24px}.qs_image{width:100%}.qs_progress{width:100%;height:15px;background-color:#cccccc;border:none;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.qs_progress_value{height:100%;background-color:#999999}.qs_textarea{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;resize:vertical;width:100%;padding:3px 5px;border:1px inset #ffffff;background-color:#ffffff;color:#000000;font-size:12px}.qs_textarea:focus{outline:none;background:#ffffff;border:1px inset #ffffff}.qs_color{position:absolute;left:-999999px}.qs_color_label{width:100%;height:20px;display:block;border:1px solid #aaaaaa;cursor:pointer;padding:0 0 0 5px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.qs_file_chooser{position:absolute;left:-999999px}.qs_file_chooser_label{background-color:#f6f6f6;color:#000000;height:30px;border:1px solid #aaaaaa;font:12px sans-serif;width:100%;display:block;cursor:pointer;padding:7px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}",j={_version:"3.0.2",_topZ:1,_panel:null,_titleBar:null,_content:null,_startX:0,_startY:0,_hidden:!1,_collapsed:!1,_controls:null,_keyCode:-1,_draggable:!0,_collapsible:!0,_globalChangeHandler:null,useExtStyleSheet:function(){h=!0},create:function(a,b,c,d){var e=Object.create(this);return e._init(a,b,c,d),e},destroy:function(){this._panel.parentElement&&this._panel.parentElement.removeChild(this._panel);for(var a in this)this[a]=null},_init:function(a,b,c,d){h||g(),this._bindHandlers(),this._createPanel(a,b,d),this._createTitleBar(c||"QuickSettings"),this._createContent()},_bindHandlers:function(){this._startDrag=this._startDrag.bind(this),this._drag=this._drag.bind(this),this._endDrag=this._endDrag.bind(this),this._doubleClickTitle=this._doubleClickTitle.bind(this),this._onKeyUp=this._onKeyUp.bind(this)},getValuesAsJSON:function(a){var b={};for(var c in this._controls)this._controls[c].getValue&&(b[c]=this._controls[c].getValue());return a&&(b=JSON.stringify(b)),b},setValuesFromJSON:function(a){"string"==typeof a&&(a=JSON.parse(a));for(var b in a)this._controls[b]&&this._controls[b].setValue&&this._controls[b].setValue(a[b]);return this},saveInLocalStorage:function(a){return this._localStorageName=a,this._readFromLocalStorage(a),this},clearLocalStorage:function(a){return localStorage.removeItem(a),this},_saveInLocalStorage:function(a){localStorage.setItem(a,this.getValuesAsJSON(!0))},_readFromLocalStorage:function(a){var b=localStorage.getItem(a);b&&this.setValuesFromJSON(b)},_createPanel:function(a,b,d){this._panel=c("div",null,"qs_main",d||document.body),this._panel.style.zIndex=++j._topZ,this.setPosition(a||0,b||0),this._controls={}},_createTitleBar:function(a){this._titleBar=c("div",null,"qs_title_bar",this._panel),this._titleBar.textContent=a,this._titleBar.addEventListener("mousedown",this._startDrag),this._titleBar.addEventListener("dblclick",this._doubleClickTitle)},_createContent:function(){this._content=c("div",null,"qs_content",this._panel)},_createContainer:function(){var a=c("div",null,"qs_container");return a.addEventListener("focus",function(){this.className+=" qs_container_selected"},!0),a.addEventListener("blur",function(){var a=this.className.indexOf(" qs_container_selected");a>-1&&(this.className=this.className.substr(0,a))},!0),this._content.appendChild(a),a},setPosition:function(a,b){return this._panel.style.left=a+"px",this._panel.style.top=Math.max(b,0)+"px",this},setSize:function(a,b){return this._panel.style.width=a+"px",this._content.style.width=a+"px",this._content.style.height=b-this._titleBar.offsetHeight+"px",this},setWidth:function(a){return this._panel.style.width=a+"px",this._content.style.width=a+"px",this},setHeight:function(a){return this._content.style.height=a-this._titleBar.offsetHeight+"px",this},setDraggable:function(a){return this._draggable=a,this._draggable||this._collapsible?this._titleBar.style.cursor="pointer":this._titleBar.style.cursor="default",this},_startDrag:function(a){this._draggable&&(this._panel.style.zIndex=++j._topZ,document.addEventListener("mousemove",this._drag),document.addEventListener("mouseup",this._endDrag),this._startX=a.clientX,this._startY=a.clientY),a.preventDefault()},_drag:function(a){var b=parseInt(this._panel.style.left),c=parseInt(this._panel.style.top),d=a.clientX,e=a.clientY;this.setPosition(b+d-this._startX,c+e-this._startY),this._startX=d,this._startY=e,a.preventDefault()},_endDrag:function(a){document.removeEventListener("mousemove",this._drag),document.removeEventListener("mouseup",this._endDrag),a.preventDefault()},setGlobalChangeHandler:function(a){return this._globalChangeHandler=a,this},_callGCH:function(a){this._localStorageName&&this._saveInLocalStorage(this._localStorageName),this._globalChangeHandler&&this._globalChangeHandler(a)},hide:function(){return this._panel.style.visibility="hidden",this._hidden=!0,this},show:function(){return this._panel.style.visibility="visible",this._panel.style.zIndex=++j._topZ,this._hidden=!1,this},toggleVisibility:function(){return this._hidden?this.show():this.hide(),this},setCollapsible:function(a){return this._collapsible=a,this._draggable||this._collapsible?this._titleBar.style.cursor="pointer":this._titleBar.style.cursor="default",this},collapse:function(){return this._panel.removeChild(this._content),this._collapsed=!0,this},expand:function(){return this._panel.appendChild(this._content),this._collapsed=!1,this},toggleCollapsed:function(){return this._collapsed?this.expand():this.collapse(),this},setKey:function(a){return this._keyCode=a.toUpperCase().charCodeAt(0),document.addEventListener("keyup",this._onKeyUp),this},_onKeyUp:function(a){a.keyCode===this._keyCode&&["INPUT","SELECT","TEXTAREA"].indexOf(a.target.tagName)<0&&this.toggleVisibility()},_doubleClickTitle:function(){this._collapsible&&this.toggleCollapsed()},removeControl:function(a){if(this._controls[a])var b=this._controls[a].container;return b&&b.parentElement&&b.parentElement.removeChild(b),this._controls[a]=null,this},enableControl:function(a){return this._controls[a]&&(this._controls[a].control.disabled=!1),this},disableControl:function(a){return this._controls[a]&&(this._controls[a].control.disabled=!0),this},hideControl:function(a){return this._controls[a]&&(this._controls[a].container.style.display="none"),this},showControl:function(a){return this._controls[a]&&(this._controls[a].container.style.display="block"),this},overrideStyle:function(a,b,c){return this._controls[a]&&(this._controls[a].control.style[b]=c),this},hideTitle:function(a){var b=this._controls[a].label;return b&&(b.style.display="none"),this},showTitle:function(a){var b=this._controls[a].label;return b&&(b.style.display="block"),this},hideAllTitles:function(){for(var a in this._controls){var b=this._controls[a].label;b&&(b.style.display="none")}return this},showAllTitles:function(){for(var a in this._controls){var b=this._controls[a].label;b&&(b.style.display="block")}return this},getValue:function(a){return this._controls[a].getValue()},setValue:function(a,b){return this._controls[a].setValue(b),this._callGCH(a),this},addBoolean:function(a,d,e){var f=this._createContainer(),g=c("label",null,"qs_checkbox_label",f);g.textContent=a,g.setAttribute("for",a);var h=c("label",null,"qs_checkbox",f);h.setAttribute("for",a);var i=b("checkbox",a,null,h);i.checked=d;c("span",null,null,h);this._controls[a]={container:f,control:i,getValue:function(){return this.control.checked},setValue:function(a){this.control.checked=a,e&&e(a)}};var j=this;return i.addEventListener("change",function(){e&&e(i.checked),j._callGCH(a)}),this},bindBoolean:function(a,b,c){return this.addBoolean(a,b,function(b){c[a]=b})},addButton:function(a,c){var d=this._createContainer(),e=b("button",a,"qs_button",d);e.value=a,this._controls[a]={container:d,control:e};var f=this;return e.addEventListener("click",function(){c&&c(e),f._callGCH(a)}),this},addColor:function(g,h,i){if(e()||f()||d())return this.addText(g,h,i);var j=this._createContainer(),k=a(""+g+": "+h,j),l=b("color",g,"qs_color",j);l.value=h||"#ff0000";var m=c("label",null,"qs_color_label",j);m.setAttribute("for",g),m.style.backgroundColor=l.value,this._controls[g]={container:j,control:l,colorLabel:m,label:k,title:g,getValue:function(){return this.control.value},setValue:function(a){this.control.value=a,this.colorLabel.style.backgroundColor=l.value,this.label.innerHTML=""+this.title+": "+this.control.value,i&&i(a)}};var n=this;return l.addEventListener("input",function(){k.innerHTML=""+g+": "+l.value,m.style.backgroundColor=l.value,i&&i(l.value),n._callGCH(g)}),this},bindColor:function(a,b,c){return this.addColor(a,b,function(b){c[a]=b})},addDate:function(c,e,f){var g;if(e instanceof Date){var h=e.getFullYear(),i=e.getMonth()+1;i<10&&(i="0"+i);var j=e.getDate();g=h+"-"+i+"-"+j}else g=e;if(d())return this.addText(c,g,f);var k=this._createContainer(),l=a(""+c+"",k),m=b("date",c,"qs_text_input",k);m.value=g||"",this._controls[c]={container:k,control:m,label:l,getValue:function(){return this.control.value},setValue:function(a){var b;if(a instanceof Date){var c=a.getFullYear(),d=a.getMonth()+1;d<10&&(d="0"+d);var e=a.getDate();e<10&&(e="0"+e),b=c+"-"+d+"-"+e}else b=a;this.control.value=b||"",f&&f(b)}};var n=this;return m.addEventListener("input",function(){f&&f(m.value),n._callGCH(c)}),this},bindDate:function(a,b,c){return this.addDate(a,b,function(b){c[a]=b})},addDropDown:function(b,d,e){for(var f=this._createContainer(),g=a(""+b+"",f),h=c("select",null,"qs_select",f),i=0;i"+b+"",d);return d.appendChild(c),this._controls[b]={container:d,label:e},this},addFileChooser:function(d,e,f,g){var h=this._createContainer(),i=a(""+d+"",h),j=b("file",d,"qs_file_chooser",h);f&&(j.accept=f);var k=c("label",null,"qs_file_chooser_label",h);k.setAttribute("for",d),k.textContent=e||"Choose a file...",this._controls[d]={container:h,control:j,label:i,getValue:function(){return this.control.files[0]}};var l=this;return j.addEventListener("change",function(){j.files&&j.files.length&&(k.textContent=j.files[0].name,g&&g(j.files[0]),l._callGCH(d))}),this},addHTML:function(b,d){var e=this._createContainer(),f=a(""+b+": ",e),g=c("div",null,null,e);return g.innerHTML=d,this._controls[b]={container:e,label:f,control:g,getValue:function(){return this.control.innerHTML},setValue:function(a){this.control.innerHTML=a}},this},addImage:function(b,d,e){var f=this._createContainer(),g=a(""+b+"",f);return img=c("img",null,"qs_image",f),img.src=d,this._controls[b]={container:f,control:img,label:g,getValue:function(){return this.control.src},setValue:function(a){this.control.src=a,e&&img.addEventListener("load",function b(){img.removeEventListener("load",b),e(a)})}},this},addRange:function(a,b,c,d,e,f){return this._addNumber("range",a,b,c,d,e,f)},addNumber:function(a,b,c,d,e,f){return this._addNumber("number",a,b,c,d,e,f)},_addNumber:function(c,e,f,g,h,i,j){var k=this._createContainer(),l=a("",k),m="range"===c?"qs_range":"qs_text_input qs_number",n=b(c,e,m,k);n.min=f||0,n.max=g||100,n.step=i||1,n.value=h||0,l.innerHTML=""+e+": "+n.value,this._controls[e]={container:k,control:n,label:l,title:e,callback:j,getValue:function(){return parseFloat(this.control.value)},setValue:function(a){this.control.value=a,this.label.innerHTML=""+this.title+": "+this.control.value,j&&j(parseFloat(a))}};var o="input";"range"===c&&d()&&(o="change");var p=this;return n.addEventListener(o,function(){l.innerHTML=""+e+": "+n.value,j&&j(parseFloat(n.value)),p._callGCH(e)}),this},bindRange:function(a,b,c,d,e,f){return this.addRange(a,b,c,d,e,function(b){f[a]=b})},bindNumber:function(a,b,c,d,e,f){return this.addNumber(a,b,c,d,e,function(b){f[a]=b})},setRangeParameters:function(a,b,c,d){return this.setNumberParameters(a,b,c,d)},setNumberParameters:function(a,b,c,d){var e=this._controls[a],f=e.control.value;return e.control.min=b,e.control.max=c,e.control.step=d,e.control.value!==f&&e.callback&&e.callback(e.control.value),this},addPassword:function(a,b,c){return this._addText("password",a,b,c)},bindPassword:function(a,b,c){return this.addPassword(a,b,function(b){c[a]=b})},addProgressBar:function(b,d,e,f){var g=this._createContainer(),h=a("",g),i=c("div",null,"qs_progress",g),j=c("div",null,"qs_progress_value",i);return j.style.width=e/d*100+"%","numbers"===f?h.innerHTML=""+b+": "+e+" / "+d:"percent"===f?h.innerHTML=""+b+": "+Math.round(e/d*100)+"%":h.innerHTML=""+b+"",this._controls[b]={container:g,control:i,valueDiv:j,valueDisplay:f,label:h,value:e,max:d,title:b,getValue:function(){return this.value},setValue:function(a){this.value=Math.max(0,Math.min(a,this.max)),this.valueDiv.style.width=this.value/this.max*100+"%","numbers"===this.valueDisplay?this.label.innerHTML=""+this.title+": "+this.value+" / "+this.max:"percent"===this.valueDisplay&&(this.label.innerHTML=""+this.title+": "+Math.round(this.value/this.max*100)+"%")}},this},setProgressMax:function(a,b){var c=this._controls[a];return c.max=b,c.value=Math.min(c.value,c.max),c.valueDiv.style.width=c.value/c.max*100+"%","numbers"===c.valueDisplay?c.label.innerHTML=""+c.title+": "+c.value+" / "+c.max:"percent"===c.valueDisplay?c.label.innerHTML=""+c.title+": "+Math.round(c.value/c.max*100)+"%":c.label.innerHTML=""+c.title+"",this},addText:function(a,b,c){return this._addText("text",a,b,c)},_addText:function(d,e,f,g){var h,i=this._createContainer(),j=a(""+e+"",i);"textarea"===d?(h=c("textarea",e,"qs_textarea",i),h.rows=5):h=b(d,e,"qs_text_input",i),h.value=f||"",this._controls[e]={container:i,control:h,label:j,getValue:function(){return this.control.value},setValue:function(a){this.control.value=a,g&&g(a)}};var k=this;return h.addEventListener("input",function(){g&&g(h.value),k._callGCH(e)}),this},bindText:function(a,b,c){return this.addText(a,b,function(b){c[a]=b})},addTextArea:function(a,b,c){return this._addText("textarea",a,b,c)},setTextAreaRows:function(a,b){return this._controls[a].control.rows=b,this},bindTextArea:function(a,b,c){return this.addTextArea(a,b,function(b){c[a]=b})},addTime:function(c,e,f){var g;if(e instanceof Date){var h=e.getHours();h<10&&(h="0"+h);var i=e.getMinutes();i<10&&(i="0"+i);var j=e.getSeconds();j<10&&(j="0"+j),g=h+":"+i+":"+j}else g=e;if(d())return this.addText(c,g,f);var k=this._createContainer(),l=a(""+c+"",k),m=b("time",c,"qs_text_input",k);m.value=g||"",this._controls[c]={container:k,control:m,label:l,getValue:function(){return this.control.value},setValue:function(a){var b;if(a instanceof Date){var c=a.getHours();c<10&&(c="0"+c);var d=a.getMinutes();d<10&&(d="0"+d);var e=a.getSeconds();e<10&&(e="0"+e),b=c+":"+d+":"+e}else b=a;this.control.value=b||"",f&&f(b)}};var n=this;return m.addEventListener("input",function(){f&&f(m.value),n._callGCH(c)}),this},bindTime:function(a,b,c){return this.addTime(a,b,function(b){c[a]=b})}};"object"==typeof exports&&"object"==typeof module?module.exports=j:"function"==typeof define&&define.amd?define(j):window.QuickSettings=j}(); \ No newline at end of file diff --git a/mrq/dashboard/static/js/views/workergroups.js b/mrq/dashboard/static/js/views/workergroups.js index d9d96718..d7ec71d2 100644 --- a/mrq/dashboard/static/js/views/workergroups.js +++ b/mrq/dashboard/static/js/views/workergroups.js @@ -1,4 +1,4 @@ -define(["jquery", "underscore", "models", "views/generic/page"],function($, _, Models, Page) { +define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"],function($, _, Models, Page, QuickSettings) { return Page.extend({ @@ -11,10 +11,39 @@ define(["jquery", "underscore", "models", "views/generic/page"],function($, _, M }, render: function() { - var self = this; + var _this = this; + + self.$("button")[0].setAttribute('style', 'top:80px;left:10px;'); + self.workers_panel = [] + + self.command_panel = QuickSettings.create(50, 80, "Actions", self.$(_this.el)[0]) + .addButton("Save") + .addButton("Reload") + .setWidth(100); + $.get("/api/workergroups").done(function(data) { - self.renderTemplate(); - self.$("textarea").val(JSON.stringify(data["workergroups"], null, 8)); + var i = 0; + _.forEach(data["workergroups"], function(workgroup, workgroup_name) { + var worker_panel = QuickSettings.create(200 + i * 350, 80, "Worker group configuration", self.$(_this.el)[0]) + .addText("Workgroup Name", workgroup_name) + .hideTitle("Workgroup Name") + .setDraggable(false) + .setHeight(850) + .setWidth(300); + _.forEach(workgroup["profiles"], function(profile, profile_name) { + worker_panel.addHTML("separator", "") + .hideTitle("separator") + .addText("Profile Name", profile_name) + .addText("Memory", profile["memory"]) + .addRange("MinCount", 0, 100, profile["min_count"], 1) + .addRange("MaxCount", 0, 100, profile["max_count"], 1) + .addRange("CPU", 0, 1000, profile["cpu"], 100) + .addTextArea("Command", profile["command"]) + }); + self.workers_panel.push(worker_panel); + console.log(workgroup_name, workgroup); + i++; + }); }); }, @@ -33,5 +62,4 @@ define(["jquery", "underscore", "models", "views/generic/page"],function($, _, M }); } }); - }); diff --git a/mrq/dashboard/templates/index.html b/mrq/dashboard/templates/index.html index c6e45145..717e8ba9 100644 --- a/mrq/dashboard/templates/index.html +++ b/mrq/dashboard/templates/index.html @@ -258,10 +258,6 @@

Current I/O operations

Worker groups

-
- - -
From 2ea541f7a905e4270e645ee13aed626436bb50fe Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Fri, 7 Jul 2017 17:57:42 +0200 Subject: [PATCH 11/19] Adding buttons to delete/add workers groups (dashboard/static/js/views/workergroups.js) --- mrq/dashboard/static/js/views/workergroups.js | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/mrq/dashboard/static/js/views/workergroups.js b/mrq/dashboard/static/js/views/workergroups.js index d7ec71d2..f135da7e 100644 --- a/mrq/dashboard/static/js/views/workergroups.js +++ b/mrq/dashboard/static/js/views/workergroups.js @@ -16,29 +16,34 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] self.$("button")[0].setAttribute('style', 'top:80px;left:10px;'); self.workers_panel = [] - self.command_panel = QuickSettings.create(50, 80, "Actions", self.$(_this.el)[0]) + self.command_panel = QuickSettings.create(25, 80, "Actions", self.$(_this.el)[0]) + .addButton("Add a Worker Group") .addButton("Save") .addButton("Reload") - .setWidth(100); + .setDraggable(false) + .setWidth(150); $.get("/api/workergroups").done(function(data) { var i = 0; _.forEach(data["workergroups"], function(workgroup, workgroup_name) { - var worker_panel = QuickSettings.create(200 + i * 350, 80, "Worker group configuration", self.$(_this.el)[0]) + var worker_panel = QuickSettings.create(225 + i * 350, 80, "Worker group configuration", self.$(_this.el)[0]) .addText("Workgroup Name", workgroup_name) .hideTitle("Workgroup Name") + .addButton("Remove this Worker Group") + .addText("Process Termination Timeout", workgroup["process_termination_timeout"]) .setDraggable(false) .setHeight(850) .setWidth(300); _.forEach(workgroup["profiles"], function(profile, profile_name) { - worker_panel.addHTML("separator", "") - .hideTitle("separator") - .addText("Profile Name", profile_name) - .addText("Memory", profile["memory"]) - .addRange("MinCount", 0, 100, profile["min_count"], 1) - .addRange("MaxCount", 0, 100, profile["max_count"], 1) - .addRange("CPU", 0, 1000, profile["cpu"], 100) - .addTextArea("Command", profile["command"]) + worker_panel.addHTML("separator", "
") + .hideTitle("separator") + .addText("Profile Name", profile_name) + .addText("Memory", profile["memory"]) + .addText("CPU", profile["cpu"]) + .addRange("MinCount", 0, 100, profile["min_count"], 1) + .addRange("MaxCount", 0, 100, profile["max_count"], 1) + .addTextArea("Command", profile["command"]) + .addButton("Remove this Profile"); }); self.workers_panel.push(worker_panel); console.log(workgroup_name, workgroup); From 2d1b322ef80c1b6b4aa36dd3c51c434f8741a71e Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Mon, 10 Jul 2017 17:03:39 +0200 Subject: [PATCH 12/19] Updating POST on /api/workergroups (dashboard/app.py) --- mrq/dashboard/app.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mrq/dashboard/app.py b/mrq/dashboard/app.py index 52f3553d..588a07c3 100644 --- a/mrq/dashboard/app.py +++ b/mrq/dashboard/app.py @@ -136,9 +136,16 @@ def get_workergroups(): @requires_auth def post_workergroups(): workergroups = json.loads(request.form["workergroups"]) + + # Remove workergroups which hasn't be sent but were present previously (supposed deleted) + workergroup_list_json = list(workergroups.keys()) + workergroup_list_mongo = [document["_id"] for document in connections.mongodb_jobs.mrq_workergroups.find()] + workergroup_to_delete_list = list(set(workergroup_list_mongo) - set(workergroup_list_json)) + for workergroup_id in workergroup_to_delete_list: + connections.mongodb_jobs.mrq_workergroups.delete_one({"_id": workergroup_id}) + for k, v in workergroups.iteritems(): connections.mongodb_jobs.mrq_workergroups.update_one({"_id": k}, {"$set": v}, upsert=True) - return jsonify({"status": "ok"}) From 9bdea97a0c7c516c5171dd716717baa4cf698701 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Mon, 10 Jul 2017 17:04:21 +0200 Subject: [PATCH 13/19] Updating UI for worker groups configuration, working and nearly finished! --- mrq/dashboard/static/js/views/workergroups.js | 158 +++++++++++++----- 1 file changed, 115 insertions(+), 43 deletions(-) diff --git a/mrq/dashboard/static/js/views/workergroups.js b/mrq/dashboard/static/js/views/workergroups.js index f135da7e..cd1eca3d 100644 --- a/mrq/dashboard/static/js/views/workergroups.js +++ b/mrq/dashboard/static/js/views/workergroups.js @@ -10,60 +10,132 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] "click .submit": "submit" }, - render: function() { + addCommandPanel: function() { var _this = this; - self.$("button")[0].setAttribute('style', 'top:80px;left:10px;'); - self.workers_panel = [] + if (typeof this.commandPanel === "undefined") + { + this.commandPanel = QuickSettings.create(25, 80, "Actions", $(this.el)[0]) + .addButton("Add a Worker Group", function() { + _this.addPanel(); + }) + .addButton("Save", function() { + _this.save(); + }) + .addButton("Reload", function() { + _this.reload(); + }) + .addHTML("Status", "OK") + .setDraggable(false) + .setWidth(150); + } + }, - self.command_panel = QuickSettings.create(25, 80, "Actions", self.$(_this.el)[0]) - .addButton("Add a Worker Group") - .addButton("Save") - .addButton("Reload") - .setDraggable(false) - .setWidth(150); + addPanel: function(workergroup = null, workergroupName = "") { + var _this = this; + var panelIndex = this.workergroupPanels.length; + var workerPanel = QuickSettings.create(225 + this.workergroupPanels.length * 350, 80, "Worker group configuration", $(this.el)[0]) + .addText("Workgroup Name", workergroupName) + .hideTitle("Workgroup Name") + .addButton("Remove this Worker Group", function() { + _this.workergroupPanels[panelIndex].destroy(); + try { + delete _this.workergroupPanels[panelIndex]; + } + catch (e) {} + _this.workergroupPanels[panelIndex] = null; + }) + .addText("Process Termination Timeout", workergroup ? workergroup["process_termination_timeout"] : 0) + .addButton("Add a Profile", function() { + _this.addProfileToPanel(_this.workergroupPanels[panelIndex]); + }) + .setDraggable(false) + .setHeight(850) + .setWidth(300); + workerPanel.profilesNumber = 0; + this.workergroupPanels.push(workerPanel); + if (workergroup != null) + { + _.forEach(workergroup["profiles"], function(profile, profileName) { + this.addProfileToPanel(workerPanel, profile, profileName); + }, this); + } + }, - $.get("/api/workergroups").done(function(data) { - var i = 0; - _.forEach(data["workergroups"], function(workgroup, workgroup_name) { - var worker_panel = QuickSettings.create(225 + i * 350, 80, "Worker group configuration", self.$(_this.el)[0]) - .addText("Workgroup Name", workgroup_name) - .hideTitle("Workgroup Name") - .addButton("Remove this Worker Group") - .addText("Process Termination Timeout", workgroup["process_termination_timeout"]) - .setDraggable(false) - .setHeight(850) - .setWidth(300); - _.forEach(workgroup["profiles"], function(profile, profile_name) { - worker_panel.addHTML("separator", "
") - .hideTitle("separator") - .addText("Profile Name", profile_name) - .addText("Memory", profile["memory"]) - .addText("CPU", profile["cpu"]) - .addRange("MinCount", 0, 100, profile["min_count"], 1) - .addRange("MaxCount", 0, 100, profile["max_count"], 1) - .addTextArea("Command", profile["command"]) - .addButton("Remove this Profile"); - }); - self.workers_panel.push(worker_panel); - console.log(workgroup_name, workgroup); - i++; - }); - }); + addProfileToPanel: function(workerPanel, profile = null, profileName = "") { + workerPanel.profilesNumber += 1; + header = "Profile " + String(workerPanel.profilesNumber) + " - "; + workerPanel.addHTML("separator", "
") + .hideTitle("separator") + .addText(header + "Profile Name", profileName) + .addText(header + "Memory", profile ? profile["memory"]: 0) + .addText(header + "CPU", profile ? profile["cpu"] : 0) + .addRange(header + "MinCount", 0, 100, profile ? profile["min_count"] : 0, 1) + .addRange(header + "MaxCount", 0, 100, profile ? profile["max_count"] : 0, 1) + .addTextArea(header + "Command", profile ? profile["command"]: "") + .addButton("Remove this Profile"); }, - submit: function(el) { - var self = this; + remove_profile: function() { + }, + + reload: function() { + if (confirm('It will discard every changes that hasn\'t be saved. Are you sure?')) { + _.forEach(this.workergroupPanels, function(panel) { + if (panel != null && panel != undefined) + panel.destroy(); + delete panel; + }) + this.render(); + } + }, - self.$("button")[0].innerHTML = "Wait..."; + save: function() { + this.commandPanel._controls["Status"].setValue("Saving..."); - var val = self.$("textarea").val(); + data = {}; + _.forEach(this.workergroupPanels, function(panel) { + if (panel != null && panel != undefined ) + { + panelJSON = panel.getValuesAsJSON(); + workergroup = { + "profiles" : {}, + "process_termination_timeout": parseInt(panelJSON["Process Termination Timeout"], 10) + } + _.forEach(_.range(1, panel.profilesNumber + 1), function(index) { + profile = {}; + header = "Profile " + String(index) + " - "; + profile["memory"] = parseInt(panelJSON[header + "Memory"], 10); + profile["cpu"] = parseInt(panelJSON[header + "CPU"], 10); + profile["min_count"] = parseInt(panelJSON[header + "MinCount"], 10); + profile["max_count"] = parseInt(panelJSON[header + "MaxCount"], 10); + profile["command"] = panelJSON[header + "Command"]; + workergroup["profiles"][panelJSON[header + "Profile Name"]] = profile; + }) + data[panelJSON["Workgroup Name"]] = workergroup; + } + }) + console.log(data) + console.log(JSON.stringify(data)) - $.post("/api/workergroups", {"workergroups": val}).done(function(data) { - if (data.status != "ok") { + $.post("/api/workergroups", {"workergroups": JSON.stringify(data)}).done(function(result) { + if (result.status != "ok") { return alert("There was an error while saving!"); } - self.$("button")[0].innerHTML = "Save"; + }); + this.commandPanel._controls["Status"].setValue("Saved"); + }, + + render: function() { + var _this = this; + + this.workergroupPanels = []; + this.addCommandPanel(); + + $.get("/api/workergroups").done(function(data) { + _.forEach(data["workergroups"], function(workergroup, workergroupName) { + _this.addPanel(workergroup, workergroupName); + }); }); } }); From 026592e5594e1773c2f08038d979540d0fe1abb0 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Mon, 10 Jul 2017 17:52:05 +0200 Subject: [PATCH 14/19] Don't send profiles with an empty name (Worker group configuration UI) --- mrq/dashboard/static/js/views/workergroups.js | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mrq/dashboard/static/js/views/workergroups.js b/mrq/dashboard/static/js/views/workergroups.js index cd1eca3d..49f8ce25 100644 --- a/mrq/dashboard/static/js/views/workergroups.js +++ b/mrq/dashboard/static/js/views/workergroups.js @@ -95,7 +95,7 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] data = {}; _.forEach(this.workergroupPanels, function(panel) { - if (panel != null && panel != undefined ) + if (panel != null && panel != undefined) { panelJSON = panel.getValuesAsJSON(); workergroup = { @@ -103,20 +103,23 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] "process_termination_timeout": parseInt(panelJSON["Process Termination Timeout"], 10) } _.forEach(_.range(1, panel.profilesNumber + 1), function(index) { - profile = {}; header = "Profile " + String(index) + " - "; - profile["memory"] = parseInt(panelJSON[header + "Memory"], 10); - profile["cpu"] = parseInt(panelJSON[header + "CPU"], 10); - profile["min_count"] = parseInt(panelJSON[header + "MinCount"], 10); - profile["max_count"] = parseInt(panelJSON[header + "MaxCount"], 10); - profile["command"] = panelJSON[header + "Command"]; - workergroup["profiles"][panelJSON[header + "Profile Name"]] = profile; + console.log(panelJSON[header + "Profile Name"]) + if (panelJSON[header + "Profile Name"] != null && panelJSON[header + "Profile Name"] != "") + { + profile = {}; + profile["memory"] = parseInt(panelJSON[header + "Memory"], 10); + profile["cpu"] = parseInt(panelJSON[header + "CPU"], 10); + profile["min_count"] = parseInt(panelJSON[header + "MinCount"], 10); + profile["max_count"] = parseInt(panelJSON[header + "MaxCount"], 10); + profile["command"] = panelJSON[header + "Command"]; + workergroup["profiles"][panelJSON[header + "Profile Name"]] = profile; + } }) data[panelJSON["Workgroup Name"]] = workergroup; } }) console.log(data) - console.log(JSON.stringify(data)) $.post("/api/workergroups", {"workergroups": JSON.stringify(data)}).done(function(result) { if (result.status != "ok") { From d51866a5d0c7378800f6f9a3f41fdff157cfa98a Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Tue, 11 Jul 2017 10:16:22 +0200 Subject: [PATCH 15/19] Don't send workergroups with an empty name (Dashboard - Worker group configuration UI) --- mrq/dashboard/static/js/views/workergroups.js | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/mrq/dashboard/static/js/views/workergroups.js b/mrq/dashboard/static/js/views/workergroups.js index 49f8ce25..35635133 100644 --- a/mrq/dashboard/static/js/views/workergroups.js +++ b/mrq/dashboard/static/js/views/workergroups.js @@ -90,33 +90,38 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] } }, + // Check for "continue" usage instead of nested ifs save: function() { this.commandPanel._controls["Status"].setValue("Saving..."); data = {}; _.forEach(this.workergroupPanels, function(panel) { - if (panel != null && panel != undefined) + // Equivalent of python's "not in" + if ($.inArray(panel, [null, undefined]) == -1) { panelJSON = panel.getValuesAsJSON(); - workergroup = { - "profiles" : {}, - "process_termination_timeout": parseInt(panelJSON["Process Termination Timeout"], 10) - } - _.forEach(_.range(1, panel.profilesNumber + 1), function(index) { - header = "Profile " + String(index) + " - "; - console.log(panelJSON[header + "Profile Name"]) - if (panelJSON[header + "Profile Name"] != null && panelJSON[header + "Profile Name"] != "") - { - profile = {}; - profile["memory"] = parseInt(panelJSON[header + "Memory"], 10); - profile["cpu"] = parseInt(panelJSON[header + "CPU"], 10); - profile["min_count"] = parseInt(panelJSON[header + "MinCount"], 10); - profile["max_count"] = parseInt(panelJSON[header + "MaxCount"], 10); - profile["command"] = panelJSON[header + "Command"]; - workergroup["profiles"][panelJSON[header + "Profile Name"]] = profile; + if ($.inArray(panelJSON["Workgroup Name"], [null, ""]) == -1) + { + workergroup = { + "profiles" : {}, + "process_termination_timeout": parseInt(panelJSON["Process Termination Timeout"], 10) } - }) - data[panelJSON["Workgroup Name"]] = workergroup; + _.forEach(_.range(1, panel.profilesNumber + 1), function(index) { + header = "Profile " + String(index) + " - "; + console.log(panelJSON[header + "Profile Name"]) + if ($.inArray(panelJSON[header + "Profile Name"], [null, ""]) == -1) + { + profile = {}; + profile["memory"] = parseInt(panelJSON[header + "Memory"], 10); + profile["cpu"] = parseInt(panelJSON[header + "CPU"], 10); + profile["min_count"] = parseInt(panelJSON[header + "MinCount"], 10); + profile["max_count"] = parseInt(panelJSON[header + "MaxCount"], 10); + profile["command"] = panelJSON[header + "Command"]; + workergroup["profiles"][panelJSON[header + "Profile Name"]] = profile; + } + }) + data[panelJSON["Workgroup Name"]] = workergroup; + } } }) console.log(data) From 7255838b530bd069fcdfd6a01d987b62109629dd Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Tue, 11 Jul 2017 14:49:48 +0200 Subject: [PATCH 16/19] Adding serial checking to avoid configuration conflict writing (Dashboard - Worker group configuration UI) --- mrq/dashboard/app.py | 14 ++++++-- mrq/dashboard/static/js/views/workergroups.js | 34 +++++++++++++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/mrq/dashboard/app.py b/mrq/dashboard/app.py index 588a07c3..4b83e907 100644 --- a/mrq/dashboard/app.py +++ b/mrq/dashboard/app.py @@ -129,6 +129,9 @@ def get_workers(): def get_workergroups(): collection = connections.mongodb_jobs.mrq_workergroups data = {"workergroups": {str(row.pop("_id")): row for row in collection.find(sort=[("_id", 1)])}} + for workergroup_id in data["workergroups"]: + if "serial" not in data["workergroups"][workergroup_id]: + data["workergroups"][workergroup_id]["serial"] = str(int(time.time())) return jsonify(data) @@ -144,9 +147,16 @@ def post_workergroups(): for workergroup_id in workergroup_to_delete_list: connections.mongodb_jobs.mrq_workergroups.delete_one({"_id": workergroup_id}) + outdated_wgcs = [] for k, v in workergroups.iteritems(): - connections.mongodb_jobs.mrq_workergroups.update_one({"_id": k}, {"$set": v}, upsert=True) - return jsonify({"status": "ok"}) + if ("serial" not in v or v["serial"] == connections.mongodb_jobs.mrq_workergroups.find_one({"_id": k})["serial"]): + v["serial"] = str(int(time.time())) + connections.mongodb_jobs.mrq_workergroups.update_one({"_id": k}, {"$set": v}, upsert=True) + else: + outdated_wgcs.append(k) + + return jsonify({"status": "outdated" if len(outdated_wgcs) else "ok", + "outdated_wgcs": outdated_wgcs}) def build_api_datatables_query(req): diff --git a/mrq/dashboard/static/js/views/workergroups.js b/mrq/dashboard/static/js/views/workergroups.js index 35635133..ad820d61 100644 --- a/mrq/dashboard/static/js/views/workergroups.js +++ b/mrq/dashboard/static/js/views/workergroups.js @@ -56,6 +56,7 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] this.workergroupPanels.push(workerPanel); if (workergroup != null) { + this.serials[workergroupName] = workergroup["serial"] _.forEach(workergroup["profiles"], function(profile, profileName) { this.addProfileToPanel(workerPanel, profile, profileName); }, this); @@ -79,10 +80,10 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] remove_profile: function() { }, - reload: function() { - if (confirm('It will discard every changes that hasn\'t be saved. Are you sure?')) { + reload: function(force = false) { + if (force || confirm('It will discard every changes that hasn\'t be saved. Are you sure?')) { _.forEach(this.workergroupPanels, function(panel) { - if (panel != null && panel != undefined) + if (panel !== null && panel !== undefined) panel.destroy(); delete panel; }) @@ -90,8 +91,10 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] } }, + // Check for "continue" usage instead of nested ifs save: function() { + var _this = this; this.commandPanel._controls["Status"].setValue("Saving..."); data = {}; @@ -106,9 +109,12 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] "profiles" : {}, "process_termination_timeout": parseInt(panelJSON["Process Termination Timeout"], 10) } + + if (panelJSON["Workgroup Name"] in _this.serials) + workergroup["serial"] = _this.serials[panelJSON["Workgroup Name"]]; + _.forEach(_.range(1, panel.profilesNumber + 1), function(index) { header = "Profile " + String(index) + " - "; - console.log(panelJSON[header + "Profile Name"]) if ($.inArray(panelJSON[header + "Profile Name"], [null, ""]) == -1) { profile = {}; @@ -124,14 +130,27 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] } } }) - console.log(data) $.post("/api/workergroups", {"workergroups": JSON.stringify(data)}).done(function(result) { - if (result.status != "ok") { + if (result.status === "ok") + { + _this.commandPanel._controls["Status"].setValue("Saved"); + _this.reload(true); + } + else if (result.status === "outdated") + { + string = ""; + _.forEach(result.outdated_wgcs, function(wgc) { + string += "- " + wgc + "
"; + }) + _this.commandPanel._controls["Status"].setValue("These configurations were outdated and were not saved:
" + string + "
The others were saved."); + } + else + { + _this.commandPanel._controls["Status"].setValue("FAILED"); return alert("There was an error while saving!"); } }); - this.commandPanel._controls["Status"].setValue("Saved"); }, render: function() { @@ -139,6 +158,7 @@ define(["jquery", "underscore", "models", "views/generic/page", "quicksettings"] this.workergroupPanels = []; this.addCommandPanel(); + this.serials = {}; $.get("/api/workergroups").done(function(data) { _.forEach(data["workergroups"], function(workergroup, workergroupName) { From f8062956c2f3c3025447055991ae2e558e5a7344 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 12 Jul 2017 17:44:27 +0200 Subject: [PATCH 17/19] Adding a test for job killing and sleep test task (tests/{test_kill.py,tasks/general.py) --- tests/tasks/general.py | 6 ++++++ tests/test_kill.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/test_kill.py diff --git a/tests/tasks/general.py b/tests/tasks/general.py index 067d72a2..86a912d7 100644 --- a/tests/tasks/general.py +++ b/tests/tasks/general.py @@ -83,6 +83,12 @@ def run(self, params): pass +class Sleep(Task): + def run(self, params): + time.sleep(params.get("sleep", 100)) + return True + + class Retry(Task): def run(self, params): diff --git a/tests/test_kill.py b/tests/test_kill.py new file mode 100644 index 00000000..93f01351 --- /dev/null +++ b/tests/test_kill.py @@ -0,0 +1,35 @@ +from mrq.job import Job +import time + + +def test_kill_by_id(worker): + worker.start() + + job_id1 = worker.send_task("tests.tasks.general.Sleep", {}, block=False) + + Job(job_id1).kill() + time.sleep(1) + + job1 = Job(job_id1).fetch().data + + assert job1["status"] == "killed" + + +# def test_kill_by_path(worker): +# worker.start() + +# job_id1 = worker.send_task("tests.tasks.general.Sleep", {}, block=False) +# job_id2 = worker.send_task("tests.tasks.general.Sleep", {}, block=False) + +# worker.send_task("mrq.basetasks.utils.JobAction", { +# "path": "tests.tasks.general.Sleep", +# "action": "kill" +# }, block=False) + +# time.sleep(1) + +# job1 = Job(job_id1).fetch().data +# job2 = Job(job_id2).fetch().data + +# assert job1["status"] == "killed" +# assert job2["status"] == "killed" From 4e77a07e702ba0af17fdfa0b53710ebe6e3969d7 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Wed, 12 Jul 2017 17:45:07 +0200 Subject: [PATCH 18/19] Adding a killing feature for jobs (mrq/{job.py,worker.py}) --- mrq/job.py | 22 +++++++++++++++------- mrq/worker.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/mrq/job.py b/mrq/job.py index c25d6da7..ece30f22 100644 --- a/mrq/job.py +++ b/mrq/job.py @@ -249,6 +249,14 @@ def abort(self): self._attach_original_exception(exc) raise exc + def kill(self): + """ Kill the current job """ + context.connections.redis.rpush("{}:wcmd:{}".format(context.get_current_config()["redis_prefix"], + context.get_current_worker()), + "kill {}".format(self.id)) + self._save_status("killed") + pass + def cancel(self): """ Markes the current job as cancelled. Doesn't interrupt it. """ self._save_status("cancel") @@ -321,15 +329,15 @@ def perform(self): # pylint: disable=protected-access gevent.sleep(0) - current_greenlet = gevent.getcurrent() + self.current_greenlet = gevent.getcurrent() t = (datetime.datetime.utcnow() - self.datestarted).total_seconds() context.log.debug( "Job %s success: %0.6fs total, %0.6fs in greenlet, %s switches" % (self.id, t, - current_greenlet._trace_time, - current_greenlet._trace_switches - 1) + self.current_greenlet._trace_time, + self.current_greenlet._trace_switches - 1) ) else: @@ -454,13 +462,13 @@ def _save_status(self, status, updates=None, exception=False, w=None, j=None): db_updates["totaltime"] = (now - self.datestarted).total_seconds() if context.get_current_config().get("trace_greenlets"): - current_greenlet = gevent.getcurrent() + self.current_greenlet = gevent.getcurrent() # TODO are we sure the current job is doing the save_status() on itself? - if hasattr(current_greenlet, "_trace_time"): + if hasattr(self.current_greenlet, "_trace_time"): # pylint: disable=protected-access - db_updates["time"] = current_greenlet._trace_time - db_updates["switches"] = current_greenlet._trace_switches + db_updates["time"] = self.current_greenlet._trace_time + db_updates["switches"] = self.current_greenlet._trace_switches if exception: trace = traceback.format_exc() diff --git a/mrq/worker.py b/mrq/worker.py index 26922388..e67bee4e 100644 --- a/mrq/worker.py +++ b/mrq/worker.py @@ -328,6 +328,18 @@ def report_worker(self, w=0): except Exception as e: # pylint: disable=broad-except self.log.debug("Worker report failed: %s" % e) + def greenlet_command_handler(self): + """ + This greenlet is used to execute commands directly in the worker + """ + while True: + command = self.redis.blpop("{}:wcmd:{}".format(get_current_config()["redis_prefix"], + self.id) + ).split(" ") + if command[0] == "kill": + Job(command[1]).current_greenlet.kill(block=True) + pass + def greenlet_admin(self): """ This greenlet is used to get status information about the worker when --admin_port was given @@ -436,6 +448,9 @@ def work_init(self): if self.config["admin_port"]: self.greenlets["admin"] = gevent.spawn(self.greenlet_admin) + # ehould add a condition using the config + self.greenlets["command_handler"] = gevent.spawn(self.greenlet_command_handler) + self.install_signal_handlers() def work_loop(self, max_jobs=None, max_time=None): From 0d0b506fc4ca10b009e17410d5e9518efc196172 Mon Sep 17 00:00:00 2001 From: DasFranck Hochstaetter Date: Tue, 18 Jul 2017 12:15:34 +0200 Subject: [PATCH 19/19] Replacing iteritems (non-existant in Python3) (mrq/dashboard/app.py) --- mrq/dashboard/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mrq/dashboard/app.py b/mrq/dashboard/app.py index 4b83e907..c5628d6e 100644 --- a/mrq/dashboard/app.py +++ b/mrq/dashboard/app.py @@ -148,7 +148,7 @@ def post_workergroups(): connections.mongodb_jobs.mrq_workergroups.delete_one({"_id": workergroup_id}) outdated_wgcs = [] - for k, v in workergroups.iteritems(): + for k, v in iteritems(workergroups): if ("serial" not in v or v["serial"] == connections.mongodb_jobs.mrq_workergroups.find_one({"_id": k})["serial"]): v["serial"] = str(int(time.time())) connections.mongodb_jobs.mrq_workergroups.update_one({"_id": k}, {"$set": v}, upsert=True)