Add task retry

rpaas/tasks.py

@@ -141,6 +141,12 @@ def init_config(self, config=None):
                                user=self._get_conf("HCAPI_USER"),
                                password=self._get_conf("HCAPI_PASSWORD"),
                                hc_format=self._get_conf("HCAPI_FORMAT", "http://{}:8080/"))
+        self.retry_countdown = os.environ.get("RETRY_COUNTDOWN", None)
+        if self.retry_countdown:
+            self.retry_countdown = int(self.retry_countdown)
+        self.max_retries = os.environ.get("MAX_RETRIES", None)
+        if self.max_retries:
+            self.max_retries = int(self.max_retries)
 
     def _get_conf(self, key, default=config.undefined):
         return config.get_config(key, default, self.config)
@@ -203,17 +209,26 @@ def _delete_host(self, name, host, lb=None):
         finally:
             self.storage.remove_task(name)
 
+    def run(self, *args, **kwargs):
+        try:
+            self.execute(*args, **kwargs)
+        except Exception as exc:
+            self.retry(exc=exc, countdown=self.retry_countdown, max_retries=self.max_retries)
+
+    def execute(self, *args, **kwargs):
+        raise NotImplementedError
+
 
 class NewInstanceTask(BaseManagerTask):
 
-    def run(self, config, name):
+    def execute(self, config, name):
         self.init_config(config)
         self._add_host(name)
 
 
 class RemoveInstanceTask(BaseManagerTask):
 
-    def run(self, config, name):
+    def execute(self, config, name):
         self.init_config(config)
         lb = LoadBalancer.find(name, self.config)
         if lb is None:
@@ -229,7 +244,7 @@ def run(self, config, name):
 
 class ScaleInstanceTask(BaseManagerTask):
 
-    def run(self, config, name, quantity):
+    def execute(self, config, name, quantity):
         try:
             self.init_config(config)
             lb = LoadBalancer.find(name, self.config)