From 54f5cec157223422012c62cfd928f5da7f193609 Mon Sep 17 00:00:00 2001 From: otherpirate Date: Wed, 10 Oct 2018 10:11:35 -0300 Subject: [PATCH 1/2] Add task retry --- rpaas/tasks.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/rpaas/tasks.py b/rpaas/tasks.py index 8222935..ad37e4b 100644 --- a/rpaas/tasks.py +++ b/rpaas/tasks.py @@ -141,6 +141,12 @@ def init_config(self, config=None): user=self._get_conf("HCAPI_USER"), password=self._get_conf("HCAPI_PASSWORD"), hc_format=self._get_conf("HCAPI_FORMAT", "http://{}:8080/")) + self.retry_countdown = os.environ.get("RETRY_COUNTDOWN", None) + if self.retry_countdown: + self.retry_countdown = int(self.retry_countdown) + self.max_retries = os.environ.get("MAX_RETRIES", None) + if self.max_retries: + self.max_retries = int(self.max_retries) def _get_conf(self, key, default=config.undefined): return config.get_config(key, default, self.config) @@ -203,17 +209,26 @@ def _delete_host(self, name, host, lb=None): finally: self.storage.remove_task(name) + def run(self, *args, **kwargs): + try: + self.execute(*args, **kwargs) + except Exception as exc: + self.retry(exc=exc, countdown=self.retry_countdown, max_retries=self.max_retries) + + def execute(self, *args, **kwargs): + raise NotImplementedError + class NewInstanceTask(BaseManagerTask): - def run(self, config, name): + def execute(self, config, name): self.init_config(config) self._add_host(name) class RemoveInstanceTask(BaseManagerTask): - def run(self, config, name): + def execute(self, config, name): self.init_config(config) lb = LoadBalancer.find(name, self.config) if lb is None: @@ -229,7 +244,7 @@ def run(self, config, name): class ScaleInstanceTask(BaseManagerTask): - def run(self, config, name, quantity): + def execute(self, config, name, quantity): try: self.init_config(config) lb = LoadBalancer.find(name, self.config) @@ -249,7 +264,7 @@ def run(self, config, name, quantity): class RestoreMachineTask(BaseManagerTask): - def run(self, config): + def execute(self, config): self.init_config(config) lock_name = self.config.get("RESTORE_LOCK_NAME", "restore_lock") healthcheck_timeout = int(self._get_conf("RPAAS_HEALTHCHECK_TIMEOUT", 600)) @@ -300,7 +315,7 @@ def _failure_instances(self, retry_failure_query, retry_failure_delay): class CheckMachineTask(BaseManagerTask): - def run(self, config): + def execute(self, config): self.init_config(config) for node in self.consul_manager.service_healthcheck(): node_fail = False @@ -341,7 +356,7 @@ def _check_machine_exists(self, address): class DownloadCertTask(BaseManagerTask): - def run(self, config, name, plugin, csr, key, domain): + def execute(self, config, name, plugin, csr, key, domain): try: self.init_config(config) sslutils.generate_crt(self.config, name, plugin, csr, key, domain) @@ -351,7 +366,7 @@ def run(self, config, name, plugin, csr, key, domain): class RevokeCertTask(BaseManagerTask): - def run(self, config, name, plugin, domain): + def execute(self, config, name, plugin, domain): try: self.init_config(config) lb = LoadBalancer.find(name, self.config) @@ -372,7 +387,7 @@ def run(self, config, name, plugin, domain): class RenewCertsTask(BaseManagerTask): - def run(self, config): + def execute(self, config): self.init_config(config) expires_in = int(self.config.get("LE_CERTIFICATE_EXPIRATION_DAYS", 90)) limit = datetime.datetime.utcnow() - datetime.timedelta(days=expires_in - 3) @@ -394,7 +409,7 @@ def renew(self, cert, config): class SessionResumptionTask(BaseManagerTask): - def run(self, config): + def execute(self, config): self.init_config(config) session_resumption_rotate = int(self.config.get("SESSION_RESUMPTION_TICKET_ROTATE", 3600)) instances_to_rotate = self.config.get("SESSION_RESUMPTION_INSTANCES", None) From 887b9256d3726f48fd0e69113a8f6e9e51e9e76b Mon Sep 17 00:00:00 2001 From: otherpirate Date: Wed, 24 Oct 2018 09:08:36 -0300 Subject: [PATCH 2/2] Do retry only when necessary --- rpaas/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rpaas/tasks.py b/rpaas/tasks.py index ad37e4b..b76ef2f 100644 --- a/rpaas/tasks.py +++ b/rpaas/tasks.py @@ -264,7 +264,7 @@ def execute(self, config, name, quantity): class RestoreMachineTask(BaseManagerTask): - def execute(self, config): + def run(self, config): self.init_config(config) lock_name = self.config.get("RESTORE_LOCK_NAME", "restore_lock") healthcheck_timeout = int(self._get_conf("RPAAS_HEALTHCHECK_TIMEOUT", 600)) @@ -315,7 +315,7 @@ def _failure_instances(self, retry_failure_query, retry_failure_delay): class CheckMachineTask(BaseManagerTask): - def execute(self, config): + def run(self, config): self.init_config(config) for node in self.consul_manager.service_healthcheck(): node_fail = False @@ -356,7 +356,7 @@ def _check_machine_exists(self, address): class DownloadCertTask(BaseManagerTask): - def execute(self, config, name, plugin, csr, key, domain): + def run(self, config, name, plugin, csr, key, domain): try: self.init_config(config) sslutils.generate_crt(self.config, name, plugin, csr, key, domain) @@ -366,7 +366,7 @@ def execute(self, config, name, plugin, csr, key, domain): class RevokeCertTask(BaseManagerTask): - def execute(self, config, name, plugin, domain): + def run(self, config, name, plugin, domain): try: self.init_config(config) lb = LoadBalancer.find(name, self.config) @@ -387,7 +387,7 @@ def execute(self, config, name, plugin, domain): class RenewCertsTask(BaseManagerTask): - def execute(self, config): + def run(self, config): self.init_config(config) expires_in = int(self.config.get("LE_CERTIFICATE_EXPIRATION_DAYS", 90)) limit = datetime.datetime.utcnow() - datetime.timedelta(days=expires_in - 3) @@ -409,7 +409,7 @@ def renew(self, cert, config): class SessionResumptionTask(BaseManagerTask): - def execute(self, config): + def run(self, config): self.init_config(config) session_resumption_rotate = int(self.config.get("SESSION_RESUMPTION_TICKET_ROTATE", 3600)) instances_to_rotate = self.config.get("SESSION_RESUMPTION_INSTANCES", None)