From 4639873220f46498f249d7ca69fb54ac5226e12f Mon Sep 17 00:00:00 2001 From: Kirill Zaitsev Date: Tue, 13 Dec 2016 03:45:38 +0300 Subject: [PATCH 1/2] Implement runbook CRUD API It is now possible to create, update and delete runbooks in elasticsearch. Input validation is done with jsonschema. --- runbook/api/utils.py | 34 ++++++++++++ runbook/api/v1/runbook_.py | 108 ++++++++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 runbook/api/utils.py diff --git a/runbook/api/utils.py b/runbook/api/utils.py new file mode 100644 index 0000000..5f824c5 --- /dev/null +++ b/runbook/api/utils.py @@ -0,0 +1,34 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import functools + +import flask + +from runbook import config + + +def check_regions(func): + regions = set(config.get_config()["regions"]) + + @functools.wraps(func) + def checker(region, *args, **kwargs): + if region not in regions: + return flask.jsonify( + {"error": "Region {} Not Found".format(region)}), 404 + return func(region, *args, **kwargs) + + return checker diff --git a/runbook/api/v1/runbook_.py b/runbook/api/v1/runbook_.py index 19c4766..20cee93 100644 --- a/runbook/api/v1/runbook_.py +++ b/runbook/api/v1/runbook_.py @@ -13,7 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. +import elasticsearch import flask +import jsonschema + +from runbook.api import utils +from runbook import storage bp = flask.Blueprint("runbooks", __name__) @@ -21,30 +26,129 @@ def get_blueprints(): return [["/region", bp]] +RUNBOOK_SCHEMA = { + "type": "object", + "$schema": "http://json-schema.org/draft-04/schema", + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "type": {"type": "string"}, + "runbook": {"type": "string"}, + }, + "required": ["name", "description", "runbook", "type"], + "additionalProperties": False +} + + +def _convert(hit): + body = {k: v for k, v in hit["_source"].items()} + body["_id"] = hit["_id"] + return body + @bp.route("//runbooks", methods=["GET", "POST"]) +@utils.check_regions def handle_runbooks(region): - return flask.jsonify("fixme!") + es = storage.get_elasticsearch() + index_name = "ms_runbooks_{}".format(region) + + if flask.request.method == "POST": + runbook = flask.request.get_json(silent=True) + try: + jsonschema.validate(runbook, RUNBOOK_SCHEMA) + except jsonschema.ValidationError as e: + # NOTE(kzaitsev): jsonschema exception has really good unicode + # error representation + return flask.jsonify( + {"error": u"{}".format(e)}), 400 + + resp = es.index( + index=index_name, + doc_type="runbook", + body=runbook, + ) + if resp['_shards']['successful']: + # at least 1 means we're good + return flask.jsonify({"_id": resp["_id"]}), 201 + # should not really be here + return flask.jsonify({"error": "Was unable to save document"}), 500 + else: + result = es.search(index=index_name, doc_type="runbook") + hit_list = [_convert(hit) for hit in result['hits']['hits']] + return flask.jsonify(hit_list) @bp.route("//runbooks/", methods=["GET", "PUT", "DELETE"]) +@utils.check_regions def handle_single_runbook(region, book_id): - return flask.jsonify("fixme!") + es = storage.get_elasticsearch() + index_name = "ms_runbooks_{}".format(region) + + if flask.request.method == "GET": + try: + result = es.get( + index=index_name, + doc_type="runbook", + id=book_id + ) + return flask.jsonify(_convert(result)) + except elasticsearch.NotFoundError: + flask.abort(404) + + elif flask.request.method == "DELETE": + try: + es.delete( + index=index_name, + doc_type="runbook", + id=book_id + ) + return flask.jsonify({}), 204 + except elasticsearch.NotFoundError: + flask.abort(404) + + elif flask.request.method == "PUT": + runbook = flask.request.get_json(silent=True) + try: + jsonschema.validate(runbook, RUNBOOK_SCHEMA) + except jsonschema.ValidationError as e: + # NOTE(kzaitsev): jsonschema exception has really good unicode + # error representation + return flask.jsonify( + {"error": u"{}".format(e)}), 400 + + try: + resp = es.update( + index=index_name, + doc_type="runbook", + id=book_id, + body={"doc": runbook}, + ) + except elasticsearch.NotFoundError: + flask.abort(404) + + if resp['_shards']['successful'] or resp['result'] == 'noop': + # noop means nothing to update, also ok + return flask.jsonify({"_id": resp["_id"]}) + return flask.jsonify({"error": "Was unable to update document"}), 500 + return flask.jsonify({"error": "Unreachable"}), 500 @bp.route("//runbooks//run", methods=["POST"]) +@utils.check_regions def run_runbook(region, book_id): return flask.jsonify("fixme!") @bp.route("//runbooks//runs") +@utils.check_regions def runbook_runs(region, book_id): return flask.jsonify("fixme!") @bp.route("//runbooks//runs/") +@utils.check_regions def single_runbook_run(region, book_id, run_id): return flask.jsonify("fixme!") From b380265382f05ffcb3a9962356fce751fcb7d330 Mon Sep 17 00:00:00 2001 From: Kirill Zaitsev Date: Tue, 13 Dec 2016 18:34:14 +0300 Subject: [PATCH 2/2] Add API tests Also fix result conversion, from always returning the same runbook --- tests/unit/api/__init__.py | 0 tests/unit/api/v1/__init__.py | 0 tests/unit/api/v1/base.py | 60 +++++++++ tests/unit/api/v1/test_runbook.py | 208 ++++++++++++++++++++++++++++++ 4 files changed, 268 insertions(+) create mode 100644 tests/unit/api/__init__.py create mode 100644 tests/unit/api/v1/__init__.py create mode 100644 tests/unit/api/v1/base.py create mode 100644 tests/unit/api/v1/test_runbook.py diff --git a/tests/unit/api/__init__.py b/tests/unit/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/api/v1/__init__.py b/tests/unit/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/api/v1/base.py b/tests/unit/api/v1/base.py new file mode 100644 index 0000000..2a37a84 --- /dev/null +++ b/tests/unit/api/v1/base.py @@ -0,0 +1,60 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import mock +import testtools + + +TEST_CONFIG = { + "flask": { + "PORT": 5000, + "HOST": "0.0.0.0", + "DEBUG": True + }, + "backend": { + "type": "elastic", + "connection": [{"host": "127.0.0.1", "port": 9200}] + }, + "regions": [ + "region_one", + "region_two" + ] +} + + +class APITestCase(testtools.TestCase): + + def setUp(self): + super(APITestCase, self).setUp() + self.addCleanup(mock.patch.stopall) + + # NOTE(kzaitsev): mock all get_config for a ll the tests + self.patcher = mock.patch('runbook.config.get_config') + self.get_config = self.patcher.start() + self.get_config.return_value = TEST_CONFIG + + # NOTE(kzaitsev): importing this module calls get_config, so I'm + # importing it after I've mocked get_config itself + import runbook.main + self.client = runbook.main.app.test_client() + self.app = runbook.main.app + + def test_not_found(self): + resp = self.client.get('/404') + self.assertEqual({"error": "Not Found"}, + json.loads(resp.data.decode())) + self.assertEqual(404, resp.status_code) diff --git a/tests/unit/api/v1/test_runbook.py b/tests/unit/api/v1/test_runbook.py new file mode 100644 index 0000000..f9aec2a --- /dev/null +++ b/tests/unit/api/v1/test_runbook.py @@ -0,0 +1,208 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import json + +import elasticsearch +import mock + +from tests.unit.api.v1 import base + + +class RunbookTestCase(base.APITestCase): + correct_runbook = { + "name": "test", + "description": "test", + "type": "bash", + "runbook": "echo", + } + + incorrect_runbook = { + "name": "test", + "description": "test", + "type": "bash", + } + + def test_get_no_region(self): + resp = self.client.get("/api/v1/region/region_zero/runbooks") + self.assertEqual(404, resp.status_code) + + resp = self.client.post("/api/v1/region/region_zero/runbooks") + self.assertEqual(404, resp.status_code) + + def test_post_new_runbook_bad_input(self): + resp = self.client.post("/api/v1/region/region_one/runbooks") + self.assertEqual(400, resp.status_code) + + resp = self.client.post("/api/v1/region/region_one/runbooks", + data=json.dumps(self.incorrect_runbook), + content_type="application/json") + self.assertEqual(400, resp.status_code) + + resp = self.client.post("/api/v1/region/region_one/runbooks", + data=json.dumps(self.correct_runbook), + ) + self.assertEqual(400, resp.status_code) + + @mock.patch.object(elasticsearch.Elasticsearch, "index") + def test_post_new_runbook(self, es_index): + es_index.return_value = { + "_shards": {"successful": 1}, + "_id": "123", + } + resp = self.client.post("/api/v1/region/region_one/runbooks", + data=json.dumps(self.correct_runbook), + content_type="application/json") + self.assertEqual(201, resp.status_code) + resp_json = json.loads(resp.data.decode()) + self.assertEqual(resp_json["_id"], "123") + + es_index.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + body=self.correct_runbook) + + @mock.patch.object(elasticsearch.Elasticsearch, "search") + def test_get_runbooks(self, es_search): + es_search.return_value = { + "hits": {"hits": [ + { + "_id": "121", + "_source": self.correct_runbook, + }, + { + "_id": "122", + "_source": self.correct_runbook, + }, + { + "_id": "123", + "_source": self.correct_runbook, + }, + ]}, + } + resp = self.client.get("/api/v1/region/region_one/runbooks", + content_type="application/json") + self.assertEqual(200, resp.status_code) + resp_json = json.loads(resp.data.decode()) + expected = [] + for book_id in ["121", "122", "123"]: + data = copy.copy(self.correct_runbook) + data["_id"] = book_id + expected.append(data) + self.assertEqual(expected, resp_json) + + es_search.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook") + + @mock.patch.object(elasticsearch.Elasticsearch, "get") + def test_get_single_runbook_bad_id(self, es_get): + es_get.side_effect = elasticsearch.NotFoundError + resp = self.client.get("/api/v1/region/region_one/runbooks/123", + content_type="application/json") + self.assertEqual(404, resp.status_code) + + es_get.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + id="123") + + @mock.patch.object(elasticsearch.Elasticsearch, "get") + def test_get_single_runbook(self, es_get): + es_get.return_value = { + "_source": self.correct_runbook, + "_id": "123", + } + resp = self.client.get("/api/v1/region/region_one/runbooks/123", + content_type="application/json") + self.assertEqual(200, resp.status_code) + + resp_json = json.loads(resp.data.decode()) + expected = copy.copy(self.correct_runbook) + expected["_id"] = "123" + self.assertEqual(expected, resp_json) + + es_get.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + id="123") + + @mock.patch.object(elasticsearch.Elasticsearch, "get") + def test_get_single_runbook_reg_two(self, es_get): + es_get.return_value = { + "_source": self.correct_runbook, + "_id": "123", + } + resp = self.client.get("/api/v1/region/region_two/runbooks/123", + content_type="application/json") + self.assertEqual(200, resp.status_code) + + resp_json = json.loads(resp.data.decode()) + expected = copy.copy(self.correct_runbook) + expected["_id"] = "123" + self.assertEqual(expected, resp_json) + + es_get.assert_called_with(index="ms_runbooks_region_two", + doc_type="runbook", + id="123") + + @mock.patch.object(elasticsearch.Elasticsearch, "delete") + def test_del_single_runbook(self, es_delete): + es_delete.side_effect = None + resp = self.client.delete("/api/v1/region/region_one/runbooks/123", + content_type="application/json") + self.assertEqual(204, resp.status_code) + es_delete.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + id="123") + + @mock.patch.object(elasticsearch.Elasticsearch, "delete") + def test_del_single_runbook_bad_id(self, es_delete): + es_delete.side_effect = elasticsearch.NotFoundError + resp = self.client.delete("/api/v1/region/region_one/runbooks/123", + content_type="application/json") + self.assertEqual(404, resp.status_code) + es_delete.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + id="123") + + @mock.patch.object(elasticsearch.Elasticsearch, "update") + def test_put_single_runbook_bad_id(self, es_update): + es_update.side_effect = elasticsearch.NotFoundError + resp = self.client.put("/api/v1/region/region_one/runbooks/123", + data=json.dumps(self.correct_runbook), + content_type="application/json") + self.assertEqual(404, resp.status_code) + expected_body = {"doc": self.correct_runbook} + es_update.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + id="123", + body=expected_body) + + @mock.patch.object(elasticsearch.Elasticsearch, "update") + def test_put_single_runbook(self, es_update): + es_update.return_value = { + "_shards": {"successful": 1}, + "_id": "123", + } + resp = self.client.put("/api/v1/region/region_one/runbooks/123", + data=json.dumps(self.correct_runbook), + content_type="application/json") + self.assertEqual(200, resp.status_code) + resp_json = json.loads(resp.data.decode()) + self.assertEqual({"_id": "123"}, resp_json) + + expected_body = {"doc": self.correct_runbook} + es_update.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + id="123", + body=expected_body)