diff --git a/.gitignore b/.gitignore index 9ef6d59..d9366d3 100644 --- a/.gitignore +++ b/.gitignore @@ -48,10 +48,7 @@ nosetests.xml # Coverage cover -# OSF-specific -####################### -Site/Cache/* -Site/Uploads/* - # Readme build README.html + +dump.rdb diff --git a/README.rst b/README.rst index 8b0a3bb..59f2c40 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ modular-odm .. image:: https://badge.fury.io/py/modular-odm.png :target: http://badge.fury.io/py/modular-odm -A database-agnostic Document-Object Mapper for Python. +A Document-Object Mapper with support for multiple NoSQL backends. Install @@ -16,8 +16,8 @@ Install $ pip install modular-odm -Example Usage with MongoDB -========================== +Example Usage +============= Defining Models --------------- @@ -49,6 +49,8 @@ Defining Models Setting the Storage Backend --------------------------- +For **MongoDB**: + .. code-block:: python from pymongo import MongoClient @@ -59,6 +61,21 @@ Setting the Storage Backend User.set_storage(storage.MongoStorage(db, collection="user")) Comment.set_storage(storage.MongoStorage(db, collection="comment")) + +For **Redis**: + +.. note:: + To use modularodm with Redis, you must have `redis-py `_ installed. + +.. code-block:: python + + from redis import Redis + from modularodm import storage + + db = redis.Redis() + User.set_storage(storage.RedisStorage(db, collection="user")) + Comment.set_storage(storage.RedisStorage(db, collection="comment")) + Creating and Querying --------------------- @@ -91,10 +108,10 @@ TODO Development =========== -Tests require `nose `_, `invoke `_, and MongoDB. +Tests require `nose `_, `invoke `_, MongoDB, and redis. -Installing MongoDB ------------------- +Installing Dependencies +----------------------- If you are on MacOSX with `homebrew `_, run @@ -102,15 +119,23 @@ If you are on MacOSX with `homebrew `_, run $ brew update $ brew install mongodb + $ brew install redis + +Then install the Python development requirements with + +.. code-block:: bash + + $ pip install -r dev-requirements.txt Running Tests ------------- -To start mongodb, run +You must have both a mongo and redis server running to execute the tests. .. code-block:: bash $ invoke mongo + $ invoke redis Run all tests with diff --git a/dev-requirements.txt b/dev-requirements.txt index 1e031e9..76e354d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,3 +3,4 @@ tox wheel invoke sphinx +redis diff --git a/modularodm/storage/__init__.py b/modularodm/storage/__init__.py index 4f2b46c..176cfab 100644 --- a/modularodm/storage/__init__.py +++ b/modularodm/storage/__init__.py @@ -2,3 +2,4 @@ from .mongostorage import MongoStorage from .picklestorage import PickleStorage from .ephemeralstorage import EphemeralStorage +from .redisstorage import RedisStorage diff --git a/modularodm/storage/base.py b/modularodm/storage/base.py index b635547..5c84e31 100644 --- a/modularodm/storage/base.py +++ b/modularodm/storage/base.py @@ -4,6 +4,7 @@ from functools import wraps import itertools +from modularodm import exceptions from ..translators import DefaultTranslator class KeyExistsException(Exception): pass @@ -192,12 +193,19 @@ def flush(self): """Flush the database.""" raise NotImplementedError - def find_one(self, query=None, **kwargs): - """Find a single record that matches ``query``. - """ - raise NotImplementedError - def find(self, query=None, **kwargs): """Query the database and return a query set. """ raise NotImplementedError + + def find_one(self, query=None, **kwargs): + results = list(self.find(query)) + if len(results) == 1: + return results[0] + elif len(results) == 0: + raise exceptions.NoResultsFound() + else: + raise exceptions.MultipleResultsFound( + 'Query for find_one must return exactly one result; ' + 'returned {0}'.format(len(results)) + ) diff --git a/modularodm/storage/mongostorage.py b/modularodm/storage/mongostorage.py index a6d7d88..8cea893 100644 --- a/modularodm/storage/mongostorage.py +++ b/modularodm/storage/mongostorage.py @@ -56,6 +56,7 @@ def __init__(self, schema, cursor): super(MongoQuerySet, self).__init__(schema) self.data = cursor + # TODO: make this the default implementations of BaseQuerySet? def __getitem__(self, index, raw=False): super(MongoQuerySet, self).__getitem__(index) key = self.data[index][self.primary] @@ -117,7 +118,7 @@ def _ensure_index(self, key): def __init__(self, db, collection): self.collection = collection - self.store = db[self.collection] + self.store = db[self.collection] # a mongo collection def find(self, query=None, **kwargs): mongo_query = self._translate_query(query) diff --git a/modularodm/storage/picklestorage.py b/modularodm/storage/picklestorage.py index 8d3c537..72c54a0 100644 --- a/modularodm/storage/picklestorage.py +++ b/modularodm/storage/picklestorage.py @@ -146,7 +146,7 @@ def get(self, primary_name, key): return copy.deepcopy(data) def _remove_by_pk(self, key, flush=True): - """Retrieve value from store. + """Remove value from store. :param key: Key @@ -201,10 +201,10 @@ def _match(self, value, query): else: raise TypeError('Query must be a QueryGroup or Query object.') - def find(self, query=None, **kwargs): + def find(self, query=None, by_pk=False, **kwargs): """ Return generator over query results. Takes optional - by_pk keyword argument; if true, return keys rather than + by_pk keyword argument; if True, return keys rather than values. """ @@ -214,7 +214,7 @@ def find(self, query=None, **kwargs): else: for key, value in self.store.items(): if self._match(value, query): - if kwargs.get('by_pk'): + if by_pk: yield key else: yield value diff --git a/modularodm/storage/redisstorage.py b/modularodm/storage/redisstorage.py new file mode 100644 index 0000000..7a097ed --- /dev/null +++ b/modularodm/storage/redisstorage.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +from modularodm.utils import dumps, loads +from .base import Storage +from ..query.query import QueryGroup, RawQuery +from ..query.queryset import BaseQuerySet +from .picklestorage import operators + + +class RedisQuerySet(BaseQuerySet): + + def __init__(self, schema, data): + super(RedisQuerySet, self).__init__(schema, data) + self.data = list(data) + + def __getitem__(self, index, raw=False): + super(RedisQuerySet, self).__getitem__(index) + key = self.data[index][self.primary] + if raw: + return key + return self.schema.load(key) + + def __iter__(self, raw=False): + keys = [obj[self.primary] for obj in self.data] + if raw: + return keys + return (self.schema.load(key) for key in keys) + + def __len__(self): + return len(self.data) + + count = __len__ + + def get_key(self, index): + return self.__getitem__(index, raw=True) + + def get_keys(self): + return list(self.__iter__(raw=True)) + + def __repr__(self): + return "".format(repr(list(self.data))) + + def sort(self, *keys): + """Sort data by keys.""" + for key in keys[::-1]: + reverse = key.startswith("-") + sort_key = key.lstrip("-") + self.data = sorted(self.data, key=lambda rec: rec[sort_key], reverse=reverse) + return self + + def offset(self, n): + """Return the queryset offset by ``n`` items.""" + self.data = self.data[n:] + return self + + def limit(self, n): + """Return the queryset limited to ``n`` items.""" + self.data = self.data[:n] + return self + + +class RedisStorage(Storage): + '''Storage backend for Redis. Requires redis-py. + + Each record is stored as a Hash keyed by the string: + : + + In addition a set keyed by _keys stores a set of all primary + keys for the collection. As a result, the queryset returned by a + StoredObject's ``find()`` method will be unordered. + + :param redis.Redis client: The ``redis.Redis`` object from redis-py. + :param str collection: The name of the collection, e.g. "user" + ''' + QuerySet = RedisQuerySet + + def __init__(self, client, collection): + self.client = client + self.collection = collection + #: Name of set that stores the primary keys for this collection + self._key_set = "{col}_keys".format(col=self.collection) + + def from_native(self, record): + """Convert a python dictionary to a dictionary that can be stored + in as a redis hash. Values must be serialized to JSON. + + :param dict record: Dictionary representation of the record. + """ + return dict((k, dumps(v)) for k, v in record.iteritems()) + + def to_native(self, record): + """Convert a dictionary retrieved from the redis store to a native + Python dictionary. + + :param dict record: A dictionary (hash) retrieved from the redis store. + """ + return dict((key, loads(val)) for key, val in record.iteritems()) + + def get_key_set(self): + """Return the set of primary keys from the store.""" + return self.client.smembers(self._key_set) + + def get_key(self, pk): + """Get the redis key for a given primary key.""" + return u"{0}:{1}".format(self.collection, pk) + + def get(self, primary_name, key): + """Get a record as a dictionary. + + :param primary_name: The name of the primary key. + :param key: The value of the primary key + """ + hash_obj = self.client.hgetall(self.get_key(key)) + if hash_obj: + return self.to_native(hash_obj) + else: # If HASH is empty, return None + return None + + def get_by_id(self, id): + return self.get(None, id) + + def insert(self, primary_name, key, value): + '''Insert a new record. + + :param str primary_name: Name of primary key + :param key: The value of the primary key + :param dict value: The dictionary of attribute:value pairs + ''' + if primary_name not in value: + value = value.copy() + value[primary_name] = key + # Add to set of primary keys + self.client.sadd(self._key_set, key) + # : => Hash of attribute:value pairs + storeable = self.from_native(value) + self.client.hmset(self.get_key(key), storeable) + return None + + def _match(self, name, query): + """Return whether the Hash named ``name`` matches the ``query``. + """ + # TODO: Duplication of PickleStorage logic. Rethink. + if isinstance(query, QueryGroup): + matches = [self._match(name, node) for node in query.nodes] + + if query.operator == 'and': + return all(matches) + elif query.operator == 'or': + return any(matches) + elif query.operator == 'not': + return not any(matches) + else: + raise ValueError('QueryGroup operator must be , , or .') + + elif isinstance(query, RawQuery): + attribute, operator, argument = \ + query.attribute, query.operator, query.argument + attribute_value = self.client.hget(name, attribute) + comp_function = operators[operator] + # Datatype returned by redis-py might not be the type expected by a query + data_type = getattr(query, "datatype", None) + val = loads(attribute_value) + # Coerce val to be the correct datatype + if data_type: + if getattr(query, "use_list", False): + val = [data_type(v) for v in val] + else: + val = data_type(val) + # Use same operators as pickle storage + return comp_function(val, argument) + else: + raise TypeError('Query must be a QueryGroup or Query object.') + + def find(self, query=None, by_pk=False): + """Return generator over query results. Takes optional + by_pk keyword argument; if ``True``, return keys rather than + values. + + .. note:: The returned objects are unordered. + """ + if query is None: + # Generator with every object in the collection + return (pkey if by_pk else self.get_by_id(pkey) + for pkey in self.get_key_set()) + else: + # Generator with objects filtered by query + return (pkey if by_pk else self.get_by_id(pkey) + for pkey in self.get_key_set() + if self._match(self.get_key(pkey), query)) + + def _remove_from_key_set(self, *keys): + """Remove primary keys from key set. + + Redis doesn't support removing arbitrary values from a set + so overwrite the key_set with the difference between the + current key_set and the set of keys to remove. + + :param keys: The primary keys to remove + """ + tmp_name = "__modm_tmp__" + self.client.sadd(tmp_name, *keys) + self.client.sdiffstore(self._key_set, self._key_set, tmp_name) + self.client.delete(tmp_name) + return None + + def remove(self, *query): + # Iterator of primary keys + keys_to_remove = list(self.find(*query, by_pk=True)) + # List of redis keys + redis_keys = [self.get_key(key) for key in keys_to_remove] + # Remove keys + self.client.delete(*redis_keys) + self._remove_from_key_set(*keys_to_remove) + return None + + def update(self, query, data): + """Update multiple records in the store. + + :param query: The query object. + :param dict data: Attribute:value pairs. + """ + for primary_key in self.find(query, by_pk=True): + redis_key = self.get_key(primary_key) + storeable = self.from_native(data) + self.client.hmset(redis_key, storeable) + return None + + def flush(self): + pass + + def __repr__(self): + return "".format(self.collection) diff --git a/modularodm/storedobject.py b/modularodm/storedobject.py index 91e3e3b..1e59c08 100644 --- a/modularodm/storedobject.py +++ b/modularodm/storedobject.py @@ -919,11 +919,13 @@ def _process_query(cls, query): if isinstance(query, RawQuery): field = cls._fields.get(query.attribute) + query.use_list = field._list if field is None: return if field._is_foreign: if getattr(query.argument, '_fields', None): if field._is_abstract: + query.datatype = tuple query.argument = ( query.argument._primary_key, query.argument._name, diff --git a/modularodm/utils.py b/modularodm/utils.py new file mode 100644 index 0000000..0c798cf --- /dev/null +++ b/modularodm/utils.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from bson import json_util +import json + + +def dumps(val): + '''Custom JSON serialization function that allows serialization of special + types, e.g. datetimes, UUIDs, etc. + ''' + return json.dumps(val, default=json_util.default) + + +def loads(val): + '''Custom JSON deserialization function that handles special types.''' + return json.loads(val, object_hook=json_util.object_hook) diff --git a/tasks.py b/tasks.py index 82cfb39..6c80e63 100644 --- a/tasks.py +++ b/tasks.py @@ -6,14 +6,23 @@ build_dir = os.path.join(docs_dir, '_build') @task -def mongo(daemon=False, port=20771): +def mongo(daemon=False): '''Run the mongod process. ''' + port = os.environ.get('MONGO_PORT', 20771) cmd = "mongod --port {0}".format(port) if daemon: cmd += " --fork" run(cmd) + +@task +def redis(): + port = os.environ.get("REDIS_PORT", 6379) + cmd = "redis-server --port {0}".format(port) + run(cmd) + + @task def test(coverage=False, browse=False): command = "nosetests" @@ -27,7 +36,7 @@ def test(coverage=False, browse=False): def clean(): run("rm -rf build") run("rm -rf dist") - run("rm -rf marshmallow.egg-info") + run("rm -rf modularodm.egg-info") clean_docs() print("Cleaned up.") diff --git a/tests/base.py b/tests/base.py index 2786577..c0abc48 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,12 +2,15 @@ import logging import inspect import os -import pymongo import unittest import uuid +import pymongo +import redis + from modularodm import StoredObject -from modularodm.storage import MongoStorage, PickleStorage, EphemeralStorage +from modularodm.storage import (MongoStorage, PickleStorage, EphemeralStorage, + RedisStorage) logger = logging.getLogger(__name__) @@ -83,6 +86,23 @@ def clean_up_storage(self): self.mongo_client.drop_collection(c) +class RedisStorageMixin(object): + fixture_suffix = "Redis" + + # DB Settings + DB_HOST = os.environ.get("REDIS_HOST", 'localhost') + DB_PORT = os.environ.get("REDIS_PORT", 6379) + + client = redis.Redis(host=DB_HOST, port=DB_PORT) + + def make_storage(self): + collection = str(uuid.uuid4())[:8] + return RedisStorage(client=self.client, collection=collection) + + def clean_up_storage(self): + self.client.flushall() + + class MultipleBackendMeta(type): def __new__(mcs, name, bases, dct): @@ -101,6 +121,7 @@ def __new__(mcs, name, bases, dct): PickleStorageMixin, MongoStorageMixin, EphemeralStorageMixin, + RedisStorageMixin ): new_name = '{}{}'.format(name, mixin.fixture_suffix) frame.f_globals[new_name] = type.__new__( diff --git a/tests/queries/test_simple_queries.py b/tests/queries/test_simple_queries.py index 5723df7..3db37b3 100644 --- a/tests/queries/test_simple_queries.py +++ b/tests/queries/test_simple_queries.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- import logging +import datetime -from modularodm import exceptions, StoredObject -from modularodm.fields import IntegerField +from nose.tools import * # PEP8 asserts + +from modularodm import fields, exceptions, StoredObject from modularodm.query.query import RawQuery as Q from tests.base import ModularOdmTestCase @@ -14,9 +16,19 @@ class BasicQueryTestCase(ModularOdmTestCase): def define_objects(self): class Foo(StoredObject): - _id = IntegerField(primary=True) + _id = fields.IntegerField(primary=True) + + class Person(StoredObject): + _meta = {"optimistic": True} + _id = fields.StringField(primary=True, index=True) + name = fields.StringField(required=True) + age = fields.IntegerField(required=False) + created = fields.DateTimeField(default=datetime.datetime.utcnow) - return Foo, + def __repr__(self): + return "".format(self.name) + + return Foo, Person def set_up_objects(self): self.foos = [] @@ -26,6 +38,13 @@ def set_up_objects(self): foo.save() self.foos.append(foo) + self.p1 = self.Person(name="Foo", age=12) + self.p1.save() + self.p2 = self.Person(name="Bar") + self.p2.save() + self.p3 = self.Person(name="Baz") + self.p3.save() + def test_load_by_pk(self): """ Given a known primary key, ``.get(pk)`` should return the object. """ @@ -279,3 +298,27 @@ def test_sort_sort_offset(self): [27, 28, 29] ) + def test_find(self): + retrieved = self.Person.find(Q("name", "eq", "Foo")) + assert_in(self.p1, retrieved) + assert_not_in(self.p2, retrieved) + + def test_create_stored_object(self): + p = self.Person(name="Foo", age=23) + p.save() + # has an _id + assert_true(p._id) + assert_equal(p.name, "Foo") + assert_equal(p.age, 23) + + def test_update_one_stored_object(self): + self.Person.update_one(self.p1, {"name": "Boo"}) + assert_equal(self.p1.name, "Boo") + + def test_find_one_raises_error_if_no_records_found(self): + p = self.Person(name="Foo") + p2 = self.Person(name="Foo") + p.save() + p2.save() + assert_raises(exceptions.NoResultsFound, + lambda: self.Person.find_one(Q("name", "eq", "notfound"))) diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/storage/test_redisstorage.py b/tests/storage/test_redisstorage.py new file mode 100644 index 0000000..be01c82 --- /dev/null +++ b/tests/storage/test_redisstorage.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import json +import random +import datetime +import unittest +from nose.tools import * # PEP8 asserts + +import redis + +from modularodm import StoredObject, fields +from modularodm.storage import RedisStorage +from modularodm.storage.redisstorage import RedisQuerySet +from modularodm.query.querydialect import DefaultQueryDialect as Q +from modularodm import exceptions + +random.seed(1) + + +class Person(StoredObject): + _meta = {"optimistic": True} + _id = fields.StringField(primary=True, index=True) + name = fields.StringField(required=True) + age = fields.IntegerField(required=False) + created = fields.DateTimeField(default=datetime.datetime.utcnow) + + def __repr__(self): + return "".format(self.name) + +class RedisTestCase(unittest.TestCase): + # DB Settings + DB_HOST = os.environ.get("REDIS_HOST", 'localhost') + DB_PORT = os.environ.get("REDIS_PORT", 6379) + + client = redis.Redis(host=DB_HOST, port=DB_PORT) + store = RedisStorage(client=client, collection='people') + Person.set_storage(store) + + def tearDown(self): + self.client.flushall() + + +class TestRedisStorage(RedisTestCase): + + def setUp(self): + self.p1 = Person(name="Foo", age=12) + self.p1.save() + self.p2 = Person(name="Bar") + self.p2.save() + self.p3 = Person(name="Baz") + self.p3.save() + + def test_insert(self): + self.store.insert("_id", "abc123", {"name": "Steve", "age": 23}) + # Sets key => hash of attributes + name = self.client.hget("people:abc123", "name") + assert_equal(name, json.dumps("Steve")) + age = int(self.client.hget("people:abc123", "age")) + assert_equal(age, 23) + + def test_find_by_pk(self): + pks = list(self.store.find(by_pk=True)) + for each in Person.find(): + assert_in(each._primary_key, pks) + + pks = list(self.store.find(Q("name", "eq", self.p1.name), by_pk=True)) + assert_in(self.p1._primary_key, pks) + + def test_repr(self): + assert_equal(repr(self.store), "") + + def test_get_key(self): + assert_equal(self.store.get_key('abc123'), "people:abc123") + + def test_remove(self): + all_people = list(Person.find()) + assert_in(self.p1, all_people) + Person.remove_one(self.p1._primary_key) + all_people = list(Person.find()) + assert_not_in(self.p1, all_people) + # Keys were removed + assert_not_in(self.p1._primary_key, self.store.get_key_set()) + redis_key = self.store.get_key(self.p1._primary_key) + assert_not_in(redis_key, self.client.keys()) + + def test_get_key_set(self): + key_set = self.client.smembers("people_keys") + assert_equal(self.store.get_key_set(), key_set) + + def test_update(self): + query = Q("_id", "eq", self.p1._id) + self.store.update(query, {"name": "Boo", 'age': 23}) + # Record as dict + record = self.client.hgetall(self.store.get_key(self.p1._id)) + assert_equal(record['name'], json.dumps("Boo")) + assert_equal(record['age'],json.dumps(23)) + + def test_update_multiple(self): + self.client.flushall() + recs = [] + for _ in range(5): + p = Person(name="Foo") + recs.append(p) + p.save() + self.store.update(Q("name", "eq", "Foo"), {"name": "Boo"}) + for rec in recs: + rec.reload() + assert_equal(rec.name, "Boo") + + +class TestRedisQuerySet(RedisTestCase): + + def setUp(self): + for i in range(5): + p = Person(name="Foo", age=i + 1) + p.save() + Person(name="Bar", age=6).save() + self.qs = Person.find() + + def test_sort(self): + sorted_qs = self.qs.sort("age") + loaded_objects = [p for p in sorted_qs] + expected = sorted(list(Person.find()), key=lambda rec: rec.age) + assert_equal(loaded_objects, expected) + + def test_sort_reversed(self): + sorted_qs = self.qs.sort("-age") + loaded_objects = [p for p in sorted_qs] + expected = sorted(list(Person.find()), key=lambda rec: rec.age, reverse=True) + assert_equal(loaded_objects, expected) + + def test_limit(self): + limited = self.qs.sort("age").limit(3) + assert_equal(len(limited), 3) + + +if __name__ == '__main__': + unittest.main()