Extracted aws implementation from datastore.

Author: jbenet

.gitignore

@@ -0,0 +1,8 @@
+*.pyc
+build/
+*.egg-info
+dist
+.tm_properties
+pypi/
+.tox
+venv/

.travis.yml

@@ -0,0 +1 @@
+language: python

LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2013 Juan Batiz-Benet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,58 @@
+# datastore-aws
+
+## datastore implementation for aws
+
+See [datastore](https://github.com/datastore/datastore).
+
+
+### Install
+
+From pypi (using pip):
+
+    sudo pip install datastore.aws
+
+From pypi (using setuptools):
+
+    sudo easy_install datastore.aws
+
+From source:
+
+    git clone https://github.com/datastore/datastore.aws/
+    cd datastore.aws
+    sudo python setup.py install
+
+
+### License
+
+datastore.aws is under the MIT License.
+
+### Contact
+
+datastore.aws is written by [Juan Batiz-Benet](https://github.com/jbenet).
+It was extracted from [datastore](https://github.com/datastore/datastore)
+in Feb 2013.
+
+Project Homepage:
+[https://github.com/datastore/datastore.aws](https://github.com/datastore/datastore.aws)
+
+Feel free to contact me. But please file issues in github first. Cheers!
+
+
+### Hello World
+
+    >>> import datastore.aws
+    >>> from boto.s3.connection import S3Connection
+    >>>
+    >>> s3conn = S3Connection('<aws access key>', '<aws secret key>')
+    >>> s3bucket = s3conn.get_bucket('<bucket name>')
+    >>> ds = datastore.aws.S3BucketDatastore(s3bucket)
+    >>>
+    >>> hello = datastore.Key('hello')
+    >>> ds.put(hello, 'world')
+    >>> ds.contains(hello)
+    True
+    >>> ds.get(hello)
+    'world'
+    >>> ds.delete(hello)
+    >>> ds.get(hello)
+    None

datastore/__init__.py

@@ -0,0 +1,5 @@
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+from datastore.core import *
+from datastore.core import __version__

datastore/aws/__init__.py

@@ -0,0 +1,140 @@
+
+__version__ = '0.1.0'
+__author__ = 'Juan Batiz-Benet'
+__email__ = '[email protected]'
+__doc__ = '''
+aws datastore implementation.
+
+Tested with:
+* boto 2.5.2
+
+'''
+
+#TODO: Implement queries using a key index.
+#TODO: Implement TTL (and key configurations)
+
+
+from boto.s3.key import Key as S3Key
+from boto.exception import S3ResponseError
+
+import datastore
+
+
+class S3BucketDatastore(datastore.Datastore):
+  '''Simple aws s3 datastore. Does not support queries.
+
+  The s3 interface is very similar to datastore's. The only differences are:
+  - values must be strings (SerializerShimDatastore)
+  - keys must be converted into strings
+  '''
+
+  def __init__(self, s3bucket):
+    '''Initialize the datastore with given s3 bucket `s3bucket`.
+
+    Args:
+      s3bucket: An s3 bucket to use.
+
+    Example::
+
+      from boto.s3.connection import S3Connection
+      s3conn = S3Connection('<aws access key>', '<aws secret key>')
+      s3bucket = s3conn.get_bucket('<bucket name>')
+      s3ds = S3BucketDatastore(s3bucket)
+    '''
+    self._s3bucket = s3bucket
+
+  def _s3key(self, key):
+    '''Return an s3 key for given datastore key.'''
+    k = S3Key(self._s3bucket)
+    k.key = str(key)
+    return k
+
+  @classmethod
+  def _s3keys_get_contents_as_string_gen(cls, s3keys):
+    '''s3 content retriever generator.'''
+    for s3key in s3keys:
+      yield s3key.get_contents_as_string()
+
+
+  def get(self, key):
+    '''Return the object named by key or None if it does not exist.
+
+    Args:
+      key: Key naming the object to retrieve
+
+    Returns:
+      object or None
+    '''
+    try:
+      return self._s3key(key).get_contents_as_string()
+    except S3ResponseError, e:
+      return None
+
+  def put(self, key, value):
+    '''Stores the object `value` named by `key`.
+
+    Args:
+      key: Key naming `value`
+      value: the object to store.
+    '''
+    self._s3key(key).set_contents_from_string(value)
+
+  def delete(self, key):
+    '''Removes the object named by `key`.
+
+    Args:
+      key: Key naming the object to remove.
+    '''
+    self._s3key(key).delete()
+
+  def query(self, query):
+    '''Returns an iterable of objects matching criteria expressed in `query`
+
+    Implementations of query will be the largest differentiating factor
+    amongst datastores. All datastores **must** implement query, even using
+    query's worst case scenario, see :ref:class:`Query` for details.
+
+    Args:
+      query: Query object describing the objects to return.
+
+    Raturns:
+      iterable cursor with all objects matching criteria
+    '''
+    allkeys = self._s3bucket.list(prefix=str(query.key).strip('/'))
+    iterable = self._s3keys_get_contents_as_string_gen(allkeys)
+    return query(iterable) # must apply filters, order, etc naively.
+
+
+  def contains(self, key):
+    '''Returns whether the object named by `key` exists.
+
+    Args:
+      key: Key naming the object to check.
+
+    Returns:
+      boalean whether the object exists
+    '''
+    return self._s3key(key).exists()
+
+
+'''
+Hello World:
+
+    >>> import datastore.aws
+    >>> from boto.s3.connection import S3Connection
+    >>>
+    >>> s3conn = S3Connection('<aws access key>', '<aws secret key>')
+    >>> s3bucket = s3conn.get_bucket('<bucket name>')
+    >>> ds = datastore.aws.S3BucketDatastore(s3bucket)
+    >>>
+    >>> hello = datastore.Key('hello')
+    >>> ds.put(hello, 'world')
+    >>> ds.contains(hello)
+    True
+    >>> ds.get(hello)
+    'world'
+    >>> ds.delete(hello)
+    >>> ds.get(hello)
+    None
+
+'''

datastore/aws/test.py

@@ -0,0 +1,50 @@
+# NOTE: make sure you set aws s3 information
+
+import unittest
+import logging
+
+from datastore import SerializerShimDatastore
+from datastore.core.test.test_basic import TestDatastore
+
+from . import S3BucketDatastore
+from boto.s3.connection import S3Connection
+
+
+aws_access_key = '<aws access key>'
+aws_secret_key = '<aws secret key>'
+
+class TestS3BucketDatastore(TestDatastore):
+
+  s3bucketname = '<aws bucket name>'
+
+  def _deletekeys(self):
+    keys = list(self.s3bucket.list())
+    for key in keys:
+      print 'deleting', key
+      self.s3bucket.delete_key(key)
+
+  def setUp(self):
+    logging.getLogger('boto').setLevel(logging.CRITICAL)
+
+    err = 'Use a real S3 %s. Edit datastore/test/test_aws.py.'
+    assert self.s3bucketname != '<aws bucket name>', err % 'bucket'
+    assert aws_access_key != '<aws access key>', err % 'access key.'
+    assert aws_secret_key != '<aws secret key>', err % 'secret key.'
+
+    self.s3conn = S3Connection(aws_access_key, aws_secret_key)
+    self.s3bucket = self.s3conn.get_bucket(self.s3bucketname)
+    self._deletekeys() # make sure we're clean :)
+
+  def tearDown(self):
+    self._deletekeys() # clean up after ourselves :]
+    del self.s3bucket
+    del self.s3conn
+
+  def test_s3(self):
+    ds = S3BucketDatastore(self.s3bucket)
+    ser = SerializerShimDatastore(ds)
+    self.subtest_simple([ser], numelems=20)
+
+
+if __name__ == '__main__':
+  unittest.main()

requirements.txt

@@ -0,0 +1,3 @@
+datastore==0.3.0
+nose==1.2.1
+boto>=2.5.2

setup.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+
+import re
+from setuptools import setup, find_packages
+
+pkgname = 'datastore.aws'
+
+# gather the package information
+main_py = open('datastore/aws/__init__.py').read()
+metadata = dict(re.findall("__([a-z]+)__ = '([^']+)'", main_py))
+packages = filter(lambda p: p.startswith(pkgname), find_packages())
+
+# convert the readme to pypi compatible rst
+try:
+  try:
+    import pypandoc
+    readme = pypandoc.convert('README.md', 'rst')
+  except ImportError:
+    readme = open('README.md').read()
+except:
+  print 'something went wrong reading the README.md file.'
+  readme = ''
+
+setup(
+  name=pkgname,
+  version=metadata['version'],
+  description='aws datastore implementation',
+  long_description=readme,
+  author=metadata['author'],
+  author_email=metadata['email'],
+  url='http://github.com/datastore/datastore.aws',
+  keywords=[
+    'datastore',
+    'aws',
+    's3',
+  ],
+  packages=packages,
+  install_requires=['datastore>=0.3.0', 'boto>=2.5.2'],
+  test_suite='datastore.aws.test',
+  license='MIT License',
+  classifiers=[
+    'Topic :: Database',
+    'Topic :: Database :: Front-Ends',
+  ]
+)