[CD #86 #85 #44] Update CD pipeline, and cleanup

Author: subhankar

.gitlab-ci.yml

@@ -26,17 +26,19 @@ before_script:
   - psql -U postgres -c "create database dpr_db owner=dpr_user;"
 
 install:
-  - pip install -r requirements.dev.txt
+  - pip install -r requirements.txt
+  - pip install -r requirements.test.txt
 
 script: nosetests tests --with-coverage --cover-package=app
 
 after_success:
   coveralls
-
+before_deploy:
+  - export FLASK_CONFIGURATION=stage
 deploy:
   skip_cleanup: true
   provider: script
-  script: bash zappa_deploy.sh
+  script: python zappa_deploy.py
   on:
     repo: frictionlessdata/dpr-api
     branch: master

.travis.yml

@@ -9,26 +9,26 @@ Requirement python 2.7
 ```
 $ virtualenv env
 $ source env/bin/activate
-For dev use 
-$ pip install -r requirements.dev.txt
+For test use 
+$ pip install -r requirements.txt
+$ pip install -r requirements.test.txt
 else
 $ pip install -r requirements.txt
 ```
 
 ## Environment Setting:
 Plz put .env file in root directory and add environment variables to that
 ```
-API_KEY=
-AWS_ACCESS_KET_ID=
+AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 AUTH0_CLIENT_ID=
 AUTH0_CLIENT_SECRET=
 AUTH0_DOMAIN=
 AUTH0_DB_NAME=
-AUTH0_LOGIN_PAGE=
-AUTH0_CALLBACK_URL=
+AUTH0_API_AUDIENCE=
 S3_BUCKET_NAME=
+FLASKS3_BUCKET_NAME=
 SQLALCHEMY_DATABASE_URI=
 ```
 Rename the env.template file to .env file.
@@ -63,10 +63,10 @@ The lambda configuration is in zappa_settings.json
 There are different environment to deploy.
 
     # Right now only dev env config is there.
-    $ zappa deploy dev [This is for first time deployment]
+    $ zappa deploy stage [This is for first time deployment]
 
     For further deployment:
-    $  zappa update dev
+    $  zappa update stage
 
 ##### Zappa Configuration:
 ```
@@ -82,6 +82,13 @@ All api documentation is maintained by [flasgger](https://github.com/rochacbruno
 The swagger UI Url path is {host}/apidocs/index.html
 
 ## Testing:
+Before running tests plz run:
+```
+$ export FLASK_CONFIGURATION=test
+```
+. So that tests are not dependent on any env variable. By default it is dependent on
+local postgresql instance.
+
 All tests are in tests directory. We use nose for testing
 
 To run all tests plz use ```nosetests tests``` from the base directory.

README.md

@@ -11,7 +11,6 @@
 from sqlalchemy import ForeignKey
 from sqlalchemy import UniqueConstraint
 
-from sqlalchemy.dialects.postgresql import JSON
 from flask import current_app as app
 from sqlalchemy.orm import relationship
 
@@ -108,7 +107,8 @@ class Publisher(db.Model):
 
     packages = relationship("MetaDataDB", back_populates="publisher")
 
-    users = relationship("PublisherUser", back_populates="publisher")
+    users = relationship("PublisherUser", back_populates="publisher",
+                         cascade='save-update, merge, delete, delete-orphan')
 
 
 class User(db.Model):
@@ -123,7 +123,8 @@ class User(db.Model):
     full_name = db.Column(db.TEXT)
     auth0_id = db.Column(db.TEXT, index=True)
 
-    publishers = relationship("PublisherUser", back_populates="user")
+    publishers = relationship("PublisherUser", back_populates="user",
+                              cascade='save-update, merge, delete, delete-orphan')
 
     @property
     def serialize(self):
@@ -187,13 +188,15 @@ class MetaDataDB(db.Model):
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow)
     name = db.Column(db.TEXT, index=True)
-    descriptor = db.Column(JSON)
+    descriptor = db.Column(db.JSON)
     status = db.Column(db.TEXT, index=True)
     private = db.Column(db.Boolean)
     readme = db.Column(db.TEXT)
 
     publisher_id = db.Column(db.Integer, ForeignKey('publisher.id'))
-    publisher = relationship("Publisher", back_populates="packages")
+    publisher = relationship("Publisher", back_populates="packages",
+                             cascade="save-update, merge, delete, delete-orphan",
+                             single_parent=True)
 
     __table_args__ = (
         UniqueConstraint("name", "publisher_id"),

app/mod_api/models.py

@@ -51,6 +51,49 @@ def get_user(user_id):
     return response.json()
 
 
+def search_user(field, value):
+    headers = {'Authorization': "Bearer {token}".
+               format(token=get_auth0_token(None)),
+               'content-type': "application/json"}
+    url = "{AUDIENCE}users?include_fields=true&q={FIELD}:{VALUE}".\
+          format(AUDIENCE=app.config["AUTH0_API_AUDIENCE"],
+                 FIELD=field, VALUE=value)
+    response = requests.get(url=url, headers=headers)
+    if response.ok:
+        return response.json()
+    else:
+        return None
+
+
+def delete_user(user_id):
+    headers = {'Authorization': "Bearer {token}".
+               format(token=get_auth0_token(None)),
+               'content-type': "application/json"}
+    url = "{AUDIENCE}users/{USER_ID}".\
+        format(AUDIENCE=app.config["AUTH0_API_AUDIENCE"],
+               USER_ID=user_id)
+    requests.delete(url=url, headers=headers)
+
+
+def create_user(email, full_name, user_name, password):
+
+    payload = dict(user_metadata=dict(full_name=full_name),
+                   connection=app.config['AUTH0_DB_NAME'],
+                   email=email,
+                   username=user_name,
+                   password=password)
+    headers = {'Authorization': "Bearer {token}". \
+               format(token=get_auth0_token(None)),
+               'content-type': "application/json"}
+    url = "{AUDIENCE}users".format(AUDIENCE=app.config["AUTH0_API_AUDIENCE"])
+    response = requests.post(url=url, data=json.dumps(payload),
+                             headers=headers)
+    if response.ok:
+        return response.json()
+    else:
+        return None
+
+
 def update_user_secret(user_id):
     headers = {'Authorization': "Bearer {token}".\
                format(token=get_auth0_token(None)),
@@ -89,7 +132,6 @@ def get_auth0_token(jwt_token):
     token_payload = {
         'client_id': app.config['AUTH0_CLIENT_ID'],
         'client_secret': app.config['AUTH0_CLIENT_SECRET'],
-        'redirect_uri': app.config['AUTH0_CALLBACK_URL'],
         'grant_type': 'client_credentials',
         'audience': app.config['AUTH0_API_AUDIENCE']
     }

app/utils/auth0_helper.py

@@ -1,13 +1,11 @@
-API_KEY=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 AUTH0_CLIENT_ID=
 AUTH0_CLIENT_SECRET=
 AUTH0_DOMAIN=
 AUTH0_DB_NAME=
-AUTH0_LOGIN_PAGE=
-AUTH0_CALLBACK_URL=
 AUTH0_API_AUDIENCE=
 S3_BUCKET_NAME=
+FLASKS3_BUCKET_NAME=
 SQLALCHEMY_DATABASE_URI=

env.template

@@ -12,7 +12,9 @@
 from app.database import db
 from app.mod_api import models
 from app import create_app
-from app.mod_api.models import MetaDataDB, Publisher, PublisherUser
+from app.mod_api.models import MetaDataDB, Publisher, \
+    PublisherUser, User, BitStore
+from app.utils import auth0_helper
 
 dot_env_path = join(dirname(__file__), '.env')
 load_dotenv(dot_env_path)
@@ -44,25 +46,66 @@ def dropdb():
 
 @manager.command
 def populate():
-    data = json.loads(open('fixtures/datapackage.json').read())
-    readme = open('fixtures/README.md').read()
-    user = models.User()
-    user.auth0_id, user.email, user.name, user.secret, user.full_name \
-        = "auth0|123", "[email protected]", "test", \
-          "c053521f4f3331908d89df39bba922190a69f0ea99f7ca00", "Test Bob"
-    publisher = Publisher(name='demo')
 
-    metadata = MetaDataDB(name="demo-package")
-    metadata.descriptor, metadata.status, metadata.private, metadata.readme \
-        = json.dumps(data), 'active', False, readme
-    publisher.packages.append(metadata)
+    user_name, full_name, email = 'admin', 'Test Admin', '[email protected]'
+    auth0_id = 'abc' #populate_auth0(user_name, full_name, email)
+    populate_db(auth0_id, email, user_name, full_name)
+    populate_data(user_name)
+
+
+def populate_auth0(user_name, full_name, email):
+    responses = auth0_helper.search_user('username', user_name)
+    if len(responses) > 0:
+        for response in responses:
+            print (response)
+            auth0_helper.delete_user(response['user_id'])
+    user_create = auth0_helper.create_user(email, full_name,
+                                           user_name, 'Admin123')
+    return user_create['user_id']
+
+
+def populate_db(auth0_id, email, user_name, full_name):
+    user = User.query.filter_by(name=user_name).first()
+
+    publisher = Publisher.query.filter_by(name=user_name).first()
+    if publisher:
+        db.session.delete(publisher)
+        db.session.commit()
+    if user:
+        db.session.delete(user)
+        db.session.commit()
+
+    user = User()
+    user.auth0_id, user.email, user.name, user.full_name, user.secret \
+        = auth0_id, email, user_name, full_name, \
+          "c053521f4f3331908d89df39bba922190a69f0ea99f7ca00"
+
+    publisher = Publisher(name=user_name)
     association = PublisherUser(role="OWNER")
     association.publisher = publisher
     user.publishers.append(association)
 
     db.session.add(user)
+    db.session.commit()
 
+
+def populate_data(publisher_name):
+    data = json.loads(open('fixtures/datapackage.json').read())
+    data_csv = open('fixtures//data/demo-resource.csv').read()
+    readme = open('fixtures/README.md').read()
+    publisher = Publisher.query.filter_by(name=publisher_name).one()
+    metadata = MetaDataDB(name="demo-package")
+    metadata.descriptor, metadata.status, metadata.private, metadata.readme \
+        = json.dumps(data), 'active', False, readme
+
+    publisher.packages.append(metadata)
+    db.session.add(publisher)
     db.session.commit()
+    bitstore = BitStore(publisher_name, package='demo-package')
+    key = bitstore.build_s3_key('demo-resource.csv')
+    bucket_name = app.config['S3_BUCKET_NAME']
+    s3_client = app.config['S3']
+    s3_client.put_object(Bucket=bucket_name, Key=key, Body=data_csv)
 
 if __name__ == "__main__":
     manager.run()

manager.py

@@ -2,7 +2,7 @@
 
 [alembic]
 # template used to generate migration files
-file_template = %%(rev)s_%%(slug)s
+# file_template = %%(rev)s_%%(slug)s
 
 # set to 'true' to run the environment during
 # the 'revision' command, regardless of autogenerate

migrations/alembic.ini

@@ -0,0 +1,80 @@
+"""empty message
+
+Revision ID: 9f755c367f6d
+Revises: None
+Create Date: 2016-11-21 13:16:22.753496
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '9f755c367f6d'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('publisher',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('name', sa.TEXT(), nullable=False),
+    sa.Column('title', sa.Text(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_publisher_name'), 'publisher', ['name'], unique=True)
+    op.create_table('user',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('email', sa.TEXT(), nullable=True),
+    sa.Column('secret', sa.TEXT(), nullable=True),
+    sa.Column('name', sa.TEXT(), nullable=False),
+    sa.Column('full_name', sa.TEXT(), nullable=True),
+    sa.Column('auth0_id', sa.TEXT(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_user_auth0_id'), 'user', ['auth0_id'], unique=False)
+    op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=False)
+    op.create_index(op.f('ix_user_name'), 'user', ['name'], unique=True)
+    op.create_table('package',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('name', sa.TEXT(), nullable=True),
+    sa.Column('descriptor', postgresql.JSON(astext_type=sa.Text()), nullable=True),
+    sa.Column('status', sa.TEXT(), nullable=True),
+    sa.Column('private', sa.Boolean(), nullable=True),
+    sa.Column('readme', sa.TEXT(), nullable=True),
+    sa.Column('publisher_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['publisher_id'], [u'publisher.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name', 'publisher_id')
+    )
+    op.create_index(op.f('ix_package_name'), 'package', ['name'], unique=False)
+    op.create_index(op.f('ix_package_status'), 'package', ['status'], unique=False)
+    op.create_table('publisher_user',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('publisher_id', sa.Integer(), nullable=False),
+    sa.Column('role', sa.TEXT(), nullable=False),
+    sa.ForeignKeyConstraint(['publisher_id'], [u'publisher.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], [u'user.id'], ),
+    sa.PrimaryKeyConstraint('id', 'user_id', 'publisher_id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('publisher_user')
+    op.drop_index(op.f('ix_package_status'), table_name='package')
+    op.drop_index(op.f('ix_package_name'), table_name='package')
+    op.drop_table('package')
+    op.drop_index(op.f('ix_user_name'), table_name='user')
+    op.drop_index(op.f('ix_user_email'), table_name='user')
+    op.drop_index(op.f('ix_user_auth0_id'), table_name='user')
+    op.drop_table('user')
+    op.drop_index(op.f('ix_publisher_name'), table_name='publisher')
+    op.drop_table('publisher')
+    ### end Alembic commands ###