diff --git a/backend/__init__.py b/backend/__init__.py index d961db5..05726f9 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -7,8 +7,9 @@ from .config import settings from .database import SessionLocal +from .enum import FilterEnum from .dynamic_routes import create_dynamic_router -from .models import Schema, AttributeDefinition +from .models import Schema, AttributeDefinition, AttrType from .general_routes import router @@ -17,20 +18,30 @@ def load_schemas(db: Session) -> List[models.Schema]: select(Schema) .options( subqueryload(Schema.attr_defs) - .subqueryload(AttributeDefinition.attribute) ) ).scalars().all() return schemas +def generate_api_description() -> str: + description = '# Filters\n\n**Filters list**:' + for filter in FilterEnum: + description += '\n* `{}` - {}'.format(filter.value.name, filter.value.description) + + description += '\n\n**Available filters for each type**:' + for atype in AttrType: + description += '\n* `{}`: {}'.format(atype.name, ', '.join([f'`{i.value.name}`' for i in atype.value.filters])) + return description + + def load_dynamic_routes(db: Session, app: FastAPI): schemas = load_schemas(db) for schema in schemas: - create_dynamic_router(schema=schema, app=app, get_db=lambda: db) + create_dynamic_router(schema=schema, app=app) def create_app(session: Optional[Session] = None) -> FastAPI: - app = FastAPI() + app = FastAPI(description=generate_api_description()) origins = ['*'] app.add_middleware(CORSMiddleware, allow_origins=origins, diff --git a/backend/alembic/versions/5e7b010c57b8_review.py b/backend/alembic/versions/5e7b010c57b8_review.py new file mode 100644 index 0000000..d4ce601 --- /dev/null +++ b/backend/alembic/versions/5e7b010c57b8_review.py @@ -0,0 +1,136 @@ +"""review + +Revision ID: 5e7b010c57b8 +Revises: 5c72f33e6b82 +Create Date: 2022-01-26 13:49:15.993571 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5e7b010c57b8' +down_revision = '5c72f33e6b82' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('change_values_bool', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.Boolean(), nullable=True), + sa.Column('new_value', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_bool_id'), 'change_values_bool', ['id'], unique=False) + op.create_table('change_values_date', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.Date(), nullable=True), + sa.Column('new_value', sa.Date(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_date_id'), 'change_values_date', ['id'], unique=False) + op.create_table('change_values_datetime', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.DateTime(timezone=True), nullable=True), + sa.Column('new_value', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_datetime_id'), 'change_values_datetime', ['id'], unique=False) + op.create_table('change_values_fk', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.Integer(), nullable=True), + sa.Column('new_value', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_fk_id'), 'change_values_fk', ['id'], unique=False) + op.create_table('change_values_float', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.Float(), nullable=True), + sa.Column('new_value', sa.Float(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_float_id'), 'change_values_float', ['id'], unique=False) + op.create_table('change_values_int', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.Integer(), nullable=True), + sa.Column('new_value', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_int_id'), 'change_values_int', ['id'], unique=False) + op.create_table('change_values_str', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('old_value', sa.String(), nullable=True), + sa.Column('new_value', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_change_values_str_id'), 'change_values_str', ['id'], unique=False) + op.create_table('change_requests', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_by_user_id', sa.Integer(), nullable=False), + sa.Column('reviewed_by_user_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('reviewed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('status', sa.Enum('PENDING', 'DECLINED', 'APPROVED', name='changestatus'), nullable=False), + sa.Column('comment', sa.String(length=1024), nullable=True), + sa.Column('object_type', sa.Enum('SCHEMA', 'ENTITY', name='editableobjecttype'), nullable=False), + sa.Column('object_id', sa.Integer(), nullable=True), + sa.Column('change_type', sa.Enum('CREATE', 'UPDATE', 'DELETE', name='changetype'), nullable=False), + sa.CheckConstraint("object_id IS NOT NULL OR (change_type = 'CREATE' AND status <> 'APPROVED')"), + sa.ForeignKeyConstraint(['created_by_user_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['reviewed_by_user_id'], ['users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('changes', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('change_request_id', sa.Integer(), nullable=False), + sa.Column('attribute_id', sa.Integer(), nullable=True), + sa.Column('object_id', sa.Integer(), nullable=True), + sa.Column('value_id', sa.Integer(), nullable=False), + sa.Column('content_type', sa.Enum('ATTRIBUTE', 'ATTRIBUTE_DEFINITION', 'ENTITY', 'SCHEMA', name='contenttype'), nullable=False), + sa.Column('change_type', sa.Enum('CREATE', 'UPDATE', 'DELETE', name='changetype'), nullable=False), + sa.Column('field_name', sa.String(), nullable=True), + sa.Column('data_type', sa.Enum('STR', 'BOOL', 'INT', 'FLOAT', 'FK', 'DT', 'DATE', name='changeattrtype'), nullable=False), + sa.CheckConstraint('NOT(attribute_id IS NULL AND field_name IS NULL) AND NOT (attribute_id IS NOT NULL AND field_name IS NOT NULL)'), + sa.ForeignKeyConstraint(['attribute_id'], ['attributes.id'], ), + sa.ForeignKeyConstraint(['change_request_id'], ['change_requests.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('values_date', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('value', sa.Date(), nullable=True), + sa.Column('entity_id', sa.Integer(), nullable=True), + sa.Column('attribute_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['attribute_id'], ['attributes.id'], ), + sa.ForeignKeyConstraint(['entity_id'], ['entities.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_values_date_id'), 'values_date', ['id'], unique=False) + op.add_column('schemas', sa.Column('reviewable', sa.Boolean(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('schemas', 'reviewable') + op.drop_index(op.f('ix_values_date_id'), table_name='values_date') + op.drop_table('values_date') + op.drop_table('changes') + op.drop_table('change_requests') + op.drop_index(op.f('ix_change_values_str_id'), table_name='change_values_str') + op.drop_table('change_values_str') + op.drop_index(op.f('ix_change_values_int_id'), table_name='change_values_int') + op.drop_table('change_values_int') + op.drop_index(op.f('ix_change_values_float_id'), table_name='change_values_float') + op.drop_table('change_values_float') + op.drop_index(op.f('ix_change_values_fk_id'), table_name='change_values_fk') + op.drop_table('change_values_fk') + op.drop_index(op.f('ix_change_values_datetime_id'), table_name='change_values_datetime') + op.drop_table('change_values_datetime') + op.drop_index(op.f('ix_change_values_date_id'), table_name='change_values_date') + op.drop_table('change_values_date') + op.drop_index(op.f('ix_change_values_bool_id'), table_name='change_values_bool') + op.drop_table('change_values_bool') + # ### end Alembic commands ### diff --git a/backend/alembic/versions/ef9a7b3217c3_bound_fk.py b/backend/alembic/versions/ef9a7b3217c3_bound_fk.py new file mode 100644 index 0000000..20ecfd3 --- /dev/null +++ b/backend/alembic/versions/ef9a7b3217c3_bound_fk.py @@ -0,0 +1,41 @@ +"""Bound FK + +Revision ID: ef9a7b3217c3 +Revises: 5e7b010c57b8 +Create Date: 2022-02-04 08:55:06.006728 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ef9a7b3217c3' +down_revision = '5e7b010c57b8' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_bound_foreign_keys_id', table_name='bound_foreign_keys') + op.drop_table('bound_foreign_keys') + op.add_column('attr_definitions', sa.Column('bound_schema_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'attr_definitions', 'schemas', ['bound_schema_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'attr_definitions', type_='foreignkey') + op.drop_column('attr_definitions', 'bound_schema_id') + op.create_table('bound_foreign_keys', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('attr_def_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('schema_id', sa.INTEGER(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['attr_def_id'], ['attr_definitions.id'], name='bound_foreign_keys_attr_def_id_fkey'), + sa.ForeignKeyConstraint(['schema_id'], ['schemas.id'], name='bound_foreign_keys_schema_id_fkey'), + sa.PrimaryKeyConstraint('id', name='bound_foreign_keys_pkey') + ) + op.create_index('ix_bound_foreign_keys_id', 'bound_foreign_keys', ['id'], unique=False) + # ### end Alembic commands ### diff --git a/backend/auth/__init__.py b/backend/auth/__init__.py index ffff387..8c8422a 100644 --- a/backend/auth/__init__.py +++ b/backend/auth/__init__.py @@ -11,10 +11,12 @@ from ..config import settings as s from .context import pwd_context from ..database import get_db -from ..models import User, Group, Schema, Entity +from ..models import Schema, Entity from ..schemas.auth import RequirePermission from .backends import get from .crud import get_user, has_permission +from .enum import PermissionType +from .models import User, Group _enabled_backends = get() diff --git a/backend/auth/backends/abstract.py b/backend/auth/backends/abstract.py index 02d90ba..76b18d9 100644 --- a/backend/auth/backends/abstract.py +++ b/backend/auth/backends/abstract.py @@ -4,7 +4,7 @@ from fastapi.security import OAuth2PasswordBearer from sqlalchemy.orm.session import Session -from ...models import User +from ..models import User class AbstractBackend(ABC): diff --git a/backend/auth/backends/ldap.py b/backend/auth/backends/ldap.py index e91d263..a005ca3 100644 --- a/backend/auth/backends/ldap.py +++ b/backend/auth/backends/ldap.py @@ -4,9 +4,9 @@ from ldap3 import Server, Connection, LEVEL, ALL_ATTRIBUTES, SYNC, Entry from ...config import settings -from ...models import User from ...schemas.auth import UserCreateSchema from ..crud import get_or_create_user +from ..models import User from .abstract import AbstractBackend diff --git a/backend/auth/backends/local.py b/backend/auth/backends/local.py index 3978e10..61a08cc 100644 --- a/backend/auth/backends/local.py +++ b/backend/auth/backends/local.py @@ -3,7 +3,7 @@ from .abstract import AbstractBackend from ..context import pwd_context from ..crud import get_user -from ...models import User +from ..models import User class Backend(AbstractBackend): diff --git a/backend/auth/crud.py b/backend/auth/crud.py index 30e66aa..19a825d 100644 --- a/backend/auth/crud.py +++ b/backend/auth/crud.py @@ -7,7 +7,8 @@ from ..database import STATEMENTS from .. import exceptions -from ..models import User, Group, Permission, UserGroup, Schema, Entity +from ..models import Schema, Entity +from .models import User, Group, Permission, UserGroup from ..schemas.auth import BaseGroupSchema, PermissionSchema, GroupSchema, \ UserCreateSchema, RequirePermission, PermissionWithIdSchema from .enum import PermissionType, PermissionTargetType, RecipientType diff --git a/backend/auth/models.py b/backend/auth/models.py new file mode 100644 index 0000000..0baaaec --- /dev/null +++ b/backend/auth/models.py @@ -0,0 +1,87 @@ +from sqlalchemy import Column, Integer, String, ForeignKey, Boolean, Enum +from sqlalchemy.orm import backref, relationship +from sqlalchemy.sql.schema import UniqueConstraint + +from ..base_models import Base +from .enum import RecipientType, PermissionTargetType, PermissionType + +class Group(Base): + __tablename__ = 'groups' + + id = Column(Integer, primary_key=True) + name = Column(String(128), unique=True, nullable=False) + parent_id = Column(Integer, ForeignKey('groups.id')) + parent = relationship('Group', remote_side=[id], backref=backref('subgroups')) + + def __str__(self): + return self.name + + +class User(Base): + __tablename__ = 'users' + id = Column(Integer, primary_key=True) + username = Column(String(128), unique=True, nullable=False) + email = Column(String(128), unique=True, nullable=False) + password = Column(String, nullable=True) + firstname = Column(String(128), nullable=True) + lastname = Column(String(128), nullable=True) + is_active = Column(Boolean, default=True) + + def __str__(self): + return self.username + + +class Permission(Base): + __tablename__ = 'permissions' + id = Column(Integer, primary_key=True) + recipient_type = Column(Enum(RecipientType), nullable=False) + recipient_id = Column(Integer, nullable=False) + obj_type = Column(Enum(PermissionTargetType), nullable=True) + obj_id = Column(Integer, nullable=True) + permission = Column(Enum(PermissionType), nullable=False) + + user = relationship('User', + primaryjoin="and_(User.id == foreign(Permission.recipient_id), " + "Permission.recipient_type == 'USER')", + overlaps="group") + group = relationship('Group', + primaryjoin="and_(Group.id == foreign(Permission.recipient_id), " + "Permission.recipient_type == 'GROUP')", + overlaps="user") + schema = relationship('Schema', + primaryjoin="and_(Schema.id == foreign(Permission.obj_id), " + "Permission.obj_type == 'SCHEMA')", + overlaps="entity, managed_group" + ) + entity = relationship('Entity', + primaryjoin="and_(Entity.id == foreign(Permission.obj_id), " + "Permission.obj_type == 'ENTITY')", + overlaps="schema, managed_group" + ) + managed_group = relationship('Group', + primaryjoin="and_(Group.id == foreign(Permission.obj_id), " + "Permission.obj_type == 'GROUP')", + overlaps="schema, entity" + ) + + __table_args__ = ( + UniqueConstraint('recipient_type', 'recipient_id', 'obj_type', 'obj_id', 'permission'), + ) + + @property + def recipient_name(self): + return self.user.username if self.recipient_type == RecipientType.USER else self.group.name + + +class UserGroup(Base): + __tablename__ = 'user_groups' + id = Column(Integer, primary_key=True) + group_id = Column(Integer, ForeignKey('groups.id'), nullable=False) + user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + + group = relationship('Group') + user = relationship('User') + + __table_args__ = ( + UniqueConstraint('user_id', 'group_id'), + ) diff --git a/backend/base_models.py b/backend/base_models.py new file mode 100644 index 0000000..6ec3eb3 --- /dev/null +++ b/backend/base_models.py @@ -0,0 +1,33 @@ +import typing +from typing import NamedTuple + +from .database import Base + +from sqlalchemy import Column, Integer, ForeignKey +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import relationship + + +class Value(Base): + __abstract__ = True + + id = Column(Integer, primary_key=True, index=True) + + @declared_attr + def entity_id(cls): + return Column(Integer, ForeignKey('entities.id')) + + @declared_attr + def attribute_id(cls): + return Column(Integer, ForeignKey('attributes.id')) + + @declared_attr + def entity(cls): + return relationship('Entity') + + +class Mapping(NamedTuple): + model: Value + converter: type + filters: list = [] + diff --git a/backend/config.py b/backend/config.py index cdc1537..8a5af7c 100644 --- a/backend/config.py +++ b/backend/config.py @@ -42,6 +42,7 @@ class Settings(BaseSettings): test_pg_port: Optional[int] = 5432 test_pg_db: Optional[str] + query_limit: Optional[int] = 10 timezone_offset: Union[str, int] = "utc" auth_backends: str = "local" diff --git a/backend/crud.py b/backend/crud.py index d39d62c..87190d3 100644 --- a/backend/crud.py +++ b/backend/crud.py @@ -1,37 +1,35 @@ -from typing import Callable, Dict, List, Tuple, Union -from collections import defaultdict +from typing import Callable, Dict, Tuple +from collections import defaultdict, Counter +from itertools import chain -import sqlalchemy -from sqlalchemy import func, distinct, column +from psycopg2.errors import ForeignKeyViolation +from sqlalchemy import func, distinct, column, asc, desc, or_, select, update from sqlalchemy.orm import Session -from sqlalchemy import select, update +from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.sql.expression import delete, intersect from sqlalchemy.sql.selectable import CompoundSelect +from .enum import FilterEnum from .models import ( AttrType, Attribute, AttributeDefinition, - BoundFK, - Entity, Schema, Value ) from .schemas import ( AttrDefSchema, - AttrDefWithAttrDataSchema, - AttributeDefinitionUpdateSchema, EntityListSchema, SchemaCreateSchema, SchemaUpdateSchema, AttributeCreateSchema ) from .exceptions import * +from .utils import iterate_model_fields RESERVED_ATTR_NAMES = ['id', 'slug', 'deleted', 'name'] -RESERVED_SCHEMA_SLUGS = ['schemas', 'attributes'] def get_attributes(db: Session) -> List[Attribute]: @@ -84,32 +82,37 @@ def get_schema(db: Session, id_or_slug: Union[int, str]) -> Schema: return schema -def create_schema(db: Session, data: SchemaCreateSchema) -> Schema: - if data.slug in RESERVED_SCHEMA_SLUGS: - raise ReservedSchemaSlugException(slug=data.slug, reserved=RESERVED_SCHEMA_SLUGS) +def _check_bound_schema_id(db: Session, schema_id: int): try: - sch = Schema(name=data.name, slug=data.slug) + db.query(Schema.id).filter(Schema.id == schema_id, Schema.deleted == False).one() + except NoResultFound: + raise MissingSchemaException(obj_id=schema_id) + + +def create_schema(db: Session, data: SchemaCreateSchema, commit: bool = True) -> Schema: + try: + sch = Schema(name=data.name, slug=data.slug, reviewable=data.reviewable) db.add(sch) db.flush() - except sqlalchemy.exc.IntegrityError: + except IntegrityError: db.rollback() raise SchemaExistsException(name=data.name, slug=data.slug) try: attr_names = set() for attr in data.attributes: - if isinstance(attr, AttrDefSchema): - a: Attribute = db.execute(select(Attribute).where(Attribute.id == attr.attr_id)).scalar() - if a is None: - raise MissingAttributeException(attr.attr_id) - elif isinstance(attr, AttrDefWithAttrDataSchema): - a = create_attribute(db, attr, commit=False) - db.flush() + a = create_attribute(db, attr, commit=False) + db.flush() if a.name in attr_names: raise MultipleAttributeOccurencesException(a.name) attr_names.add(a.name) + if a.type == AttrType.FK: + if attr.bound_schema_id == -1: + attr.bound_schema_id = sch.id + _check_bound_schema_id(db=db, schema_id=attr.bound_schema_id) + ad = AttributeDefinition( attribute_id=a.id, schema_id=sch.id, @@ -117,28 +120,16 @@ def create_schema(db: Session, data: SchemaCreateSchema) -> Schema: list=attr.list, unique=attr.unique, key=attr.key, - description=attr.description + description=attr.description, + bound_schema_id=attr.bound_schema_id ) db.add(ad) db.flush() - if a.type == AttrType.FK: - if attr.bind_to_schema is None: - raise NoSchemaToBindException(attr_id=a.id) - if attr.bind_to_schema == -1: - s = sch - else: - s = db.execute( - select(Schema) - .where(Schema.id == attr.bind_to_schema) - .where(Schema.deleted == False) - ).scalar() - if s is None: - raise MissingSchemaException(obj_id=attr.bind_to_schema) - bfk = BoundFK(attr_def_id=ad.id, schema_id=s.id) - db.add(bfk) - - db.commit() - except sqlalchemy.exc.IntegrityError: + if commit: + db.commit() + else: + db.flush() + except IntegrityError: db.rollback() raise SchemaExistsException(name=data.name, slug=data.slug) except: @@ -147,7 +138,7 @@ def create_schema(db: Session, data: SchemaCreateSchema) -> Schema: return sch -def delete_schema(db: Session, id_or_slug: Union[int, str]) -> Schema: +def delete_schema(db: Session, id_or_slug: Union[int, str], commit: bool = True) -> Schema: q = select(Schema).where(Schema.deleted == False) if isinstance(id_or_slug, int): q = q.where(Schema.id == id_or_slug) @@ -158,113 +149,166 @@ def delete_schema(db: Session, id_or_slug: Union[int, str]) -> Schema: if schema is None: raise MissingSchemaException(obj_id=id_or_slug) - db.execute( - update(Entity).where(Entity.schema_id == schema.id).values(deleted=True) - ) + db.execute(update(Entity) + .where(Entity.schema_id == schema.id, Entity.deleted == False) + .values(deleted=True)) schema.deleted = True - db.commit() + if commit: + db.commit() + else: + db.flush() return schema -def update_schema(db: Session, id_or_slug: Union[int, str], data: SchemaUpdateSchema) -> Schema: - if data.slug in RESERVED_SCHEMA_SLUGS: - raise ReservedSchemaSlugException(slug=data.slug, reserved=RESERVED_SCHEMA_SLUGS) - - q = select(Schema).where(Schema.deleted == False) - if isinstance(id_or_slug, int): - q = q.where(Schema.id == id_or_slug) - else: - q = q.where(Schema.slug == id_or_slug) - sch: Schema = db.execute(q).scalar() - if sch is None: + +def _delete_attr_from_schema(db: Session, attr_def: AttributeDefinition, schema: Schema): + ValueModel = attr_def.attribute.type.value.model + db.execute(delete(AttributeDefinition).where(AttributeDefinition.id == attr_def.id)) + db.execute(delete(ValueModel) + .where(ValueModel.attribute_id == attr_def.attribute_id) + .where(ValueModel.entity_id == Entity.id) + .where(Entity.schema_id == schema.id) + .execution_options(synchronize_session=False) + ) + + +def _update_attr_in_schema(db: Session, attr_upd: AttrDefSchema, attr_def: AttributeDefinition): + if attr_def.list and not attr_upd.list: + raise ListedToUnlistedException(attr_def_id=attr_def.id) + + attr_def.required = attr_upd.required + attr_def.unique = False if attr_upd.list else attr_upd.unique + attr_def.list = attr_upd.list + attr_def.key = attr_upd.key + attr_def.description = attr_upd.description + + if attr_upd.name != attr_def.attribute.name: + attr_def.attribute = create_attribute( + db=db, + data=AttributeCreateSchema(name=attr_upd.name, type=attr_def.attribute.type.name), + commit=False + ) + + +def _add_attr_to_schema(db: Session, attr_schema: AttrDefSchema, schema: Schema): + attribute = create_attribute(db, attr_schema, commit=False) + db.flush() + bound_schema_id = attr_schema.bound_schema_id if attr_schema.bound_schema_id != -1 else schema.id + if bound_schema_id is not None and bound_schema_id != schema.id: + _check_bound_schema_id(db=db, schema_id=bound_schema_id) + try: + attr_def = AttributeDefinition( + attribute_id=attribute.id, + schema_id=schema.id, + required=attr_schema.required, + list=attr_schema.list, + unique=attr_schema.unique if not attr_schema.list else False, + key=attr_schema.key, + description=attr_schema.description, + bound_schema_id=bound_schema_id + ) + db.add(attr_def) + db.flush() + except IntegrityError as error: + db.rollback() + if isinstance(error.orig, ForeignKeyViolation): + raise MissingSchemaException(obj_id=attr_schema.bound_schema_id) + raise AttributeAlreadyDefinedException(attr_id=attribute.id, schema_id=schema.id) + + +def sort_attribute_definitions(schema: Schema, definitions: List[AttrDefSchema])\ + -> Tuple[List[AttrDefSchema], List[AttrDefSchema], List[AttributeDefinition]]: + existing_defs = {x.id: x for x in schema.attr_defs} + new, updated = [], [] + fields = iterate_model_fields(AttributeDefinition) + skipped_fields = ["schema_id", "attribute_id", "name", "type", "bound_schema_id"] + + for attr_def in definitions: + if not getattr(attr_def, "id", None): + new.append(attr_def) + continue + existing = existing_defs.get(attr_def.id, None) + if existing is None: + raise AttributeNotDefinedException(attr_id=attr_def.id, schema_id=schema.id) + if existing.attribute.name != attr_def.name: + updated.append(attr_def) + bound_schema_id = attr_def.bound_schema_id if attr_def.bound_schema_id != -1 else existing.bound_schema_id + if bound_schema_id != existing.bound_schema_id: + updated.append(attr_def) + continue + if any(getattr(attr_def, field) != getattr(existing, field) + for field in fields + if field not in skipped_fields): + updated.append(attr_def) + continue + + submitted_ids = {a.id for a in definitions} + deleted = [a for a in existing_defs.values() if a.id not in submitted_ids] + return new, updated, deleted + + +def update_schema(db: Session, id_or_slug: Union[int, str], data: SchemaUpdateSchema, commit: bool = True) -> Schema: + schema = get_schema(db=db, id_or_slug=id_or_slug) + if schema.deleted: raise MissingSchemaException(obj_id=id_or_slug) - + + duplicate_attr_names = [name + for name, count in Counter(a.name for a in data.attributes).items() + if count > 1] + if duplicate_attr_names: + raise MultipleAttributeOccurencesException(attr_name=duplicate_attr_names[0]) + try: - db.execute( + result = db.execute( update(Schema) - .where(Schema.id == sch.id) - .values(name=data.name, slug=data.slug) + .where(Schema.id == schema.id, + or_(Schema.name != data.name, Schema.slug != data.slug, + Schema.reviewable != (data.reviewable or Schema.reviewable.default.arg))) + .values( + name=data.name or schema.name, + slug=data.slug or schema.slug, + reviewable=data.reviewable if data.reviewable is not None else schema.reviewable) ) - except sqlalchemy.exc.IntegrityError: + except IntegrityError: db.rollback() raise SchemaExistsException(name=data.name, slug=data.slug) - attr_def_ids: Dict[int, AttributeDefinition] = {i.id: i for i in sch.attr_defs} - attr_def_names: Dict[str, AttributeDefinition] = {i.attribute.name: i for i in sch.attr_defs} - for attr in data.update_attributes: - if isinstance(attr, AttributeDefinitionUpdateSchema): - attr_def = attr_def_ids.get(attr.attr_def_id) - else: - attr_def = attr_def_names.get(attr.name) - + added, updated, deleted = sort_attribute_definitions(schema=schema, definitions=data.attributes) + + # Are there meaningful changes? + intersect_update_delete = {(d.name, d.type.name) for d in updated} & {(d.attribute.name, d.attribute.type.name) for d in deleted} + intersect_add_delete = {(d.name, d.type.name) for d in added} & {(d.attribute.name, d.attribute.type.name) for d in deleted} + + if result.rowcount == 0 and intersect_update_delete | intersect_add_delete: + raise NoOpChangeException("Trivial change: Attempt to add/update and delete an attribute at" + " the same time") + + attr_def_names: Dict[int, AttributeDefinition] = {i.id: i for i in schema.attr_defs} + + for attr_def in deleted: + _delete_attr_from_schema(db=db, attr_def=attr_def, schema=schema) + + for attr in updated: + attr_def = attr_def_names.get(attr.id) if attr_def is None: - raise AttributeNotDefinedException(attr_id=attr.attr_def_id, schema_id=sch.id) - if attr_def.list and not attr.list: - raise ListedToUnlistedException(attr_def_id=attr_def.id) - if attr.list: - attr.unique = False - attr_def.required = attr.required - attr_def.unique = attr.unique - attr_def.list = attr.list - attr_def.key = attr.key - attr_def.description = attr.description + db.rollback() + raise AttributeNotDefinedException(attr_id=attr.id, schema_id=schema.id) + _update_attr_in_schema(db=db, attr_upd=attr, attr_def=attr_def) db.flush() - - attr_names = set() - for attr in data.add_attributes: - if isinstance(attr, AttrDefSchema): - a: Attribute = db.execute(select(Attribute).where(Attribute.id == attr.attr_id)).scalar() - if a is None: - raise MissingAttributeException(obj_id=attr.attr_id) - elif isinstance(attr, AttrDefWithAttrDataSchema): - a = create_attribute(db, attr, commit=False) - db.flush() - - if a.name in attr_names: - raise MultipleAttributeOccurencesException(attr_name=a.name) - elif a.name in attr_def_names: - raise AttributeAlreadyDefinedException(attr_id=a.id, schema_id=sch.id) - attr_names.add(a.name) - - try: - ad = AttributeDefinition( - attribute_id=a.id, - schema_id=sch.id, - required=attr.required, - list=attr.list, - unique=attr.unique if not attr.list else False, - key=attr.key, - description=attr.description - ) - db.add(ad) - db.flush() - except sqlalchemy.exc.IntegrityError: - db.rollback() - raise AttributeAlreadyDefinedException(attr_id=a.id, schema_id=sch.id) - - if a.type == AttrType.FK: - if attr.bind_to_schema is None: - raise NoSchemaToBindException(attr_id=a.id) - if attr.bind_to_schema == -1: - s = sch - else: - s = db.execute( - select(Schema) - .where(Schema.id == attr.bind_to_schema) - .where(Schema.deleted == False) - ).scalar() - if s is None: - raise MissingSchemaException(obj_id=attr.bind_to_schema) - bfk = BoundFK(attr_def_id=ad.id, schema_id=s.id) - db.add(bfk) + for attr in added: + _add_attr_to_schema(db=db, attr_schema=attr, schema=schema) + try: - db.commit() - except sqlalchemy.exc.IntegrityError: + if commit: + db.commit() + else: + db.flush() + except IntegrityError: db.rollback() raise SchemaExistsException(name=data.name, slug=data.slug) - return sch + return schema def _get_entity_data(db: Session, entity: Entity, attr_names: List[str]) -> Dict[str, Any]: @@ -327,64 +371,50 @@ def _get_attr_values_batch(db: Session, entities: List[Entity], attrs_to_include return results -FILTER_MAP = { - 'eq': '__eq__', - 'lt': '__lt__', - 'gt': '__gt__', - 'le': '__le__', - 'ge': '__ge__', - 'ne': '__ne__', - 'contains': 'contains', - 'regexp': 'regexp_match' -} - -ALLOWED_FILTERS = { - AttrType.STR: ['eq', 'lt', 'gt', 'le', 'ge', 'ne', 'contains', 'regexp'], - AttrType.INT: ['eq', 'lt', 'gt', 'le', 'ge', 'ne'], - AttrType.FLOAT: ['eq', 'lt', 'gt', 'le', 'ge', 'ne'], - AttrType.DT: ['eq', 'lt', 'gt', 'le', 'ge', 'ne'], - AttrType.BOOL: ['eq', 'ne'] -} - - -def _parse_filters(filters: dict, attrs: List[str]) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]: - '''Returns tuple of two `dict`s like `{attr_name: {op1: value, op2: value}}`. +def _parse_filters(filters: dict, attrs: List[str]) \ + -> Tuple[Dict[str, Dict[FilterEnum, Any]], Dict[FilterEnum, Any]]: + ''' + Returns tuple of two `dict`s like `{attr_name: {op1: value, op2: value}}`. First `dict` is for attribute filters, second is for `Entity.name` filters ''' + filter_map = {f.value.name: f for f in FilterEnum} attrs_filters = defaultdict(dict) name_filters = {} for f, v in filters.items(): split = f.rsplit('.', maxsplit=1) attr = split[0] - filter = 'eq' if len(split) == 1 else split[-1] + filter = FilterEnum.EQ if len(split) == 1 else filter_map.get(split[-1], None) + if attr != "name" and attr not in attrs: + raise InvalidFilterAttributeException(attr=attr, allowed_attrs=attrs) + if not filter: + raise InvalidFilterOperatorException(attr=attr, filter=split[-1]) + if attr == 'name': name_filters[filter] = v continue - elif attr not in attrs: - raise InvalidFilterAttributeException(attr=attr, allowed_attrs=attrs) - elif filter not in FILTER_MAP: - raise InvalidFilterOperatorException(attr=attr, filter=filter) - attrs_filters[attr][FILTER_MAP[filter]] = v - - name_filters = {k: v for k,v in name_filters.items() if k in FILTER_MAP and k in ALLOWED_FILTERS[AttrType.STR]} + attrs_filters[attr][filter] = v + return attrs_filters, name_filters -def _query_entity_with_filters(filters: dict, schema: Schema, all: bool = False, deleted_only: bool = False) -> CompoundSelect: - '''Returns intersection query of several queries with filters +def _query_entity_with_filters(filters: dict, schema: Schema, all: bool = False, + deleted_only: bool = False) -> CompoundSelect: + ''' + Returns intersection query of several queries with filters to get entities that satisfy all conditions from `filters` ''' - - attrs = {i.attribute.name: i.attribute for i in schema.attr_defs if i.attribute.type in ALLOWED_FILTERS} - attrs_filters, name_filters = _parse_filters(filters=filters, attrs=list(attrs)) + attrs = {i.attribute.name: i.attribute + for i in schema.attr_defs if i.attribute.type.value.filters} + attrs_filters, name_filters = _parse_filters(filters=filters, attrs=attrs.keys()) selects = [] - if name_filters: # since `name` is defined in `Entity`, not in `Value` tables, we need to query it separately + # since `name` is defined in `Entity`, not in `Value` tables, we need to query it separately + if name_filters: q = select(Entity).where(Entity.schema_id == schema.id) if not all: q = q.where(Entity.deleted == deleted_only) for f, v in name_filters.items(): - q = q.where(getattr(Entity.name, FILTER_MAP[f])(v)) + q = q.where(getattr(Entity.name, f.value.op)(v)) selects.append(q) for attr_name, filters in attrs_filters.items(): @@ -394,7 +424,7 @@ def _query_entity_with_filters(filters: dict, schema: Schema, all: bool = False, if not all: q = q.where(Entity.deleted == deleted_only) for filter, value in filters.items(): - q = q.where(getattr(value_model.value, filter)(value)) + q = q.where(getattr(value_model.value, filter.value.op)(value)) q = q.where(value_model.attribute_id == attr.id) selects.append(q) return intersect(*selects) @@ -408,9 +438,15 @@ def get_entities( all: bool = False, deleted_only: bool = False, all_fields: bool = False, - filters: dict = None + filters: dict = None, + order_by: str = 'name', + ascending: bool = True, ) -> EntityListSchema: - + if order_by != 'name': + attrs = [i for i in schema.attr_defs if i.attribute.name == order_by] + if not attrs: + raise AttributeNotDefinedException(order_by, schema.id) + if filters: q = _query_entity_with_filters(filters=filters, schema=schema, all=all, deleted_only=deleted_only) else: @@ -418,23 +454,59 @@ def get_entities( if not all: q = q.where(Entity.deleted == deleted_only) total = db.execute(select(func.count(distinct(column('id')))).select_from(q.subquery())).scalar() - q = q.offset(offset).limit(limit).order_by(Entity.id) + try: + queries = q.selects + q1 = queries[0] + sub = q1.subquery(name='anon_1') + from_ = Entity.__table__.join(sub, Entity.id == sub.c.id) + for idx, i in enumerate(queries[1:]): + sub = i.subquery(name=f'anon_{idx+2}') + from_ = from_.join(sub, Entity.id == sub.c.id) + q = select(Entity).select_from(from_) + except AttributeError: + pass + if order_by != 'name': + attrs = {i.attribute.name: i.attribute for i in schema.attr_defs if i.attribute.type.value.filters} + attr = attrs[order_by] + value_model = attr.type.value.model + direction = asc if ascending else desc + q = q.order_by( + direction( + select(value_model.value) + .where(value_model.attribute_id == attr.id) + .where(value_model.entity_id == Entity.id).scalar_subquery() + ), Entity.name.asc() + ) + else: + direction = 'asc' if ascending else 'desc' + q = q.order_by(getattr(Entity.name, direction)()) + q = q.offset(offset).limit(limit) entities = db.execute(select(Entity).from_statement(q)).scalars().all() + attr_defs = schema.attr_defs if all_fields else [i for i in schema.attr_defs if i.key or i.attribute.name == order_by] + entities = _get_attr_values_batch(db, entities, attr_defs) + return EntityListSchema(total=total, entities=entities) - attr_defs = schema.attr_defs if all_fields else [i for i in schema.attr_defs if i.key] - return EntityListSchema(total=total, entities=_get_attr_values_batch(db, entities, attr_defs)) +def get_entity_by_id(db: Session, entity_id: int) -> Entity: + entity = db.execute(select(Entity).where(Entity.id == entity_id)).scalar() + if entity is None: + raise MissingEntityException(obj_id=entity_id) + return entity -def get_entity(db: Session, id_or_slug: Union[int, str], schema: Schema) -> dict: + +def get_entity_model(db: Session, id_or_slug: Union[int, str], schema: Schema) -> Optional[Entity]: + q = select(Entity).where(Entity.schema_id == schema.id) if isinstance(id_or_slug, int): - e = db.execute(select(Entity).where(Entity.id == id_or_slug)).scalar() + return db.execute(q.where(Entity.id == id_or_slug)).scalar() else: - e = db.execute(select(Entity).where(Entity.slug == id_or_slug)).scalar() + return db.execute(q.where(Entity.slug == id_or_slug)).scalar() + + +def get_entity(db: Session, id_or_slug: Union[int, str], schema: Schema) -> dict: + e = get_entity_model(db=db, id_or_slug=id_or_slug, schema=schema) if e is None: raise MissingEntityException(obj_id=id_or_slug) - if e.schema_id != schema.id: - raise MismatchingSchemaException(entity_id=id_or_slug, schema_id=schema.id) attrs = [i.attribute.name for i in schema.attr_defs] return _get_entity_data(db=db, entity=e, attr_names=attrs) @@ -444,29 +516,24 @@ def _convert_values(attr_def: AttributeDefinition, value: Any, caster: Callable) if isinstance(value, list): if not attr_def.list: raise NotListedAttributeException(attr_name=attr_def.attribute.name, schema_id=attr_def.schema_id) - return [caster(i) for i in value] + return [caster(i) for i in value if i is not None] else: - return [caster(value)] + return [caster(value)] if value is not None else [] def _check_fk_value(db: Session, attr_def: AttributeDefinition, entity_ids: List[int]): - bound_schema_id = db.execute( - select(BoundFK.schema_id) - .where(BoundFK.attr_def_id == attr_def.id) - ).scalar() + entities = {e.id: e + for e in db.query(Entity).filter(Entity.id.in_(entity_ids), + Entity.deleted == False)} for id_ in entity_ids: - entity = db.execute( - select(Entity) - .where(Entity.id == id_) - .where(Entity.deleted == False) - ).scalar() + entity = entities.get(id_, None) if entity is None: raise MissingEntityException(obj_id=id_) - if entity.schema_id != bound_schema_id: + if entity.schema_id != attr_def.bound_schema_id: raise WrongSchemaToBindException( attr_name=attr_def.attribute.name, schema_id=attr_def.schema_id, - bound_schema_id=bound_schema_id, + bound_schema_id=attr_def.bound_schema_id, passed_entity=entity ) @@ -484,7 +551,7 @@ def _check_unique_value(db: Session, attr_def: AttributeDefinition, model: Value raise UniqueValueException(attr_name=attr_def.attribute.name, schema_id=attr_def.schema_id, value=val.value) -def create_entity(db: Session, schema_id: int, data: dict) -> Entity: +def create_entity(db: Session, schema_id: int, data: dict, commit: bool = True) -> Entity: sch: Schema = db.execute( select(Schema).where(Schema.id == schema_id).where(Schema.deleted == False) ).scalar() @@ -509,7 +576,7 @@ def create_entity(db: Session, schema_id: int, data: dict) -> Entity: db.add(e) try: db.flush() - except sqlalchemy.exc.IntegrityError: + except IntegrityError: raise EntityExistsException(slug=slug) for field, value in data.items(): @@ -518,24 +585,27 @@ def create_entity(db: Session, schema_id: int, data: dict) -> Entity: raise AttributeNotDefinedException(attr_id=None, schema_id=schema_id) attr: Attribute = attr_def.attribute - model, caster = attr.type.value + model, caster, _ = attr.type.value values = _convert_values(attr_def=attr_def, value=value, caster=caster) if attr.type == AttrType.FK: _check_fk_value(db=db, attr_def=attr_def, entity_ids=values) - if attr_def.unique and not attr_def.list: + if attr_def.unique and not attr_def.list and values: _check_unique_value(db=db, attr_def=attr_def, model=model, value=values[0]) for val in values: v = model(value=val, entity_id=e.id, attribute_id=attr.id) db.add(v) - db.commit() + if commit: + db.commit() + else: + db.flush() return e -def update_entity(db: Session, id_or_slug: Union[str, int], schema_id: int, data: dict) -> Entity: +def update_entity(db: Session, id_or_slug: Union[str, int], schema_id: int, data: dict, commit: bool = True) -> Entity: q = select(Entity).where(Entity.schema_id == schema_id) q = q.where(Entity.id == id_or_slug) if isinstance(id_or_slug, int) else q.where(Entity.slug == id_or_slug) e = db.execute(q).scalar() - if e is None: + if e is None or e.deleted: raise MissingEntityException(obj_id=id_or_slug) if e.schema.deleted: raise MissingSchemaException(obj_id=e.schema.id) @@ -544,7 +614,7 @@ def update_entity(db: Session, id_or_slug: Union[str, int], schema_id: int, data name = data.pop('name', e.name) try: db.execute(update(Entity).where(Entity.id == e.id).values(slug=slug, name=name)) - except sqlalchemy.exc.IntegrityError: + except IntegrityError: db.rollback() raise EntityExistsException(slug=slug) @@ -555,7 +625,7 @@ def update_entity(db: Session, id_or_slug: Union[str, int], schema_id: int, data raise AttributeNotDefinedException(attr_id=None, schema_id=schema_id) attr: Attribute = attr_def.attribute - model, caster = attr.type.value + model, caster, _ = attr.type.value if value is None: if attr_def.required: raise RequiredFieldException(field=field) @@ -569,7 +639,7 @@ def update_entity(db: Session, id_or_slug: Union[str, int], schema_id: int, data values = _convert_values(attr_def=attr_def, value=value, caster=caster) if attr.type == AttrType.FK: _check_fk_value(db=db, attr_def=attr_def, entity_ids=values) - if attr_def.unique and not attr_def.list: + if attr_def.unique and not attr_def.list and values: _check_unique_value(db=db, attr_def=attr_def, model=model, value=values[0]) db.execute( @@ -581,11 +651,14 @@ def update_entity(db: Session, id_or_slug: Union[str, int], schema_id: int, data v = model(value=val, entity_id=e.id, attribute_id=attr.id) db.add(v) - db.commit() + if commit: + db.commit() + else: + db.flush() return e -def delete_entity(db: Session, id_or_slug: Union[int, str], schema_id: int) -> Entity: +def delete_entity(db: Session, id_or_slug: Union[int, str], schema_id: int, commit: bool = True) -> Entity: q = select(Entity).where(Entity.deleted == False).where(Entity.schema_id == schema_id) if isinstance(id_or_slug, int): q = q.where(Entity.id == id_or_slug) @@ -595,5 +668,8 @@ def delete_entity(db: Session, id_or_slug: Union[int, str], schema_id: int) -> E if e is None: raise MissingEntityException(obj_id=id_or_slug) e.deleted = True - db.commit() + if commit: + db.commit() + else: + db.flush() return e \ No newline at end of file diff --git a/backend/dynamic_routes.py b/backend/dynamic_routes.py index ff3d5f1..35cd4e0 100644 --- a/backend/dynamic_routes.py +++ b/backend/dynamic_routes.py @@ -1,21 +1,30 @@ -from typing import List, Callable, Optional, Union +from typing import List, Optional, Union from dataclasses import make_dataclass +from collections import defaultdict -from fastapi import APIRouter, Depends, HTTPException, status, Query +from fastapi import APIRouter, Depends, HTTPException, status, Query, Response from fastapi.applications import FastAPI from sqlalchemy.orm.session import Session from pydantic import create_model, Field, validator -from pydantic.main import ModelMetaclass +from pydantic.main import BaseModel, ModelMetaclass -from .auth import authorized_user +from .auth import authorized_user, authenticated_user from .auth.enum import PermissionType -from .models import AttrType, Schema, AttributeDefinition, User, Entity +from .auth.models import User +from .config import settings +from .database import get_db +from .enum import FilterEnum +from .models import AttrType, Schema, AttributeDefinition, Entity from . import crud, exceptions, schemas +from .traceability.entity import create_entity_create_request, create_entity_update_request, \ + create_entity_delete_request, apply_entity_create_request, apply_entity_update_request, \ + apply_entity_delete_request + def _make_type(attr_def: AttributeDefinition, optional: bool = False) -> tuple: '''Given `AttributeDefinition` returns a type that will be for type annotations in Pydantic model''' - + kwargs = {'description': attr_def.description, 'alias': attr_def.attribute.name} type_ = (attr_def.attribute.type # AttributeDefinition.Attribute.type -> AttrType .value.model # AttrType.value -> Mapping, Mapping.model -> Value model .value.property.columns[0].type.python_type) # get python type of value column in Value child @@ -23,7 +32,14 @@ def _make_type(attr_def: AttributeDefinition, optional: bool = False) -> tuple: type_ = List[type_] if not attr_def.required or optional: type_ = Optional[type_] - return (type_, Field(description=attr_def.description)) + return (type_, Field(**kwargs)) + + +def _fieldname_for_schema(name: str): + """Returns modified version of `name` if it shadows member of `pydantic.BaseModel`""" + if name in dir(BaseModel): + return name + '__' + return name def _get_entity_request_model(schema: Schema, name: str) -> ModelMetaclass: @@ -44,7 +60,7 @@ def _get_entity_request_model(schema: Schema, name: str) -> ModelMetaclass: in response in any case, it is advised to provide documentation in API that will list all fields and mark them as required/optional. ''' - field_names = [i.attribute.name for i in schema.attr_defs] + field_names = [_fieldname_for_schema(i.attribute.name) for i in schema.attr_defs] types = [_make_type(i, optional=True) for i in schema.attr_defs] fields_types = dict(zip(field_names, types)) @@ -52,7 +68,7 @@ def _get_entity_request_model(schema: Schema, name: str) -> ModelMetaclass: 'id': (int, Field(description='ID of this entity')), 'deleted': (bool, Field(description='Indicates whether this entity is marked as deleted')), 'slug': (str, Field(description='Slug for this entity')), - 'name': (str, Field(description='Name of this entity')) + 'name': (str, Field(description='Name of this entity')), } model = create_model( name, @@ -72,7 +88,7 @@ def _description_for_get_entity(schema: Schema) -> str: return description -def route_get_entity(router: APIRouter, schema: Schema, get_db: Callable): +def route_get_entity(router: APIRouter, schema: Schema): entity_get_schema = _get_entity_request_model(schema=schema, name=f"{schema.slug.capitalize().replace('-', '_')}Get") description = _description_for_get_entity(schema=schema) @@ -82,6 +98,7 @@ def route_get_entity(router: APIRouter, schema: Schema, get_db: Callable): tags=[schema.name], summary=f'Get {schema.name} entity by ID', description=description, + response_model_exclude_unset=True, responses={ 404: { 'description': "Entity with provided ID doesn't exist", @@ -93,7 +110,8 @@ def route_get_entity(router: APIRouter, schema: Schema, get_db: Callable): ) def get_entity(id_or_slug: Union[int, str], db: Session = Depends(get_db)): try: - return crud.get_entity(db=db, id_or_slug=id_or_slug, schema=schema) + res = crud.get_entity(db=db, id_or_slug=id_or_slug, schema=schema) + return res except exceptions.MissingEntityException as e: raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) except exceptions.MismatchingSchemaException as e: @@ -117,18 +135,6 @@ def _description_for_get_entities(schema: Schema) -> str: return description -FILTER_DESCRIPTION = { - 'eq': 'equal to', - 'lt': 'less than', - 'gt': 'greater than', - 'le': 'less than or equal to', - 'ge': 'greater than or equal to', - 'ne': 'not equal to', - 'contains': 'contains substring', - 'regexp': 'matches regular expression' -} - - def _filters_request_model(schema: Schema): '''Creates a dataclass that will be used to capture filters like `.=` @@ -136,28 +142,33 @@ def _filters_request_model(schema: Schema): ''' fields = [] - fields.append(('name', Optional[str], Query(None, description=FILTER_DESCRIPTION['eq']))) - for filter in crud.ALLOWED_FILTERS[AttrType.STR]: - fields.append((f'name_{filter}', Optional[str], Query(None, alias=f'name.{filter}', description=FILTER_DESCRIPTION[filter]))) + fields.append(('name', Optional[str], Query(None, description=FilterEnum.EQ.value.description))) + for filter in AttrType.STR.value.filters: + fields.append((f'name_{filter.value.name}', Optional[str], + Query(None, alias=f'name.{filter.value.name}', description=filter.value.description))) for attr_def in schema.attr_defs: attr = attr_def.attribute - if attr.type not in crud.ALLOWED_FILTERS: + if not attr.type.value.filters: continue type_ = (attr.type # Attribute.type -> AttrType .value.model # AttrType.value -> Mapping, Mapping.model -> Value model .value.property.columns[0].type.python_type) # get python type of value column in Value child # default filter {attr.name} which works as {attr.name}.eq, i.e. for equality filtering - fields.append((attr.name, Optional[type_], Query(None, alias=attr.name, description=FILTER_DESCRIPTION['eq']))) - for filter in crud.ALLOWED_FILTERS[attr.type]: - fields.append((f'{attr.name}_{filter}', Optional[type_], Query(None, alias=f'{attr.name}.{filter}', description=FILTER_DESCRIPTION[filter]))) - - filter_model = make_dataclass(f"{schema.slug.capitalize().replace('-', '_')}Filters", fields=fields) + fields.append((attr.name, Optional[type_], + Query(None, alias=attr.name, description=FilterEnum.EQ.value.description))) + for filter in attr.type.value.filters: + fields.append((f'{attr.name}_{filter.value.name}', Optional[type_], + Query(None, alias=f'{attr.name}.{filter.value.name}', + description=filter.value.description))) + + filter_model = make_dataclass(f"{schema.slug.capitalize().replace('-', '_')}Filters", + fields=fields) return filter_model -def route_get_entities(router: APIRouter, schema: Schema, get_db: Callable): +def route_get_entities(router: APIRouter, schema: Schema): entity_schema = _get_entity_request_model(schema=schema, name=f"{schema.slug.capitalize().replace('-', '_')}ListItem") description = _description_for_get_entities(schema=schema) filter_model = _filters_request_model(schema=schema) @@ -175,12 +186,14 @@ def route_get_entities(router: APIRouter, schema: Schema, get_db: Callable): response_model_exclude_unset=True ) def get_entities( - limit: int = Query(None, min=0, description='Limit results to `limit` entities'), + limit: int = Query(settings.query_limit, min=0, description='Limit results to `limit` entities'), offset: int = Query(None, min=0, description='Take an offset of `offset` when retreiving entities'), all: bool = Query(False, description='If true, returns both deleted and not deleted entities'), deleted_only: bool = Query(False, description='If true, returns only deleted entities. *Note:* if `all` is true `deleted_only` is not checked'), - all_fields: bool = Query(False, description='If true, returns data for all entity fields, not just key ones'), + all_fields: bool = Query(False, description='If true, returns data for all entity fields, not just key ones'), filters: filter_model = Depends(), + order_by: str = Query('name', description='Ordering field'), + ascending: bool = Query(True, description='Direction of ordering'), db: Session = Depends(get_db) ): filters = {k: v for k, v in filters.__dict__.items() if v is not None} @@ -199,8 +212,11 @@ def get_entities( all=all, deleted_only=deleted_only, all_fields=all_fields, - filters=new_filters - ) # these two exceptions are not supposed to be ever raised + filters=new_filters, + order_by=order_by, + ascending=ascending + ) + # these two exceptions are not supposed to be ever raised except exceptions.InvalidFilterAttributeException as e: raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(e)) except exceptions.InvalidFilterOperatorException as e: @@ -219,7 +235,7 @@ def _create_entity_request_model(schema: Schema) -> ModelMetaclass: This model will raise exception if passed fields that don't belong to it. ''' - field_names = [i.attribute.name for i in schema.attr_defs] + field_names = [_fieldname_for_schema(i.attribute.name) for i in schema.attr_defs] types = [_make_type(i) for i in schema.attr_defs] fields_types = dict(zip(field_names, types)) @@ -237,14 +253,23 @@ class Config: return model -def route_create_entity(router: APIRouter, schema: Schema, get_db: Callable): +def route_create_entity(router: APIRouter, schema: Schema): entity_create_schema = _create_entity_request_model(schema=schema) + req_permission = authenticated_user + if schema.reviewable: + req_permission = authorized_user(schemas.RequirePermission( + permission=PermissionType.CREATE_ENTITY, + target=Schema() + )) + @router.post( f'/{schema.slug}', - response_model=schemas.EntityBaseSchema, + response_model=Union[schemas.EntityBaseSchema, schemas.ChangeRequestSchema], tags=[schema.name], summary=f'Create new {schema.name} entity', responses={ + 200: {"description": "Entity was created"}, + 202: {"description": "Request to create entity was stored"}, 404: { 'description': '''Can be returned when: @@ -269,10 +294,17 @@ def route_create_entity(router: APIRouter, schema: Schema, get_db: Callable): } } ) - def create_entity(data: entity_create_schema, db: Session = Depends(get_db), - user: User = Depends(authorized_user(schemas.RequirePermission(permission=PermissionType.CREATE_ENTITY, target=Schema())))): + def create_entity(data: entity_create_schema, response: Response, db: Session = Depends(get_db), + user: User = Depends(req_permission)): try: - return crud.create_entity(db=db, schema_id=schema.id, data=data.dict()) + change_request = create_entity_create_request( + db=db, schema_id=schema.id, data=data.dict(), created_by=user, commit=False) + if not schema.reviewable: + return apply_entity_create_request(db=db, change_request=change_request, + reviewed_by=user, comment='Autosubmit')[1] + db.commit() + response.status_code = status.HTTP_202_ACCEPTED + return change_request except exceptions.MissingSchemaException as e: raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) except exceptions.EntityExistsException as e: @@ -301,7 +333,7 @@ def _update_entity_request_model(schema: Schema) -> ModelMetaclass: This model will raise exception if passed fields that don't belong to it. ''' - field_names = [i.attribute.name for i in schema.attr_defs] + field_names = [_fieldname_for_schema(i.attribute.name) for i in schema.attr_defs] types = [_make_type(i, optional=True) for i in schema.attr_defs] fields_types = dict(zip(field_names, types)) @@ -319,14 +351,23 @@ class Config: return model -def route_update_entity(router: APIRouter, schema: Schema, get_db: Callable): +def route_update_entity(router: APIRouter, schema: Schema): entity_update_schema = _update_entity_request_model(schema=schema) + req_permission = authenticated_user + if schema.reviewable: + req_permission = authorized_user(schemas.RequirePermission( + permission=PermissionType.UPDATE_ENTITY, + target=Entity() + )) + @router.put( f'/{schema.slug}/{{id_or_slug}}', - response_model=schemas.EntityBaseSchema, + response_model=Union[schemas.EntityBaseSchema, schemas.ChangeRequestSchema], tags=[schema.name], summary=f'Update {schema.name} entity', responses={ + 200: {"description": "Entity was updated"}, + 202: {"description": "Request to update entity was stored"}, 404: { 'description': '''Can be returned when: @@ -354,11 +395,20 @@ def route_update_entity(router: APIRouter, schema: Schema, get_db: Callable): } } ) - def update_entity(id_or_slug: Union[int, str], data: entity_update_schema, - db: Session = Depends(get_db), - user: User = Depends(authorized_user(schemas.RequirePermission(permission=PermissionType.UPDATE_ENTITY, target=Entity())))): + def update_entity(id_or_slug: Union[int, str], data: entity_update_schema, response: Response, + db: Session = Depends(get_db), user: User = Depends(req_permission)): try: - return crud.update_entity(db=db, id_or_slug=id_or_slug, schema_id=schema.id, data=data.dict(exclude_unset=True)) + change_request = create_entity_update_request( + db=db, id_or_slug=id_or_slug, schema_id=schema.id, created_by=user, + data=data.dict(exclude_unset=True), commit=False + ) + if not schema.reviewable: + return apply_entity_update_request( + db=db, change_request=change_request, reviewed_by=user, comment='Autosubmit' + )[1] + db.commit() + response.status_code = status.HTTP_202_ACCEPTED + return change_request except exceptions.MissingEntityException as e: raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) except exceptions.MissingSchemaException as e: @@ -373,44 +423,63 @@ def update_entity(id_or_slug: Union[int, str], data: entity_update_schema, raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(e)) -def route_delete_entity(router: APIRouter, schema: Schema, get_db: Callable): +def route_delete_entity(router: APIRouter, schema: Schema): + req_permission = authenticated_user + if schema.reviewable: + req_permission = authorized_user(schemas.RequirePermission( + permission=PermissionType.DELETE_ENTITY, + target=Entity() + )) + @router.delete( f'/{schema.slug}/{{id_or_slug}}', - response_model=schemas.EntityBaseSchema, + response_model=Union[schemas.EntityBaseSchema, schemas.ChangeRequestSchema], tags=[schema.name], summary=f'Delete {schema.name} entity', responses={ + 200: {"description": "Entity was deleted"}, + 202: {"description": "Request to delete entity was stored"}, 404: { 'description': "entity with provided id/slug doesn't exist on current schema" } } ) - def delete_entity(id_or_slug: Union[int, str], db: Session = Depends(get_db), - user: User = Depends(authorized_user(schemas.RequirePermission(permission=PermissionType.DELETE_ENTITY, target=Entity())))): + def delete_entity(id_or_slug: Union[int, str], response: Response, + db: Session = Depends(get_db), + user: User = Depends(req_permission)): try: - return crud.delete_entity(db=db, id_or_slug=id_or_slug, schema_id=schema.id) + change_request = create_entity_delete_request( + db=db, id_or_slug=id_or_slug, schema_id=schema.id, created_by=user, commit=False + ) + if not schema.reviewable: + return apply_entity_delete_request( + db=db, change_request=change_request, reviewed_by=user, comment='Autosubmit' + )[1] + db.commit() + response.status_code = status.HTTP_202_ACCEPTED + return change_request except exceptions.MissingEntityException as e: raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) -def create_dynamic_router(schema: Schema, app: FastAPI, get_db: Callable, old_slug: str = None): +def create_dynamic_router(schema: Schema, app: FastAPI, old_slug: str = None): router = APIRouter() - route_get_entities(router=router, schema=schema, get_db=get_db) - route_get_entity(router=router, schema=schema, get_db=get_db) - route_create_entity(router=router, schema=schema, get_db=get_db) - route_update_entity(router=router, schema=schema, get_db=get_db) - route_delete_entity(router=router, schema=schema, get_db=get_db) + route_get_entities(router=router, schema=schema) + route_get_entity(router=router, schema=schema) + route_create_entity(router=router, schema=schema) + route_update_entity(router=router, schema=schema) + route_delete_entity(router=router, schema=schema) router_routes = [(r.path, r.methods) for r in router.routes] routes_to_remove = [] for route in app.routes: if (route.path, route.methods) in router_routes: routes_to_remove.append(route) - elif old_slug and (route.path.startswith(f'/{old_slug}/') or route.path == f'/{old_slug}'): + elif old_slug and (route.path.startswith(f'/entity/{old_slug}/') or route.path == f'/entity/{old_slug}'): routes_to_remove.append(route) for route in routes_to_remove: app.routes.remove(route) - app.include_router(router, prefix='') + app.include_router(router, prefix='/entity') app.openapi_schema = None diff --git a/backend/enum.py b/backend/enum.py new file mode 100644 index 0000000..a3cee8b --- /dev/null +++ b/backend/enum.py @@ -0,0 +1,22 @@ +from enum import Enum +from typing import NamedTuple + + +class Filter(NamedTuple): + name: str + op: str + description: str + + +class FilterEnum(Enum): + EQ = Filter('eq', '__eq__', 'equal to') + LT = Filter('lt', '__lt__', 'less than') + GT = Filter('gt', '__gt__', 'greater than') + LE = Filter('le', '__le__', 'less than of equal to') + GE = Filter('ge', '__ge__', 'greater than or equal to') + NE = Filter('ne', '__ne__', 'not equal to') + CONTAINS = Filter('contains', 'contains', 'contains substring') + REGEXP = Filter('regexp', 'regexp_match', 'matches regular expression') + STARTS = Filter('starts', 'startswith', 'starts with substring') + + diff --git a/backend/exceptions.py b/backend/exceptions.py index 7c29357..dacb42b 100644 --- a/backend/exceptions.py +++ b/backend/exceptions.py @@ -46,6 +46,45 @@ class MissingEntityException(MissingObjectException): class MissingAttributeException(MissingObjectException): obj_type = 'Attribute' + +class MissingChangeException(MissingObjectException): + obj_type = 'Change' + + +class MissingChangeRequestException(MissingObjectException): + obj_type = "ChangeRequest" + + +class MissingEntityUpdateRequestException(MissingObjectException): + def __str__(self) -> str: + return f'There is no entity update request with id {self.obj_id}' + + +class MissingEntityDeleteRequestException(MissingObjectException): + def __str__(self) -> str: + return f'There is no entity delete request with id {self.obj_id}' + + +class MissingEntityCreateRequestException(MissingObjectException): + def __str__(self) -> str: + return f'There is no entity create request with id {self.obj_id}' + + +class MissingSchemaCreateRequestException(MissingObjectException): + def __str__(self) -> str: + return f'There is no schema create request with id {self.obj_id}' + + +class MissingSchemaUpdateRequestException(MissingObjectException): + def __str__(self) -> str: + return f'There is no schema update request with id {self.obj_id}' + + +class MissingSchemaDeleteRequestException(MissingObjectException): + def __str__(self) -> str: + return f'There is no schema delete request with id {self.obj_id}' + + class MissingUserException(MissingObjectException): obj_type = 'User' @@ -162,15 +201,6 @@ def __str__(self) -> str: return f"Can't create attribute `{self.attr_name}`. List of reserved attribute names: {', '.join(self.reserved)}" -class ReservedSchemaSlugException(Exception): - def __init__(self, slug: str, reserved: List[str]): - self.slug = slug - self.reserved = reserved - - def __str__(self) -> str: - return f"Can't create schema with slug `{self.slug}`. List of reserved schema slugs: {', '.join(self.reserved)}" - - class InvalidFilterOperatorException(Exception): def __init__(self, attr: str, filter: str): self.attr = attr diff --git a/backend/general_routes.py b/backend/general_routes.py index 17094be..3b6797f 100644 --- a/backend/general_routes.py +++ b/backend/general_routes.py @@ -8,19 +8,28 @@ from .config import settings, VERSION from . import crud, schemas, exceptions, auth from .database import get_db -from .models import User, Schema, Group, Entity +from .enum import FilterEnum +from .models import Schema, AttrType from .dynamic_routes import create_dynamic_router from .auth import authenticated_user, authorized_user, crud as auth_crud from .auth.enum import PermissionType, RecipientType, PermissionTargetType +from .auth.models import User, Group from .schemas.auth import ( GroupSchema, RequirePermission, PermissionSchema, - # GroupDetailsSchema, - UserSchema, UserIDSchema, + UserSchema, UserCreateSchema, BaseGroupSchema, Token ) -from .schemas.info import InfoModel +from .schemas.info import InfoModel, FilterModel +from .schemas.traceability import ChangeRequestSchema, SchemaChangeRequestSchema +from .traceability.crud import review_changes, get_pending_change_requests, \ + is_user_authorized_to_review +from .traceability.entity import get_recent_entity_changes, entity_change_details +from .traceability.enum import ContentType +from .traceability.schema import create_schema_create_request, create_schema_update_request, \ + create_schema_delete_request, apply_schema_create_request, apply_schema_update_request, \ + apply_schema_delete_request, get_recent_schema_changes, schema_change_details router = APIRouter() @@ -28,7 +37,13 @@ @router.get("/info", response_model=InfoModel) def get_info(): - return {"version": VERSION} + return { + "version": VERSION, + "filters": [FilterModel(name=f.value.name, description=f.value.description) + for f in FilterEnum], + "filters_per_type": {atype.name: [filter.value.name for filter in atype.value.filters] + for atype in AttrType} + } @router.get( @@ -53,7 +68,7 @@ def get_attribute(attr_id: int, db: Session = Depends(get_db)): @router.get( - '/schemas', + '/schema', response_model=List[schemas.SchemaForListSchema], tags=['General routes'] ) @@ -66,18 +81,20 @@ def get_schemas( @router.post( - '/schemas', + '/schema', response_model=schemas.SchemaForListSchema, tags=['General routes'] ) def create_schema(data: schemas.SchemaCreateSchema, request: Request, db: Session = Depends(get_db), user: User = Depends(authorized_user(RequirePermission(permission=PermissionType.CREATE_SCHEMA)))): try: - schema = crud.create_schema(db=db, data=data) - create_dynamic_router(schema=schema, app=request.app, get_db=get_db) + change_request = create_schema_create_request(db=db, data=data, created_by=user, + commit=False) + _, schema = apply_schema_create_request(db=db, change_request=change_request, + reviewed_by=user, comment='Autosubmit') + db.commit() + create_dynamic_router(schema=schema, app=request.app) return schema - except exceptions.ReservedSchemaSlugException as e: - raise HTTPException(status.HTTP_409_CONFLICT, str(e)) except exceptions.SchemaExistsException as e: raise HTTPException(status.HTTP_409_CONFLICT, str(e)) except exceptions.MissingAttributeException as e: @@ -91,7 +108,7 @@ def create_schema(data: schemas.SchemaCreateSchema, request: Request, db: Sessio @router.get( - '/schemas/{id_or_slug}', + '/schema/{id_or_slug}', response_model=schemas.SchemaDetailSchema, tags=['General routes'] ) @@ -103,20 +120,30 @@ def get_schema(id_or_slug: Union[int, str], db: Session = Depends(get_db)): @router.put( - '/schemas/{id_or_slug}', - response_model=schemas.SchemaDetailSchema, + '/schema/{id_or_slug}', + response_model=schemas.SchemaBaseSchema, tags=['General routes'] ) -def update_schema(data: schemas.SchemaUpdateSchema, id_or_slug: Union[int, str], request: Request, - db: Session = Depends(get_db), - user: User = Depends(authorized_user(RequirePermission(permission=PermissionType.UPDATE_SCHEMA, target=Schema())))): +def update_schema( + data: schemas.SchemaUpdateSchema, + id_or_slug: Union[int, str], + request: Request, + db: Session = Depends(get_db), + user: User = Depends(authorized_user(RequirePermission(permission=PermissionType.UPDATE_SCHEMA, target=Schema()))) + ): try: old_slug = crud.get_schema(db=db, id_or_slug=id_or_slug).slug - schema = crud.update_schema(db=db, id_or_slug=id_or_slug, data=data) - create_dynamic_router(schema=schema, old_slug=old_slug, app=request.app, get_db=get_db) + change_request = create_schema_update_request( + db=db, id_or_slug=id_or_slug, data=data, created_by=user, commit=False + ) + _, schema = apply_schema_update_request( + db=db, change_request=change_request, reviewed_by=user, comment='Autosubmit' + ) + db.commit() + create_dynamic_router(schema=schema, old_slug=old_slug, app=request.app) return schema - except exceptions.ReservedSchemaSlugException as e: - raise HTTPException(status.HTTP_409_CONFLICT, str(e)) + except exceptions.NoOpChangeException as e: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(e)) except exceptions.MissingAttributeException as e: raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) except exceptions.MissingSchemaException as e: @@ -136,7 +163,7 @@ def update_schema(data: schemas.SchemaUpdateSchema, id_or_slug: Union[int, str], @router.delete( - '/schemas/{id_or_slug}', + '/schema/{id_or_slug}', response_model=schemas.SchemaDetailSchema, response_model_exclude=['attributes', 'attr_defs'], tags=['General routes'] @@ -144,11 +171,95 @@ def update_schema(data: schemas.SchemaUpdateSchema, id_or_slug: Union[int, str], def delete_schema(id_or_slug: Union[int, str], db: Session = Depends(get_db), user: User = Depends(authorized_user(RequirePermission(permission=PermissionType.DELETE_SCHEMA)))): try: - return crud.delete_schema(db=db, id_or_slug=id_or_slug) + change_request = create_schema_delete_request( + db=db, id_or_slug=id_or_slug, created_by=user, commit=False + ) + _, schema = apply_schema_delete_request(db=db, change_request=change_request, + reviewed_by=user, comment='Autosubmit') + db.commit() + return schema + except exceptions.MissingSchemaException as e: + raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) + + +@router.post('/changes/review/{request_id}', tags=["Reviews & Changes"], response_model=ChangeRequestSchema) +def reviewchanges(request_id: int, review: schemas.ChangeReviewSchema, response: Response, + db: Session = Depends(get_db), user: User = Depends(authenticated_user)): + try: + is_user_authorized_to_review(db=db, user=user, request_id=request_id) + change_request, changed = review_changes(db=db, change_request_id=request_id, review=review, + reviewed_by=user) + if not changed: + response.status_code = status.HTTP_208_ALREADY_REPORTED + return change_request + except exceptions.MissingObjectException as e: + raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) + + +@router.get('/changes/pending', tags=["Reviews & Changes"], + response_model=List[schemas.ChangeRequestSchema]) +def get_pending_changerequests( + obj_type: Optional[ContentType] = Query(None), + limit: Optional[int] = Query(10), + offset: Optional[int] = Query(0), + all: Optional[bool] = Query(False), + db: Session = Depends(get_db) +): + return get_pending_change_requests(obj_type=obj_type, limit=limit, offset=offset, + all=all, db=db) + + +@router.get('/changes/schema/{id_or_slug}', tags=["Reviews & Changes"], + response_model=schemas.SchemaChangeRequestSchema) +def get_schema_changes(id_or_slug: Union[int, str], count: Optional[int] = Query(5), + db: Session = Depends(get_db)): + try: + schema = crud.get_schema(db=db, id_or_slug=id_or_slug) + schema_changes, pending_entity_requests = get_recent_schema_changes(db=db, schema_id=schema.id, count=count) + return SchemaChangeRequestSchema(schema_changes=schema_changes, pending_entity_requests=pending_entity_requests) except exceptions.MissingSchemaException as e: raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) +@router.get('/changes/detail/schema/{change_id}', tags=["Reviews & Changes"]) +def get_schema_change_details(change_id: int, db: Session = Depends(get_db)): + try: + return schema_change_details(db=db, change_request_id=change_id) + except exceptions.MissingObjectException as e: + raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) + + +@router.get( + '/changes/entity/{schema_id_or_slug}/{entity_id_or_slug}', tags=["Reviews & Changes"], + response_model=List[schemas.ChangeRequestSchema] +) +def get_entity_changes(schema_id_or_slug: Union[int, str], entity_id_or_slug: Union[int, str], count: Optional[int] = Query(5), db: Session = Depends(get_db)): + try: + schema = crud.get_schema(db=db, id_or_slug=schema_id_or_slug) + entity = crud.get_entity_model(db=db, id_or_slug=entity_id_or_slug, schema=schema) + if entity is None: + raise exceptions.MissingEntityException(obj_id=entity_id_or_slug) + return get_recent_entity_changes(db=db, entity_id=entity.id, count=count) + except exceptions.MissingObjectException as e: + raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) + + +@router.get( + '/changes/detail/entity/{change_id}', tags=["Reviews & Changes"], + response_model=schemas.EntityChangeDetailSchema +) +def get_entity_change_details(change_id: int, db: Session = Depends(get_db)): + try: + return entity_change_details(db=db, change_request_id=change_id) + except exceptions.MissingObjectException as e: + raise HTTPException(status.HTTP_404_NOT_FOUND, str(e)) + + +@router.get('/entities/{entity_id}', response_model=schemas.EntityByIdSchema) +def get_entity_by_id(entity_id: int, db: Session = Depends(get_db)): + return crud.get_entity_by_id(db=db, entity_id=entity_id) + + @router.get('/groups', response_model=List[GroupSchema], tags=["Auth"]) def get_groups(db: Session = Depends(get_db), user: User = Depends(authenticated_user)): return auth_crud.get_groups(db=db) diff --git a/backend/models.py b/backend/models.py index 3912eab..880e676 100644 --- a/backend/models.py +++ b/backend/models.py @@ -1,37 +1,18 @@ import enum -from datetime import datetime -from typing import List, Union, Optional, NamedTuple +from typing import List, Union, Optional from sqlalchemy import ( - select, Enum, DateTime, + select, Enum, DateTime, Date, Boolean, Column, ForeignKey, Integer, String, Float) -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship from sqlalchemy.orm.session import Session -from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.sql.schema import UniqueConstraint -from .auth.enum import PermissionType, PermissionTargetType, RecipientType -from .config import settings +from .base_models import Value, Mapping from .database import Base - - -class Value(Base): - __abstract__ = True - - id = Column(Integer, primary_key=True, index=True) - - @declared_attr - def entity_id(cls): - return Column(Integer, ForeignKey('entities.id')) - - @declared_attr - def attribute_id(cls): - return Column(Integer, ForeignKey('attributes.id')) - - @declared_attr - def entity(cls): - return relationship('Entity') +from .enum import FilterEnum +from .utils import make_aware_datetime class ValueBool(Value): @@ -63,42 +44,25 @@ class ValueDatetime(Value): __tablename__ = 'values_datetime' value = Column(DateTime(timezone=True)) - -class Mapping(NamedTuple): - model: Value - converter: type - - -def make_aware_datetime(dt: datetime) -> datetime: - if dt and dt.tzinfo is None: - return dt.replace(tzinfo=settings.timezone) - return dt +class ValueDate(Value): + __tablename__ = 'values_date' + value = Column(Date) class AttrType(enum.Enum): - STR = Mapping(ValueStr, str) - BOOL = Mapping(ValueBool, bool) - INT = Mapping(ValueInt, int) - FLOAT = Mapping(ValueFloat, float) + STR = Mapping(ValueStr, str, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, FilterEnum.LE, + FilterEnum.GE, FilterEnum.NE, FilterEnum.CONTAINS, + FilterEnum.REGEXP, FilterEnum.STARTS]) + BOOL = Mapping(ValueBool, bool, [FilterEnum.EQ, FilterEnum.NE]) + INT = Mapping(ValueInt, int, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, FilterEnum.LE, + FilterEnum.GE, FilterEnum.NE]) + FLOAT = Mapping(ValueFloat, float, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, FilterEnum.LE, + FilterEnum.GE, FilterEnum.NE]) FK = Mapping(ValueForeignKey, int) - DT = Mapping(ValueDatetime, make_aware_datetime) - - -class BoundFK(Base): - __tablename__ = 'bound_foreign_keys' - - id = Column(Integer, primary_key=True, index=True) - attr_def_id = Column(Integer, ForeignKey('attr_definitions.id')) - schema_id = Column(Integer, ForeignKey('schemas.id')) - - attr_def = relationship('AttributeDefinition') - schema = relationship( - 'Schema', - doc='''Points to schema that is bound - to BoundFK.attr_def which is - not necessarily the same as - BoundFK.attr_def.schema''' - ) + DT = Mapping(ValueDatetime, make_aware_datetime, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, + FilterEnum.LE, FilterEnum.GE, FilterEnum.NE]) + DATE = Mapping(ValueDate, lambda x: x, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, + FilterEnum.LE, FilterEnum.GE, FilterEnum.NE]) class Schema(Base): @@ -108,9 +72,11 @@ class Schema(Base): name = Column(String(128), unique=True) slug = Column(String(128), unique=True) deleted = Column(Boolean, default=False) + reviewable = Column(Boolean, default=False) entities = relationship('Entity', back_populates='schema') - attr_defs = relationship('AttributeDefinition', back_populates='schema') + attr_defs = relationship('AttributeDefinition', back_populates='schema', + foreign_keys="[AttributeDefinition.schema_id]") class Entity(Base): @@ -172,93 +138,13 @@ class AttributeDefinition(Base): unique = Column(Boolean) key = Column(Boolean) list = Column(Boolean, default=False) - description = Column(String) + description = Column(String(128)) + bound_schema_id = Column(Integer, ForeignKey('schemas.id'), nullable=True) - schema = relationship('Schema', back_populates='attr_defs') + schema = relationship('Schema', back_populates='attr_defs', foreign_keys=[schema_id]) + bound_schema = relationship('Schema', foreign_keys=[bound_schema_id]) attribute = relationship('Attribute', back_populates='attr_defs') __table_args__ = ( UniqueConstraint('schema_id', 'attribute_id'), ) - - -class Group(Base): - __tablename__ = 'groups' - - id = Column(Integer, primary_key=True) - name = Column(String(128), unique=True, nullable=False) - parent_id = Column(Integer, ForeignKey('groups.id')) - parent = relationship('Group', remote_side=[id], backref=backref('subgroups')) - - def __str__(self): - return self.name - - -class User(Base): - __tablename__ = 'users' - id = Column(Integer, primary_key=True) - username = Column(String(128), unique=True, nullable=False) - email = Column(String(128), unique=True, nullable=False) - password = Column(String, nullable=True) - firstname = Column(String(128), nullable=True) - lastname = Column(String(128), nullable=True) - is_active = Column(Boolean, default=True) - - def __str__(self): - return self.username - - -class Permission(Base): - __tablename__ = 'permissions' - id = Column(Integer, primary_key=True) - recipient_type = Column(Enum(RecipientType), nullable=False) - recipient_id = Column(Integer, nullable=False) - obj_type = Column(Enum(PermissionTargetType), nullable=True) - obj_id = Column(Integer, nullable=True) - permission = Column(Enum(PermissionType), nullable=False) - - user = relationship('User', - primaryjoin="and_(User.id == foreign(Permission.recipient_id), " - "Permission.recipient_type == 'USER')", - overlaps="group") - group = relationship('Group', - primaryjoin="and_(Group.id == foreign(Permission.recipient_id), " - "Permission.recipient_type == 'GROUP')", - overlaps="user") - schema = relationship('Schema', - primaryjoin="and_(Schema.id == foreign(Permission.obj_id), " - "Permission.obj_type == 'SCHEMA')", - overlaps="entity, managed_group" - ) - entity = relationship('Entity', - primaryjoin="and_(Entity.id == foreign(Permission.obj_id), " - "Permission.obj_type == 'ENTITY')", - overlaps="schema, managed_group" - ) - managed_group = relationship('Group', - primaryjoin="and_(Group.id == foreign(Permission.obj_id), " - "Permission.obj_type == 'GROUP')", - overlaps="schema, entity" - ) - - __table_args__ = ( - UniqueConstraint('recipient_type', 'recipient_id', 'obj_type', 'obj_id', 'permission'), - ) - - @property - def recipient_name(self): - return self.user.username if self.recipient_type == RecipientType.USER else self.group.name - - -class UserGroup(Base): - __tablename__ = 'user_groups' - id = Column(Integer, primary_key=True) - group_id = Column(Integer, ForeignKey('groups.id'), nullable=False) - user_id = Column(Integer, ForeignKey('users.id'), nullable=False) - - group = relationship('Group') - user = relationship('User') - - __table_args__ = ( - UniqueConstraint('user_id', 'group_id'), - ) diff --git a/backend/schemas/__init__.py b/backend/schemas/__init__.py index 93a08bd..634d63b 100644 --- a/backend/schemas/__init__.py +++ b/backend/schemas/__init__.py @@ -1,5 +1,5 @@ from .schema import * -# from .entity import * -# from .traceability import * +from .entity import * +from .traceability import * from .auth import * from .info import * diff --git a/backend/schemas/entity.py b/backend/schemas/entity.py new file mode 100644 index 0000000..d045127 --- /dev/null +++ b/backend/schemas/entity.py @@ -0,0 +1,50 @@ +import re +from typing import Dict, List, Union + +from pydantic import BaseModel, validator, Field + +from ..models import Schema + + +def validate_slug(cls, slug: str): + if slug is None: + return slug + if re.match('^[a-zA-Z]+[0-9]*(-[a-zA-Z0-9]+)*$', slug) is None: + raise ValueError(f'`{slug}` is invalid value for slug field') + return slug + + +class EntityBaseSchema(BaseModel): + id: int + slug: str + name: str + deleted: bool + + slug_validator = validator('slug', allow_reuse=True)(validate_slug) + + class Config: + orm_mode = True + + +class EntityByIdSchema(EntityBaseSchema): + schema_slug: str = Field(alias='schema') + + class Config: + orm_mode = True + + @validator('schema_slug', pre=True) + def convert_to_str(cls, v): + if isinstance(v, Schema): + return v.slug + else: + return v + + +class EntityListSchema(BaseModel): + total: int + entities: List[dict] + + +class FilterFields(BaseModel): + operators: Dict[str, List[str]] + fields: Dict[str, Dict[str, str]] diff --git a/backend/schemas/info.py b/backend/schemas/info.py index d056562..b2c9947 100644 --- a/backend/schemas/info.py +++ b/backend/schemas/info.py @@ -1,5 +1,14 @@ +import typing + from pydantic import BaseModel +class FilterModel(BaseModel): + name: str + description: str + + class InfoModel(BaseModel): version: str + filters: typing.List[FilterModel] + filters_per_type: typing.Dict[str, typing.List[str]] diff --git a/backend/schemas/schema.py b/backend/schemas/schema.py index e1644d0..a6eba62 100644 --- a/backend/schemas/schema.py +++ b/backend/schemas/schema.py @@ -1,11 +1,12 @@ import re from enum import Enum -from typing import List, Optional, Union, Any +from typing import List, Optional, Any -from pydantic import BaseModel, validator, Field +from pydantic import BaseModel, validator, Field, root_validator from ..models import AttrType, AttributeDefinition + class AttrTypeMapping(Enum): STR = 'STR' BOOL = 'BOOL' @@ -13,33 +14,49 @@ class AttrTypeMapping(Enum): FLOAT = 'FLOAT' FK = 'FK' DT = 'DT' + DATE = 'DATE' + assert set(AttrType.__members__.keys()) == set(AttrTypeMapping.__members__.keys()) +def validate_attribute_name(cls, v: str): + if v.isidentifier() and re.match('(^_.*)|(.*_$)', v) is None: + return v + raise ValueError( + 'Attribute name must be a valid Python identifier and must not start/end with underscore') + + class AttributeCreateSchema(BaseModel): name: str type: AttrTypeMapping + validate_attribute_name_ = validator('name', allow_reuse=True)(validate_attribute_name) + class AttributeDefinitionBase(BaseModel): required: bool unique: bool list: bool key: bool - description: Optional[str] - bind_to_schema: Optional[int] + description: Optional[str] = None + bound_schema_id: Optional[int] = None + id: Optional[int] = None + + @root_validator(pre=True) + def check_type_and_bound_id(cls, values): + if values.get("type", None) in (AttrTypeMapping.FK, 'FK') \ + and values.get("bound_schema_id") is None: + raise ValueError("Attribute type FK must be bound to a specific schema") + + return values class Config: orm_mode = True allow_population_by_field_name = True -class AttrDefSchema(AttributeDefinitionBase): - attr_id: int = Field(alias='attribute_id') - - -class AttrDefWithAttrDataSchema(AttributeDefinitionBase, AttributeCreateSchema): +class AttrDefSchema(AttributeDefinitionBase, AttributeCreateSchema): @classmethod def from_orm(cls, obj: Any): if isinstance(obj, AttributeDefinition): @@ -48,15 +65,9 @@ def from_orm(cls, obj: Any): return super().from_orm(obj) -class AttributeDefinitionUpdateSchema(AttributeDefinitionBase): - attr_def_id: int - - -class AttributeDefinitionUpdateWithNameSchema(AttributeDefinitionBase): - name: str - - def validate_slug(cls, slug: str): + if slug is None: + return slug if re.match('^[a-zA-Z]+[0-9]*(-[a-zA-Z0-9]+)*$', slug) is None: raise ValueError(f'`{slug}` is invalid value for slug field') return slug @@ -65,17 +76,17 @@ def validate_slug(cls, slug: str): class SchemaCreateSchema(BaseModel): name: str slug: str - attributes: List[Union[AttrDefSchema, AttrDefWithAttrDataSchema]] + reviewable: bool = False + attributes: List[AttrDefSchema] slug_validator = validator('slug', allow_reuse=True)(validate_slug) class SchemaUpdateSchema(BaseModel): - name: str - slug: str - - update_attributes: List[Union[AttributeDefinitionUpdateSchema, AttributeDefinitionUpdateWithNameSchema]] - add_attributes: List[Union[AttrDefSchema, AttrDefWithAttrDataSchema]] + name: Optional[str] + slug: Optional[str] + reviewable: Optional[bool] + attributes: List[AttrDefSchema] = [] slug_validator = validator('slug', allow_reuse=True)(validate_slug) @@ -84,6 +95,7 @@ class SchemaBaseSchema(BaseModel): id: int name: str slug: str + deleted: bool slug_validator = validator('slug', allow_reuse=True)(validate_slug) @@ -98,7 +110,8 @@ class SchemaForListSchema(SchemaBaseSchema): class SchemaDetailSchema(SchemaBaseSchema): deleted: bool - attr_defs: List[AttrDefWithAttrDataSchema] = Field(alias='attributes') + reviewable: bool + attr_defs: List[AttrDefSchema] = Field(alias='attributes') class AttributeSchema(BaseModel): @@ -106,6 +119,8 @@ class AttributeSchema(BaseModel): name: str type: str + validate_attribute_name_ = validator('name', allow_reuse=True)(validate_attribute_name) + @validator('type', pre=True) def convert_to_str(cls, v): if isinstance(v, AttrType) or isinstance(v, AttrTypeMapping): @@ -116,20 +131,4 @@ def convert_to_str(cls, v): raise ValueError('valid types for type field: str, AttrType, AttrTypeMapping') class Config: - orm_mode = True - - -class EntityBaseSchema(BaseModel): - id: int - slug: str - name: str - deleted: bool - - slug_validator = validator('slug', allow_reuse=True)(validate_slug) - class Config: - orm_mode = True - - -class EntityListSchema(BaseModel): - total: int - entities: List[dict] \ No newline at end of file + orm_mode = True \ No newline at end of file diff --git a/backend/schemas/traceability.py b/backend/schemas/traceability.py new file mode 100644 index 0000000..4b65780 --- /dev/null +++ b/backend/schemas/traceability.py @@ -0,0 +1,61 @@ +from datetime import datetime +from typing import List, Optional, Union, Any + +from pydantic import BaseModel, Field, validator + +from ..auth.models import User +from .schema import SchemaBaseSchema +from ..traceability.enum import EditableObjectType, ChangeStatus, ChangeType, ReviewResult + + +class ChangeRequestSchema(BaseModel): + id: Optional[int] + created_at: datetime + reviewed_at: Optional[datetime] + created_by: str + reviewed_by: Optional[str] + status: ChangeStatus + comment: Optional[str] + object_type: EditableObjectType + change_type: ChangeType + + class Config: + orm_mode = True + + @validator('created_by', 'reviewed_by', pre=True) + def convert_to_str(cls, v): + if isinstance(v, User): + return v.username + return v + + +class SchemaChangeRequestSchema(BaseModel): + schema_changes: List[ChangeRequestSchema] + pending_entity_requests: List[ChangeRequestSchema] + + +class ChangeReviewSchema(BaseModel): + result: ReviewResult + comment: Optional[str] + + +class ChangedEntitySchema(BaseModel): + slug: str + name: str + schema_slug: str = Field(alias='schema') + + +class ChangeSchema(BaseModel): + new: Optional[Union[Any, List[Any]]] + old: Optional[Union[Any, List[Any]]] + current: Optional[Union[Any, List[Any]]] + + +class EntityChangeDetailSchema(ChangeRequestSchema): + entity: ChangedEntitySchema + changes: dict[str, ChangeSchema] + + +class SchemaChangeDetailSchema(ChangeRequestSchema): + schema_: Optional[SchemaBaseSchema] = Field(alias='schema') + changes: dict[str, ChangeSchema] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index e813043..ae3e7fa 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone, timedelta import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -6,10 +7,13 @@ from ..auth import get_password_hash, authenticated_user, authorized_user from ..auth.crud import get_or_create_user, get_user, grant_permission from ..auth.enum import RecipientType, PermissionType +from ..auth.models import User from ..database import get_db from ..models import * from ..config import settings as s from ..schemas.auth import UserCreateSchema, PermissionSchema +from ..traceability.enum import EditableObjectType, ChangeType, ContentType +from ..traceability.models import ChangeRequest, Change, ChangeAttrType, ChangeValueInt from .. import create_app @@ -115,7 +119,8 @@ def populate_db(db: Session): required=True, unique=False, list=True, - key=False + key=False, + bound_schema_id=person.id ) nickname_ = AttributeDefinition( schema_id=person.id, @@ -135,8 +140,6 @@ def populate_db(db: Session): ) db.add_all([age_, born_, friends_, nickname_, fav_color_]) db.flush() - bfk = BoundFK(attr_def_id=friends_.id, schema_id=person.id) - db.add(bfk) p1 = Entity(schema_id=person.id, slug='Jack', name='Jack') db.add(p1) @@ -159,7 +162,132 @@ def populate_db(db: Session): unperson = Schema(name="UnPerson", slug="unperson") db.add(unperson) + + time = datetime.now(timezone.utc) + + # Some change requests for entities + for i in range(-12, -3): + change_request = ChangeRequest( + created_at=time+timedelta(hours=i), + created_by=user, + object_type=EditableObjectType.ENTITY, + object_id=1, + change_type=ChangeType.UPDATE + ) + change_1 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.UPDATE, + content_type=ContentType.ENTITY, + field_name='name', + data_type=ChangeAttrType.STR, + value_id=998 + ) + change_2 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.UPDATE, + content_type=ContentType.ENTITY, + field_name='slug', + data_type=ChangeAttrType.STR, + value_id=999 + ) + db.add_all([change_request, change_1, change_2]) + + change_request = ChangeRequest( + created_at=time+timedelta(hours=-9), + created_by=user, + object_type=EditableObjectType.ENTITY, + object_id=1, + change_type=ChangeType.CREATE + ) + change_1 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.CREATE, + content_type=ContentType.ENTITY, + field_name='deleted', + data_type=ChangeAttrType.STR, + value_id=998 + ) + change_2 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.CREATE, + content_type=ContentType.ENTITY, + field_name='deleted', + data_type=ChangeAttrType.STR, + value_id=999 + ) + schema_value = ChangeValueInt(new_value=1) + db.add(schema_value) db.flush() + change_3 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.CREATE, + content_type=ContentType.ENTITY, + field_name='schema_id', + data_type=ChangeAttrType.INT, + value_id=schema_value.id + ) + db.add_all([change_request, change_1, change_2, change_3]) + + # And some change requests for schemas + for i in range(-12, -3): + change_request = ChangeRequest( + created_at=time+timedelta(hours=i), + created_by=user, + object_type=EditableObjectType.SCHEMA, + object_id=1, + change_type=ChangeType.UPDATE + ) + change_1 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.UPDATE, + content_type=ContentType.SCHEMA, + field_name='name', + data_type=ChangeAttrType.STR, + value_id=998 + ) + change_2 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.UPDATE, + content_type=ContentType.SCHEMA, + field_name='slug', + data_type=ChangeAttrType.STR, + value_id=999 + ) + db.add_all([change_request, change_1, change_2]) + + change_request = ChangeRequest( + created_at=time+timedelta(hours=9), + created_by=user, + object_type=EditableObjectType.SCHEMA, + object_id=1, + change_type=ChangeType.CREATE, + ) + change_1 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.CREATE, + content_type=ContentType.SCHEMA, + field_name='deleted', + data_type=ChangeAttrType.STR, + value_id=998 + ) + change_2 = Change( + change_request=change_request, + object_id=1, + change_type=ChangeType.CREATE, + content_type=ContentType.SCHEMA, + field_name='deleted', + data_type=ChangeAttrType.STR, + value_id=999 + ) + db.add_all([change_request, change_1, change_2]) db.commit() @@ -191,6 +319,11 @@ def dbsession(engine, tables) -> Session: session.close() +@pytest.fixture +def testuser(dbsession) -> User: + return get_user(db=dbsession, username=TEST_USER.username) + + @pytest.fixture def client(dbsession): app = create_app(session=dbsession) diff --git a/backend/tests/test_crud_entity.py b/backend/tests/test_crud_entity.py index 361a962..eb4d9c4 100644 --- a/backend/tests/test_crud_entity.py +++ b/backend/tests/test_crud_entity.py @@ -1,11 +1,26 @@ +from datetime import timezone, timedelta, datetime + import pytest -from ..config import * from ..crud import * from ..models import * from ..exceptions import * +def asserts_after_entities_create(db: Session): + born = datetime(1990, 6, 30, tzinfo=timezone.utc) + tz_born = datetime(1983, 10, 31, tzinfo=timezone(timedelta(hours=2))) + persons = db.execute(select(Entity).where(Entity.schema_id == 1)).scalars().all() + assert len(persons) == 5 + assert persons[-2].name == 'John' + assert persons[-2].slug == 'John' + assert persons[-2].get('nickname', db).value == 'john' + assert persons[-2].get('age', db).value == 10 + assert persons[-2].get('born', db).value == born + assert isinstance(persons[-2].get('age', db), ValueInt) + assert [i.value for i in persons[-2].get('friends', db)] == [persons[-3].id, 1] + assert persons[-1].get('born', db).value.astimezone(timezone.utc) == tz_born.astimezone(timezone.utc) + class TestEntityCreate: def test_create(self, dbsession): born = datetime(1990, 6, 30, tzinfo=timezone.utc) @@ -27,6 +42,7 @@ def test_create(self, dbsession): 'born': born } p2 = create_entity(dbsession, schema_id=1, data=p2) + p3 = { 'name': 'Pumpkin Jack', 'slug': 'pumpkin-jack', @@ -37,17 +53,29 @@ def test_create(self, dbsession): } p3 = create_entity(dbsession, schema_id=1, data=p3) - persons = dbsession.execute(select(Entity).where(Entity.schema_id == 1)).scalars().all() - assert len(persons) == 5 - assert persons[-2].name == 'John' - assert persons[-2].slug == 'John' - assert persons[-2].get('nickname', dbsession).value == 'john' - assert persons[-2].get('age', dbsession).value == 10 - assert persons[-2].get('born', dbsession).value == born - assert isinstance(persons[-2].get('age', dbsession), ValueInt) - assert [i.value for i in persons[-2].get('friends', dbsession)] == [p1.id, 1] - assert persons[-1].get('born', dbsession).value == tz_born + asserts_after_entities_create(dbsession) + def test_no_raise_with_empty_optional_single_fk_field(self, dbsession): + attr = dbsession.execute(select(Attribute).where(Attribute.name == 'address')).scalar() + attr_def = AttributeDefinition( + schema_id=1, + attribute=attr, + required=False, + key=False, + unique=False, + list=False + ) + dbsession.add(attr_def) + dbsession.commit() + data = { + 'name': 'Mike', + 'slug': 'Mike', + 'nickname': 'mike', + 'age': 10, + 'friends': [] + } + create_entity(dbsession, schema_id=1, data=data) + def test_raise_on_non_unique_slug(self, dbsession): p1 = { 'name': 'Jack', @@ -228,7 +256,7 @@ def test_raise_on_entity_doesnt_belong_to_schema(self, dbsession): s = Schema(name='test', slug='test') dbsession.add(s) dbsession.flush() - with pytest.raises(MismatchingSchemaException): + with pytest.raises(MissingEntityException): get_entity(dbsession, id_or_slug=1, schema=s) def test_get_entities(self, dbsession): @@ -397,6 +425,31 @@ def test_raise_on_invalid_filters(self, dbsession): get_entities(dbsession, schema=schema, filters=filters).entities +def asserts_after_entities_update(db: Session, born_time: datetime): + e = db.execute(select(Entity).where(Entity.id == 1)).scalar() + assert e.slug == 'test' + assert e.get('age', db).value == 10 + assert e.get('born', db).value.astimezone(timezone.utc) == born_time.astimezone(timezone.utc) + assert [i.value for i in e.get('friends', db)] == [1, 2] + assert e.get('nickname', db) is None + nicknames = db.execute( + select(ValueStr) + .where(Attribute.name == 'nickname') + .join(Attribute) + ).scalars().all() + assert len(nicknames) == 1, "nickname for entity 1 wasn't deleted from database" + + e = db.execute(select(Entity).where(Entity.id == 2)).scalar() + assert e.slug == 'test2' + assert e.get('nickname', db).value == 'test' + nicknames = db.execute( + select(ValueStr) + .where(Attribute.name == 'nickname') + .join(Attribute) + ).scalars().all() + assert len(nicknames) == 1, "nickname for entity 2 wasn't deleted from database" + + class TestEntityUpdate: def test_update(self, dbsession): time = datetime.now(timezone(timedelta(hours=-4))) @@ -407,33 +460,32 @@ def test_update(self, dbsession): 'friends': [1, 2], } update_entity(dbsession, id_or_slug=1, schema_id=1, data=data) - e = dbsession.execute(select(Entity).where(Entity.id == 1)).scalar() - assert e.slug == 'test' - assert e.get('age', dbsession).value == 10 - assert e.get('born', dbsession).value == time - assert [i.value for i in e.get('friends', dbsession)] == [1, 2] - assert e.get('nickname', dbsession) == None - nicknames = dbsession.execute( - select(ValueStr) - .where(Attribute.name == 'nickname') - .join(Attribute) - ).scalars().all() - assert len(nicknames) == 1, "nickname for entity 1 wasn't deleted from database" data = { 'slug': 'test2', 'nickname': 'test' } update_entity(dbsession, id_or_slug='Jane', schema_id=1, data=data) - e = dbsession.execute(select(Entity).where(Entity.id == 2)).scalar() - assert e.slug == 'test2' - assert e.get('nickname', dbsession).value == 'test' - nicknames = dbsession.execute( - select(ValueStr) - .where(Attribute.name == 'nickname') - .join(Attribute) - ).scalars().all() - assert len(nicknames) == 1, "nickname for entity 2 wasn't deleted from database" + asserts_after_entities_update(dbsession, born_time=time) + + def test_no_raise_with_empty_optional_single_fk_field(self, dbsession): + attr = dbsession.execute(select(Attribute).where(Attribute.name == 'address')).scalar() + attr_def = AttributeDefinition( + schema_id=1, + attribute=attr, + required=False, + key=False, + unique=False, + list=False + ) + dbsession.add(attr_def) + dbsession.commit() + data = { + 'name': 'Mike', + 'slug': 'Mike', + 'address': None, + } + update_entity(dbsession, id_or_slug=1, schema_id=1, data=data) def test_raise_on_entity_doesnt_exist(self, dbsession): with pytest.raises(MissingEntityException): @@ -511,15 +563,17 @@ def test_no_raise_on_non_unique_if_existing_is_deleted(self, dbsession): assert e.get('nickname', dbsession).value == 'jane' +def asserts_after_entity_delete(db: Session): + entities = db.execute(select(Entity)).scalars().all() + assert len(entities) == 2 + e = db.execute(select(Entity).where(Entity.id == 1)).scalar() + assert e.deleted + class TestEntityDelete: @pytest.mark.parametrize('id_or_slug', [1, 'Jack']) def test_delete(self, dbsession, id_or_slug): delete_entity(dbsession, id_or_slug=id_or_slug, schema_id=1) - - entities = dbsession.execute(select(Entity)).scalars().all() - assert len(entities) == 2 - e = dbsession.execute(select(Entity).where(Entity.id == 1)).scalar() - assert e.deleted + asserts_after_entity_delete(db=dbsession) @pytest.mark.parametrize('id_or_slug', [1234567, 'qwertyu']) def test_raise_on_entity_doesnt_exist(self, dbsession, id_or_slug): diff --git a/backend/tests/test_crud_schema.py b/backend/tests/test_crud_schema.py index ac0f089..2e835f7 100644 --- a/backend/tests/test_crud_schema.py +++ b/backend/tests/test_crud_schema.py @@ -1,24 +1,21 @@ +from pydantic.error_wrappers import ValidationError import pytest +from sqlalchemy import select, update +from sqlalchemy.orm import Session -from ..config import * -from ..crud import * -from ..models import * -from ..schemas import * -from ..exceptions import * +from ..crud import create_schema, get_schema, get_schemas, update_schema, delete_schema +from ..exceptions import SchemaExistsException, NoSchemaToBindException, MissingSchemaException, \ + NoOpChangeException, ListedToUnlistedException, MultipleAttributeOccurencesException +from ..models import Schema, AttributeDefinition, Attribute, AttrType, Entity +from .. schemas import AttrDefSchema, SchemaCreateSchema, AttrTypeMapping, SchemaUpdateSchema class TestSchemaCreate: - def data_for_test(self, db: Session) -> dict: - color = Attribute(name='color', type=AttrType.STR) - max_speed = Attribute(name='max_speed', type=AttrType.INT) - release_year = Attribute(name='release_year', type=AttrType.DT) - owner = Attribute(name='owner', type=AttrType.FK) - - db.add_all([color, max_speed, release_year, owner]) - db.commit() - + @staticmethod + def data_for_test() -> dict: color_ = AttrDefSchema( - attr_id=color.id, + name='color', + type='STR', required=False, unique=False, list=False, @@ -26,34 +23,31 @@ def data_for_test(self, db: Session) -> dict: description='Color of this car' ) max_speed_ = AttrDefSchema( - attr_id=max_speed.id, + name='max_speed', + type='INT', required=True, unique=False, list=False, key=False ) release_year_ = AttrDefSchema( - attr_id=release_year.id, + name='release_year', + type='DT', required=False, unique=False, list=False, key=False ) owner_ = AttrDefSchema( - attr_id=owner.id, + name='owner', + type='FK', required=True, unique=False, list=False, key=False, - bind_to_schema=1 + bound_schema_id=1 ) return { - 'attrs': { - 'color': color, - 'max_speed': max_speed, - 'release_year': release_year, - 'owner': owner - }, 'attr_defs': { 'color': color_, 'max_speed': max_speed_, @@ -63,98 +57,42 @@ def data_for_test(self, db: Session) -> dict: } def test_create(self, dbsession): - data = self.data_for_test(dbsession) - car = SchemaCreateSchema(name='Car', slug='car', attributes=list(data['attr_defs'].values())) - create_schema(dbsession, data=car) - - car = dbsession.execute(select(Schema).where(Schema.name == 'Car')).scalar() - assert car is not None + data = self.data_for_test() + car = create_schema(dbsession, data=SchemaCreateSchema( + name='Car', slug='car', attributes=list(data['attr_defs'].values()) + )) - attr_defs = dbsession.execute(select(AttributeDefinition).where(AttributeDefinition.schema_id == car.id)).scalars().all() + assert car is not None and not car.reviewable + + attr_defs = dbsession.execute(select(AttributeDefinition).where( + AttributeDefinition.schema_id == car.id)).scalars().all() assert sorted([i.attribute.name for i in attr_defs]) == sorted(data['attr_defs']) color = dbsession.execute( select(AttributeDefinition) - .where(AttributeDefinition.schema_id == car.id) - .where(AttributeDefinition.attribute_id == data['attrs']['color'].id) + .where(AttributeDefinition.schema_id == car.id) + .join(Attribute) + .where(Attribute.name == 'color') ).scalar() assert not any([color.required, color.unique, color.list, color.key]) assert color.description == 'Color of this car' ry = dbsession.execute( select(AttributeDefinition) - .where(AttributeDefinition.schema_id == car.id) - .where(AttributeDefinition.attribute_id == data['attrs']['release_year'].id) + .where(AttributeDefinition.schema_id == car.id) + .join(Attribute) + .where(Attribute.name == 'release_year') ).scalar() assert not any([ry.required, ry.unique, ry.list, ry.key]) assert ry.description is None owner = dbsession.execute( select(AttributeDefinition) - .where(AttributeDefinition.schema_id == car.id) - .where(AttributeDefinition.attribute_id == data['attrs']['owner'].id) + .where(AttributeDefinition.schema_id == car.id) + .join(Attribute) + .where(Attribute.name == 'owner') ).scalar() - bfk = dbsession.execute(select(BoundFK).where(BoundFK.attr_def_id == owner.id)).scalars().all() - assert len(bfk) == 1 and bfk[0].schema.name == 'Person' - - def test_create_with_attr_data(self, dbsession): - data = self.data_for_test(dbsession) - test = SchemaCreateSchema( - name='Test', - slug='test', - attributes=[ - AttrDefWithAttrDataSchema( - name='test1', - type=AttrTypeMapping.STR, - required=True, - unique=True, - list=False, - key=True, - description='Test 1' - ), - AttrDefWithAttrDataSchema( - name='test2', - type=AttrTypeMapping.STR, - required=True, - unique=True, - list=False, - key=True, - description='Test 2' - ), - data['attr_defs']['color'] - ] - ) - create_schema(dbsession, data=test) - - schemas = dbsession.execute(select(Schema)).scalars().all() - assert len(schemas) == 3 - - schema = dbsession.execute(select(Schema).where(Schema.name == 'Test')).scalar() - assert schema is not None - - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'test1')).scalar() - assert attr is not None - - attr2 = dbsession.execute(select(Attribute).where(Attribute.name == 'test2')).scalar() - assert attr2 is not None - - attr_defs = dbsession.execute( - select(AttributeDefinition).where(AttributeDefinition.schema_id == schema.id) - ).scalars().all() - assert len(attr_defs) == 3 - - attr_def = attr_defs[0] - assert attr_def is not None - assert attr_def.attribute == attr - assert all([attr_def.required, attr_def.unique, attr_def.key]) - assert not attr_def.list - assert attr_def.description == 'Test 1' - - def test_raise_on_reserved_slug(self, dbsession): - for i in RESERVED_SCHEMA_SLUGS: - sch = SchemaCreateSchema(name='Person', slug=i, attributes=[]) - with pytest.raises(ReservedSchemaSlugException): - create_schema(dbsession, data=sch) + assert owner.bound_schema.name == 'Person' def test_raise_on_duplicate_name_or_slug(self, dbsession): sch = SchemaCreateSchema(name='Person', slug='test', attributes=[]) @@ -167,43 +105,28 @@ def test_raise_on_duplicate_name_or_slug(self, dbsession): create_schema(dbsession, data=sch) dbsession.rollback() - def test_raise_on_nonexistent_attr_id(self, dbsession): - nonexistent = AttrDefSchema( - attr_id=99999, - required=True, - unique=True, - list=False, - key=True, - description='Nonexistent attribute' - ) - sch = SchemaCreateSchema(name='Test', slug='test', attributes=[nonexistent]) - with pytest.raises(MissingAttributeException): - create_schema(dbsession, data=sch) - def test_raise_on_empty_schema_when_binding(self, dbsession): - data = self.data_for_test(dbsession) - no_schema = AttrDefSchema( - attr_id=data['attrs']['owner'].id, - required=True, - unique=True, - list=False, - key=True, - description='No schema passed for binding', - ) - sch = SchemaCreateSchema(name='Test', slug='test', attributes=[no_schema]) - with pytest.raises(NoSchemaToBindException): - create_schema(dbsession, data=sch) + with pytest.raises(ValidationError): + no_schema = AttrDefSchema( + name='friends', + type='FK', + required=True, + unique=True, + list=False, + key=True, + description='No schema passed for binding', + ) def test_raise_on_nonexistent_schema_when_binding(self, dbsession): - data = self.data_for_test(dbsession) nonexistent = AttrDefSchema( - attr_id=data['attrs']['owner'].id, + name='owner', + type='FK', required=True, unique=True, list=False, key=True, description='Nonexistent schema to bind to', - bind_to_schema=9999999999 + bound_schema_id=9999999999 ) sch = SchemaCreateSchema(name='Test', slug='test', attributes=[nonexistent]) with pytest.raises(MissingSchemaException): @@ -211,14 +134,14 @@ def test_raise_on_nonexistent_schema_when_binding(self, dbsession): def test_raise_on_passed_deleted_schema_for_binding(self, dbsession): dbsession.execute(update(Schema).where(Schema.id == 1).values(deleted=True)) - data = self.data_for_test(dbsession) attr_def = AttrDefSchema( - attr_id=data['attrs']['owner'].id, + name='owner', + type='FK', required=True, unique=True, list=False, key=True, - bind_to_schema=1 + bound_schema_id=1 ) sch = SchemaCreateSchema(name='Test', slug='test', attributes=[attr_def]) @@ -226,13 +149,13 @@ def test_raise_on_passed_deleted_schema_for_binding(self, dbsession): create_schema(dbsession, data=sch) def test_raise_on_multiple_attrs_with_same_name(self, dbsession): - data = self.data_for_test(dbsession) + data = self.data_for_test() test = SchemaCreateSchema( name='Test', slug='test', attributes=[ - AttrDefWithAttrDataSchema( + AttrDefSchema( name='test1', type=AttrTypeMapping.STR, required=True, @@ -241,7 +164,7 @@ def test_raise_on_multiple_attrs_with_same_name(self, dbsession): key=True, description='Test 1' ), - AttrDefWithAttrDataSchema( + AttrDefSchema( name='test1', type=AttrTypeMapping.INT, required=True, @@ -260,7 +183,7 @@ def test_raise_on_multiple_attrs_with_same_name(self, dbsession): name='Test', slug='test', attributes=[ - AttrDefWithAttrDataSchema( + AttrDefSchema( name='color', type=AttrTypeMapping.INT, required=True, @@ -331,191 +254,256 @@ def test_get_deleted_only(self, dbsession): class TestSchemaUpdate: + default_attributes = [ + AttrDefSchema( + id=1, + name='age', + type="INT", + required=True, + unique=False, + list=False, + key=True, + description='Age of this person' + ), + AttrDefSchema( + id=2, + name='born', + type='DT', + required=False, + unique=False, + list=False, + key=False + ), + AttrDefSchema( + id=3, + name='friends', + type='FK', + required=True, + unique=False, + list=True, + key=False, + bound_schema_id=-1 + ), + AttrDefSchema( + id=4, + name='nickname', + type='STR', + required=False, + unique=True, + list=False, + key=False + ), + AttrDefSchema( + id=5, + name='fav_color', + type='STR', + required=False, + unique=False, + list=True, + key=False + ) + ] + def test_update(self, dbsession): - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'address')).scalar() - attr_def = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[ - AttributeDefinitionUpdateSchema( - attr_def_id=attr_def.id, + attributes = [a for a in self.default_attributes if a.name != "age"] + [ + AttrDefSchema( + id=1, + name='age', + type='INT', required=False, unique=False, list=False, key=False, description='Age of this person' - ) - ], - add_attributes=[ + ), AttrDefSchema( - attr_id=attr.id, + name='address', + type='FK', required=True, unique=True, list=True, key=True, - bind_to_schema=-1 + bound_schema_id=-1 ) ] + upd_schema = SchemaUpdateSchema( + slug='test', + reviewable=True, + attributes=attributes, + delete_attributes=['friends'] ) update_schema(dbsession, id_or_slug='person', data=upd_schema) - age_def = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert age_def is not None - assert not any([age_def.required, age_def.unique, age_def.list, age_def.key]) + schema = dbsession.query(Schema).filter(Schema.slug == "test").one() + assert schema.name == 'Person' and schema.slug == 'test' - address_def = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.attribute_id == attr.id) - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert address_def is not None - assert all([address_def.list, address_def.key, address_def.required]) - assert not address_def.unique - - bfk = dbsession.execute( - select(BoundFK) - .where(BoundFK.schema_id == 1) - .where(BoundFK.attr_def_id == address_def.id) - ).scalar() - assert bfk is not None + attrs = {d.attribute.name: d.attribute.type for d in schema.attr_defs} + assert attrs.get("friends") == AttrType.FK + assert attrs.get("address") == AttrType.FK + assert attrs.get("age") == AttrType.INT - sch = dbsession.execute(select(Schema).where(Schema.id == 1)).scalar() - assert sch.name == 'Test' and sch.slug == 'test' + age = next(iter(d for d in schema.attr_defs if d.attribute.name == "age")) + assert not any([age.required, age.unique, age.list, age.key]) - def test_update_attr_def_with_name(self, dbsession): - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[ - AttributeDefinitionUpdateWithNameSchema( - name='age', + address = next(iter(d for d in schema.attr_defs if d.attribute.name == "address")) + assert all([address.list, address.key, address.required]) + assert not address.unique + assert address.bound_schema_id == schema.id + + def test_update_with_renaming(self, dbsession): + attributes = [a for a in self.default_attributes if a.id != 4] + [ + AttrDefSchema( + id=4, + name='nick', + type='STR', required=False, unique=False, list=False, key=False, - description='Age of this person' + description='updated' ) - ], - add_attributes=[] + ] + upd_schema = SchemaUpdateSchema( + name='Test', + slug='test', + attributes=attributes, ) update_schema(dbsession, id_or_slug=1, data=upd_schema) - - age_def = dbsession.execute( + nickname = dbsession.execute(select(Attribute).where(Attribute.name == 'nickname')).scalar() + assert nickname is not None, 'nickname must be still present in DB' + attr_def = dbsession.execute( select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') .where(AttributeDefinition.schema_id == 1) + .join(Attribute) + .where(Attribute.name == 'nick') ).scalar() - assert age_def is not None - assert not any([age_def.required, age_def.unique, age_def.list, age_def.key]) + assert attr_def is not None + assert not any([attr_def.required, attr_def.unique, attr_def.list, attr_def.key]) + assert attr_def.description == 'updated' - def test_update_with_attr_data(self, dbsession, engine): - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'address')).scalar() - attr_def = dbsession.execute( + def test_update_with_renaming_and_adding_new_with_old_name(self, dbsession): + nickname_id = dbsession.execute( select(AttributeDefinition) .join(Attribute) - .where(Attribute.name == 'age') + .where(Attribute.name == 'nickname') .where(AttributeDefinition.schema_id == 1) - ).scalar() - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[ - AttributeDefinitionUpdateSchema( - attr_def_id=attr_def.id, + ).scalar().id + + attributes = [a for a in self.default_attributes if a.id != 4] + [ + AttrDefSchema( + id=4, + name='nick', + type='STR', required=False, unique=False, list=False, key=False, - description='Age of this person' - ) - ], - add_attributes=[ + description='updated' + ), AttrDefSchema( - attr_id=attr.id, + name='nickname', + type='DT', required=True, unique=True, list=True, - key=True, - bind_to_schema=-1 - ), - AttrDefWithAttrDataSchema( - name='test', - type=AttrTypeMapping.FK, - required=False, - unique=False, - list=False, - key=False, - description='Test', - bind_to_schema=-1 + key=True ) ] + + upd_schema = SchemaUpdateSchema( + name='Test', + slug='test', + attributes=attributes ) update_schema(dbsession, id_or_slug=1, data=upd_schema) - - address_def = dbsession.execute( + dbsession.expire_all() + nick_id = dbsession.execute( select(AttributeDefinition) - .where(AttributeDefinition.attribute_id == attr.id) .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert address_def is not None - assert not address_def.unique and address_def.list - - age_def = dbsession.execute( - select(AttributeDefinition) .join(Attribute) - .where(Attribute.name == 'age') + .where(Attribute.name == 'nick') + ).scalar().id + assert nickname_id == nick_id + + nickname = dbsession.execute( + select(AttributeDefinition) .where(AttributeDefinition.schema_id == 1) + .join(Attribute) + .where(Attribute.name == 'nickname') ).scalar() - assert age_def is not None - assert not any([age_def.required, age_def.unique, age_def.list, age_def.key]) - - sch = dbsession.execute(select(Schema).where(Schema.id == 1)).scalar() - assert sch.name == 'Test' and sch.slug == 'test' + assert nickname.attribute.type == AttrType.DT + assert all([nickname.required, nickname.list, nickname.key]) + + def test_raise_on_renaming_to_already_present_attr(self, dbsession): + attributes = [a for a in self.default_attributes if a.id != 4] + [ + AttrDefSchema( + id=4, + name='friends', + type='INT', + required=False, + unique=False, + list=False, + key=False, + description='updated' + ) + ] + upd_schema = SchemaUpdateSchema( + name='Test', + slug='test', + attributes=attributes + ) + with pytest.raises(MultipleAttributeOccurencesException): + update_schema(dbsession, id_or_slug=1, data=upd_schema) + dbsession.rollback() - test_def = dbsession.execute( + def test_update_with_deleting_attr(self, dbsession): + initial_count = len( + dbsession.execute( + select(AttributeDefinition) + .where(AttributeDefinition.schema_id == 1) + ).scalars().all() + ) + init_entities_count = len(dbsession.execute(select(Entity).where(Entity.schema_id == 1)).scalars().all()) + upd_schema = SchemaUpdateSchema( + name='Test', + slug='test', + attributes=[a for a in self.default_attributes if a.name not in ['age', 'born']] + ) + update_schema(dbsession, id_or_slug=1, data=upd_schema) + attr_defs = dbsession.execute( select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'test') .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert test_def is not None - assert test_def.attribute.type == AttrType.FK - bfk = dbsession.execute( - select(BoundFK) - .where(BoundFK.schema_id == 1) - .where(BoundFK.attr_def_id == test_def.id) - ).scalar() - assert bfk - - def test_raise_on_reserved_slig(self, dbsession): - for i in RESERVED_SCHEMA_SLUGS: - upd_schema = SchemaUpdateSchema( - name='Test', - slug=i, - update_attributes=[], - add_attributes=[] - ) - with pytest.raises(ReservedSchemaSlugException): - update_schema(dbsession, id_or_slug=1, data=upd_schema) + ).scalars().all() + assert len(attr_defs) == initial_count - 2 + names = [i.attribute.name for i in attr_defs] + assert 'age' not in names and 'born' not in names + dbsession.expire_all() + new_entities_count = len(dbsession.execute(select(Entity).where(Entity.schema_id == 1)).scalars().all()) + assert init_entities_count == new_entities_count + + def test_raise_on_deleting_and_creating_same_attr(self, dbsession): + attributes = [a for a in self.default_attributes if a.id != 1] + [ + AttrDefSchema( + name='age', + type='INT', + required=True, + unique=True, + list=True, + key=True + )] + upd_schema = SchemaUpdateSchema( + name='Person', + slug='person', + attributes=attributes + ) + with pytest.raises(NoOpChangeException): + update_schema(dbsession, id_or_slug=1, data=upd_schema) def test_raise_on_schema_doesnt_exist(self, dbsession): upd_schema = SchemaUpdateSchema( name='Test', - slug='test', - update_attributes=[], - add_attributes=[] + slug='test', + attributes=[] ) with pytest.raises(MissingSchemaException): update_schema(dbsession, id_or_slug=99999999, data=upd_schema) @@ -525,132 +513,73 @@ def test_raise_on_existing_slug_or_name(self, dbsession): dbsession.add(new_sch) dbsession.flush() - upd_schema = SchemaUpdateSchema(name='Person', slug='test', update_attributes=[], add_attributes=[]) + upd_schema = SchemaUpdateSchema(name='Person', slug='test', attributes=[]) with pytest.raises(SchemaExistsException): update_schema(dbsession, id_or_slug=new_sch.id, data=upd_schema) dbsession.rollback() dbsession.add(new_sch) dbsession.flush() - upd_schema = SchemaUpdateSchema(name='Test', slug='person', update_attributes=[], add_attributes=[]) + upd_schema = SchemaUpdateSchema(name='Test', slug='person', attributes=[]) with pytest.raises(SchemaExistsException): update_schema(dbsession, id_or_slug=new_sch.id, data=upd_schema) - def test_raise_on_attr_def_doesnt_exist(self, dbsession): - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[ - AttributeDefinitionUpdateSchema( - attr_def_id=9999999, - required=True, - unique=True, - list=True, - key=True, - ) - ], - add_attributes=[] - ) - with pytest.raises(AttributeNotDefinedException): - update_schema(dbsession, id_or_slug=1, data=upd_schema) - def test_raise_on_convert_list_to_single(self, dbsession): - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'friends')).scalar() - attr_def = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.schema_id == 1) - .where(AttributeDefinition.attribute_id == attr.id) - ).scalar() - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[ - AttributeDefinitionUpdateSchema( - attr_def_id=attr_def.id, + attributes = [a for a in self.default_attributes if a.id != 3] + [ + AttrDefSchema( + id=3, + name='friends', + type="FK", required=True, unique=True, list=False, key=True, + bound_schema_id=-1 ) - ], - add_attributes=[] - ) - with pytest.raises(ListedToUnlistedException): - update_schema(dbsession, id_or_slug=1, data=upd_schema) - - def test_raise_on_attr_doesnt_exist(self, dbsession): + ] upd_schema = SchemaUpdateSchema( name='Test', slug='test', - update_attributes=[], - add_attributes=[ - AttrDefSchema( - attr_id=99999999999, - required=True, - unique=True, - list=True, - key=True - ) - ] + attributes=attributes ) - with pytest.raises(MissingAttributeException): + with pytest.raises(ListedToUnlistedException): update_schema(dbsession, id_or_slug=1, data=upd_schema) def test_raise_on_attr_def_already_exists(self, dbsession): - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'born')).scalar() - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[], - add_attributes=[ + attributes = self.default_attributes + [ AttrDefSchema( - attr_id=attr.id, + name='born', + type='INT', required=True, unique=True, list=True, key=True ) ] - ) - with pytest.raises(AttributeAlreadyDefinedException): - update_schema(dbsession, id_or_slug=1, data=upd_schema) - dbsession.rollback() - upd_schema = SchemaUpdateSchema( name='Test', - slug='test', - update_attributes=[], - add_attributes=[ - AttrDefWithAttrDataSchema( - name='nickname', - type=AttrTypeMapping.DT, - required=True, - unique=True, - list=True, - key=True - ) - ] + slug='test', + attributes=attributes ) - with pytest.raises(AttributeAlreadyDefinedException): + # TODO currently works the same way as raise_on_multiple_attrs_with_same_name + # this one can be removed + with pytest.raises(MultipleAttributeOccurencesException): update_schema(dbsession, id_or_slug=1, data=upd_schema) + dbsession.rollback() def test_raise_on_nonexistent_schema_when_binding(self, dbsession): - attr = dbsession.execute( - select(Attribute) - .where(Attribute.name == 'address') - ).scalar() upd_schema = SchemaUpdateSchema( name='Test', - slug='test', - update_attributes=[], - add_attributes=[ + slug='test', + attributes=self.default_attributes + [ AttrDefSchema( - attr_id=attr.id, + name='address', + type='FK', required=True, unique=True, list=True, key=True, - bind_to_schema=999999 + bound_schema_id=999999 ) ] ) @@ -658,75 +587,65 @@ def test_raise_on_nonexistent_schema_when_binding(self, dbsession): update_schema(dbsession, id_or_slug=1, data=upd_schema) def test_raise_on_schema_not_passed_when_binding(self, dbsession): - attr = dbsession.execute( - select(Attribute) - .where(Attribute.name == 'address') - ).scalar() - upd_schema = SchemaUpdateSchema( - name='Test', - slug='test', - update_attributes=[], - add_attributes=[ - AttrDefSchema( - attr_id=attr.id, - required=True, - unique=True, - list=True, - key=True, - ) - ] - ) - with pytest.raises(NoSchemaToBindException): - update_schema(dbsession, id_or_slug=1, data=upd_schema) - - def test_raise_on_multiple_attrs_with_same_name(self, dbsession): - address = dbsession.execute( - select(Attribute) - .where(Attribute.name == 'address') - ).scalar() + with pytest.raises(ValidationError): + SchemaUpdateSchema( + name='Test', + slug='test', + attributes=[ + AttrDefSchema( + name='address2', + type='FK', + required=True, + unique=True, + list=True, + key=True, + ) + ] + ) + def test_raise_on_duplicate_attrs(self, dbsession): upd_schema = SchemaUpdateSchema( name='Test', - slug='test', - update_attributes=[], - add_attributes=[ + slug='test', + attributes=[ AttrDefSchema( - attr_id=address.id, + name='address', + type='FK', required=True, unique=True, list=True, key=True, - bind_to_schema=-1 + bound_schema_id=-1 ), AttrDefSchema( - attr_id=address.id, + name='address', + type='FK', required=True, unique=True, list=True, key=True, - bind_to_schema=-1 + bound_schema_id=-1 ) ] ) with pytest.raises(MultipleAttributeOccurencesException): update_schema(dbsession, id_or_slug=1, data=upd_schema) - dbsession.rollback() - + def test_raise_on_multiple_attrs_with_same_name(self, dbsession): upd_schema = SchemaUpdateSchema( name='Test', - slug='test', - update_attributes=[], - add_attributes=[ + slug='test', + attributes=[ AttrDefSchema( - attr_id=address.id, + name='address', + type='FK', required=True, unique=True, list=True, key=True, - bind_to_schema=-1 + bound_schema_id=-1 ), - AttrDefWithAttrDataSchema( + AttrDefSchema( name='address', type=AttrTypeMapping.DT, required=True, @@ -743,11 +662,14 @@ def test_raise_on_multiple_attrs_with_same_name(self, dbsession): class TestSchemaDelete: @pytest.mark.parametrize('id_or_slug', [1, 'person']) def test_delete(self, dbsession, id_or_slug): - delete_schema(dbsession, id_or_slug=id_or_slug) + schema = delete_schema(dbsession, id_or_slug=id_or_slug) + assert schema.deleted - schemas = dbsession.execute(select(Schema).order_by(Schema.id)).scalars().all() + schemas = dbsession.execute(select(Schema)).scalars().all() + deleted_schema = [s for s in schemas if schema.id == s.id] assert len(schemas) == 2 - assert schemas[0].deleted + assert deleted_schema + assert deleted_schema[0].deleted entities = dbsession.execute(select(Entity).where(Entity.schema_id == 1)).scalars().all() assert len(entities) == 2 @@ -761,7 +683,3 @@ def test_raise_on_already_deleted(self, dbsession): def test_raise_on_delete_nonexistent(self, dbsession): with pytest.raises(MissingSchemaException): delete_schema(dbsession, id_or_slug=999999999) - - - - \ No newline at end of file diff --git a/backend/tests/test_dynamic_routes.py b/backend/tests/test_dynamic_routes.py index e1f1160..1a37b8f 100644 --- a/backend/tests/test_dynamic_routes.py +++ b/backend/tests/test_dynamic_routes.py @@ -1,4 +1,4 @@ -from datetime import timezone +from datetime import datetime, timezone, timedelta import pytest from sqlalchemy import update @@ -6,6 +6,11 @@ from ..models import * from ..dynamic_routes import * from .. import load_schemas +from .test_crud_entity import ( + asserts_after_entities_create, + asserts_after_entities_update, + asserts_after_entity_delete +) class TestRouteBasics: @@ -18,19 +23,161 @@ def test_load_schema_data(self, dbsession, client): def test_routes_were_generated(self, dbsession, client): routes = [ - '/person/{id_or_slug}', - '/person', - '/unperson/{id_or_slug}', - '/unperson', + '/entity/person/{id_or_slug}', + '/entity/person', + '/entity/unperson/{id_or_slug}', + '/entity/unperson', '/attributes', '/attributes/{attr_id}', - '/schemas/{id_or_slug}', - '/schemas/{id_or_slug}', - '/schemas' + '/schema/{id_or_slug}', + '/schema/{id_or_slug}', + '/schema' ] assert all([i in [route.path for route in client.app.routes] for i in routes]) +class TestRouteCreateEntity: + def test_create_without_review(self, dbsession, authorized_client): + p1 = { + 'name': 'Mike', + 'slug': 'Mike', + 'nickname': 'mike', + 'age': 10, + 'friends': [], + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 200 + json = response.json() + mike_id = json.pop('id') + assert json == {'slug': 'Mike', 'name': 'Mike', 'deleted': False} + + born = datetime(1990, 6, 30, tzinfo=timezone.utc) + tz_born = datetime(1983, 10, 31, tzinfo=timezone(timedelta(hours=2))) + p2 = { + 'name': 'John', + 'slug': 'John', + 'nickname': 'john', + 'age': 10, + 'friends': [mike_id, 1], + 'born': str(born), + } + response = authorized_client.post(f'/entity/person', json=p2) + assert response.status_code == 200 + json = response.json() + john_id = json.pop('id') + assert json == {'slug': 'John', 'name': 'John', 'deleted': False} + + p3 = { + 'name': 'Pumpkin Jack', + 'slug': 'pumpkin-jack', + 'nickname': 'pumpkin', + 'age': 38, + 'friends': [mike_id, john_id], + 'born': str(tz_born) + } + response = authorized_client.post(f'/entity/person', json=p3) + assert response.status_code == 200 + json = response.json() + del json['id'] + assert json == {'slug': 'pumpkin-jack', 'name': 'Pumpkin Jack', 'deleted': False} + + asserts_after_entities_create(dbsession) + + def test_raise_on_non_unique_slug(self, dbsession, authorized_client): + p1 = { + 'name': 'name', + 'slug': 'Jack', + 'nickname': 'test', + 'age': 10, + 'friends': [] + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 409 + assert 'already exists in this schema' in response.json()['detail'] + + def test_no_raise_on_same_slug_in_different_schemas(self, dbsession, authorized_client): + data = {'slug': 'Jack', 'name': 'name'} + response = authorized_client.post(f'/entity/unperson', json=data) + assert response.status_code == 200 + + entity = dbsession.execute(select(Entity).where(Entity.schema_id == 2)).scalar() + assert entity.slug == 'Jack' and entity.name == 'name' + + def test_raise_on_non_unique_field(self, dbsession, authorized_client): + p1 = { + 'name': 'name', + 'slug': 'Jake', + 'nickname': 'jack', # <-- already exists in db + 'age': 10, + 'friends': [] + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 409 + assert 'Got non-unique value for field' in response.json()['detail'] + + def test_no_raise_on_non_unique_value_if_it_is_deleted(self, dbsession, authorized_client): + jacks = dbsession.execute(select(ValueStr).where(ValueStr.value == 'jack')).scalars().all() + assert len(jacks) == 1 + + dbsession.execute(update(Entity).where(Entity.id == 1).values(deleted=True)) + dbsession.commit() + p1 = { + 'name': 'name', + 'slug': 'Jackie', + 'nickname': 'jack', # <-- already exists in db, but for deleted entity + 'age': 10, + 'friends': [] + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 200 + + entity = dbsession.execute(select(Entity).where(Entity.slug == 'Jackie')).scalar() + assert entity is not None + + def test_raise_on_fk_entity_doesnt_exist(self, dbsession, authorized_client): + p1 = { + 'name': 'name', + 'slug': 'Mike', + 'nickname': 'mike', + 'age': 10, + 'friends': [99999999] + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 404 + assert "doesn't exist or was deleted" in response.json()['detail'] + + def test_raise_on_fk_entity_is_deleted(self, dbsession, authorized_client): + dbsession.execute(update(Entity).where(Entity.id == 1).values(deleted=True)) + dbsession.commit() + p1 = { + 'name': 'name', + 'slug': 'Mike', + 'nickname': 'mike', + 'age': 10, + 'friends': [1] + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 404 + assert "doesn't exist or was deleted" in response.json()['detail'] + + def test_raise_on_fk_entity_from_wrong_schema(self, dbsession, authorized_client): + schema = Schema(name='Test', slug='test') + entity = Entity(schema=schema, slug='test', name='test') + dbsession.add_all([schema, entity]) + dbsession.commit() + + p1 = { + 'name': 'name', + 'slug': 'Mike', + 'nickname': 'mike', + 'age': 10, + 'friends': [entity.id] + } + response = authorized_client.post(f'/entity/person', json=p1) + assert response.status_code == 422 + assert 'got instead entity' in response.json()['detail'] + + class TestRouteGetEntity: def test_get_entity(self, dbsession, client): data = { @@ -44,16 +191,16 @@ def test_get_entity(self, dbsession, client): 'nickname': 'jack', 'fav_color': ['red', 'blue'] } - response = client.get('/person/1') + response = client.get('/entity/person/1') assert response.status_code == 200 - assert response.json() == data + assert response.json() == data - response = client.get(f'/person/Jack') + response = client.get(f'/entity/person/Jack') assert response.status_code == 200 - assert response.json() == data + assert response.json() == data def test_raise_on_entity_doesnt_exist(self, dbsession, client): - response = client.get('/person/99999999') + response = client.get('/entity/person/99999999') assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] @@ -63,12 +210,13 @@ def test_raise_on_entity_doesnt_belong_to_schema(self, dbsession, client): dbsession.add_all([e, s]) dbsession.commit() - response = client.get(f'/person/{e.id}') - assert response.status_code == 409 - assert "doesn't belong to specified Schema" in response.json()['detail'] + response = client.get(f'/entity/person/{e.id}') + assert response.status_code == 404 + assert "doesn't exist or was deleted" in response.json()['detail'] class TestRouteGetEntities: + # TODO tests for ordering @pytest.fixture def jack(self): return {'age': 10, 'id': 1, 'deleted': False, 'slug': 'Jack', 'name': 'Jack'} @@ -86,7 +234,7 @@ def test_get_entities(self, dbsession, client): {'age': 10, 'id': 1, 'deleted': False, 'slug': 'Jack', 'name': 'Jack'}, {'age': 12, 'id': 2, 'deleted': False, 'slug': 'Jane', 'name': 'Jane'} ] - response = client.get(f'/person/') + response = client.get(f'/entity/person/') assert response.status_code == 200 assert response.json()['entities'] == data @@ -95,7 +243,7 @@ def test_get_deleted_only(self, dbsession, client): dbsession.commit() data = [{'age': 12, 'id': 2, 'deleted': True, 'slug': 'Jane', 'name': 'Jane'}] - response = client.get(f'/person?deleted_only=True') + response = client.get(f'/entity/person?deleted_only=True') assert response.status_code == 200 assert response.json()['entities'] == data @@ -107,16 +255,16 @@ def test_get_all(self, dbsession, client): {'age': 10, 'id': 1, 'deleted': False, 'slug': 'Jack', 'name': 'Jack'}, {'age': 12, 'id': 2, 'deleted': True, 'slug': 'Jane', 'name': 'Jane'} ]} - response = client.get(f'/person?all=True') + response = client.get(f'/entity/person?all=True') assert response.status_code == 200 assert response.json() == data - response = client.get(f'/person?all=True&deleted_only=True') + response = client.get(f'/entity/person?all=True&deleted_only=True') assert response.status_code == 200 assert response.json() == data def test_get_all_fields(self, dbsession, client): - data = {'total': 2, 'entities': [ + data = {'total': 2, 'entities': [ { 'age': 10, 'born': None, @@ -139,7 +287,7 @@ def test_get_all_fields(self, dbsession, client): 'fav_color': ['red', 'black'] } ]} - response = client.get(f'/person?all_fields=True') + response = client.get(f'/entity/person?all_fields=True') assert response.status_code == 200 assert response.json() == data @@ -147,19 +295,18 @@ def test_offset_and_limit(self, dbsession, client): jack = {'age': 10, 'id': 1, 'deleted': False, 'slug': 'Jack', 'name': 'Jack'} jane = {'age': 12, 'id': 2, 'deleted': False, 'slug': 'Jane', 'name': 'Jane'} - response = client.get(f'/person?limit=1') + response = client.get(f'/entity/person?limit=1') assert response.status_code == 200 assert response.json() == {'total': 2, 'entities': [jack]} - response = client.get(f'/person?limit=1&offset=1') + response = client.get(f'/entity/person?limit=1&offset=1') assert response.status_code == 200 assert response.json() == {'total': 2, 'entities': [jane]} - response = client.get(f'/person?offset=10') + response = client.get(f'/entity/person?offset=10') assert response.status_code == 200 assert response.json() == {'total': 2, 'entities': []} - @pytest.mark.parametrize(['q', 'resp'], [ ('age=10', ['jack']), ('age.lt=10', []), @@ -179,14 +326,14 @@ def test_offset_and_limit(self, dbsession, client): def test_get_with_filter(self, dbsession, client, request, q, resp): ents = [request.getfixturevalue(i) for i in resp] resp = {'total': len(ents), 'entities': ents} - response = client.get(f'/person?{q}') + response = client.get(f'/entity/person?{q}') assert response.json() == resp def test_get_with_multiple_filters_for_same_attr(self, dbsession, client, jane): - response = client.get('/person?age.gt=9&age.ne=10') + response = client.get('/entity/person?age.gt=9&age.ne=10') assert response.json()['entities'] == [jane] - response = client.get('/person?age.gt=9&age.ne=10&age.lt=12') + response = client.get('/entity/person?age.gt=9&age.ne=10&age.lt=12') assert response.json()['entities'] == [] @pytest.mark.parametrize(['q', 'resp'], [ @@ -197,17 +344,17 @@ def test_get_with_multiple_filters_for_same_attr(self, dbsession, client, jane): ]) def test_get_with_multiple_filters(self, dbsession, client, request, q, resp): resp = [request.getfixturevalue(i) for i in resp] - response = client.get(f'/person?{q}') + response = client.get(f'/entity/person?{q}') assert response.json()['entities'] == resp def test_get_with_filters_and_offset_limit(self, dbsession, client, jack, jane): - response = client.get('/person?age.gt=0&age.lt=20&limit=1') + response = client.get('/entity/person?age.gt=0&age.lt=20&limit=1') assert response.json()['entities'] == [jack] - response = client.get('/person?age.gt=0&age.lt=20&limit=1&offset=1') + response = client.get('/entity/person?age.gt=0&age.lt=20&limit=1&offset=1') assert response.json()['entities'] == [jane] - response = client.get('/person?age.gt=0&age.lt=20&offset=2') + response = client.get('/entity/person?age.gt=0&age.lt=20&offset=2') assert response.json()['entities'] == [] @pytest.mark.parametrize(['q', 'resp'], [ @@ -221,214 +368,24 @@ def test_get_with_filters_and_deleted(self, dbsession, client, request, q, resp) dbsession.execute(update(Entity).where(Entity.slug == 'Jack').values(deleted=True)) dbsession.commit() resp = [request.getfixturevalue(i) for i in resp] - response = client.get(f'/person?{q}') + response = client.get(f'/entity/person?{q}') assert response.json()['entities'] == resp def test_ignore_invalid_filter(self, dbsession, client, jack, jane): - response = client.get('/person?age.gt=0&age.lt=20&qwe.rty=123') + response = client.get('/entity/person?age.gt=0&age.lt=20&qwe.rty=123') assert response.json()['entities'] == [jack, jane] - response = client.get('/person?age.gt=0&age.lt=20&friends.lt=1234') + response = client.get('/entity/person?age.gt=0&age.lt=20&friends.lt=1234') assert response.json()['entities'] == [jack, jane] - response = client.get('/person?age.qwe=1234') + response = client.get('/entity/person?age.qwe=1234') assert response.json()['entities'] == [jack, jane] - def test_ignore_filters_for_list_and_fk(self, dbsession, client, jack, jane): - response = client.get('/person?friends=1') + def test_ignore_filters_for_fk(self, dbsession, client, jack, jane): + response = client.get('/entity/person?friends=1') assert response.json()['entities'] == [jack, jane] -class TestRouteCreateEntity: - def test_create(self, dbsession, authorized_client): - p1 = { - 'name': 'mike', - 'slug': 'Mike', - 'nickname': 'mike', - 'age': 10, - 'friends': [], - } - response = authorized_client.post(f'/person', json=p1) - assert response.json() == {'id': 3, 'slug': 'Mike', 'name': 'mike', 'deleted': False} - - mike = dbsession.execute(select(Entity).where(Entity.id == 3)).scalar() - assert mike.get('nickname', dbsession).value == 'mike' - assert mike.get('age', dbsession).value == 10 - assert mike.get('friends', dbsession) == [] - - p2 = { - 'name': 'john', - 'slug': 'John', - 'nickname': 'john', - 'age': 10, - 'friends': [3, 1], - 'born': '2021-10-20T13:52:17', - } - response = authorized_client.post(f'/person', json=p2) - assert response.json() == {'id': 4, 'slug': 'John', 'name': 'john', 'deleted': False} - - john = dbsession.execute(select(Entity).where(Entity.id == 4)).scalar() - assert john.get('nickname', dbsession).value == 'john' - assert john.get('age', dbsession).value == 10 - assert [i.value for i in john.get('friends', dbsession)] == [3, 1] - assert john.get('born', dbsession).value == datetime(2021, 10, 20, 13, 52, 17, tzinfo=timezone.utc) - - def test_raise_on_non_unique_slug(self, dbsession, authorized_client): - p1 = { - 'name': 'name', - 'slug': 'Jack', - 'nickname': 'test', - 'age': 10, - 'friends': [] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 409 - assert 'already exists in this schema' in response.json()['detail'] - - def test_no_raise_on_same_slug_in_different_schemas(self, authorized_client): - data = {'slug': 'Jack', 'name': 'name'} - response = authorized_client.post(f'/unperson', json=data) - assert response.status_code == 200 - - def test_raise_on_invalid_slug(self, dbsession, authorized_client): - p1 = { - 'slug': '-Jake-', - 'nickname': 'jackie', - 'age': 10, - 'friends': [] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 422 - assert 'is invalid value for slug field' in response.json()['detail'][0]['msg'] - - def test_raise_on_non_unique_field(self, dbsession, authorized_client): - p1 = { - 'name': 'name', - 'slug': 'Jake', - 'nickname': 'jack', # <-- already exists in db - 'age': 10, - 'friends': [] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 409 - assert 'Got non-unique value for field' in response.json()['detail'] - - def test_no_raise_on_non_unique_value_if_it_is_deleted(self, dbsession, authorized_client): - jacks = dbsession.execute(select(ValueStr).where(ValueStr.value == 'jack')).scalars().all() - assert len(jacks) == 1 - - dbsession.execute(update(Entity).where(Entity.id == 1).values(deleted=True)) - dbsession.commit() - p1 = { - 'name': 'name', - 'slug': 'Jackie', - 'nickname': 'jack', # <-- already exists in db, but for deleted entity - 'age': 10, - 'friends': [] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 200 - - def test_raise_on_attr_doesnt_exist(self, dbsession, authorized_client): - p = { - 'name': 'name', - 'slug': 'SomeName', - 'nickname': 'somename', - 'age': 10, - 'friends': [1], - 'nonexistent': True - } - response = authorized_client.post(f'/person', json=p) - assert response.status_code == 422 - assert 'extra fields not permitted' in response.json()['detail'][0]['msg'] - - def test_raise_on_value_cast(self, dbsession, authorized_client): - p = { - 'name': 'name', - 'slug': 'SomeName', - 'nickname': 'somename', - 'age': 'INVALID VALUE', - 'friends': [1], - } - response = authorized_client.post(f'/person', json=p) - assert response.status_code == 422 - assert 'value is not a valid integer' in response.json()['detail'][0]['msg'] - - def test_raise_on_passed_list_for_single_value_attr(self, dbsession, authorized_client): - p = { - 'name': 'name', - 'slug': 'Some name', - 'nickname': 'somename', - 'age': [1, 2, 3], - 'friends': [1], - } - response = authorized_client.post(f'/person', json=p) - assert response.status_code == 422 - assert 'value is not a valid integer' in response.json()['detail'][0]['msg'] - - def test_raise_on_fk_entity_doesnt_exist(self, dbsession, authorized_client): - p1 = { - 'name': 'name', - 'slug': 'Mike', - 'nickname': 'mike', - 'age': 10, - 'friends': [99999999] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 404 - assert "doesn't exist or was deleted" in response.json()['detail'] - - def test_raise_on_fk_entity_is_deleted(self, dbsession, authorized_client): - dbsession.execute(update(Entity).where(Entity.id == 1).values(deleted=True)) - dbsession.commit() - p1 = { - 'name': 'name', - 'slug': 'Mike', - 'nickname': 'mike', - 'age': 10, - 'friends': [1] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 404 - assert "doesn't exist or was deleted" in response.json()['detail'] - - def test_raise_on_fk_entity_from_wrong_schema(self, dbsession, authorized_client): - schema = Schema(name='Test', slug='test') - entity = Entity(schema=schema, slug='test', name='test') - dbsession.add_all([schema, entity]) - dbsession.commit() - - p1 = { - 'name': 'name', - 'slug': 'Mike', - 'nickname': 'mike', - 'age': 10, - 'friends': [entity.id] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 422 - assert 'got instead entity' in response.json()['detail'] - - def test_raise_on_slug_not_provided(self, dbsession, authorized_client): - p1 = { - 'nickname': 'mike', - 'age': 10, - 'friends': [1] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 422 - assert 'field required' in response.json()['detail'][0]['msg'] - - def test_raise_on_required_field_not_provided(self, dbsession, authorized_client): - p1 = { - 'slug': 'Mike', - 'friends': [1] - } - response = authorized_client.post(f'/person', json=p1) - assert response.status_code == 422 - assert 'field required' in response.json()['detail'][0]['msg'] - - class TestRouteUpdateEntity: def test_update(self, dbsession, authorized_client): data = { @@ -438,49 +395,27 @@ def test_update(self, dbsession, authorized_client): 'born': '2021-10-20T13:52:17+03', 'friends': [1, 2], } - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('entity/person/1', json=data) assert response.status_code == 200 assert response.json() == {'id': 1, 'slug': 'test', 'name': 'test', 'deleted': False} - e = dbsession.execute(select(Entity).where(Entity.id == 1)).scalar() - assert e.name == 'test' - assert e.slug == 'test' - assert e.get('age', dbsession).value == 10 - assert e.get('born', dbsession).value == datetime(2021, 10, 20, 10, 52, 17, tzinfo=timezone.utc) - assert [i.value for i in e.get('friends', dbsession)] == [1, 2] - assert e.get('nickname', dbsession) == None - nicknames = dbsession.execute( - select(ValueStr) - .where(Attribute.name == 'nickname') - .join(Attribute) - ).scalars().all() - assert len(nicknames) == 1, "nickname for entity 1 wasn't deleted from database" + born_utc = datetime(2021, 10, 20, 10, 52, 17, tzinfo=timezone.utc) data = { 'slug': 'test2', 'nickname': 'test' } - response = authorized_client.put('person/Jane', json=data) + response = authorized_client.put('/entity/person/Jane', json=data) assert response.status_code == 200 assert response.json() == {'id': 2, 'slug': 'test2', 'name': 'Jane', 'deleted': False} - - e = dbsession.execute(select(Entity).where(Entity.id == 2)).scalar() - assert e.slug == 'test2' - assert e.name == 'Jane' - assert e.get('nickname', dbsession).value == 'test' - nicknames = dbsession.execute( - select(ValueStr) - .where(Attribute.name == 'nickname') - .join(Attribute) - ).scalars().all() - assert len(nicknames) == 1, "nickname for entity 2 wasn't deleted from database" + asserts_after_entities_update(dbsession, born_time=born_utc) def test_raise_on_entity_doesnt_exist(self, dbsession, authorized_client): - response = authorized_client.put('person/99999999999', json={}) + response = authorized_client.put('/entity/person/99999999999', json={}) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] - response = authorized_client.put('person/qwertyuiop', json={}) + response = authorized_client.put('/entity/person/qwertyuiop', json={}) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] @@ -488,62 +423,51 @@ def test_raise_on_entity_doesnt_exist(self, dbsession, authorized_client): e = Entity(slug='test', schema=s, name='test') dbsession.add_all([s, e]) dbsession.commit() - response = authorized_client.put('person/test', json={}) + response = authorized_client.put('/entity/person/test', json={}) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] def test_raise_on_schema_is_deleted(self, dbsession, authorized_client): dbsession.execute(update(Schema).where(Schema.id == 1).values(deleted=True)) dbsession.commit() - response = authorized_client.put('person/1', json={}) + response = authorized_client.put('/entity/person/1', json={}) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] def test_raise_on_entity_already_exists(self, dbsession, authorized_client): data = {'slug': 'Jane'} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('/entity/person/1', json=data) assert response.status_code == 409 assert 'already exists in this schema' in response.json()['detail'] def test_no_raise_on_changing_to_same_slug(self, dbsession, authorized_client): data = {'slug': 'Jack'} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('/entity/person/1', json=data) assert response.status_code == 200 - def test_raise_on_attribute_not_defined_on_schema(self, dbsession, authorized_client): - data = {'not_existing_attr': 50000} - response = authorized_client.put('person/1', json=data) - assert response.status_code == 422 - assert 'extra fields not permitted' in response.json()['detail'][0]['msg'] - - data = {'address': 1234} - response = authorized_client.put('person/1', json=data) - assert response.status_code == 422 - assert 'extra fields not permitted' in response.json()['detail'][0]['msg'] - def test_raise_on_invalid_slug(self, dbsession, authorized_client): p1 = { 'slug': '-Jake-', } - response = authorized_client.put(f'/person/1', json=p1) + response = authorized_client.put('/entity/person/1', json=p1) assert response.status_code == 422 assert 'is invalid value for slug field' in response.json()['detail'][0]['msg'] def test_raise_on_deleting_required_value(self, dbsession, authorized_client): data = {'age': None} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('entity/person/1', json=data) assert response.status_code == 422 assert 'Missing required field' in response.json()['detail'] def test_raise_on_passing_list_for_not_listed_attr(self, dbsession, authorized_client): data = {'age': [1, 2, 3, 4, 5]} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('entity/person/1', json=data) assert response.status_code == 422 assert 'value is not a valid integer' in response.json()['detail'][0]['msg'] def test_raise_on_fk_entity_doesnt_exist(self, dbsession, authorized_client): data = {'friends': [9999999999]} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('/entity/person/1', json=data) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] @@ -554,13 +478,13 @@ def test_raise_on_fk_entity_is_from_wrong_schema(self, dbsession, authorized_cli dbsession.commit() data = {'friends': [e.id]} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('/entity/person/1', json=data) assert response.status_code == 422 assert "got instead entity" in response.json()['detail'] def test_raise_on_non_unique_value(self, dbsession, authorized_client): data = {'nickname': 'jane'} - response = authorized_client.put('person/1', json=data) + response = authorized_client.put('/entity/person/1', json=data) assert response.status_code == 409 assert 'Got non-unique value' in response.json()['detail'] @@ -569,7 +493,7 @@ def test_no_raise_on_non_unique_if_existing_is_deleted(self, dbsession, authoriz dbsession.commit() data = {'nickname': 'jane'} - response = authorized_client.put('person/Jack', json=data) + response = authorized_client.put('/entity/person/Jack', json=data) assert response.status_code == 200 e = dbsession.execute(select(Entity).where(Entity.slug == 'Jack')).scalar() @@ -579,22 +503,20 @@ def test_no_raise_on_non_unique_if_existing_is_deleted(self, dbsession, authoriz class TestRouteDeleteEntity: @pytest.mark.parametrize('entity', [1, 'Jack']) def test_delete(self, dbsession, authorized_client, entity): - response = authorized_client.delete(f'/person/{entity}') + response = authorized_client.delete(f'/entity/person/{entity}') + assert response.status_code == 200 assert response.json() == {'id': 1, 'slug': 'Jack', 'name': 'Jack', 'deleted': True} - entities = dbsession.execute(select(Entity)).scalars().all() - assert len(entities) == 2 - e = dbsession.execute(select(Entity).where(Entity.id == 1)).scalar() - assert e.deleted + asserts_after_entity_delete(db=dbsession) @pytest.mark.parametrize('entity', [1234567, 'qwertyu']) def test_raise_on_entity_doesnt_exist(self, dbsession, authorized_client, entity): - response = authorized_client.delete(f'/person/{entity}') + response = authorized_client.delete(f'/entity/person/{entity}') assert response.status_code == 404 @pytest.mark.parametrize('entity', [1, 'Jack']) def test_raise_on_already_deleted(self, dbsession, authorized_client, entity): dbsession.execute(update(Entity).where(Entity.id == 1).values(deleted=True)) dbsession.commit() - response = authorized_client.delete(f'/person/{entity}') + response = authorized_client.delete(f'/entity/person/{entity}') assert response.status_code == 404 diff --git a/backend/tests/test_entity_listing.py b/backend/tests/test_entity_listing.py new file mode 100644 index 0000000..cfa37f1 --- /dev/null +++ b/backend/tests/test_entity_listing.py @@ -0,0 +1,191 @@ +from random import choice +import random + +from ..config import * +from ..crud import * +from ..models import * +from ..schemas import * +from ..exceptions import * + +from copy import copy + +def data_for_test(db: Session, count: int): + random.seed(42) + schema = create_schema(db=db, data=SchemaCreateSchema( + name='test', slug='test', attributes=[ + AttrDefSchema( + name='int_field', + type='INT', + required=False, + unique=False, + list=False, + key=False + ), + AttrDefSchema( + name='string_field', + type='STR', + required=False, + unique=False, + list=False, + key=False + ) + ] + )) + entities = [] + names = [ + 'john', 'jane', 'jim', + 'mike', 'miles', 'monty', + 'nick', 'nicolas', 'nelson', + 'dave', 'david', 'dan' + 'alan', 'alyx', 'alice'] + for i in range(count): + entities.append( + { + 'name': f'{choice(names)}_{i}', + 'slug': f'{choice(names)}_{i}', + 'int_field': i, + 'string_field': f'{choice(names)}' + } + ) + for e in entities: + + ent = create_entity(db=db, schema_id=schema.id, data=copy(e)) + e['id'] = ent.id + + return entities, schema + + +def test_stuff(dbsession): + entities, schema = data_for_test(dbsession, 1000) + + # test 1 + ## ascending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field' + ).entities + assert [i['id'] for i in result] == [i['id'] for i in entities] + + ### pagination 10, 0 + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + limit=10, + offset=0 + ).entities + assert [i['id'] for i in result] == [i['id'] for i in entities][0:10] + + ### pagination 10, 20 + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + limit=10, + offset=20 + ).entities + assert [i['id'] for i in result] == [i['id'] for i in entities][20:30] + + ## descenging + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + ascending=False + ).entities + assert [i['id'] for i in result] == [i['id'] for i in entities][::-1] + + + # test 2 + ## ascending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + filters={'name.contains': 'j'} + ).entities + filtered = [i for i in entities if 'j' in i['name']] + a = 5 + assert [i['id'] for i in result] == [i['id'] for i in filtered] + + ## descending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + filters={'name.contains': 'j'}, + ascending=False + ).entities + filtered = [i for i in entities if 'j' in i['name']][::-1] + assert [i['id'] for i in result] == [i['id'] for i in filtered] + + # test 3 + ## ascending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + filters={'name.contains': 'j', 'string_field.starts': 'a'} + ).entities + filtered = [i for i in entities if 'j' in i['name'] and i['string_field'].startswith('a')] + assert [i['id'] for i in result] == [i['id'] for i in filtered] + + ## descending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + filters={'name.contains': 'j', 'string_field.starts': 'a'}, + ascending=False + ).entities + filtered = [i for i in entities if 'j' in i['name'] and i['string_field'].startswith('a')][::-1] + assert [i['id'] for i in result] == [i['id'] for i in filtered] + + # test 4 + ## ascending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + filters={ + 'name.contains': 'j', + 'string_field.starts': 'a', + 'int_field.gt': 50, + 'int_field.lt': 550 + } + ).entities + filtered = [i for i in entities if 'j' in i['name'] + and i['string_field'].startswith('a') + and i['int_field'] > 50 + and i['int_field'] < 550] + assert [i['id'] for i in result] == [i['id'] for i in filtered] + + ## descending + result = get_entities( + db=dbsession, + schema=schema, + all_fields=True, + order_by='int_field', + filters={ + 'name.contains': 'j', + 'string_field.starts': 'a', + 'int_field.gt': 50, + 'int_field.lt': 550 + }, + ascending=False + ).entities + filtered = [i for i in entities if 'j' in i['name'] + and i['string_field'].startswith('a') + and i['int_field'] > 50 + and i['int_field'] < 550][::-1] + assert [i['id'] for i in result] == [i['id'] for i in filtered] \ No newline at end of file diff --git a/backend/tests/test_general_routes.py b/backend/tests/test_general_routes.py index b089658..694bd33 100644 --- a/backend/tests/test_general_routes.py +++ b/backend/tests/test_general_routes.py @@ -1,12 +1,24 @@ -from sqlalchemy import update +from datetime import timedelta, timezone, datetime + +from fastapi.testclient import TestClient +from dateutil import parser +from sqlalchemy import update, select +from sqlalchemy.orm import Session import pytest -from ..crud import RESERVED_SCHEMA_SLUGS -from ..models import * +from ..auth.models import User +from ..models import Schema +from ..schemas import AttrDefSchema, SchemaCreateSchema, SchemaUpdateSchema +from ..traceability.entity import create_entity_update_request, create_entity_create_request, \ + create_entity_delete_request +from ..traceability.enum import ChangeStatus +from ..traceability.models import ChangeRequest, Change +from ..traceability.schema import create_schema_create_request, create_schema_delete_request, \ + create_schema_update_request class TestRouteAttributes: - def test_get_attributes(self, dbsession, client): + def test_get_attributes(self, dbsession: Session, client: TestClient): attrs = [ {'id': 1, 'name': 'age', 'type': 'FLOAT'}, {'id': 2, 'name': 'age', 'type': 'INT'}, @@ -20,143 +32,145 @@ def test_get_attributes(self, dbsession, client): response = client.get('/attributes') assert response.json() == attrs - def test_get_attribute(self, dbsession, client): + def test_get_attribute(self, dbsession: Session, client: TestClient): response = client.get('/attributes/1') assert response.json() == {'id': 1, 'name': 'age', 'type': 'FLOAT'} - def test_raise_on_attribute_doesnt_exist(self, dbsession, client): + def test_raise_on_attribute_doesnt_exist(self, dbsession: Session, client: TestClient): response = client.get('/attributes/123456789') assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] class TestRouteSchemasGet: - def test_get_schemas(self, dbsession, client): + def test_get_schemas(self, dbsession: Session, client: TestClient): test = Schema(name='Test', slug='test', deleted=True) dbsession.add(test) dbsession.commit() - response = client.get('/schemas') + response = client.get('/schema') assert response.status_code == 200 - assert response.json() == [{'id': 1, 'name': 'Person', 'slug': 'person'}, - {'id': 2, 'name': 'UnPerson', 'slug': 'unperson'},] + assert response.json() == [ + {'id': 1, 'name': 'Person', 'slug': 'person', 'deleted': False}, + {'id': 2, 'name': 'UnPerson', 'slug': 'unperson', 'deleted': False} + ] - def test_get_all(self, dbsession, client): + def test_get_all(self, dbsession: Session, client: TestClient): test = Schema(name='Test', slug='test', deleted=True) dbsession.add(test) dbsession.commit() - response = client.get('/schemas?all=True') + expected = [ + {'id': 1, 'name': 'Person', 'slug': 'person', 'deleted': False}, + {'id': 2, 'name': 'UnPerson', 'slug': 'unperson', 'deleted': False}, + {'id': 3, 'name': 'Test', 'slug': 'test', 'deleted': True} + ] + + response = client.get('/schema?all=True') assert response.status_code == 200 - assert response.json() == [{'id': 1, 'name': 'Person', 'slug': 'person'}, - {'id': 2, 'name': 'UnPerson', 'slug': 'unperson'}, - {'id': 3, 'name': 'Test', 'slug': 'test'}] + assert response.json() == expected - response = client.get('/schemas?all=True&deleted_only=True') + response = client.get('/schema?all=True&deleted_only=True') assert response.status_code == 200 - assert response.json() == [{'id': 1, 'name': 'Person', 'slug': 'person'}, - {'id': 2, 'name': 'UnPerson', 'slug': 'unperson'}, - {'id': 3, 'name': 'Test', 'slug': 'test'}] + assert response.json() == expected - def test_get_deleted_only(self, dbsession, client): + def test_get_deleted_only(self, dbsession: Session, client: TestClient): test = Schema(name='Test', slug='test', deleted=True) dbsession.add(test) dbsession.commit() - response = client.get('/schemas?deleted_only=True') + response = client.get('/schema?deleted_only=True') assert response.status_code == 200 - assert response.json() == [{'id': 3, 'name': 'Test', 'slug': 'test'}] + assert response.json() == [{'id': 3, 'name': 'Test', 'slug': 'test', 'deleted': True}] - def test_get_schema(self, dbsession, client): + def test_get_schema(self, dbsession: Session, client: TestClient): + attrs = [ + { + 'bound_schema_id': None, + 'description': 'Age of this person', + 'key': True, + 'list': False, + 'name': 'age', + 'required': True, + 'type': 'INT', + 'unique': False + }, + { + 'bound_schema_id': None, + 'description': None, + 'key': False, + 'list': False, + 'name': 'born', + 'required': False, + 'type': 'DT', + 'unique': False + }, + { + 'bound_schema_id': 1, + 'description': None, + 'key': False, + 'list': True, + 'name': 'friends', + 'required': True, + 'type': 'FK', + 'unique': False + }, + { + 'bound_schema_id': None, + 'description': None, + 'key': False, + 'list': False, + 'name': 'nickname', + 'required': False, + 'type': 'STR', + 'unique': True + }, + { + 'bound_schema_id': None, + 'description': None, + 'key': False, + 'list': True, + 'name': 'fav_color', + 'required': False, + 'type': 'STR', + 'unique': False + } + ] schema = { - 'attributes': [ - {'bind_to_schema': None, - 'description': 'Age of this person', - 'key': True, - 'list': False, - 'name': 'age', - 'required': True, - 'type': 'INT', - 'unique': False - }, - {'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'born', - 'required': False, - 'type': 'DT', - 'unique': False - }, - {'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'friends', - 'required': True, - 'type': 'FK', - 'unique': False - }, - {'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'nickname', - 'required': False, - 'type': 'STR', - 'unique': True - }, - {'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'fav_color', - 'required': False, - 'type': 'STR', - 'unique': False - } - ], 'deleted': False, 'id': 1, 'name': 'Person', - 'slug': 'person' + 'slug': 'person', + 'reviewable': False } + attrs = sorted(attrs, key=lambda x: x['name']) - response = client.get('/schemas/1') - assert response.json() == schema - - response = client.get('/schemas/person') - assert response.json() == schema + for id_or_slug in ('1', 'person'): + response = client.get(f'/schema/{id_or_slug}') + json = response.json() + assert {a["name"] for a in json['attributes']} == {a["name"] for a in attrs} + del json['attributes'] + assert json == schema def test_raise_on_schema_doesnt_exist(self, dbsession, client): - response = client.get('/schemas/12345678') + response = client.get('/schema/12345678') assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] - response = client.get('/schemas/qwertyui') + response = client.get('/schema/qwertyui') assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] class TestRouteSchemaCreate: - def attrs(self, db: Session) -> List[Attribute]: - color = Attribute(name='color', type=AttrType.STR) - max_speed = Attribute(name='max_speed', type=AttrType.INT) - release_year = Attribute(name='release_year', type=AttrType.DT) - owner = Attribute(name='owner', type=AttrType.FK) - - db.add_all([color, max_speed, release_year, owner]) - db.commit() - return [color, max_speed, release_year, owner] - - def test_create(self, dbsession, authorized_client): - color, max_speed, release_year, owner = self.attrs(dbsession) + def test_create(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Car', 'slug': 'car', 'attributes': [ { - 'attribute_id': color.id, # this + 'name': 'color', + 'type': 'STR', 'required': False, 'unique': False, 'list': False, @@ -164,138 +178,41 @@ def test_create(self, dbsession, authorized_client): 'description': 'Color of this car' }, { - 'attr_id': max_speed.id, # and this are the same + 'name': 'max_speed', + 'type': 'INT', 'required': True, 'unique': False, 'list': False, 'key': False, }, { - 'attr_id': release_year.id, + 'name': 'release_year', + 'type': 'DT', 'required': False, 'unique': False, 'list': False, 'key': False, }, { - 'attr_id': owner.id, + 'name': 'owner', + 'type': 'FK', 'required': True, 'unique': False, 'list': False, 'key': False, - 'bind_to_schema': 1 + 'bound_schema_id': 1 } ] } - response = authorized_client.post('/schemas', json=data) + response = authorized_client.post('/schema', json=data) assert response.status_code == 200 - assert response.json() == {'id': 3, 'name': 'Car', 'slug': 'car'} - assert '/car' in [i.path for i in authorized_client.app.routes] - - - car = dbsession.execute(select(Schema).where(Schema.name == 'Car')).scalar() - assert car is not None and car.id == 3 and car.slug == 'car' - - attr_defs = dbsession.execute(select(AttributeDefinition).where(AttributeDefinition.schema_id == car.id)).scalars().all() - assert len(attr_defs) == 4 - - color_ = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.schema_id == car.id) - .where(AttributeDefinition.attribute_id == color.id) - ).scalar() - assert not any([color_.required, color_.unique, color_.list, color_.key]) - assert color_.description == 'Color of this car' - - ry = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.schema_id == car.id) - .where(AttributeDefinition.attribute_id == release_year.id) - ).scalar() - assert not any([ry.required, ry.unique, ry.list, ry.key]) - assert ry.description is None - - owner_ = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.schema_id == car.id) - .where(AttributeDefinition.attribute_id == owner.id) - ).scalar() - bfk = dbsession.execute(select(BoundFK).where(BoundFK.attr_def_id == owner_.id)).scalars().all() - assert len(bfk) == 1 and bfk[0].schema.name == 'Person' - - def test_create_with_attr_data(self, dbsession, authorized_client): - color, *_ = self.attrs(dbsession) - data = { - 'name': 'Test', - 'slug': 'test', - 'attributes': [ - { - 'name': 'test1', - 'type': 'STR', - 'required': True, - 'unique': True, - 'list': False, - 'key': True, - 'description': 'Test 1' - }, - { - 'name': 'test2', - 'type': 'STR', - 'required': True, - 'unique': True, - 'list': False, - 'key': True, - 'description': 'Test 2' - }, - { - 'attribute_id': color.id, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - 'description': 'Color of this car' - } - ] - } + json = response.json() + del json['id'] + assert json == {'name': 'Car', 'slug': 'car', 'deleted': False} + assert '/entity/car' in [i.path for i in authorized_client.app.routes] - response = authorized_client.post('/schemas', json=data) + response = authorized_client.get('/entity/car') assert response.status_code == 200 - assert response.json() == {'id': 3, 'name': 'Test', 'slug': 'test'} - - schemas = dbsession.execute(select(Schema)).scalars().all() - assert len(schemas) == 3 - - schema = dbsession.execute(select(Schema).where(Schema.name == 'Test')).scalar() - assert schema is not None - - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'test1')).scalar() - assert attr is not None - - attr2 = dbsession.execute(select(Attribute).where(Attribute.name == 'test2')).scalar() - assert attr2 is not None - - attr_defs = dbsession.execute( - select(AttributeDefinition).where(AttributeDefinition.schema_id == schema.id) - ).scalars().all() - assert len(attr_defs) == 3 - - attr_def = attr_defs[0] - assert attr_def is not None - assert attr_def.attribute == attr - assert all([attr_def.required, attr_def.unique, attr_def.key]) - assert not attr_def.list - assert attr_def.description == 'Test 1' - - def test_raise_on_reserved_slug(self, dbsession, authorized_client): - for i in RESERVED_SCHEMA_SLUGS: - data = { - 'name': 'Person', - 'slug': i, - 'attributes': [] - } - response = authorized_client.post('/schemas', json=data) - assert response.status_code == 409 - assert "Can't create schema with slug" in response.json()['detail'] def test_raise_on_duplicate_name_or_slug(self, dbsession, authorized_client): data = { @@ -303,7 +220,7 @@ def test_raise_on_duplicate_name_or_slug(self, dbsession, authorized_client): 'slug': 'test', 'attributes': [] } - response = authorized_client.post('/schemas', json=data) + response = authorized_client.post('/schema', json=data) assert dbsession.query(Schema).filter(Schema.name == "Person").count() == 1 assert response.status_code == 409 assert 'already exists' in response.json()['detail'] @@ -313,94 +230,73 @@ def test_raise_on_duplicate_name_or_slug(self, dbsession, authorized_client): 'slug': 'person', 'attributes': [] } - response = authorized_client.post('/schemas', json=data) - # dbsession.begin() + response = authorized_client.post('/schema', json=data) assert dbsession.query(Schema).filter(Schema.slug == "person").count() == 1 assert response.status_code == 409 assert 'already exists' in response.json()['detail'] - def test_raise_on_nonexistent_attr_id(self, dbsession, authorized_client): - data = { - 'name': 'Test', - 'slug': 'test', - 'attributes': [ - { - 'attribute_id': 123456789, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - 'description': 'Nonexistent attribute' - } - ] - } - response = authorized_client.post('/schemas', json=data) - assert response.status_code == 404 - assert "doesn't exist or was deleted" in response.json()['detail'] - - def test_raise_on_empty_schema_when_binding(self, dbsession, authorized_client): - *_, owner = self.attrs(dbsession) + def test_raise_on_empty_schema_when_binding(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', 'attributes': [ { - 'attribute_id': owner.id, - 'required':False, + 'name': 'owner', + 'type': 'FK', + 'required': False, 'unique': False, 'list': False, 'key': False, } ] } - response = authorized_client.post('/schemas', json=data) + response = authorized_client.post('/schema', json=data) assert response.status_code == 422 - assert "You must bind attribute" in response.json()['detail'] + assert "Attribute type FK must be bound to a specific schema" in response.text - def test_raise_on_nonexistent_schema_when_binding(self, dbsession, authorized_client): - *_, owner = self.attrs(dbsession) + def test_raise_on_nonexistent_schema_when_binding(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', 'attributes': [ { - 'attribute_id': owner.id, - 'required':False, + 'name': 'owner', + 'type': 'FK', + 'required': False, 'unique': False, 'list': False, 'key': False, - 'bind_to_schema': 123456789 + 'bound_schema_id': 123456789 } ] } - response = authorized_client.post('/schemas', json=data) + response = authorized_client.post('/schema', json=data) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] - def test_raise_on_passed_deleted_schema_for_binding(self, dbsession, authorized_client): + def test_raise_on_passed_deleted_schema_for_binding(self, dbsession: Session, authorized_client: TestClient): dbsession.execute(update(Schema).where(Schema.id == 1).values(deleted=True)) dbsession.commit() - *_, owner = self.attrs(dbsession) data = { 'name': 'Test', 'slug': 'test', 'attributes': [ { - 'attribute_id': owner.id, + 'name': 'owner', + 'type': 'FK', 'required':False, 'unique': False, 'list': False, 'key': False, - 'bind_to_schema': 1 + 'bound_schema_id': 1 } ] } - response = authorized_client.post('/schemas', json=data) + response = authorized_client.post('/schema', json=data) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] - def test_raise_on_multiple_attrs_with_same_name(self, dbsession, authorized_client): - color, *_ = self.attrs(dbsession) + def test_raise_on_multiple_attrs_with_same_name(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', @@ -408,7 +304,7 @@ def test_raise_on_multiple_attrs_with_same_name(self, dbsession, authorized_clie { 'name': 'test1', 'type': 'STR', - 'required':False, + 'required': False, 'unique': False, 'list': False, 'key': False, @@ -416,737 +312,466 @@ def test_raise_on_multiple_attrs_with_same_name(self, dbsession, authorized_clie { 'name': 'test1', 'type': 'INT', - 'required':False, + 'required': False, 'unique': False, 'list': False, 'key': False, } ] } - response = authorized_client.post('/schemas', json=data) + response = authorized_client.post('/schema', json=data) assert response.status_code == 409 assert dbsession.query(Schema).filter(Schema.slug == "test").count() == 0 assert "Found multiple occurrences of attribute" in response.json()['detail'] - data = { - 'name': 'Test', - 'slug': 'test', - 'attributes': [ - { - 'name': 'color', - 'type': 'INT', - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - }, - { - 'attr_id': color.id, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - } - ] - } - response = authorized_client.post('/schemas', json=data) - assert response.status_code == 409 - assert "Found multiple occurrences of attribute" in response.json()['detail'] - - data = { - 'name': 'Test', - 'slug': 'test', - 'attributes': [ - { - 'attr_id': color.id, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - }, - { - 'attr_id': color.id, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - } - ] - } - response = authorized_client.post('/schemas', json=data) - assert response.status_code == 409 - assert "Found multiple occurrences of attribute" in response.json()['detail'] - class TestRouteSchemaUpdate: - def test_update(self, dbsession, authorized_client): - address = dbsession.execute(select(Attribute).where(Attribute.name == 'address')).scalar() - age = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - - data = { - 'name': 'Test', - 'slug': 'test', - 'update_attributes': [ - { - 'attr_def_id': age.id, - 'required': False, - 'unique': False, - 'list': False, - 'key': False, - 'description': 'Age of this person' - } - ], - 'add_attributes': [ - { - 'attribute_id': address.id, - 'required': True, - 'unique': True, - 'list': True, - 'key': True, - 'bind_to_schema': -1 - } - ] - } - result = { - 'attributes': [ - { - 'bind_to_schema': None, - 'description': 'Age of this person', - 'key': False, - 'list': False, - 'name': 'age', - 'required': False, - 'type': 'INT', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'born', - 'required': False, - 'type': 'DT', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'friends', - 'required': True, - 'type': 'FK', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'nickname', - 'required': False, - 'type': 'STR', - 'unique': True - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'fav_color', - 'required': False, - 'type': 'STR', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': True, - 'list': True, - 'name': 'address', - 'required': True, - 'type': 'FK', - 'unique': False - } - ], - 'deleted': False, - 'id': 1, - 'name': 'Test', - 'slug': 'test' + default_attributes = [ + { + "id": 1, + "name": 'age', + "type": "INT", + "required": True, + "unique": False, + "list": False, + "key": True, + "description": 'Age of this person' + }, + { + "id": 2, + "name": 'born', + "type": 'DT', + "required": False, + "unique": False, + "list": False, + "key": False + }, + { + "id": 3, + "name": 'friends', + "type": 'FK', + "required": True, + "unique": False, + "list": True, + "key": False, + "bound_schema_id": -1 + }, + { + "id": 4, + "name": 'nickname', + "type": 'STR', + "required": False, + "unique": True, + "list": False, + "key": False + }, + { + "id": 5, + "name": 'fav_color', + "type": 'STR', + "required": False, + "unique": False, + "list": True, + "key": False } + ] - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 200 - assert response.json() == result - routes = [i.path for i in authorized_client.app.routes] - assert '/test' in routes - assert '/person' not in routes - - dbsession.expire_all() - age_def = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert age_def is not None - assert not any([age_def.required, age_def.unique, age_def.list, age_def.key]) - - address_def = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.attribute_id == address.id) - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert address_def is not None - assert all([address_def.list, address_def.key, address_def.required]) - assert not address_def.unique - - bfk = dbsession.execute( - select(BoundFK) - .where(BoundFK.schema_id == 1) - .where(BoundFK.attr_def_id == address_def.id) - ).scalar() - assert bfk is not None - - sch = dbsession.execute(select(Schema).where(Schema.id == 1)).scalar() - assert sch.name == 'Test' and sch.slug == 'test' - - def test_update_attr_def_with_name(self, dbsession, authorized_client): + def test_update(self, dbsession: Session, authorized_client: TestClient): + attributes = [a for a in self.default_attributes if a["id"] != 1] data = { - 'name': 'Test', 'slug': 'test', - 'update_attributes': [ - { - 'name': 'age', - 'required': False, - 'unique': False, - 'list': False, - 'key': False, - 'description': 'Age of this person' - } - ], - 'add_attributes': [] - } - result = { - 'attributes': [ + 'reviewable': True, + 'attributes': attributes + [ { - 'bind_to_schema': None, - 'description': 'Age of this person', - 'key': False, - 'list': False, + 'id': 1, 'name': 'age', - 'required': False, 'type': 'INT', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'born', - 'required': False, - 'type': 'DT', - 'unique': False}, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'friends', - 'required': True, - 'type': 'FK', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'nickname', - 'required': False, - 'type': 'STR', - 'unique': True - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'fav_color', - 'required': False, - 'type': 'STR', - 'unique': False - } - ], - 'deleted': False, - 'id': 1, - 'name': 'Test', - 'slug': 'test' - } - - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 200 - assert response.json() == result - - - age_def = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert age_def is not None - assert not any([age_def.required, age_def.unique, age_def.list, age_def.key]) - - def test_update_with_attr_data(self, dbsession, authorized_client): - address = dbsession.execute(select(Attribute).where(Attribute.name == 'address')).scalar() - age = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - data = { - 'name': 'Test', - 'slug': 'test', - 'update_attributes': [ - { - 'attr_def_id': age.id, 'required': False, 'unique': False, 'list': False, 'key': False, 'description': 'Age of this person' - } - ], - 'add_attributes': [ - { - 'attribute_id': address.id, - 'required': True, - 'unique': True, - 'list': True, - 'key': True, - 'bind_to_schema': -1 }, { - 'name': 'test', + 'name': 'address', 'type': 'FK', - 'required': False, - 'unique': False, - 'list': False, - 'key': False, - 'description': 'test', - 'bind_to_schema': -1 - } - ] - } - result = { - 'attributes': [ - { - 'bind_to_schema': None, - 'description': 'Age of this person', - 'key': False, - 'list': False, - 'name': 'age', - 'required': False, - 'type': 'INT', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'born', - 'required': False, - 'type': 'DT', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': True, - 'name': 'friends', 'required': True, - 'type': 'FK', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, - 'list': False, - 'name': 'nickname', - 'required': False, - 'type': 'STR', - 'unique': True - }, - { - 'bind_to_schema': None, - 'description': None, - 'key': False, + 'unique': True, 'list': True, - 'name': 'fav_color', - 'required': False, - 'type': 'STR', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': None, 'key': True, - 'list': True, - 'name': 'address', - 'required': True, - 'type': 'FK', - 'unique': False - }, - { - 'bind_to_schema': None, - 'description': 'test', - 'key': False, - 'list': False, - 'name': 'test', - 'required': False, - 'type': 'FK', - 'unique': False + 'bound_schema_id': -1 } ], - 'deleted': False, - 'id': 1, - 'name': 'Test', - 'slug': 'test' + 'delete_attributes': ['friends'] } - - response = authorized_client.put('/schemas/1', json=data) + result = {'name': 'Person', 'slug': 'test', 'deleted': False} + response = authorized_client.put('/schema/1', json=data) assert response.status_code == 200 - assert response.json() == result - - dbsession.expire_all() - address_def = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.attribute_id == address.id) - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert address_def is not None - assert not address_def.unique and address_def.list - age_def = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'age') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert age_def is not None - assert not any([age_def.required, age_def.unique, age_def.list, age_def.key]) - - sch = dbsession.execute(select(Schema).where(Schema.id == 1)).scalar() - assert sch.name == 'Test' and sch.slug == 'test' - - test_def = dbsession.execute( - select(AttributeDefinition) - .join(Attribute) - .where(Attribute.name == 'test') - .where(AttributeDefinition.schema_id == 1) - ).scalar() - assert test_def is not None - assert test_def.attribute.type == AttrType.FK - bfk = dbsession.execute( - select(BoundFK) - .where(BoundFK.schema_id == 1) - .where(BoundFK.attr_def_id == test_def.id) - ).scalar() - assert bfk - - def test_raise_on_reserved_slug(self, dbsession, authorized_client): - for i in RESERVED_SCHEMA_SLUGS: - data = { - 'name': 'Person', - 'slug': i, - 'update_attributes': [], - 'add_attributes': [] - } - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 409 - assert "Can't create schema with slug" in response.json()['detail'] + json = response.json() + del json['id'] + assert json == result + + routes = [i.path for i in authorized_client.app.routes] + assert '/entity/test' in routes + assert '/entity/person' not in routes def test_raise_on_schema_doesnt_exist(self, dbsession, authorized_client): data = { 'name': 'Test', 'slug': 'person', - 'update_attributes': [], - 'add_attributes': [] + 'attributes': [] } - response = authorized_client.put('/schemas/12345678', json=data) + response = authorized_client.put('/schema/12345678', json=data) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] - def test_raise_on_existing_slug_or_name(self, dbsession, authorized_client): + def test_raise_on_existing_slug_or_name(self, dbsession: Session, authorized_client: TestClient): new_sch = Schema(name='Test', slug='test') dbsession.add(new_sch) dbsession.commit() data = { 'name': 'Test', - 'slug': 'person', - 'update_attributes': [], - 'add_attributes': [] + 'slug': 'person' } - response = authorized_client.put('/schemas/1', json=data) + response = authorized_client.put('/schema/1', json=data) assert response.status_code == 409 assert 'already exists' in response.json()['detail'] - ata = { + data = { 'name': 'Person', 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [] + 'attributes': [] } - response = authorized_client.put('/schemas/person', json=data) + response = authorized_client.put('/schema/person', json=data) assert response.status_code == 409 assert 'already exists' in response.json()['detail'] - def test_raise_on_attr_def_doesnt_exist(self, dbsession, authorized_client): + def test_raise_on_attr_not_defined_on_schema(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', - 'update_attributes': [ + 'attributes': self.default_attributes + [ { - 'attr_def_id': 12345678, + 'id': 56789, + 'name': 'address', + 'type': 'STR', 'required': True, 'unique': True, 'list': False, 'key': True } - ], - 'add_attributes': [] + ] } - response = authorized_client.put('/schemas/1', json=data) + response = authorized_client.put('/schema/1', json=data) + print("===DEBUG===", response.json()) assert response.status_code == 404 assert "is not defined on schema" in response.json()['detail'] - def test_raise_on_convert_list_to_single(self, dbsession, authorized_client): - attr_def = dbsession.execute( - select(AttributeDefinition) - .where(AttributeDefinition.schema_id == 1) - .join(Attribute) - .where(Attribute.name == 'friends') - ).scalar() + def test_raise_on_convert_list_to_single(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', - 'update_attributes': [ + 'attributes': [a for a in self.default_attributes if a["id"] != 3] + [ { - 'attr_def_id': attr_def.id, + 'id': 3, + 'name': 'friends', + 'type': 'STR', 'required': True, 'unique': True, 'list': False, 'key': True } - ], - 'add_attributes': [] - } - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 409 - assert "is listed, can't make unlisted" in response.json()['detail'] - - def test_raise_on_attr_doesnt_exist(self, dbsession, authorized_client): - data = { - 'name': 'Test', - 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ - { - 'attr_id': 12345678, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - } ] } - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 404 - assert "doesn't exist or was deleted" in response.json()['detail'] - - def test_raise_on_attr_def_already_exists(self, dbsession, authorized_client): - attr = dbsession.execute(select(Attribute).where(Attribute.name == 'born')).scalar() - data = { - 'name': 'Test', - 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ - { - 'attr_id': attr.id, - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - } - ] - } - response = authorized_client.put('/schemas/1', json=data) + response = authorized_client.put('/schema/1', json=data) assert response.status_code == 409 - assert "already defined" in response.json()['detail'] + assert "is listed, can't make unlisted" in response.text + def test_raise_on_nonexistent_schema_when_binding(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ + 'attributes': self.default_attributes + [ { - 'name': 'born', - 'type': 'STR', - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - } - ] - } - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 409 - assert "already defined" in response.json()['detail'] - - def test_raise_on_nonexistent_schema_when_binding(self, dbsession, authorized_client): - attr = dbsession.execute( - select(Attribute) - .where(Attribute.name == 'address') - ).scalar() - data = { - 'name': 'Test', - 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ - { - 'attr_id': attr.id, - 'required':False, + 'name': 'address', + 'type': 'FK', + 'required': False, 'unique': False, 'list': False, 'key': False, - 'bind_to_schema': 123456789 + 'bound_schema_id': 123456789 } ] } - response = authorized_client.put('/schemas/1', json=data) + response = authorized_client.put('/schema/1', json=data) assert response.status_code == 404 assert "doesn't exist or was deleted" in response.json()['detail'] - def test_raise_on_schema_not_passed_when_binding(self, dbsession, authorized_client): - attr = dbsession.execute( - select(Attribute) - .where(Attribute.name == 'address') - ).scalar() + def test_raise_on_schema_not_passed_when_binding(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ + 'attributes': self.default_attributes + [ { - 'attr_id': attr.id, - 'required':False, + 'name': 'address', + 'type': 'FK', + 'required': False, 'unique': False, 'list': False, 'key': False, } ] } - response = authorized_client.put('/schemas/1', json=data) + response = authorized_client.put('/schema/1', json=data) assert response.status_code == 422 - assert "You must bind attribute" in response.json()['detail'] + assert "Attribute type FK must be bound to a specific schema" in response.text - def test_raise_on_multiple_attrs_with_same_name(self, dbsession, authorized_client): - address = dbsession.execute( - select(Attribute) - .where(Attribute.name == 'address') - ).scalar() + def test_raise_on_multiple_attrs_with_same_name(self, dbsession: Session, authorized_client: TestClient): data = { 'name': 'Test', 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ + 'attributes': self.default_attributes + [ { - 'attr_id': address.id, + 'name': 'address', 'type': 'STR', - 'required':False, + 'required': False, 'unique': False, 'list': False, 'key': False, - 'bind_to_schema': -1 + 'bound_schema_id': -1 }, - { - 'attribute_id': address.id, - 'type': 'INT', - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - 'bind_to_schema': -1 - } - ] - } - response = authorized_client.put('/schemas/1', json=data) - assert response.status_code == 409 - assert "Found multiple occurrences of attribute" in response.json()['detail'] - - data = { - 'name': 'Test', - 'slug': 'test', - 'update_attributes': [], - 'add_attributes': [ { 'name': 'address', - 'type': 'DT', - 'required':False, - 'unique': False, - 'list': False, - 'key': False, - }, - { - 'attr_id': address.id, - 'required':False, + 'type': 'INT', + 'required': False, 'unique': False, 'list': False, 'key': False, - 'bind_to_schema': -1 + 'bound_schema_id': -1 } ] } - response = authorized_client.put('/schemas/1', json=data) + response = authorized_client.put('/schema/1', json=data) assert response.status_code == 409 assert "Found multiple occurrences of attribute" in response.json()['detail'] class TestRouteSchemaDelete: @pytest.mark.parametrize('id_or_slug', [1, 'person']) - def test_delete(self, dbsession, authorized_client, id_or_slug): - response = authorized_client.delete(f'/schemas/{id_or_slug}') + def test_delete(self, dbsession: Session, authorized_client: TestClient, id_or_slug): + response = authorized_client.delete(f'/schema/{id_or_slug}') assert response.status_code == 200 - assert response.json() == {'id': 1, 'name': 'Person', 'slug': 'person', 'deleted': True} - - schemas = dbsession.execute(select(Schema).order_by(Schema.id)).scalars().all() - assert len(schemas) == 2 - assert schemas[0].deleted - - entities = dbsession.execute(select(Entity).where(Entity.schema_id == 1)).scalars().all() - assert len(entities) == 2 - assert all([i.deleted for i in entities]) + assert response.json() == {'id': 1, 'name': 'Person', 'slug': 'person', 'deleted': True, 'reviewable': False} @pytest.mark.parametrize('id_or_slug', [1, 'person']) - def test_raise_on_already_deleted(self, dbsession, authorized_client, id_or_slug): + def test_raise_on_already_deleted(self, dbsession: Session, authorized_client: TestClient, id_or_slug): dbsession.execute(update(Schema).where(Schema.id == 1).values(deleted=True)) dbsession.commit() - response = authorized_client.delete(f'/schemas/{id_or_slug}') + response = authorized_client.delete(f'/schema/{id_or_slug}') assert response.status_code == 404 @pytest.mark.parametrize('id_or_slug', [1234567, 'qwerty']) def test_raise_on_delete_nonexistent(self, dbsession, authorized_client, id_or_slug): - response = authorized_client.delete(f'/schemas/{id_or_slug}') + response = authorized_client.delete(f'/schema/{id_or_slug}') + assert response.status_code == 404 + + +class TestRouteGetEntityChanges: + default_request_data = { + "name": "Jackson", + "age": 42, + "fav_color": ['violet', 'cyan'] + } + + def test_get_recent_changes(self, dbsession: Session, client: TestClient, testuser: User): + response = client.get('/changes/entity/person/Jack?count=1') + changes = response.json() + parsed_timestamp = parser.parse(changes[0]['created_at']) + assert parsed_timestamp.tzinfo is not None + + def test_get_update_details(self, dbsession: Session, client: TestClient, testuser: User): + user = dbsession.execute(select(User)).scalar() + now = datetime.utcnow().replace(tzinfo=timezone.utc) + change_request = create_entity_update_request( + db=dbsession, id_or_slug=1, schema_id=1, data=self.default_request_data.copy(), + created_by=testuser + ) + + url = f'/changes/detail/entity/{change_request.id}' + response = client.get(url) + change = response.json() + assert change['status'] == 'PENDING' + + dbsession.execute(update(ChangeRequest).values(status=ChangeStatus.APPROVED)) + dbsession.commit() + + response = client.get(url) + change = response.json() + assert change['status'] == 'APPROVED' + + dbsession.execute(update(ChangeRequest).values(status=ChangeStatus.DECLINED)) + dbsession.commit() + + response = client.get(url) + change = response.json() + assert change['status'] == 'DECLINED' + + def test_get_create_details(self, dbsession: Session, client: TestClient, testuser: User): + data = self.default_request_data.copy() + data.update({"slug": "jackson", "age": 42, "friends": [2]}) + change_request = create_entity_create_request( + db=dbsession, schema_id=1, data=data.copy(), + created_by=testuser) + + url = f'/changes/detail/entity/{change_request.id}' + response = client.get(url) + change = response.json() + assert change['status'] == 'PENDING' + assert change['entity']['name'] == '' + assert change['entity']['slug'] == '' + assert change['entity']['schema'] == 'person' + + dbsession.execute(update(ChangeRequest).values(status=ChangeStatus.DECLINED)) + dbsession.commit() + + response = client.get(f'/changes/detail/entity/{change_request.id}') + change = response.json() + assert change['status'] == 'DECLINED' + assert change['entity']['name'] == '' + assert change['entity']['slug'] == '' + assert change['entity']['schema'] == 'person' + + def test_get_delete_details(self, dbsession: Session, client: TestClient, testuser: User): + now = datetime.now(timezone.utc) + change_request = create_entity_delete_request(db=dbsession, id_or_slug=1, schema_id=1, + created_by=testuser) + + response = client.get(f'/changes/detail/entity/{change_request.id}') + change = response.json() + assert parser.parse(change['created_at']) >= now + assert change['created_by'] == testuser.username + assert change['reviewed_at'] == change['reviewed_by'] == change['comment'] == None + assert change['status'] == 'PENDING' + assert change['entity']['name'] == 'Jack' + assert change['entity']['slug'] == 'Jack' + assert change['entity']['schema'] == 'person' + assert len(change['changes']) == 1 + deleted = change['changes']['deleted'] + assert deleted['new'] and not deleted['old'] and not deleted['current'] + + def test_raise_on_change_doesnt_exist(self, dbsession: Session, client: TestClient): + response = client.get('/changes/detail/entity/12345678') + assert response.status_code == 404 + + +class TestRouteGetSchemaChanges: + def test_get_recent_changes(self, dbsession: Session, client: TestClient, testuser: User): + response = client.get('/changes/schema/person?count=1') + changes = response.json() + assert changes['schema_changes'][0]['created_at'] is not None + entity_requests = changes['pending_entity_requests'] + assert len(entity_requests) == 1 + + response = client.get('/changes/schema/person') + changes = response.json() + entity_requests = changes['pending_entity_requests'] + assert len(entity_requests) == 1 + assert all(change['created_at'] is not None for change in changes['schema_changes']) + + def test_get_update_details(self, dbsession: Session, client: TestClient, testuser: User): + change_request = create_schema_update_request( + db=dbsession, id_or_slug="person", data=SchemaUpdateSchema(name="Hello", attributes=[]), + created_by=testuser + ) + response = client.get(f'/changes/detail/schema/{change_request.id}') + assert response.status_code == 200 + change = response.json() + assert change['created_at'] is not None + assert change['created_by'] == testuser.username + assert change['reviewed_at'] == change['reviewed_by'] == change['comment'] == None + assert change['status'] == 'PENDING' + assert change['schema']['name'] == 'Person' + assert change['schema']['slug'] == 'person' + assert change['changes']['name']["new"] == "Hello" + + def test_get_create_details(self, dbsession: Session, client: TestClient, testuser: User): + change_request = create_schema_create_request( + db=dbsession, + data=SchemaCreateSchema(name="Test", slug="test", attributes=[]), + created_by=testuser + ) + response = client.get(f'/changes/detail/schema/{change_request.id}') + assert response.status_code == 200 + change = response.json() + + assert change['created_at'] is not None + assert change['created_by'] == testuser.username + assert change['reviewed_at'] == change['reviewed_by'] == change['comment'] == None + assert change['status'] == 'PENDING' + assert change["schema"] is None + + def test_get_delete_details(self, dbsession: Session, authorized_client: TestClient, + testuser: User): + change_request = create_schema_delete_request(db=dbsession, id_or_slug="person", + created_by=testuser) + response = authorized_client.get(f'/changes/detail/schema/{change_request.id}') + assert response.status_code == 200 + change = response.json() + + assert change['created_at'] is not None + assert change['created_by'] == testuser.username + assert change['reviewed_at'] == change['reviewed_by'] == change['comment'] == None + assert change['status'] == 'PENDING' + assert len(change['changes']) == 1 + deleted = change['changes']['deleted'] + assert deleted['new'] is True and deleted['old'] is False and deleted['current'] is False + + def test_raise_on_change_doesnt_exist(self, dbsession: Session, client: TestClient): + response = client.get('/changes/schema/12345678') + assert response.status_code == 404 + + +class TestTraceabilityRoutes: + def test_review_changes(self, dbsession: Session, authorized_client: TestClient, + testuser: User): + data = {"name": "Føø Bar"} + change_request = create_entity_update_request( + db=dbsession, id_or_slug=1, schema_id=1, data=data.copy(), created_by=testuser + ) + + # APPROVE + review = { + 'result': 'APPROVE', + 'comment': 'test' + } + + response = authorized_client.post(f'/changes/review/{change_request.id}', json=review) + assert response.status_code == 200 + data = response.json() + expected = { + 'created_by': 'tester', 'reviewed_by': 'tester', 'status': 'APPROVED', + 'comment': 'test', 'object_type': 'ENTITY', 'change_type': 'UPDATE' + } + for key, value in expected.items(): + assert data.get(key, None) == value + + dbsession.commit() + change_request.status = ChangeStatus.PENDING + dbsession.commit() + + # DECLINE + review['result'] = 'DECLINE' + review['comment'] = 'test2' + response = authorized_client.post(f'/changes/review/{change_request.id}', json=review) + json = response.json() + assert json['status'] == 'DECLINED' + assert json['comment'] == 'test2' + + def test_raise_on_change_doesnt_exist(self, dbsession: Session, authorized_client: TestClient): + review = { + 'result': 'APPROVE', + 'comment': 'test' + } + response = authorized_client.post('/changes/review/23456', json=review) assert response.status_code == 404 diff --git a/backend/tests/test_traceability.py b/backend/tests/test_traceability.py new file mode 100644 index 0000000..6895594 --- /dev/null +++ b/backend/tests/test_traceability.py @@ -0,0 +1,185 @@ +from datetime import timedelta, datetime + +import pytest +from sqlalchemy import select, update +from sqlalchemy.orm import Session + +from ..auth.models import User +from ..schemas.traceability import ChangeReviewSchema +from ..traceability.crud import decline_change_request, review_changes, get_pending_change_requests +from ..traceability.enum import ChangeType, ContentType, EditableObjectType, ChangeStatus, \ + ReviewResult +from ..traceability.models import ChangeRequest, Change, ChangeAttrType, ChangeValueStr + + +@pytest.mark.parametrize(['content_type', 'change_type'], [ + (ContentType.ENTITY, ChangeType.CREATE), + (ContentType.ENTITY, ChangeType.UPDATE), + (ContentType.ENTITY, ChangeType.DELETE), + (ContentType.SCHEMA, ChangeType.CREATE), + (ContentType.SCHEMA, ChangeType.UPDATE), + (ContentType.SCHEMA, ChangeType.DELETE), +]) +def test_decline_change_request(dbsession: Session, content_type, change_type): + user = dbsession.execute(select(User)).scalar() + change_request = ChangeRequest( + created_at=datetime.now(), + created_by=user, + object_type=EditableObjectType[content_type.name], + change_type=change_type + ) + if change_type != ChangeType.CREATE: + change_request.object_id = 1 + v = ChangeValueStr() + dbsession.add_all([change_request, v]) + dbsession.flush() + change = Change( + change_request=change_request, + value_id=v.id, + change_type=change_type, + content_type=content_type, + field_name='test', + data_type=ChangeAttrType.STR + ) + if change_type != ChangeType.CREATE: + change.object_id = 1 + dbsession.add(change) + dbsession.commit() + + decline_change_request( + db=dbsession, + change_request=change_request, + reviewed_by=user, + comment='test' + ) + + assert change_request.status == ChangeStatus.DECLINED + assert change_request.comment == 'test' + assert change_request.reviewed_at >= change_request.created_at + assert change_request.reviewed_by == user + + +def make_change_for_review(db: Session) -> ChangeRequest: + user = db.execute(select(User)).scalar() + change_request = ChangeRequest( + created_at=datetime.utcnow(), + created_by=user, + object_type=EditableObjectType.ENTITY, + object_id=1, + change_type=ChangeType.UPDATE + ) + v = ChangeValueStr(new_value='test') + db.add_all([change_request, v]) + db.flush() + change = Change( + change_request=change_request, + value_id=v.id, + change_type=ChangeType.UPDATE, + object_id=1, + content_type=ContentType.ENTITY, + field_name='name', + data_type=ChangeAttrType.STR + ) + db.add(change) + db.commit() + return change_request + + +def test_review_changes(dbsession: Session): + user = dbsession.execute(select(User)).scalar() + change_request = make_change_for_review(db=dbsession) + + review = ChangeReviewSchema( + result=ReviewResult.APPROVE, + comment='test' + ) + # approve + review_changes( + db=dbsession, + change_request_id=change_request.id, + review=review, + reviewed_by=user + ) + assert change_request.status == ChangeStatus.APPROVED + assert change_request.comment == 'test' + assert change_request.reviewed_at >= change_request.created_at + assert change_request.reviewed_by == user + + change_request.status = ChangeStatus.PENDING + dbsession.flush() + # decline + review.result = ReviewResult.DECLINE + review.comment = 'test2' + review_changes( + db=dbsession, + change_request_id=change_request.id, + review=review, + reviewed_by=user + ) + + assert change_request.status == ChangeStatus.DECLINED + assert change_request.comment == 'test2' + assert change_request.reviewed_at >= change_request.created_at + assert change_request.reviewed_by == user + + +def test_get_pending_change_requests(dbsession: Session): + # 10 requests, 9 UPD, 1 CREATE + # 10 requests, 9 UPD, 1 CREATE + schema_requests = dbsession.query(ChangeRequest)\ + .filter(ChangeRequest.object_type == EditableObjectType.SCHEMA) + entity_requests = dbsession.query(ChangeRequest)\ + .filter(ChangeRequest.object_type == EditableObjectType.ENTITY) + + # limit 10 offset 0 + requests = get_pending_change_requests(dbsession) + assert len(requests) == 10 + assert {i.id for i in schema_requests} == {i.id for i in requests} + + # limit 10, offset 10 + requests = get_pending_change_requests(dbsession, offset=10) + assert len(requests) == 10 + assert {i.id for i in entity_requests} == {i.id for i in requests} + + # limit 1, offset 0 + requests = get_pending_change_requests(dbsession, limit=1) + assert requests[0] == schema_requests[-1] + + # limit 1, offset 19 + requests = get_pending_change_requests(dbsession, limit=1, offset=19) + assert requests[0] == entity_requests[0] + + # limit 20, all types + requests = get_pending_change_requests(dbsession, limit=20) + assert requests == schema_requests[::-1] + entity_requests[::-1] + + # no limit, all types + requests = get_pending_change_requests(dbsession, all=True) + assert requests == schema_requests[::-1] + entity_requests[::-1] + + # limit 20, offset 20, all types + requests = get_pending_change_requests(dbsession, limit=20, offset=20) + assert requests == [] + + # limit 20, only schemas + requests = get_pending_change_requests(dbsession, obj_type=ContentType.SCHEMA, limit=20) + assert requests == schema_requests[::-1] + + # limit 1, offset 1, only schemas + requests = get_pending_change_requests(dbsession, obj_type=ContentType.SCHEMA, limit=1, offset=1) + assert requests == [schema_requests[-2]] + + # limit 20, only entities + requests = get_pending_change_requests(dbsession, obj_type=ContentType.ENTITY, limit=20) + assert requests == entity_requests[::-1] + + # limit 1, offset 1, only entities + requests = get_pending_change_requests(dbsession, obj_type=ContentType.ENTITY, limit=1, offset=1) + assert requests == [entity_requests[-2]] + + dbsession.execute(update(ChangeRequest).values(status=ChangeStatus.APPROVED)) + dbsession.commit() + + # no limit, all types + requests = get_pending_change_requests(dbsession, all=True) + assert requests == [] diff --git a/backend/tests/test_traceability_entity.py b/backend/tests/test_traceability_entity.py new file mode 100644 index 0000000..c71efc1 --- /dev/null +++ b/backend/tests/test_traceability_entity.py @@ -0,0 +1,346 @@ +import typing +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ..auth.models import User +from ..crud import get_entity_by_id, get_entity +from ..exceptions import MissingEntityUpdateRequestException, AttributeNotDefinedException, \ + MissingEntityCreateRequestException +from ..models import AttributeDefinition, Attribute, AttrType, Entity, Schema +from ..traceability.entity import get_recent_entity_changes, entity_change_details, \ + create_entity_update_request, apply_entity_update_request, create_entity_delete_request, \ + apply_entity_delete_request, create_entity_create_request, apply_entity_create_request +from ..traceability.enum import EditableObjectType, ChangeType, ContentType, ChangeStatus +from ..traceability.models import ChangeRequest, Change, ChangeAttrType, ChangeValueInt, \ + ChangeValueStr + + +class TestUpdateEntityTraceability: + default_data = { + 'slug': 'test', + 'nickname': None, + 'born': datetime(1983, 10, 31, tzinfo=timezone.utc), + 'friends': [1, 2], + } + + def _create_request(self, dbsession: Session, testuser: User, + data: typing.Optional[dict] = None) -> ChangeRequest: + return create_entity_update_request(dbsession, 1, 1, (data or self.default_data).copy(), + testuser) + + def test_create_request(self, dbsession: Session, testuser: User): + change_request = self._create_request(dbsession, testuser) + + assert change_request.created_by == testuser + assert change_request.created_at is not None + assert change_request.status == ChangeStatus.PENDING + assert change_request.reviewed_by is None + assert change_request.reviewed_at is None + assert change_request.object_type == EditableObjectType.ENTITY + assert change_request.object_id == 1 + assert change_request.change_type == ChangeType.UPDATE + + schema = dbsession.query(Schema).filter(Schema.id == 1).one() + old_data = {key: value + for key, value in get_entity(db=dbsession, id_or_slug=1, schema=schema).items() + if key in self.default_data} + details = entity_change_details(db=dbsession, change_request_id=change_request.id) + + assert self.default_data == {key: value.new for key, value in details.changes.items()} + assert old_data == {key: value.old for key, value in details.changes.items()} + assert old_data == {key: value.current for key, value in details.changes.items()} + + def test_apply_request(self, dbsession: Session, testuser: User): + change_request = self._create_request(dbsession, testuser) + apply_entity_update_request(dbsession, change_request=change_request, reviewed_by=testuser, + comment='Autosubmit') + + assert change_request.status == ChangeStatus.APPROVED + assert change_request.reviewed_at >= change_request.created_at + assert change_request.reviewed_by == testuser + assert change_request.comment == 'Autosubmit' + + schema = dbsession.query(Schema).filter(Schema.id == 1).one() + entity = get_entity(db=dbsession, id_or_slug=1, schema=schema) + + assert entity == { + 'id': 1, + 'slug': 'test', + 'deleted': False, + 'name': 'Jack', + 'age': 10, + 'born': datetime(1983, 10, 31, tzinfo=timezone.utc), + 'friends': [1, 2], + 'nickname': None, + 'fav_color': ['red', 'blue'] + } + + def test_raise_on_missing_change(self, dbsession: Session, testuser: User): + schema_change = ChangeRequest( + created_by=testuser, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.SCHEMA, + object_id=1, + change_type=ChangeType.UPDATE + ) + dbsession.add(schema_change) + dbsession.flush() + + # raise on wrong change + with pytest.raises(MissingEntityUpdateRequestException): + apply_entity_update_request(db=dbsession, change_request=schema_change, + reviewed_by=testuser) + + def test_raise_on_attribute_not_defined(self, dbsession: Session, testuser: User): + r = ChangeRequest( + created_by=testuser, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.ENTITY, + object_id=1, + change_type=ChangeType.UPDATE + ) + name_val = ChangeValueStr(new_value='test', old_value="Jack") + attr_val = ChangeValueStr(new_value='test') + dbsession.add_all([name_val, attr_val]) + dbsession.flush() + + name = Change( + change_request=r, value_id=name_val.id, field_name='name', + data_type=ChangeAttrType.STR, + content_type=ContentType.ENTITY, + change_type=ChangeType.UPDATE, + object_id=1 + ) + + a = Attribute(name='test', type=AttrType.STR) + attr = Change( + change_request=r, value_id=attr_val.id, attribute=a, + data_type=ChangeAttrType.STR, + content_type=ContentType.ENTITY, + change_type=ChangeType.UPDATE, + object_id=1 + ) + dbsession.add_all([r, name, a, attr]) + dbsession.flush() + + with pytest.raises(AttributeNotDefinedException): + apply_entity_update_request(db=dbsession, change_request=r, reviewed_by=testuser) + + +class TestDeleteEntityTraceability: + def _create_request(self, dbsession: Session, testuser: User) -> ChangeRequest: + return create_entity_delete_request(dbsession, id_or_slug=1, schema_id=1, + created_by=testuser) + + def test_create_request(self, dbsession: Session, testuser: User): + change_request = self._create_request(dbsession, testuser) + + assert change_request.created_by == testuser + assert change_request.created_at is not None + assert change_request.status == ChangeStatus.PENDING + assert change_request.reviewed_by is None + assert change_request.reviewed_at is None + assert change_request.object_type == EditableObjectType.ENTITY + assert change_request.object_id == 1 + assert change_request.change_type == ChangeType.DELETE + + changes = entity_change_details(db=dbsession, change_request_id=change_request.id) + + assert changes.changes["deleted"].dict() == {'new': True, 'old': False, 'current': False} + + def test_apply_request(self, dbsession: Session, testuser: User): + change_request = self._create_request(dbsession, testuser) + apply_entity_delete_request(dbsession, change_request=change_request, reviewed_by=testuser, + comment='test') + + assert change_request.reviewed_by == testuser + assert change_request.reviewed_at >= change_request.created_at + assert change_request.status == ChangeStatus.APPROVED + assert change_request.comment == "test" + + entity = get_entity_by_id(db=dbsession, entity_id=1) + assert entity.deleted + + def test_raise_on_missing_change(self, dbsession: Session, testuser: User): + schema_change = ChangeRequest( + created_by=testuser, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.SCHEMA, + object_id=1, + change_type=ChangeType.DELETE + ) + dbsession.add(schema_change) + dbsession.flush() + + # raise on wrong change + with pytest.raises(MissingEntityUpdateRequestException): + apply_entity_update_request(db=dbsession, change_request=schema_change, + reviewed_by=testuser, comment=None) + + +class TestCreateEntityTraceability: + default_data = { + 'name': 'John', + 'slug': 'John', + 'nickname': 'john', + 'age': 10, + 'friends': [1], + 'born': datetime(1990, 6, 30, tzinfo=timezone.utc) + } + + def _create_request(self, dbsession: Session, user: User, data: typing.Optional[dict] = None) \ + -> ChangeRequest: + return create_entity_create_request(db=dbsession, data=(data or self.default_data).copy(), + schema_id=1, created_by=user) + + def test_create_request(self, dbsession: Session, testuser: User): + start_time = datetime.now(timezone.utc) + change_request = self._create_request(dbsession=dbsession, user=testuser) + + assert 0 == dbsession.query(Entity.id).filter(Entity.slug == "John").count() + + assert change_request.created_by == testuser + assert change_request.created_at.astimezone(timezone.utc) >= start_time + assert change_request.status == ChangeStatus.PENDING + assert change_request.reviewed_by is None + assert change_request.reviewed_at is None + assert change_request.object_type == EditableObjectType.ENTITY + assert change_request.object_id is None + assert change_request.change_type == ChangeType.CREATE + + changes = dbsession.query(Change).filter(Change.change_request_id == change_request.id) + assert 7 == changes.count() + changed_values = {} + for c in changes: + key = c.field_name or c.attribute.name + Model = c.data_type.value.model + value = dbsession.query(Model).filter(Model.id == c.value_id).one() + assert value.old_value is None + if isinstance(value.new_value, datetime): + changed_values[key] = value.new_value.astimezone(timezone.utc) + elif key == "schema_id": + assert 1 == value.new_value + elif isinstance(self.default_data.get(key, None), list): + if key not in changed_values: + changed_values[key] = [] + changed_values[key].append(value.new_value) + else: + changed_values[key] = value.new_value + + assert self.default_data == changed_values + + def test_approve_request(self, dbsession: Session, testuser: User): + change_request = self._create_request(dbsession=dbsession, user=testuser) + start_time = datetime.now(timezone.utc) + apply_entity_create_request(db=dbsession, change_request=change_request, + reviewed_by=testuser, comment='Looks good to me 👌') + + dbsession.refresh(change_request) + assert change_request.status == ChangeStatus.APPROVED + assert change_request.reviewed_by == testuser + assert change_request.created_at.astimezone(timezone.utc) <= start_time + assert start_time <= change_request.reviewed_at.astimezone(timezone.utc) + assert change_request.object_id is not None + + changes = dbsession.query(Change).filter(Change.change_request_id == change_request.id) + assert 7 == changes.count() + assert 0 == changes.filter(Change.object_id == None).count() + + entity = get_entity_by_id(db=dbsession, entity_id=change_request.object_id) + assert entity.name == self.default_data["name"] + assert entity.slug == self.default_data["slug"] + + def test_raise_on_missing_change(self, dbsession: Session, testuser: User): + schema_change = ChangeRequest( + created_by=testuser, + created_at=datetime.utcnow(), + object_type=EditableObjectType.SCHEMA, + change_type=ChangeType.CREATE + ) + dbsession.add(schema_change) + dbsession.flush() + + # raise on wrong change + with pytest.raises(MissingEntityCreateRequestException): + apply_entity_create_request(db=dbsession, change_request=schema_change, + reviewed_by=testuser) + dbsession.rollback() + # raise on nonexistent change + with pytest.raises(MissingEntityCreateRequestException): + fake = ChangeRequest(id=42) + apply_entity_create_request(db=dbsession, change_request=fake, reviewed_by=testuser) + + def test_raise_on_attribute_not_defined(self, dbsession: Session, testuser: User): + r = ChangeRequest( + created_by=testuser, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.ENTITY, + change_type=ChangeType.CREATE + ) + name_val = ChangeValueStr(new_value='test') + slug_val = ChangeValueStr(new_value='test') + schema_val = ChangeValueInt(new_value=1) + attr_val = ChangeValueStr(new_value='test') + dbsession.add_all([name_val, slug_val, schema_val, attr_val]) + dbsession.flush() + + name = Change( + change_request=r, value_id=name_val.id, field_name='name', + data_type=ChangeAttrType.STR, + content_type=ContentType.ENTITY, + change_type=ChangeType.CREATE + ) + slug = Change( + change_request=r, value_id=slug_val.id, field_name='slug', + data_type=ChangeAttrType.STR, + content_type=ContentType.ENTITY, + change_type=ChangeType.CREATE + ) + schema = Change( + change_request=r, value_id=schema_val.id, field_name='schema_id', + data_type=ChangeAttrType.INT, + content_type=ContentType.ENTITY, + change_type=ChangeType.CREATE + ) + + a = Attribute(name='test', type=AttrType.STR) + attr = Change( + change_request=r, value_id=attr_val.id, attribute=a, + data_type=ChangeAttrType.STR, + content_type=ContentType.ENTITY, + change_type=ChangeType.CREATE + ) + dbsession.add_all([r, name, slug, schema, a, attr]) + dbsession.flush() + + with pytest.raises(AttributeNotDefinedException): + apply_entity_create_request(db=dbsession, change_request=r, reviewed_by=testuser) + + def test_get_entity_create_details(self, dbsession: Session, testuser: User): + data = { + "name": "Jackson", + "slug": "jackson", + "fav_color": ["violet", "cyan"], + "age": 42, + "friends": [1] + } + change_request = self._create_request(dbsession=dbsession, user=testuser, data=data) + + change = entity_change_details(db=dbsession, change_request_id=change_request.id) + assert all(value.old is None + for key, value in change.changes.items() + if key != "schema_id" and not isinstance(data[key], list)) + assert all(value.old == [] + for key, value in change.changes.items() + if key != "schema_id" and isinstance(data[key], list)) + assert all(value.current is None + for key, value in change.changes.items() + if key not in ["name", "slug", "schema_id"] and not isinstance(data[key], list)) + assert all(value.current == [] + for key, value in change.changes.items() + if isinstance(data.get(key, None), list)) + new_values = {key: value.new for key, value in change.changes.items() if key != "schema_id"} + assert new_values == data diff --git a/backend/tests/test_traceability_schema.py b/backend/tests/test_traceability_schema.py new file mode 100644 index 0000000..0e210f4 --- /dev/null +++ b/backend/tests/test_traceability_schema.py @@ -0,0 +1,328 @@ +from sqlalchemy.orm import Session + +from ..auth.models import User +from ..models import Schema +from ..schemas.schema import AttrDefSchema, SchemaCreateSchema, SchemaUpdateSchema +from ..schemas.traceability import ChangeSchema +from ..traceability.enum import ChangeType, ChangeStatus +from ..traceability.models import ChangeRequest, Change +from ..traceability.schema import get_recent_schema_changes, schema_change_details, \ + create_schema_create_request, apply_schema_create_request, \ + create_schema_update_request, apply_schema_update_request, create_schema_delete_request, \ + apply_schema_delete_request + + +def test_get_recent_schema_changes(dbsession: Session): + changes, entity_requests = get_recent_schema_changes(db=dbsession, schema_id=1, count=50) + assert len(changes) == 10 + assert len(entity_requests) == 1 + + +class TestCreateSchema: + default_data = { + "name": "Car", + "slug": "car", + "attributes": [ + AttrDefSchema( + name='color', + type='STR', + required=False, + unique=False, + list=False, + key=False, + description='Color of this car' + ), + AttrDefSchema( + name='max_speed', + type='INT', + required=True, + unique=False, + list=False, + key=False + ), + AttrDefSchema( + name="release_year", + type='DT', + required=False, + unique=False, + list=False, + key=False + ), + AttrDefSchema( + name='owner', + type='FK', + required=True, + unique=False, + list=False, + key=False, + bound_schema_id=1 + ), + AttrDefSchema( + name="extras", + type="STR", + required=False, + unique=False, + list=True, + key=False, + description='Special extras of car' + ) + ] + } + + def create_request(self, dbsession: Session, testuser: User) -> ChangeRequest: + return create_schema_create_request(dbsession, SchemaCreateSchema(**self.default_data), + testuser) + + def test_create_request(self, dbsession: Session, testuser: User): + change_request = self.create_request(dbsession=dbsession, testuser=testuser) + + assert change_request.id is not None + assert change_request.created_by == testuser + assert change_request.created_at is not None + assert change_request.status == ChangeStatus.PENDING + assert change_request.object_id is None + + changes = schema_change_details(db=dbsession, change_request_id=change_request.id) + expected_schema_new = { + 'name': 'Car', + 'slug': 'car', + 'reviewable': False, + 'color.required': False, + 'color.unique': False, + 'color.list': False, + 'color.key': False, + 'color.description': 'Color of this car', + 'color.bound_schema_id': None, + 'color.name': 'color', + 'color.type': 'STR', + 'max_speed.required': True, + 'max_speed.unique': False, + 'max_speed.list': False, + 'max_speed.key': False, + 'max_speed.description': None, + 'max_speed.bound_schema_id': None, + 'max_speed.name': 'max_speed', + 'max_speed.type': 'INT', + 'release_year.required': False, + 'release_year.unique': False, + 'release_year.list': False, + 'release_year.key': False, + 'release_year.description': None, + 'release_year.bound_schema_id': None, + 'release_year.name': 'release_year', + 'release_year.type': 'DT', + 'owner.required': True, + 'owner.unique': False, + 'owner.list': False, + 'owner.key': False, + 'owner.description': None, + 'owner.bound_schema_id': 1, + 'owner.name': 'owner', + 'owner.type': 'FK', + 'extras.required': False, + 'extras.unique': False, + 'extras.list': True, + 'extras.key': False, + 'extras.description': 'Special extras of car', + 'extras.bound_schema_id': None, + 'extras.name': 'extras', + 'extras.type': 'STR', + } + assert expected_schema_new == {key: value.new for key, value in changes.changes.items()} + assert all(v.old is None for v in changes.changes.values()) + assert all(v.current is None for v in changes.changes.values()) + + def test_appy_request(self, dbsession: Session, testuser: User): + change_request = self.create_request(dbsession=dbsession, testuser=testuser) + apply_schema_create_request(db=dbsession, change_request=change_request, + reviewed_by=testuser, comment='test') + assert change_request.comment == "test" + assert change_request.reviewed_by == testuser + assert change_request.status == ChangeStatus.APPROVED + assert change_request.reviewed_at >= change_request.created_at + + +class TestUpdateSchema: + default_data = { + "slug": "test", + "reviewable": True, + "attributes": [ + # changed + AttrDefSchema( + id=1, + name='age', + type="INT", + required=False, + unique=False, + list=False, + key=False, + description='Age of this person' + ), + # added + AttrDefSchema( + name='address2', + type='FK', + required=True, + unique=False, + list=True, + key=True, + bound_schema_id=-1 + ), + # unchanged + AttrDefSchema( + id=3, + name='friends', + type='FK', + required=True, + unique=False, + list=True, + key=False, + bound_schema_id=-1 + ), + AttrDefSchema( + id=4, + name='nickname', + type='STR', + required=False, + unique=True, + list=False, + key=False + ), + AttrDefSchema( + id=5, + name='fav_color', + type='STR', + required=False, + unique=False, + list=True, + key=False + ) + # deleted: born + ] + } + + def create_request(self, dbsession: Session, testuser: User) -> ChangeRequest: + return create_schema_update_request(db=dbsession, id_or_slug=1, created_by=testuser, + data=SchemaUpdateSchema(**self.default_data)) + + def test_create_request(self, dbsession: Session, testuser: User): + change_request = self.create_request(dbsession=dbsession, testuser=testuser) + assert change_request.created_by is not None + assert change_request.created_at is not None + assert change_request.status == ChangeStatus.PENDING + + details = schema_change_details(db=dbsession, change_request_id=change_request.id) + assert details.change_type == ChangeType.UPDATE + assert details.created_at is not None + assert details.created_by == testuser.username + assert details.reviewed_at is None + assert details.reviewed_by is None + assert details.comment is None + assert details.status == ChangeStatus.PENDING + assert details.schema_.name == 'Person' and details.schema_.slug == 'person' + + expected = { + 'slug': ChangeSchema(new='test', old='person', current='person'), + 'reviewable': ChangeSchema(new=True, old=False, current=False), + 'age.key': ChangeSchema(new=False, old=True, current=True), + 'age.required': ChangeSchema(new=False, old=True, current=True), + 'born': ChangeSchema(new=None, old='born', current='born'), + 'address2.unique': ChangeSchema(new=False, old=None, current=None), + 'address2.list': ChangeSchema(new=True, old=None, current=None), + 'address2.key': ChangeSchema(new=True, old=None, current=None), + 'address2.description': ChangeSchema(new=None, old=None, current=None), + 'address2.bound_schema_id': ChangeSchema(new=-1, old=None, current=None), + 'address2.name': ChangeSchema(new='address2', old=None, current=None), + 'address2.type': ChangeSchema(new='FK', old=None, current=None), + 'address2.required': ChangeSchema(new=True, old=None, current=None) + } + assert expected == details.changes + + changes = {c.field_name or c.attribute.name: c.change_type + for c in dbsession.query(Change) + .filter(Change.change_request_id == change_request.id)} + assert changes == { + 'name': ChangeType.UPDATE, 'slug': ChangeType.UPDATE, 'reviewable': ChangeType.UPDATE, + 'age.required': ChangeType.UPDATE, 'age.key': ChangeType.UPDATE, + 'address2.name': ChangeType.CREATE, + 'address2.type': ChangeType.CREATE, 'address2.required': ChangeType.CREATE, + 'address2.unique': ChangeType.CREATE, 'address2.list': ChangeType.CREATE, + 'address2.key': ChangeType.CREATE, 'address2.description': ChangeType.CREATE, + 'address2.bound_schema_id': ChangeType.CREATE, 'born': ChangeType.DELETE + } + + def test_apply_request(self, dbsession: Session, testuser: User): + change_request = self.create_request(dbsession=dbsession, testuser=testuser) + apply_schema_update_request(db=dbsession, change_request=change_request, + reviewed_by=testuser, comment='test') + + assert change_request.reviewed_at >= change_request.created_at + assert change_request.reviewed_by == testuser + assert change_request.comment == "test" + assert change_request.status == ChangeStatus.APPROVED + + schema = dbsession.query(Schema).filter(Schema.id == 1).one() + attr_names = {d.attribute.name for d in schema.attr_defs} + assert "born" not in attr_names + assert "address2" in attr_names + + details = schema_change_details(db=dbsession, change_request_id=change_request.id) + expected = { + 'slug': ChangeSchema(new='test', old='person', current='test'), + 'reviewable': ChangeSchema(new=True, old=False, current=True), + 'age.key': ChangeSchema(new=False, old=True, current=False), + 'age.required': ChangeSchema(new=False, old=True, current=False), + 'born': ChangeSchema(new=None, old='born', current=None), + 'address2.unique': ChangeSchema(new=False, old=None, current=False), + 'address2.list': ChangeSchema(new=True, old=None, current=True), + 'address2.key': ChangeSchema(new=True, old=None, current=True), + 'address2.description': ChangeSchema(new=None, old=None, current=None), + 'address2.bound_schema_id': ChangeSchema(new=-1, old=None, current=1), + 'address2.name': ChangeSchema(new='address2', old=None, current='address2'), + 'address2.type': ChangeSchema(new='FK', old=None, current='FK'), + 'address2.required': ChangeSchema(new=True, old=None, current=True) + } + assert expected == details.changes + + +class TestDeleteSchema: + def create_request(self, dbsession: Session, testuser: User) -> ChangeRequest: + return create_schema_delete_request(db=dbsession, id_or_slug=1, created_by=testuser) + + def test_create_request(self, dbsession: Session, testuser: User): + change_request = self.create_request(dbsession=dbsession, testuser=testuser) + assert change_request.status == ChangeStatus.PENDING + assert change_request.created_at is not None + assert change_request.created_by == testuser + + details = schema_change_details(db=dbsession, change_request_id=change_request.id) + assert details.change_type == ChangeType.DELETE + assert details.created_at is not None + assert details.created_by == testuser.username + assert details.reviewed_at is None + assert details.reviewed_by is None + assert details.comment is None + assert details.status == ChangeStatus.PENDING + assert details.schema_.name == 'Person' and details.schema_.slug == 'person' + assert details.changes == {"deleted": ChangeSchema(new=True, old=False, current=False)} + + def test_apply_request(self, dbsession: Session, testuser: User): + change_request = self.create_request(dbsession=dbsession, testuser=testuser) + apply_schema_delete_request(db=dbsession, change_request=change_request, + reviewed_by=testuser, comment='test') + + assert change_request.reviewed_at >= change_request.created_at + assert change_request.reviewed_by == testuser + assert change_request.status == ChangeStatus.APPROVED + assert change_request.comment == "test" + + details = schema_change_details(db=dbsession, change_request_id=change_request.id) + assert details.change_type == ChangeType.DELETE + assert details.created_at is not None + assert details.created_by == testuser.username + assert details.reviewed_at is not None + assert details.reviewed_by == testuser.username + assert details.comment == "test" + assert details.status == ChangeStatus.APPROVED + assert details.schema_.name == 'Person' and details.schema_.slug == 'person' + assert details.changes == {"deleted": ChangeSchema(new=True, old=False, current=True)} diff --git a/backend/traceability/__init__.py b/backend/traceability/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/traceability/crud.py b/backend/traceability/crud.py new file mode 100644 index 0000000..c93acfe --- /dev/null +++ b/backend/traceability/crud.py @@ -0,0 +1,126 @@ +import typing +from datetime import datetime, timezone +from typing import Optional, List + +from fastapi.exceptions import HTTPException +from sqlalchemy import select, desc +from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import Session +from starlette.status import HTTP_403_FORBIDDEN + +from ..exceptions import MissingChangeException, MissingChangeRequestException +from ..auth.crud import has_permission +from ..auth.enum import PermissionType +from ..auth.models import User +from ..models import Schema, Entity +from ..schemas.auth import RequirePermission +from ..schemas.traceability import ChangeReviewSchema + +from .enum import EditableObjectType, ChangeType, ChangeStatus, ReviewResult, ContentType +from .models import ChangeRequest, Change, ChangeValueInt +from .entity import apply_entity_create_request, apply_entity_update_request, \ + apply_entity_delete_request +from .schema import apply_schema_create_request, apply_schema_update_request, \ + apply_schema_delete_request + + +REQUIRED_PERMISSIONS = { + (EditableObjectType.SCHEMA, ChangeType.CREATE): (PermissionType.CREATE_SCHEMA, Schema), + (EditableObjectType.SCHEMA, ChangeType.UPDATE): (PermissionType.UPDATE_SCHEMA, Schema), + (EditableObjectType.SCHEMA, ChangeType.DELETE): (PermissionType.DELETE_SCHEMA, Schema), + (EditableObjectType.ENTITY, ChangeType.CREATE): (PermissionType.CREATE_ENTITY, Schema), + (EditableObjectType.ENTITY, ChangeType.UPDATE): (PermissionType.UPDATE_ENTITY, Entity), + (EditableObjectType.ENTITY, ChangeType.DELETE): (PermissionType.DELETE_ENTITY, Entity), +} + + +def is_user_authorized_to_review(db: Session, user: User, request_id: int) -> None: + request = get_change_request(db=db, request_id=request_id) + req_perm, Model = REQUIRED_PERMISSIONS.get((request.object_type, request.change_type)) + req_perm = RequirePermission(permission=req_perm) + if request.object_type == EditableObjectType.ENTITY \ + and request.change_type == ChangeType.CREATE: + try: + schema_id = db\ + .query(ChangeValueInt.new_value)\ + .join(Change, ChangeValueInt.id == Change.value_id)\ + .filter(Change.change_request_id == request.id, Change.field_name == "schema_id")\ + .scalar() + except NoResultFound: + schema_id = None + req_perm.target = Model(id=schema_id) + else: + req_perm.target = Model(id=request.object_id) + + if not has_permission(db=db, user=user, permission=req_perm): + raise HTTPException(status_code=HTTP_403_FORBIDDEN, + detail="You are not authorized for this action") + + +def get_change_request(db: Session, request_id: int) -> ChangeRequest: + try: + return db.query(ChangeRequest).filter(ChangeRequest.id == request_id).one() + except NoResultFound: + raise MissingChangeRequestException(obj_id=request_id) + + +def decline_change_request(db: Session, change_request: ChangeRequest, reviewed_by: User, + comment: Optional[str]) -> ChangeRequest: + change_request.status = ChangeStatus.DECLINED + change_request.comment = comment + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.reviewed_by = reviewed_by + db.commit() + return change_request + + +def review_changes(db: Session, change_request_id: int, review: ChangeReviewSchema, + reviewed_by: User) -> typing.Tuple[ChangeRequest, bool]: + try: + change_request = db.query(ChangeRequest).filter(ChangeRequest.id == change_request_id).one() + except NoResultFound: + raise MissingChangeRequestException(obj_id=change_request_id) + if change_request.status != ChangeStatus.PENDING: + return change_request, False + + if review.result == ReviewResult.DECLINE: + decline_change_request( + db=db, + change_request=change_request, + reviewed_by=reviewed_by, + comment=review.comment + ) + return change_request, True + kwargs = {'db': db, 'change_request': change_request, 'reviewed_by': reviewed_by, + 'comment': review.comment} + + changed = False + if change_request.object_type == EditableObjectType.SCHEMA: + if change_request.change_type == ChangeType.CREATE: + changed |= apply_schema_create_request(**kwargs)[0] + elif change_request.change_type == ChangeType.UPDATE: + changed |= apply_schema_update_request(**kwargs)[0] + elif change_request.change_type == ChangeType.DELETE: + changed |= apply_schema_delete_request(**kwargs)[0] + + elif change_request.object_type == EditableObjectType.ENTITY: + if change_request.change_type == ChangeType.CREATE: + changed |= apply_entity_create_request(**kwargs)[0] + elif change_request.change_type == ChangeType.UPDATE: + changed |= apply_entity_update_request(**kwargs)[0] + elif change_request.change_type == ChangeType.DELETE: + changed |= apply_entity_delete_request(**kwargs)[0] + + return change_request, changed + + +def get_pending_change_requests(db: Session, obj_type: Optional[ContentType] = None, + limit: Optional[int] = 10, offset: Optional[int] = 0, + all: Optional[bool] = False) -> List[ChangeRequest]: + q = select(ChangeRequest).where(ChangeRequest.status == ChangeStatus.PENDING).order_by(desc(ChangeRequest.id)) + if obj_type is not None: + q = q.join(Change).where(Change.content_type == obj_type).distinct() + if not all: + q = q.offset(offset).limit(limit) + q = q.order_by(ChangeRequest.created_at.desc()) + return db.execute(q).scalars().all() diff --git a/backend/traceability/entity.py b/backend/traceability/entity.py new file mode 100644 index 0000000..af36830 --- /dev/null +++ b/backend/traceability/entity.py @@ -0,0 +1,543 @@ +from collections import defaultdict +from copy import deepcopy +from datetime import datetime, timezone +from typing import List, Tuple, Optional, Dict, Any, Union + +from sqlalchemy import select +from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import Session + +from .. import crud +from .. import dynamic_routes +from ..auth.models import User +from ..exceptions import MissingChangeException, MissingEntityCreateRequestException, \ + AttributeNotDefinedException, MissingEntityUpdateRequestException, \ + MissingEntityDeleteRequestException, MissingChangeRequestException +from ..models import Entity, AttributeDefinition, Schema, Attribute +from ..schemas.traceability import EntityChangeDetailSchema + +from .enum import EditableObjectType, ContentType, ChangeType, ChangeStatus +from .models import ChangeRequest, Change, ChangeAttrType, ChangeValueInt, ChangeValueBool, \ + ChangeValueStr + +def get_recent_entity_changes(db: Session, entity_id: int, count: int = 5) -> List[ChangeRequest]: + return db.execute( + select(ChangeRequest) + .where(ChangeRequest.object_id == entity_id) + .where(ChangeRequest.object_type == EditableObjectType.ENTITY) + .order_by(ChangeRequest.created_at.desc()).limit(count) + .distinct() + ).scalars().all() + + +def _fill_in_change_request_info(change: dict, change_request: ChangeRequest, entity: Entity): + change['object_type'] = change_request.object_type.name + change['change_type'] = change_request.change_type.name + change['reviewed_at'] = change_request.reviewed_at + change['created_at'] = change_request.created_at + change['status'] = change_request.status + change['comment'] = change_request.comment + change['created_by'] = change_request.created_by.username + change['reviewed_by'] = change_request.reviewed_by.username if change_request.reviewed_by else None + change['entity'] = {'slug': entity.slug, 'name': entity.name, 'schema': entity.schema.slug} + +def _fill_in_entity_change(change: dict, entity_change: Change, entity: Entity, db: Session): + field = entity_change.field_name + ValueModel = entity_change.data_type.value.model + v = db.execute(select(ValueModel).where(ValueModel.id == entity_change.value_id)).scalar() + change['changes'][field] = {'new': v.new_value, + 'old': v.old_value, + 'current': getattr(entity, field, None) if entity.id else None} + +def _fill_in_field_change(change: dict, entity_change: Change, entity: Entity, listed_changes: dict[int, List[Change]], checked_listed: List[int], db: Session): + attr = entity_change.attribute + ValueModel = entity_change.data_type.value.model + if attr.id not in listed_changes: + v = db.execute(select(ValueModel).where(ValueModel.id == entity_change.value_id)).scalar() + try: + current = get_old_value(db, entity, attr.name) + except KeyError: + current = None + change['changes'][attr.name] = {'new': v.new_value, 'old': v.old_value, 'current': current} + return + + if attr.id in checked_listed: + return + checked_listed.append(attr.id) + changes = listed_changes[attr.id] + values = db.execute(select(ValueModel).where(ValueModel.id.in_([i.value_id for i in changes]))).scalars().all() + values = [i.new_value for i in values if i.new_value is not None] + change['changes'][attr.name] = {'new': values, + 'old': [] if attr.id in listed_changes else None, + 'current': [] if attr.id in listed_changes else None} + + +def entity_change_details(db: Session, change_request_id: int) -> EntityChangeDetailSchema: + try: + change_request = db.query(ChangeRequest).filter(ChangeRequest.id == change_request_id).one() + except NoResultFound: + raise MissingChangeRequestException(obj_id=change_request_id) + + changes_query = (select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.content_type == ContentType.ENTITY)) + + if change_request.change_type == ChangeType.CREATE and change_request.status != ChangeStatus.APPROVED: + changes_query = changes_query.where(Change.object_id == None) + else: + changes_query = changes_query.where(Change.object_id != None) + + entity_changes = db.execute(changes_query.where(Change.field_name != None)).scalars().all() + fields_changes = db.execute(changes_query.where(Change.attribute_id != None)).scalars().all() + if not entity_changes and not fields_changes: + raise MissingChangeException(obj_id=change_request_id) + + # WARNING: technically, these changes within one ChangeRequest can reference different entities + # although, they should not + if change_request.change_type == ChangeType.CREATE and change_request.status != ChangeStatus.APPROVED: + entity = Entity(name='', slug='', deleted=None) + schema_change = db.execute( + changes_query + .where(Change.change_type == ChangeType.CREATE) + .where(Change.field_name == 'schema_id') + .where(Change.data_type == ChangeAttrType.INT) + ).scalar() + schema_id = db.execute(select(ChangeValueInt).where(ChangeValueInt.id == schema_change.value_id)).scalar() + schema = crud.get_schema(db=db, id_or_slug=schema_id.new_value) + entity.schema = schema + entity.schema_id = schema_id.new_value + elif entity_changes: + entity = crud.get_entity_by_id(db=db, entity_id=entity_changes[0].object_id) + else: + entity = crud.get_entity_by_id(db=db, entity_id=fields_changes[0].object_id) + + change_ = {'changes': {}} + _fill_in_change_request_info(change=change_, change_request=change_request, entity=entity) + + deleted = [i for i in entity_changes if i.field_name == 'deleted'] + if deleted: + deleted = deleted[0] + v = db.execute(select(ChangeValueBool).where(ChangeValueBool.id == deleted.value_id)).scalar() + change_['changes']['deleted'] = {'new': v.new_value, 'old': v.old_value, 'current': entity.deleted} + return EntityChangeDetailSchema(**change_) + + for change in entity_changes: + _fill_in_entity_change(change=change_, entity_change=change, entity=entity, db=db) + + attr_defs = db.execute( + select(AttributeDefinition) + .where(AttributeDefinition.schema_id == entity.schema_id) + .where(AttributeDefinition.attribute_id.in_([i.attribute_id for i in fields_changes])) + ).scalars().all() + listed_ids = {i.attribute_id for i in attr_defs if i.list} + listed_changes = {i: [j for j in fields_changes if j.attribute_id == i] for i in listed_ids} + checked_listed = [] + + for change in fields_changes: + _fill_in_field_change(change=change_, entity_change=change, entity=entity, listed_changes=listed_changes, checked_listed=checked_listed, db=db) + db.expunge_all() + return EntityChangeDetailSchema(**change_) + + +def get_old_value(db: Session, entity: Entity, attr_name: str) -> Any: + val = entity.get(attr_name, db) + if isinstance(val, list): + return None + return val.value if val is not None else None + + +def create_entity_create_request(db: Session, data: dict, schema_id: int, created_by: User, commit: bool = True) -> ChangeRequest: + crud.create_entity(db=db, schema_id=schema_id, data=deepcopy(data), commit=False) + db.rollback() + + sch: Schema = db.execute( + select(Schema).where(Schema.id == schema_id).where(Schema.deleted == False) + ).scalar() + + attr_defs: Dict[str, AttributeDefinition] = {i.attribute.name: i for i in sch.attr_defs} + change_request = ChangeRequest( + created_by=created_by, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.ENTITY, + change_type=ChangeType.CREATE + ) + + entity_change_kwargs = { + 'change_request': change_request, + 'content_type': ContentType.ENTITY, + 'change_type': ChangeType.CREATE + } + + name_val = ChangeValueStr(new_value=data.pop('name')) + db.add(name_val) + db.flush() + name_change = Change( + value_id=name_val.id, + field_name='name', + data_type=ChangeAttrType.STR, + **entity_change_kwargs + ) + + slug_val = ChangeValueStr(new_value=data.pop('slug')) + db.add(slug_val) + db.flush() + slug_change = Change( + value_id=slug_val.id, + field_name='slug', + data_type=ChangeAttrType.STR, + **entity_change_kwargs + ) + + schema_val = ChangeValueInt(new_value=schema_id) + db.add(schema_val) + db.flush() + schema_change = Change( + value_id=schema_val.id, + field_name='schema_id', + data_type=ChangeAttrType.INT, + **entity_change_kwargs + ) + + db.add_all([change_request, name_change, slug_change, schema_change]) + + for field, value in data.items(): + attr_def = attr_defs.get(field) + attr: Attribute = attr_def.attribute + model, caster, _ = attr.type.value + model = ChangeAttrType[attr.type.name].value.model + values = crud._convert_values(attr_def=attr_def, value=value, caster=caster) or [None] + for val in values: + v = model(new_value=val) + db.add(v) + db.flush() + change = Change( + change_request=change_request, + value_id=v.id, + attribute=attr, + data_type=ChangeAttrType[attr.type.name], + content_type=ContentType.ENTITY, + change_type=ChangeType.CREATE + ) + db.add_all([v, change]) + + if commit: + db.commit() + else: + db.flush() + return change_request + + +def apply_entity_create_request(db: Session, change_request: ChangeRequest, reviewed_by: User, + comment: Optional[str] = None) -> Tuple[bool, Entity]: + entity_change = ( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.content_type == ContentType.ENTITY) + .where(Change.change_type == ChangeType.CREATE) + ) + name_change = db.execute( + entity_change + .where(Change.field_name == 'name') + .where(Change.data_type == ChangeAttrType.STR) + ).scalar() + slug_change = db.execute( + entity_change + .where(Change.field_name == 'slug') + .where(Change.data_type == ChangeAttrType.STR) + ).scalar() + schema_change = db.execute( + entity_change + .where(Change.field_name == 'schema_id') + .where(Change.data_type == ChangeAttrType.INT) + ).scalar() + if not all([name_change, slug_change, schema_change]): + raise MissingEntityCreateRequestException(obj_id=change_request.id) + + schema_id = db.execute( + select(ChangeValueInt).where(ChangeValueInt.id == schema_change.value_id) + ).scalar().new_value + schema = crud.get_schema(db=db, id_or_slug=schema_id) + + name = db.execute( + select(ChangeValueStr) + .where(ChangeValueStr.id == name_change.value_id) + ).scalar().new_value + slug = db.execute( + select(ChangeValueStr) + .where(ChangeValueStr.id == slug_change.value_id) + ).scalar().new_value + + value_changes = db.execute( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.attribute_id != None) + .where(Change.content_type == ContentType.ENTITY) + .where(Change.change_type == ChangeType.CREATE) + ).scalars().all() + + single_changes = [] + listed_changes = defaultdict(list) + for change in value_changes: + attr_def = db.execute( + select(AttributeDefinition) + .where(AttributeDefinition.schema_id == schema.id) + .where(AttributeDefinition.attribute_id == change.attribute_id) + ).scalar() + if attr_def is None: + raise AttributeNotDefinedException(attr_id=change.attribute_id, schema_id=schema.id) + + if attr_def.list: + listed_changes[change.attribute.name].append(change) + else: + single_changes.append(change) + + data = {'name': name, 'slug': slug} + for change in single_changes: + ChangeValueModel = change.data_type.value.model + v = db.execute(select(ChangeValueModel).where(ChangeValueModel.id == change.value_id)).scalar() + data[change.attribute.name] = v.new_value + + for attr_name, changes in listed_changes.items(): + ChangeValueModel = changes[0].data_type.value.model + value_ids = [i.value_id for i in changes] + values = db.execute(select(ChangeValueModel).where(ChangeValueModel.id.in_(value_ids))).scalars().all() + data[attr_name] = [i.new_value for i in values if i.new_value is not None] + + EntityCreateModel = dynamic_routes._create_entity_request_model(schema=schema) + + e = crud.create_entity( + db=db, + schema_id=schema.id, + data=EntityCreateModel(**data).dict(), + commit=False + ) + name_change.object_id = e.id # setting object_id is required + slug_change.object_id = e.id # to be able to show details + schema_change.object_id = e.id + for change in value_changes: # for this change request + change.object_id = e.id + change_request.object_id = e.id + change_request.status = ChangeStatus.APPROVED + change_request.reviewed_by = reviewed_by + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.comment = comment + db.commit() + return True, e + + +def create_entity_update_request(db: Session, id_or_slug: Union[int, str], schema_id: int, data: dict, created_by: User, commit: bool = True) -> ChangeRequest: + crud.update_entity(db=db, id_or_slug=id_or_slug, schema_id=schema_id, data=deepcopy(data), commit=False) + db.rollback() + if isinstance(id_or_slug, int): + q = select(Entity).where(Entity.id == id_or_slug) + else: + q = select(Entity).where(Entity.slug == id_or_slug) + entity = db.execute(q).scalar() + + change_request = ChangeRequest( + created_by=created_by, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.ENTITY, + object_id=entity.id, + change_type=ChangeType.UPDATE, + ) + db.add(change_request) + + entity_fields = {'name': data.pop('name', None), 'slug': data.pop('slug', None)} + entity_fields = {k: v for k, v in entity_fields.items() if v is not None} + for field, value in entity_fields.items(): + v = ChangeValueStr(old_value=getattr(entity, field), new_value=value) + db.add(v) + db.flush() + change = Change( + change_request=change_request, + object_id=entity.id, + field_name=field, + value_id=v.id, + data_type=ChangeAttrType.STR, + content_type=ContentType.ENTITY, + change_type=ChangeType.UPDATE + ) + db.add(change) + + + attr_defs: Dict[str, AttributeDefinition] = {i.attribute.name: i for i in entity.schema.attr_defs} + for field, value in data.items(): + attr_def = attr_defs.get(field) + attr: Attribute = attr_def.attribute + model, caster, _ = attr.type.value + model = ChangeAttrType[attr.type.name].value.model + + if value is None: + old_value = get_old_value(db=db, entity=entity, attr_name=attr.name) + v = model(old_value=old_value, new_value=None) + db.add(v) + db.flush() + change = Change( + change_request=change_request, + object_id=entity.id, + attribute_id=attr.id, + value_id=v.id, + data_type=ChangeAttrType[attr.type.name], + content_type=ContentType.ENTITY, + change_type=ChangeType.UPDATE + ) + db.add(change) + continue + + values = crud._convert_values(attr_def=attr_def, value=value, caster=caster) + for val in values: + old_value = get_old_value(db=db, entity=entity, attr_name=attr.name) + v = model(old_value=old_value, new_value=val) + db.add(v) + db.flush() + change = Change( + change_request=change_request, + object_id=entity.id, + attribute_id=attr.id, + value_id=v.id, + data_type=ChangeAttrType[attr.type.name], + content_type=ContentType.ENTITY, + change_type=ChangeType.UPDATE + ) + db.add(change) + if commit: + db.commit() + else: + db.flush() + return change_request + + +def apply_entity_update_request(db: Session, change_request: ChangeRequest, reviewed_by: User, comment: Optional[str] = None) -> Tuple[bool, Entity]: + changes_query = (select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.content_type == ContentType.ENTITY) + .where(Change.change_type == ChangeType.UPDATE)) + entity_fields_changes = db.execute( + changes_query.where(Change.field_name != None) + ).scalars().all() + other_fields_changes = db.execute( + changes_query.where(Change.attribute_id != None) + ).scalars().all() + + if not entity_fields_changes and not other_fields_changes: + raise MissingEntityUpdateRequestException(obj_id=change_request.id) + + if entity_fields_changes: + entity = crud.get_entity_by_id(db=db, entity_id=entity_fields_changes[0].object_id) + else: + entity = crud.get_entity_by_id(db=db, entity_id=other_fields_changes[0].object_id) + + single_changes = [] + listed_changes = defaultdict(list) + for change in other_fields_changes: + attr_def = db.execute( + select(AttributeDefinition) + .where(AttributeDefinition.schema_id == entity.schema_id) + .where(AttributeDefinition.attribute_id == change.attribute_id) + ).scalar() + if attr_def is None: + raise AttributeNotDefinedException(attr_id=change.attribute_id, schema_id=entity.schema_id) + + if attr_def.list: + listed_changes[change.attribute.name].append(change) + else: + single_changes.append(change) + + data = {} + for change in entity_fields_changes: + ValueModel = change.data_type.value.model + value = db.execute(select(ValueModel).where(ValueModel.id == change.value_id)).scalar() + value.old_value = getattr(entity, change.field_name) + data[change.field_name] = value.new_value + + for change in single_changes: + ValueModel = change.data_type.value.model + v = db.execute(select(ValueModel).where(ValueModel.id == change.value_id)).scalar() + data[change.attribute.name] = v.new_value + v.old_value = get_old_value(db=db, entity=entity, attr_name=change.attribute.name) + + for attr_name, changes in listed_changes.items(): + ValueModel = changes[0].data_type.value.model + value_ids = [i.value_id for i in changes] + values = db.execute(select(ValueModel).where(ValueModel.id.in_(value_ids))).scalars().all() + data[attr_name] = [i.new_value for i in values if i.new_value is not None] + + UpdateModel = dynamic_routes._update_entity_request_model(schema=entity.schema) + entity = crud.update_entity( + db=db, + id_or_slug=entity.id, + schema_id=entity.schema_id, + data=UpdateModel(**data).dict(exclude_unset=True), + commit=False + ) + change_request.status = ChangeStatus.APPROVED + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.reviewed_by_user_id = reviewed_by.id + change_request.comment = comment + db.commit() + return True, entity + + +def create_entity_delete_request(db: Session, id_or_slug: Union[int, str], schema_id: int, created_by: User, commit: bool = True) -> ChangeRequest: + crud.delete_entity(db=db, id_or_slug=id_or_slug, schema_id=schema_id, commit=False) + db.rollback() + schema = crud.get_schema(db=db, id_or_slug=schema_id) + entity = crud.get_entity_model(db=db, id_or_slug=id_or_slug, schema=schema) + + change_request = ChangeRequest( + created_by=created_by, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.ENTITY, + object_id=entity.id, + change_type=ChangeType.DELETE + ) + db.add(change_request) + + val = ChangeValueBool(old_value=entity.deleted, new_value=True) + db.add(val) + db.flush() + change = Change( + change_request=change_request, + data_type=ChangeAttrType.BOOL, + change_type=ChangeType.DELETE, + content_type=ContentType.ENTITY, + object_id=entity.id, + field_name='deleted', + value_id=val.id + ) + db.add(change) + if commit: + db.commit() + else: + db.flush() + return change_request + +def apply_entity_delete_request(db: Session, change_request: ChangeRequest, reviewed_by: User, comment: Optional[str]) -> Tuple[bool, Entity]: + change = db.execute( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.data_type == ChangeAttrType.BOOL) + .where(Change.change_type == ChangeType.DELETE) + .where(Change.content_type == ContentType.ENTITY) + .where(Change.field_name == 'deleted') + .where(Change.object_id != None) + ).scalar() + + if change is None: + raise MissingEntityDeleteRequestException(obj_id=change_request.id) + entity = crud.get_entity_by_id(db=db, entity_id=change.object_id) + v = db.execute(select(ChangeValueBool).where(ChangeValueBool.id == change.value_id)).scalar() + v.old_value = entity.deleted + entity = crud.delete_entity( + db=db, + id_or_slug=entity.id, + schema_id=entity.schema_id, + commit=False + ) + change_request.status = ChangeStatus.APPROVED + change_request.reviewed_by = reviewed_by + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.comment = comment + db.commit() + return True, entity diff --git a/backend/traceability/enum.py b/backend/traceability/enum.py new file mode 100644 index 0000000..592ef04 --- /dev/null +++ b/backend/traceability/enum.py @@ -0,0 +1,30 @@ +import enum + + +class EditableObjectType(enum.Enum): + SCHEMA = 'SCHEMA' + ENTITY = 'ENTITY' + + +class ContentType(enum.Enum): + ATTRIBUTE = 'ATTRIBUTE' + ATTRIBUTE_DEFINITION = 'ATTRIBUTE_DEFINITION' + ENTITY = 'ENTITY' + SCHEMA = 'SCHEMA' + + +class ReviewResult(enum.Enum): + APPROVE = 'APPROVE' + DECLINE = 'DECLINE' + + +class ChangeStatus(enum.Enum): + PENDING = 'PENDING' + DECLINED = 'DECLINED' + APPROVED = 'APPROVED' + + +class ChangeType(enum.Enum): + CREATE = 'CREATE' + UPDATE = 'UPDATE' + DELETE = 'DELETE' diff --git a/backend/traceability/models.py b/backend/traceability/models.py new file mode 100644 index 0000000..ef7f41a --- /dev/null +++ b/backend/traceability/models.py @@ -0,0 +1,121 @@ +import enum + +from sqlalchemy import Column, Integer, Float, String, DateTime, Date, Boolean, ForeignKey, Enum +from sqlalchemy.orm import relationship +from sqlalchemy.sql.schema import CheckConstraint + +from ..base_models import Mapping +from ..database import Base +from ..utils import make_aware_datetime + +from .enum import ChangeStatus, EditableObjectType, ChangeType, ContentType + + +class ChangeValue(Base): + __abstract__ = True + + id = Column(Integer, primary_key=True, index=True) + + +class ChangeValueBool(ChangeValue): + __tablename__ = 'change_values_bool' + old_value = Column(Boolean, nullable=True) + new_value = Column(Boolean, nullable=True) + + +class ChangeValueInt(ChangeValue): + __tablename__ = 'change_values_int' + old_value = Column(Integer, nullable=True) + new_value = Column(Integer, nullable=True) + + +class ChangeValueFloat(ChangeValue): + __tablename__ = 'change_values_float' + old_value = Column(Float, nullable=True) + new_value = Column(Float, nullable=True) + + +class ChangeValueForeignKey(ChangeValue): + __tablename__ = 'change_values_fk' + old_value = Column(Integer, nullable=True) + new_value = Column(Integer, nullable=True) + + +class ChangeValueStr(ChangeValue): + __tablename__ = 'change_values_str' + old_value = Column(String, nullable=True) + new_value = Column(String, nullable=True) + + +class ChangeValueDatetime(ChangeValue): + __tablename__ = 'change_values_datetime' + old_value = Column(DateTime(timezone=True), nullable=True) + new_value = Column(DateTime(timezone=True), nullable=True) + + +class ChangeValueDate(ChangeValue): + __tablename__ = 'change_values_date' + old_value = Column(Date, nullable=True) + new_value = Column(Date, nullable=True) + + +class ChangeAttrType(enum.Enum): + STR = Mapping(ChangeValueStr, str) + BOOL = Mapping(ChangeValueBool, bool) + INT = Mapping(ChangeValueInt, int) + FLOAT = Mapping(ChangeValueFloat, float) + FK = Mapping(ChangeValueForeignKey, int) + DT = Mapping(ChangeValueDatetime, make_aware_datetime) + DATE = Mapping(ChangeValueDate, lambda x: x) + + +class ChangeRequest(Base): + __tablename__ = 'change_requests' + id = Column(Integer, primary_key=True) + created_by_user_id = Column(Integer, ForeignKey('users.id'), nullable=False) + reviewed_by_user_id = Column(Integer, ForeignKey('users.id')) + + created_at = Column(DateTime(timezone=True), nullable=False) + reviewed_at = Column(DateTime(timezone=True), nullable=True) + status = Column(Enum(ChangeStatus), default=ChangeStatus.PENDING, nullable=False) + comment = Column(String(1024), nullable=True) + object_type = Column(Enum(EditableObjectType), nullable=False) + object_id = Column(Integer, nullable=True) + change_type = Column(Enum(ChangeType), nullable=False) + + created_by = relationship('User', foreign_keys=[created_by_user_id]) + reviewed_by = relationship('User', foreign_keys=[reviewed_by_user_id]) + entity = relationship('Entity', + primaryjoin="and_(Entity.id == foreign(ChangeRequest.object_id), " + "ChangeRequest.object_type == 'ENTITY')", + overlaps="schema") + schema = relationship('Schema', + primaryjoin="and_(Schema.id == foreign(ChangeRequest.object_id), " + "ChangeRequest.object_type == 'SCHEMA')", + overlaps="entity") + + __table_args__ = ( + CheckConstraint("object_id IS NOT NULL OR (change_type = 'CREATE' AND status <> 'APPROVED')"), + ) + + +class Change(Base): + __tablename__ = 'changes' + id = Column(Integer, primary_key=True) + change_request_id = Column(Integer, ForeignKey('change_requests.id'), nullable=False) + attribute_id = Column(Integer, ForeignKey('attributes.id'), nullable=True) + + object_id = Column(Integer, nullable=True) + value_id = Column(Integer, nullable=False) + content_type = Column(Enum(ContentType), nullable=False) + change_type = Column(Enum(ChangeType), nullable=False) + field_name = Column(String, nullable=True) + data_type = Column(Enum(ChangeAttrType), nullable=False) + + change_request = relationship('ChangeRequest') + attribute = relationship('Attribute') + + __table_args__ = ( + CheckConstraint('NOT(attribute_id IS NULL AND field_name IS NULL)' + ' AND NOT (attribute_id IS NOT NULL AND field_name IS NOT NULL)'), + ) diff --git a/backend/traceability/schema.py b/backend/traceability/schema.py new file mode 100644 index 0000000..b253c2e --- /dev/null +++ b/backend/traceability/schema.py @@ -0,0 +1,533 @@ +import enum +from datetime import datetime, timezone +from itertools import groupby, chain +import typing + +from sqlalchemy import select +from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import Session + +from ..auth.models import User +from ..models import Schema +from ..schemas.schema import AttrDefSchema, SchemaCreateSchema, SchemaUpdateSchema, \ + AttributeDefinition, SchemaBaseSchema +from ..schemas.traceability import SchemaChangeDetailSchema, ChangeSchema +from .. import crud +from .. import exceptions + +from .enum import EditableObjectType, ContentType, ChangeType, ChangeStatus +from .models import ChangeRequest, Change, ChangeValueInt, ChangeAttrType, ChangeValueBool, \ + ChangeValueStr, ChangeValue + + +SCHEMA_FIELDS = [ + ('name', ChangeAttrType.STR), + ('slug', ChangeAttrType.STR), + ('reviewable', ChangeAttrType.BOOL) +] +ATTRIBUTE_FIELDS = [ + ('name', ChangeAttrType.STR), + ('type', ChangeAttrType.STR), +] +DEFINITION_FIELDS = [ + ('required', ChangeAttrType.BOOL), + ('unique', ChangeAttrType.BOOL), + ('list', ChangeAttrType.BOOL), + ('key', ChangeAttrType.BOOL), + ('description', ChangeAttrType.STR), + ('bound_schema_id', ChangeAttrType.INT), +] +FIELD_MAP = { + 'bound_schema_id': 'bound_schema_id' +} + + +def get_pending_entity_create_requests_for_schema(db: Session, schema_id: int) -> typing.List[ChangeRequest]: + q = ( + select(ChangeRequest) + .where(ChangeRequest.status == ChangeStatus.PENDING) + .where(ChangeRequest.change_type == ChangeType.CREATE) + .where(ChangeRequest.object_type == EditableObjectType.ENTITY) + .join(Change) + .where(Change.field_name == 'schema_id') + .join(ChangeValueInt, Change.value_id == ChangeValueInt.id) + .where(ChangeValueInt.new_value == schema_id) + ) + return db.execute(q).scalars().all() + + +def get_recent_schema_changes(db: Session, schema_id: int, count: int = 5) \ + -> typing.Tuple[typing.List[ChangeRequest], typing.List[ChangeRequest]]: + schema_changes = db.execute( + select(ChangeRequest) + .where(ChangeRequest.object_id == schema_id) + .where(ChangeRequest.object_type == EditableObjectType.SCHEMA) + .order_by(ChangeRequest.created_at.desc()).limit(count) + .distinct() + ).scalars().all() + pending_entity_requests = get_pending_entity_create_requests_for_schema(db=db, schema_id=schema_id) + return schema_changes, pending_entity_requests + + +def schema_change_details(db: Session, change_request_id: int) -> SchemaChangeDetailSchema: + try: + change_request = db\ + .query(ChangeRequest)\ + .filter(ChangeRequest.id == change_request_id, + ChangeRequest.object_type == EditableObjectType.SCHEMA)\ + .one() + except NoResultFound: + raise exceptions.MissingChangeRequestException(obj_id=change_request_id) + + schema_changes = db\ + .query(Change)\ + .filter(Change.change_request_id == change_request.id, + Change.field_name != None, + Change.field_name != 'id', + Change.content_type == ContentType.SCHEMA) + + attr_changes = db.execute( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.content_type == ContentType.ATTRIBUTE_DEFINITION) + .order_by(Change.object_id) + ).scalars().all() + + schema, attr_defs = None, None + if change_request.object_id: + schema = crud.get_schema(db=db, id_or_slug=change_request.object_id) + attr_defs = {d.attribute.name: d for d in schema.attr_defs} + + change_ = { + 'changes': {}, + 'object_type': change_request.object_type.name, + 'change_type': change_request.change_type.name, + 'reviewed_at': change_request.reviewed_at, + 'created_at': change_request.created_at, + 'status': change_request.status, + 'comment': change_request.comment, + 'created_by': change_request.created_by.username, + 'reviewed_by': change_request.reviewed_by.username if change_request.reviewed_by else None, + 'schema': SchemaBaseSchema.from_orm(schema) if schema else None + } + + deleted = [i for i in schema_changes if i.field_name == 'deleted'] + if deleted: + deleted = deleted[0] + v = db.execute(select(ChangeValueBool).where(ChangeValueBool.id == deleted.value_id)).scalar() + change_['changes']['deleted'] = {'new': v.new_value, 'old': v.old_value, + 'current': schema.deleted} + return SchemaChangeDetailSchema(**change_) + + for change in schema_changes: + v = get_value_for_change(change, db) + if v.new_value is None: + continue + change_['changes'][change.field_name] = {'old': v.old_value, 'new': v.new_value, + 'current': getattr(schema, change.field_name, None)} + + for change in attr_changes: + v = get_value_for_change(change, db) + if change.field_name: + attr_name, field_name = change.field_name.split(".", maxsplit=1) + else: + attr_name, field_name = change.attribute.name, None + + if not attr_defs or attr_name not in attr_defs: + current_value = None + elif field_name in ["name", "type"]: + current_value = getattr(attr_defs.get(attr_name).attribute, field_name) + elif field_name is None: + current_value = attr_name + else: + current_value = getattr(attr_defs.get(attr_name), field_name, None) + if isinstance(current_value, enum.Enum): + current_value = current_value.name + change_["changes"][change.field_name or attr_name] = ChangeSchema( + old=v.old_value, + current=current_value, + new=v.new_value + ) + + return SchemaChangeDetailSchema(**change_) + + +def create_schema_create_request(db: Session, data: SchemaCreateSchema, created_by: User, + commit: bool = True) -> ChangeRequest: + crud.create_schema(db=db, data=data, commit=False) + db.rollback() + + change_request = ChangeRequest( + created_by=created_by, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.SCHEMA, + change_type=ChangeType.CREATE + ) + db.add(change_request) + + for field, type_ in SCHEMA_FIELDS: + ValueModel = type_.value.model + v = ValueModel(new_value=getattr(data, field)) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name=field, + value_id=v.id, + data_type=type_, + content_type=ContentType.SCHEMA, + change_type=ChangeType.CREATE + )) + + for attr in data.attributes: + for field, type_ in chain(ATTRIBUTE_FIELDS, DEFINITION_FIELDS): + ValueModel = type_.value.model + new_value = getattr(attr, field) + if isinstance(new_value, enum.Enum): + new_value = new_value.value + v = ValueModel(new_value=new_value) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name=f"{attr.name}.{field}", + value_id=v.id, + data_type=type_, + content_type=ContentType.ATTRIBUTE_DEFINITION, + change_type=ChangeType.CREATE + )) + if commit: + db.commit() + else: + db.flush() + return change_request + + +def get_value_for_change(change: Change, db: Session) -> ChangeValue: + ValueModel = change.data_type.value.model + return db.execute(select(ValueModel).where(ValueModel.id == change.value_id)).scalar() + + +def apply_schema_create_request(db: Session, change_request: ChangeRequest, reviewed_by: User, + comment: typing.Optional[str] = None, commit: bool = True) \ + -> typing.Tuple[bool, Schema]: + schema_changes = db.execute( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.field_name != None) + .where(Change.object_id == None) + .where(Change.content_type == ContentType.SCHEMA) + .where(Change.change_type == ChangeType.CREATE) + ).scalars().all() + if not schema_changes: + raise exceptions.MissingSchemaCreateRequestException(obj_id=change_request.id) + + data = {'attributes': []} + for change in schema_changes: + v = get_value_for_change(change=change, db=db) + data[change.field_name] = v.new_value + + attr_changes = db.execute( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.field_name != None) + .where(Change.object_id == None) + .where(Change.content_type == ContentType.ATTRIBUTE_DEFINITION) + .where(Change.change_type == ChangeType.CREATE) + ).scalars().all() + grouped_attr_changes = groupby(attr_changes, key=lambda x: x.field_name.split(".", maxsplit=1)[0]) + for attr_name, changes in grouped_attr_changes: + attr_data = {"name": attr_name} + for change in changes: + attr_name2, field_name = change.field_name.split(".", maxsplit=1) + assert attr_name2 == attr_name + attr_data[field_name] = get_value_for_change(change=change, db=db).new_value + data["attributes"].append(attr_data) + data = SchemaCreateSchema(**data) + + schema = crud.create_schema(db=db, data=data, commit=False) + change_request.object_id = schema.id + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.reviewed_by_user_id = reviewed_by.id + change_request.status = ChangeStatus.APPROVED + change_request.comment = comment + + attr_defs = {d.attribute.name: d.id for d in schema.attr_defs} + for change in schema_changes: # setting object_id is required + change.object_id = schema.id # to be able to show details + for change in attr_changes: # for this change request + attr_name = change.field_name.split(".", maxsplit=1)[0] + change.object_id = attr_defs.get(attr_name) + + v = ChangeValueInt(new_value=schema.id) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name='id', + value_id=v.id, + object_id=schema.id, + data_type=ChangeAttrType.INT, + content_type=ContentType.SCHEMA, + change_type=ChangeType.CREATE + )) + if commit: + db.commit() + else: + db.flush() + return True, schema + + +def create_schema_update_request(db: Session, id_or_slug: typing.Union[int, str], + data: SchemaUpdateSchema, created_by: User, commit: bool = True) \ + -> ChangeRequest: + crud.update_schema(db=db, id_or_slug=id_or_slug, data=data, commit=False) + db.rollback() + schema = crud.get_schema(db=db, id_or_slug=id_or_slug) + + change_request = ChangeRequest( + created_by=created_by, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.SCHEMA, + object_id=schema.id, + change_type=ChangeType.UPDATE + ) + db.add(change_request) + + for field, type_ in SCHEMA_FIELDS: + new_value, old_value = getattr(data, field), getattr(schema, field) + if new_value == old_value: + continue + ValueModel = type_.value.model + v = ValueModel(new_value=new_value, old_value=old_value) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name=field, + value_id=v.id, + object_id=schema.id, + data_type=type_, + content_type=ContentType.SCHEMA, + change_type=ChangeType.UPDATE + )) + + attr_map: typing.Dict[int, AttributeDefinition] = {i.id: i for i in schema.attr_defs} + + added, updated, deleted = crud.sort_attribute_definitions(schema=schema, + definitions=data.attributes) + for attr in updated: + attr_def = attr_map.get(attr.id) + for field, type_ in ATTRIBUTE_FIELDS: + cfield = FIELD_MAP.get(field, field) + ValueModel = type_.value.model + new_value = getattr(attr, field) + old_value = getattr(attr_def.attribute, cfield) + if isinstance(new_value, enum.Enum): + new_value = new_value.name + if isinstance(old_value, enum.Enum): + old_value = old_value.name + if old_value == new_value: + continue + v = ValueModel(new_value=new_value, old_value=old_value) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name=f"{attr.name}.{field}", + object_id=attr_def.id, + value_id=v.id, + data_type=type_, + content_type=ContentType.ATTRIBUTE_DEFINITION, + change_type=ChangeType.UPDATE + )) + for field, type_ in DEFINITION_FIELDS: + cfield = FIELD_MAP.get(field, field) + ValueModel = type_.value.model + new_value = getattr(attr, field) + old_value = getattr(attr_def, cfield) + if isinstance(new_value, enum.Enum): + new_value = new_value.name + if isinstance(old_value, enum.Enum): + old_value = old_value.name + if new_value == old_value: + continue + v = ValueModel(new_value=new_value, old_value=old_value) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name=f"{attr.name}.{field}", + object_id=attr_def.id, + value_id=v.id, + data_type=type_, + content_type=ContentType.ATTRIBUTE_DEFINITION, + change_type=ChangeType.UPDATE + )) + + for attr in added: + for field, type_ in chain(ATTRIBUTE_FIELDS, DEFINITION_FIELDS): + ValueModel = type_.value.model + new_value = getattr(attr, field) + if isinstance(new_value, enum.Enum): + new_value = new_value.name + v = ValueModel(new_value=new_value) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + field_name=f"{attr.name}.{field}", + value_id=v.id, + data_type=type_, + content_type=ContentType.ATTRIBUTE_DEFINITION, + change_type=ChangeType.CREATE + )) + + for attr_def in deleted: + v = ChangeValueStr(old_value=attr_def.attribute.name) + db.add(v) + db.flush() + db.add(Change( + change_request=change_request, + object_id=attr_def.id, + attribute_id=attr_def.attribute_id, + value_id=v.id, + data_type=ChangeAttrType.STR, + content_type=ContentType.ATTRIBUTE_DEFINITION, + change_type=ChangeType.DELETE + )) + + if commit: + db.commit() + else: + db.flush() + return change_request + + +def apply_schema_update_request(db: Session, change_request: ChangeRequest, reviewed_by: User, + comment: typing.Optional[str] = None) -> typing.Tuple[bool, Schema]: + schema_query = ( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.field_name != None) + .where(Change.object_id != None) + .where(Change.content_type == ContentType.SCHEMA) + .where(Change.change_type == ChangeType.UPDATE) + ) + + schema = change_request.schema + attr_defs = {d.id: d for d in schema.attr_defs} + schema_changes = db.execute(schema_query.where(Change.field_name != 'id')).scalars().all() + + attr_changes = db.query(Change)\ + .filter(Change.change_request_id == change_request.id, + Change.content_type == ContentType.ATTRIBUTE_DEFINITION)\ + .all() + + if not schema or not any([schema_changes, attr_changes]): + raise exceptions.MissingSchemaUpdateRequestException(obj_id=change_request.id) + + attributes = [] + for key, changes in groupby((a for a in attr_changes if a.change_type == ChangeType.UPDATE), + key=lambda x: x.object_id): + attr_data = AttrDefSchema.from_orm(attr_defs.get(key)) + for change in changes: + attr_name, field_name = change.field_name.split(".", maxsplit=1) + if attr_name not in attr_data: + setattr(attr_data, "name", attr_name) + setattr(attr_data, field_name, get_value_for_change(change=change, db=db).new_value) + attributes.append(attr_data) + for key, changes in groupby((a for a in attr_changes if a.change_type == ChangeType.CREATE), + key=lambda x: x.field_name.split(".", maxsplit=1)[0]): + attr_data = {"name": key} + for change in changes: + attr_name, field_name = change.field_name.split(".", maxsplit=1) + assert key == attr_name + attr_data[field_name] = get_value_for_change(change=change, db=db).new_value + attributes.append(AttrDefSchema(**attr_data)) + + unchanged_attributes = [AttrDefSchema.from_orm(a) + for a_id, a in attr_defs.items() + if a_id not in [_a.object_id for _a in attr_changes if _a.object_id]] + + data = {"attributes": attributes + unchanged_attributes} + for change in schema_changes: + data[change.field_name] = get_value_for_change(change=change, db=db).new_value + + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.reviewed_by = reviewed_by + change_request.status = ChangeStatus.APPROVED + change_request.comment = comment + schema = crud.update_schema(db=db, id_or_slug=schema.id, data=SchemaUpdateSchema(**data), + commit=False) + db.refresh(schema) + created_attr_defs = {d.attribute.name: d.id for d in schema.attr_defs} + for change in attr_changes: + if change.change_type == ChangeType.CREATE and not change.object_id: + attr_name = change.field_name.split(".", maxsplit=1)[0] + change.object_id = created_attr_defs.get(attr_name) + if change.object_id is None: + raise ValueError() + db.commit() + return True, schema + + +def create_schema_delete_request(db: Session, id_or_slug: typing.Union[int, str], created_by: User, + commit: bool = True) -> ChangeRequest: + crud.delete_schema(db=db, id_or_slug=id_or_slug, commit=False) + db.rollback() + schema = crud.get_schema(db=db, id_or_slug=id_or_slug) + change_request = ChangeRequest( + created_by=created_by, + created_at=datetime.now(timezone.utc), + object_type=EditableObjectType.SCHEMA, + object_id=schema.id, + change_type=ChangeType.DELETE + ) + db.add(change_request) + v = ChangeValueBool(old_value=schema.deleted, new_value=True) + db.add(v) + db.flush() + db.add( + Change( + change_request=change_request, + field_name='deleted', + value_id=v.id, + object_id=schema.id, + data_type=ChangeAttrType.BOOL, + content_type=ContentType.SCHEMA, + change_type=ChangeType.DELETE + ) + ) + if commit: + db.commit() + else: + db.flush() + return change_request + + +def apply_schema_delete_request(db: Session, change_request: ChangeRequest, reviewed_by: User, + comment: typing.Optional[str]) -> typing.Tuple[bool, Schema]: + change = db.execute( + select(Change) + .where(Change.change_request_id == change_request.id) + .where(Change.field_name == 'deleted') + .where(Change.object_id != None) + .where(Change.data_type == ChangeAttrType.BOOL) + .where(Change.content_type == ContentType.SCHEMA) + .where(Change.change_type == ChangeType.DELETE) + ).scalar() + if change is None: + raise exceptions.MissingSchemaDeleteRequestException(obj_id=change_request.id) + + v = get_value_for_change(change=change, db=db) + if not v.new_value: + raise exceptions.MissingSchemaDeleteRequestException(obj_id=change_request.id) + + schema = crud.delete_schema(db=db, id_or_slug=change.object_id, commit=False) + change_request.status = ChangeStatus.APPROVED + change_request.reviewed_by = reviewed_by + change_request.reviewed_at = datetime.now(timezone.utc) + change_request.comment = comment + db.commit() + return True, schema diff --git a/backend/utils.py b/backend/utils.py new file mode 100644 index 0000000..6b24b26 --- /dev/null +++ b/backend/utils.py @@ -0,0 +1,20 @@ +from datetime import datetime +import typing + +from sqlalchemy.orm import class_mapper +from sqlalchemy.orm.properties import StrategizedProperty, ColumnProperty + +from .config import settings +from .database import Base + + +def make_aware_datetime(dt: datetime) -> datetime: + if dt and dt.tzinfo is None: + return dt.replace(tzinfo=settings.timezone) + return dt + + +def iterate_model_fields(model: Base) -> typing.Dict[str, StrategizedProperty]: + return {prop.key: prop + for prop in class_mapper(model).iterate_properties + if isinstance(prop, ColumnProperty)} diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 10adaee..1830ba6 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -54,7 +54,9 @@ export default { components: {SchemaList, AlertDisplay, AuthNav, ReviewNav, HelpNav}, data: function () { return { - activeSchema: null + activeSchema: null, + schemaDetails: {}, + apiInfo: null } }, provide() { @@ -62,7 +64,8 @@ export default { activeSchema: computed(() => this.activeSchema), availableSchemas: computed(() => this.$refs.schemalist.schemas), pendingRequests: computed(() => this.$refs.pendingrequests.changes), - updatePendingRequests: this.onPendingReviews + updatePendingRequests: this.onPendingReviews, + apiInfo: computed(() => this.apiInfo) } }, computed: { @@ -87,19 +90,12 @@ export default { if (!schemaSlug) { return null; } - if (Object.keys(this.availableSchemas).length < 1) { - if (!this.$refs.schemalist) { - console.warn("Schema list not ready, yet") - return - } - await this.$refs.schemalist.load(); - } - try { - // First, try to reuse data in storage - this.activeSchema = this.availableSchemas[schemaSlug]; - } catch (e) { - this.activeSchema = await this.$api.getSchema({slugOrId: schemaSlug}); + + if (!(schemaSlug in this.schemaDetails)){ + this.schemaDetails[schemaSlug] = await this.$api.getSchema({slugOrId: schemaSlug}); } + + this.activeSchema = this.schemaDetails[schemaSlug]; } }, watch: { @@ -107,6 +103,9 @@ export default { handler: "getSchemaFromRoute", immediate: true, // runs immediately with mount() instead of calling method on mount hook }, + }, + async created() { + this.apiInfo = await this.$api.getInfo(); } } \ No newline at end of file diff --git a/frontend/src/components/EditAttributes.vue b/frontend/src/components/EditAttributes.vue index da4f6d2..ba0a6d8 100644 --- a/frontend/src/components/EditAttributes.vue +++ b/frontend/src/components/EditAttributes.vue @@ -1,19 +1,20 @@