Case-insensitive search (fixes #95, #87) (#96)

* Add filter options for case-insensitive searching
* Also support filtering by entity name and slug
* Offer advanced filter options in the UI for name and slug

backend/crud.py

@@ -410,23 +410,24 @@ def _parse_filters(filters: dict, attrs: List[str]) \
     First `dict` is for attribute filters, second is for `Entity.name` filters
     '''
     filter_map = {f.value.name: f for f in FilterEnum}
-    attrs_filters = defaultdict(dict)
-    name_filters = {}
+    entity_fields = ('name', 'slug')
+    attr_filters = defaultdict(dict)
+    entity_filters = defaultdict(dict)
     for f, v in filters.items():
         split = f.rsplit('.', maxsplit=1)
         attr = split[0]
         filter = FilterEnum.EQ if len(split) == 1 else filter_map.get(split[-1], None)
-        if attr != "name" and attr not in attrs:
+        if attr not in entity_fields and attr not in attrs:
             raise InvalidFilterAttributeException(attr=attr, allowed_attrs=attrs)
         if not filter:
             raise InvalidFilterOperatorException(attr=attr, filter=split[-1])
 
-        if attr == 'name':
-            name_filters[filter] = v
-            continue
-        attrs_filters[attr][filter] = v
+        if attr in entity_fields:
+            entity_filters[attr][filter] = v
+        else:
+            attr_filters[attr][filter] = v
 
-    return attrs_filters, name_filters
+    return attr_filters, entity_filters
 
 
 def _query_entity_with_filters(filters: dict, schema: Schema, all: bool = False,
@@ -437,19 +438,25 @@ def _query_entity_with_filters(filters: dict, schema: Schema, all: bool = False,
     '''
     attrs = {i.attribute.name: i.attribute
              for i in schema.attr_defs if i.attribute.type.value.filters}
-    attrs_filters, name_filters = _parse_filters(filters=filters, attrs=attrs.keys())
+    attr_filters, entity_filters = _parse_filters(filters=filters, attrs=attrs.keys())
     selects = []
 
-    # since `name` is defined in `Entity`, not in `Value` tables, we need to query it separately
-    if name_filters:
+    # Add filters for entity model
+    if entity_filters:
         q = select(Entity).where(Entity.schema_id == schema.id)
         if not all:
             q = q.where(Entity.deleted == deleted_only)
-        for f, v in name_filters.items():
-            q = q.where(getattr(Entity.name, f.value.op)(v))
+        for field_name, filters in entity_filters.items():
+            for f, v in filters.items():
+                field = getattr(Entity, field_name, None)
+                if field is None:
+                    raise AttributeError(f"Entity has no field {field_name}")
+
+                q = q.where(getattr(Entity.name, f.value.op)(v))
         selects.append(q)
 
-    for attr_name, filters in attrs_filters.items():
+    # Add filters for attribute values
+    for attr_name, filters in attr_filters.items():
         attr = attrs[attr_name]
         value_model = attr.type.value.model
         q = select(Entity).where(Entity.schema_id == schema.id).join(value_model)
@@ -569,6 +576,7 @@ def _check_fk_value(db: Session, attr_def: AttributeDefinition, entity_ids: List
                 passed_entity=entity
             )
 
+
 def _check_unique_value(db: Session, attr_def: AttributeDefinition, model: Value, value: Any):
     existing = db.execute(
         select(model)
@@ -721,4 +729,4 @@ def delete_entity(db: Session, id_or_slug: Union[int, str], schema_id: int, comm
         db.commit()
     else:
         db.flush()
-    return e
+    return e

backend/enum.py

@@ -15,9 +15,10 @@ class FilterEnum(Enum):
     LE = Filter('le', '__le__', 'less than of equal to')
     GE = Filter('ge', '__ge__', 'greater than or equal to')
     NE = Filter('ne', '__ne__', 'not equal to')
-    CONTAINS = Filter('contains', 'contains', 'contains substring')
-    REGEXP = Filter('regexp', 'regexp_match', 'matches regular expression')
-    STARTS = Filter('starts', 'startswith', 'starts with substring')
+    CONTAINS = Filter('contains', 'icontains', 'contains substring')
+    REGEXP = Filter('regexp', 'iregexp_match', 'matches regular expression')
+    STARTS = Filter('starts', 'istartswith', 'starts with substring')
+    IEQ = Filter('ieq', 'ieq', 'equal to (case insensitive)')
 
 
 class ModelVariant(Enum):

backend/models.py

@@ -1,12 +1,14 @@
 import enum
+from operator import eq
 from typing import List, Union, Optional
 
 from sqlalchemy import (
-    select, Enum, DateTime, Date,
+    func, select, Enum, DateTime, Date,
     Boolean, Column, ForeignKey,
     Integer, String, Float)
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.session import Session
+from sqlalchemy.sql.operators import startswith_op, endswith_op, contains_op, regexp_match_op
 from sqlalchemy.sql.schema import UniqueConstraint
 
 from .base_models import Value, Mapping
@@ -15,6 +17,35 @@
 from .utils import make_aware_datetime
 
 
+class CaseInsensitiveComparator(String.Comparator):
+    def ioperate(self, op, *other, **kwargs):
+        return op(
+            func.lower(self.__clause_element__()),
+            *[func.lower(o) for o in other],
+            **kwargs
+        )
+
+    def istartswith(self, other, **kwargs):
+        return self.ioperate(startswith_op, other, **kwargs)
+
+    def iendswith(self, other, **kwargs):
+        return self.ioperate(endswith_op, other, **kwargs)
+
+    def icontains(self, other, **kwargs):
+        return self.ioperate(contains_op, other, **kwargs)
+
+    def iregexp_match(self, other, **kwargs):
+        return self.ioperate(regexp_match_op, other, **kwargs)
+
+    def ieq(self, other, **kwargs):
+        return self.ioperate(eq, other, **kwargs)
+
+
+class CaseInsensitiveString(String):
+    Comparator = CaseInsensitiveComparator
+    comparator_factory = CaseInsensitiveComparator
+
+
 class ValueBool(Value):
     __tablename__ = 'values_bool'
     value = Column(Boolean)
@@ -37,21 +68,22 @@ class ValueForeignKey(Value):
 
 class ValueStr(Value):
     __tablename__ = 'values_str'
-    value = Column(String)
+    value = Column(CaseInsensitiveString)
 
 
 class ValueDatetime(Value):
     __tablename__ = 'values_datetime'
     value = Column(DateTime(timezone=True))
 
+
 class ValueDate(Value):
     __tablename__ = 'values_date'
     value = Column(Date)
 
 
 class AttrType(enum.Enum):
     STR = Mapping(ValueStr, str, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, FilterEnum.LE,
-                                  FilterEnum.GE, FilterEnum.NE, FilterEnum.CONTAINS,
+                                  FilterEnum.GE, FilterEnum.NE, FilterEnum.CONTAINS, FilterEnum.IEQ,
                                   FilterEnum.REGEXP, FilterEnum.STARTS])
     BOOL = Mapping(ValueBool, bool, [FilterEnum.EQ, FilterEnum.NE])
     INT = Mapping(ValueInt, int, [FilterEnum.EQ, FilterEnum.LT, FilterEnum.GT, FilterEnum.LE,
@@ -86,8 +118,8 @@ class Entity(Base):
     __tablename__ = 'entities'
 
     id = Column(Integer, primary_key=True, index=True)
-    name = Column(String(128), nullable=False)
-    slug = Column(String(128), nullable=False)
+    name = Column(CaseInsensitiveString(128), nullable=False)
+    slug = Column(CaseInsensitiveString(128), nullable=False)
     schema_id = Column(Integer, ForeignKey('schemas.id'))
     deleted = Column(Boolean, default=False)
 

backend/tests/test_crud_entity.py

@@ -369,6 +369,8 @@ def test_offset_and_limit(self, dbsession):
         ({'age.le': 10},              1, ['Jack']),
         ({'age.lt': 10},              0, []),
         ({'age.ne': 10},              1, ['Jane']),
+        ({'slug.starts': 'Ja'},       2, ['Jack', 'Jane']),
+        ({'slug.contains': 'ck'},     1, ['Jack']),
         ({'name': 'Jane'},            1, ['Jane']),
         ({'nickname': 'jane'},        1, ['Jane']),
         ({'nickname.ne': 'jack'},     1, ['Jane']),
@@ -641,4 +643,3 @@ def test_raise_on_already_deleted(self, dbsession):
         dbsession.flush()
         with pytest.raises(MissingEntityException):
             delete_entity(dbsession, id_or_slug=entity.id, schema_id=entity.schema_id)
-

backend/tests/test_dynamic_routes.py

@@ -345,7 +345,8 @@ def test_get_with_filters_and_offset_limit(self, dbsession, client):
         ('deleted_only=true&page=2&age.gt=0&age.lt=20',   set()),
     ])
     def test_get_with_filters_and_deleted(self, dbsession, client, q, slugs):
-        dbsession.execute(update(Entity).where(Entity.slug == 'Jack').values(deleted=True))
+        jack = dbsession.scalars(select(Entity).where(Entity.slug == 'Jack')).one()
+        jack.deleted = True
         dbsession.commit()
         response = client.get(f'/entity/person?{q}')
         assert {i["slug"] for i in response.json()['items']} == slugs
@@ -360,6 +361,28 @@ def test_ignore_filters_for_fk(self, dbsession, client):
         response = client.get('/entity/person?friends=1')
         assert {i["slug"] for i in response.json()['items']} == {'Jack', 'Jane'}
 
+    @pytest.mark.parametrize(['q', 'slugs'], [
+        ('name=Jack', {'Jack'}),
+        ('name=jack', set()),
+        ('name.starts=Ja', {'Jack', 'Jane'}),
+        ('name.starts=ja', {'Jack', 'Jane'}),
+        ('name.regexp=^Jac', {'Jack'}),
+        ('name.regexp=^jac', {'Jack'}),
+        ('name.ieq=Jack', {'Jack'}),
+        ('name.ieq=jack', {'Jack'}),
+        ('nickname=Jack', set()),
+        ('nickname=jack', {'Jack'}),
+        ('nickname.starts=Ja', {'Jack', 'Jane'}),
+        ('nickname.starts=ja', {'Jack', 'Jane'}),
+        ('nickname.regexp=^Jac', {'Jack'}),
+        ('nickname.regexp=^jac', {'Jack'}),
+        ('nickname.ieq=Jack', {'Jack'}),
+        ('nickname.ieq=jack', {'Jack'})
+    ])
+    def test_get_with_caseinsensitive_filter(self, dbsession, client, q, slugs):
+        response = client.get(f'/entity/person?{q}')
+        assert {i["slug"] for i in response.json()['items']} == slugs
+
 
 class TestRouteUpdateEntity(DefaultMixin):
     def test_update(self, dbsession, authorized_client):

frontend/src/components/SearchPanel.vue

@@ -123,14 +123,12 @@ export default {
       return params;
     },
     fieldOptions() {
-      let r = this.schema?.attributes.map(x => {
-        return {value: x.name}
-      });
-      r.unshift({value: '', text: '-- select one --'})
+      let r = Object.keys(this.fieldTypes).map(x => ({value: x}));
+      r.unshift({value: '', text: '-- select one --'});
       return r;
     },
     fieldTypes() {
-      const t = {};
+      const t = {'name': 'STR', 'slug': 'STR'};
       for (let a of (this.schema?.attributes || [])) {
         t[a.name] = a.type;
       }