Merge pull request #53 from flowintel/add-galaxy-cluster-select

refactor tag selector, support galaxy clusters

api/app/repositories/galaxies.py

@@ -6,6 +6,7 @@
 
 from app.models import event as events_models
 from app.models import galaxy as galaxies_models
+from app.models import tag as tags_models
 from app.repositories import tags as tags_repository
 from app.schemas import galaxy as galaxies_schemas
 from app.schemas import user as users_schemas
@@ -16,13 +17,18 @@
 logger = logging.getLogger(__name__)
 
 
-def get_galaxies(db: Session, filter: str = Query(None)) -> galaxies_models.Galaxy:
+def get_galaxies(
+    db: Session, enabled: bool = Query(None), filter: str = Query(None)
+) -> galaxies_models.Galaxy:
     query = db.query(galaxies_models.Galaxy)
 
     if filter:
-        query = query.filter(galaxies_models.Galaxy.namespace.ilike(f"%{filter}%"))
+        query = query.filter(galaxies_models.Galaxy.name.ilike(f"%{filter}%"))
+
+    if enabled is not None:
+        query = query.filter(galaxies_models.Galaxy.enabled == enabled)
 
-    query = query.order_by(galaxies_models.Galaxy.namespace)
+    query = query.order_by(galaxies_models.Galaxy.name)
 
     return paginate(
         query,
@@ -239,6 +245,47 @@ def update_galaxies(
     return galaxies
 
 
+def enable_galaxy_tags(db: Session, galaxy: galaxies_models.Galaxy):
+    for cluster in galaxy.clusters:
+        galaxy_cluster_tag = f'misp-galaxy:{galaxy.type}="{cluster.value}"'
+        db_galaxy_cluster_tag = (
+            db.query(tags_models.Tag)
+            .filter(
+                tags_models.Tag.name == galaxy_cluster_tag,
+            )
+            .first()
+        )
+
+        if db_galaxy_cluster_tag is None:
+            db_galaxy_cluster_tag = tags_models.Tag(
+                name=galaxy_cluster_tag,
+                colour="#BBBBBB",
+                exportable=False,
+                hide_tag=False,
+                is_galaxy=True,
+                is_custom_galaxy=False,
+                local_only=False,
+            )
+
+            db.add(db_galaxy_cluster_tag)
+            db.commit()
+
+
+def disable_galaxy_tags(db: Session, galaxy: galaxies_models.Galaxy):
+    # delete all tags from the galaxy
+    db_galaxy_tags = (
+        db.query(tags_models.Tag)
+        .filter(tags_models.Tag.name.ilike(f"misp-galaxy:{galaxy.type}%"))
+        .all()
+    )
+
+    for tag in db_galaxy_tags:
+        tag.hide_tag = True
+        db.add(tag)
+
+    db.commit()
+
+
 def update_galaxy(
     db: Session,
     galaxy_id: int,
@@ -255,6 +302,12 @@ def update_galaxy(
     for key, value in galaxy_patch.items():
         setattr(db_galaxy, key, value)
 
+    # if galaxy is enabled, update the tags
+    if db_galaxy.enabled and galaxy_patch["enabled"]:
+        enable_galaxy_tags(db, db_galaxy)
+    elif not db_galaxy.enabled and not galaxy_patch["enabled"]:
+        disable_galaxy_tags(db, db_galaxy)
+
     db.add(db_galaxy)
     db.commit()
     db.refresh(db_galaxy)

api/app/repositories/tags.py

@@ -3,14 +3,25 @@
 from app.models import tag as tag_models
 from app.models import user as user_models
 from app.schemas import tag as tag_schemas
-from fastapi import HTTPException, status
+from fastapi import HTTPException, Query, status
+from fastapi_pagination.ext.sqlalchemy import paginate
 from pymisp import MISPTag
 from sqlalchemy import func
 from sqlalchemy.orm import Session
 
 
-def get_tags(db: Session, skip: int = 0, limit: int = 100):
-    return db.query(tag_models.Tag).offset(skip).limit(limit).all()
+def get_tags(db: Session, hidden: bool = Query(None), filter: str = Query(None)):
+    query = db.query(tag_models.Tag)
+
+    if hidden is not None:
+        query = query.filter(tag_models.Tag.hide_tag == hidden)
+
+    if filter:
+        query = query.filter(tag_models.Tag.name.ilike(f"%{filter}%"))
+
+    query = query.order_by(tag_models.Tag.name)
+
+    return paginate(query)
 
 
 def get_tag_by_id(db: Session, tag_id: int):

api/app/repositories/taxonomies.py

@@ -60,8 +60,8 @@ def get_or_create_predicate(db: Session, db_taxonomy, raw_predicate):
     return db_predicate
 
 
-def get_or_create_predicate_tag(db: Session, db_taxonomy, raw_predicate):
-    predicate_tag = f'{db_taxonomy.namespace}:{raw_predicate["value"]}'
+def get_or_create_predicate_tag(db: Session, db_taxonomy, db_predicate):
+    predicate_tag = f"{db_taxonomy.namespace}:{db_predicate.value}"
     db_predicate_tag = (
         db.query(tags_models.Tag)
         .filter(
@@ -73,16 +73,13 @@ def get_or_create_predicate_tag(db: Session, db_taxonomy, raw_predicate):
     if db_predicate_tag is None:
         db_predicate_tag = tags_models.Tag(
             name=predicate_tag,
-            colour=(raw_predicate["colour"] if "colour" in raw_predicate else ""),
+            colour=db_predicate.colour,
             exportable=False,
             hide_tag=False,
             is_galaxy=False,
             is_custom_galaxy=False,
             local_only=False,
         )
-        db.add(db_predicate_tag)
-        db.commit()
-        db.refresh(db_predicate_tag)
 
     return db_predicate_tag
 
@@ -108,13 +105,16 @@ def get_or_create_entry(db: Session, db_predicate, raw_entry):
                 raw_entry["description"] if "description" in raw_entry else ""
             ),
         )
+
     return db_entry
 
 
 def get_or_create_predicate_entry_tag(
-    db: Session, db_predicate_tag, raw_entry, db_predicate
+    db: Session, db_taxonomy, db_predicate, db_predicate_entry
 ):
-    predicate_entry_tag = f'{db_predicate_tag}:{raw_entry["value"]}'
+    predicate_entry_tag = (
+        f"{db_taxonomy.namespace}:{db_predicate.value}:{db_predicate_entry.value}"
+    )
     db_predicate_entry_tag = (
         db.query(tags_models.Tag)
         .filter(
@@ -126,15 +126,14 @@ def get_or_create_predicate_entry_tag(
     if db_predicate_entry_tag is None:
         db_predicate_entry_tag = tags_models.Tag(
             name=predicate_entry_tag,
-            colour=(
-                raw_entry["colour"] if "colour" in raw_entry else db_predicate.colour
-            ),
+            colour=db_predicate_entry.colour,
             exportable=False,
             hide_tag=False,
             is_galaxy=False,
             is_custom_galaxy=False,
             local_only=False,
         )
+
     return db_predicate_entry_tag
 
 
@@ -198,12 +197,6 @@ def update_taxonomies(db: Session):
 
                 predicates.append(db_predicate)
 
-                # check if the predicate exists in the tags table
-                db_predicate_tag = get_or_create_predicate_tag(
-                    db, db_taxonomy, raw_predicate
-                )
-                db.add(db_predicate_tag)
-
             # process entries
             if "values" not in raw_taxonomy:
                 continue
@@ -221,17 +214,46 @@ def update_taxonomies(db: Session):
                     db_entry = get_or_create_entry(db, db_predicate, raw_entry)
                     db.add(db_entry)
 
-                    # check if the predicate entry exists in the tags table
-                    db_predicate_entry_tag = get_or_create_predicate_entry_tag(
-                        db, db_predicate_tag, raw_entry, db_predicate
-                    )
-                    db.add(db_predicate_entry_tag)
-
                 db.commit()
 
     return taxonomies
 
 
+def enable_taxonomy_tags(db: Session, db_taxonomy):
+
+    for db_predicate in db_taxonomy.predicates:
+        # check if the predicate exists in the tags table
+        db_predicate_tag = get_or_create_predicate_tag(db, db_taxonomy, db_predicate)
+        db.add(db_predicate_tag)
+
+        for db_predicate_entry in db_predicate.entries:
+            # check if the predicate entry exists in the tags table
+            db_predicate_entry_tag = get_or_create_predicate_entry_tag(
+                db,
+                db_taxonomy,
+                db_predicate,
+                db_predicate_entry,
+            )
+            db.add(db_predicate_entry_tag)
+
+        db.commit()
+
+
+def disable_taxonomy_tags(db: Session, db_taxonomy):
+    # delete all tags from the taxonomy
+    db_taxonomy_tags = (
+        db.query(tags_models.Tag)
+        .filter(tags_models.Tag.name.ilike(f"{db_taxonomy.namespace}:%"))
+        .all()
+    )
+
+    for tag in db_taxonomy_tags:
+        tag.hide_tag = True
+        db.add(tag)
+
+    db.commit()
+
+
 def update_taxonomy(
     db: Session,
     taxonomy_id: int,
@@ -248,6 +270,12 @@ def update_taxonomy(
     for key, value in taxonomy_patch.items():
         setattr(db_taxonomy, key, value)
 
+    # if taxonomy is enabled, update the tags
+    if db_taxonomy.enabled and taxonomy_patch["enabled"]:
+        enable_taxonomy_tags(db, db_taxonomy)
+    elif not db_taxonomy.enabled and not taxonomy_patch["enabled"]:
+        disable_taxonomy_tags(db, db_taxonomy)
+
     db.add(db_taxonomy)
     db.commit()
     db.refresh(db_taxonomy)

api/app/routers/galaxies.py

@@ -5,14 +5,19 @@
 from app.schemas import user as user_schemas
 from fastapi import APIRouter, Depends, HTTPException, Query, Security, status
 from fastapi_pagination import Page
-from fastapi_pagination.customization import CustomizedPage, UseModelConfig
+from fastapi_pagination.customization import (
+    CustomizedPage,
+    UseModelConfig,
+    UseParamsFields,
+)
 from sqlalchemy.orm import Session
 
 router = APIRouter()
 
 Page = CustomizedPage[
     Page,
     UseModelConfig(extra="allow"),
+    UseParamsFields(size=Query(le=1000, default=20)),
 ]
 
 
@@ -22,9 +27,10 @@ def get_galaxies(
     user: user_schemas.User = Security(
         get_current_active_user, scopes=["galaxies:read"]
     ),
+    enabled: bool = Query(None),
     filter: str = Query(None),
 ):
-    return galaxies_repository.get_galaxies(db, filter=filter)
+    return galaxies_repository.get_galaxies(db, enabled=enabled, filter=filter)
 
 
 @router.get("/galaxies/{galaxy_id}", response_model=galaxies_schemas.Galaxy)

api/app/routers/tags.py

@@ -3,20 +3,32 @@
 from app.repositories import tags as tags_repository
 from app.schemas import tag as tag_schemas
 from app.schemas import user as user_schemas
-from fastapi import APIRouter, Depends, HTTPException, Security, status
+from fastapi import APIRouter, Depends, HTTPException, Query, Security, status
+from fastapi_pagination import Page
+from fastapi_pagination.customization import (
+    CustomizedPage,
+    UseModelConfig,
+    UseParamsFields,
+)
 from sqlalchemy.orm import Session
 
 router = APIRouter()
 
+Page = CustomizedPage[
+    Page,
+    UseModelConfig(extra="allow"),
+    UseParamsFields(size=Query(le=1000, default=20)),
+]
+
 
-@router.get("/tags/", response_model=list[tag_schemas.Tag])
+@router.get("/tags/", response_model=Page[tag_schemas.Tag])
 def get_tags(
-    skip: int = 0,
-    limit: int = 100,
     db: Session = Depends(get_db),
+    hidden: bool = Query(None),
+    filter: str = Query(None),
     user: user_schemas.User = Security(get_current_active_user, scopes=["tags:read"]),
 ):
-    return tags_repository.get_tags(db, skip=skip, limit=limit)
+    return tags_repository.get_tags(db, hidden=hidden, filter=filter)
 
 
 @router.get("/tags/{tag_id}", response_model=tag_schemas.Tag)

api/app/tests/api/test_tags.py

@@ -23,16 +23,16 @@ def test_get_tags(
 
         assert response.status_code == status.HTTP_200_OK
 
-        assert len(data) == 1
-        assert data[0]["id"] == tlp_white_tag.id
-        assert data[0]["name"] == tlp_white_tag.name
-        assert data[0]["colour"] == tlp_white_tag.colour
-        assert data[0]["exportable"] == tlp_white_tag.exportable
-        assert data[0]["hide_tag"] == tlp_white_tag.hide_tag
-        assert data[0]["is_galaxy"] == tlp_white_tag.is_galaxy
-        assert data[0]["is_custom_galaxy"] == tlp_white_tag.is_custom_galaxy
-        assert data[0]["numerical_value"] == tlp_white_tag.numerical_value
-        assert data[0]["local_only"] == tlp_white_tag.local_only
+        assert len(data["items"]) == 1
+        assert data["items"][0]["id"] == tlp_white_tag.id
+        assert data["items"][0]["name"] == tlp_white_tag.name
+        assert data["items"][0]["colour"] == tlp_white_tag.colour
+        assert data["items"][0]["exportable"] == tlp_white_tag.exportable
+        assert data["items"][0]["hide_tag"] == tlp_white_tag.hide_tag
+        assert data["items"][0]["is_galaxy"] == tlp_white_tag.is_galaxy
+        assert data["items"][0]["is_custom_galaxy"] == tlp_white_tag.is_custom_galaxy
+        assert data["items"][0]["numerical_value"] == tlp_white_tag.numerical_value
+        assert data["items"][0]["local_only"] == tlp_white_tag.local_only
 
     @pytest.mark.parametrize("scopes", [["tags:read"]])
     def test_get_tag_by_id(

frontend/.eslintrc.cjs → frontend/.eslintrc.js

@@ -1,6 +1,3 @@
-/* eslint-env node */
-require("@rushstack/eslint-patch/modern-module-resolution");
-
 module.exports = {
   root: true,
   extends: [
@@ -11,7 +8,7 @@ module.exports = {
   ],
   overrides: [
     {
-      files: ["cypress/e2e/**.{cy,spec}.{js,ts,jsx,tsx}"],
+      files: ["cypress/e2e/**/*.{cy,spec}.{js,ts,jsx,tsx}"],
       extends: ["plugin:cypress/recommended"],
     },
   ],