add: add galaxy clusters, relations, relations tags

api/alembic/versions/39cbeba07d8b_add_galaxies_tables.py

@@ -62,7 +62,7 @@ def upgrade():
         sa.Column("tag_name", sa.String(255), nullable=False),
         sa.Column("description", sa.String(), nullable=False),
         sa.Column("galaxy_id", sa.Integer(), nullable=False),
-        sa.Column("source", sa.String(255), nullable=False),
+        sa.Column("source", sa.String(255), nullable=True),
         sa.Column("authors", sa.JSON(), nullable=False, default={}),
         sa.Column("version", sa.Integer(), nullable=True),
         sa.Column(
@@ -113,7 +113,7 @@ def upgrade():
         "galaxy_cluster_relations",
         sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
         sa.Column("galaxy_cluster_id", sa.Integer(), nullable=False),
-        sa.Column("referenced_galaxy_cluster_id", sa.Integer(), nullable=False),
+        sa.Column("referenced_galaxy_cluster_id", sa.Integer(), nullable=True),
         sa.Column(
             "referenced_galaxy_cluster_uuid",
             sa.types.Uuid(as_uuid=False),
@@ -127,7 +127,7 @@ def upgrade():
             nullable=False,
         ),
         sa.Column("sharing_group_id", sa.Integer(), nullable=True),
-        sa.Column("enabled", sa.Boolean(), nullable=False),
+        sa.Column("default", sa.Boolean(), nullable=False),
         sa.PrimaryKeyConstraint("id"),
         sa.ForeignKeyConstraint(
             ["galaxy_cluster_id"],
@@ -139,6 +139,18 @@ def upgrade():
         ),
     )
 
+    op.create_table(
+        "galaxy_cluster_relation_tags",
+        sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+        sa.Column("galaxy_cluster_relation_id", sa.Integer(), nullable=False),
+        sa.Column("tag_id", sa.Integer, nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.ForeignKeyConstraint(
+            ["galaxy_cluster_relation_id"],
+            ["galaxy_cluster_relations.id"],
+        ),
+    )
+
     op.create_table(
         "galaxy_cluster_blocklists",
         sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
@@ -153,6 +165,7 @@ def upgrade():
 
 def downgrade():
     op.drop_table("galaxy_cluster_blocklists")
+    op.drop_table("galaxy_cluster_relation_tags")
     op.drop_table("galaxy_cluster_relations")
     op.drop_table("galaxy_elements")
     op.drop_table("galaxy_clusters")

api/app/models/galaxy.py

@@ -13,7 +13,7 @@
     String,
 )
 from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 
 class Galaxy(Base):
@@ -42,35 +42,78 @@ class Galaxy(Base):
         nullable=False,
         default=DistributionLevel.INHERIT_EVENT,
     )
-    # predicates = relationship("TaxonomyPredicate", lazy="subquery")
+    clusters = relationship("GalaxyCluster", lazy="subquery")
 
 
-# class TaxonomyPredicate(Base):
-#     __tablename__ = "taxonomy_predicates"
+class GalaxyCluster(Base):
+    __tablename__ = "galaxy_clusters"
+
+    id = Column(Integer, primary_key=True, index=True)
+    uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4)
+    collection_uuid = Column(UUID(as_uuid=True), nullable=True)
+    type = Column(String, nullable=False)
+    value = Column(String, nullable=False)
+    tag_name = Column(String, nullable=False)
+    description = Column(String, nullable=False)
+    galaxy_id = Column(Integer, ForeignKey("galaxies.id"), index=True, nullable=False)
+    source = Column(String, nullable=True)
+    authors = Column(JSON, nullable=True, default={})
+    version = Column(Integer, nullable=True)
+    distribution: Mapped[DistributionLevel] = mapped_column(
+        Enum(DistributionLevel, name="distribution_level"),
+        nullable=False,
+        default=DistributionLevel.INHERIT_EVENT,
+    )
+    sharing_group_id = Column(
+        Integer, ForeignKey("sharing_groups.id"), index=True, nullable=True
+    )
+    org_id = Column(Integer, ForeignKey("organisations.id"), index=True, nullable=False)
+    orgc_id = Column(
+        Integer, ForeignKey("organisations.id"), index=True, nullable=False
+    )
+    extends_uuid = Column(UUID(as_uuid=True), nullable=True)
+    extends_version = Column(Integer, nullable=True)
+    published = Column(Boolean, nullable=False, default=False)
+    deleted = Column(Boolean, nullable=False, default=False)
 
-#     id = Column(Integer, primary_key=True, index=True)
-#     taxonomy_id = Column(
-#         Integer, ForeignKey("taxonomies.id"), index=True, nullable=True
-#     )
-#     value = Column(String, nullable=False)
-#     expanded = Column(String, nullable=False)
-#     colour = Column(String, nullable=False)
-#     description = Column(String, nullable=False)
-#     exclusive = Column(Boolean, nullable=False, default=False)
-#     numerical_value = Column(Integer, index=True)
 
-#     entries = relationship("TaxonomyEntry", lazy="subquery")
+class GalaxyElement(Base):
+    __tablename__ = "galaxy_elements"
+    id = Column(Integer, primary_key=True, index=True)
+    key = Column(String, nullable=False)
+    value = Column(String, nullable=False)
+    galaxy_cluster_id = Column(
+        Integer, ForeignKey("galaxy_clusters.id"), index=True, nullable=False
+    )
 
 
-# class TaxonomyEntry(Base):
-#     __tablename__ = "taxonomy_entries"
+class GalaxyClusterRelation(Base):
+    __tablename__ = "galaxy_cluster_relations"
+    id = Column(Integer, primary_key=True, index=True)
+    galaxy_cluster_id = Column(
+        Integer, ForeignKey("galaxy_clusters.id"), index=True, nullable=False
+    )
+    referenced_galaxy_cluster_id = Column(
+        Integer, ForeignKey("galaxy_clusters.id"), index=True, nullable=False
+    )
+    referenced_galaxy_cluster_uuid = Column(UUID(as_uuid=True), nullable=False)
+    referenced_galaxy_cluster_type = Column(String, nullable=False)
+    galaxy_cluster_uuid = Column(UUID(as_uuid=True), nullable=False)
+    distribution: Mapped[DistributionLevel] = mapped_column(
+        Enum(DistributionLevel, name="distribution_level"),
+        nullable=False,
+        default=DistributionLevel.INHERIT_EVENT,
+    )
+    sharing_group_id = Column(
+        Integer, ForeignKey("sharing_groups.id"), index=True, nullable=True
+    )
+    default = Column(Boolean, nullable=False, default=False)
 
-#     id = Column(Integer, primary_key=True, index=True)
-#     taxonomy_predicate_id = Column(
-#         Integer, ForeignKey("taxonomy_predicates.id"), index=True, nullable=True
-#     )
-#     value = Column(String, nullable=False)
-#     expanded = Column(String, nullable=False)
-#     colour = Column(String, nullable=False)
-#     description = Column(String, nullable=False)
-#     numerical_value = Column(Integer, index=True)
+
+class GalaxyClusterRelationTag(Base):
+    __tablename__ = "galaxy_cluster_relation_tags"
+    id = Column(Integer, primary_key=True, index=True)
+    galaxy_cluster_relation_id = Column(
+        Integer, ForeignKey("galaxy_cluster_relations.id"), index=True, nullable=False
+    )
+    tag_id = Column(Integer, ForeignKey("tags.id"), index=True, nullable=False)

api/app/repositories/galaxies.py

@@ -2,7 +2,9 @@
 import os
 from datetime import datetime
 
+from app.models import event as events_models
 from app.models import galaxy as galaxies_models
+from app.repositories import tags as tags_repository
 from app.schemas import galaxy as galaxies_schemas
 from app.schemas import user as users_schemas
 from fastapi import HTTPException, Query, status
@@ -37,6 +39,7 @@ def update_galaxies(
 ) -> list[galaxies_schemas.Galaxy]:
     galaxies = []
     galaxies_dir = "app/submodules/misp-galaxy/galaxies"
+    galaxies_clusters_dir = "app/submodules/misp-galaxy/clusters"
 
     for root, __, files in os.walk(galaxies_dir):
         for galaxy_file in files:
@@ -71,11 +74,136 @@ def update_galaxies(
                 db.commit()
                 db.refresh(galaxy)
 
+                # parse galaxy clusters file
+                with open(os.path.join(galaxies_clusters_dir, galaxy_file)) as f:
+                    clusters_data_raw = json.load(f)
+
+                    if "values" in clusters_data_raw:
+                        for cluster in clusters_data_raw["values"]:
+                            galaxy_cluster = galaxies_models.GalaxyCluster(
+                                galaxy_id=galaxy.id,
+                                uuid=cluster["uuid"],
+                                value=cluster["value"],
+                                type=(
+                                    clusters_data_raw["type"]
+                                    if "type" in clusters_data_raw
+                                    else galaxy.type
+                                ),
+                                description=(
+                                    cluster["description"]
+                                    if "description" in cluster
+                                    else ""
+                                ),
+                                source=(
+                                    clusters_data_raw["source"]
+                                    if "source" in clusters_data_raw
+                                    else None
+                                ),
+                                version=clusters_data_raw["version"],
+                                authors=(
+                                    clusters_data_raw["authors"]
+                                    if "authors" in clusters_data_raw
+                                    else None
+                                ),
+                                tag_name=f"misp-galaxy:{galaxy.type}={cluster['uuid']}",
+                                org_id=user.org_id,
+                                orgc_id=user.org_id,
+                                collection_uuid=(
+                                    clusters_data_raw["collection_uuid"]
+                                    if "collection_uuid" in clusters_data_raw
+                                    else None
+                                ),
+                                extends_uuid=(
+                                    clusters_data_raw["extends_uuid"]
+                                    if "extends_uuid" in clusters_data_raw
+                                    else None
+                                ),
+                                extends_version=(
+                                    clusters_data_raw["extends_version"]
+                                    if "extends_version" in clusters_data_raw
+                                    else None
+                                ),
+                            )
+                            db.add(galaxy_cluster)
+                            db.flush()
+
+                            # add galaxy elements
+                            if "meta" in cluster:
+                                for element in cluster["meta"]:
+                                    galaxy_element = galaxies_models.GalaxyElement(
+                                        galaxy_cluster_id=galaxy_cluster.id,
+                                        key=element,
+                                        value=(
+                                            cluster["meta"][element]
+                                            if isinstance(cluster["meta"][element], str)
+                                            else json.dumps(cluster["meta"][element])
+                                        ),
+                                    )
+                                    db.add(galaxy_element)
+
+                            # commit galaxy elements
+                            db.commit()
+
+                            # add galaxy relations
+                            if "related" in cluster:
+                                for relation in cluster["related"]:
+
+                                    # check if valid uuid
+                                    if (
+                                        "dest-uuid" not in relation
+                                        or not relation["dest-uuid"]
+                                    ):
+                                        continue
+
+                                    galaxy_relation = galaxies_models.GalaxyClusterRelation(
+                                        galaxy_cluster_id=galaxy_cluster.id,
+                                        galaxy_cluster_uuid=cluster["uuid"],
+                                        referenced_galaxy_cluster_uuid=relation[
+                                            "dest-uuid"
+                                        ],
+                                        referenced_galaxy_cluster_type=relation["type"],
+                                        default=True,
+                                        distribution=events_models.DistributionLevel.ALL_COMMUNITIES,
+                                    )
+                                    db.add(galaxy_relation)
+                                    db.flush()
+
+                                    if "tags" in relation:
+                                        for tag in relation["tags"]:
+                                            tag = tags_repository.get_tag_by_name(
+                                                db, tag_name=tag
+                                            )
+
+                                            if tag:
+                                                galaxy_relation_tag = galaxies_models.GalaxyClusterRelationTag(
+                                                    galaxy_cluster_relation_id=galaxy_relation.id,
+                                                    tag=tag,
+                                                )
+
+                                                db.add(galaxy_relation_tag)
+
+                            # commit galaxy relations and tags
+                            db.commit()
+
+                        # commit galaxy clusters
+                        db.commit()
+
                 galaxies.append(galaxy)
 
-        # db.add(db_entry)
-        # db.commit()
-        # db.refresh(db_entry)
+    # fix galaxy cluster relations references to galaxy clusters
+    relations = db.query(galaxies_models.GalaxyClusterRelation).all()
+    for relation in relations:
+        galaxy_cluster = (
+            db.query(galaxies_models.GalaxyCluster)
+            .filter(
+                galaxies_models.GalaxyCluster.uuid
+                == relation.referenced_galaxy_cluster_uuid
+            )
+            .first()
+        )
+        relation.referenced_galaxy_cluster_id = galaxy_cluster.id
+        db.add(relation)
+    db.commit()
 
     return galaxies
 

api/app/repositories/tags.py

@@ -1,3 +1,5 @@
+from functools import lru_cache
+
 from app.models import attribute as attribute_models
 from app.models import event as event_models
 from app.models import tag as tag_models
@@ -146,6 +148,7 @@ def capture_tag(db: Session, tag: MISPTag, user: user_models.User) -> tag_models
     return db_tag
 
 
+@lru_cache
 def get_tag_by_name(db: Session, tag_name: str):
     return (
         db.query(tag_models.Tag)

api/app/schemas/galaxy.py

@@ -34,3 +34,20 @@ class GalaxyUpdate(BaseModel):
     default: Optional[bool] = None
     enabled: Optional[bool] = None
     local_only: Optional[bool] = None
+
+
+class GalaxyClusterBase(BaseModel):
+    name: str
+    description: str
+    version: int
+    icon: str
+    namespace: str
+    enabled: bool
+    local_only: bool
+    kill_chain_order: Optional[dict] = {}
+    default: bool
+    org_id: int
+    orgc_id: int
+    created: datetime
+    modified: datetime
+    distribution: DistributionLevel