WIP: 1155 implement project backend

.gitignore

@@ -59,6 +59,7 @@ scripts/keycloak/data/*
 # ignore clowder chart deps
 deployments/kubernetes/charts/clowder2/charts
 deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml
+*secret*.yaml
 
 # Environments
 .env

.run/Python tests in tests.run.xml

@@ -3,9 +3,10 @@
     <module name="clowder2" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
-    <option name="SDK_HOME" value="" />
+    <option name="SDK_HOME" value="$USER_HOME$/.virtualenvs/clowder2-Ai7Xc8E9/bin/python" />
+    <option name="SDK_NAME" value="Python 3.7 (clowder2-Ai7Xc8E9)" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/backend/app/tests" />
-    <option name="IS_MODULE_SDK" value="true" />
+    <option name="IS_MODULE_SDK" value="false" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
@@ -14,4 +15,4 @@
     <option name="_new_targetType" value="&quot;PATH&quot;" />
     <method v="2" />
   </configuration>
-</component>
+</component>

CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v2.0.0-beta.3] - 2024-07-29
+
+### Added
+
+- License management
+- Release dataset with versions
+- Enable and disable user account through Keycloak
+- Jupyterhub integration
+- Interface for creating and editing matching criteria for triggering extractors
+- Interface for editing metadata definitions
+- My dataset tab listing all the datasets created by the user
+- Drag and drop upload multiple files
+- Footer with links to documentation, source code, and contact information
+- Documentation through MKDocs
+
+### Changed
+
+- Allow public datasets and files to be searchable
+- List all the extractors with the ability to enable/disable the extractors
+- Filter listeners based on their support for file or dataset
+- Helm chart updated to support custom existing secret
+
+### Fixed
+
+- Clowder registration link on the top bar
+- Case-insensitive search
+- Download count immediately increments after download
+
 ## [v2.0.0-beta.2] - 2024-02-16
 
 ### Added

backend/app/config.py

@@ -9,7 +9,7 @@ class Settings(BaseSettings):
     API_V2_STR: str = "/api/v2"
     admin_email: str = "[email protected]"
     frontend_url: str = "http://localhost:3000"
-    version: str = "2.0.0-beta.2"
+    version: str = "2.0.0-beta.3"
 
     # Unique secret for hashing API keys. Generate with `openssl rand -hex 32`
     local_auth_secret = "clowder_secret_key"

backend/app/main.py

@@ -26,6 +26,7 @@
     MetadataDefinitionDB,
     MetadataFreezeDB,
 )
+from app.models.project import ProjectDB
 from app.models.thumbnails import ThumbnailDB, ThumbnailDBViewList, ThumbnailFreezeDB
 from app.models.tokens import TokenDB
 from app.models.users import ListenerAPIKeyDB, UserAPIKeyDB, UserDB
@@ -55,6 +56,7 @@
     metadata,
     metadata_datasets,
     metadata_files,
+    projects,
     public_datasets,
     public_elasticsearch,
     public_files,
@@ -86,7 +88,7 @@
     description="A cloud native data management framework to support any research domain. Clowder was "
     "developed to help researchers and scientists in data intensive domains manage raw data, complex "
     "metadata, and automatic data pipelines. ",
-    version="2.0.0-beta.2",
+    version="2.0.0-beta.3",
     contact={"name": "Clowder", "url": "https://clowderframework.org/"},
     license_info={
         "name": "Apache 2.0",
@@ -250,6 +252,11 @@
     prefix="/public_thumbnails",
     tags=["public_thumbnails"],
 )
+api_router.include_router(
+    projects.router,
+    prefix="/projects",
+    tags=["projects"],
+)
 api_router.include_router(
     licenses.router,
     prefix="/licenses",
@@ -315,6 +322,7 @@ async def startup_beanie():
             ThumbnailFreezeDB,
             ThumbnailDBViewList,
             LicenseDB,
+            ProjectDB,
         ],
         recreate_views=True,
     )

backend/app/models/project.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+from enum import Enum, auto
+from typing import List, Optional
+
+import pymongo
+from app.models.authorization import AuthorizationDB, RoleType
+from app.models.groups import GroupOut
+from app.models.users import UserOut
+from beanie import Document, PydanticObjectId, View
+from pydantic import BaseModel, Field
+
+
+class Member(BaseModel):
+    user: UserOut
+    editor: bool = False
+
+
+class ProjectBase(BaseModel):
+    id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="_id")
+    name: str
+    description: Optional[str] = None
+    created: datetime = Field(default_factory=datetime.utcnow)
+    modified: datetime = Field(default_factory=datetime.utcnow)
+    dataset_ids: Optional[List[PydanticObjectId]] = []
+    folder_ids: Optional[List[PydanticObjectId]] = []
+    file_ids: Optional[List[PydanticObjectId]] = []
+    creator: UserOut
+    users: List[Member] = []
+
+
+class ProjectDB(Document, ProjectBase):
+    class Settings:
+        name = "projects"
+
+
+class ProjectIn(ProjectBase):
+    pass
+
+
+class ProjectOut(ProjectDB):
+    class Config:
+        fields = {"id": "id"}