Skip to content

Commit bb8f99d

Browse files
amalcarazamalcaraz
authored
Feature: include file size in cost endpoints (#1013)
* fix: tests * feat: Allow paying STORE with credits * fix: STORE credit tests + updated prices * fix: tests * fix: linter * fix: alembic ids * fix: gt-only linter errors * fix: skip exhaustive time checking for now * fix: tests * fix: tests * featch: change the store cutoff date to 2027-01-01 00:00:00 UTC to properly comunicate the changes to users * fix: tests * fix: tests * feat: min 25Mib cost for credit payed storage * feat: include file size in cost endpoints * fix: tests * fix: tests * fix: linter * feat: calculate size_mib for all cost types * fix: filter costs by credits by default + fix storage size calculation * fix: tests * fix: lint fix * fix: tests * fix: some sizes not being properly calculated + restrict detail level 2 to filtered sets
1 parent af05571 commit bb8f99d

7 files changed

Lines changed: 1068 additions & 27 deletions

File tree

‎src/aleph/db/accessors/cost.py‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
from decimal import Decimal
2-
from typing import Iterable, List, Optional
2+
from typing import Iterable, List, Optional, Tuple
33

44
from aleph_message.models import PaymentType
5-
from sqlalchemy import asc, delete, func, select
5+
from sqlalchemy import and_, asc, delete, func, select
66
from sqlalchemy.dialects.postgresql import insert
77
from sqlalchemy.sql import Insert
88

99
from aleph.db.models import ChainTxDb, message_confirmations
1010
from aleph.db.models.account_costs import AccountCostsDb
11+
from aleph.db.models.files import FilePinDb, FilePinType, StoredFileDb
1112
from aleph.db.models.messages import MessageStatusDb
1213
from aleph.toolkit.costs import format_cost, format_cost_str
14+
from aleph.types.cost import CostType
1315
from aleph.types.db_session import DbSession
1416
from aleph.types.message_status import MessageStatus
1517

@@ -80,6 +82,50 @@ def get_message_costs(session: DbSession, item_hash: str) -> Iterable[AccountCos
8082
return (session.execute(select_stmt)).scalars().all()
8183

8284

85+
# Types where the file size comes from the DB (via file_pins → files).
86+
# For these, account_costs.ref is the item_hash of the linked STORE message,
87+
# and file_pins.item_hash resolves to the actual file content hash.
88+
# EXECUTION_INSTANCE_VOLUME_ROOTFS is intentionally excluded: its ref points to
89+
# the parent base image STORE message, but the billed size is the allocated
90+
# rootfs disk size (content.rootfs.size_mib), which must come from the message content.
91+
_DB_SIZED_COST_TYPES = [
92+
CostType.STORAGE,
93+
CostType.EXECUTION_PROGRAM_VOLUME_CODE,
94+
CostType.EXECUTION_PROGRAM_VOLUME_RUNTIME,
95+
CostType.EXECUTION_PROGRAM_VOLUME_DATA,
96+
CostType.EXECUTION_VOLUME_INMUTABLE,
97+
]
98+
99+
100+
def get_message_costs_with_file_sizes(
101+
session: DbSession, item_hash: str
102+
) -> List[Tuple[AccountCostsDb, Optional[int]]]:
103+
"""Return cost rows for a message with file sizes (bytes) resolved in a single query.
104+
105+
Joins account_costs → file_pins → files for the types listed in
106+
``_DB_SIZED_COST_TYPES``. For all other types (EXECUTION, PERSISTENT,
107+
ROOTFS, DISCOUNT) ``file_size`` is ``None``; callers should fall back to
108+
the message content for ROOTFS and PERSISTENT.
109+
"""
110+
stmt = (
111+
select(
112+
AccountCostsDb,
113+
StoredFileDb.size.label("file_size"),
114+
)
115+
.outerjoin(
116+
FilePinDb,
117+
and_(
118+
FilePinDb.item_hash == AccountCostsDb.ref,
119+
FilePinDb.type == FilePinType.MESSAGE.value,
120+
AccountCostsDb.type.in_(_DB_SIZED_COST_TYPES),
121+
),
122+
)
123+
.outerjoin(StoredFileDb, StoredFileDb.hash == FilePinDb.file_hash)
124+
.where(AccountCostsDb.item_hash == item_hash)
125+
)
126+
return [(row.AccountCostsDb, row.file_size) for row in session.execute(stmt).all()]
127+
128+
83129
def make_costs_upsert_query(costs: List[AccountCostsDb]) -> Insert:
84130
costs_dict = [
85131
cost.to_dict(

‎src/aleph/schemas/api/costs.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,20 @@ class GetCostsQueryParams(BaseModel):
1414
item_hash: Optional[str] = Field(
1515
default=None, description="Filter by specific resource item_hash"
1616
)
17-
payment_type: Optional[PaymentType] = Field(
18-
default=None, description="Filter by payment type (hold, superfluid, credit)"
17+
payment_type: PaymentType = Field(
18+
default=PaymentType.credit,
19+
description="Filter by payment type (hold, superfluid, credit)",
1920
)
2021
include_details: int = Field(
2122
default=0,
2223
ge=0,
2324
le=2,
2425
description="Detail level: 0=summary only, 1=include resource list, 2=include resource list with cost breakdown per component",
2526
)
27+
include_size: bool = Field(
28+
default=False,
29+
description="Include size_mib in cost component details (requires include_details=2 for /costs or applies to /price/:hash). Adds a file_pins+files join; disabled by default for performance.",
30+
)
2631
pagination: int = Field(
2732
default=100,
2833
ge=10,
@@ -68,6 +73,10 @@ class CostComponentDetail(BaseModel):
6873
cost_hold: str = Field(description="Hold cost for this component")
6974
cost_stream: str = Field(description="Streaming cost for this component")
7075
cost_credit: str = Field(description="Credit cost for this component")
76+
size_mib: Optional[float] = Field(
77+
default=None,
78+
description="Storage size in MiB (populated for volume/storage-related components: STORAGE, EXECUTION_INSTANCE_VOLUME_ROOTFS, EXECUTION_PROGRAM_VOLUME_CODE, EXECUTION_PROGRAM_VOLUME_RUNTIME, EXECUTION_PROGRAM_VOLUME_DATA, EXECUTION_VOLUME_PERSISTENT, EXECUTION_VOLUME_INMUTABLE)",
79+
)
7180

7281
@field_validator("cost_hold", "cost_stream", "cost_credit", mode="before")
7382
def check_format_price(cls, v):
@@ -121,6 +130,10 @@ class EstimatedCostDetailResponse(BaseModel):
121130
cost_hold: str
122131
cost_stream: str
123132
cost_credit: str
133+
size_mib: Optional[float] = Field(
134+
default=None,
135+
description="Storage size in MiB (populated for volume/storage-related components: STORAGE, EXECUTION_INSTANCE_VOLUME_ROOTFS, EXECUTION_PROGRAM_VOLUME_CODE, EXECUTION_PROGRAM_VOLUME_RUNTIME, EXECUTION_PROGRAM_VOLUME_DATA, EXECUTION_VOLUME_PERSISTENT, EXECUTION_VOLUME_INMUTABLE)",
136+
)
124137

125138
@field_validator("cost_hold", "cost_stream", "cost_credit", mode="before")
126139
def check_format_price(cls, v):

‎src/aleph/services/cost.py‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,157 @@ def calculate_storage_size(
801801
return storage_mib
802802

803803

804+
def _get_size_from_file_ref(session: DbSession, file_hash: str) -> Optional[float]:
805+
"""Get file size in MiB from file hash."""
806+
file = get_file(session, file_hash)
807+
if not file:
808+
return None
809+
return float(Decimal(file.size) / MiB)
810+
811+
812+
def _get_estimated_size_from_content(
813+
cost: AccountCostsDb, content: CostComputableContent
814+
) -> Optional[float]:
815+
"""Get estimated size from content based on cost type."""
816+
if cost.type == CostType.STORAGE:
817+
if isinstance(content, CostEstimationStoreContent):
818+
if content.estimated_size_mib:
819+
return float(content.estimated_size_mib)
820+
821+
elif cost.type == CostType.EXECUTION_PROGRAM_VOLUME_CODE:
822+
if isinstance(content, CostEstimationProgramContent):
823+
if content.code.estimated_size_mib:
824+
return float(content.code.estimated_size_mib)
825+
826+
elif cost.type == CostType.EXECUTION_PROGRAM_VOLUME_RUNTIME:
827+
if isinstance(content, CostEstimationProgramContent):
828+
if content.runtime.estimated_size_mib:
829+
return float(content.runtime.estimated_size_mib)
830+
831+
elif cost.type == CostType.EXECUTION_PROGRAM_VOLUME_DATA:
832+
if isinstance(content, CostEstimationProgramContent):
833+
if content.data and content.data.estimated_size_mib:
834+
return float(content.data.estimated_size_mib)
835+
836+
elif cost.type == CostType.EXECUTION_VOLUME_INMUTABLE:
837+
if isinstance(
838+
content,
839+
(
840+
InstanceContent,
841+
CostEstimationInstanceContent,
842+
ProgramContent,
843+
CostEstimationProgramContent,
844+
),
845+
):
846+
# Extract volume index from name (format: "#0:/mount/path")
847+
try:
848+
if cost.name and ":" in cost.name:
849+
index_str = cost.name.split(":")[0].replace("#", "")
850+
volume_index = int(index_str)
851+
if volume_index < len(content.volumes):
852+
volume = content.volumes[volume_index]
853+
if (
854+
isinstance(volume, CostEstimationImmutableVolume)
855+
and volume.estimated_size_mib
856+
):
857+
return float(volume.estimated_size_mib)
858+
except (ValueError, IndexError):
859+
pass
860+
861+
return None
862+
863+
864+
def get_cost_component_size_mib(
865+
session: DbSession,
866+
cost: AccountCostsDb,
867+
content: Optional[CostComputableContent] = None,
868+
) -> Optional[float]:
869+
"""
870+
Retrieve the size in MiB for a cost component.
871+
872+
Returns size for volume/storage-related cost types:
873+
- STORAGE
874+
- EXECUTION_INSTANCE_VOLUME_ROOTFS
875+
- EXECUTION_PROGRAM_VOLUME_CODE
876+
- EXECUTION_PROGRAM_VOLUME_RUNTIME
877+
- EXECUTION_PROGRAM_VOLUME_DATA
878+
- EXECUTION_VOLUME_PERSISTENT
879+
- EXECUTION_VOLUME_INMUTABLE
880+
881+
Priority: Real file size first, then estimated size from content as fallback.
882+
883+
Args:
884+
session: Database session
885+
cost: Cost component from database
886+
content: Optional message content (for estimation and content-based sizes)
887+
888+
Returns:
889+
Size in MiB as float, or None if not applicable/unavailable
890+
"""
891+
# EXECUTION_INSTANCE_VOLUME_ROOTFS: always from content
892+
if cost.type == CostType.EXECUTION_INSTANCE_VOLUME_ROOTFS:
893+
if content and isinstance(
894+
content, (InstanceContent, CostEstimationInstanceContent)
895+
):
896+
return float(content.rootfs.size_mib)
897+
return None
898+
899+
# EXECUTION_VOLUME_PERSISTENT: always from content
900+
if cost.type == CostType.EXECUTION_VOLUME_PERSISTENT:
901+
if content and isinstance(
902+
content,
903+
(
904+
InstanceContent,
905+
CostEstimationInstanceContent,
906+
ProgramContent,
907+
CostEstimationProgramContent,
908+
),
909+
):
910+
# Extract volume index from name (format: "#0:/mount/path")
911+
try:
912+
if cost.name and ":" in cost.name:
913+
index_str = cost.name.split(":")[0].replace("#", "")
914+
volume_index = int(index_str)
915+
if volume_index < len(content.volumes):
916+
volume = content.volumes[volume_index]
917+
if hasattr(volume, "size_mib"):
918+
return float(volume.size_mib)
919+
except (ValueError, IndexError):
920+
pass
921+
return None
922+
923+
# For ref-based types: try real file size first, then estimated size
924+
ref_based_types = {
925+
CostType.STORAGE,
926+
CostType.EXECUTION_PROGRAM_VOLUME_CODE,
927+
CostType.EXECUTION_PROGRAM_VOLUME_RUNTIME,
928+
CostType.EXECUTION_PROGRAM_VOLUME_DATA,
929+
CostType.EXECUTION_VOLUME_INMUTABLE,
930+
}
931+
932+
if cost.type in ref_based_types:
933+
# Try to get real size from file (only if session is available).
934+
# cost.ref (and cost.item_hash for STORAGE) is the item_hash of the linked
935+
# STORE message, not the file content hash. Resolve through file_pins first.
936+
if session:
937+
store_msg_hash = cost.ref or (
938+
cost.item_hash if cost.type == CostType.STORAGE else None
939+
)
940+
if store_msg_hash:
941+
pin = get_message_file_pin(session, store_msg_hash)
942+
if pin:
943+
size = _get_size_from_file_ref(session, pin.file_hash)
944+
if size is not None:
945+
return size
946+
947+
# Fall back to estimated size from content
948+
if content:
949+
return _get_estimated_size_from_content(cost, content)
950+
951+
# For all other types (EXECUTION, EXECUTION_VOLUME_DISCOUNT), return None
952+
return None
953+
954+
804955
def get_detailed_costs(
805956
session: DbSession,
806957
content: CostComputableContent,

0 commit comments

Comments
 (0)