diff --git a/Cargo.lock b/Cargo.lock index 9162869b2275..0230f3a03255 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19451,6 +19451,7 @@ dependencies = [ "schemars 0.8.22", "semver", "serde", + "serde_bytes", "serde_cbor", "serde_json", "sha2 0.10.9", @@ -19513,6 +19514,7 @@ dependencies = [ "ic-http-endpoints-public", "ic-https-outcalls-adapter", "ic-https-outcalls-adapter-client", + "ic-https-outcalls-pricing", "ic-https-outcalls-service", "ic-icp-index", "ic-icrc1-index-ng", diff --git a/packages/pocket-ic/BUILD.bazel b/packages/pocket-ic/BUILD.bazel index 383384d6254d..65819b49e17b 100644 --- a/packages/pocket-ic/BUILD.bazel +++ b/packages/pocket-ic/BUILD.bazel @@ -84,6 +84,7 @@ rust_test( "//rs/registry/helpers", "//rs/registry/proto_data_provider", "//rs/types/base_types", + "//rs/types/management_canister_types", "@crate_index//:bitcoin-0.28.2", "@crate_index//:candid", "@crate_index//:ed25519-dalek", @@ -98,6 +99,7 @@ rust_test( "@crate_index//:k256", "@crate_index//:reqwest", "@crate_index//:serde", + "@crate_index//:serde_bytes", "@crate_index//:serde_cbor", "@crate_index//:sha2", "@crate_index//:tempfile", diff --git a/packages/pocket-ic/CHANGELOG.md b/packages/pocket-ic/CHANGELOG.md index e1e9e46bc29f..ab4b9e232139 100644 --- a/packages/pocket-ic/CHANGELOG.md +++ b/packages/pocket-ic/CHANGELOG.md @@ -9,8 +9,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added the `SubnetCoolingDown` variant to the `ErrorCode` enum: ingress messages addressed to a subnet that is "cooling down" are rejected with this error code. +- The function `PocketIc::mock_flexible_canister_http_response` and the type `MockFlexibleCanisterHttpResponse` to mock the responses + of the committee nodes of a pending *flexible* canister HTTP outcall, i.e. one made through the `flexible_http_request` + management canister endpoint. +- The field `replication` of type `CanisterHttpReplication` on `CanisterHttpRequest`, describing how a pending canister HTTP + outcall is replicated across the nodes of its subnet: `FullyReplicated`, `NonReplicated`, or `Flexible` with the outcall's + `total_requests`, `min_responses`, and `max_responses`. The committee size of a flexible outcall (`total_requests`) + is the number of responses `PocketIc::mock_flexible_canister_http_response` accepts. +- The field `pricing_version` of type `CanisterHttpPricingVersion` on `CanisterHttpRequest`, reporting whether a pending + canister HTTP outcall is priced with the `Legacy` or the `PayAsYouGo` pricing model. +- Enabling beta features (`IcpConfig::beta_features` passed to `PocketIcBuilder::with_icp_config`) now also enables the + pay-as-you-go pricing model, which covers both the `flexible_http_request` management canister endpoint (whose outcalls are + always priced that way) and the `pricing_version` field of `http_request` (through which a fully replicated or non-replicated + outcall can select it). ### Changed +- Mocked canister HTTP responses report the cycles their node actually spent on the outcall, instead of reporting no spend at all. + This applies to both `PocketIc::mock_canister_http_response` and `PocketIc::mock_flexible_canister_http_response`. +- Mocking a canister HTTP response whose reject message exceeds 1 KiB now fails. Such a response is not one any node could have + reported. +- A node that cannot pay for gossiping its mocked reject reports an out-of-cycles reject instead of the mocked one, matching what + a node of a real subnet does under pay-as-you-go pricing. - No hard TTL is set on PocketIC servers started implicitly by the library (e.g. by `PocketIc::new` or `PocketIcBuilder::build`); previously a default of 10 minutes was used. The hard TTL is an absolute deadline measured from the server's launch which is not extended by activity, so a test suite whose total runtime exceeded it had its server terminated while still serving requests, failing in-flight calls with `Connection reset by peer`. Orphaned servers remain bounded by the (activity-based) soft TTL. diff --git a/packages/pocket-ic/Cargo.toml b/packages/pocket-ic/Cargo.toml index ef455179503f..db0404f22d5a 100644 --- a/packages/pocket-ic/Cargo.toml +++ b/packages/pocket-ic/Cargo.toml @@ -72,6 +72,7 @@ ic-vetkeys = { workspace = true } icrc-ledger-types = { path = "../icrc-ledger-types" } k256 = { workspace = true } registry-canister = { path = "../../rs/registry/canister" } +serde_bytes = { workspace = true } wat = { workspace = true } [target.'cfg(not(windows))'.dev-dependencies] diff --git a/packages/pocket-ic/HOWTO.md b/packages/pocket-ic/HOWTO.md index 5e48ac98a901..d1fb76797c94 100644 --- a/packages/pocket-ic/HOWTO.md +++ b/packages/pocket-ic/HOWTO.md @@ -378,6 +378,106 @@ e.g., 13 for a regular application subnet. assert_eq!(err, expected); ``` +### Flexible canister HTTP outcalls + +A canister can also make a *flexible* HTTP outcall through the `flexible_http_request` management canister +endpoint. Such an outcall is performed by a committee of `total_requests` nodes of the subnet, whose responses +need not agree, and the calling canister is handed between `min_responses` and `max_responses` of them. + +Flexible outcalls are priced with the pay-as-you-go pricing model, which is still gated. So is the +`pricing_version` field of a regular `http_request`, through which a fully replicated or non-replicated outcall +can select that same pricing model. To use either on a regular application subnet, enable beta features on the +instance: + +```rust + let pic = PocketIcBuilder::new() + .with_application_subnet() + .with_icp_config(IcpConfig { + beta_features: Some(IcpConfigFlag::Enabled), + ..Default::default() + }) + .build(); +``` + +Without beta features, flexible outcalls are only available on subnets where HTTP outcalls are free (system +subnets and subnets created with a free cycles cost schedule), where they fall back to the legacy pricing model, +and an `http_request` asking for pay-as-you-go pricing silently falls back to the legacy pricing model as well. +`CanisterHttpRequest::pricing_version` reports which pricing model a pending outcall actually ended up with. + +Under the pay-as-you-go pricing model, a base fee is charged up front, a per-replica cycles allowance is +withheld from the payment, and whatever the responding nodes do not spend is refunded. If the attached cycles +do not cover the cost of delivering a response, the outcall fails with an out-of-cycles error: a `flexible_http_request` +returns the `out_of_cycles` error described below, while an `http_request` is rejected with `SysTransient` and +a message starting with `Out of cycles:`. This applies to calls made under pay-as-you-go pricing. + +A pending flexible outcall reports its replication in `CanisterHttpRequest::replication`, from which the size +of its committee can be read, and its responses are mocked with `PocketIc::mock_flexible_canister_http_response`: + +```rust + let canister_http_requests = pic.get_canister_http(); + assert_eq!(canister_http_requests.len(), 1); + let canister_http_request = &canister_http_requests[0]; + + let CanisterHttpReplication::Flexible { total_requests, .. } = + canister_http_request.replication + else { + panic!("expected a flexible canister http outcall"); + }; + + let http_reply = |body: &[u8]| { + CanisterHttpResponse::CanisterHttpReply(CanisterHttpReply { + status: 200, + headers: vec![], + body: body.to_vec(), + }) + }; + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: canister_http_request.subnet_id, + request_id: canister_http_request.request_id, + // One response per committee node. The responses may differ. + responses: vec![http_reply(b"hello"); total_requests as usize], + }); + + // The calling canister receives at least `min_responses` (and at most + // `max_responses`) of the mocked responses, smallest first. + let reply = pic.await_call(call_id).unwrap(); +``` + +Unlike `PocketIc::mock_canister_http_response`, which delivers one response per node of the subnet, +this takes *at most* `total_requests` responses. Providing fewer models the remaining committee nodes never +responding: +- with at least `min_responses` successful responses among them, the outcall succeeds; +- with more rejects than the slack between `total_requests` and `min_responses` allows, i.e. with more than + `total_requests - min_responses` of them, the outcall fails with a `too_many_rejects` error reporting the + rejecting nodes; +- if the `min_responses` smallest successful responses do not fit into the 2 MiB a block has for HTTP outcall + responses, the outcall fails with a `responses_too_large` error — note that each individual response may + still be within the 2 MB cap that applies to a single response; +- if the cycles attached to the outcall do not cover the cost of putting its responses into a block, it fails + with an `out_of_cycles` error; +- with too few responses to decide any of the above, the outcall stays pending until, after advancing the time + past its 60 second timeout, it fails with a `timeout` error. + +*Warning.* All responses to an outcall must be provided in a single call: once any response to it has been +mocked, the outcall no longer shows up in `PocketIc::get_canister_http` and further responses to it cannot +be mocked. + +*Note.* A mocked reject message must fit into the 1 KiB a node truncates its reject messages to; mocking a +longer one fails, since no node could have reported it. This applies to both `PocketIc::mock_canister_http_response` +and `PocketIc::mock_flexible_canister_http_response`. + +Note that a flexible outcall never rejects the calling canister's call: every outcome above, including the +errors, is delivered as a `flexible_http_request_result` reply. Only a synchronous failure (invalid arguments, +insufficient cycles attached, or the endpoint not being enabled on the subnet) rejects the call. + +*Warning.* The cycles a node reports having spent on a mocked outcall include a term for how long the +response took to arrive, which PocketIC derives from how long it took to run the mocked outcall in process. +Avoid asserting on exact cycles balances after any outcall priced with the pay-as-you-go pricing model — +whether it is a flexible one or an `http_request` mocked with `PocketIc::mock_canister_http_response`. +Outcalls priced with the legacy pricing model are unaffected, since it ignores the reported spend. + +### Live mode + In the live mode (see the section "Live Mode" for more details), the canister HTTP outcalls are processed by actually making an HTTP request to the URL specified in the canister HTTP outcall. diff --git a/packages/pocket-ic/src/common/rest.rs b/packages/pocket-ic/src/common/rest.rs index ab9d2c12a7ec..f35b9205e3ac 100644 --- a/packages/pocket-ic/src/common/rest.rs +++ b/packages/pocket-ic/src/common/rest.rs @@ -1174,6 +1174,47 @@ pub struct CanisterHttpHeader { pub value: String, } +/// How a canister HTTP outcall is replicated across the nodes of its subnet. +#[derive( + Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, JsonSchema, +)] +pub enum CanisterHttpReplication { + /// Every node of the subnet performs the outcall and a response is delivered + /// once `n - f` of them agree on it. Too many differing responses make the + /// outcall fail with a "no consensus could be reached" rejection instead. + /// Mock such an outcall with `PocketIc::mock_canister_http_response`. + FullyReplicated, + /// A single node performs the outcall (`is_replicated = false`) and its + /// response is the one delivered. + NonReplicated, + /// A committee of `total_requests` nodes performs the outcall and between + /// `min_responses` and `max_responses` of their (potentially differing) + /// responses are delivered to the calling canister. Mock such an outcall + /// with `PocketIc::mock_flexible_canister_http_response`. + Flexible { + /// The number of nodes performing the outcall. + total_requests: u32, + /// The number of responses required to deliver a result. + min_responses: u32, + /// The largest number of responses that may be delivered. + max_responses: u32, + }, +} + +/// The pricing model applied to a canister HTTP outcall. +#[derive( + Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, JsonSchema, +)] +pub enum CanisterHttpPricingVersion { + /// The whole cost of the outcall is charged up front, based on the largest + /// response it could receive. + Legacy, + /// The outcall is charged for the resources it actually consumes: a base fee is + /// charged up front, a per-replica cycles allowance is withheld from the + /// payment, and whatever the responding nodes do not spend is refunded. + PayAsYouGo, +} + #[derive(Clone, Serialize, Deserialize, Debug, JsonSchema)] pub struct RawCanisterHttpRequest { pub subnet_id: RawSubnetId, @@ -1185,6 +1226,8 @@ pub struct RawCanisterHttpRequest { #[serde(serialize_with = "base64::serialize")] pub body: Vec, pub max_response_bytes: Option, + pub replication: CanisterHttpReplication, + pub pricing_version: CanisterHttpPricingVersion, } #[derive(Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] @@ -1198,6 +1241,8 @@ pub struct CanisterHttpRequest { #[serde(serialize_with = "base64::serialize")] pub body: Vec, pub max_response_bytes: Option, + pub replication: CanisterHttpReplication, + pub pricing_version: CanisterHttpPricingVersion, } impl From for CanisterHttpRequest { @@ -1212,6 +1257,8 @@ impl From for CanisterHttpRequest { headers: raw_canister_http_request.headers, body: raw_canister_http_request.body, max_response_bytes: raw_canister_http_request.max_response_bytes, + replication: raw_canister_http_request.replication, + pricing_version: raw_canister_http_request.pricing_version, } } } @@ -1226,6 +1273,8 @@ impl From for RawCanisterHttpRequest { headers: canister_http_request.headers, body: canister_http_request.body, max_response_bytes: canister_http_request.max_response_bytes, + replication: canister_http_request.replication, + pricing_version: canister_http_request.pricing_version, } } } @@ -1246,6 +1295,7 @@ pub struct CanisterHttpReply { )] pub struct CanisterHttpReject { pub reject_code: u64, + /// Bounded by the 1 KiB a node truncates its reject messages to. pub message: String, } @@ -1299,6 +1349,61 @@ impl From for RawMockCanisterHttpResponse { } } +#[derive(Clone, Serialize, Deserialize, Debug, JsonSchema)] +pub struct RawMockFlexibleCanisterHttpResponse { + pub subnet_id: RawSubnetId, + pub request_id: u64, + pub responses: Vec, +} + +/// Mocked responses to a pending *flexible* canister HTTP outcall, i.e. one made +/// through the `flexible_http_request` management canister endpoint. +/// +/// Unlike a fully replicated outcall, which delivers the one response its nodes +/// agree on, a flexible outcall is performed by a committee of `total_requests` +/// nodes whose responses may differ and need not all arrive (see +/// [`CanisterHttpReplication::Flexible`]). +#[derive(Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub struct MockFlexibleCanisterHttpResponse { + pub subnet_id: Principal, + pub request_id: u64, + /// One response per committee node that responded. Each response is attributed + /// to a different committee node. + /// + /// There may be at most `total_requests` responses; providing fewer models a + /// committee whose remaining nodes never respond. With at least `min_responses` + /// successful ones among them the outcall still succeeds; with fewer it stays + /// pending until the time is advanced past its 60 second timeout. + pub responses: Vec, +} + +impl From for MockFlexibleCanisterHttpResponse { + fn from(raw_mock_flexible_canister_http_response: RawMockFlexibleCanisterHttpResponse) -> Self { + Self { + subnet_id: candid::Principal::from_slice( + &raw_mock_flexible_canister_http_response.subnet_id.subnet_id, + ), + request_id: raw_mock_flexible_canister_http_response.request_id, + responses: raw_mock_flexible_canister_http_response.responses, + } + } +} + +impl From for RawMockFlexibleCanisterHttpResponse { + fn from(mock_flexible_canister_http_response: MockFlexibleCanisterHttpResponse) -> Self { + Self { + subnet_id: RawSubnetId { + subnet_id: mock_flexible_canister_http_response + .subnet_id + .as_slice() + .to_vec(), + }, + request_id: mock_flexible_canister_http_response.request_id, + responses: mock_flexible_canister_http_response.responses, + } + } +} + #[derive(Clone, Serialize, Deserialize, Debug, JsonSchema)] pub struct RawCanisterSnapshotDownload { pub sender: RawPrincipalId, diff --git a/packages/pocket-ic/src/lib.rs b/packages/pocket-ic/src/lib.rs index 1b2246c427c9..33c7dd912f3b 100644 --- a/packages/pocket-ic/src/lib.rs +++ b/packages/pocket-ic/src/lib.rs @@ -57,8 +57,9 @@ use crate::{ common::rest::{ AutoProgressConfig, BlobCompression, BlobId, CanisterHttpRequest, ExtendedSubnetConfigSet, HttpsConfig, IcpConfig, IcpFeatures, InitialTime, InstanceHttpGatewayConfig, InstanceId, - MockCanisterHttpResponse, RawEffectivePrincipal, RawMessageId, RawSenderInfo, - RawSubnetBlockmakers, RawTickConfigs, RawTime, SubnetId, SubnetKind, SubnetSpec, Topology, + MockCanisterHttpResponse, MockFlexibleCanisterHttpResponse, RawEffectivePrincipal, + RawMessageId, RawSenderInfo, RawSubnetBlockmakers, RawTickConfigs, RawTime, SubnetId, + SubnetKind, SubnetSpec, Topology, }, nonblocking::PocketIc as PocketIcAsync, }; @@ -1640,7 +1641,9 @@ impl PocketIc { /// Note that, unless a PocketIC instance is in auto progress mode, /// a response to the pending canister HTTP outcalls /// must be produced by the test driver and passed on to the PocketIC instace - /// using `PocketIc::mock_canister_http_response`. + /// using `PocketIc::mock_canister_http_response`, or, for a *flexible* outcall + /// (`CanisterHttpReplication::Flexible`), using + /// `PocketIc::mock_flexible_canister_http_response`. /// In auto progress mode, the PocketIC server produces a response for every /// pending canister HTTP outcall by actually making an HTTP request /// to the specified URL. @@ -1650,7 +1653,11 @@ impl PocketIc { runtime.block_on(async { self.pocket_ic.get_canister_http().await }) } - /// Mock a response to a pending canister HTTP outcall. + /// Mock a response to a pending canister HTTP outcall: the same response for + /// every node of the subnet, or one response per node if + /// `MockCanisterHttpResponse::additional_responses` is non-empty. For a + /// *flexible* outcall, whose committee nodes are answered individually, see + /// `PocketIc::mock_flexible_canister_http_response`. #[instrument(ret, skip(self), fields(instance_id=self.pocket_ic.instance_id))] pub fn mock_canister_http_response( &self, @@ -1664,6 +1671,34 @@ impl PocketIc { }) } + /// Mock the responses of the committee nodes of a pending *flexible* canister + /// HTTP outcall, i.e. one made through the `flexible_http_request` management + /// canister endpoint. + /// + /// This takes at most one response per node of the outcall's committee (whose + /// size is the `total_requests` of the outcall's `CanisterHttpReplication::Flexible` + /// replication). Providing fewer responses than the committee size + /// models the remaining committee nodes never responding: with at least + /// `min_responses` successful ones among them the outcall still succeeds, and + /// with fewer it stays pending until the time is advanced past its 60 second + /// timeout, at which point it fails with a timeout error. + /// + /// All responses to an outcall must be provided in a single call: once any + /// response to it has been mocked, the outcall no longer shows up in + /// `PocketIc::get_canister_http` and further responses to it cannot be mocked. + #[instrument(ret, skip(self), fields(instance_id=self.pocket_ic.instance_id))] + pub fn mock_flexible_canister_http_response( + &self, + mock_flexible_canister_http_response: MockFlexibleCanisterHttpResponse, + ) { + let runtime = self.runtime.clone(); + runtime.block_on(async { + self.pocket_ic + .mock_flexible_canister_http_response(mock_flexible_canister_http_response) + .await + }) + } + /// Download a canister snapshot to a given snapshot directory. /// The sender must be a controller of the canister. /// The snapshot directory must be empty if it exists. diff --git a/packages/pocket-ic/src/nonblocking.rs b/packages/pocket-ic/src/nonblocking.rs index 097f1ecea487..1b5f50af8837 100644 --- a/packages/pocket-ic/src/nonblocking.rs +++ b/packages/pocket-ic/src/nonblocking.rs @@ -3,12 +3,13 @@ use crate::common::rest::{ ApiResponse, AutoProgressConfig, BlobCompression, BlobId, CanisterHttpRequest, CreateHttpGatewayResponse, CreateInstanceResponse, ExtendedSubnetConfigSet, HttpGatewayBackend, HttpGatewayConfig, HttpGatewayInfo, HttpsConfig, IcpConfig, IcpFeatures, InitialTime, - InstanceConfig, InstanceHttpGatewayConfig, InstanceId, MockCanisterHttpResponse, RawAddCycles, - RawCanisterCall, RawCanisterHttpRequest, RawCanisterId, RawCanisterResult, - RawCanisterSnapshotDownload, RawCanisterSnapshotId, RawCanisterSnapshotUpload, RawCycles, - RawEffectivePrincipal, RawIngressStatusArgs, RawMessageId, RawMockCanisterHttpResponse, - RawPrincipalId, RawSenderInfo, RawSetStableMemory, RawStableMemory, RawSubnetId, - RawTickConfigs, RawTime, RawVerifyCanisterSigArg, SubnetId, Topology, + InstanceConfig, InstanceHttpGatewayConfig, InstanceId, MockCanisterHttpResponse, + MockFlexibleCanisterHttpResponse, RawAddCycles, RawCanisterCall, RawCanisterHttpRequest, + RawCanisterId, RawCanisterResult, RawCanisterSnapshotDownload, RawCanisterSnapshotId, + RawCanisterSnapshotUpload, RawCycles, RawEffectivePrincipal, RawIngressStatusArgs, + RawMessageId, RawMockCanisterHttpResponse, RawMockFlexibleCanisterHttpResponse, RawPrincipalId, + RawSenderInfo, RawSetStableMemory, RawStableMemory, RawSubnetId, RawTickConfigs, RawTime, + RawVerifyCanisterSigArg, SubnetId, Topology, }; #[cfg(windows)] use crate::wsl_path; @@ -1866,7 +1867,9 @@ impl PocketIc { /// Note that, unless a PocketIC instance is in auto progress mode, /// a response to the pending canister HTTP outcalls /// must be produced by the test driver and passed on to the PocketIC instace - /// using `PocketIc::mock_canister_http_response`. + /// using `PocketIc::mock_canister_http_response`, or, for a *flexible* outcall + /// (`CanisterHttpReplication::Flexible`), using + /// `PocketIc::mock_flexible_canister_http_response`. /// In auto progress mode, the PocketIC server produces a response for every /// pending canister HTTP outcall by actually making an HTTP request /// to the specified URL. @@ -1877,7 +1880,11 @@ impl PocketIc { res.into_iter().map(|r| r.into()).collect() } - /// Mock a response to a pending canister HTTP outcall. + /// Mock a response to a pending canister HTTP outcall: the same response for + /// every node of the subnet, or one response per node if + /// `MockCanisterHttpResponse::additional_responses` is non-empty. For a + /// *flexible* outcall, whose committee nodes are answered individually, see + /// `PocketIc::mock_flexible_canister_http_response`. #[instrument(ret, skip(self), fields(instance_id=self.instance_id))] pub async fn mock_canister_http_response( &self, @@ -1889,6 +1896,33 @@ impl PocketIc { self.post(endpoint, raw_mock_canister_http_response).await } + /// Mock the responses of the committee nodes of a pending *flexible* canister + /// HTTP outcall, i.e. one made through the `flexible_http_request` management + /// canister endpoint. + /// + /// This takes at most one response per node of the outcall's committee (whose + /// size is the `total_requests` of the outcall's `CanisterHttpReplication::Flexible` + /// replication). Providing fewer responses than the committee size + /// models the remaining committee nodes never responding: with at least + /// `min_responses` successful ones among them the outcall still succeeds, and + /// with fewer it stays pending until the time is advanced past its 60 second + /// timeout, at which point it fails with a timeout error. + /// + /// All responses to an outcall must be provided in a single call: once any + /// response to it has been mocked, the outcall no longer shows up in + /// `PocketIc::get_canister_http` and further responses to it cannot be mocked. + #[instrument(ret, skip(self), fields(instance_id=self.instance_id))] + pub async fn mock_flexible_canister_http_response( + &self, + mock_flexible_canister_http_response: MockFlexibleCanisterHttpResponse, + ) { + let endpoint = "update/mock_flexible_canister_http"; + let raw_mock_flexible_canister_http_response: RawMockFlexibleCanisterHttpResponse = + mock_flexible_canister_http_response.into(); + self.post(endpoint, raw_mock_flexible_canister_http_response) + .await + } + /// Download a canister snapshot to a given snapshot directory. /// The sender must be a controller of the canister. /// The snapshot directory must be empty if it exists. diff --git a/packages/pocket-ic/test_canister/canister.did b/packages/pocket-ic/test_canister/canister.did index 3ca860df1fe2..016411f930cc 100644 --- a/packages/pocket-ic/test_canister/canister.did +++ b/packages/pocket-ic/test_canister/canister.did @@ -92,6 +92,10 @@ type HttpResponseResult = variant { Ok : HttpResponse; Err : record {RejectionCode; text}; }; +type RawHttpResponseResult = variant { + Ok : blob; + Err : record {RejectionCode; text}; +}; type TransformArgs = record { response : HttpResponse; @@ -123,6 +127,8 @@ service : { node_metrics_history_proxy: (NodeMetricsHistoryArgs) -> (vec NodeMetricsHistoryResponse); canister_http : (text) -> (HttpResponseResult); canister_http_with_transform : (text) -> (HttpResponse); + canister_http_raw : (blob, nat) -> (RawHttpResponseResult); + flexible_canister_http : (blob, nat) -> (RawHttpResponseResult); transform : (TransformArgs) -> (HttpResponse) query; whoami : () -> (text); whois : (principal) -> (text); diff --git a/packages/pocket-ic/test_canister/src/canister.rs b/packages/pocket-ic/test_canister/src/canister.rs index abd4ecffd2e3..6ef3d4a2bef2 100644 --- a/packages/pocket-ic/test_canister/src/canister.rs +++ b/packages/pocket-ic/test_canister/src/canister.rs @@ -50,8 +50,8 @@ impl RejectionCode { /// Translates a failed call into the reject code and message `canister_http` /// reports back over Candid. -fn map_call_error(err: CallError) -> (RejectionCode, String) { - match err { +fn map_call_error(err: impl Into) -> (RejectionCode, String) { + match err.into() { CallError::CallRejected(rejected) => ( RejectionCode::from_raw(rejected.raw_reject_code()), rejected.reject_message().to_string(), @@ -429,6 +429,44 @@ async fn canister_http_with_transform(http_server_addr: String) -> HttpRequestRe canister_http_outcall(&arg).await.unwrap() } +/// Makes an HTTP outcall, passing `args` (a Candid-encoded `http_request_args`) to +/// the management canister verbatim and attaching `cycles` cycles to the call. +/// +/// Unlike `canister_http`, this lets the caller set fields that +/// `ic-cdk-management-canister` does not expose, such as `pricing_version`. +#[update] +async fn canister_http_raw( + args: ByteBuf, + cycles: u128, +) -> Result { + Call::unbounded_wait(Principal::management_canister(), "http_request") + .with_raw_args(&args) + .with_cycles(cycles) + .await + .map(|response| ByteBuf::from(response.into_bytes())) + .map_err(map_call_error) +} + +/// Makes a flexible HTTP outcall, passing `args` (a Candid-encoded +/// `flexible_http_request_args`) to the management canister verbatim and +/// attaching `cycles` cycles to the call. +/// +/// Both the argument and the result are passed through undecoded so that callers +/// can use the authoritative Candid types instead of copies maintained here; +/// `ic-cdk-management-canister` does not expose flexible HTTP outcalls (yet). +#[update] +async fn flexible_canister_http( + args: ByteBuf, + cycles: u128, +) -> Result { + Call::unbounded_wait(Principal::management_canister(), "flexible_http_request") + .with_raw_args(&args) + .with_cycles(cycles) + .await + .map(|response| ByteBuf::from(response.into_bytes())) + .map_err(map_call_error) +} + // inter-canister calls #[update] diff --git a/packages/pocket-ic/tests/tests.rs b/packages/pocket-ic/tests/tests.rs index 92505906df13..292603f614b3 100644 --- a/packages/pocket-ic/tests/tests.rs +++ b/packages/pocket-ic/tests/tests.rs @@ -8,6 +8,11 @@ use ic_management_canister_types::{ HttpRequestResult, ProvisionalCreateCanisterWithCyclesArgs, SchnorrAlgorithm, SchnorrAux, SchnorrKeyId as SchnorrPublicKeyArgsKeyId, SchnorrPublicKeyResult, }; +use ic_management_canister_types_private::{ + BoundedHttpHeaders, CanisterHttpRequestArgs, CanisterHttpResponsePayload, + FlexibleCanisterHttpRequestArgs, FlexibleHttpGlobalError, FlexibleHttpRequestResult, + HttpMethod, PRICING_VERSION_PAY_AS_YOU_GO, ReplicationCounts, TransformContext, TransformFunc, +}; use ic_transport_types::EnvelopeContent::{Call, ReadState}; use ic_transport_types::{CallResponse, Envelope}; use ic_utils::interfaces::ManagementCanister; @@ -17,10 +22,12 @@ use pocket_ic::{ IngressStatusResult, PocketIc, PocketIcBuilder, PocketIcState, RejectCode, StartServerParams, Time, common::rest::{ - AutoProgressConfig, BlobCompression, CanisterCyclesCostSchedule, CanisterHttpReply, - CanisterHttpResponse, CanisterIdRange, CreateInstanceResponse, ExtendedSubnetConfigSet, - HttpGatewayDetails, HttpsConfig, IcpFeatures, IcpFeaturesConfig, InitialTime, - InstanceConfig, InstanceHttpGatewayConfig, MockCanisterHttpResponse, RawEffectivePrincipal, + AutoProgressConfig, BlobCompression, CanisterCyclesCostSchedule, + CanisterHttpPricingVersion, CanisterHttpReject, CanisterHttpReplication, CanisterHttpReply, + CanisterHttpRequest, CanisterHttpResponse, CanisterIdRange, CreateInstanceResponse, + ExtendedSubnetConfigSet, HttpGatewayDetails, HttpsConfig, IcpConfig, IcpConfigFlag, + IcpFeatures, IcpFeaturesConfig, InitialTime, InstanceConfig, InstanceHttpGatewayConfig, + MockCanisterHttpResponse, MockFlexibleCanisterHttpResponse, RawEffectivePrincipal, RawMessageId, SubnetConfigSet, SubnetKind, SubnetSpec, }, nonblocking::PocketIc as PocketIcAsync, @@ -30,6 +37,7 @@ use reqwest::blocking::Response; use reqwest::header::CONTENT_LENGTH; use reqwest::{Method, StatusCode, Url}; use serde::Serialize; +use serde_bytes::ByteBuf; use sha2::{Digest, Sha256}; use std::{ io::Read, @@ -1717,6 +1725,919 @@ fn test_canister_http_timeout() { assert_eq!(err, "Canister http request timed out"); } +// --------------------------------------------------------------------------- +// Flexible canister HTTP outcalls +// --------------------------------------------------------------------------- + +/// The cycles attached to a flexible outcall in these tests. Every cycle beyond +/// the outcall's base fee and per-replica allowances is refunded immediately, so +/// this only has to be comfortably above what the outcall can cost. +const FLEXIBLE_OUTCALL_CYCLES: u128 = 1_000_000_000_000; + +/// The arguments of a flexible HTTP outcall to `example.com` with the given +/// replication. +fn flexible_args(replication: Option) -> FlexibleCanisterHttpRequestArgs { + FlexibleCanisterHttpRequestArgs { + url: "example.com".to_string(), + max_response_bytes: None, + headers: BoundedHttpHeaders::new(vec![]), + body: None, + method: HttpMethod::GET, + transform: None, + replication, + } +} + +/// A PocketIC instance with a single application subnet on which flexible HTTP +/// outcalls (and thus pay-as-you-go pricing) are enabled. +fn flexible_outcalls_pic() -> PocketIc { + PocketIcBuilder::new() + .with_application_subnet() + .with_icp_config(IcpConfig { + beta_features: Some(IcpConfigFlag::Enabled), + ..Default::default() + }) + .build() +} + +fn deploy_test_canister(pic: &PocketIc) -> Principal { + let canister_id = pic.create_canister(); + pic.add_cycles(canister_id, INIT_CYCLES); + pic.install_canister(canister_id, test_canister_wasm(), vec![], None); + canister_id +} + +/// The argument of the test canister's `flexible_canister_http` method: the +/// Candid-encoded outcall arguments and the cycles to attach. +fn flexible_call_arg_with_cycles(args: FlexibleCanisterHttpRequestArgs, cycles: u128) -> Vec { + Encode!(&ByteBuf::from(Encode!(&args).unwrap()), &cycles).unwrap() +} + +fn flexible_call_arg(args: FlexibleCanisterHttpRequestArgs) -> Vec { + flexible_call_arg_with_cycles(args, FLEXIBLE_OUTCALL_CYCLES) +} + +/// Submits a flexible HTTP outcall through the test canister, returning the +/// message ID of the pending update call and the pending outcall. +fn submit_flexible_outcall( + pic: &PocketIc, + canister_id: Principal, + args: FlexibleCanisterHttpRequestArgs, +) -> (RawMessageId, CanisterHttpRequest) { + submit_flexible_outcall_with_cycles(pic, canister_id, args, FLEXIBLE_OUTCALL_CYCLES) +} + +fn submit_flexible_outcall_with_cycles( + pic: &PocketIc, + canister_id: Principal, + args: FlexibleCanisterHttpRequestArgs, + cycles: u128, +) -> (RawMessageId, CanisterHttpRequest) { + let call_id = pic + .submit_call( + canister_id, + Principal::anonymous(), + "flexible_canister_http", + flexible_call_arg_with_cycles(args, cycles), + ) + .unwrap(); + + // We need a pair of ticks for the test canister method to make the http outcall + // and for the management canister to start processing the http outcall. + pic.tick(); + pic.tick(); + let mut canister_http_requests = pic.get_canister_http(); + assert_eq!(canister_http_requests.len(), 1); + (call_id, canister_http_requests.pop().unwrap()) +} + +fn reply(body: &[u8]) -> CanisterHttpResponse { + CanisterHttpResponse::CanisterHttpReply(CanisterHttpReply { + status: 200, + headers: vec![], + body: body.to_vec(), + }) +} + +fn reject(message: &str) -> CanisterHttpResponse { + CanisterHttpResponse::CanisterHttpReject(CanisterHttpReject { + // `SysTransient`, i.e. what a connection failure produces. + reject_code: 2, + message: message.to_string(), + }) +} + +/// Decodes the reply of the test canister's `flexible_canister_http` method into +/// the `flexible_http_request_result` the calling canister observed. +fn decode_flexible_result(reply: &[u8]) -> FlexibleHttpRequestResult { + let result: Result = decode_one(reply).unwrap(); + let bytes = result.expect("the flexible outcall was rejected synchronously"); + Decode!(&bytes, FlexibleHttpRequestResult).unwrap() +} + +/// Submits a non-flexible HTTP outcall through the test canister that asks for the +/// given `pricing_version`, returning the message ID of the pending update call and +/// the pending outcall. +fn submit_pay_as_you_go_outcall( + pic: &PocketIc, + canister_id: Principal, + pricing_version: u32, + cycles: u128, +) -> (RawMessageId, CanisterHttpRequest) { + submit_raw_outcall(pic, canister_id, None, Some(pricing_version), cycles) +} + +fn submit_raw_outcall( + pic: &PocketIc, + canister_id: Principal, + is_replicated: Option, + pricing_version: Option, + cycles: u128, +) -> (RawMessageId, CanisterHttpRequest) { + let args = CanisterHttpRequestArgs { + url: "example.com".to_string(), + max_response_bytes: None, + headers: BoundedHttpHeaders::new(vec![]), + body: None, + method: HttpMethod::GET, + transform: None, + is_replicated, + pricing_version, + }; + let call_id = pic + .submit_call( + canister_id, + Principal::anonymous(), + "canister_http_raw", + Encode!(&ByteBuf::from(Encode!(&args).unwrap()), &cycles).unwrap(), + ) + .unwrap(); + + // We need a pair of ticks for the test canister method to make the http outcall + // and for the management canister to start processing the http outcall. + pic.tick(); + pic.tick(); + let mut canister_http_requests = pic.get_canister_http(); + assert_eq!(canister_http_requests.len(), 1); + (call_id, canister_http_requests.pop().unwrap()) +} + +/// Decodes the reply of the test canister's `canister_http_raw` method into the +/// outcome the calling canister observed. +fn decode_raw_http_result( + reply: &[u8], +) -> Result { + let result: Result = decode_one(reply).unwrap(); + result.map(|bytes| Decode!(&bytes, CanisterHttpResponsePayload).unwrap()) +} + +/// Decodes the reply of the test canister's `flexible_canister_http` method into +/// the synchronous rejection the calling canister observed. +fn decode_flexible_rejection(reply: &[u8]) -> (RejectionCode, String) { + let result: Result = decode_one(reply).unwrap(); + result.expect_err("the flexible outcall was not rejected") +} + +/// With the default replication every node of the subnet performs the outcall and +/// `floor(2N/3) + 1` responses suffice, so mocking all of them delivers between +/// `min_responses` and `max_responses` payloads. +#[test] +fn test_flexible_canister_http() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let (call_id, request) = submit_flexible_outcall(&pic, canister_id, flexible_args(None)); + + // The subnet has 13 nodes, so the default replication is + // `total_requests = max_responses = 13`, `min_responses = floor(2*13/3) + 1 = 9`. + let CanisterHttpReplication::Flexible { + total_requests, + min_responses, + max_responses, + } = request.replication + else { + panic!("expected a flexible outcall, got {:?}", request.replication); + }; + assert_eq!(total_requests, 13); + assert_eq!(min_responses, 9); + assert_eq!(max_responses, 13); + + let body = b"hello".to_vec(); + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(&body); total_requests as usize], + }); + + // There should be no more pending canister http outcalls. + assert!(pic.get_canister_http().is_empty()); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Ok(payloads) = decode_flexible_result(&reply) else { + panic!("expected a successful flexible outcall"); + }; + assert!(payloads.len() >= min_responses as usize); + assert!(payloads.len() <= max_responses as usize); + for payload in &payloads { + assert_eq!(payload.status, 200); + assert_eq!(payload.body, body); + } +} + +/// A flexible outcall is answered as soon as `min_responses` of its committee have +/// responded, and the responses need not agree. +#[test] +fn test_flexible_canister_http_partial_diverging_responses() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 4, + min_responses: 2, + max_responses: 4, + }; + let (call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + assert_eq!( + request.replication, + CanisterHttpReplication::Flexible { + total_requests: 4, + min_responses: 2, + max_responses: 4, + } + ); + + // Only two of the four committee nodes respond, with differing bodies. + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"aa"), reply(b"bb")], + }); + + assert!(pic.get_canister_http().is_empty()); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Ok(payloads) = decode_flexible_result(&reply) else { + panic!("expected a successful flexible outcall"); + }; + // The payloads are delivered smallest first; both have the same size here, so + // only the set of bodies is determined. + let mut bodies: Vec<_> = payloads.iter().map(|p| p.body.clone()).collect(); + bodies.sort(); + assert_eq!(bodies, vec![b"aa".to_vec(), b"bb".to_vec()]); +} + +/// Once more nodes reject than the slack between `total_requests` and +/// `min_responses` allows, the outcall fails with `TooManyRejects` and reports the +/// rejecting nodes. +#[test] +fn test_flexible_canister_http_too_many_rejects() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 4, + min_responses: 3, + max_responses: 4, + }; + let (call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + + // The slack is `4 - 3 = 1`, so two rejects are one too many. + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![ + reject("Connection refused"), + reject("Connection refused"), + reply(b"hello"), + reply(b"hello"), + ], + }); + + assert!(pic.get_canister_http().is_empty()); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Err(err) = decode_flexible_result(&reply) else { + panic!("expected the flexible outcall to fail"); + }; + assert_eq!( + err.global_error, + Some(FlexibleHttpGlobalError::TooManyRejects(candid::Reserved)) + ); + assert_eq!(err.node_details.len(), 2); + for detail in &err.node_details { + let node_error = detail.error.as_ref().expect("expected a per-node error"); + assert_eq!(node_error.code, "SysTransient"); + assert_eq!(node_error.message, "Connection refused"); + } +} + +/// Fewer than `min_responses` mocked responses leave the outcall pending until it +/// times out. +#[test] +fn test_flexible_canister_http_timeout() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 4, + min_responses: 3, + max_responses: 4, + }; + let (call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + + // Only one of the three required responses arrives. + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"hello")], + }); + pic.tick(); + + // Advance time so that the canister http outcall times out. + pic.advance_time(std::time::Duration::from_secs(180)); + pic.tick(); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Err(err) = decode_flexible_result(&reply) else { + panic!("expected the flexible outcall to time out"); + }; + assert_eq!( + err.global_error, + Some(FlexibleHttpGlobalError::Timeout(candid::Reserved)) + ); +} + +/// The calling canister's transform function is applied to every mocked response. +#[test] +fn test_flexible_canister_http_with_transform() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 2, + min_responses: 2, + max_responses: 2, + }; + let context = b"this is my transform context".to_vec(); + let mut args = flexible_args(Some(replication)); + args.transform = Some(TransformContext { + function: TransformFunc(candid::Func { + method: "transform".to_string(), + principal: canister_id, + }), + context: context.clone(), + }); + let (call_id, request) = submit_flexible_outcall(&pic, canister_id, args); + + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"hello"), reply(b"hello")], + }); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Ok(payloads) = decode_flexible_result(&reply) else { + panic!("expected a successful flexible outcall"); + }; + assert_eq!(payloads.len(), 2); + for payload in &payloads { + // The transform function clears the response headers and replaces the body + // with its transform context. + assert!(payload.headers.is_empty()); + assert_eq!(payload.body, context); + } +} + +/// A flexible outcall withholds a per-replica cycles allowance from its payment +/// and refunds whatever the responding nodes did not spend. +#[test] +fn test_flexible_canister_http_cycles_refund() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let balance_before = pic.cycle_balance(canister_id); + let replication = ReplicationCounts { + total_requests: 4, + min_responses: 4, + max_responses: 4, + }; + let (call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + // While the outcall is in flight, its base fee and the per-replica allowances + // are withheld from the canister. + let balance_in_flight = pic.cycle_balance(canister_id); + assert!(balance_in_flight < balance_before); + + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"hello"); 4], + }); + + let reply = pic.await_call(call_id).unwrap(); + assert!(matches!( + decode_flexible_result(&reply), + FlexibleHttpRequestResult::Ok(_) + )); + + // The unspent part of the allowances is refunded, so the canister ends up with + // more than it had while the outcall was in flight, but with less than it + // started with: the base fee, the nodes' reported spend and the cost of + // putting the responses into a block are not refunded. + let balance_after = pic.cycle_balance(canister_id); + assert!( + balance_after > balance_in_flight, + "expected a refund: {balance_in_flight} -> {balance_after}" + ); + assert!( + balance_after < balance_before, + "expected the outcall to cost something: {balance_after} >= {balance_before}" + ); +} + +/// Once the responses that would have to be delivered no longer fit into a block, +/// the outcall fails with `ResponsesTooLarge`. +#[test] +fn test_flexible_canister_http_responses_too_large() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 2, + min_responses: 2, + max_responses: 2, + }; + let (call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + + // Both responses are well below the 2 MB cap on a single response + // (`MAX_CANISTER_HTTP_RESPONSE_BYTES`), but the two of them together exceed the + // 2 MiB a block has for HTTP outcall responses + // (`MAX_CANISTER_HTTP_PAYLOAD_SIZE`), and both of them have to be delivered + // (`min_responses == 2`). + let body = vec![b'x'; 1_100_000]; + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(&body); 2], + }); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Err(err) = decode_flexible_result(&reply) else { + panic!("expected the flexible outcall to fail"); + }; + assert_eq!( + err.global_error, + Some(FlexibleHttpGlobalError::ResponsesTooLarge(candid::Reserved)) + ); +} + +/// If the cycles attached to the outcall do not cover the cost of delivering its +/// responses, the outcall fails with `OutOfCycles`. +#[test] +fn test_flexible_canister_http_out_of_cycles() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + // Enough to cover the outcall's base fee (~50M cycles on a 13-node subnet with + // this replication) and to let every node perform the outcall, but nowhere near + // enough to also cover the cost of putting 13 responses into a block. Revisit + // if the fees in `rs/https_outcalls/pricing/src/fees.rs` change. + const CYCLES: u128 = 100_000_000; + let replication = ReplicationCounts { + total_requests: 13, + min_responses: 13, + max_responses: 13, + }; + let (call_id, request) = submit_flexible_outcall_with_cycles( + &pic, + canister_id, + flexible_args(Some(replication)), + CYCLES, + ); + + let body = vec![b'x'; 1_000]; + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(&body); 13], + }); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Err(err) = decode_flexible_result(&reply) else { + panic!("expected the flexible outcall to fail"); + }; + assert_eq!( + err.global_error, + Some(FlexibleHttpGlobalError::OutOfCycles(candid::Reserved)) + ); +} + +/// A non-flexible outcall can select the pay-as-you-go pricing model through its +/// `pricing_version`, and is then refunded whatever the responding nodes did not +/// spend. +#[test] +fn test_canister_http_pay_as_you_go() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let balance_before = pic.cycle_balance(canister_id); + let (call_id, request) = submit_pay_as_you_go_outcall( + &pic, + canister_id, + PRICING_VERSION_PAY_AS_YOU_GO, + FLEXIBLE_OUTCALL_CYCLES, + ); + assert_eq!( + request.replication, + CanisterHttpReplication::FullyReplicated + ); + assert_eq!( + request.pricing_version, + CanisterHttpPricingVersion::PayAsYouGo + ); + let balance_in_flight = pic.cycle_balance(canister_id); + assert!(balance_in_flight < balance_before); + + // A fully replicated outcall is mocked as usual: one response per subnet node. + let body = b"hello".to_vec(); + pic.mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + response: reply(&body), + additional_responses: vec![], + }); + + let reply = pic.await_call(call_id).unwrap(); + let payload = decode_raw_http_result(&reply).expect("the outcall was rejected"); + assert_eq!(payload.body, body); + + // The unspent part of the per-replica allowances is refunded. + let balance_after = pic.cycle_balance(canister_id); + assert!( + balance_after > balance_in_flight, + "expected a refund: {balance_in_flight} -> {balance_after}" + ); + assert!( + balance_after < balance_before, + "expected the outcall to cost something: {balance_after} >= {balance_before}" + ); +} + +/// A non-replicated outcall (`is_replicated = false`) can select the pay-as-you-go +/// pricing model too, and is answered by the single node it was delegated to. +#[test] +fn test_canister_http_non_replicated_pay_as_you_go() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let (call_id, request) = submit_raw_outcall( + &pic, + canister_id, + Some(false), + Some(PRICING_VERSION_PAY_AS_YOU_GO), + FLEXIBLE_OUTCALL_CYCLES, + ); + assert_eq!(request.replication, CanisterHttpReplication::NonReplicated); + assert_eq!( + request.pricing_version, + CanisterHttpPricingVersion::PayAsYouGo + ); + + // Only the delegated node's response is delivered, but which node that is is + // not exposed, so the same response is mocked for every node of the subnet. + let body = b"hello".to_vec(); + pic.mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + response: reply(&body), + additional_responses: vec![], + }); + + let reply = pic.await_call(call_id).unwrap(); + let payload = decode_raw_http_result(&reply).expect("the outcall was rejected"); + assert_eq!(payload.body, body); +} + +/// Without the pay-as-you-go pricing model enabled, a non-flexible outcall asking +/// for it silently falls back to the legacy pricing model. +#[test] +fn test_canister_http_pay_as_you_go_disabled() { + let pic = PocketIc::new(); + let canister_id = deploy_test_canister(&pic); + + let (_call_id, request) = submit_pay_as_you_go_outcall( + &pic, + canister_id, + PRICING_VERSION_PAY_AS_YOU_GO, + FLEXIBLE_OUTCALL_CYCLES, + ); + assert_eq!(request.pricing_version, CanisterHttpPricingVersion::Legacy); +} + +/// A pay-as-you-go outcall whose attached cycles do not cover the cost of +/// delivering a response is rejected once the nodes have reported their spend. +#[test] +fn test_canister_http_pay_as_you_go_out_of_cycles() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + // Enough to cover the outcall's base fee (~38M cycles on a 13-node subnet) and + // to let every node perform the outcall, but nowhere near enough to also cover + // the cost of putting a 10 KB response into a block. Revisit if the fees in + // `rs/https_outcalls/pricing/src/fees.rs` change. + const CYCLES: u128 = 80_000_000; + let (call_id, request) = + submit_pay_as_you_go_outcall(&pic, canister_id, PRICING_VERSION_PAY_AS_YOU_GO, CYCLES); + assert_eq!( + request.pricing_version, + CanisterHttpPricingVersion::PayAsYouGo + ); + + let body = vec![b'x'; 10_000]; + pic.mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + response: reply(&body), + additional_responses: vec![], + }); + + let reply = pic.await_call(call_id).unwrap(); + let (reject_code, message) = + decode_raw_http_result(&reply).expect_err("expected the outcall to be rejected"); + assert!( + matches!(reject_code, RejectionCode::SysTransient), + "unexpected reject code {reject_code:?} (message: {message})" + ); + assert!( + message.contains("Out of cycles:"), + "unexpected rejection message {message:?}" + ); +} + +/// Invalid replication counts are rejected synchronously. +#[test] +fn test_flexible_canister_http_invalid_replication_counts() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + for (replication, expected) in [ + ( + ReplicationCounts { + total_requests: 0, + min_responses: 0, + max_responses: 0, + }, + "total_requests (0) must be at least 1", + ), + ( + ReplicationCounts { + total_requests: 14, + min_responses: 1, + max_responses: 1, + }, + "total_requests (14) must not exceed the number of available nodes (13)", + ), + ( + ReplicationCounts { + total_requests: 4, + min_responses: 3, + max_responses: 2, + }, + "min_responses (3) must not exceed max_responses (2)", + ), + ( + ReplicationCounts { + total_requests: 2, + min_responses: 1, + max_responses: 3, + }, + "max_responses (3) must not exceed total_requests (2)", + ), + ] { + let reply = pic + .update_call( + canister_id, + Principal::anonymous(), + "flexible_canister_http", + flexible_call_arg(flexible_args(Some(replication))), + ) + .unwrap(); + let (reject_code, message) = decode_flexible_rejection(&reply); + assert!( + matches!(reject_code, RejectionCode::CanisterReject), + "unexpected reject code {reject_code:?} (message: {message})" + ); + assert!( + message.contains(expected), + "rejection message {message:?} does not contain {expected:?}" + ); + } +} + +/// Without the beta features enabled, flexible outcalls are still available on a +/// subnet where HTTP outcalls are free, falling back to the legacy pricing model. +#[test] +fn test_flexible_canister_http_on_system_subnet() { + let pic = PocketIcBuilder::new().with_system_subnet().build(); + let system_subnet = pic.topology().get_system_subnets()[0]; + let canister_id = pic.create_canister_on_subnet(None, None, system_subnet); + pic.add_cycles(canister_id, INIT_CYCLES); + pic.install_canister(canister_id, test_canister_wasm(), vec![], None); + + let replication = ReplicationCounts { + total_requests: 3, + min_responses: 2, + max_responses: 3, + }; + let (call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"hello"); 3], + }); + + let reply = pic.await_call(call_id).unwrap(); + let FlexibleHttpRequestResult::Ok(payloads) = decode_flexible_result(&reply) else { + panic!("expected a successful flexible outcall"); + }; + assert_eq!(payloads.len(), 3); + for payload in &payloads { + assert_eq!(payload.body, b"hello".to_vec()); + } +} + +/// Without the beta features enabled, flexible outcalls are unavailable on a +/// subnet that charges for HTTP outcalls. +#[test] +fn test_flexible_canister_http_disabled() { + let pic = PocketIc::new(); + let canister_id = deploy_test_canister(&pic); + + let reply = pic + .update_call( + canister_id, + Principal::anonymous(), + "flexible_canister_http", + flexible_call_arg(flexible_args(None)), + ) + .unwrap(); + let (reject_code, message) = decode_flexible_rejection(&reply); + assert!( + matches!(reject_code, RejectionCode::CanisterError), + "unexpected reject code {reject_code:?} (message: {message})" + ); + assert!( + message.contains("This API is not enabled on this subnet"), + "unexpected rejection message {message:?}" + ); +} + +/// Mocking more responses than the outcall's committee has nodes is an error. +#[test] +#[should_panic(expected = "TooManyMockCanisterHttpResponses((3, 2))")] +fn test_flexible_canister_http_too_many_responses() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 2, + min_responses: 1, + max_responses: 2, + }; + let (_call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"hello"); 3], + }); +} + +/// A fully replicated outcall cannot be mocked through the flexible endpoint. +#[test] +#[should_panic(expected = "NotAFlexibleCanisterHttpRequest")] +fn test_flexible_mock_of_fully_replicated_outcall() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + pic.submit_call( + canister_id, + Principal::anonymous(), + "canister_http", + encode_one("example.com").unwrap(), + ) + .unwrap(); + pic.tick(); + pic.tick(); + let canister_http_requests = pic.get_canister_http(); + assert_eq!(canister_http_requests.len(), 1); + let request = &canister_http_requests[0]; + assert_eq!( + request.replication, + CanisterHttpReplication::FullyReplicated + ); + + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reply(b"hello")], + }); +} + +/// A mocked reject is priced too: a rejecting node reports having downloaded +/// nothing, and a fully replicated outcall does not gossip its responses either, +/// so nothing beyond the base fee and the consensus cost of putting the reject +/// into a block is charged. +#[test] +fn test_canister_http_pay_as_you_go_reject() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let balance_before = pic.cycle_balance(canister_id); + let (call_id, request) = submit_pay_as_you_go_outcall( + &pic, + canister_id, + PRICING_VERSION_PAY_AS_YOU_GO, + FLEXIBLE_OUTCALL_CYCLES, + ); + let balance_in_flight = pic.cycle_balance(canister_id); + assert!(balance_in_flight < balance_before); + + pic.mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + response: reject("Connection refused"), + additional_responses: vec![], + }); + + let reply = pic.await_call(call_id).unwrap(); + let (reject_code, message) = + decode_raw_http_result(&reply).expect_err("expected the outcall to be rejected"); + assert!( + matches!(reject_code, RejectionCode::SysTransient), + "unexpected reject code {reject_code:?} (message: {message})" + ); + assert_eq!(message, "Connection refused"); + + let balance_after = pic.cycle_balance(canister_id); + assert!( + balance_after > balance_in_flight, + "expected a refund: {balance_in_flight} -> {balance_after}" + ); + assert!( + balance_after < balance_before, + "expected the outcall to cost something: {balance_after} >= {balance_before}" + ); +} + +/// A reject message longer than the 1 KiB a node truncates its reject messages to +/// is not one any node could have reported, so mocking it is an error. +#[test] +#[should_panic(expected = "CanisterHttpRejectMessageTooLong((1025, 1024))")] +fn test_canister_http_reject_message_too_long() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let (_call_id, request) = + submit_raw_outcall(&pic, canister_id, None, None, FLEXIBLE_OUTCALL_CYCLES); + + pic.mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + response: reject(&"x".repeat(1025)), + additional_responses: vec![], + }); +} + +/// The same holds for the flexible mock. +#[test] +#[should_panic(expected = "CanisterHttpRejectMessageTooLong((1025, 1024))")] +fn test_flexible_canister_http_reject_message_too_long() { + let pic = flexible_outcalls_pic(); + let canister_id = deploy_test_canister(&pic); + + let replication = ReplicationCounts { + total_requests: 2, + min_responses: 1, + max_responses: 2, + }; + let (_call_id, request) = + submit_flexible_outcall(&pic, canister_id, flexible_args(Some(replication))); + + pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse { + subnet_id: request.subnet_id, + request_id: request.request_id, + responses: vec![reject(&"x".repeat(1025))], + }); +} + #[test] fn subnet_metrics() { let pic = PocketIcBuilder::new().with_application_subnet().build(); diff --git a/rs/execution_environment/src/execution_environment.rs b/rs/execution_environment/src/execution_environment.rs index d0f38945b84c..57ac545892ff 100644 --- a/rs/execution_environment/src/execution_environment.rs +++ b/rs/execution_environment/src/execution_environment.rs @@ -1351,6 +1351,10 @@ impl ExecutionEnvironment { | SubnetType::VerifiedApplication | SubnetType::CloudEngine => state.get_own_cost_schedule(), }; + // The pay-as-you-go pricing model is gated behind the same + // feature flag as flexible outcalls + let pay_as_you_go_enabled = + self.config.flexible_http_requests == FlagStatus::Enabled; match CanisterHttpRequestContext::generate_from_args( state.time(), request.as_ref(), @@ -1359,6 +1363,7 @@ impl ExecutionEnvironment { registry_settings.registry_version, cost_schedule, rng, + pay_as_you_go_enabled, ) { Err(err) => ExecuteSubnetMessageResult::Finished { response: Err(err.into()), diff --git a/rs/execution_environment/src/execution_environment/tests.rs b/rs/execution_environment/src/execution_environment/tests.rs index e5d98d0d2137..09c580b4e8f8 100644 --- a/rs/execution_environment/src/execution_environment/tests.rs +++ b/rs/execution_environment/src/execution_environment/tests.rs @@ -10,10 +10,10 @@ use ic_management_canister_types_private::{ CanisterIdRecord, CanisterMetadataRequest, CanisterMetadataResponse, CanisterStatusResultV2, CanisterStatusType, CreateCanisterArgs, DerivationPath, EcdsaCurve, EcdsaKeyId, EmptyBlob, FetchCanisterLogsRequest, FlexibleCanisterHttpRequestArgs, HttpMethod, IC_00, LogVisibilityV2, - MasterPublicKeyId, Method, Payload as Ic00Payload, ProvisionalCreateCanisterWithCyclesArgs, - ProvisionalTopUpCanisterArgs, ReplicationCounts, SchnorrAlgorithm, SchnorrKeyId, - TakeCanisterSnapshotArgs, TransformContext, TransformFunc, UploadChunkArgs, VetKdCurve, - VetKdKeyId, + MasterPublicKeyId, Method, PRICING_VERSION_PAY_AS_YOU_GO, Payload as Ic00Payload, + ProvisionalCreateCanisterWithCyclesArgs, ProvisionalTopUpCanisterArgs, ReplicationCounts, + SchnorrAlgorithm, SchnorrKeyId, TakeCanisterSnapshotArgs, TransformContext, TransformFunc, + UploadChunkArgs, VetKdCurve, VetKdKeyId, }; use ic_registry_routing_table::{CanisterIdRange, RoutingTable, canister_id_into_u64}; use ic_registry_subnet_type::SubnetType; @@ -3832,6 +3832,111 @@ fn execute_canister_http_request_caps_allowance_at_worst_case_cost() { ); } +fn http_request_args_with_pricing_version( + caller_canister: CanisterId, + pricing_version: Option, +) -> CanisterHttpRequestArgs { + CanisterHttpRequestArgs { + url: "https://example.com".to_string(), + max_response_bytes: Some(1024), + headers: BoundedHttpHeaders::new(vec![]), + body: None, + method: HttpMethod::GET, + transform: Some(TransformContext { + function: TransformFunc(candid::Func { + principal: caller_canister.get().0, + method: "transform".to_string(), + }), + context: vec![0, 1, 2], + }), + is_replicated: None, + pricing_version, + } +} + +/// The pay-as-you-go pricing model is gated behind the `flexible_http_requests` +/// feature flag for non-flexible outcalls too: a request selecting it is honored +/// once the flag is enabled, and falls back to legacy pricing until then. +#[test] +fn execute_canister_http_request_pay_as_you_go_is_gated() { + for (flexible_http_requests_enabled, expected) in [ + (false, PricingVersion::Legacy), + (true, PricingVersion::PayAsYouGo), + ] { + let own_subnet = subnet_test_id(1); + let caller_canister = canister_test_id(10); + let mut builder = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet) + .with_caller(own_subnet, caller_canister); + if flexible_http_requests_enabled { + builder = builder.with_flexible_http_requests_enabled(); + } + let mut test = builder.build(); + std::sync::Arc::make_mut(&mut test.state_mut().metadata.own_subnet_info) + .subnet_features + .http_requests = true; + + let args = http_request_args_with_pricing_version( + caller_canister, + Some(PRICING_VERSION_PAY_AS_YOU_GO), + ); + test.inject_call_to_ic00( + Method::HttpRequest, + args.encode(), + Cycles::new(1_000_000_000), + ); + test.execute_all(); + + let canister_http_request_contexts = &test + .state() + .metadata + .subnet_call_context_manager + .canister_http_request_contexts; + assert_eq!(canister_http_request_contexts.len(), 1); + let http_request_context = canister_http_request_contexts + .get(&CallbackId::from(0)) + .unwrap(); + assert_eq!( + http_request_context.pricing_version, expected, + "unexpected pricing version with the feature flag \ + {flexible_http_requests_enabled}" + ); + } +} + +/// An unknown pricing version falls back to the default one, whether or not the +/// pay-as-you-go pricing model is enabled. +#[test] +fn execute_canister_http_request_unknown_pricing_version_falls_back() { + let own_subnet = subnet_test_id(1); + let caller_canister = canister_test_id(10); + let mut test = ExecutionTestBuilder::new() + .with_own_subnet_id(own_subnet) + .with_caller(own_subnet, caller_canister) + .with_flexible_http_requests_enabled() + .build(); + std::sync::Arc::make_mut(&mut test.state_mut().metadata.own_subnet_info) + .subnet_features + .http_requests = true; + + let args = http_request_args_with_pricing_version(caller_canister, Some(42)); + test.inject_call_to_ic00( + Method::HttpRequest, + args.encode(), + Cycles::new(1_000_000_000), + ); + test.execute_all(); + + let http_request_context = test + .state() + .metadata + .subnet_call_context_manager + .canister_http_request_contexts + .get(&CallbackId::from(0)) + .unwrap(); + assert_eq!(http_request_context.pricing_version, PricingVersion::Legacy); +} + #[test] fn execute_flexible_canister_http_request_free_subnet_uses_legacy() { // On a free subnet, flexible outcalls are available by default (without the diff --git a/rs/https_outcalls/client/src/client.rs b/rs/https_outcalls/client/src/client.rs index 7ed7bc233714..49e2308d1700 100644 --- a/rs/https_outcalls/client/src/client.rs +++ b/rs/https_outcalls/client/src/client.rs @@ -154,7 +154,6 @@ impl NonBlockingChannel for CanisterHttpAdapterClientImpl { body: request_body, http_method: request_http_method, transform: request_transform, - pricing_version: request_pricing_version, replication: request_replication, max_response_bytes: request_max_response_bytes, .. @@ -165,31 +164,6 @@ impl NonBlockingChannel for CanisterHttpAdapterClientImpl { let max_response_size_bytes = request_max_response_bytes .map_or(MAX_CANISTER_HTTP_RESPONSE_BYTES, |bytes| bytes.get()); - if request_pricing_version == ic_types::canister_http::PricingVersion::PayAsYouGo { - warn!( - log, - "Canister HTTP request with PayAsYouGo pricing is not supported yet: \ - request_id {}, sender {}, process_id: {}", - request_id, - request_sender, - std::process::id(), - ); - let _ = permit.send(( - CanisterHttpResponse { - id: request_id, - canister_id: request_sender, - content: CanisterHttpResponseContent::Reject(CanisterHttpReject { - reject_code: RejectCode::SysFatal, - message: - "Canister HTTP request with PayAsYouGo pricing is not supported" - .to_string(), - }), - }, - budget.create_payment_receipt(), - )); - return; - } - let mut payload = async { // Execute the HTTP request and get the adapter response. let (adapter_response, downloaded_bytes, elapsed) = execute_http_request( diff --git a/rs/pocket_ic_server/BUILD.bazel b/rs/pocket_ic_server/BUILD.bazel index f023da5fb118..3151c84e3231 100644 --- a/rs/pocket_ic_server/BUILD.bazel +++ b/rs/pocket_ic_server/BUILD.bazel @@ -21,6 +21,7 @@ LIB_DEPENDENCIES = [ "//rs/http_endpoints/public", "//rs/https_outcalls/adapter:adapter_with_http", "//rs/https_outcalls/client", + "//rs/https_outcalls/pricing", "//rs/https_outcalls/service", "//rs/interfaces", "//rs/interfaces/adapter_client", diff --git a/rs/pocket_ic_server/CHANGELOG.md b/rs/pocket_ic_server/CHANGELOG.md index 8185d6cad4b9..10e9400679e6 100644 --- a/rs/pocket_ic_server/CHANGELOG.md +++ b/rs/pocket_ic_server/CHANGELOG.md @@ -11,6 +11,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added +- The endpoint `/instances//update/mock_flexible_canister_http` to mock the responses of the committee nodes + of a pending *flexible* canister HTTP outcall, i.e. one made through the `flexible_http_request` management canister endpoint. +- The endpoint `/instances//read/get_canister_http` reports two additional fields for every pending canister HTTP + outcall: `replication`, describing how the outcall is replicated across the nodes of its subnet (`FullyReplicated`, + `NonReplicated`, or `Flexible` with the outcall's `total_requests`, `min_responses`, and `max_responses`), and + `pricing_version`, reporting whether it is priced with the `Legacy` or the `PayAsYouGo` pricing model. +- Enabling beta features now also enables the pay-as-you-go pricing model, which covers both the `flexible_http_request` management + canister endpoint (whose outcalls are always priced that way) and the `pricing_version` field of `http_request` (through which a fully replicated or non-replicated outcall can select it). + +### Changed +- Mocked canister HTTP responses report the cycles their node actually spent on the outcall, instead of reporting no spend at + all. This applies to `/instances//update/mock_canister_http` and +`/instances//update/mock_flexible_canister_http`. +- Mocking a canister HTTP response whose reject message exceeds 1 KiB now fails. Such a response is not one any node could have + reported. +- A node that cannot pay for gossiping its mocked reject reports an out-of-cycles reject instead of the mocked one, matching + what a node of a real subnet does under pay-as-you-go pricing. + ## 15.0.0 - 2026-06-26 diff --git a/rs/pocket_ic_server/Cargo.toml b/rs/pocket_ic_server/Cargo.toml index a8bac94861ba..7665861ee5f4 100644 --- a/rs/pocket_ic_server/Cargo.toml +++ b/rs/pocket_ic_server/Cargo.toml @@ -44,6 +44,7 @@ ic-gateway = { workspace = true } ic-http-endpoints-public = { path = "../http_endpoints/public" } ic-https-outcalls-adapter = { path = "../https_outcalls/adapter" } ic-https-outcalls-adapter-client = { path = "../https_outcalls/client" } +ic-https-outcalls-pricing = { path = "../https_outcalls/pricing" } ic-https-outcalls-service = { path = "../https_outcalls/service" } ic-icp-index = { path = "../ledger_suite/icp/index" } ic-icrc1-index-ng = { path = "../ledger_suite/icrc1/index-ng" } diff --git a/rs/pocket_ic_server/src/beta_features.rs b/rs/pocket_ic_server/src/beta_features.rs index 5ce0fc0af4cb..2ffee3406e5b 100644 --- a/rs/pocket_ic_server/src/beta_features.rs +++ b/rs/pocket_ic_server/src/beta_features.rs @@ -1,10 +1,14 @@ use ic_config::embedders::FeatureFlags; +use ic_config::flag_status::FlagStatus; use ic_config::{ embedders::Config as EmbeddersConfig, execution_environment::Config as HypervisorConfig, }; pub fn hypervisor_config() -> HypervisorConfig { HypervisorConfig { + // Enables the `flexible_http_request` management canister endpoint, and the + // option for `http_request` to choose between legacy and pay-as-you-go pricing. + flexible_http_requests: FlagStatus::Enabled, embedders_config: EmbeddersConfig { feature_flags: FeatureFlags { ..Default::default() diff --git a/rs/pocket_ic_server/src/pocket_ic.rs b/rs/pocket_ic_server/src/pocket_ic.rs index 491a49499e79..0c20a6623e64 100644 --- a/rs/pocket_ic_server/src/pocket_ic.rs +++ b/rs/pocket_ic_server/src/pocket_ic.rs @@ -53,6 +53,7 @@ use ic_https_outcalls_adapter::{ start_server as start_canister_http_server, }; use ic_https_outcalls_adapter_client::{CanisterHttpAdapterClientImpl, setup_canister_http_client}; +use ic_https_outcalls_pricing::{NetworkUsage, PricingError, PricingFactory}; use ic_https_outcalls_service::HttpsOutcallRequest; use ic_https_outcalls_service::HttpsOutcallResponse; use ic_https_outcalls_service::HttpsOutcallResult; @@ -118,12 +119,15 @@ use ic_types::messages::{ CertificateDelegationFormat, CertificateDelegationMetadata, SignedSenderInfo, }; use ic_types::{ - CanisterId, Height, NumInstructions, PrincipalId, RegistryVersion, SnapshotId, SubnetId, + CanisterId, Height, NodeId, NumInstructions, PrincipalId, RegistryVersion, SnapshotId, + SubnetId, artifact::UnvalidatedArtifactMutation, canister_http::{ CanisterHttpPaymentReceipt, CanisterHttpReject, - CanisterHttpRequest as AdapterCanisterHttpRequest, CanisterHttpRequestId, - CanisterHttpResponse as AdapterCanisterHttpResponse, CanisterHttpResponseContent, + CanisterHttpRequest as AdapterCanisterHttpRequest, CanisterHttpRequestContext, + CanisterHttpRequestId, CanisterHttpResponse as AdapterCanisterHttpResponse, + CanisterHttpResponseContent, MAXIMUM_CANISTER_HTTP_ERROR_MESSAGE_BYTES, PricingVersion, + Replication, ReplicationKind, }, crypto::{BasicSig, BasicSigOf, CryptoResult, Signable, threshold_sig::IcRootOfTrust}, malicious_flags::MaliciousFlags, @@ -133,7 +137,7 @@ use ic_types::{ }, time::GENESIS, }; -use ic_types::{NumBytes, Time}; +use ic_types::{CountBytes, NumBytes, Time}; use ic_types_cycles::{CanisterCyclesCostSchedule, Cycles}; use ic_validator_http_request_test_utils::icp_mainnet_root_public_key_for_testing; use ic_validator_ingress_message::StandaloneIngressSigVerifier; @@ -141,9 +145,10 @@ use icp_ledger::{AccountIdentifier, LedgerCanisterInitPayloadBuilder, Subaccount use icrc_ledger_types::icrc1::account::Account; use itertools::Itertools; use pocket_ic::common::rest::{ - self, BinaryBlob, BlobCompression, CanisterHttpHeader, CanisterHttpMethod, CanisterHttpRequest, - CanisterHttpResponse, ExtendedSubnetConfigSet, IcpConfig, IcpConfigFlag, IcpFeatures, - IcpFeaturesConfig, IncompleteStateFlag, MockCanisterHttpResponse, RawAddCycles, + self, BinaryBlob, BlobCompression, CanisterHttpHeader, CanisterHttpMethod, + CanisterHttpPricingVersion, CanisterHttpReplication, CanisterHttpRequest, CanisterHttpResponse, + ExtendedSubnetConfigSet, IcpConfig, IcpConfigFlag, IcpFeatures, IcpFeaturesConfig, + IncompleteStateFlag, MockCanisterHttpResponse, MockFlexibleCanisterHttpResponse, RawAddCycles, RawCanisterCall, RawCanisterId, RawEffectivePrincipal, RawMessageId, RawSenderInfo, RawSetStableMemory, SubnetInstructionConfig, SubnetKind, Topology, }; @@ -3747,6 +3752,31 @@ fn http_header_from( } } +fn replication_from(replication: &Replication) -> CanisterHttpReplication { + // `Replication::kind()` already derives the committee size, so the conversion + // only has to rename the variants. + match replication.kind() { + ReplicationKind::FullyReplicated => CanisterHttpReplication::FullyReplicated, + ReplicationKind::NonReplicated => CanisterHttpReplication::NonReplicated, + ReplicationKind::Flexible { + total_requests, + min_responses, + max_responses, + } => CanisterHttpReplication::Flexible { + total_requests, + min_responses, + max_responses, + }, + } +} + +fn pricing_version_from(pricing_version: &PricingVersion) -> CanisterHttpPricingVersion { + match pricing_version { + PricingVersion::Legacy => CanisterHttpPricingVersion::Legacy, + PricingVersion::PayAsYouGo => CanisterHttpPricingVersion::PayAsYouGo, + } +} + fn get_canister_http_requests(pic: &PocketIc) -> Vec { let mut res = vec![]; for subnet in pic.subnets.get_all() { @@ -3765,6 +3795,8 @@ fn get_canister_http_requests(pic: &PocketIc) -> Vec { headers: c.headers.iter().map(http_header_from).collect(), body: c.body.unwrap_or_default(), max_response_bytes: c.max_response_bytes.map(|b| b.get()), + replication: replication_from(&c.replication), + pricing_version: pricing_version_from(&c.pricing_version), }) .collect(); res.append(&mut cur); @@ -3783,6 +3815,16 @@ impl Operation for GetCanisterHttp { } } +/// The nodes of `sm` that perform the HTTP outcall described by `context`, i.e. +/// the ones that produce a response to it. +fn outcall_nodes(sm: &StateMachine, context: &CanisterHttpRequestContext) -> Vec { + match &context.replication { + Replication::FullyReplicated => sm.nodes.iter().map(|node| node.node_id).collect(), + Replication::NonReplicated(node_id) => vec![*node_id], + Replication::Flexible { committee, .. } => committee.iter().copied().collect(), + } +} + /// The operation `ProcessCanisterHttpInternal` changes the instance state in a non-deterministic way! /// It should only be used internally in auto-progress mode /// which changes the instance state in a non-deterministic way anyway. @@ -3815,14 +3857,22 @@ impl Operation for ProcessCanisterHttpInternal { Err(_) => { break; } - Ok((response, _payment_receipt)) => { + Ok((response, payment_receipt)) => { canister_http.pending.remove(&response.id); if let Some(context) = sm.canister_http_request_contexts().get(&response.id) { - sm.mock_canister_http_response( + // Only one real outcall is made, so every node that would have + // performed it reports the same response and the same spend. + let responses = outcall_nodes(&sm, context) + .into_iter() + .map(|node_id| { + (node_id, (response.content.clone(), payment_receipt.clone())) + }) + .collect(); + sm.mock_canister_http_response_for_nodes( response.id.get(), context.request.sender, - vec![response.content; sm.nodes.len()], + responses, ); } } @@ -3895,46 +3945,48 @@ async fn setup_adapter_mock( // END COPY -fn process_mock_canister_https_response( - pic: &PocketIc, - mock_canister_http_response: &MockCanisterHttpResponse, -) -> OpOut { - let response_to_reject_code = |response: &CanisterHttpResponse| match response { - CanisterHttpResponse::CanisterHttpReply(_) => None, - CanisterHttpResponse::CanisterHttpReject(reject) => Some(reject.reject_code), - }; - let mut reject_codes: Vec<_> = mock_canister_http_response - .additional_responses - .iter() - .filter_map(response_to_reject_code) - .collect(); - if let Some(reject_code) = response_to_reject_code(&mock_canister_http_response.response) { - reject_codes.push(reject_code) - } - for reject_code in reject_codes { - if ic_error_types::RejectCode::try_from(reject_code).is_err() { - return OpOut::Error(PocketIcError::InvalidRejectCode(reject_code)); +/// Checks that every reject in `responses` is one a node could have produced, i.e. +/// that its reject code is a valid one and that its message is within the size a +/// node truncates its reject messages to. +fn validate_mock_canister_http_rejects<'a>( + responses: impl Iterator, +) -> Result<(), OpOut> { + for response in responses { + let CanisterHttpResponse::CanisterHttpReject(reject) = response else { + continue; + }; + if ic_error_types::RejectCode::try_from(reject.reject_code).is_err() { + return Err(OpOut::Error(PocketIcError::InvalidRejectCode( + reject.reject_code, + ))); + } + // A node prunes an oversized reject message before signing and gossiping + // it, so a longer message is not something any node could have reported: + // it would both be priced above what a node can be charged for gossiping + // a reject and produce a response share that peers reject as too large. + if reject.message.len() > MAXIMUM_CANISTER_HTTP_ERROR_MESSAGE_BYTES { + return Err(OpOut::Error( + PocketIcError::CanisterHttpRejectMessageTooLong(( + reject.message.len(), + MAXIMUM_CANISTER_HTTP_ERROR_MESSAGE_BYTES, + )), + )); } } - let subnet_id = - ic_types::SubnetId::new(ic_types::PrincipalId(mock_canister_http_response.subnet_id)); - let Some(subnet) = pic.subnets.get(subnet_id) else { - return OpOut::Error(PocketIcError::SubnetNotFound( - mock_canister_http_response.subnet_id, - )); - }; - let canister_http_request_id = - CanisterHttpRequestId::from(mock_canister_http_response.request_id); - let contexts = subnet.canister_http_request_contexts(); - let Some(context) = contexts.get(&canister_http_request_id) else { - return OpOut::Error(PocketIcError::InvalidCanisterHttpRequestId(( - subnet_id, - canister_http_request_id, - ))); - }; - let canister_id = context.request.sender; + Ok(()) +} - let response_to_content = |response: &CanisterHttpResponse| match response { +/// Turns one mocked response into the response content a node would have +/// produced, together with the cycles that node reports having spent on the +/// outcall. +fn mock_canister_http_response_content( + pic: &PocketIc, + subnet: &StateMachine, + canister_http_request_id: CanisterHttpRequestId, + context: &CanisterHttpRequestContext, + response: &CanisterHttpResponse, +) -> (CanisterHttpResponseContent, CanisterHttpPaymentReceipt) { + match response { CanisterHttpResponse::CanisterHttpReply(reply) => { let response = HttpsOutcallResponse { status: reply.status.into(), @@ -3982,22 +4034,126 @@ fn process_mock_canister_https_response( socks_proxy_addrs: vec![], }) .unwrap(); - let response = loop { + loop { match client.try_receive() { Err(_) => std::thread::sleep(Duration::from_millis(10)), - Ok((r, _payment_receipt)) => { - break r; + Ok((response, payment_receipt)) => { + break (response.content, payment_receipt); } } - }; - response.content + } } CanisterHttpResponse::CanisterHttpReject(reject) => { - CanisterHttpResponseContent::Reject(CanisterHttpReject { + // The reject code was checked by `validate_mock_canister_http_rejects` + // before any response was converted. + let reject = CanisterHttpReject { reject_code: ic_error_types::RejectCode::try_from(reject.reject_code).unwrap(), message: reject.message.clone(), - }) + }; + let mut budget = + PricingFactory::new(&MetricsRegistry::new(), subnet.replica_logger.clone()) + .new_tracker(context); + // A rejecting node ran no transform and, unlike a real one, spent no + // time on an outcall that downloaded nothing. It does gossip its reject + // body to its peers though, which is what it is charged for here. + budget + .subtract_network_usage(NetworkUsage { + response_size: NumBytes::from(0), + response_time: Duration::ZERO, + }) + .expect("an outcall that consumed no network resources is never charged"); + // A node that cannot pay for gossiping its reject reports an + // out-of-cycles reject instead of the one it produced, just like the + // HTTPS outcalls client does. + let reject = + match budget.subtract_gossip_usage(NumBytes::from(reject.count_bytes() as u64)) { + Ok(()) => reject, + Err(PricingError::InsufficientCycles) => CanisterHttpReject { + reject_code: ic_error_types::RejectCode::CanisterReject, + message: "Insufficient cycles".to_string(), + }, + }; + ( + CanisterHttpResponseContent::Reject(reject), + budget.create_payment_receipt(), + ) } + } +} + +/// The pending canister HTTP outcall a mock refers to: the subnet it was made on, +/// its request ID, and its request context. +type PendingCanisterHttpRequest = ( + Arc, + CanisterHttpRequestId, + CanisterHttpRequestContext, +); + +/// Resolves the subnet and the pending canister HTTP outcall a mock refers to. +fn pending_canister_http_request( + pic: &PocketIc, + raw_subnet_id: Principal, + request_id: u64, +) -> Result { + let subnet_id = ic_types::SubnetId::new(ic_types::PrincipalId(raw_subnet_id)); + let Some(subnet) = pic.subnets.get(subnet_id) else { + return Err(OpOut::Error(PocketIcError::SubnetNotFound(raw_subnet_id))); + }; + let canister_http_request_id = CanisterHttpRequestId::from(request_id); + let Some(context) = subnet + .canister_http_request_contexts() + .remove(&canister_http_request_id) + else { + return Err(OpOut::Error(PocketIcError::InvalidCanisterHttpRequestId(( + subnet_id, + canister_http_request_id, + )))); + }; + Ok((subnet, canister_http_request_id, context)) +} + +fn process_mock_canister_https_response( + pic: &PocketIc, + mock_canister_http_response: &MockCanisterHttpResponse, +) -> OpOut { + if let Err(err) = validate_mock_canister_http_rejects( + std::iter::once(&mock_canister_http_response.response) + .chain(mock_canister_http_response.additional_responses.iter()), + ) { + return err; + } + let (subnet, canister_http_request_id, context) = match pending_canister_http_request( + pic, + mock_canister_http_response.subnet_id, + mock_canister_http_response.request_id, + ) { + Ok(request) => request, + Err(err) => return err, + }; + let canister_id = context.request.sender; + + // The number of responses is checked before any of them is converted, so that a + // mismatch does not run the transform function of the calling canister. + let num_responses = if mock_canister_http_response.additional_responses.is_empty() { + subnet.nodes.len() + } else { + mock_canister_http_response.additional_responses.len() + 1 + }; + if num_responses != subnet.nodes.len() { + return OpOut::Error(PocketIcError::InvalidMockCanisterHttpResponses(( + num_responses, + subnet.nodes.len(), + ))); + } + + let response_to_content = |response: &CanisterHttpResponse| { + mock_canister_http_response_content( + pic, + &subnet, + canister_http_request_id, + &context, + response, + ) }; let content = response_to_content(&mock_canister_http_response.response); let mut contents: Vec<_> = if !mock_canister_http_response.additional_responses.is_empty() { @@ -4010,16 +4166,80 @@ fn process_mock_canister_https_response( vec![content.clone(); subnet.nodes.len() - 1] }; contents.push(content); - if contents.len() != subnet.nodes.len() { - return OpOut::Error(PocketIcError::InvalidMockCanisterHttpResponses(( - contents.len(), - subnet.nodes.len(), + // Every node of the subnet answers, which is the contract of this + // (non-flexible) mock. That is exactly what a fully replicated outcall needs, + // since its committee is the whole node set. For a non-replicated or flexible + // outcall the payload builder ignores the shares of the nodes outside the + // designated node / the committee, so the surplus responses are harmless (see + // `mock_flexible_canister_http_response` for mocking a flexible outcall + // committee by committee). + let responses = std::iter::zip(subnet.nodes.iter(), contents) + .map(|(node, content)| (node.node_id, content)) + .collect(); + subnet.mock_canister_http_response_for_nodes( + mock_canister_http_response.request_id, + canister_id, + responses, + ); + OpOut::NoOutput +} + +fn process_mock_flexible_canister_https_response( + pic: &PocketIc, + mock_flexible_canister_http_response: &MockFlexibleCanisterHttpResponse, +) -> OpOut { + if let Err(err) = + validate_mock_canister_http_rejects(mock_flexible_canister_http_response.responses.iter()) + { + return err; + } + let (subnet, canister_http_request_id, context) = match pending_canister_http_request( + pic, + mock_flexible_canister_http_response.subnet_id, + mock_flexible_canister_http_response.request_id, + ) { + Ok(request) => request, + Err(err) => return err, + }; + let Replication::Flexible { committee, .. } = &context.replication else { + return OpOut::Error(PocketIcError::NotAFlexibleCanisterHttpRequest(( + subnet.get_subnet_id(), + canister_http_request_id, + ))); + }; + let num_responses = mock_flexible_canister_http_response.responses.len(); + if num_responses > committee.len() { + return OpOut::Error(PocketIcError::TooManyMockCanisterHttpResponses(( + num_responses, + committee.len(), ))); } - subnet.mock_canister_http_response( - mock_canister_http_response.request_id, + let canister_id = context.request.sender; + + // The responses are assigned to the committee's nodes one each, in the + // deterministic order in which the `BTreeSet` iterates them. The assigned + // node IDs remain observable in per-node error details. + let responses = std::iter::zip( + committee.iter(), + mock_flexible_canister_http_response.responses.iter(), + ) + .map(|(node_id, response)| { + ( + *node_id, + mock_canister_http_response_content( + pic, + &subnet, + canister_http_request_id, + &context, + response, + ), + ) + }) + .collect(); + subnet.mock_canister_http_response_for_nodes( + mock_flexible_canister_http_response.request_id, canister_id, - contents, + responses, ); OpOut::NoOutput } @@ -4042,6 +4262,27 @@ impl Operation for MockCanisterHttp { } } +#[derive(Clone, Debug)] +pub struct MockFlexibleCanisterHttp { + pub mock_flexible_canister_http_response: MockFlexibleCanisterHttpResponse, +} + +impl Operation for MockFlexibleCanisterHttp { + fn compute(&self, pic: &mut PocketIc) -> OpOut { + process_mock_flexible_canister_https_response( + pic, + &self.mock_flexible_canister_http_response, + ) + } + + fn id(&self) -> OpId { + OpId(format!( + "mock_flexible_canister_http({:?})", + self.mock_flexible_canister_http_response + )) + } +} + #[derive(Copy, Clone, Debug)] pub struct PubKey { pub subnet_id: SubnetId, diff --git a/rs/pocket_ic_server/src/state_api/routes.rs b/rs/pocket_ic_server/src/state_api/routes.rs index 38d28386e9ef..bc9ac49f39b2 100644 --- a/rs/pocket_ic_server/src/state_api/routes.rs +++ b/rs/pocket_ic_server/src/state_api/routes.rs @@ -12,8 +12,8 @@ use crate::pocket_ic::{ AddCycles, AwaitIngressMessage, CallRequest, CallRequestVersion, CanisterReadStateRequest, CanisterSnapshotDownload, CanisterSnapshotUpload, DashboardRequest, DeleteSubnet, GetCanisterHttp, GetControllers, GetCyclesBalance, GetStableMemory, GetSubnet, GetTime, - GetTopology, IngressMessageStatus, MockCanisterHttp, PubKey, Query, QueryRequest, - SetCertifiedTime, SetStableMemory, SetTime, StatusRequest, SubmitIngressMessage, + GetTopology, IngressMessageStatus, MockCanisterHttp, MockFlexibleCanisterHttp, PubKey, Query, + QueryRequest, SetCertifiedTime, SetStableMemory, SetTime, StatusRequest, SubmitIngressMessage, SubnetReadStateRequest, Tick, }; use crate::{ @@ -45,11 +45,11 @@ use pocket_ic::RejectResponse; use pocket_ic::common::rest::{ self, ApiResponse, AutoProgressConfig, ExtendedSubnetConfigSet, HttpGatewayConfig, HttpGatewayDetails, IcpConfig, IcpFeatures, InitialTime, InstanceConfig, - MockCanisterHttpResponse, RawAddCycles, RawCanisterCall, RawCanisterHttpRequest, RawCanisterId, - RawCanisterResult, RawCanisterSnapshotDownload, RawCanisterSnapshotId, - RawCanisterSnapshotUpload, RawCycles, RawIngressStatusArgs, RawMessageId, - RawMockCanisterHttpResponse, RawPrincipalId, RawSetStableMemory, RawStableMemory, RawSubnetId, - RawTickConfigs, RawTime, Topology, + MockCanisterHttpResponse, MockFlexibleCanisterHttpResponse, RawAddCycles, RawCanisterCall, + RawCanisterHttpRequest, RawCanisterId, RawCanisterResult, RawCanisterSnapshotDownload, + RawCanisterSnapshotId, RawCanisterSnapshotUpload, RawCycles, RawIngressStatusArgs, + RawMessageId, RawMockCanisterHttpResponse, RawMockFlexibleCanisterHttpResponse, RawPrincipalId, + RawSetStableMemory, RawStableMemory, RawSubnetId, RawTickConfigs, RawTime, Topology, }; use serde::Serialize; use slog::Level; @@ -138,6 +138,10 @@ where .directory_route("/set_stable_memory", post(handler_set_stable_memory)) .directory_route("/tick", post(handler_tick)) .directory_route("/mock_canister_http", post(handler_mock_canister_http)) + .directory_route( + "/mock_flexible_canister_http", + post(handler_mock_flexible_canister_http), + ) .directory_route( "/canister_snapshot_download", post(handler_canister_snapshot_download), @@ -710,6 +714,24 @@ pub async fn handler_mock_canister_http( (code, Json(response)) } +pub async fn handler_mock_flexible_canister_http( + State(AppState { api_state, .. }): State, + headers: HeaderMap, + Path(instance_id): Path, + axum::extract::Json(raw_mock_flexible_canister_http_response): axum::extract::Json< + RawMockFlexibleCanisterHttpResponse, + >, +) -> (StatusCode, Json>) { + let timeout = timeout_or_default(headers); + let mock_flexible_canister_http_response: MockFlexibleCanisterHttpResponse = + raw_mock_flexible_canister_http_response.into(); + let op = MockFlexibleCanisterHttp { + mock_flexible_canister_http_response, + }; + let (code, response) = run_operation(api_state, instance_id, timeout, op).await; + (code, Json(response)) +} + pub async fn handler_get_controllers( State(AppState { api_state, .. }): State, Path(instance_id): Path, diff --git a/rs/pocket_ic_server/src/state_api/state.rs b/rs/pocket_ic_server/src/state_api/state.rs index de4e17ba1844..601e9b2baf82 100644 --- a/rs/pocket_ic_server/src/state_api/state.rs +++ b/rs/pocket_ic_server/src/state_api/state.rs @@ -285,7 +285,10 @@ pub enum PocketIcError { SubnetRequestRoutingError(String), InvalidCanisterHttpRequestId((SubnetId, CanisterHttpRequestId)), InvalidMockCanisterHttpResponses((usize, usize)), + NotAFlexibleCanisterHttpRequest((SubnetId, CanisterHttpRequestId)), + TooManyMockCanisterHttpResponses((usize, usize)), InvalidRejectCode(u64), + CanisterHttpRejectMessageTooLong((usize, usize)), SettingTimeIntoPast((u64, u64)), Forbidden(String), BlockmakerNotFound(NodeId), @@ -348,9 +351,30 @@ impl std::fmt::Debug for OpOut { "InvalidMockCanisterHttpResponses(actual={actual},expected={expected})" ) } + OpOut::Error(PocketIcError::NotAFlexibleCanisterHttpRequest(( + subnet_id, + canister_http_request_id, + ))) => { + write!( + f, + "NotAFlexibleCanisterHttpRequest({subnet_id},{canister_http_request_id:?})" + ) + } + OpOut::Error(PocketIcError::TooManyMockCanisterHttpResponses((actual, max))) => { + write!( + f, + "TooManyMockCanisterHttpResponses(actual={actual},max={max})" + ) + } OpOut::Error(PocketIcError::InvalidRejectCode(code)) => { write!(f, "InvalidRejectCode({code})") } + OpOut::Error(PocketIcError::CanisterHttpRejectMessageTooLong((actual, max))) => { + write!( + f, + "CanisterHttpRejectMessageTooLong(actual={actual},max={max})" + ) + } OpOut::Error(PocketIcError::SettingTimeIntoPast((current, set))) => { write!(f, "SettingTimeIntoPast(current={current},set={set})") } diff --git a/rs/state_machine_tests/src/lib.rs b/rs/state_machine_tests/src/lib.rs index 2838bc958c22..55cab0672697 100644 --- a/rs/state_machine_tests/src/lib.rs +++ b/rs/state_machine_tests/src/lib.rs @@ -2764,6 +2764,11 @@ impl StateMachine { .push(msg, self.get_time(), self.nodes[0].node_id); } + /// Injects one response share per node of the subnet, all reporting that the + /// node spent nothing on the outcall. + /// + /// For an outcall performed by only a subset of the nodes, or one whose nodes + /// report having spent cycles, see [`Self::mock_canister_http_response_for_nodes`]. pub fn mock_canister_http_response( &self, request_id: u64, @@ -2771,7 +2776,37 @@ impl StateMachine { contents: Vec, ) { assert_eq!(contents.len(), self.nodes.len()); - for (node, content) in std::iter::zip(self.nodes.iter(), contents) { + let responses = std::iter::zip(self.nodes.iter(), contents) + .map(|(node, content)| { + ( + node.node_id, + (content, CanisterHttpPaymentReceipt::default()), + ) + }) + .collect(); + self.mock_canister_http_response_for_nodes(request_id, canister_id, responses); + } + + /// Injects one response share per entry of `responses`, signed by the node it + /// is keyed by and carrying that node's payment receipt. + /// + /// Unlike [`Self::mock_canister_http_response`], this does not require exactly + /// one response per subnet node, which is what non-fully-replicated outcalls + /// need: only the nodes of the outcall's committee produce a response, their + /// responses may differ, and some of them may not respond at all. + pub fn mock_canister_http_response_for_nodes( + &self, + request_id: u64, + canister_id: CanisterId, + responses: BTreeMap, + ) { + for node_id in responses.keys() { + assert!( + self.nodes.iter().any(|node| node.node_id == *node_id), + "cannot respond as {node_id}, which is not a node of this subnet" + ); + } + for (node_id, (content, payment_receipt)) in responses { let registry_version = self.registry_client.get_latest_version(); let response = CanisterHttpResponse { id: CanisterHttpRequestId::from(request_id), @@ -2786,10 +2821,10 @@ impl StateMachine { is_reject: content.is_reject(), replica_version: ReplicaVersion::default(), }, - payment_receipt: CanisterHttpPaymentReceipt::default(), + payment_receipt, }; let signature = CryptoReturningOk::default() - .sign(&receipt_share, node.node_id, registry_version) + .sign(&receipt_share, node_id, registry_version) .unwrap(); let share = Signed { content: receipt_share, diff --git a/rs/tests/networking/canister_http_correctness_test.rs b/rs/tests/networking/canister_http_correctness_test.rs index ff29c7fb6c89..e2cef1bcaab6 100644 --- a/rs/tests/networking/canister_http_correctness_test.rs +++ b/rs/tests/networking/canister_http_correctness_test.rs @@ -2818,6 +2818,10 @@ fn expected_cycle_cost( RegistryVersion::from(1), CanisterCyclesCostSchedule::Normal, &mut rand::thread_rng(), + // Only the request size is read off this context, so the pricing model it + // would be charged with does not matter. + /* pay_as_you_go_enabled = */ + false, ) .unwrap(); let req_size = dummy_context.variable_parts_size(); diff --git a/rs/types/management_canister_types/src/http.rs b/rs/types/management_canister_types/src/http.rs index 2ee699a273e9..fb4142e3ac61 100644 --- a/rs/types/management_canister_types/src/http.rs +++ b/rs/types/management_canister_types/src/http.rs @@ -66,11 +66,18 @@ pub const PRICING_VERSION_PAY_AS_YOU_GO: u32 = 2; /// Described in . pub const DEFAULT_HTTP_OUTCALLS_PRICING_VERSION: u32 = PRICING_VERSION_LEGACY; -/// A set of all allowed pricing versions for HTTP outcalls. +/// The pricing versions an HTTP outcall may select on a subnet where the +/// pay-as-you-go pricing model is *not* enabled. /// /// If the pricing version provided in the request is not in this set, the request will use the default pricing version. pub const ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS: &[u32] = &[PRICING_VERSION_LEGACY]; +/// The pricing versions an HTTP outcall may select on a subnet where the +/// pay-as-you-go pricing model *is* enabled, i.e. one whose +/// `flexible_http_requests` feature flag is on. +pub const ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS_WITH_PAY_AS_YOU_GO: &[u32] = + &[PRICING_VERSION_LEGACY, PRICING_VERSION_PAY_AS_YOU_GO]; + /// HTTP headers bounded by total size. pub type BoundedHttpHeaders = BoundedVec< HTTP_HEADERS_MAX_NUMBER, @@ -431,8 +438,6 @@ pub struct FlexibleHttpRequestErr { /// Why the flexible HTTP outcall failed globally. #[derive(Clone, Eq, PartialEq, Hash, Debug, CandidType, Deserialize, Serialize)] pub enum FlexibleHttpGlobalError { - #[serde(rename = "invalid_parameters")] - InvalidParameters(candid::Reserved), #[serde(rename = "timeout")] Timeout(candid::Reserved), #[serde(rename = "out_of_cycles")] diff --git a/rs/types/management_canister_types/src/lib.rs b/rs/types/management_canister_types/src/lib.rs index 5742a153ce1c..9b6acd4252a6 100644 --- a/rs/types/management_canister_types/src/lib.rs +++ b/rs/types/management_canister_types/src/lib.rs @@ -10,8 +10,9 @@ pub use bounded_vec::*; use candid::{CandidType, Decode, DecoderConfig, Deserialize, Encode, Reserved}; pub use data_size::*; pub use http::{ - ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS, BoundedHttpHeaders, CanisterHttpRequestArgs, - CanisterHttpResponsePayload, DEFAULT_HTTP_OUTCALLS_PRICING_VERSION, + ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS, + ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS_WITH_PAY_AS_YOU_GO, BoundedHttpHeaders, + CanisterHttpRequestArgs, CanisterHttpResponsePayload, DEFAULT_HTTP_OUTCALLS_PRICING_VERSION, FlexibleCanisterHttpRequestArgs, FlexibleHttpGlobalError, FlexibleHttpNodeDetail, FlexibleHttpNodeError, FlexibleHttpRequestErr, FlexibleHttpRequestResult, HttpHeader, HttpMethod, HttpRequestResourceReport, PRICING_VERSION_LEGACY, PRICING_VERSION_PAY_AS_YOU_GO, diff --git a/rs/types/types/src/canister_http.rs b/rs/types/types/src/canister_http.rs index fd9b98e63bfb..bc6f1b8648c6 100644 --- a/rs/types/types/src/canister_http.rs +++ b/rs/types/types/src/canister_http.rs @@ -54,7 +54,8 @@ use ic_error_types::{ErrorCode, RejectCode, UserError}; #[cfg(test)] use ic_exhaustive_derive::ExhaustiveSet; use ic_management_canister_types_private::{ - ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS, CanisterHttpRequestArgs, + ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS, + ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS_WITH_PAY_AS_YOU_GO, CanisterHttpRequestArgs, DEFAULT_HTTP_OUTCALLS_PRICING_VERSION, DataSize, FlexibleCanisterHttpRequestArgs, HttpHeader, HttpMethod, PRICING_VERSION_LEGACY, PRICING_VERSION_PAY_AS_YOU_GO, ReplicationCounts, TransformContext, @@ -657,6 +658,7 @@ impl CanisterHttpRequestContext { registry_version: RegistryVersion, cost_schedule: CanisterCyclesCostSchedule, rng: &mut dyn RngCore, + pay_as_you_go_enabled: bool, ) -> Result { validate_transform_principal(&args.transform, request.sender.get())?; validate_url_length(&args.url)?; @@ -708,9 +710,14 @@ impl CanisterHttpRequestContext { time, replication, pricing_version: { + let allowed_versions = if pay_as_you_go_enabled { + ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS_WITH_PAY_AS_YOU_GO + } else { + ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS + }; let final_version_u32 = args .pricing_version - .filter(|v| ALLOWED_HTTP_OUTCALLS_PRICING_VERSIONS.contains(v)) + .filter(|v| allowed_versions.contains(v)) .unwrap_or(DEFAULT_HTTP_OUTCALLS_PRICING_VERSION); PricingVersion::from_repr(final_version_u32).unwrap_or(PricingVersion::Legacy) }, @@ -2238,6 +2245,7 @@ mod tests { RegistryVersion::from(1), CanisterCyclesCostSchedule::Normal, &mut ReproducibleRng::new(), + /* pay_as_you_go_enabled = */ false, ) }