feat: CON-1700 Support flexible outcalls and pay-as-you-go pricing in pocket-ic

packages/pocket-ic/BUILD.bazel

@@ -84,6 +84,7 @@ rust_test(
         "//rs/registry/helpers",
         "//rs/registry/proto_data_provider",
         "//rs/types/base_types",
+        "//rs/types/management_canister_types",
         "@crate_index//:bitcoin-0.28.2",
         "@crate_index//:candid",
         "@crate_index//:ed25519-dalek",
@@ -98,6 +99,7 @@ rust_test(
         "@crate_index//:k256",
         "@crate_index//:reqwest",
         "@crate_index//:serde",
+        "@crate_index//:serde_bytes",
         "@crate_index//:serde_cbor",
         "@crate_index//:sha2",
         "@crate_index//:tempfile",

packages/pocket-ic/CHANGELOG.md

@@ -9,8 +9,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Added the `SubnetCoolingDown` variant to the `ErrorCode` enum: ingress messages addressed to a subnet that is "cooling down" are rejected with this error code.
+- The function `PocketIc::mock_flexible_canister_http_response` and the type `MockFlexibleCanisterHttpResponse` to mock the responses
+  of the committee nodes of a pending *flexible* canister HTTP outcall, i.e. one made through the `flexible_http_request`
+  management canister endpoint.
+- The field `replication` of type `CanisterHttpReplication` on `CanisterHttpRequest`, describing how a pending canister HTTP
+  outcall is replicated across the nodes of its subnet: `FullyReplicated`, `NonReplicated`, or `Flexible` with the outcall's
+  `total_requests`, `min_responses`, and `max_responses`. The committee size of a flexible outcall (`total_requests`)
+  is the number of responses `PocketIc::mock_flexible_canister_http_response` accepts.
+- The field `pricing_version` of type `CanisterHttpPricingVersion` on `CanisterHttpRequest`, reporting whether a pending
+  canister HTTP outcall is priced with the `Legacy` or the `PayAsYouGo` pricing model.
+- Enabling beta features (`IcpConfig::beta_features` passed to `PocketIcBuilder::with_icp_config`) now also enables the
+  pay-as-you-go pricing model, which covers both the `flexible_http_request` management canister endpoint (whose outcalls are
+  always priced that way) and the `pricing_version` field of `http_request` (through which a fully replicated or non-replicated
+  outcall can select it).
 
 ### Changed
+- Mocked canister HTTP responses report the cycles their node actually spent on the outcall, instead of reporting no spend at all.
+  This applies to both `PocketIc::mock_canister_http_response` and `PocketIc::mock_flexible_canister_http_response`.
+- Mocking a canister HTTP response whose reject message exceeds 1 KiB now fails. Such a response is not one any node could have
+  reported.
+- A node that cannot pay for gossiping its mocked reject reports an out-of-cycles reject instead of the mocked one, matching what
+  a node of a real subnet does under pay-as-you-go pricing.
 - No hard TTL is set on PocketIC servers started implicitly by the library (e.g. by `PocketIc::new` or `PocketIcBuilder::build`); previously a default of 10 minutes was used.
   The hard TTL is an absolute deadline measured from the server's launch which is not extended by activity, so a test suite whose total runtime exceeded it had its server terminated while still serving requests, failing in-flight calls with `Connection reset by peer`.
   Orphaned servers remain bounded by the (activity-based) soft TTL.

packages/pocket-ic/Cargo.toml

@@ -72,6 +72,7 @@ ic-vetkeys = { workspace = true }
 icrc-ledger-types = { path = "../icrc-ledger-types" }
 k256 = { workspace = true }
 registry-canister = { path = "../../rs/registry/canister" }
+serde_bytes = { workspace = true }
 wat = { workspace = true }
 
 [target.'cfg(not(windows))'.dev-dependencies]

packages/pocket-ic/HOWTO.md

@@ -378,6 +378,106 @@ e.g., 13 for a regular application subnet.
     assert_eq!(err, expected);
 ```
 
+### Flexible canister HTTP outcalls
+
+A canister can also make a *flexible* HTTP outcall through the `flexible_http_request` management canister
+endpoint. Such an outcall is performed by a committee of `total_requests` nodes of the subnet, whose responses
+need not agree, and the calling canister is handed between `min_responses` and `max_responses` of them.
+
+Flexible outcalls are priced with the pay-as-you-go pricing model, which is still gated. So is the
+`pricing_version` field of a regular `http_request`, through which a fully replicated or non-replicated outcall
+can select that same pricing model. To use either on a regular application subnet, enable beta features on the
+instance:
+
+```rust
+    let pic = PocketIcBuilder::new()
+        .with_application_subnet()
+        .with_icp_config(IcpConfig {
+            beta_features: Some(IcpConfigFlag::Enabled),
+            ..Default::default()
+        })
+        .build();
+```
+
+Without beta features, flexible outcalls are only available on subnets where HTTP outcalls are free (system
+subnets and subnets created with a free cycles cost schedule), where they fall back to the legacy pricing model,
+and an `http_request` asking for pay-as-you-go pricing silently falls back to the legacy pricing model as well.
+`CanisterHttpRequest::pricing_version` reports which pricing model a pending outcall actually ended up with.
+
+Under the pay-as-you-go pricing model, a base fee is charged up front, a per-replica cycles allowance is
+withheld from the payment, and whatever the responding nodes do not spend is refunded. If the attached cycles
+do not cover the cost of delivering a response, the outcall fails with an out-of-cycles error: a `flexible_http_request`
+returns the `out_of_cycles` error described below, while an `http_request` is rejected with `SysTransient` and
+a message starting with `Out of cycles:`. This applies to calls made under pay-as-you-go pricing.
+
+A pending flexible outcall reports its replication in `CanisterHttpRequest::replication`, from which the size
+of its committee can be read, and its responses are mocked with `PocketIc::mock_flexible_canister_http_response`:
+
+```rust
+    let canister_http_requests = pic.get_canister_http();
+    assert_eq!(canister_http_requests.len(), 1);
+    let canister_http_request = &canister_http_requests[0];
+
+    let CanisterHttpReplication::Flexible { total_requests, .. } =
+        canister_http_request.replication
+    else {
+        panic!("expected a flexible canister http outcall");
+    };
+
+    let http_reply = |body: &[u8]| {
+        CanisterHttpResponse::CanisterHttpReply(CanisterHttpReply {
+            status: 200,
+            headers: vec![],
+            body: body.to_vec(),
+        })
+    };
+    pic.mock_flexible_canister_http_response(MockFlexibleCanisterHttpResponse {
+        subnet_id: canister_http_request.subnet_id,
+        request_id: canister_http_request.request_id,
+        // One response per committee node. The responses may differ.
+        responses: vec![http_reply(b"hello"); total_requests as usize],
+    });
+
+    // The calling canister receives at least `min_responses` (and at most
+    // `max_responses`) of the mocked responses, smallest first.
+    let reply = pic.await_call(call_id).unwrap();
+```
+
+Unlike `PocketIc::mock_canister_http_response`, which delivers one response per node of the subnet,
+this takes *at most* `total_requests` responses. Providing fewer models the remaining committee nodes never
+responding:
+- with at least `min_responses` successful responses among them, the outcall succeeds;
+- with more rejects than the slack between `total_requests` and `min_responses` allows, i.e. with more than
+  `total_requests - min_responses` of them, the outcall fails with a `too_many_rejects` error reporting the
+  rejecting nodes;
+- if the `min_responses` smallest successful responses do not fit into the 2 MiB a block has for HTTP outcall
+  responses, the outcall fails with a `responses_too_large` error — note that each individual response may
+  still be within the 2 MB cap that applies to a single response;
+- if the cycles attached to the outcall do not cover the cost of putting its responses into a block, it fails
+  with an `out_of_cycles` error;
+- with too few responses to decide any of the above, the outcall stays pending until, after advancing the time
+  past its 60 second timeout, it fails with a `timeout` error.
+
+*Warning.* All responses to an outcall must be provided in a single call: once any response to it has been
+mocked, the outcall no longer shows up in `PocketIc::get_canister_http` and further responses to it cannot
+be mocked.
+
+*Note.* A mocked reject message must fit into the 1 KiB a node truncates its reject messages to; mocking a
+longer one fails, since no node could have reported it. This applies to both `PocketIc::mock_canister_http_response`
+and `PocketIc::mock_flexible_canister_http_response`.
+
+Note that a flexible outcall never rejects the calling canister's call: every outcome above, including the
+errors, is delivered as a `flexible_http_request_result` reply. Only a synchronous failure (invalid arguments,
+insufficient cycles attached, or the endpoint not being enabled on the subnet) rejects the call.
+
+*Warning.* The cycles a node reports having spent on a mocked outcall include a term for how long the
+response took to arrive, which PocketIC derives from how long it took to run the mocked outcall in process.
+Avoid asserting on exact cycles balances after any outcall priced with the pay-as-you-go pricing model —
+whether it is a flexible one or an `http_request` mocked with `PocketIc::mock_canister_http_response`.
+Outcalls priced with the legacy pricing model are unaffected, since it ignores the reported spend.
+
+### Live mode
+
 In the live mode (see the section "Live Mode" for more details), the canister HTTP outcalls are processed
 by actually making an HTTP request to the URL specified in the canister HTTP outcall.
 

packages/pocket-ic/src/common/rest.rs

@@ -1174,6 +1174,47 @@ pub struct CanisterHttpHeader {
     pub value: String,
 }
 
+/// How a canister HTTP outcall is replicated across the nodes of its subnet.
+#[derive(
+    Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, JsonSchema,
+)]
+pub enum CanisterHttpReplication {
+    /// Every node of the subnet performs the outcall and a response is delivered
+    /// once `n - f` of them agree on it. Too many differing responses make the
+    /// outcall fail with a "no consensus could be reached" rejection instead.
+    /// Mock such an outcall with `PocketIc::mock_canister_http_response`.
+    FullyReplicated,
+    /// A single node performs the outcall (`is_replicated = false`) and its
+    /// response is the one delivered.
+    NonReplicated,
+    /// A committee of `total_requests` nodes performs the outcall and between
+    /// `min_responses` and `max_responses` of their (potentially differing)
+    /// responses are delivered to the calling canister. Mock such an outcall
+    /// with `PocketIc::mock_flexible_canister_http_response`.
+    Flexible {
+        /// The number of nodes performing the outcall.
+        total_requests: u32,
+        /// The number of responses required to deliver a result.
+        min_responses: u32,
+        /// The largest number of responses that may be delivered.
+        max_responses: u32,
+    },
+}
+
+/// The pricing model applied to a canister HTTP outcall.
+#[derive(
+    Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, JsonSchema,
+)]
+pub enum CanisterHttpPricingVersion {
+    /// The whole cost of the outcall is charged up front, based on the largest
+    /// response it could receive.
+    Legacy,
+    /// The outcall is charged for the resources it actually consumes: a base fee is
+    /// charged up front, a per-replica cycles allowance is withheld from the
+    /// payment, and whatever the responding nodes do not spend is refunded.
+    PayAsYouGo,
+}
+
 #[derive(Clone, Serialize, Deserialize, Debug, JsonSchema)]
 pub struct RawCanisterHttpRequest {
     pub subnet_id: RawSubnetId,
@@ -1185,6 +1226,8 @@ pub struct RawCanisterHttpRequest {
     #[serde(serialize_with = "base64::serialize")]
     pub body: Vec<u8>,
     pub max_response_bytes: Option<u64>,
+    pub replication: CanisterHttpReplication,
+    pub pricing_version: CanisterHttpPricingVersion,
 }
 
 #[derive(Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
@@ -1198,6 +1241,8 @@ pub struct CanisterHttpRequest {
     #[serde(serialize_with = "base64::serialize")]
     pub body: Vec<u8>,
     pub max_response_bytes: Option<u64>,
+    pub replication: CanisterHttpReplication,
+    pub pricing_version: CanisterHttpPricingVersion,
 }
 
 impl From<RawCanisterHttpRequest> for CanisterHttpRequest {
@@ -1212,6 +1257,8 @@ impl From<RawCanisterHttpRequest> for CanisterHttpRequest {
             headers: raw_canister_http_request.headers,
             body: raw_canister_http_request.body,
             max_response_bytes: raw_canister_http_request.max_response_bytes,
+            replication: raw_canister_http_request.replication,
+            pricing_version: raw_canister_http_request.pricing_version,
         }
     }
 }
@@ -1226,6 +1273,8 @@ impl From<CanisterHttpRequest> for RawCanisterHttpRequest {
             headers: canister_http_request.headers,
             body: canister_http_request.body,
             max_response_bytes: canister_http_request.max_response_bytes,
+            replication: canister_http_request.replication,
+            pricing_version: canister_http_request.pricing_version,
         }
     }
 }
@@ -1246,6 +1295,7 @@ pub struct CanisterHttpReply {
 )]
 pub struct CanisterHttpReject {
     pub reject_code: u64,
+    /// Bounded by the 1 KiB a node truncates its reject messages to.
     pub message: String,
 }
 
@@ -1299,6 +1349,61 @@ impl From<MockCanisterHttpResponse> for RawMockCanisterHttpResponse {
     }
 }
 
+#[derive(Clone, Serialize, Deserialize, Debug, JsonSchema)]
+pub struct RawMockFlexibleCanisterHttpResponse {
+    pub subnet_id: RawSubnetId,
+    pub request_id: u64,
+    pub responses: Vec<CanisterHttpResponse>,
+}
+
+/// Mocked responses to a pending *flexible* canister HTTP outcall, i.e. one made
+/// through the `flexible_http_request` management canister endpoint.
+///
+/// Unlike a fully replicated outcall, which delivers the one response its nodes
+/// agree on, a flexible outcall is performed by a committee of `total_requests`
+/// nodes whose responses may differ and need not all arrive (see
+/// [`CanisterHttpReplication::Flexible`]).
+#[derive(Clone, Serialize, Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
+pub struct MockFlexibleCanisterHttpResponse {
+    pub subnet_id: Principal,
+    pub request_id: u64,
+    /// One response per committee node that responded. Each response is attributed
+    /// to a different committee node.
+    ///
+    /// There may be at most `total_requests` responses; providing fewer models a
+    /// committee whose remaining nodes never respond. With at least `min_responses`
+    /// successful ones among them the outcall still succeeds; with fewer it stays
+    /// pending until the time is advanced past its 60 second timeout.
+    pub responses: Vec<CanisterHttpResponse>,
+}
+
+impl From<RawMockFlexibleCanisterHttpResponse> for MockFlexibleCanisterHttpResponse {
+    fn from(raw_mock_flexible_canister_http_response: RawMockFlexibleCanisterHttpResponse) -> Self {
+        Self {
+            subnet_id: candid::Principal::from_slice(
+                &raw_mock_flexible_canister_http_response.subnet_id.subnet_id,
+            ),
+            request_id: raw_mock_flexible_canister_http_response.request_id,
+            responses: raw_mock_flexible_canister_http_response.responses,
+        }
+    }
+}
+
+impl From<MockFlexibleCanisterHttpResponse> for RawMockFlexibleCanisterHttpResponse {
+    fn from(mock_flexible_canister_http_response: MockFlexibleCanisterHttpResponse) -> Self {
+        Self {
+            subnet_id: RawSubnetId {
+                subnet_id: mock_flexible_canister_http_response
+                    .subnet_id
+                    .as_slice()
+                    .to_vec(),
+            },
+            request_id: mock_flexible_canister_http_response.request_id,
+            responses: mock_flexible_canister_http_response.responses,
+        }
+    }
+}
+
 #[derive(Clone, Serialize, Deserialize, Debug, JsonSchema)]
 pub struct RawCanisterSnapshotDownload {
     pub sender: RawPrincipalId,