Add new version of space-based weighting algorithm

include/os/freebsd/spl/sys/mod.h

@@ -77,6 +77,9 @@
 #define	param_set_active_allocator_args(var) \
     CTLTYPE_STRING, NULL, 0, param_set_active_allocator, "A"
 
+#define	param_set_active_weightfunc_args(var) \
+    CTLTYPE_STRING, NULL, 0, param_set_active_weightfunc, "A"
+
 #define	param_set_deadman_synctime_args(var) \
     CTLTYPE_U64, NULL, 0, param_set_deadman_synctime, "QU"
 

include/sys/metaslab.h

@@ -44,9 +44,13 @@ typedef struct metaslab_ops {
 	uint64_t (*msop_alloc)(metaslab_t *, uint64_t, uint64_t, uint64_t *);
 } metaslab_ops_t;
 
-
 extern const metaslab_ops_t zfs_metaslab_ops;
 
+typedef struct metaslab_wfs {
+	const char *mswf_name;
+	uint64_t (*mswf_func)(metaslab_t *);
+} metaslab_wfs_t;
+
 int metaslab_init(metaslab_group_t *, uint64_t, uint64_t, uint64_t,
     metaslab_t **);
 void metaslab_fini(metaslab_t *);
@@ -103,7 +107,7 @@ void metaslab_trace_init(zio_alloc_list_t *);
 void metaslab_trace_fini(zio_alloc_list_t *);
 
 metaslab_class_t *metaslab_class_create(spa_t *, const char *,
-    const metaslab_ops_t *, boolean_t);
+    const metaslab_ops_t *, const metaslab_wfs_t *, boolean_t);
 void metaslab_class_destroy(metaslab_class_t *);
 void metaslab_class_validate(metaslab_class_t *);
 void metaslab_class_balance(metaslab_class_t *mc, boolean_t onsync);

include/sys/metaslab_impl.h

@@ -77,11 +77,14 @@ typedef enum trace_alloc_type {
 #define	METASLAB_WEIGHT_PRIMARY		(1ULL << 63)
 #define	METASLAB_WEIGHT_SECONDARY	(1ULL << 62)
 #define	METASLAB_WEIGHT_CLAIM		(1ULL << 61)
-#define	METASLAB_WEIGHT_TYPE		(1ULL << 60)
+#define	METASLAB_WEIGHT_MASK		((1ULL << 60) | 1ULL << 59)
 #define	METASLAB_ACTIVE_MASK		\
 	(METASLAB_WEIGHT_PRIMARY | METASLAB_WEIGHT_SECONDARY | \
 	METASLAB_WEIGHT_CLAIM)
 
+#define	METASLAB_WEIGHT_MAX_IDX	58
+#define	METASLAB_WEIGHT_MAX	((1ULL << (METASLAB_WEIGHT_MAX_IDX + 1)) - 1)
+
 /*
  * The metaslab weight is used to encode the amount of free space in a
  * metaslab, such that the "best" metaslab appears first when sorting the
@@ -103,18 +106,30 @@ typedef enum trace_alloc_type {
  *
  *      64      56      48      40      32      24      16      8       0
  *      +-------+-------+-------+-------+-------+-------+-------+-------+
- *      |PSC1|                  weighted-free space                     |
+ *      |PSC10|                   weighted-free space                   |
  *      +-------+-------+-------+-------+-------+-------+-------+-------+
  *
  *	PS - indicates primary and secondary activation
  *	C - indicates activation for claimed block zio
  *	space - the fragmentation-weighted space
  *
+ * Space-based weight v2:
+ *
+ *      64      56      48      40      32      24      16      8       0
+ *      +-------+-------+-------+-------+-------+-------+-------+-------+
+ *      |PSC11|                weighted-free space                | idx |
+ *      +-------+-------+-------+-------+-------+-------+-------+-------+
+ *
+ *	PS - indicates primary and secondary activation
+ *	C - indicates activation for claimed block zio
+ *	idx - index for the highest bucket in the histogram
+ *	space - the fragmentation-weighted space
+ *
  * Segment-based weight:
  *
  *      64      56      48      40      32      24      16      8       0
  *      +-------+-------+-------+-------+-------+-------+-------+-------+
- *      |PSC0| idx|            count of segments in region              |
+ *      |PSC00| idx|            count of segments in region             |
  *      +-------+-------+-------+-------+-------+-------+-------+-------+
  *
  *	PS - indicates primary and secondary activation
@@ -125,17 +140,22 @@ typedef enum trace_alloc_type {
 #define	WEIGHT_GET_ACTIVE(weight)		BF64_GET((weight), 61, 3)
 #define	WEIGHT_SET_ACTIVE(weight, x)		BF64_SET((weight), 61, 3, x)
 
+#define	WEIGHT_GET_TYPE(weight)			BF64_GET((weight), 59, 2)
+#define	WEIGHT_SET_TYPE(weight, x)		BF64_SET((weight), 59, 2, x)
 #define	WEIGHT_IS_SPACEBASED(weight)		\
-	((weight) == 0 || BF64_GET((weight), 60, 1))
-#define	WEIGHT_SET_SPACEBASED(weight)		BF64_SET((weight), 60, 1, 1)
+	((weight) == 0 || WEIGHT_GET_TYPE((weight)))
+#define	WEIGHT_SET_SPACEBASED(weight)		WEIGHT_SET_TYPE((weight), 2)
+#define	WEIGHT_IS_SPACEBASED_V2(weight)		\
+	((weight) == 0 || WEIGHT_GET_TYPE((weight)) == 3)
+#define	WEIGHT_SET_SPACEBASED_V2(weight)	WEIGHT_SET_TYPE((weight), 3)
 
 /*
  * These macros are only applicable to segment-based weighting.
  */
-#define	WEIGHT_GET_INDEX(weight)		BF64_GET((weight), 54, 6)
-#define	WEIGHT_SET_INDEX(weight, x)		BF64_SET((weight), 54, 6, x)
-#define	WEIGHT_GET_COUNT(weight)		BF64_GET((weight), 0, 54)
-#define	WEIGHT_SET_COUNT(weight, x)		BF64_SET((weight), 0, 54, x)
+#define	WEIGHT_GET_INDEX(weight)		BF64_GET((weight), 53, 6)
+#define	WEIGHT_SET_INDEX(weight, x)		BF64_SET((weight), 53, 6, x)
+#define	WEIGHT_GET_COUNT(weight)		BF64_GET((weight), 0, 53)
+#define	WEIGHT_SET_COUNT(weight, x)		BF64_SET((weight), 0, 53, x)
 
 /*
  * Per-allocator data structure.
@@ -183,6 +203,7 @@ struct metaslab_class {
 	spa_t			*mc_spa;
 	const char		*mc_name;
 	const metaslab_ops_t	*mc_ops;
+	const metaslab_wfs_t	*mc_wfs;
 
 	/*
 	 * Track the number of metaslab groups that have been initialized

include/sys/spa.h

@@ -1124,6 +1124,8 @@ extern uint64_t spa_dirty_data(spa_t *spa);
 extern spa_autotrim_t spa_get_autotrim(spa_t *spa);
 extern int spa_get_allocator(spa_t *spa);
 extern void spa_set_allocator(spa_t *spa, const char *allocator);
+extern int spa_get_weightfunc(spa_t *spa);
+extern void spa_set_weightfunc(spa_t *spa, const char *weightfunc);
 
 /* Miscellaneous support routines */
 extern void spa_load_failed(spa_t *spa, const char *fmt, ...)
@@ -1282,6 +1284,7 @@ int param_set_deadman_synctime(ZFS_MODULE_PARAM_ARGS);
 int param_set_slop_shift(ZFS_MODULE_PARAM_ARGS);
 int param_set_deadman_failmode(ZFS_MODULE_PARAM_ARGS);
 int param_set_active_allocator(ZFS_MODULE_PARAM_ARGS);
+int param_set_active_weightfunc(ZFS_MODULE_PARAM_ARGS);
 
 #ifdef ZFS_DEBUG
 #define	dprintf_bp(bp, fmt, ...) do {				\

include/sys/spa_impl.h

@@ -275,6 +275,7 @@ struct spa {
 	spa_allocs_use_t *spa_allocs_use;
 	int		spa_alloc_count;
 	int		spa_active_allocator;	/* selectable allocator */
+	int		spa_active_weightfunc;	/* selectable weight function */
 
 	/* per-allocator sync thread taskqs */
 	taskq_t		*spa_sync_tq;
@@ -501,7 +502,9 @@ extern void spa_set_deadman_synctime(hrtime_t ns);
 extern void spa_set_deadman_ziotime(hrtime_t ns);
 extern const char *spa_history_zone(void);
 extern const char *zfs_active_allocator;
+extern const char *zfs_active_weightfunc;
 extern int param_set_active_allocator_common(const char *val);
+extern int param_set_active_weightfunc_common(const char *val);
 
 #ifdef	__cplusplus
 }

man/man4/zfs.4

@@ -325,6 +325,8 @@ Prevent log spacemaps from being destroyed during pool exports and destroys.
 .
 .It Sy zfs_metaslab_segment_weight_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable/disable segment-based metaslab selection.
+Note that this tunable has been deprecated in favor of
+.Sy zfs_active_weightfunc .
 .
 .It Sy zfs_metaslab_switch_threshold Ns = Ns Sy 2 Pq int
 When using segment-based metaslab selection, continue allocating
@@ -720,6 +722,25 @@ depends on kernel configuration.
 This is the minimum allocation size that will use scatter (page-based) ABDs.
 Smaller allocations will use linear ABDs.
 .
+.It Sy zfs_active_weightfunc Ns = Ns Sy auto Pq string
+Selects the metaslab weighting function to use.
+The weighting function can be one of auto, space, space_v2, and segment.
+This determines what weighting algorithm will be used to sort metaslabs for
+selection during the allocation process.
+The space_v2 and segment algorithms require the spacemap histogram feature to be
+enabled, and will fall back to the space algorithm if that feature is not
+enabled on the pool.
+.Pp
+The space algorithm combines the total free space in the metaslab with the
+fragmentation metric to balance toward metaslabs with more contiguous free
+space.
+The segment algorithm uses the spacemap histograms to calculate the weight from
+the largest free segment size, bucketed to powers of two, and the number of
+segments in that bucket.
+The space_v2 algorithm considers not only the largest free segment bucket, but
+the smaller ones as well, providing a higher weight to larger contiguous chunks.
+Auto provides the recommended algorithm (currently space_v2).
+.
 .It Sy zfs_arc_dnode_limit Ns = Ns Sy 0 Ns B Pq u64
 When the number of bytes consumed by dnodes in the ARC exceeds this number of
 bytes, try to unpin some of it in response to demand for non-metadata.

module/os/freebsd/zfs/sysctl_os.c

@@ -273,6 +273,24 @@ param_set_active_allocator(SYSCTL_HANDLER_ARGS)
 	return (param_set_active_allocator_common(buf));
 }
 
+int
+param_set_active_weightfunc(SYSCTL_HANDLER_ARGS)
+{
+	char buf[16];
+	int rc;
+
+	if (req->newptr == NULL)
+		strlcpy(buf, zfs_active_weightfunc, sizeof (buf));
+
+	rc = sysctl_handle_string(oidp, buf, sizeof (buf), req);
+	if (rc || req->newptr == NULL)
+		return (rc);
+	if (strcmp(buf, zfs_active_weightfunc) == 0)
+		return (0);
+
+	return (param_set_active_weightfunc_common(buf));
+}
+
 /*
  * In pools where the log space map feature is not enabled we touch
  * multiple metaslabs (and their respective space maps) with each

module/os/linux/zfs/spa_misc_os.c

@@ -116,6 +116,18 @@ param_set_active_allocator(const char *val, zfs_kernel_param_t *kp)
 	return (error);
 }
 
+int
+param_set_active_weightfunc(const char *val, zfs_kernel_param_t *kp)
+{
+	int error;
+
+	error = -param_set_active_weightfunc_common(val);
+	if (error == 0)
+		error = param_set_charp(val, kp);
+
+	return (error);
+}
+
 const char *
 spa_history_zone(void)
 {