btrfs-progs: cmds/qgroup: use sysfs to detect inconsistent status early

.github/workflows/codeql.yml

@@ -0,0 +1,105 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "devel" ]
+  pull_request:
+    branches: [ "devel" ]
+  schedule:
+    - cron: '37 22 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: c-cpp
+          build-mode: manual
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Add any setup steps before running the `github/codeql-action/init` action.
+    # This includes steps like installing compilers or runtimes (`actions/setup-node`
+    # or others). This is typically only required for manual builds.
+    # - name: Setup runtime (example)
+    #   uses: actions/setup-example@v1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    - name: Update package cache
+      run: sudo apt-get update -qqq
+    - name: Add more build dependencies
+      run: sudo apt-get install -y liblzo2-dev libzstd-dev libblkid-dev uuid-dev zlib1g-dev libext2fs-dev e2fsprogs libudev-dev
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        ./autogen.sh
+        ./configure --disable-documentation --enable-experimental
+        make -j 4
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/codespell.yml

@@ -1,5 +1,7 @@
 # Check for typos on devel and pull rquests
 # - codespell configuration is within .codespellrc
+# - the workflow can be run separately from other workflows that trigger on
+#   ci/* branches, use codespell/* as the prefix
 
 name: Codespell
 run-name: Codespell
@@ -9,12 +11,11 @@ on:
     branches:
       - devel
       - 'ci/**'
-      - 'CI/**'
+      - 'codespell/**'
   pull_request:
     branches:
       - devel
       - 'ci/**'
-      - 'CI/**'
 
 permissions:
   contents: read

Documentation/DocConventions.rst

@@ -67,7 +67,7 @@ Referencing:
 
 -  generic links can use the free form link syntax with description ```Link text <https://example.com>`__``
    (note the double underscore, this is *anonymous* link and does not create a reference)
-   or plain link that will auto-render to a clikable link https://example.com
+   or plain link that will auto-render to a clickable link https://example.com
 
 -  in manual pages: always use full link as it's meant to be read in terminal
    output and must allow copy&paste

Documentation/Kernel-by-version.rst

@@ -853,7 +853,7 @@ Pull requests:
 `v5.4-rc7 <https://git.kernel.org/linus/00aff6836241ae5654895dcea10e6d4fc5878ca6>`__,
 `v5.4-rc8 <https://git.kernel.org/linus/afd7a71872f14062cc12cac126bb8e219e7dacf6>`__
 
-- tree checker: adde sanity checks for tree items, extent items, and references
+- tree checker: added sanity checks for tree items, extent items, and references
 - deprecated subvolume creation mode BTRFS_SUBVOL_CREATE_ASYNC
 - qgroup relation deletion tries harder, orphan entries are removed too
 - space handling improvements (ticket reservations, flushing, overcommit logic)
@@ -1023,7 +1023,7 @@ Pull requests:
 Highlights:
 
 - performance improvements in fsync (dbench workload: higher throughput, lower latency)
-- sysfs exports current exclusive operataion (balance, resize, device add/del/...)
+- sysfs exports current exclusive operation (balance, resize, device add/del/...)
 - sysfs exports supported send stream version
 
 Core:
@@ -1083,7 +1083,7 @@ Features:
 
 Core changes:
 
-- subpage block size support peparations
+- subpage block size support preparations
 
 Fixes:
 
@@ -1434,7 +1434,7 @@ Qgroup:
 - limits are shared upon snapshot
 - allow to remove qgroup which has parent but no child
 - fix status of qgroup consistency after rescan
-- fix quota status bits after dsiabling
+- fix quota status bits after disabling
 - mark qgroups inconsistent after assign/delete actions
 - code cleanups
 
@@ -1475,7 +1475,7 @@ Fixes:
 ^^^^^^^^^^^^^^
 
 - send fixes: cloning, sending with parent
-- improved handling of framgented space using bitmaps
+- improved handling of fragmented space using bitmaps
 - new mount option for debugging: fragment=data|metadata|all
 - updated balance filters: limit, stripes, usage
 - more bugfixes and cleanups
@@ -1501,7 +1501,7 @@ Fixes:
     writes (now does: nologreplay)
 
 - default inline limit is now 2048 (instead of page size, usually 4096)
-- /dev/btrfs-control now understands the GET_SUPPORTE_FEATURES ioctl
+- /dev/btrfs-control now understands the GET_SUPPORTED_FEATURES ioctl
 - get rid of harmless message "''could not find root %llu''"
 - preparatory work for subpage-blocksize patchset
 - fix bug when using overlayfs
@@ -1574,7 +1574,7 @@ Fixes:
 ^^^^^^^^^^^^^^^
 
 - new tracepoints: file item
-- fix qgoup accounting when inode_cache is in use
+- fix qgroup accounting when inode_cache is in use
 - fix incorrect number report in stat::t_blocks under certain conditions
 - raid56 fixes:
 
@@ -1877,7 +1877,7 @@ Fixes:
 ^^^^^^^^^^^^^^^
 
 * ''fiemap'' exports information about shared extents
-* bugfix and stability foucsed release
+* bugfix and stability focused release
 
 3.14 (Mar 2014)
 ^^^^^^^^^^^^^^^

Documentation/Send-receive.rst

@@ -22,4 +22,4 @@ truncate), whole file operations (rename, delete). The stream can be sent over
 network, piped directly to the receive command or saved to a file. Each command
 in the stream is protected by a CRC32C checksum, with 0 as the initial value
 and no inversion. See :doc:`btrfs-send` and :doc:`btrfs-receive` for more,
-for protocol description :doc:`dev-send-stream`.
+for protocol description :doc:`dev/dev-send-stream`.

Documentation/btrfs-filesystem.rst

@@ -140,7 +140,7 @@ defragment [options] <file>|<dir> [<file>|<dir>...]
                 logic. Reasonable values are from tens to hundreds of megabytes.
 
         --step SIZE
-                Perform defragmention in the range in SIZE steps and flush (*-f*) after each one.
+                Perform defragmentation in the range in SIZE steps and flush (*-f*) after each one.
                 The range is default (the whole file) or given by *-s* and *-l*, split into
                 the steps or done in one go if the step is larger. Minimum range size is 256KiB.
 

Documentation/btrfs-ioctl.rst

@@ -550,7 +550,7 @@ BTRFS_IOC_ADD_DEV
 
 Add a given block device to the filesystem. Unlike the command :command:`btrfs device add`
 there's are no safety checks (like existence of another filesystem on the
-device), device preparataion (like TRIM or zone reset), so use it with care.
+device), device preparation (like TRIM or zone reset), so use it with care.
 
 This is a filesystem-exclusive operation and it will fail if there's another
 one already running, with one exception, when there's a paused balance.
@@ -968,7 +968,7 @@ Use cases (:ref:`definition of constants<struct_btrfs_ioctl_subvol_wait>`):
    * - ioctl args
      - :ref:`struct btrfs_ioctl_subvol_wait<struct_btrfs_ioctl_subvol_wait>`
    * - args.subvolid
-     - Depending on the mode, the numeric id of subolume to wait for, or
+     - Depending on the mode, the numeric id of subvolume to wait for, or
        the one queried by *PEEK* modes
    * - args.mode
      - mode of operation described above

Documentation/btrfs-man5.rst

@@ -220,7 +220,7 @@ SYSFS INTERFACE
 
 .. include:: ch-sysfs.rst
 
-.. _man-btrfs5-fileysstem-exclusive-operations:
+.. _man-btrfs5-filesystem-exclusive-operations:
 
 FILESYSTEM EXCLUSIVE OPERATIONS
 -------------------------------

Documentation/btrfs-qgroup.rst

@@ -31,6 +31,12 @@ ownership. For example a fresh snapshot shares almost all the blocks with the
 original subvolume, new writes to either subvolume will raise towards the
 exclusive limit.
 
+.. note::
+   Qgroup limit only works when qgroup is in a consistent state.
+   If some workload marks qgroup inconsistent (like assigning a qgroup to another
+   qgroup), the limit will no longer work until the inconsistent flag is cleared
+   by :command:`btrfs quota rescan`.
+
 The qgroup identifiers conform to *level/id* where level 0 is reserved to the
 qgroups associated with subvolumes. Such qgroups are created automatically.
 

Documentation/ch-balance-filters.rst

@@ -79,10 +79,26 @@ convert=<profile>
                 Starting with kernel 4.5, the ``data`` chunks can be converted to/from the
                 ``DUP`` profile on a single device.
 
-        .. note::
                 Starting with kernel 4.6, all profiles can be converted to/from ``DUP`` on
                 multi-device filesystems.
 
+	.. warning::
+                Bad or missing device are not detected immediately during
+                runtime and this depends on some later event like failed write
+                or failed transaction commit. If there's a known failing
+                device, or a device deleted by :file:`/sys/block/<dev>/device/delete` interface,
+                the device will be still accessed and written to.
+
+                In such case, one should not convert to a profile with lower
+                redundancy (e.g. from *RAID1* to *SINGLE*),
+                as attempts to create new chunks on the new devices will cause
+                various problems.
+
+                The proper action is to use :command:`btrfs replace` or
+                :command:`btrfs device remove` to handle the failing/missing
+                device first. Then convert will work with all devices
+                correctly.
+
 limit=<number>, limit=<range>
         Process only given number of chunks, after all filters are applied. This can be
         used to specifically target a chunk in connection with other filters (``drange``,

Documentation/ch-sysfs.rst

@@ -5,26 +5,31 @@ The top level path is :file:`/sys/fs/btrfs/`, and the main directory layout is t
 =============================  ===================================  ========
 Relative Path                  Description                          Version
 =============================  ===================================  ========
-features/                      All supported features               3.14+
-<UUID>/                        Mounted fs UUID                      3.14+
-<UUID>/allocation/             Space allocation info                3.14+
-<UUID>/features/               Features of the filesystem           3.14+
-<UUID>/devices/<DEVID>/        Symlink to each block device sysfs   5.6+
-<UUID>/devinfo/<DEVID>/        Btrfs specific info for each device  5.6+
-<UUID>/qgroups/                Global qgroup info                   5.9+
-<UUID>/qgroups/<LEVEL>_<ID>/   Info for each qgroup                 5.9+
-<UUID>/discard/                Discard stats and tunables           6.1+
+features/                      All supported features               3.14
+<UUID>/                        Mounted fs UUID                      3.14
+<UUID>/allocation/             Space allocation info                3.14
+<UUID>/bdi/                    Backing device info (writeback)      5.9
+<UUID>/devices/<DEVID>/        Symlink to each block device sysfs   5.6
+<UUID>/devinfo/<DEVID>/        Btrfs specific info for each device  5.6
+<UUID>/discard/                Discard stats and tunables           6.1
+<UUID>/features/               Features of the filesystem           3.14
+<UUID>/qgroups/                Global qgroup info                   5.9
+<UUID>/qgroups/<LEVEL>_<ID>/   Info for each qgroup                 5.9
 =============================  ===================================  ========
 
 For :file:`/sys/fs/btrfs/features/` directory, each file means a supported feature
-for the current kernel.
+of the current kernel. Most files have value 0. Otherwise it depends on the file,
+value *1* typically means the feature can be turned on a mounted filesystem.
 
 For :file:`/sys/fs/btrfs/<UUID>/features/` directory, each file means an enabled
-feature for the mounted filesystem.
+feature on the mounted filesystem.
 
-The features shares the same name in section
+The features share the same name in section
 :ref:`FILESYSTEM FEATURES<man-btrfs5-filesystem-features>`.
 
+UUID
+^^^^
+
 Files in :file:`/sys/fs/btrfs/<UUID>/` directory are:
 
 bg_reclaim_threshold
@@ -72,7 +77,7 @@ exclusive_operation
 
         Shows the running exclusive operation.
         Check section
-        :ref:`FILESYSTEM EXCLUSIVE OPERATIONS<man-btrfs5-fileysstem-exclusive-operations>`
+        :ref:`FILESYSTEM EXCLUSIVE OPERATIONS<man-btrfs5-filesystem-exclusive-operations>`
         for details.
 
 generation
@@ -88,7 +93,7 @@ label
 metadata_uuid
         (RO, since: 5.0)
 
-        Shows the metadata uuid of the mounted filesystem.
+        Shows the metadata UUID of the mounted filesystem.
         Check `metadata_uuid` feature for more details.
 
 nodesize
@@ -107,20 +112,24 @@ read_policy
         (RW, since: 5.11)
 
         Shows the current balance policy for reads.
-        Currently only "pid" (balance using pid value) is supported.
+        Currently only ``pid`` (balance using the process id (pid) value) is
+        supported. More balancing policies are available in experimental
+        build, namely round-robin.
 
 sectorsize
         (RO, since: 3.14)
 
         Shows the sectorsize of the mounted filesystem.
 
-
 temp_fsid
         (RO, since 6.7)
 
         Indicate that this filesystem got assigned a temporary FSID at mount time,
         making possible to mount devices with the same FSID.
 
+UUID/allocations
+^^^^^^^^^^^^^^^^
+
 Files and directories in :file:`/sys/fs/btrfs/<UUID>/allocations` directory are:
 
 global_rsv_reserved
@@ -136,8 +145,10 @@ global_rsv_size
 `data/`, `metadata/` and `system/` directories
         (RO, since: 5.14)
 
-        Space info accounting for the 3 chunk types.
-        Mostly for debug purposes.
+        Space info accounting for the 3 block group types.
+
+UUID/allocations/{data,metadata,system}
+"""""""""""""""""""""""""""""""""""""""
 
 Files in :file:`/sys/fs/btrfs/<UUID>/allocations/{data,metadata,system}` directory are:
 
@@ -152,13 +163,20 @@ bytes_*
         (RO)
 
         Values of the corresponding data structures for the given block group
-        type and profile.
+        type and profile that are used internally and may change rapidly depending
+        on the load.
+
+        Complete list: bytes_may_use, bytes_pinned, bytes_readonly,
+        bytes_reserved, bytes_used, bytes_zone_unusable
 
 chunk_size
         (RW, since: 6.0)
 
-        Shows the chunk size. Can be changed for data and metadata.
-        Cannot be set for zoned devices.
+        Shows the chunk size. Can be changed for data and metadata (independently)
+        and cannot be set for system block group type.
+        Cannot be set for zoned devices as it depends on the fixed device zone size.
+        Upper bound is 10% of the filesystem size, the value must be multiple of 256MiB
+        and greater than 0.
 
 size_classes
         (RO, since: 6.3)
@@ -173,6 +191,27 @@ size_classes
                 medium 282
                 large 93
 
+UUID/bdi
+^^^^^^^^
+
+Symlink to the sysfs directory of the backing device info (BDI), which is
+related to writeback process and infrastructure.
+
+UUID/devices
+^^^^^^^^^^^^
+
+Files in :file:`/sys/fs/btrfs/<UUID>/devices` directory are symlinks named
+after device nodes (e.g. sda, dm-0) and pointing to their sysfs directory.
+
+UUID/devinfo
+^^^^^^^^^^^^
+
+The directory contains subdirectories named after device ids (numeric values). Each
+subdirectory has information about the device of the given *devid*.
+
+UUID/devinfo/DEVID
+""""""""""""""""""
+
 Files in :file:`/sys/fs/btrfs/<UUID>/devinfo/<DEVID>` directory are:
 
 error_stats:
@@ -192,44 +231,47 @@ fsid:
         (RO, since: 5.17)
 
         Shows the fsid which the device belongs to.
-        It can be different than the `<UUID>` if it's a seed device.
+        It can be different than the ``UUID`` if it's a seed device.
 
 in_fs_metadata
         (RO, since: 5.6)
 
         Shows whether we have found the device.
-        Should always be 1, as if this turns to 0, the `<DEVID>` directory
+        Should always be 1, as if this turns to 0, the :file:`DEVID` directory
         would get removed automatically.
 
 missing
         (RO, since: 5.6)
 
-        Shows whether the device is missing.
+        Shows whether the device is considered missing by the kernel module.
 
 replace_target
         (RO, since: 5.6)
 
         Shows whether the device is the replace target.
-        If no dev-replace is running, this value should be 0.
+        If no device replace is running, this value is 0.
 
 scrub_speed_max
         (RW, since: 5.14)
 
         Shows the scrub speed limit for this device. The unit is Bytes/s.
-        0 means no limit. The value can be set but is not persisent.
+        0 means no limit. The value can be set but is not persistent.
 
 writeable
         (RO, since: 5.6)
 
         Show if the device is writeable.
 
+UUID/qgroups
+^^^^^^^^^^^^
+
 Files in :file:`/sys/fs/btrfs/<UUID>/qgroups/` directory are:
 
 enabled
         (RO, since: 6.1)
 
         Shows if qgroup is enabled.
-        Also, if qgroup is disabled, the `qgroups` directory would
+        Also, if qgroup is disabled, the :file:`qgroups` directory will
         be removed automatically.
 
 inconsistent
@@ -249,12 +291,17 @@ drop_subtree_threshold
         we will not trigger qgroup account at all, but mark qgroup inconsistent to
         avoid the huge workload.
 
-        Default value is 8, where no subtree drop can trigger qgroup.
+        Default value is 3, which means that trees of low height will be accounted
+        properly as this is sufficiently fast. The value was 8 until 6.13 where
+        no subtree drop can trigger qgroup rescan making it less useful.
 
         Lower value can reduce qgroup workload, at the cost of extra qgroup rescan
         to re-calculate the numbers.
 
-Files in :file:`/sys/fs/btrfs/<UUID>/<LEVEL>_<ID>/` directory are:
+UUID/qgroups/LEVEL_ID
+"""""""""""""""""""""
+
+Files in each :file:`/sys/fs/btrfs/<UUID>/qgroups/<LEVEL>_<ID>/` directory are:
 
 exclusive
         (RO, since: 5.9)
@@ -297,6 +344,9 @@ rsv_meta_prealloc
 
         Shows the reserved bytes for preallocated metadata.
 
+UUID/discard
+^^^^^^^^^^^^
+
 Files in :file:`/sys/fs/btrfs/<UUID>/discard/` directory are:
 
 discardable_bytes

Documentation/dev/CmdLineConventions.rst

@@ -58,7 +58,7 @@ or change descriptions.
 
 *  brief but descriptive
 *  long and descriptive can be used in justified cases (e.g. conversion options
-   in :doc:`btrfstune` because of the single command without subcommands)
+   in :doc:`../btrfstune` because of the single command without subcommands)
 
 Option parameters
 ^^^^^^^^^^^^^^^^^

Documentation/dev/Developer-s-FAQ.rst

@@ -64,7 +64,7 @@ candidate for a CC: of the bugreport.
 
 **Do**: talk to the maintainer if he forgot to add this tag to the final patch.
 Reviews do take time and the patches land in various branches early after
-they're sent to the mailingslist for testing, but the reviews are always
+they're sent to the mailinglist for testing, but the reviews are always
 welcome.
 
 **Do**: collect the Reviewed-by tags for patches that get resent unchanged e.g.
@@ -236,7 +236,7 @@ Good practices, contribution hints
 -  be patient if your patch is not accepted immediately, try to send a gentle
    ping if there's a significant time without any action
 -  if you want to start contributing but are not sure about how to do that,
-   lurk in the mailingist or on the IRC channel
+   lurk in the mailinglist or on the IRC channel
 -  every patch should implement one thing -- this is vaguely defined, you may
    receive comments about patch splitting or merging with other
 -  every patch must be compilable when applied, possibly with all related
@@ -451,7 +451,7 @@ copyright holders of changes in a given file. The code is usually heavily
 changed over time in smaller portions, slowly morphing into something that does
 not resemble the original code anymore though it shares a lot of the core ideas
 and implemented logic.  A copyright notice by a company that does not exist
-anymore from 10 years ago is a clear example of uselesness for the developers.
+anymore from 10 years ago is a clear example of uselessness for the developers.
 
 When code is moved verbatim from a file to another file, in the new file it
 appears to be contributed by a single author while it is in most cases code

Documentation/dev/Development-notes.rst

@@ -120,7 +120,7 @@ There are several ways how to react to the unexpected conditions:
    This would cause the filesystem to be flipped read-only to prevent further
    corruption.
 
-   Additionally call trace would be dumpped for the first btrfs_abort_transaction()
+   Additionally call trace would be dumped for the first btrfs_abort_transaction()
    call site.
 
 -  ASSERT()

Documentation/dev/dev-btrees.rst

@@ -114,7 +114,7 @@ Log Root Tree
 UUID Tree
 ---------
 
-The tree storesc correspondence between UUIDs and subvolumes. Used for
+The tree stores correspondence between UUIDs and subvolumes. Used for
 quick lookup during send.
 
 

Documentation/dev/dev-json.rst

@@ -3,12 +3,17 @@ JSON output
 
 Supported types:
 
+* any: a valid *printf* format, parameters to *fmt_print()* must match the number (and are not validated)
 * number: ``%llu``
 * string: ``%s``
-* string: ``str`` (escaped special characters)
-* bool: ``bool`` (unquoted native json value)
-* qgroupid: ``qgroupid`` (split to 48/16 for level/subvolid)
-* size: ``size``
+* string: ``str`` - backslash escaped special characters (0x08, 0x08, 0x0a, 0x0c, 0x0d, 0x0b),
+  the rest of range from *0x00* to *0x1f* as *\\uXXXX* encoding
+* bool: ``bool`` - unquoted native json values *true* or *false*
+* qgroupid: ``qgroupid`` - split to 48/16 for level/subvolid
+* size: ``size`` - size with SI/IEC size suffix
+* size: ``size-or-none`` - same as *size* but for 0 it's *none*
+* UUID: ``uuid`` - if all zeros then *null* (native json), or properly formatted UUID string
+* date + time: ``date-time`` - timestamp formatted as *YYYY-MM-DD HH:MM:SS TIMEZONE*
 
 Commands that support json output
 ---------------------------------
@@ -23,16 +28,24 @@ Commands that support json output
 Recommendations
 ---------------
 
+Keys and formatting are defined as an array of *struct rowspec*.
+
 * key names
-   * should be unified if they mean the same thing
+   * should be unified with the printed value if they mean the same thing
    * not abbreviated (e.g. *generation* instead of *gen*)
    * referring to existing and well known names (qgroupid, devid, ...)
+   * using ``-`` as word separator, or ``_`` if it's better to keep the same name of the value
 * values
    * numbers without suffix or other transformation, i.e. no *KiB*
+   * formatted by the types if possible
+   * any *printf* format is possible but should be avoided or a new type should
+     be defined
 * printing more data about an item is better than printing less, assuming the
   filtering is done on the user side
 * structure of json output may not reflect the way it's printed in plain text,
   in that case do two separate printer functions
 * if plain and json output roughly follow the same style, e.g. line oriented
   that is easy to transform to a map, then both outputs should use the same
   rowspec
+* one value can be printed by multiple rowspecs that may do different
+  formatting depending on the context

Documentation/mkfs.btrfs.rst

@@ -150,6 +150,9 @@ OPTIONS
         Populate the toplevel subvolume with files from *rootdir*.  This does not
         require root permissions to write the new files or to mount the filesystem.
 
+        Directories can be created as subvolumes, see also option *--subvol*.
+        Hardlinks are detected and created in the filesystem image.
+
         .. note::
                 This option may enlarge the image or file to ensure it's big enough to
                 contain the files from *rootdir*. Since version 4.14.1 the filesystem size is
@@ -160,6 +163,10 @@ OPTIONS
         *no* (the default), *zstd*, *lzo* or *zlib*.  The optional value *level* is a
         compression level, 1..15 for *zstd*, 1..9 for *zlib*.
 
+        It is recommended to use the highest level to achieve maximum space savings.
+        Compression at mkfs time is not as constrained as in kernel where it's
+        desirable to use the less CPU load. Otherwise the default level is 3.
+
         As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when
         they would be larger than the uncompressed versions, and will set file attribute
         *NOCOMPRESS* if its beginning is found to be incompressible.
@@ -173,7 +180,7 @@ OPTIONS
         directory.  The option *--rootdir* must also be specified, and *subdir* must be an
         existing subdirectory within it.  This option can be specified multiple times.
 
-        *type* is an optional additional modifier. Valid choices are:
+        The *type* is an optional additional modifier. Valid choices are:
 
         * *default*: create as default subvolume
         * *ro*: create as read-only subvolume
@@ -183,24 +190,26 @@ OPTIONS
         Only one of *default* and *default-ro* may be specified.
 
         If you wish to create a subvolume with a name containing a colon and you don't
-        want this to be parsed as containing a modifier, you can prefix the path with `./`:
+        want this to be parsed as containing a modifier, you can prefix the path with :file:`./`:
 
         .. code-block:: bash
 
                 $ mkfs.btrfs --rootdir dir --subvol ./ro:subdir /dev/loop0
 
-	If there are hard links inside *rootdir* and *subdir* will split the
-	subvolumes, like the following case::
+        If there are hardlinks inside *rootdir* and *subdir* will split the
+        subvolumes, like the following case:
+
+        .. code-block:: none
 
 		rootdir/
 		|- hardlink1
 		|- hardlink2
 		|- subdir/  <- will be a subvolume
 		   |- hardlink3
 
-	In that case we cannot create `hardlink3` as hardlinks of
-	`hardlink1` and `hardlink2` because hardlink3 will be inside a new
-	subvolume.
+        In that case we cannot create :file:`hardlink3` as hardlinks of
+        :file:`hardlink1` and :file:`hardlink2` because :file:`hardlink3` will
+        be inside a new subvolume.
 
 --shrink
         Shrink the filesystem to its minimal size, only works with *--rootdir* option.
@@ -605,7 +614,7 @@ hopefully not destroying both copies of particular data in case of DUP.
 
 The wear levelling techniques can also lead to reduced redundancy, even if the
 device does not do any deduplication. The controllers may put data written in
-a short timespan into the same physical storage unit (cell, block etc). In case
+a short time span into the same physical storage unit (cell, block etc). In case
 this unit dies, both copies are lost. BTRFS does not add any artificial delay
 between metadata writes.
 

README.md

@@ -6,6 +6,7 @@ Btrfs-progs
 [![codecov](https://codecov.io/gh/kdave/btrfs-progs/branch/coverage-test/graph/badge.svg?token=fhLI8V9s0k)](https://codecov.io/gh/kdave/btrfs-progs)
 [![static](https://github.com/kdave/btrfs-progs/actions/workflows/artifacts-static-build.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/artifacts-static-build.yml)
 [![release](https://github.com/kdave/btrfs-progs/actions/workflows/ci-build-test.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/ci-build-test.yml)
+[![codeql](https://github.com/kdave/btrfs-progs/actions/workflows/codeql.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/codeql.yml)
 
 Userspace utilities to manage btrfs filesystems.
 License: GPLv2.

ci/images/ci-centos-7-x86_64/Dockerfile

@@ -29,7 +29,7 @@ COPY ./test-build .
 COPY ./run-tests .
 COPY ./devel.tar.gz .
 
-CMD ./test-build devel --disable-documentation --disable-libudev
+CMD ["./test-build", "devel", "--disable-documentation", "--disable-libudev"]
 
 # Continue with:
 # cd /tmp

ci/images/ci-centos-8-x86_64/Dockerfile

@@ -32,7 +32,7 @@ COPY ./test-build .
 COPY ./run-tests .
 COPY ./devel.tar.gz .
 
-CMD ./test-build devel --disable-documentation --disable-libudev --disable-zoned
+CMD ["./test-build", "devel", "--disable-documentation", "--disable-libudev", "--disable-zoned"]
 
 # Continue with:
 # cd /tmp

ci/images/ci-musl-i386/Dockerfile

@@ -16,4 +16,4 @@ RUN apk add wget
 COPY ./test-build .
 COPY ./devel.tar.gz .
 
-CMD ./test-build devel --disable-documentation --disable-backtrace --disable-libudev
+CMD ["./test-build", "devel", "--disable-documentation", "--disable-backtrace", "--disable-libudev"]

ci/images/ci-musl-x86_64/Dockerfile

@@ -16,4 +16,4 @@ RUN apk add wget
 COPY ./test-build .
 COPY ./devel.tar.gz .
 
-CMD ./test-build devel --disable-documentation --disable-backtrace --disable-libudev
+CMD ["./test-build", "devel", "--disable-documentation", "--disable-backtrace", "--disable-libudev"]

ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile

@@ -26,7 +26,7 @@ COPY ./run-tests .
 COPY ./devel.tar.gz .
 
 # The blkzoned.h exists but blk_zone.capacity is missing, disable zoned mode explicitly
-CMD ./test-build devel --disable-documentation --disable-zoned
+CMD ["./test-build", "devel", "--disable-documentation", "--disable-zoned"]
 
 # Continue with:
 # cd /tmp

ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile

@@ -26,7 +26,7 @@ COPY ./run-tests .
 COPY ./devel.tar.gz .
 
 # The blkzoned.h exists but blk_zone.capacity is missing, disable zoned mode explicitly
-CMD ./test-build devel --disable-documentation --disable-zoned
+CMD ["./test-build", "devel", "--disable-documentation", "--disable-zoned"]
 
 # Continue with:
 # cd /tmp

ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile

@@ -33,7 +33,7 @@ COPY ./test-build .
 COPY ./run-tests .
 COPY ./devel.tar.gz .
 
-CMD ./test-build devel --disable-documentation
+CMD ["./test-build", "devel", "--disable-documentation"]
 
 # Continue with:
 # cd /tmp

ci/images/ci-rockylinux-9-x86_64/Dockerfile

@@ -29,7 +29,7 @@ COPY ./test-build .
 COPY ./run-tests .
 COPY ./devel.tar.gz .
 
-CMD ./test-build devel --disable-documentation
+CMD ["./test-build", "devel", "--disable-documentation"]
 
 # Continue with:
 # cd /tmp

cmds/qgroup.c

@@ -36,6 +36,7 @@
 #include "kernel-shared/uapi/btrfs.h"
 #include "kernel-shared/ctree.h"
 #include "common/open-utils.h"
+#include "common/sysfs-utils.h"
 #include "common/utils.h"
 #include "common/help.h"
 #include "common/units.h"
@@ -1610,6 +1611,33 @@ static void print_all_qgroups_json(struct qgroup_lookup *qgroup_lookup)
 	fmt_end(&fctx);
 }
 
+/*
+ * Tree search based inconsistent flag is only updated at transaction commit
+ * time.
+ * Thus even if the qgroup_status flag shows consistent, the qgroup may already
+ * be in an inconsistent status.
+ */
+static void check_qgroup_sysfs_inconsistent(int fd,
+		const struct qgroup_lookup *qgroup_lookup)
+{
+	u64 value;
+	int sysfs_fd;
+	int ret;
+
+	if (qgroup_lookup->flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)
+		return;
+	sysfs_fd = sysfs_open_fsid_file(fd, "qgroups/inconsistent");
+	if (fd < 0)
+		return;
+	ret = sysfs_read_fsid_file_u64(fd, "qgroups/inconsistent", &value);
+	if (ret < 0)
+		goto out;
+	if (value)
+		warning("qgroup data inconsistent, rescan recommended");
+out:
+	close(sysfs_fd);
+}
+
 static int show_qgroups(int fd,
 		       struct btrfs_qgroup_filter_set *filter_set,
 		       struct btrfs_qgroup_comparer_set *comp_set)
@@ -1622,6 +1650,8 @@ static int show_qgroups(int fd,
 	ret = qgroups_search_all(fd, &qgroup_lookup);
 	if (ret)
 		return ret;
+
+	check_qgroup_sysfs_inconsistent(fd, &qgroup_lookup);
 	__filter_and_sort_qgroups(&qgroup_lookup, &sort_tree,
 				  filter_set, comp_set);
 	if (bconf.output_format == CMD_FORMAT_JSON)

cmds/scrub.c

@@ -204,9 +204,10 @@ static void print_scrub_summary(struct btrfs_scrub_progress *p, struct scrub_sta
 		pr_verbose(LOG_DEFAULT, "Total to scrub:   %s\n",
 			pretty_size_mode(bytes_total, unit_mode));
 	}
+
 	/*
 	 * Rate and size units are disproportionate so they are affected only
-	 * by --raw, otherwise it's human readable
+	 * by --raw, otherwise it's human readable (respecting the SI or IEC mode).
 	 */
 	if (unit_mode == UNITS_RAW) {
 		pr_verbose(LOG_DEFAULT, "Rate:             %s/s",
@@ -218,11 +219,16 @@ static void print_scrub_summary(struct btrfs_scrub_progress *p, struct scrub_sta
 			pr_verbose(LOG_DEFAULT, " (some device limits set)");
 		pr_verbose(LOG_DEFAULT, "\n");
 	} else {
+		unsigned int mode = UNITS_HUMAN_DECIMAL;
+
+		if (unit_mode & UNITS_BINARY)
+			mode = UNITS_HUMAN_BINARY;
+
 		pr_verbose(LOG_DEFAULT, "Rate:             %s/s",
-			pretty_size_mode(bytes_per_sec, unit_mode));
+			pretty_size_mode(bytes_per_sec, mode));
 		if (limit > 1)
 			pr_verbose(LOG_DEFAULT, " (limit %s/s)",
-				   pretty_size_mode(limit, unit_mode));
+				   pretty_size_mode(limit, mode));
 		else if (limit == 1)
 			pr_verbose(LOG_DEFAULT, " (some device limits set)");
 		pr_verbose(LOG_DEFAULT, "\n");

common/units.h

@@ -31,9 +31,9 @@
 #define UNITS_TBYTES			(5)
 #define UNITS_RAW			(1U << UNITS_MODE_SHIFT)
 #define UNITS_BINARY			(2U << UNITS_MODE_SHIFT)
-#define UNITS_DECIMAL			(3U << UNITS_MODE_SHIFT)
+#define UNITS_DECIMAL			(4U << UNITS_MODE_SHIFT)
 /* Interpret the u64 value as s64 */
-#define UNITS_NEGATIVE			(4U << UNITS_MODE_SHIFT)
+#define UNITS_NEGATIVE			(8U << UNITS_MODE_SHIFT)
 #define UNITS_MODE_MASK			((1U << UNITS_MODE_SHIFT) - 1)
 #define UNITS_MODE_SHIFT		(8)
 #define UNITS_HUMAN_BINARY		(UNITS_BINARY)

kernel-shared/ctree.h

@@ -831,6 +831,16 @@ static inline u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
 	return val;
 }
 
+static inline void btrfs_set_dev_stats_value(struct extent_buffer *eb,
+					     struct btrfs_dev_stats_item *ptr,
+					     int index, u64 val)
+{
+	write_extent_buffer(eb, &val,
+			    offsetof(struct btrfs_dev_stats_item, values) +
+			     ((unsigned long)ptr) + (index * sizeof(u64)),
+			    sizeof(val));
+}
+
 /* struct btrfs_ioctl_search_header */
 static inline u64 btrfs_search_header_transid(struct btrfs_ioctl_search_header *sh)
 {

kernel-shared/volumes.c

@@ -1135,6 +1135,7 @@ int btrfs_add_device(struct btrfs_trans_handle *trans,
 	int ret;
 	struct btrfs_path *path;
 	struct btrfs_dev_item *dev_item;
+	struct btrfs_dev_stats_item *dev_stats;
 	struct extent_buffer *leaf;
 	struct btrfs_key key;
 	struct btrfs_root *root = fs_info->chunk_root;
@@ -1149,6 +1150,7 @@ int btrfs_add_device(struct btrfs_trans_handle *trans,
 	if (ret)
 		goto out;
 
+	/* Add DEV_ITEM. */
 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
 	key.type = BTRFS_DEV_ITEM_KEY;
 	key.offset = free_devid;
@@ -1182,6 +1184,27 @@ int btrfs_add_device(struct btrfs_trans_handle *trans,
 			    BTRFS_UUID_SIZE);
 	btrfs_mark_buffer_dirty(leaf);
 	fs_info->fs_devices->total_rw_bytes += device->total_bytes;
+
+	btrfs_release_path(path);
+
+	/* Add DEV STATS item. */
+	key.objectid = BTRFS_DEV_STATS_OBJECTID;
+	key.type = BTRFS_PERSISTENT_ITEM_KEY;
+	key.offset = free_devid;
+
+	ret = btrfs_insert_empty_item(trans, fs_info->dev_root, path, &key,
+				      sizeof(*dev_stats));
+	if (ret)
+		goto out;
+
+	leaf = path->nodes[0];
+	dev_stats = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_stats_item);
+
+	for (int i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
+		btrfs_set_dev_stats_value(leaf, dev_stats, i, 0);
+
+	btrfs_mark_buffer_dirty(leaf);
+
 	ret = 0;
 
 out:

mkfs/common.c

@@ -127,11 +127,13 @@ static int btrfs_create_tree_root(int fd, struct btrfs_mkfs_config *cfg,
 			memcpy(root_item.uuid, uuid, BTRFS_UUID_SIZE);
 			btrfs_set_stack_timespec_sec(&root_item.otime, now);
 			btrfs_set_stack_timespec_sec(&root_item.ctime, now);
+			btrfs_set_stack_inode_flags(inode_item, BTRFS_INODE_ROOT_ITEM_INIT);
 		} else {
 			memset(uuid, 0, BTRFS_UUID_SIZE);
 			memcpy(root_item.uuid, uuid, BTRFS_UUID_SIZE);
 			btrfs_set_stack_timespec_sec(&root_item.otime, 0);
 			btrfs_set_stack_timespec_sec(&root_item.ctime, 0);
+			btrfs_set_stack_inode_flags(inode_item, 0);
 		}
 		write_extent_buffer(buf, &root_item,
 			btrfs_item_ptr_offset(buf, nritems),
@@ -367,6 +369,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
 	struct btrfs_chunk *chunk;
 	struct btrfs_dev_item *dev_item;
 	struct btrfs_dev_extent *dev_extent;
+	struct btrfs_dev_stats_item *dev_stats;
 	enum btrfs_mkfs_block blocks[MKFS_BLOCK_COUNT];
 	u8 chunk_tree_uuid[BTRFS_UUID_SIZE];
 	u8 *ptr;
@@ -671,8 +674,25 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
 	memset(buf->data + sizeof(struct btrfs_header), 0,
 		cfg->nodesize - sizeof(struct btrfs_header));
 	nritems = 0;
-	itemoff = cfg->leaf_data_size - sizeof(struct btrfs_dev_extent);
+	itemoff = cfg->leaf_data_size;
+
+	/* Add a DEV_STATS item for device 1. */
+	itemoff -= sizeof(struct btrfs_dev_stats_item);
+	btrfs_set_disk_key_objectid(&disk_key, BTRFS_DEV_STATS_OBJECTID);
+	btrfs_set_disk_key_type(&disk_key, BTRFS_PERSISTENT_ITEM_KEY);
+	btrfs_set_disk_key_offset(&disk_key, 1);
+	btrfs_set_item_key(buf, &disk_key, nritems);
+	btrfs_set_item_offset(buf, nritems, itemoff);
+	btrfs_set_item_size(buf, nritems, sizeof(struct btrfs_dev_stats_item));
+	dev_stats = btrfs_item_ptr(buf, nritems, struct btrfs_dev_stats_item);
+
+	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
+		btrfs_set_dev_stats_value(buf, dev_stats, i, 0);
+
+	nritems++;
 
+	/* Add the DEV_EXTENT item for the system chunk. */
+	itemoff -= sizeof(struct btrfs_dev_extent);
 	btrfs_set_disk_key_objectid(&disk_key, 1);
 	btrfs_set_disk_key_offset(&disk_key, system_group_offset);
 	btrfs_set_disk_key_type(&disk_key, BTRFS_DEV_EXTENT_KEY);

mkfs/main.c

@@ -902,7 +902,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info)
 	}
 	ret = btrfs_create_root(trans, fs_info, BTRFS_QUOTA_TREE_OBJECTID);
 	if (ret < 0) {
-		error("failed to create quota root: %d (%m)", ret);
+		errno = -ret;
+		error("failed to create quota root: %m");
 		goto fail;
 	}
 	quota_root = fs_info->quota_root;
@@ -914,7 +915,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info)
 	ret = btrfs_insert_empty_item(trans, quota_root, &path, &key,
 				      sizeof(*qsi));
 	if (ret < 0) {
-		error("failed to insert qgroup status item: %d (%m)", ret);
+		errno = -ret;
+		error("failed to insert qgroup status item: %m");
 		goto fail;
 	}
 
@@ -938,7 +940,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info)
 	/* Currently mkfs will only create one subvolume */
 	ret = insert_qgroup_items(trans, fs_info, BTRFS_FS_TREE_OBJECTID);
 	if (ret < 0) {
-		error("failed to insert qgroup items: %d (%m)", ret);
+		errno = -ret;
+		error("failed to insert qgroup items: %m");
 		goto fail;
 	}
 
@@ -953,7 +956,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info)
 	if (simple) {
 		ret = touch_root_subvol(fs_info);
 		if (ret) {
-			error("failed to touch root dir for simple quota accounting %d (%m)", ret);
+			errno = -ret;
+			error("failed to touch root dir for simple quota accounting: %m");
 			goto fail;
 		}
 	}
@@ -964,12 +968,15 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info)
 	 */
 	ret = qgroup_verify_all(fs_info);
 	if (ret < 0) {
-		error("qgroup rescan failed: %d (%m)", ret);
+		errno = -ret;
+		error("qgroup rescan failed: %m");
 		return ret;
 	}
 	ret = repair_qgroups(fs_info, &qgroup_repaired, true);
-	if (ret < 0)
-		error("failed to fill qgroup info: %d (%m)", ret);
+	if (ret < 0) {
+		errno = -ret;
+		error("failed to fill qgroup info: %m");
+	}
 	return ret;
 fail:
 	btrfs_abort_transaction(trans, ret);
@@ -1142,7 +1149,7 @@ static int parse_subvolume(const char *path, struct list_head *subvols,
 
 	if (subvol->is_default) {
 		if (*has_default_subvol) {
-			error("default subvol can only be specified once");
+			error("default subvolume can only be specified once");
 			free(subvol);
 			return 1;
 		}
@@ -1453,25 +1460,25 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 		struct rootdir_subvol *rds2;
 
 		if (path_cat_out(path, source_dir, rds->dir)) {
-			error("path invalid");
+			error("path invalid: %s", path);
 			ret = 1;
 			goto error;
 		}
 
 		if (!realpath(path, rds->full_path)) {
-			error("could not get canonical path to %s", rds->dir);
+			error("could not get canonical path: %s", rds->dir);
 			ret = 1;
 			goto error;
 		}
 
 		if (!path_exists(rds->full_path)) {
-			error("subvolume %s does not exist", rds->dir);
+			error("subvolume path does not exist: %s", rds->dir);
 			ret = 1;
 			goto error;
 		}
 
 		if (!path_is_dir(rds->full_path)) {
-			error("subvolume %s is not a directory", rds->dir);
+			error("subvolume is not a directory: %s", rds->dir);
 			ret = 1;
 			goto error;
 		}
@@ -1486,7 +1493,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 		     rds2 != rds;
 		     rds2 = list_next_entry(rds2, list)) {
 			if (strcmp(rds2->full_path, rds->full_path) == 0) {
-				error("subvolume %s specified more than once", rds->dir);
+				error("subvolume specified more than once: %s", rds->dir);
 				ret = 1;
 				goto error;
 			}
@@ -1658,6 +1665,10 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 		goto error;
 	}
 
+	/* This is also fixed in kernel, but the flag has no real meaning anymore. */
+	if (nodesize > sysconf(_SC_PAGE_SIZE))
+		features.incompat_flags |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
+
 	if (sectorsize < sizeof(struct btrfs_super_block)) {
 		error("sectorsize smaller than superblock: %u < %zu",
 				sectorsize, sizeof(struct btrfs_super_block));
@@ -1897,14 +1908,16 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 
 	ret = create_metadata_block_groups(root, mixed, &allocation);
 	if (ret) {
-		error("failed to create default block groups: %d", ret);
+		errno = -ret;
+		error("failed to create default block groups: %m");
 		goto error;
 	}
 
 	if (features.incompat_flags & BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE) {
 		ret = setup_raid_stripe_tree_root(fs_info);
 		if (ret < 0) {
-			error("failed to initialize raid-stripe-tree: %d (%m)", ret);
+			errno = -ret;
+			error("failed to initialize raid-stripe-tree: %m");
 			goto out;
 		}
 	}
@@ -1919,21 +1932,24 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 
 	ret = create_data_block_groups(trans, root, mixed, &allocation);
 	if (ret) {
-		error("failed to create default data block groups: %d", ret);
+		errno = -ret;
+		error("failed to create default data block groups: %m");
 		goto error;
 	}
 
 	if (features.incompat_flags & BTRFS_FEATURE_INCOMPAT_EXTENT_TREE_V2) {
 		ret = create_global_roots(trans, nr_global_roots);
 		if (ret) {
-			error("failed to create global roots: %d", ret);
+			errno = -ret;
+			error("failed to create global roots: %m");
 			goto error;
 		}
 	}
 
 	ret = make_root_dir(trans, root);
 	if (ret) {
-		error("failed to setup the root directory: %d", ret);
+		errno = -ret;
+		error("failed to setup the root directory: %m");
 		goto error;
 	}
 
@@ -1976,8 +1992,9 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 					prepare_ctx[i].file, dev_byte_count,
 					sectorsize, sectorsize, sectorsize);
 		if (ret) {
-			error("unable to add %s to filesystem: %d",
-			      prepare_ctx[i].file, ret);
+			errno = -ret;
+			error("unable to add %s to filesystem: %m",
+			      prepare_ctx[i].file);
 			goto error;
 		}
 		if (bconf.verbose >= LOG_INFO) {
@@ -1996,7 +2013,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 	ret = create_raid_groups(trans, root, data_profile,
 			 metadata_profile, mixed, &allocation);
 	if (ret) {
-		error("unable to create raid groups: %d", ret);
+		errno = -ret;
+		error("unable to create raid groups: %m");
 		goto out;
 	}
 
@@ -2031,7 +2049,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 	ret = btrfs_make_subvolume(trans, BTRFS_DATA_RELOC_TREE_OBJECTID,
 				   false);
 	if (ret) {
-		error("unable to create data reloc tree: %d", ret);
+		errno = -ret;
+		error("unable to create data reloc tree: %m");
 		goto out;
 	}
 
@@ -2061,11 +2080,21 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 				   pretty_size_mode(compression_level, UNITS_RAW) :
 				   "");
 
+		/* Print subvolumes now as btrfs_mkfs_fill_dir() deletes the list. */
+		list_for_each_entry(rds, &subvols, list) {
+			pr_verbose(LOG_DEFAULT, "  Subvolume (%s%s):  %s%s\n",
+				   rds->is_default ? "d" : "",
+				   rds->readonly ? "ro" : "rw",
+				   rds->is_default ? "" : " ",
+				   rds->dir);
+		}
+
 		ret = btrfs_mkfs_fill_dir(trans, source_dir, root,
 					  &subvols, compression,
 					  compression_level);
 		if (ret) {
-			error("error while filling filesystem: %d", ret);
+			errno = -ret;
+			error("error while filling filesystem: %m");
 			btrfs_abort_transaction(trans, ret);
 			goto out;
 		}
@@ -2077,18 +2106,13 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 			goto out;
 		}
 
-		list_for_each_entry(rds, &subvols, list) {
-			pr_verbose(LOG_DEFAULT, "  Subvolume:        %s\n",
-				   rds->full_path);
-		}
-
 		if (shrink_rootdir) {
 			pr_verbose(LOG_DEFAULT, "  Shrink:           yes\n");
 			ret = btrfs_mkfs_shrink_fs(fs_info, &shrink_size,
 						   shrink_rootdir);
 			if (ret < 0) {
-				error("error while shrinking filesystem: %d",
-					ret);
+				errno = -ret;
+				error("error while shrinking filesystem: %m");
 				goto out;
 			}
 		} else {
@@ -2103,15 +2127,17 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 	ret = cleanup_temp_chunks(fs_info, &allocation, data_profile,
 				  metadata_profile, metadata_profile);
 	if (ret < 0) {
-		error("failed to cleanup temporary chunks: %d", ret);
+		errno = -ret;
+		error("failed to cleanup temporary chunks: %m");
 		goto out;
 	}
 
 	if (features.runtime_flags & BTRFS_FEATURE_RUNTIME_QUOTA ||
 	    features.incompat_flags & BTRFS_FEATURE_INCOMPAT_SIMPLE_QUOTA) {
 		ret = setup_quota_root(fs_info);
 		if (ret < 0) {
-			error("failed to initialize quota: %d (%m)", ret);
+			errno = -ret;
+			error("failed to initialize quota: %m");
 			goto out;
 		}
 	}
@@ -2184,8 +2210,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv)
 
 	if (!ret && close_ret) {
 		ret = close_ret;
-		error("failed to close ctree, the filesystem may be inconsistent: %d",
-		      ret);
+		errno = -ret;
+		error("failed to close ctree, filesystem may be inconsistent: %m");
 	}
 
 	btrfs_close_all_devices();

tests/README.md

@@ -8,6 +8,12 @@ management. In some cases optional features are turned on by mkfs and the
 filesystem image could be mounted, such tests might fail if there's lack of
 support.
 
+The testsuite follows the principles:
+
+- gather all output to logs
+- stop on first failure (do not clean up until the problem is investigated)
+- detect available capabilities in the system, skip if missing
+
 ## Quick start
 
 Run the tests from the top directory:
@@ -199,7 +205,7 @@ command `losetup` and eventually delete all existing loop devices with `losetup
 ### Prototyping tests, quick tests
 
 There's a script `test-console.sh` that will run shell commands in a loop and
-logs the output with the testing environment set up. It sources the common
+log the output with the testing environment set up. It sources the common
 helper scripts so the shell functions are available.
 
 ### Runtime dependencies
@@ -213,7 +219,8 @@ specific tests need the following packages installed: `acl`, `attr`,
 
 1. Pick the category for the new test or fallback to `misc-tests` if not sure. For
 an easy start copy an existing `test.sh` script from some test that might be
-close to the purpose of your new test. The environment setup includes the
+close to the purpose of your new test, or use one of the templates in the
+[tests/template/](template/) directory. The environment setup includes the
 common scripts and/or prepares the test devices. Other scripts contain examples
 how to do mkfs, mount, unmount, check, loop device management etc.
 
@@ -229,22 +236,23 @@ infrastructure.
 
 5. **Test your test.** Use the `TEST` variable to jump right to your test:
 ```shell
-$ make TEST=012\* tests-misc           # from top directory
-$ TEST=012\* ./misc-tests.sh           # from tests/
+$ make TEST=012\* tests-misc           # from the top or tests/ directory
+$ TEST=012\* ./misc-tests.sh           # alternatively from tests/
 ```
 
 6. The commit changelog should reference a commit that either introduced or
-  fixed the bug (or both). Subject line of the shall mention the name of the
-  new directory for ease of search, e.g. `btrfs-progs: tests: add 012-subvolume-sync-must-wait`
+  fixed the bug (or both). Subject line of the could mention the name of the
+  new directory for ease of search (e.g. `btrfs-progs: tests: add 012-subvolume-sync-must-wait`)
+  or a brief description of the test case (e.g. `btrfs-progs: tests: add case for waiting on subvolume sync after deletion`)
 
-7. A commit that fixes a bug should be applied before the test that verifies
+7. A commit that fixes a bug should be applied *before* the test that verifies
   the fix. This is to keep the git history bisectable.
 
 
 ### Test images
 
 Most tests should be able to create the test images from scratch, using regular
-commands and file operation. The commands also document the test case and use
+commands and file operations. The commands also document the test case and use
 the test code and kernel of the environment.
 
 In other cases, a pre-created image may be the right way if the above does not
@@ -295,7 +303,8 @@ There are some utilities that are not distributed but are necessary for the
 tests. They are in the top level directory of the testsuite and their path
 cannot be set.
 
-The tests assume write access to their directories.
+The tests assume write access to their directories and an existing `/tmp`
+directory.
 
 
 # Coding style, best practices
@@ -305,9 +314,10 @@ The tests assume write access to their directories.
 * quote all variables by default, any path, even the TOP could need that, and
   we use it everywhere
   * even if the variable is safe, use quotes for consistency and to ease
-    reading the code
+    reading the code (syntax highlighting in editors)
   * there are exceptions:
     * `$SUDO_HELPER` as it might be intentionally unset
+    * conditionally set options that are stored in a variable
 * use `#!/bin/bash` explicitly
 * check for all external dependencies (`check_global_prereq`)
 * check for internal dependencies (`check_prereq`), though the basic set is

tests/convert-tests/027-large-symbol-link/test.sh → tests/convert-tests/027-large-symbolic-link/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Make sure btrfs-convert can handle a symbol link which is 4095 bytes large
+# Make sure btrfs-convert can handle a symbolic link which is 4095 bytes long
 
 source "$TEST_TOP/common" || exit
 source "$TEST_TOP/common.convert" || exit
@@ -12,16 +12,16 @@ check_global_prereq mkfs.ext4
 link_target=$(printf "%0.sb" {1..4095})
 
 convert_test_prep_fs ext4 mke2fs -t ext4 -b 4096
-run_check $SUDO_HELPER ln -s "$link_target" "$TEST_MNT/symbol_link"
+run_check $SUDO_HELPER ln -s "$link_target" "$TEST_MNT/symbolic_link"
 run_check_umount_test_dev
 
 # For unpatched btrfs-convert, it will always append one byte to the
-# link target, causing above 4095 target to be 4096, exactly one sector,
-# resulting a regular file extent.
+# link target, causing the above 4095 target to be 4096, exactly one sector,
+# resulting in a regular file extent.
 convert_test_do_convert
 
 run_check_mount_test_dev
 # If the unpatched btrfs-convert created a regular extent, and the kernel is
-# newer enough, such readlink will be rejected by kernel.
-run_check $SUDO_HELPER readlink "$TEST_MNT/symbol_link"
+# new enough, readlink will be rejected by kernel.
+run_check $SUDO_HELPER readlink "$TEST_MNT/symbolic_link"
 run_check_umount_test_dev