1 // SPDX-License-Identifier: GPL-2.0
 
   3  * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc.
 
   8 #include "xfs_shared.h"
 
   9 #include "xfs_format.h"
 
  10 #include "xfs_log_format.h"
 
  11 #include "xfs_trans_resv.h"
 
  12 #include "xfs_mount.h"
 
  13 #include "xfs_btree.h"
 
  14 #include "xfs_btree_staging.h"
 
  15 #include "xfs_alloc_btree.h"
 
  16 #include "xfs_alloc.h"
 
  17 #include "xfs_extent_busy.h"
 
  18 #include "xfs_error.h"
 
  19 #include "xfs_health.h"
 
  20 #include "xfs_trace.h"
 
  21 #include "xfs_trans.h"
 
  24 static struct kmem_cache        *xfs_allocbt_cur_cache;
 
  26 STATIC struct xfs_btree_cur *
 
  28         struct xfs_btree_cur    *cur)
 
  30         return xfs_bnobt_init_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ag.agbp,
 
  34 STATIC struct xfs_btree_cur *
 
  36         struct xfs_btree_cur    *cur)
 
  38         return xfs_cntbt_init_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ag.agbp,
 
  45         struct xfs_btree_cur            *cur,
 
  46         const union xfs_btree_ptr       *ptr,
 
  49         struct xfs_buf          *agbp = cur->bc_ag.agbp;
 
  50         struct xfs_agf          *agf = agbp->b_addr;
 
  54         if (xfs_btree_is_bno(cur->bc_ops)) {
 
  55                 agf->agf_bno_root = ptr->s;
 
  56                 be32_add_cpu(&agf->agf_bno_level, inc);
 
  57                 cur->bc_ag.pag->pagf_bno_level += inc;
 
  59                 agf->agf_cnt_root = ptr->s;
 
  60                 be32_add_cpu(&agf->agf_cnt_level, inc);
 
  61                 cur->bc_ag.pag->pagf_cnt_level += inc;
 
  64         xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS);
 
  68 xfs_allocbt_alloc_block(
 
  69         struct xfs_btree_cur            *cur,
 
  70         const union xfs_btree_ptr       *start,
 
  71         union xfs_btree_ptr             *new,
 
  77         /* Allocate the new block from the freelist. If we can't, give up.  */
 
  78         error = xfs_alloc_get_freelist(cur->bc_ag.pag, cur->bc_tp,
 
  79                         cur->bc_ag.agbp, &bno, 1);
 
  83         if (bno == NULLAGBLOCK) {
 
  88         atomic64_inc(&cur->bc_mp->m_allocbt_blks);
 
  89         xfs_extent_busy_reuse(cur->bc_ag.pag, bno, 1, false);
 
  91         new->s = cpu_to_be32(bno);
 
  98 xfs_allocbt_free_block(
 
  99         struct xfs_btree_cur    *cur,
 
 102         struct xfs_buf          *agbp = cur->bc_ag.agbp;
 
 106         bno = xfs_daddr_to_agbno(cur->bc_mp, xfs_buf_daddr(bp));
 
 107         error = xfs_alloc_put_freelist(cur->bc_ag.pag, cur->bc_tp, agbp, NULL,
 
 112         atomic64_dec(&cur->bc_mp->m_allocbt_blks);
 
 113         xfs_extent_busy_insert(cur->bc_tp, agbp->b_pag, bno, 1,
 
 114                               XFS_EXTENT_BUSY_SKIP_DISCARD);
 
 119 xfs_allocbt_get_minrecs(
 
 120         struct xfs_btree_cur    *cur,
 
 123         return cur->bc_mp->m_alloc_mnr[level != 0];
 
 127 xfs_allocbt_get_maxrecs(
 
 128         struct xfs_btree_cur    *cur,
 
 131         return cur->bc_mp->m_alloc_mxr[level != 0];
 
 135 xfs_allocbt_init_key_from_rec(
 
 136         union xfs_btree_key             *key,
 
 137         const union xfs_btree_rec       *rec)
 
 139         key->alloc.ar_startblock = rec->alloc.ar_startblock;
 
 140         key->alloc.ar_blockcount = rec->alloc.ar_blockcount;
 
 144 xfs_bnobt_init_high_key_from_rec(
 
 145         union xfs_btree_key             *key,
 
 146         const union xfs_btree_rec       *rec)
 
 150         x = be32_to_cpu(rec->alloc.ar_startblock);
 
 151         x += be32_to_cpu(rec->alloc.ar_blockcount) - 1;
 
 152         key->alloc.ar_startblock = cpu_to_be32(x);
 
 153         key->alloc.ar_blockcount = 0;
 
 157 xfs_cntbt_init_high_key_from_rec(
 
 158         union xfs_btree_key             *key,
 
 159         const union xfs_btree_rec       *rec)
 
 161         key->alloc.ar_blockcount = rec->alloc.ar_blockcount;
 
 162         key->alloc.ar_startblock = 0;
 
 166 xfs_allocbt_init_rec_from_cur(
 
 167         struct xfs_btree_cur    *cur,
 
 168         union xfs_btree_rec     *rec)
 
 170         rec->alloc.ar_startblock = cpu_to_be32(cur->bc_rec.a.ar_startblock);
 
 171         rec->alloc.ar_blockcount = cpu_to_be32(cur->bc_rec.a.ar_blockcount);
 
 175 xfs_allocbt_init_ptr_from_cur(
 
 176         struct xfs_btree_cur    *cur,
 
 177         union xfs_btree_ptr     *ptr)
 
 179         struct xfs_agf          *agf = cur->bc_ag.agbp->b_addr;
 
 181         ASSERT(pag_agno(cur->bc_ag.pag) == be32_to_cpu(agf->agf_seqno));
 
 183         if (xfs_btree_is_bno(cur->bc_ops))
 
 184                 ptr->s = agf->agf_bno_root;
 
 186                 ptr->s = agf->agf_cnt_root;
 
 191         struct xfs_btree_cur            *cur,
 
 192         const union xfs_btree_key       *key)
 
 194         struct xfs_alloc_rec_incore     *rec = &cur->bc_rec.a;
 
 195         const struct xfs_alloc_rec      *kp = &key->alloc;
 
 197         return (int64_t)be32_to_cpu(kp->ar_startblock) - rec->ar_startblock;
 
 202         struct xfs_btree_cur            *cur,
 
 203         const union xfs_btree_key       *key)
 
 205         struct xfs_alloc_rec_incore     *rec = &cur->bc_rec.a;
 
 206         const struct xfs_alloc_rec      *kp = &key->alloc;
 
 209         diff = (int64_t)be32_to_cpu(kp->ar_blockcount) - rec->ar_blockcount;
 
 213         return (int64_t)be32_to_cpu(kp->ar_startblock) - rec->ar_startblock;
 
 217 xfs_bnobt_diff_two_keys(
 
 218         struct xfs_btree_cur            *cur,
 
 219         const union xfs_btree_key       *k1,
 
 220         const union xfs_btree_key       *k2,
 
 221         const union xfs_btree_key       *mask)
 
 223         ASSERT(!mask || mask->alloc.ar_startblock);
 
 225         return (int64_t)be32_to_cpu(k1->alloc.ar_startblock) -
 
 226                         be32_to_cpu(k2->alloc.ar_startblock);
 
 230 xfs_cntbt_diff_two_keys(
 
 231         struct xfs_btree_cur            *cur,
 
 232         const union xfs_btree_key       *k1,
 
 233         const union xfs_btree_key       *k2,
 
 234         const union xfs_btree_key       *mask)
 
 238         ASSERT(!mask || (mask->alloc.ar_blockcount &&
 
 239                          mask->alloc.ar_startblock));
 
 241         diff =  be32_to_cpu(k1->alloc.ar_blockcount) -
 
 242                 be32_to_cpu(k2->alloc.ar_blockcount);
 
 246         return  be32_to_cpu(k1->alloc.ar_startblock) -
 
 247                 be32_to_cpu(k2->alloc.ar_startblock);
 
 250 static xfs_failaddr_t
 
 254         struct xfs_mount        *mp = bp->b_mount;
 
 255         struct xfs_btree_block  *block = XFS_BUF_TO_BLOCK(bp);
 
 256         struct xfs_perag        *pag = bp->b_pag;
 
 260         if (!xfs_verify_magic(bp, block->bb_magic))
 
 261                 return __this_address;
 
 263         if (xfs_has_crc(mp)) {
 
 264                 fa = xfs_btree_agblock_v5hdr_verify(bp);
 
 270          * The perag may not be attached during grow operations or fully
 
 271          * initialized from the AGF during log recovery. Therefore we can only
 
 272          * check against maximum tree depth from those contexts.
 
 274          * Otherwise check against the per-tree limit. Peek at one of the
 
 275          * verifier magic values to determine the type of tree we're verifying
 
 278         level = be16_to_cpu(block->bb_level);
 
 279         if (pag && xfs_perag_initialised_agf(pag)) {
 
 280                 unsigned int    maxlevel, repair_maxlevel = 0;
 
 283                  * Online repair could be rewriting the free space btrees, so
 
 284                  * we'll validate against the larger of either tree while this
 
 287                 if (bp->b_ops->magic[0] == cpu_to_be32(XFS_ABTC_MAGIC)) {
 
 288                         maxlevel = pag->pagf_cnt_level;
 
 289 #ifdef CONFIG_XFS_ONLINE_REPAIR
 
 290                         repair_maxlevel = pag->pagf_repair_cnt_level;
 
 293                         maxlevel = pag->pagf_bno_level;
 
 294 #ifdef CONFIG_XFS_ONLINE_REPAIR
 
 295                         repair_maxlevel = pag->pagf_repair_bno_level;
 
 299                 if (level >= max(maxlevel, repair_maxlevel))
 
 300                         return __this_address;
 
 301         } else if (level >= mp->m_alloc_maxlevels)
 
 302                 return __this_address;
 
 304         return xfs_btree_agblock_verify(bp, mp->m_alloc_mxr[level != 0]);
 
 308 xfs_allocbt_read_verify(
 
 313         if (!xfs_btree_agblock_verify_crc(bp))
 
 314                 xfs_verifier_error(bp, -EFSBADCRC, __this_address);
 
 316                 fa = xfs_allocbt_verify(bp);
 
 318                         xfs_verifier_error(bp, -EFSCORRUPTED, fa);
 
 322                 trace_xfs_btree_corrupt(bp, _RET_IP_);
 
 326 xfs_allocbt_write_verify(
 
 331         fa = xfs_allocbt_verify(bp);
 
 333                 trace_xfs_btree_corrupt(bp, _RET_IP_);
 
 334                 xfs_verifier_error(bp, -EFSCORRUPTED, fa);
 
 337         xfs_btree_agblock_calc_crc(bp);
 
 341 const struct xfs_buf_ops xfs_bnobt_buf_ops = {
 
 343         .magic = { cpu_to_be32(XFS_ABTB_MAGIC),
 
 344                    cpu_to_be32(XFS_ABTB_CRC_MAGIC) },
 
 345         .verify_read = xfs_allocbt_read_verify,
 
 346         .verify_write = xfs_allocbt_write_verify,
 
 347         .verify_struct = xfs_allocbt_verify,
 
 350 const struct xfs_buf_ops xfs_cntbt_buf_ops = {
 
 352         .magic = { cpu_to_be32(XFS_ABTC_MAGIC),
 
 353                    cpu_to_be32(XFS_ABTC_CRC_MAGIC) },
 
 354         .verify_read = xfs_allocbt_read_verify,
 
 355         .verify_write = xfs_allocbt_write_verify,
 
 356         .verify_struct = xfs_allocbt_verify,
 
 360 xfs_bnobt_keys_inorder(
 
 361         struct xfs_btree_cur            *cur,
 
 362         const union xfs_btree_key       *k1,
 
 363         const union xfs_btree_key       *k2)
 
 365         return be32_to_cpu(k1->alloc.ar_startblock) <
 
 366                be32_to_cpu(k2->alloc.ar_startblock);
 
 370 xfs_bnobt_recs_inorder(
 
 371         struct xfs_btree_cur            *cur,
 
 372         const union xfs_btree_rec       *r1,
 
 373         const union xfs_btree_rec       *r2)
 
 375         return be32_to_cpu(r1->alloc.ar_startblock) +
 
 376                 be32_to_cpu(r1->alloc.ar_blockcount) <=
 
 377                 be32_to_cpu(r2->alloc.ar_startblock);
 
 381 xfs_cntbt_keys_inorder(
 
 382         struct xfs_btree_cur            *cur,
 
 383         const union xfs_btree_key       *k1,
 
 384         const union xfs_btree_key       *k2)
 
 386         return be32_to_cpu(k1->alloc.ar_blockcount) <
 
 387                 be32_to_cpu(k2->alloc.ar_blockcount) ||
 
 388                 (k1->alloc.ar_blockcount == k2->alloc.ar_blockcount &&
 
 389                  be32_to_cpu(k1->alloc.ar_startblock) <
 
 390                  be32_to_cpu(k2->alloc.ar_startblock));
 
 394 xfs_cntbt_recs_inorder(
 
 395         struct xfs_btree_cur            *cur,
 
 396         const union xfs_btree_rec       *r1,
 
 397         const union xfs_btree_rec       *r2)
 
 399         return be32_to_cpu(r1->alloc.ar_blockcount) <
 
 400                 be32_to_cpu(r2->alloc.ar_blockcount) ||
 
 401                 (r1->alloc.ar_blockcount == r2->alloc.ar_blockcount &&
 
 402                  be32_to_cpu(r1->alloc.ar_startblock) <
 
 403                  be32_to_cpu(r2->alloc.ar_startblock));
 
 406 STATIC enum xbtree_key_contig
 
 407 xfs_allocbt_keys_contiguous(
 
 408         struct xfs_btree_cur            *cur,
 
 409         const union xfs_btree_key       *key1,
 
 410         const union xfs_btree_key       *key2,
 
 411         const union xfs_btree_key       *mask)
 
 413         ASSERT(!mask || mask->alloc.ar_startblock);
 
 415         return xbtree_key_contig(be32_to_cpu(key1->alloc.ar_startblock),
 
 416                                  be32_to_cpu(key2->alloc.ar_startblock));
 
 419 const struct xfs_btree_ops xfs_bnobt_ops = {
 
 421         .type                   = XFS_BTREE_TYPE_AG,
 
 423         .rec_len                = sizeof(xfs_alloc_rec_t),
 
 424         .key_len                = sizeof(xfs_alloc_key_t),
 
 425         .ptr_len                = XFS_BTREE_SHORT_PTR_LEN,
 
 427         .lru_refs               = XFS_ALLOC_BTREE_REF,
 
 428         .statoff                = XFS_STATS_CALC_INDEX(xs_abtb_2),
 
 429         .sick_mask              = XFS_SICK_AG_BNOBT,
 
 431         .dup_cursor             = xfs_bnobt_dup_cursor,
 
 432         .set_root               = xfs_allocbt_set_root,
 
 433         .alloc_block            = xfs_allocbt_alloc_block,
 
 434         .free_block             = xfs_allocbt_free_block,
 
 435         .get_minrecs            = xfs_allocbt_get_minrecs,
 
 436         .get_maxrecs            = xfs_allocbt_get_maxrecs,
 
 437         .init_key_from_rec      = xfs_allocbt_init_key_from_rec,
 
 438         .init_high_key_from_rec = xfs_bnobt_init_high_key_from_rec,
 
 439         .init_rec_from_cur      = xfs_allocbt_init_rec_from_cur,
 
 440         .init_ptr_from_cur      = xfs_allocbt_init_ptr_from_cur,
 
 441         .key_diff               = xfs_bnobt_key_diff,
 
 442         .buf_ops                = &xfs_bnobt_buf_ops,
 
 443         .diff_two_keys          = xfs_bnobt_diff_two_keys,
 
 444         .keys_inorder           = xfs_bnobt_keys_inorder,
 
 445         .recs_inorder           = xfs_bnobt_recs_inorder,
 
 446         .keys_contiguous        = xfs_allocbt_keys_contiguous,
 
 449 const struct xfs_btree_ops xfs_cntbt_ops = {
 
 451         .type                   = XFS_BTREE_TYPE_AG,
 
 453         .rec_len                = sizeof(xfs_alloc_rec_t),
 
 454         .key_len                = sizeof(xfs_alloc_key_t),
 
 455         .ptr_len                = XFS_BTREE_SHORT_PTR_LEN,
 
 457         .lru_refs               = XFS_ALLOC_BTREE_REF,
 
 458         .statoff                = XFS_STATS_CALC_INDEX(xs_abtc_2),
 
 459         .sick_mask              = XFS_SICK_AG_CNTBT,
 
 461         .dup_cursor             = xfs_cntbt_dup_cursor,
 
 462         .set_root               = xfs_allocbt_set_root,
 
 463         .alloc_block            = xfs_allocbt_alloc_block,
 
 464         .free_block             = xfs_allocbt_free_block,
 
 465         .get_minrecs            = xfs_allocbt_get_minrecs,
 
 466         .get_maxrecs            = xfs_allocbt_get_maxrecs,
 
 467         .init_key_from_rec      = xfs_allocbt_init_key_from_rec,
 
 468         .init_high_key_from_rec = xfs_cntbt_init_high_key_from_rec,
 
 469         .init_rec_from_cur      = xfs_allocbt_init_rec_from_cur,
 
 470         .init_ptr_from_cur      = xfs_allocbt_init_ptr_from_cur,
 
 471         .key_diff               = xfs_cntbt_key_diff,
 
 472         .buf_ops                = &xfs_cntbt_buf_ops,
 
 473         .diff_two_keys          = xfs_cntbt_diff_two_keys,
 
 474         .keys_inorder           = xfs_cntbt_keys_inorder,
 
 475         .recs_inorder           = xfs_cntbt_recs_inorder,
 
 476         .keys_contiguous        = NULL, /* not needed right now */
 
 480  * Allocate a new bnobt cursor.
 
 482  * For staging cursors tp and agbp are NULL.
 
 484 struct xfs_btree_cur *
 
 485 xfs_bnobt_init_cursor(
 
 486         struct xfs_mount        *mp,
 
 487         struct xfs_trans        *tp,
 
 488         struct xfs_buf          *agbp,
 
 489         struct xfs_perag        *pag)
 
 491         struct xfs_btree_cur    *cur;
 
 493         cur = xfs_btree_alloc_cursor(mp, tp, &xfs_bnobt_ops,
 
 494                         mp->m_alloc_maxlevels, xfs_allocbt_cur_cache);
 
 495         cur->bc_ag.pag = xfs_perag_hold(pag);
 
 496         cur->bc_ag.agbp = agbp;
 
 498                 struct xfs_agf          *agf = agbp->b_addr;
 
 500                 cur->bc_nlevels = be32_to_cpu(agf->agf_bno_level);
 
 506  * Allocate a new cntbt cursor.
 
 508  * For staging cursors tp and agbp are NULL.
 
 510 struct xfs_btree_cur *
 
 511 xfs_cntbt_init_cursor(
 
 512         struct xfs_mount        *mp,
 
 513         struct xfs_trans        *tp,
 
 514         struct xfs_buf          *agbp,
 
 515         struct xfs_perag        *pag)
 
 517         struct xfs_btree_cur    *cur;
 
 519         cur = xfs_btree_alloc_cursor(mp, tp, &xfs_cntbt_ops,
 
 520                         mp->m_alloc_maxlevels, xfs_allocbt_cur_cache);
 
 521         cur->bc_ag.pag = xfs_perag_hold(pag);
 
 522         cur->bc_ag.agbp = agbp;
 
 524                 struct xfs_agf          *agf = agbp->b_addr;
 
 526                 cur->bc_nlevels = be32_to_cpu(agf->agf_cnt_level);
 
 532  * Install a new free space btree root.  Caller is responsible for invalidating
 
 533  * and freeing the old btree blocks.
 
 536 xfs_allocbt_commit_staged_btree(
 
 537         struct xfs_btree_cur    *cur,
 
 538         struct xfs_trans        *tp,
 
 539         struct xfs_buf          *agbp)
 
 541         struct xfs_agf          *agf = agbp->b_addr;
 
 542         struct xbtree_afakeroot *afake = cur->bc_ag.afake;
 
 544         ASSERT(cur->bc_flags & XFS_BTREE_STAGING);
 
 546         if (xfs_btree_is_bno(cur->bc_ops)) {
 
 547                 agf->agf_bno_root = cpu_to_be32(afake->af_root);
 
 548                 agf->agf_bno_level = cpu_to_be32(afake->af_levels);
 
 550                 agf->agf_cnt_root = cpu_to_be32(afake->af_root);
 
 551                 agf->agf_cnt_level = cpu_to_be32(afake->af_levels);
 
 553         xfs_alloc_log_agf(tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS);
 
 555         xfs_btree_commit_afakeroot(cur, tp, agbp);
 
 558 /* Calculate number of records in an alloc btree block. */
 
 559 static inline unsigned int
 
 560 xfs_allocbt_block_maxrecs(
 
 561         unsigned int            blocklen,
 
 565                 return blocklen / sizeof(xfs_alloc_rec_t);
 
 566         return blocklen / (sizeof(xfs_alloc_key_t) + sizeof(xfs_alloc_ptr_t));
 
 570  * Calculate number of records in an alloc btree block.
 
 574         struct xfs_mount        *mp,
 
 575         unsigned int            blocklen,
 
 578         blocklen -= XFS_ALLOC_BLOCK_LEN(mp);
 
 579         return xfs_allocbt_block_maxrecs(blocklen, leaf);
 
 582 /* Free space btrees are at their largest when every other block is free. */
 
 583 #define XFS_MAX_FREESP_RECORDS  ((XFS_MAX_AG_BLOCKS + 1) / 2)
 
 585 /* Compute the max possible height for free space btrees. */
 
 587 xfs_allocbt_maxlevels_ondisk(void)
 
 589         unsigned int            minrecs[2];
 
 590         unsigned int            blocklen;
 
 592         blocklen = min(XFS_MIN_BLOCKSIZE - XFS_BTREE_SBLOCK_LEN,
 
 593                        XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN);
 
 595         minrecs[0] = xfs_allocbt_block_maxrecs(blocklen, true) / 2;
 
 596         minrecs[1] = xfs_allocbt_block_maxrecs(blocklen, false) / 2;
 
 598         return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_FREESP_RECORDS);
 
 601 /* Calculate the freespace btree size for some records. */
 
 603 xfs_allocbt_calc_size(
 
 604         struct xfs_mount        *mp,
 
 605         unsigned long long      len)
 
 607         return xfs_btree_calc_size(mp->m_alloc_mnr, len);
 
 611 xfs_allocbt_init_cur_cache(void)
 
 613         xfs_allocbt_cur_cache = kmem_cache_create("xfs_bnobt_cur",
 
 614                         xfs_btree_cur_sizeof(xfs_allocbt_maxlevels_ondisk()),
 
 617         if (!xfs_allocbt_cur_cache)
 
 623 xfs_allocbt_destroy_cur_cache(void)
 
 625         kmem_cache_destroy(xfs_allocbt_cur_cache);
 
 626         xfs_allocbt_cur_cache = NULL;