Kernel 6.11 "Allocator stuck? Waited for 30 seconds"

[truongsinh]

I'm running 6.11.0-061100rc7daily20240915-generic, is this problem the lock contention that was rejected during 6.11 RC window?

Allocator stuck? Waited for 30 seconds
[12833.040409] Allocator stuck? Waited for 30 seconds
[12833.040420] Allocator debug:
[12833.040421]   capacity8344943698
[12833.040422]   reserved             725647278
[12833.040505]   hidden               16229376
[12833.040506]   btree                55331840
[12833.040508]   data                 7198854472
[12833.040509]   cached               0
[12833.040510]   reserved             0
[12833.040511]   online_reserved      33344
[12833.040512]   nr_inodes            0
[12833.040514]   
[12833.040515]   freelist_wait        waiting
[12833.040516]   open buckets allocated11
[12833.040518]   open buckets total   1024
[12833.040519]   open_buckets_wait    empty
[12833.040520]   open_buckets_btree   2
[12833.040521]   open_buckets_user    8
[12833.040522]   btree reserve cache  1
[12833.040567] 
[12833.040569] Dev 0:
[12833.040570]                      buckets         sectors      fragmented
[12833.040571]   free                 67502               0               0
[12833.040572]   sb                      13            6152             504
[12833.040573]   journal               8192         4194304               0
[12833.040574]   btree                    0               0               0
[12833.040575]   user               4240325      2171043656            2744
[12833.040576]   cached                   0               0               0
[12833.040577]   parity                   0               0               0
[12833.040578]   stripe                   0               0               0
[12833.040579]   need_gc_gens             0               0               0
[12833.040580]   need_discard             0               0               0
[12833.040601]   unstriped                0               0               0
[12833.040603]   capacity           4316032
[12833.040603]   
[12833.040604]   reserves:
[12833.040605]   stripe              134932
[12833.040606]   normal               67494
[12833.040607]   copygc                  56
[12833.040608]   btree                   28
[12833.040608]   btree_copygc             0
[12833.040609]   reclaim                  0
[12833.040610]   interior_updates         0
[12833.040611]   
[12833.040612]   open buckets             8
[12833.040613]   buckets to invalidate    0
[12833.040614] 
[12833.040615] Dev 1:
[12833.040616]                      buckets         sectors      fragmented
[12833.040617]   free                 15180               0               0
[12833.040627]   sb                      13            6152             504
[12833.040628]   journal               7631         3907072               0
[12833.040630]   btree                26798        13720576               0
[12833.040631]   user                927166       474708992               0
[12833.040632]   cached                   0               0               0
[12833.040634]   parity                   0               0               0
[12833.040635]   stripe                   0               0               0
[12833.040636]   need_gc_gens             0               0               0
[12833.040637]   need_discard             0               0               0
[12833.040639]   unstriped                0               0               0
[12833.040640]   capacity            976788
[12833.040641]   
[12833.040641]   reserves:
[12833.040643]   stripe               30580
[12833.040644]   normal               15318
[12833.040644]   copygc                  56
[12833.040645]   btree                   28
[12833.040646]   btree_copygc             0
[12833.040647]   reclaim                  0
[12833.040648]   interior_updates         0
[12833.040649]   
[12833.040650]   open buckets             1
[12833.040651]   buckets to invalidate    0
[12833.040652] 
[12833.040653] Dev 2:
[12833.040654]                      buckets         sectors      fragmented
[12833.040655]   free                 15191               0               0
[12833.040657]   sb                      13            6152             504
[12833.040658]   journal               7631         3907072               0
[12833.040659]   btree                27455        14056960               0
[12833.040660]   user                926498       474366976               0
[12833.040662]   cached                   0               0               0
[12833.040663]   parity                   0               0               0
[12833.040664]   stripe                   0               0               0
[12833.040665]   need_gc_gens             0               0               0
[12833.040666]   need_discard             0               0               0
[12833.040667]   unstriped                0               0               0
[12833.040669]   capacity            976788
[12833.040669]   
[12833.040670]   reserves:
[12833.040671]   stripe               30580
[12833.040673]   normal               15318
[12833.040674]   copygc                  56
[12833.040674]   btree                   28
[12833.040675]   btree_copygc             0
[12833.040676]   reclaim                  0
[12833.040677]   interior_updates         0
[12833.040686]   
[12833.040686]   open buckets             0
[12833.040688]   buckets to invalidate    0
[12833.040689] 
[12833.040690] Dev 3:
[12833.040691]                      buckets         sectors      fragmented
[12833.040692]   free                119199               0               0
[12833.040693]   sb                      13            6152             504
[12833.040694]   journal               8192         4194304               0
[12833.040695]   btree                53817        27554304               0
[12833.040697]   user               7449691      3814241792               0
[12833.040698]   cached                   0               0               0
[12833.040699]   parity                   0               0               0
[12833.040700]   stripe                   0               0               0
[12833.040700]   need_gc_gens             0               0               0
[12833.040701]   need_discard             0               0               0
[12833.040702]   unstriped                0               0               0
[12833.040710]   capacity           7630912
[12833.040711]   
[12833.040711]   reserves:
[12833.040712]   stripe              238522
[12833.040713]   normal              119289
[12833.040714]   copygc                  56
[12833.040716]   btree                   28
[12833.040716]   btree_copygc             0
[12833.040717]   reclaim                  0
[12833.040718]   interior_updates         0
[12833.040718]   
[12833.040719]   open buckets             1
[12833.040720]   buckets to invalidate    0
[12833.040721] 
[12833.040722] Dev 4:
[12833.040723]                      buckets         sectors      fragmented
[12833.040724]   free     18446744073709293322            0               0
[12833.040725]   sb                       0               0               0
[12833.040726]   journal                  0               0               0
[12833.040727]   btree                    0               0               0
[12833.040728]   user                258294       264493056               0
[12833.040729]   cached                   0               0               0
[12833.040730]   parity                   0               0               0
[12833.040731]   stripe                   0               0               0
[12833.040732]   need_gc_gens             0               0               0
[12833.040739]   need_discard             0               0               0
[12833.040745]   unstriped                0               0               0
[12833.040746]   capacity           1907739
[12833.040747]   
[12833.040753]   reserves:
[12833.040754]   stripe               59644
[12833.040755]   normal               29836
[12833.040756]   copygc                  28
[12833.040757]   btree                   14
[12833.040758]   btree_copygc             0
[12833.040758]   reclaim                  0
[12833.040759]   interior_updates         0
[12833.040760]   
[12833.040761]   open buckets             0
[12833.040762]   buckets to invalidate    0
[12833.040763] 
[12833.040763] Copygc debug:
[12833.040765]   running: 0
[12833.040765]   copygc_wait:0
[12833.040766]   copygc_wait_at:0
[12833.040767]   Currently waiting for:0 B
[12833.040767]   Currently waiting since:5.60 TiB
[12833.040768]   Currently calculated wait:0 B
[12833.040769] 
[12833.040770] Journal debug:
[12833.040770]   flags:                   replay_done,running,may_skip_flush
[12833.040771]   dirty journal entries:   0/32768
[12833.040772]   seq:                     1031358
[12833.040773]   seq_ondisk:              1031358
[12833.040774]   last_seq:                1031359
[12833.040774]   last_seq_ondisk:         1031358
[12833.040775]   flushed_seq_ondisk:      1031358
[12833.040776]   watermark:               stripe
[12833.040777]   each entry reserved:     321
[12833.040778]   nr flush writes:         12091
[12833.040779]   nr noflush writes:       3033
[12833.040780]   average write size:      69.7 KiB
[12833.040781]   nr direct reclaim:       0
[12833.040781]   nr background reclaim:   408443
[12833.040782]   reclaim kicked:          0
[12833.040783]   reclaim runs in:         0 ms
[12833.040784]   blocked:                 0
[12833.040784]   current entry sectors:   512
[12833.040785]   current entry error:     ok
[12833.040786]   current entry:           closed
[12833.040786]   unwritten entries:
[12833.040787]   last buf closed
[12833.040788]   space:
[12833.040788]     discarded              512:4193792
[12833.040789]     clean ondisk           512:4193792
[12833.040790]     clean                  512:4193792
[12833.040805]     total                  512:4194304
[12833.040806]   dev 0:
[12833.040807]     nr                     8192
[12833.040807]     bucket size            512
[12833.040809]     available              8190:512
[12833.040809]     discard_idx            0
[12833.040810]     dirty_ondisk           0 (seq 0)
[12833.040811]     dirty_idx              0 (seq 0)
[12833.040812]     cur_idx                0 (seq 0)
[12833.040813]   dev 1:
[12833.040814]     nr                     7631
[12833.040815]     bucket size            512
[12833.040816]     available              7629:0
[12833.040817]     discard_idx            6033
[12833.040818]     dirty_ondisk           6033 (seq 1031335)
[12833.040825]     dirty_idx              6033 (seq 1031335)
[12833.040826]     cur_idx                6033 (seq 1031335)
[12833.040828]   dev 2:
[12833.040828]     nr                     7631
[12833.040830]     bucket size            512
[12833.040831]     available              7629:240
[12833.040832]     discard_idx            6636
[12833.040833]     dirty_ondisk           6636 (seq 1031358)
[12833.040834]     dirty_idx              6636 (seq 1031358)
[12833.040835]     cur_idx                6636 (seq 1031358)
[12833.040836]   dev 3:
[12833.040837]     nr                     8192
[12833.040837]     bucket size            512
[12833.040839]     available              8190:240
[12833.040840]     discard_idx            1574
[12833.040841]     dirty_ondisk           1574 (seq 1031358)
[12833.040842]     dirty_idx              1574 (seq 1031358)
[12833.040842]     cur_idx                1574 (seq 1031358)
[12833.040843]   dev 4:
[12833.040844]     nr                     8192
[12833.040845]     bucket size            1024
[12833.040846]     available              8190:1024
[12833.040847]     discard_idx            0
[12833.040847]     dirty_ondisk           0 (seq 0)
[12833.040848]     dirty_idx              0 (seq 0)
[12833.040849]     cur_idx                0 (seq 0)
[12833.040850] 
[12833.623726] Allocator stuck? Waited for 30 seconds
[12833.623736] Allocator debug:
[12833.623737]   capacity8344943698
[12833.623739]   reserved             725647278
[12833.623778]   hidden               16229376
[12833.623779]   btree                55331840
[12833.623780]   data                 7198854472
[12833.623781]   cached               0
[12833.623782]   reserved             0
[12833.623783]   online_reserved      33344
[12833.623783]   nr_inodes            0
[12833.623785]   
[12833.623785]   freelist_wait        waiting
[12833.623786]   open buckets allocated11
[12833.623788]   open buckets total   1024
[12833.623788]   open_buckets_wait    empty
[12833.623789]   open_buckets_btree   2
[12833.623790]   open_buckets_user    8
[12833.623791]   btree reserve cache  1
[12833.623792] 
[12833.623793] Dev 0:
[12833.623794]                      buckets         sectors      fragmented
[12833.623795]   free                 67502               0               0
[12833.623796]   sb                      13            6152             504
[12833.623797]   journal               8192         4194304               0
[12833.623798]   btree                    0               0               0
[12833.623799]   user               4240325      2171043656            2744
[12833.623800]   cached                   0               0               0
[12833.623801]   parity                   0               0               0
[12833.623802]   stripe                   0               0               0
[12833.623803]   need_gc_gens             0               0               0
[12833.623804]   need_discard             0               0               0
[12833.623805]   unstriped                0               0               0
[12833.623806]   capacity           4316032
[12833.623807]   
[12833.623808]   reserves:
[12833.623809]   stripe              134932
[12833.623810]   normal               67494
[12833.623810]   copygc                  56
[12833.623811]   btree                   28
[12833.623812]   btree_copygc             0
[12833.623813]   reclaim                  0
[12833.623814]   interior_updates         0
[12833.623815]   
[12833.623816]   open buckets             8
[12833.623816]   buckets to invalidate    0
[12833.623817] 
[12833.623818] Dev 1:
[12833.623819]                      buckets         sectors      fragmented
[12833.623820]   free                 15180               0               0
[12833.623821]   sb                      13            6152             504
[12833.623832]   journal               7631         3907072               0
[12833.623833]   btree                26798        13720576               0
[12833.623834]   user                927166       474708992               0
[12833.623835]   cached                   0               0               0
[12833.623836]   parity                   0               0               0
[12833.623837]   stripe                   0               0               0
[12833.623852]   need_gc_gens             0               0               0
[12833.623853]   need_discard             0               0               0
[12833.623854]   unstriped                0               0               0
[12833.623855]   capacity            976788
[12833.623856]   
[12833.623857]   reserves:
[12833.623858]   stripe               30580
[12833.623859]   normal               15318
[12833.623860]   copygc                  56
[12833.623862]   btree                   28
[12833.623864]   btree_copygc             0
[12833.623867]   reclaim                  0
[12833.623869]   interior_updates         0
[12833.623869]   
[12833.623870]   open buckets             1
[12833.623871]   buckets to invalidate    0
[12833.623873] 
[12833.623875] Dev 2:
[12833.623876]                      buckets         sectors      fragmented
[12833.623877]   free                 15191               0               0
[12833.623879]   sb                      13            6152             504
[12833.623887]   journal               7631         3907072               0
[12833.623888]   btree                27455        14056960               0
[12833.623888]   user                926498       474366976               0
[12833.623890]   cached                   0               0               0
[12833.623891]   parity                   0               0               0
[12833.623892]   stripe                   0               0               0
[12833.623893]   need_gc_gens             0               0               0
[12833.623894]   need_discard             0               0               0
[12833.623895]   unstriped                0               0               0
[12833.623896]   capacity            976788
[12833.623897]   
[12833.623898]   reserves:
[12833.623899]   stripe               30580
[12833.623900]   normal               15318
[12833.623900]   copygc                  56
[12833.623901]   btree                   28
[12833.623902]   btree_copygc             0
[12833.623903]   reclaim                  0
[12833.623904]   interior_updates         0
[12833.623911]   
[12833.623911]   open buckets             0
[12833.623912]   buckets to invalidate    0
[12833.623913] 
[12833.623914] Dev 3:
[12833.623915]                      buckets         sectors      fragmented
[12833.623916]   free                119199               0               0
[12833.623919]   sb                      13            6152             504
[12833.623921]   journal               8192         4194304               0
[12833.623922]   btree                53817        27554304               0
[12833.623923]   user               7449691      3814241792               0
[12833.623924]   cached                   0               0               0
[12833.623925]   parity                   0               0               0
[12833.623926]   stripe                   0               0               0
[12833.623927]   need_gc_gens             0               0               0
[12833.623928]   need_discard             0               0               0
[12833.623929]   unstriped                0               0               0
[12833.623930]   capacity           7630912
[12833.623931]   
[12833.623932]   reserves:
[12833.623933]   stripe              238522
[12833.623935]   normal              119289
[12833.623935]   copygc                  56
[12833.623936]   btree                   28
[12833.623937]   btree_copygc             0
[12833.623938]   reclaim                  0
[12833.623939]   interior_updates         0
[12833.623940]   
[12833.623941]   open buckets             1
[12833.623942]   buckets to invalidate    0
[12833.623943] 
[12833.623944] Dev 4:
[12833.623944]                      buckets         sectors      fragmented
[12833.623945]   free     18446744073709293322            0               0
[12833.623946]   sb                       0               0               0
[12833.623948]   journal                  0               0               0
[12833.623949]   btree                    0               0               0
[12833.623950]   user                258294       264493056               0
[12833.623951]   cached                   0               0               0
[12833.623952]   parity                   0               0               0
[12833.623953]   stripe                   0               0               0
[12833.623960]   need_gc_gens             0               0               0
[12833.623962]   need_discard             0               0               0
[12833.623963]   unstriped                0               0               0
[12833.623964]   capacity           1907739
[12833.623965]   
[12833.623966]   reserves:
[12833.623972]   stripe               59644
[12833.623973]   normal               29836
[12833.623974]   copygc                  28
[12833.623974]   btree                   14
[12833.623975]   btree_copygc             0
[12833.623977]   reclaim                  0
[12833.623977]   interior_updates         0
[12833.623978]   
[12833.623979]   open buckets             0
[12833.623980]   buckets to invalidate    0
[12833.623981] 
[12833.623982] Copygc debug:
[12833.623983]   running: 0
[12833.623984]   copygc_wait:0
[12833.623985]   copygc_wait_at:0
[12833.623985]   Currently waiting for:0 B
[12833.623987]   Currently waiting since:5.60 TiB
[12833.623988]   Currently calculated wait:0 B
[12833.623991] 
[12833.623993] Journal debug:
[12833.623993]   flags:                   replay_done,running,may_skip_flush
[12833.623994]   dirty journal entries:   0/32768
[12833.623995]   seq:                     1031358
[12833.623996]   seq_ondisk:              1031358
[12833.623997]   last_seq:                1031359
[12833.623998]   last_seq_ondisk:         1031358
[12833.624000]   flushed_seq_ondisk:      1031358
[12833.624001]   watermark:               stripe
[12833.624002]   each entry reserved:     321
[12833.624003]   nr flush writes:         12091
[12833.624004]   nr noflush writes:       3033
[12833.624005]   average write size:      69.7 KiB
[12833.624006]   nr direct reclaim:       0
[12833.624007]   nr background reclaim:   408443
[12833.624008]   reclaim kicked:          0
[12833.624009]   reclaim runs in:         0 ms
[12833.624010]   blocked:                 0
[12833.624011]   current entry sectors:   512
[12833.624012]   current entry error:     ok
[12833.624013]   current entry:           closed
[12833.624014]   unwritten entries:
[12833.624015]   last buf closed
[12833.624016]   space:
[12833.624016]     discarded              512:4193792
[12833.624018]     clean ondisk           512:4193792
[12833.624019]     clean                  512:4193792
[12833.624026]     total                  512:4194304
[12833.624026]   dev 0:
[12833.624027]     nr                     8192
[12833.624028]     bucket size            512
[12833.624029]     available              8190:512
[12833.624030]     discard_idx            0
[12833.624031]     dirty_ondisk           0 (seq 0)
[12833.624032]     dirty_idx              0 (seq 0)
[12833.624033]     cur_idx                0 (seq 0)
[12833.624034]   dev 1:
[12833.624035]     nr                     7631
[12833.624036]     bucket size            512
[12833.624036]     available              7629:0
[12833.624037]     discard_idx            6033
[12833.624038]     dirty_ondisk           6033 (seq 1031335)
[12833.624039]     dirty_idx              6033 (seq 1031335)
[12833.624040]     cur_idx                6033 (seq 1031335)
[12833.624041]   dev 2:
[12833.624042]     nr                     7631
[12833.624043]     bucket size            512
[12833.624044]     available              7629:240
[12833.624045]     discard_idx            6636
[12833.624045]     dirty_ondisk           6636 (seq 1031358)
[0] 0:bash*                                             "qemuvm" 03:17 22-Sep-24
[12833.624047]     cur_idx                6636 (seq 1031358)
[12833.624048]   dev 3:
[12833.624049]     nr                     8192
[12833.624050]     bucket size            512
[12833.624051]     available              8190:240
[12833.624052]     discard_idx            1574
[12833.624053]     dirty_ondisk           1574 (seq 1031358)
[12833.624054]     dirty_idx              1574 (seq 1031358)
[12833.624055]     cur_idx                1574 (seq 1031358)
[12833.624056]   dev 4:
[12833.624056]     nr                     8192
[12833.624057]     bucket size            1024
[12833.624058]     available              8190:1024
[12833.624059]     discard_idx            0
[12833.624060]     dirty_ondisk           0 (seq 0)
[12833.624067]     dirty_idx              0 (seq 0)
[12833.624068]     cur_idx                0 (seq 0)
[12833.624069] 

[0] 0:bash*                                             "qemuvm" 03:16 22

[truongsinh]

This time, it's kernel task of rebalance:

root@qemuvm:~# [  861.228704] INFO: task bch-rebalance/1:837 blocked for more than 122 seconds.
[  861.236360]       Not tainted 6.11.0-061100rc7daily20240915-generic #202409150203
[  861.250846] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  987.179736] Allocator stuck? Waited for 30 seconds
[  987.179752] Allocator debug:
[  987.179757]   capacity8344943698
[  987.179761]   reserved             725647278
[  987.179766]   hidden               16231424
[  987.179770]   btree                55331840
[  987.179774]   data                 7198854472
[  987.179779]   cached               0
[  987.179785]   reserved             0
[  987.179788]   online_reserved      0
[  987.179792]   nr_inodes            0
[  987.179796]   
[  987.179800]   freelist_wait        waiting
[  987.179805]   open buckets allocated4
[  987.179809]   open buckets total   1024
[  987.179814]   open_buckets_wait    empty
[  987.179818]   open_buckets_btree   2
[  987.179822]   open_buckets_user    1
[  987.179826]   btree reserve cache  1
[  987.179830] 
[  987.179834] Dev 0:
[  987.179838]                      buckets         sectors      fragmented
[  987.179843]   free                 67509               0               0
[  987.179925]   sb                      13            6152             504
[  987.179931]   journal               8192         4194304               0
[  987.179936]   btree                    0               0               0
[  987.179941]   user               4240318      2171042816               0
[  987.179945]   cached                   0               0               0
[  987.179950]   parity                   0               0               0
[  987.179954]   stripe                   0               0               0
[  987.179959]   need_gc_gens             0               0               0
[  987.179964]   need_discard             0               0               0
[  987.179968]   unstriped                0               0               0
[  987.179973]   capacity           4316032
[  987.179977]   
[  987.179981]   reserves:
[  987.179984]   stripe              134932
[  987.179990]   normal               67494
[  987.179994]   copygc                  56
[  987.179999]   btree                   28
[  987.180002]   btree_copygc             0
[  987.180007]   reclaim                  0
[  987.180011]   interior_updates         0
[  987.180014]   
[  987.180018]   open buckets             0
[  987.180023]   buckets to invalidate    0
[  987.180026] 
[  987.180030] Dev 1:
[  987.180034]                      buckets         sectors      fragmented
[  987.180039]   free                 15179               0               0
[  987.180044]   sb                      13            6152             504
[  987.180048]   journal               7631         3907072               0
[  987.180053]   btree                26798        13720576               0
[  987.180058]   user                927167       474709504               0
[  987.180062]   cached                   0               0               0
[  987.180067]   parity                   0               0               0
[  987.180071]   stripe                   0               0               0
[  987.180076]   need_gc_gens             0               0               0
[  987.180081]   need_discard             0               0               0
[  987.180085]   unstriped                0               0               0
[  987.180090]   capacity            976788
[  987.180094]   
[  987.180097]   reserves:
[  987.180101]   stripe               30580
[  987.180105]   normal               15318
[  987.180109]   copygc                  56
[  987.180114]   btree                   28
[  987.180118]   btree_copygc             0
[  987.180121]   reclaim                  0
[  987.180126]   interior_updates         0
[  987.180129]   
[  987.180133]   open buckets             1
[  987.180137]   buckets to invalidate    0
[  987.180141] 
[  987.180144] Dev 2:
[  987.180148]                      buckets         sectors      fragmented
[  987.180153]   free                 15190               0               0
[  987.180158]   sb                      13            6152             504
[  987.180162]   journal               7631         3907072               0
[  987.180167]   btree                27455        14056960               0
[  987.180172]   user                926499       474367304             184
[  987.180176]   cached                   0               0               0
[  987.180181]   parity                   0               0               0
[  987.180185]   stripe                   0               0               0
[  987.180190]   need_gc_gens             0               0               0
[  987.180194]   need_discard             0               0               0
[  987.180199]   unstriped                0               0               0
[  987.180203]   capacity            976788
[  987.180207]   
[  987.180211]   reserves:
[  987.180215]   stripe               30580
[  987.180219]   normal               15318
[  987.180223]   copygc                  56
[  987.180227]   btree                   28
[  987.180231]   btree_copygc             0
[  987.180235]   reclaim                  0
[  987.180239]   interior_updates         0
[  987.180242]   
[  987.180246]   open buckets             2
[  987.180250]   buckets to invalidate    0
[  987.180254] 
[  987.180257] Dev 3:
[  987.180261]                      buckets         sectors      fragmented
[  987.180266]   free                119199               0               0
[  987.180270]   sb                      13            6152             504
[  987.180275]   journal               8192         4194304               0
[  987.180280]   btree                53817        27554304               0
[  987.180284]   user               7449691      3814241792               0
[  987.180289]   cached                   0               0               0
[  987.180294]   parity                   0               0               0
[  987.180298]   stripe                   0               0               0
[  987.180303]   need_gc_gens             0               0               0
[  987.180307]   need_discard             0               0               0
[  987.180312]   unstriped                0               0               0
[  987.180316]   capacity           7630912
[  987.180320]   
[  987.180324]   reserves:
[  987.180328]   stripe              238522
[  987.180332]   normal              119289
[  987.180336]   copygc                  56
[  987.180340]   btree                   28
[  987.180344]   btree_copygc             0
[  987.180347]   reclaim                  0
[  987.180352]   interior_updates         0
[  987.180355]   
[  987.180359]   open buckets             0
[  987.180363]   buckets to invalidate    0
[  987.180367] 
[  987.180370] Dev 4:
[  987.180374]                      buckets         sectors      fragmented
[  987.180379]   free     18446744073709294247            0               0
[  987.180384]   sb                       1            1024               0
[  987.180388]   journal                  1            1024               0
[  987.180393]   btree                    0               0               0
[  987.180397]   user                258294       264493056               0
[  987.180402]   cached                   0               0               0
[  987.180406]   parity                   0               0               0
[  987.180411]   stripe                   0               0               0
[  987.180416]   need_gc_gens             0               0               0
[  987.180420]   need_discard             0               0               0
[  987.180425]   unstriped                0               0               0
[  987.180429]   capacity           1907739
[  987.180433]   
[  987.180437]   reserves:
[  987.180441]   stripe               59644
[  987.180445]   normal               29836
[  987.180449]   copygc                  28
[  987.180453]   btree                   14
[  987.180457]   btree_copygc             0
[  987.180461]   reclaim                  0
[  987.180465]   interior_updates         0
[  987.180469]   
[  987.180472]   open buckets             0
[  987.180476]   buckets to invalidate    0
[  987.180480] 
[  987.180483] Copygc debug:
[  987.180487]   running: 0
[  987.180491]   copygc_wait:0
[  987.180495]   copygc_wait_at:0
[  987.180499]   Currently waiting for:0 B
[  987.180503]   Currently waiting since:5.60 TiB
[  987.180507]   Currently calculated wait:0 B
[  987.180511] 
[  987.180515] Journal debug:
[  987.180519]   flags:                   replay_done,running,may_skip_flush
[  987.180524]   dirty journal entries:   0/32768
[  987.180528]   seq:                     1031382
[  987.180532]   seq_ondisk:              1031382
[  987.180536]   last_seq:                1031383
[  987.180540]   last_seq_ondisk:         1031382
[  987.180544]   flushed_seq_ondisk:      1031382
[  987.180548]   watermark:               stripe
[  987.180553]   each entry reserved:     321
[  987.180557]   nr flush writes:         15
[  987.180561]   nr noflush writes:       0
[  987.180565]   average write size:      6.69 KiB
[  987.180569]   nr direct reclaim:       0
[  987.180573]   nr background reclaim:   145
[  987.180577]   reclaim kicked:          0
[  987.180581]   reclaim runs in:         0 ms
[  987.180585]   blocked:                 0
[  987.180589]   current entry sectors:   512
[  987.180593]   current entry error:     ok
[  987.180597]   current entry:           closed
[  987.180602]   unwritten entries:
[  987.180606]   last buf closed
[  987.180609]   space:
[  987.180613]     discarded              512:4193792
[  987.180618]     clean ondisk           512:4193792
[  987.180622]     clean                  512:4193792
[  987.180626]     total                  512:4194304
[  987.180630]   dev 0:
[  987.180634]     nr                     8192
[  987.180638]     bucket size            512
[  987.180642]     available              8190:512
[  987.180646]     discard_idx            0
[  987.180651]     dirty_ondisk           0 (seq 0)
[  987.180655]     dirty_idx              0 (seq 0)
[  987.180659]     cur_idx                0 (seq 0)
[  987.180663]   dev 1:
[  987.180667]     nr                     7631
[  987.180671]     bucket size            512
[  987.180675]     available              7629:336
[  987.180679]     discard_idx            0
[  987.180683]     dirty_ondisk           0 (seq 1031381)
[  987.180688]     dirty_idx              0 (seq 1031381)
[  987.180693]     cur_idx                0 (seq 1031381)
[  987.180697]   dev 2:
[  987.180700]     nr                     7631
[  987.180704]     bucket size            512
[  987.180709]     available              7629:48
[  987.180713]     discard_idx            6636
[  987.180717]     dirty_ondisk           6636 (seq 1031382)
[  987.180721]     dirty_idx              6636 (seq 1031382)
[  987.180726]     cur_idx                6636 (seq 1031382)
[  987.180730]   dev 3:
[  987.180734]     nr                     8192
[  987.180738]     bucket size            512
[  987.180742]     available              8190:96
[  987.180746]     discard_idx            1574
[  987.180750]     dirty_ondisk           1574 (seq 1031382)
[  987.180755]     dirty_idx              1574 (seq 1031382)
[  987.180759]     cur_idx                1574 (seq 1031382)
[  987.180763]   dev 4:
[  987.180767]     nr                     8192
[  987.180771]     bucket size            1024
[  987.180775]     available              8190:1024
[  987.180779]     discard_idx            0
[  987.180783]     dirty_ondisk           0 (seq 0)
[  987.180787]     dirty_idx              0 (seq 0)
[  987.180791]     cur_idx                0 (seq 0)
[  987.180795]