From 05ecfd7acb1cd556aaeabb5fa5fdf248ee7fa4d9fc499a4a46186cb52bac7765 Mon Sep 17 00:00:00 2001 From: Mark Fasheh Date: Fri, 17 Jun 2016 21:21:18 +0000 Subject: [PATCH] - btrfsck updates for qgroup verification and repair (fate#318144) * We can now check all qgroup levels * btrfsck will write out corrected qgroups when run with --repair - Added patch: 0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch - Added patch: 0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch - Added patch: 0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch OBS-URL: https://build.opensuse.org/package/show/filesystems/btrfsprogs?expand=0&rev=241 --- ...-progs-free-qgroup-counts-in-btrfsck.patch | 62 +++ ...btrfsck-verify-qgroups-above-level-0.patch | 493 ++++++++++++++++++ ...sck-write-corrected-qgroup-info-to-d.patch | 418 +++++++++++++++ btrfsprogs.changes | 10 + btrfsprogs.spec | 7 + 5 files changed, 990 insertions(+) create mode 100644 0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch create mode 100644 0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch create mode 100644 0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch diff --git a/0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch b/0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch new file mode 100644 index 0000000..528e10b --- /dev/null +++ b/0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch @@ -0,0 +1,62 @@ +From 4995239a8e53ef58b788a6df6263318cb656cd79 Mon Sep 17 00:00:00 2001 +From: Mark Fasheh +Date: Wed, 15 Jun 2016 13:28:28 -0700 +Subject: [PATCH 1/3] btrfs-progs: free qgroup counts in btrfsck + +Signed-off-by: Mark Fasheh +--- + cmds-check.c | 1 + + qgroup-verify.c | 13 +++++++++++++ + qgroup-verify.h | 2 ++ + 3 files changed, 16 insertions(+) + +diff --git a/cmds-check.c b/cmds-check.c +index bada87e..7b65f89 100644 +--- a/cmds-check.c ++++ b/cmds-check.c +@@ -9890,6 +9890,7 @@ out: + (unsigned long long)data_bytes_allocated, + (unsigned long long)data_bytes_referenced); + ++ free_qgroup_counts(); + free_root_recs_tree(&root_cache); + close_out: + close_ctree(root); +diff --git a/qgroup-verify.c b/qgroup-verify.c +index 1a0d38c..7b78504 100644 +--- a/qgroup-verify.c ++++ b/qgroup-verify.c +@@ -1095,6 +1095,19 @@ int report_qgroups(int all) + return ret; + } + ++void free_qgroup_counts(void) ++{ ++ struct rb_node *node; ++ struct qgroup_count *c; ++ node = rb_first(&counts.root); ++ while (node) { ++ c = rb_entry(node, struct qgroup_count, rb_node); ++ node = rb_next(node); ++ rb_erase(&c->rb_node, &counts.root); ++ free(c); ++ } ++} ++ + int qgroup_verify_all(struct btrfs_fs_info *info) + { + int ret; +diff --git a/qgroup-verify.h b/qgroup-verify.h +index 3747465..0f8ff9b 100644 +--- a/qgroup-verify.h ++++ b/qgroup-verify.h +@@ -27,4 +27,6 @@ int report_qgroups(int all); + + int print_extent_state(struct btrfs_fs_info *info, u64 subvol); + ++void free_qgroup_counts(void); ++ + #endif +-- +2.1.4 + diff --git a/0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch b/0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch new file mode 100644 index 0000000..6fe117e --- /dev/null +++ b/0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch @@ -0,0 +1,493 @@ +From 84064263d0128536113ae7ced9ad2f0d17f7663e Mon Sep 17 00:00:00 2001 +From: Mark Fasheh +Date: Wed, 15 Jun 2016 13:37:55 -0700 +Subject: [PATCH 2/3] btrfs-progs: btrfsck: verify qgroups above level 0 + +At the moment we only check subvolume quota groups (level 0). With this +patch we can check groups above 0, thus verifying the entire qgroup +hierarchy on a file system. The accounting portion of this patch is modeled +after the kernel - we are essentially reimplementing the 'quota rescan' case +here. Most other sections of this code went unchanged, in particular the +root counting works independently of the accounting. + +Signed-off-by: Mark Fasheh +--- + qgroup-verify.c | 309 +++++++++++++++++++++++++++++++++++++++++++++----------- + 1 file changed, 252 insertions(+), 57 deletions(-) + +diff --git a/qgroup-verify.c b/qgroup-verify.c +index 7b78504..23f2961 100644 +--- a/qgroup-verify.c ++++ b/qgroup-verify.c +@@ -35,7 +35,8 @@ + /*#define QGROUP_VERIFY_DEBUG*/ + static unsigned long tot_extents_scanned = 0; + +-static void add_bytes(u64 root_objectid, u64 num_bytes, int exclusive); ++struct qgroup_count; ++static struct qgroup_count *find_count(u64 qgroupid); + + struct qgroup_info { + u64 referenced; +@@ -54,6 +55,16 @@ struct qgroup_count { + struct qgroup_info info; + + struct rb_node rb_node; ++ ++ struct list_head groups; /* Parents when we are a child group */ ++ ++ /* ++ * Children when we are a parent group (not currently used but ++ * maintained to mirror kernel handling of qgroups) ++ */ ++ struct list_head members; ++ ++ u64 cur_refcnt; + }; + + static struct counts_tree { +@@ -66,6 +77,39 @@ static struct counts_tree { + static struct rb_root by_bytenr = RB_ROOT; + + /* ++ * Glue structure to represent the relations between qgroups. Mirrored ++ * from kernel. ++ */ ++struct btrfs_qgroup_list { ++ struct list_head next_group; ++ struct list_head next_member; ++ struct qgroup_count *group; /* Parent group */ ++ struct qgroup_count *member; ++}; ++ ++/* Allow us to reset ref counts during accounting without zeroing each group. */ ++static u64 qgroup_seq = 1ULL; ++ ++static inline void update_cur_refcnt(struct qgroup_count *c) ++{ ++ if (c->cur_refcnt < qgroup_seq) ++ c->cur_refcnt = qgroup_seq; ++ c->cur_refcnt++; ++} ++ ++static inline u64 group_get_cur_refcnt(struct qgroup_count *c) ++{ ++ if (c->cur_refcnt < qgroup_seq) ++ return 0; ++ return c->cur_refcnt - qgroup_seq; ++} ++ ++static void inc_qgroup_seq(int root_count) ++{ ++ qgroup_seq += root_count + 1; ++} ++ ++/* + * List of interior tree blocks. We walk this list after loading the + * extent tree to resolve implied refs. For each interior node we'll + * place a shared ref in the ref tree against each child object. This +@@ -296,9 +340,10 @@ static void find_parent_roots(struct ulist *roots, u64 parent) + } + + do { +- if (ref->root) +- ulist_add(roots, ref->root, 0, 0); +- else ++ if (ref->root) { ++ if (is_fstree(ref->root)) ++ ulist_add(roots, ref->root, 0, 0); ++ } else + find_parent_roots(roots, ref->parent); + + node = rb_next(node); +@@ -307,6 +352,114 @@ static void find_parent_roots(struct ulist *roots, u64 parent) + } while (node && ref->bytenr == parent); + } + ++static int account_one_extent(struct ulist *roots, u64 bytenr, u64 num_bytes) ++{ ++ int ret; ++ u64 id, nr_roots, nr_refs; ++ struct qgroup_count *count; ++ struct ulist *counts = ulist_alloc(0); ++ struct ulist *tmp = ulist_alloc(0); ++ struct ulist_iterator uiter; ++ struct ulist_iterator tmp_uiter; ++ struct ulist_node *unode; ++ struct ulist_node *tmp_unode; ++ struct btrfs_qgroup_list *glist; ++ ++ if (!counts || !tmp) { ++ ulist_free(counts); ++ ulist_free(tmp); ++ return ENOMEM; ++ } ++ ++ ULIST_ITER_INIT(&uiter); ++ while ((unode = ulist_next(roots, &uiter))) { ++ BUG_ON(unode->val == 0ULL); ++ ++ /* ++ * For each root, find their corresponding tracking group and ++ * add it to our qgroups list. ++ */ ++ count = find_count(unode->val); ++ if (!count) ++ continue; ++ ++ BUG_ON(!is_fstree(unode->val)); ++ ret = ulist_add(counts, count->qgroupid, ptr_to_u64(count), 0); ++ if (ret < 0) ++ goto out; ++ ++ /* ++ * Now we look for parents (and parents of those...). Use a tmp ++ * ulist here to avoid re-walking (and re-incrementing) our ++ * already added items on every loop iteration. ++ */ ++ ulist_reinit(tmp); ++ ret = ulist_add(tmp, count->qgroupid, ptr_to_u64(count), 0); ++ if (ret < 0) ++ goto out; ++ ++ ULIST_ITER_INIT(&tmp_uiter); ++ while ((tmp_unode = ulist_next(tmp, &tmp_uiter))) { ++ /* Bump the refcount on a node every time we see it. */ ++ count = u64_to_ptr(tmp_unode->aux); ++ update_cur_refcnt(count); ++ ++ list_for_each_entry(glist, &count->groups, next_group) { ++ struct qgroup_count *parent; ++ parent = glist->group; ++ id = parent->qgroupid; ++ ++ BUG_ON(!count); ++ ++ ret = ulist_add(counts, id, ptr_to_u64(parent), ++ 0); ++ if (ret < 0) ++ goto out; ++ ret = ulist_add(tmp, id, ptr_to_u64(parent), ++ 0); ++ if (ret < 0) ++ goto out; ++ } ++ } ++ } ++ ++ /* ++ * Now that we have gathered up and counted all the groups, we ++ * can add bytes for this ref. ++ */ ++ nr_roots = roots->nnodes; ++ ULIST_ITER_INIT(&uiter); ++ while ((unode = ulist_next(counts, &uiter))) { ++ count = u64_to_ptr(unode->aux); ++ ++ nr_refs = group_get_cur_refcnt(count); ++ if (nr_refs) { ++ count->info.referenced += num_bytes; ++ count->info.referenced_compressed += num_bytes; ++ ++ if (nr_refs == nr_roots) { ++ count->info.exclusive += num_bytes; ++ count->info.exclusive_compressed += num_bytes; ++ } ++ } ++#ifdef QGROUP_VERIFY_DEBUG ++ printf("account (%llu, %llu), qgroup %llu/%llu, rfer %llu," ++ " excl %llu, refs %llu, roots %llu\n", bytenr, num_bytes, ++ btrfs_qgroup_level(count->qgroupid), ++ btrfs_qgroup_subvid(count->qgroupid), ++ count->info.referenced, count->info.exclusive, nr_refs, ++ nr_roots); ++#endif ++ } ++ ++ inc_qgroup_seq(roots->nnodes); ++ ret = 0; ++out: ++ ulist_free(counts); ++ ulist_free(tmp); ++ return ret; ++} ++ + static void print_subvol_info(u64 subvolid, u64 bytenr, u64 num_bytes, + struct ulist *roots); + /* +@@ -318,18 +471,15 @@ static void print_subvol_info(u64 subvolid, u64 bytenr, u64 num_bytes, + * - resolve all possible roots for shared refs, insert each + * of those into ref_roots ulist (this is a recursive process) + * +- * - Walk ref_roots ulist, adding extent bytes to each qgroup count that +- * cooresponds to a found root. ++ * - With all roots resolved we can account the ref - this is done in ++ * account_one_extent(). + */ +-static void account_all_refs(int do_qgroups, u64 search_subvol) ++static int account_all_refs(int do_qgroups, u64 search_subvol) + { +- int exclusive; + struct ref *ref; + struct rb_node *node; + u64 bytenr, num_bytes; + struct ulist *roots = ulist_alloc(0); +- struct ulist_iterator uiter; +- struct ulist_node *unode; + + node = rb_first(&by_bytenr); + while (node) { +@@ -347,10 +497,14 @@ static void account_all_refs(int do_qgroups, u64 search_subvol) + do { + BUG_ON(ref->bytenr != bytenr); + BUG_ON(ref->num_bytes != num_bytes); +- if (ref->root) +- ulist_add(roots, ref->root, 0, 0); +- else ++ if (ref->root) { ++ if (is_fstree(ref->root)) { ++ if (ulist_add(roots, ref->root, 0, 0) < 0) ++ goto enomem; ++ } ++ } else { + find_parent_roots(roots, ref->parent); ++ } + + /* + * When we leave this inner loop, node is set +@@ -362,29 +516,22 @@ static void account_all_refs(int do_qgroups, u64 search_subvol) + ref = rb_entry(node, struct ref, bytenr_node); + } while (node && ref->bytenr == bytenr); + +- /* +- * Now that we have all roots, we can properly account +- * this extent against the corresponding qgroups. +- */ +- if (roots->nnodes == 1) +- exclusive = 1; +- else +- exclusive = 0; +- + if (search_subvol) + print_subvol_info(search_subvol, bytenr, num_bytes, + roots); + +- ULIST_ITER_INIT(&uiter); +- while ((unode = ulist_next(roots, &uiter))) { +- BUG_ON(unode->val == 0ULL); +- /* We only want to account fs trees */ +- if (is_fstree(unode->val) && do_qgroups) +- add_bytes(unode->val, num_bytes, exclusive); +- } ++ if (!do_qgroups) ++ continue; ++ ++ if (account_one_extent(roots, bytenr, num_bytes)) ++ goto enomem; + } + + ulist_free(roots); ++ return 0; ++enomem: ++ error("Out of memory while accounting refs for qgroups!\n"); ++ return -ENOMEM; + } + + static u64 resolve_one_root(u64 bytenr) +@@ -668,6 +815,8 @@ static struct qgroup_count *alloc_count(struct btrfs_disk_key *key, + item->exclusive = btrfs_qgroup_info_exclusive(leaf, disk); + item->exclusive_compressed = + btrfs_qgroup_info_exclusive_compressed(leaf, disk); ++ INIT_LIST_HEAD(&c->groups); ++ INIT_LIST_HEAD(&c->members); + + if (insert_count(c)) { + free(c); +@@ -677,29 +826,30 @@ static struct qgroup_count *alloc_count(struct btrfs_disk_key *key, + return c; + } + +-static void add_bytes(u64 root_objectid, u64 num_bytes, int exclusive) ++static int add_qgroup_relation(u64 memberid, u64 parentid) + { +- struct qgroup_count *count = find_count(root_objectid); +- struct qgroup_info *qg; ++ struct qgroup_count *member; ++ struct qgroup_count *parent; ++ struct btrfs_qgroup_list *list; + +- BUG_ON(num_bytes < 4096); /* Random sanity check. */ ++ if (memberid > parentid) ++ return 0; + +- if (!count) +- return; ++ member = find_count(memberid); ++ parent = find_count(parentid); ++ if (!member || !parent) ++ return -ENOENT; + +- qg = &count->info; ++ list = calloc(1, sizeof(*list)); ++ if (!list) ++ return -ENOMEM; + +- qg->referenced += num_bytes; +- /* +- * count of compressed bytes is unimplemented, so we do the +- * same as kernel. +- */ +- qg->referenced_compressed += num_bytes; ++ list->group = parent; ++ list->member = member; ++ list_add_tail(&list->next_group, &member->groups); ++ list_add_tail(&list->next_member, &parent->members); + +- if (exclusive) { +- qg->exclusive += num_bytes; +- qg->exclusive_compressed += num_bytes; +- } ++ return 0; + } + + static void read_qgroup_status(struct btrfs_path *path, +@@ -728,11 +878,18 @@ static int load_quota_info(struct btrfs_fs_info *info) + struct btrfs_qgroup_info_item *item; + struct qgroup_count *count; + int i, nr; ++ int search_relations = 0; + ++loop: ++ /* ++ * Do 2 passes, the first allocates group counts and reads status ++ * items. The 2nd pass picks up relation items and glues them ++ * to their respective count structures. ++ */ + btrfs_init_path(&path); + + key.offset = 0; +- key.objectid = 0; ++ key.objectid = search_relations ? 0 : BTRFS_QGROUP_RELATION_KEY; + key.type = 0; + + ret = btrfs_search_slot(NULL, root, &key, &path, 0, 0); +@@ -749,17 +906,26 @@ static int load_quota_info(struct btrfs_fs_info *info) + btrfs_item_key(leaf, &disk_key, i); + btrfs_disk_key_to_cpu(&key, &disk_key); + ++ if (search_relations) { ++ if (key.type == BTRFS_QGROUP_RELATION_KEY) { ++ ret = add_qgroup_relation(key.objectid, ++ key.offset); ++ if (ret) { ++ error("out of memory\n"); ++ goto out; ++ } ++ } ++ continue; ++ } ++ + if (key.type == BTRFS_QGROUP_STATUS_KEY) { + read_qgroup_status(&path, &counts); + continue; + } +- if (key.type == BTRFS_QGROUP_RELATION_KEY) +- printf("Ignoring qgroup relation key %llu\n", +- key.objectid); + + /* +- * Ignore: BTRFS_QGROUP_LIMIT_KEY, +- * BTRFS_QGROUP_RELATION_KEY ++ * At this point, we can ignore anything that ++ * isn't a qgroup info. + */ + if (key.type != BTRFS_QGROUP_INFO_KEY) + continue; +@@ -791,6 +957,12 @@ static int load_quota_info(struct btrfs_fs_info *info) + + ret = 0; + btrfs_release_path(&path); ++ ++ if (!search_relations) { ++ search_relations = 1; ++ goto loop; ++ } ++ + out: + return ret; + } +@@ -1035,6 +1207,11 @@ static void print_fields_signed(long long bytes, + prefix, type, bytes, type, bytes_compressed); + } + ++static inline int qgroup_printable(struct qgroup_count *c) ++{ ++ return !!(c->subvol_exists || btrfs_qgroup_level(c->qgroupid)); ++} ++ + static int report_qgroup_difference(struct qgroup_count *count, int verbose) + { + int is_different; +@@ -1045,9 +1222,10 @@ static int report_qgroup_difference(struct qgroup_count *count, int verbose) + + is_different = excl_diff || ref_diff; + +- if (verbose || (is_different && count->subvol_exists)) { +- printf("Counts for qgroup id: %llu %s\n", +- (unsigned long long)count->qgroupid, ++ if (verbose || (is_different && qgroup_printable(count))) { ++ printf("Counts for qgroup id: %llu/%llu %s\n", ++ btrfs_qgroup_level(count->qgroupid), ++ btrfs_qgroup_subvid(count->qgroupid), + is_different ? "are different" : ""); + + print_fields(info->referenced, info->referenced_compressed, +@@ -1099,10 +1277,27 @@ void free_qgroup_counts(void) + { + struct rb_node *node; + struct qgroup_count *c; ++ struct btrfs_qgroup_list *glist, *tmpglist; ++ + node = rb_first(&counts.root); + while (node) { + c = rb_entry(node, struct qgroup_count, rb_node); ++ ++ list_for_each_entry_safe(glist, tmpglist, &c->groups, ++ next_group) { ++ list_del(&glist->next_group); ++ list_del(&glist->next_member); ++ free(glist); ++ } ++ list_for_each_entry_safe(glist, tmpglist, &c->members, ++ next_group) { ++ list_del(&glist->next_group); ++ list_del(&glist->next_member); ++ free(glist); ++ } ++ + node = rb_next(node); ++ + rb_erase(&c->rb_node, &counts.root); + free(c); + } +@@ -1143,7 +1338,7 @@ int qgroup_verify_all(struct btrfs_fs_info *info) + goto out; + } + +- account_all_refs(1, 0); ++ ret = account_all_refs(1, 0); + + out: + /* +@@ -1215,7 +1410,7 @@ int print_extent_state(struct btrfs_fs_info *info, u64 subvol) + } + + printf("Offset\t\tLen\tRoot Refs\tRoots\n"); +- account_all_refs(0, subvol); ++ ret = account_all_refs(0, subvol); + + out: + free_tree_blocks(); +-- +2.1.4 + diff --git a/0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch b/0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch new file mode 100644 index 0000000..f93aff1 --- /dev/null +++ b/0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch @@ -0,0 +1,418 @@ +From 822afe3408638be63e0fcebf98ac9780af5afc8d Mon Sep 17 00:00:00 2001 +From: Mark Fasheh +Date: Thu, 16 Jun 2016 16:06:28 -0700 +Subject: [PATCH 3/3] btrfs-progs: btrfsck: write corrected qgroup info to disk + +Now that we can verify all qgroups, we can write the corrected qgroups out +to disk when '--repair' is specified. The qgroup status item is also updated +to clear any out-of-date state. The repair_ functions were modeled after the +inode repair code in cmds-check.c. + +I also renamed the 'scan' member of qgroup_status_item to 'rescan' in order +to keep consistency with the kernel. + +Testing this was easy, I just reproduced qgroup inconsistencies via the +usual routes and had btrfsck fix them. + +Signed-off-by: Mark Fasheh +--- + cmds-check.c | 15 +++-- + ctree.h | 10 ++-- + print-tree.c | 2 +- + qgroup-verify.c | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- + qgroup-verify.h | 3 +- + repair.c | 2 + + repair.h | 2 + + 7 files changed, 189 insertions(+), 23 deletions(-) + +diff --git a/cmds-check.c b/cmds-check.c +index 7b65f89..b7f4bd5 100644 +--- a/cmds-check.c ++++ b/cmds-check.c +@@ -67,7 +67,6 @@ static u64 data_bytes_referenced = 0; + static int found_old_backref = 0; + static LIST_HEAD(duplicate_extents); + static LIST_HEAD(delete_items); +-static int repair = 0; + static int no_holes = 0; + static int init_extent_tree = 0; + static int check_data_csum = 0; +@@ -9543,6 +9542,7 @@ int cmd_check(int argc, char **argv) + int init_csum_tree = 0; + int readonly = 0; + int qgroup_report = 0; ++ int qgroups_repaired = 0; + enum btrfs_open_ctree_flags ctree_flags = OPEN_CTREE_EXCLUSIVE; + + while(1) { +@@ -9698,7 +9698,7 @@ int cmd_check(int argc, char **argv) + uuidbuf); + ret = qgroup_verify_all(info); + if (ret == 0) +- ret = report_qgroups(1); ++ report_qgroups(1); + goto close_out; + } + if (subvolid) { +@@ -9852,6 +9852,10 @@ int cmd_check(int argc, char **argv) + err = qgroup_verify_all(info); + if (err) + goto out; ++ report_qgroups(0); ++ err = repair_qgroups(info, &qgroups_repaired); ++ if (err) ++ goto out; + } + + if (!list_empty(&root->fs_info->recow_ebs)) { +@@ -9860,10 +9864,9 @@ int cmd_check(int argc, char **argv) + } + out: + /* Don't override original ret */ +- if (ret) +- report_qgroups(0); +- else +- ret = report_qgroups(0); ++ if (!ret && qgroups_repaired) ++ ret = qgroups_repaired; ++ + if (found_old_backref) { /* + * there was a disk format change when mixed + * backref was in testing tree. The old format +diff --git a/ctree.h b/ctree.h +index 86227f8..34c6b73 100644 +--- a/ctree.h ++++ b/ctree.h +@@ -897,7 +897,7 @@ struct btrfs_qgroup_status_item { + __le64 version; + __le64 generation; + __le64 flags; +- __le64 scan; /* progress during scanning */ ++ __le64 rescan; /* progress during scanning */ + } __attribute__ ((__packed__)); + + struct btrfs_block_group_item { +@@ -2130,8 +2130,8 @@ BTRFS_SETGET_FUNCS(qgroup_status_generation, struct btrfs_qgroup_status_item, + generation, 64); + BTRFS_SETGET_FUNCS(qgroup_status_flags, struct btrfs_qgroup_status_item, + flags, 64); +-BTRFS_SETGET_FUNCS(qgroup_status_scan, struct btrfs_qgroup_status_item, +- scan, 64); ++BTRFS_SETGET_FUNCS(qgroup_status_rescan, struct btrfs_qgroup_status_item, ++ rescan, 64); + + BTRFS_SETGET_STACK_FUNCS(stack_qgroup_status_version, + struct btrfs_qgroup_status_item, version, 64); +@@ -2139,8 +2139,8 @@ BTRFS_SETGET_STACK_FUNCS(stack_qgroup_status_generation, + struct btrfs_qgroup_status_item, generation, 64); + BTRFS_SETGET_STACK_FUNCS(stack_qgroup_status_flags, + struct btrfs_qgroup_status_item, flags, 64); +-BTRFS_SETGET_STACK_FUNCS(stack_qgroup_status_scan, +- struct btrfs_qgroup_status_item, scan, 64); ++BTRFS_SETGET_STACK_FUNCS(stack_qgroup_status_rescan, ++ struct btrfs_qgroup_status_item, rescan, 64); + + /* btrfs_qgroup_info_item */ + BTRFS_SETGET_FUNCS(qgroup_info_generation, struct btrfs_qgroup_info_item, +diff --git a/print-tree.c b/print-tree.c +index 746f25b..9f9e11e 100644 +--- a/print-tree.c ++++ b/print-tree.c +@@ -1037,7 +1037,7 @@ void btrfs_print_leaf(struct btrfs_root *root, struct extent_buffer *l) + btrfs_qgroup_status_generation(l, qg_status), + flags_str, + (unsigned long long) +- btrfs_qgroup_status_scan(l, qg_status)); ++ btrfs_qgroup_status_rescan(l, qg_status)); + break; + case BTRFS_QGROUP_RELATION_KEY: + break; +diff --git a/qgroup-verify.c b/qgroup-verify.c +index 23f2961..12921ad 100644 +--- a/qgroup-verify.c ++++ b/qgroup-verify.c +@@ -29,6 +29,8 @@ + #include "utils.h" + #include "ulist.h" + #include "rbtree-utils.h" ++#include "repair.h" ++#include "transaction.h" + + #include "qgroup-verify.h" + +@@ -65,6 +67,8 @@ struct qgroup_count { + struct list_head members; + + u64 cur_refcnt; ++ ++ struct list_head bad_list; + }; + + static struct counts_tree { +@@ -74,6 +78,8 @@ static struct counts_tree { + unsigned int qgroup_inconsist:1; + } counts = { .root = RB_ROOT }; + ++static LIST_HEAD(bad_qgroups); ++ + static struct rb_root by_bytenr = RB_ROOT; + + /* +@@ -817,6 +823,7 @@ static struct qgroup_count *alloc_count(struct btrfs_disk_key *key, + btrfs_qgroup_info_exclusive_compressed(leaf, disk); + INIT_LIST_HEAD(&c->groups); + INIT_LIST_HEAD(&c->members); ++ INIT_LIST_HEAD(&c->bad_list); + + if (insert_count(c)) { + free(c); +@@ -1243,34 +1250,36 @@ static int report_qgroup_difference(struct qgroup_count *count, int verbose) + print_fields_signed(excl_diff, excl_diff, + "diff:", "exclusive"); + } +- return (is_different && count->subvol_exists); ++ ++ return is_different; + } + +-int report_qgroups(int all) ++void report_qgroups(int all) + { + struct rb_node *node; + struct qgroup_count *c; +- int ret = 0; + +- if (counts.rescan_running) { ++ if (!repair && counts.rescan_running) { + if (all) { + printf( +- "Qgroup rescan is running, qgroup counts difference is expected\n"); ++ "Qgroup rescan is running, a difference in qgroup counts is expected\n"); + } else { + printf( +- "Qgroup rescan is running, ignore qgroup check\n"); +- return ret; ++ "Qgroup rescan is running, qgroups will not be printed.\n"); ++ return; + } + } + if (counts.qgroup_inconsist && !counts.rescan_running) +- fprintf(stderr, "Qgroup is already inconsistent before checking\n"); ++ fprintf(stderr, "Qgroup are marked as inconsistent.\n"); + node = rb_first(&counts.root); + while (node) { + c = rb_entry(node, struct qgroup_count, rb_node); +- ret |= report_qgroup_difference(c, all); ++ ++ if (report_qgroup_difference(c, all)) ++ list_add_tail(&c->bad_list, &bad_qgroups); ++ + node = rb_next(node); + } +- return ret; + } + + void free_qgroup_counts(void) +@@ -1283,6 +1292,8 @@ void free_qgroup_counts(void) + while (node) { + c = rb_entry(node, struct qgroup_count, rb_node); + ++ list_del(&c->bad_list); ++ + list_for_each_entry_safe(glist, tmpglist, &c->groups, + next_group) { + list_del(&glist->next_group); +@@ -1418,3 +1429,150 @@ out: + return ret; + } + ++static int repair_qgroup_info(struct btrfs_fs_info *info, ++ struct qgroup_count *count) ++{ ++ int ret; ++ struct btrfs_root *root = info->quota_root; ++ struct btrfs_trans_handle *trans; ++ struct btrfs_path *path; ++ struct btrfs_qgroup_info_item *info_item; ++ struct btrfs_key key; ++ ++ printf("Repair qgroup %llu/%llu\n", btrfs_qgroup_level(count->qgroupid), ++ btrfs_qgroup_subvid(count->qgroupid)); ++ ++ path = btrfs_alloc_path(); ++ if (!path) ++ return -ENOMEM; ++ ++ trans = btrfs_start_transaction(root, 1); ++ if (IS_ERR(trans)) { ++ btrfs_free_path(path); ++ return PTR_ERR(trans); ++ } ++ ++ key.objectid = 0; ++ key.type = BTRFS_QGROUP_INFO_KEY; ++ key.offset = count->qgroupid; ++ ret = btrfs_search_slot(trans, root, &key, path, 0, 1); ++ if (ret) { ++ error("Could not find disk item for qgroup %llu/%llu.\n", ++ btrfs_qgroup_level(count->qgroupid), ++ btrfs_qgroup_subvid(count->qgroupid)); ++ if (ret > 0) ++ ret = -ENOENT; ++ goto out; ++ } ++ ++ info_item = btrfs_item_ptr(path->nodes[0], path->slots[0], ++ struct btrfs_qgroup_info_item); ++ ++ btrfs_set_qgroup_info_generation(path->nodes[0], info_item, ++ trans->transid); ++ ++ btrfs_set_qgroup_info_referenced(path->nodes[0], info_item, ++ count->info.referenced); ++ btrfs_set_qgroup_info_referenced_compressed(path->nodes[0], info_item, ++ count->info.referenced_compressed); ++ ++ btrfs_set_qgroup_info_exclusive(path->nodes[0], info_item, ++ count->info.exclusive); ++ btrfs_set_qgroup_info_exclusive_compressed(path->nodes[0], info_item, ++ count->info.exclusive_compressed); ++ ++ btrfs_mark_buffer_dirty(path->nodes[0]); ++ ++out: ++ btrfs_commit_transaction(trans, root); ++ btrfs_free_path(path); ++ ++ return ret; ++} ++ ++static int repair_qgroup_status(struct btrfs_fs_info *info) ++{ ++ int ret; ++ struct btrfs_root *root = info->quota_root; ++ struct btrfs_trans_handle *trans; ++ struct btrfs_path *path; ++ struct btrfs_key key; ++ struct btrfs_qgroup_status_item *status_item; ++ ++ printf("Repair qgroup status item\n"); ++ ++ path = btrfs_alloc_path(); ++ if (!path) ++ return -ENOMEM; ++ ++ trans = btrfs_start_transaction(root, 1); ++ if (IS_ERR(trans)) { ++ btrfs_free_path(path); ++ return PTR_ERR(trans); ++ } ++ ++ key.objectid = 0; ++ key.type = BTRFS_QGROUP_STATUS_KEY; ++ key.offset = 0; ++ ret = btrfs_search_slot(trans, root, &key, path, 0, 1); ++ if (ret) { ++ error("Could not find qgroup status item\n"); ++ if (ret > 0) ++ ret = -ENOENT; ++ goto out; ++ } ++ ++ status_item = btrfs_item_ptr(path->nodes[0], path->slots[0], ++ struct btrfs_qgroup_status_item); ++ btrfs_set_qgroup_status_flags(path->nodes[0], status_item, ++ BTRFS_QGROUP_STATUS_FLAG_ON); ++ btrfs_set_qgroup_status_rescan(path->nodes[0], status_item, 0); ++ btrfs_set_qgroup_status_generation(path->nodes[0], status_item, ++ trans->transid); ++ ++ btrfs_mark_buffer_dirty(path->nodes[0]); ++ ++out: ++ btrfs_commit_transaction(trans, root); ++ btrfs_free_path(path); ++ ++ return ret; ++} ++ ++int repair_qgroups(struct btrfs_fs_info *info, int *repaired) ++{ ++ int ret; ++ struct qgroup_count *count, *tmpcount; ++ ++ *repaired = 0; ++ ++ if (!repair) ++ return 0; ++ ++ list_for_each_entry_safe(count, tmpcount, &bad_qgroups, bad_list) { ++ ret = repair_qgroup_info(info, count); ++ if (ret) { ++ goto out; ++ } ++ ++ (*repaired)++; ++ ++ list_del_init(&count->bad_list); ++ } ++ ++ /* ++ * Do this step last as we want the latest transaction id on ++ * our qgroup status to avoid a (useless) warning after ++ * mount. ++ */ ++ if (*repaired || counts.qgroup_inconsist || counts.rescan_running) { ++ ret = repair_qgroup_status(info); ++ if (ret) ++ goto out; ++ ++ (*repaired)++; ++ } ++ ++out: ++ return ret; ++} +diff --git a/qgroup-verify.h b/qgroup-verify.h +index 0f8ff9b..d7d83a4 100644 +--- a/qgroup-verify.h ++++ b/qgroup-verify.h +@@ -23,7 +23,8 @@ + #include "ctree.h" + + int qgroup_verify_all(struct btrfs_fs_info *info); +-int report_qgroups(int all); ++void report_qgroups(int all); ++int repair_qgroups(struct btrfs_fs_info *info, int *repaired); + + int print_extent_state(struct btrfs_fs_info *info, u64 subvol); + +diff --git a/repair.c b/repair.c +index 4f74742..07a1232 100644 +--- a/repair.c ++++ b/repair.c +@@ -21,6 +21,8 @@ + #include "utils.h" + #include "repair.h" + ++int repair = 0; ++ + int btrfs_add_corrupt_extent_record(struct btrfs_fs_info *info, + struct btrfs_key *first_key, + u64 start, u64 len, int level) +diff --git a/repair.h b/repair.h +index 3fc0e8b..355bbf2 100644 +--- a/repair.h ++++ b/repair.h +@@ -21,6 +21,8 @@ + + #include "ctree.h" + ++extern int repair; /* repair mode */ ++ + struct btrfs_corrupt_block { + struct cache_extent cache; + struct btrfs_key key; +-- +2.1.4 + diff --git a/btrfsprogs.changes b/btrfsprogs.changes index abe6a71..31d435e 100644 --- a/btrfsprogs.changes +++ b/btrfsprogs.changes @@ -1,3 +1,13 @@ +------------------------------------------------------------------- +Fri Jun 17 21:16:12 UTC 2016 - mfasheh@suse.com + +- btrfsck updates for qgroup verification and repair (fate#318144) + * We can now check all qgroup levels + * btrfsck will write out corrected qgroups when run with --repair + - Added patch: 0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch + - Added patch: 0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch + - Added patch: 0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch + ------------------------------------------------------------------- Mon May 16 00:00:00 CEST 2016 - dsterba@suse.cz diff --git a/btrfsprogs.spec b/btrfsprogs.spec index 0179cda..0d82f0a 100644 --- a/btrfsprogs.spec +++ b/btrfsprogs.spec @@ -37,6 +37,10 @@ Source: https://www.kernel.org/pub/linux/kernel/people/kdave/btrfs-progs Source1: boot-btrfs.sh Source4: setup-btrfs.sh +Patch1: 0001-btrfs-progs-free-qgroup-counts-in-btrfsck.patch +Patch2: 0002-btrfs-progs-btrfsck-verify-qgroups-above-level-0.patch +Patch3: 0003-btrfs-progs-btrfsck-write-corrected-qgroup-info-to-d.patch + Patch163: 0163-btrfs-progs-fsck-fix-segfault.patch Patch167: 0167-Btrfs-progs-make-find_and_setup_root-return-an-error.patch Patch168: 0168-Btrfs-progs-don-t-bug-out-if-we-can-t-find-the-last-.patch @@ -111,6 +115,9 @@ build applications to interface with btrfs. %prep %setup -q -n btrfs-progs-v%{version} +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 %patch163 -p1 %patch167 -p1 %patch168 -p1