mm/vmstat: add events for THP migration without split

Add following new vmstat events which will help in validating THP
migration without split.  Statistics reported through these new VM events
will help in performance debugging.

1. THP_MIGRATION_SUCCESS
2. THP_MIGRATION_FAILURE
3. THP_MIGRATION_SPLIT

In addition, these new events also update normal page migration statistics
appropriately via PGMIGRATE_SUCCESS and PGMIGRATE_FAILURE.  While here,
this updates current trace event 'mm_migrate_pages' to accommodate now
available THP statistics.

[akpm@linux-foundation.org: s/hpage_nr_pages/thp_nr_pages/]
[ziy@nvidia.com: v2]
Link: http://lkml.kernel.org/r/C5E3C65C-8253-4638-9D3C-71A61858BB8B@nvidia.com
[anshuman.khandual@arm.com: s/thp_nr_pages/hpage_nr_pages/]
Link: http://lkml.kernel.org/r/1594287583-16568-1-git-send-email-anshuman.khandual@arm.com
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
Signed-off-by: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Daniel Jordan <daniel.m.jordan@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Zi Yan <ziy@nvidia.com>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Link: http://lkml.kernel.org/r/1594080415-27924-1-git-send-email-anshuman.khandual@arm.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/Documentation/vm/page_migration.rst b/Documentation/vm/page_migration.rst
index 1d6cd7db4e43b98e77daaf8847a3008e71bc9a7e..68883ac485fae6c12a8c956bb4ee6d714637c2f3 100644 (file)
--- a/Documentation/vm/page_migration.rst
+++ b/Documentation/vm/page_migration.rst
@@ -253,5 +253,32 @@ which are function pointers of struct address_space_operations.
      PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
      for own purpose.
 
+Monitoring Migration
+=====================
+
+The following events (counters) can be used to monitor page migration.
+
+1. PGMIGRATE_SUCCESS: Normal page migration success. Each count means that a
+   page was migrated. If the page was a non-THP page, then this counter is
+   increased by one. If the page was a THP, then this counter is increased by
+   the number of THP subpages. For example, migration of a single 2MB THP that
+   has 4KB-size base pages (subpages) will cause this counter to increase by
+   512.
+
+2. PGMIGRATE_FAIL: Normal page migration failure. Same counting rules as for
+   _SUCCESS, above: this will be increased by the number of subpages, if it was
+   a THP.
+
+3. THP_MIGRATION_SUCCESS: A THP was migrated without being split.
+
+4. THP_MIGRATION_FAIL: A THP could not be migrated nor it could be split.
+
+5. THP_MIGRATION_SPLIT: A THP was migrated, but not as such: first, the THP had
+   to be split. After splitting, a migration retry was used for it's sub-pages.
+
+THP_MIGRATION_* events also update the appropriate PGMIGRATE_SUCCESS or
+PGMIGRATE_FAIL events. For example, a THP migration failure will cause both
+THP_MIGRATION_FAIL and PGMIGRATE_FAIL to increase.
+
 Christoph Lameter, May 8, 2006.
 Minchan Kim, Mar 28, 2016.

diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 24fc7c3ae7d647da305f3097026f482fa52c2fe6..2e6ca53b9bbd5e9257902809edac87172d103176 100644 (file)
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -56,6 +56,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 #endif
 #ifdef CONFIG_MIGRATION
                PGMIGRATE_SUCCESS, PGMIGRATE_FAIL,
+               THP_MIGRATION_SUCCESS,
+               THP_MIGRATION_FAIL,
+               THP_MIGRATION_SPLIT,
 #endif
 #ifdef CONFIG_COMPACTION
                COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED,

diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index 705b33d1e395e86cd8889d9dff3b2f93bc28a608..4d434398d64d0bb042f2bc2a76510dca1e726858 100644 (file)
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -46,13 +46,18 @@ MIGRATE_REASON
 TRACE_EVENT(mm_migrate_pages,
 
        TP_PROTO(unsigned long succeeded, unsigned long failed,
-                enum migrate_mode mode, int reason),
+                unsigned long thp_succeeded, unsigned long thp_failed,
+                unsigned long thp_split, enum migrate_mode mode, int reason),
 
-       TP_ARGS(succeeded, failed, mode, reason),
+       TP_ARGS(succeeded, failed, thp_succeeded, thp_failed,
+               thp_split, mode, reason),
 
        TP_STRUCT__entry(
                __field(        unsigned long,          succeeded)
                __field(        unsigned long,          failed)
+               __field(        unsigned long,          thp_succeeded)
+               __field(        unsigned long,          thp_failed)
+               __field(        unsigned long,          thp_split)
                __field(        enum migrate_mode,      mode)
                __field(        int,                    reason)
        ),
@@ -60,13 +65,19 @@ TRACE_EVENT(mm_migrate_pages,
        TP_fast_assign(
                __entry->succeeded      = succeeded;
                __entry->failed         = failed;
+               __entry->thp_succeeded  = thp_succeeded;
+               __entry->thp_failed     = thp_failed;
+               __entry->thp_split      = thp_split;
                __entry->mode           = mode;
                __entry->reason         = reason;
        ),
 
-       TP_printk("nr_succeeded=%lu nr_failed=%lu mode=%s reason=%s",
+       TP_printk("nr_succeeded=%lu nr_failed=%lu nr_thp_succeeded=%lu nr_thp_failed=%lu nr_thp_split=%lu mode=%s reason=%s",
                __entry->succeeded,
                __entry->failed,
+               __entry->thp_succeeded,
+               __entry->thp_failed,
+               __entry->thp_split,
                __print_symbolic(__entry->mode, MIGRATE_MODE),
                __print_symbolic(__entry->reason, MIGRATE_REASON))
 );

diff --git a/mm/migrate.c b/mm/migrate.c
index b52f30113366e839a67fd0627252fd66772c7dab..5ea2758783831b120127a92cfb56172ee9996125 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1418,22 +1418,35 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
                enum migrate_mode mode, int reason)
 {
        int retry = 1;
+       int thp_retry = 1;
        int nr_failed = 0;
        int nr_succeeded = 0;
+       int nr_thp_succeeded = 0;
+       int nr_thp_failed = 0;
+       int nr_thp_split = 0;
        int pass = 0;
+       bool is_thp = false;
        struct page *page;
        struct page *page2;
        int swapwrite = current->flags & PF_SWAPWRITE;
-       int rc;
+       int rc, nr_subpages;
 
        if (!swapwrite)
                current->flags |= PF_SWAPWRITE;
 
-       for(pass = 0; pass < 10 && retry; pass++) {
+       for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
                retry = 0;
+               thp_retry = 0;
 
                list_for_each_entry_safe(page, page2, from, lru) {
 retry:
+                       /*
+                        * THP statistics is based on the source huge page.
+                        * Capture required information that might get lost
+                        * during migration.
+                        */
+                       is_thp = PageTransHuge(page);
+                       nr_subpages = hpage_nr_pages(page);
                        cond_resched();
 
                        if (PageHuge(page))
@@ -1464,15 +1477,30 @@ retry:
                                        unlock_page(page);
                                        if (!rc) {
                                                list_safe_reset_next(page, page2, lru);
+                                               nr_thp_split++;
                                                goto retry;
                                        }
                                }
+                               if (is_thp) {
+                                       nr_thp_failed++;
+                                       nr_failed += nr_subpages;
+                                       goto out;
+                               }
                                nr_failed++;
                                goto out;
                        case -EAGAIN:
+                               if (is_thp) {
+                                       thp_retry++;
+                                       break;
+                               }
                                retry++;
                                break;
                        case MIGRATEPAGE_SUCCESS:
+                               if (is_thp) {
+                                       nr_thp_succeeded++;
+                                       nr_succeeded += nr_subpages;
+                                       break;
+                               }
                                nr_succeeded++;
                                break;
                        default:
@@ -1482,19 +1510,27 @@ retry:
                                 * removed from migration page list and not
                                 * retried in the next outer loop.
                                 */
+                               if (is_thp) {
+                                       nr_thp_failed++;
+                                       nr_failed += nr_subpages;
+                                       break;
+                               }
                                nr_failed++;
                                break;
                        }
                }
        }
-       nr_failed += retry;
+       nr_failed += retry + thp_retry;
+       nr_thp_failed += thp_retry;
        rc = nr_failed;
 out:
-       if (nr_succeeded)
-               count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
-       if (nr_failed)
-               count_vm_events(PGMIGRATE_FAIL, nr_failed);
-       trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
+       count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
+       count_vm_events(PGMIGRATE_FAIL, nr_failed);
+       count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
+       count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
+       count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
+       trace_mm_migrate_pages(nr_succeeded, nr_failed, nr_thp_succeeded,
+                              nr_thp_failed, nr_thp_split, mode, reason);
 
        if (!swapwrite)
                current->flags &= ~PF_SWAPWRITE;

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 3bb034c998876d47992aaf3cb5f76bb18ca7e7f9..727a26d1ec1d2d3c22f58656d2ae921582a2bcf3 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1277,6 +1277,9 @@ const char * const vmstat_text[] = {
 #ifdef CONFIG_MIGRATION
        "pgmigrate_success",
        "pgmigrate_fail",
+       "thp_migration_success",
+       "thp_migration_fail",
+       "thp_migration_split",
 #endif
 #ifdef CONFIG_COMPACTION
        "compact_migrate_scanned",