//include/linux/mmzone.h
enum zone_type {/** ZONE_DMA and ZONE_DMA32 are used when there are peripherals not able* to DMA to all of the addressable memory (ZONE_NORMAL).* On architectures where this area covers the whole 32 bit address* space ZONE_DMA32 is used. ZONE_DMA is left for the ones with smaller* DMA addressing constraints. This distinction is important as a 32bit* DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit* platforms may need both zones as they support peripherals with* different DMA addressing limitations.*/
#ifdef CONFIG_ZONE_DMAZONE_DMA,
#endif
#ifdef CONFIG_ZONE_DMA32ZONE_DMA32,
#endif/** Normal addressable memory is in ZONE_NORMAL. DMA operations can be* performed on pages in ZONE_NORMAL if the DMA devices support* transfers to all addressable memory.*/ZONE_NORMAL,
#ifdef CONFIG_HIGHMEM/** A memory area that is only addressable by the kernel through* mapping portions into its own address space. This is for example* used by i386 to allow the kernel to address the memory beyond* 900MB. The kernel will set up special mappings (page* table entries on i386) for each page that the kernel needs to* access.*/ZONE_HIGHMEM,
#endif/** ZONE_MOVABLE is similar to ZONE_NORMAL, except that it contains* movable pages with few exceptional cases described below. Main use* cases for ZONE_MOVABLE are to make memory offlining/unplug more* likely to succeed, and to locally limit unmovable allocations - e.g.,* to increase the number of THP/huge pages. Notable special cases are:** 1. Pinned pages: (long-term) pinning of movable pages might*    essentially turn such pages unmovable. Memory offlining might*    retry a long time.* 2. memblock allocations: kernelcore/movablecore setups might create*    situations where ZONE_MOVABLE contains unmovable allocations*    after boot. Memory offlining and allocations fail early.* 3. Memory holes: kernelcore/movablecore setups might create very rare*    situations where ZONE_MOVABLE contains memory holes after boot,*    for example, if we have sections that are only partially*    populated. Memory offlining and allocations fail early.* 4. PG_hwpoison pages: while poisoned pages can be skipped during*    memory offlining, such pages cannot be allocated.* 5. Unmovable PG_offline pages: in paravirtualized environments,*    hotplugged memory blocks might only partially be managed by the*    buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The*    parts not manged by the buddy are unmovable PG_offline pages. In*    some cases (virtio-mem), such pages can be skipped during*    memory offlining, however, cannot be moved/allocated. These*    techniques might use alloc_contig_range() to hide previously*    exposed pages from the buddy again (e.g., to implement some sort*    of memory unplug in virtio-mem).** In general, no unmovable allocations that degrade memory offlining* should end up in ZONE_MOVABLE. Allocators (like alloc_contig_range())* have to expect that migrating pages in ZONE_MOVABLE can fail (even* if has_unmovable_pages() states that there are no unmovable pages,* there can be false negatives).*/ZONE_MOVABLE,
#ifdef CONFIG_ZONE_DEVICEZONE_DEVICE,
#endif__MAX_NR_ZONES //__MAX_NR_ZONES是充当结束标记, 在内核中想要迭代系统中所有内存域, 会用到该常量};

​

//include/linux/gfp.h
#define GFP_USER    (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL)
#define GFP_HIGHUSER    (GFP_USER | __GFP_HIGHMEM)
#define GFP_HIGHUSER_MOVABLE    (GFP_HIGHUSER | __GFP_MOVABLE | \__GFP_SKIP_KASAN_POISON)

/** This finds a zone that can be used for ZONE_MOVABLE pages. The* assumption is made that zones within a node are ordered in monotonic* increasing memory addresses so that the "highest" populated zone is used *注释这里也写了：期望node里面的zone按照单调递增的内存地址排序，以便选用“highest”zone被选来当ZONE_MOVABLE*/
static void __init find_usable_zone_for_movable(void)
{int zone_index;for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {if (zone_index == ZONE_MOVABLE)continue;if (arch_zone_highest_possible_pfn[zone_index] >arch_zone_lowest_possible_pfn[zone_index]) //这里体现break;}VM_BUG_ON(zone_index == -1);movable_zone = zone_index;
}

//msm-kernel/mm/Kconfig
config ZONE_DEVICEbool "Device memory (pmem, HMM, etc...) hotplug support"depends on MEMORY_HOTPLUGdepends on MEMORY_HOTREMOVEdepends on SPARSEMEM_VMEMMAPdepends on ARCH_HAS_PTE_DEVMAPselect XARRAY_MULTIhelpDevice memory hotplug support allows for establishing pmem,or other device driver discovered memory regions, in thememmap. This allows pfn_to_page() lookups of otherwise"device-physical" addresses which is needed for using a DAXmapping in an O_DIRECT operation, among other things.If FS_DAX is enabled, then say Y.

//include/linux/mmzone.h
struct zone {/* Read-mostly fields *//* zone watermarks, access with *_wmark_pages(zone) macros */unsigned long _watermark[NR_WMARK];unsigned long watermark_boost;unsigned long nr_reserved_highatomic;/** We don't know if the memory that we're going to allocate will be* freeable or/and it will be released eventually, so to avoid totally* wasting several GB of ram we must reserve some of the lower zone* memory (otherwise we risk to run OOM on the lower zones despite* there being tons of freeable ram on the higher zones).  This array is* recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl* changes.*/long lowmem_reserve[MAX_NR_ZONES];#ifdef CONFIG_NEED_MULTIPLE_NODESint node;
#endifstruct pglist_data    *zone_pgdat;struct per_cpu_pageset __percpu *pageset;#ifndef CONFIG_SPARSEMEM/** Flags for a pageblock_nr_pages block. See pageblock-flags.h.* In SPARSEMEM, this map is stored in struct mem_section*/unsigned long        *pageblock_flags;
#endif /* CONFIG_SPARSEMEM *//* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */unsigned long        zone_start_pfn;/** spanned_pages is the total pages spanned by the zone, including* holes, which is calculated as:*     spanned_pages = zone_end_pfn - zone_start_pfn;** present_pages is physical pages existing within the zone, which* is calculated as:*   present_pages = spanned_pages - absent_pages(pages in holes);** managed_pages is present pages managed by the buddy system, which* is calculated as (reserved_pages includes pages allocated by the* bootmem allocator):*  managed_pages = present_pages - reserved_pages;** cma pages is present pages that are assigned for CMA use* (MIGRATE_CMA).** So present_pages may be used by memory hotplug or memory power* management logic to figure out unmanaged pages by checking* (present_pages - managed_pages). And managed_pages should be used* by page allocator and vm scanner to calculate all kinds of watermarks* and thresholds.** Locking rules:** zone_start_pfn and spanned_pages are protected by span_seqlock.* It is a seqlock because it has to be read outside of zone->lock,* and it is done in the main allocator path.  But, it is written* quite infrequently.** The span_seq lock is declared along with zone->lock because it is* frequently read in proximity to zone->lock.  It's good to* give them a chance of being in the same cacheline.** Write access to present_pages at runtime should be protected by* mem_hotplug_begin/end(). Any reader who can't tolerant drift of* present_pages should get_online_mems() to get a stable value.*/atomic_long_t        managed_pages;unsigned long     spanned_pages;unsigned long     present_pages;
#ifdef CONFIG_CMAunsigned long      cma_pages;
#endifconst char        *name;#ifdef CONFIG_MEMORY_ISOLATION/** Number of isolated pageblock. It is used to solve incorrect* freepage counting problem due to racy retrieving migratetype* of pageblock. Protected by zone->lock.*/unsigned long     nr_isolate_pageblock;
#endif#ifdef CONFIG_MEMORY_HOTPLUG/* see spanned/present_pages for more description */seqlock_t     span_seqlock;
#endifint initialized;/* Write-intensive fields used from the page allocator */ZONE_PADDING(_pad1_)/* free areas of different sizes */struct free_area  free_area[MAX_ORDER];/* zone flags, see below */unsigned long       flags;/* Primarily protects free_area */spinlock_t      lock;/* Write-intensive fields used by compaction and vmstats. */ZONE_PADDING(_pad2_)/** When free pages are below this point, additional steps are taken* when reading the number of free pages to avoid per-cpu counter* drift allowing watermarks to be breached*/unsigned long percpu_drift_mark;#if defined CONFIG_COMPACTION || defined CONFIG_CMA/* pfn where compaction free scanner should start */unsigned long       compact_cached_free_pfn;/* pfn where compaction migration scanner should start */unsigned long      compact_cached_migrate_pfn[ASYNC_AND_SYNC];unsigned long        compact_init_migrate_pfn;unsigned long      compact_init_free_pfn;
#endif#ifdef CONFIG_COMPACTION/** On compaction failure, 1<<compact_defer_shift compactions* are skipped before trying again. The number attempted since* last failure is tracked with compact_considered.* compact_order_failed is the minimum compaction failed order.*/unsigned int        compact_considered;unsigned int     compact_defer_shift;int         compact_order_failed;
#endif#if defined CONFIG_COMPACTION || defined CONFIG_CMA/* Set to true when the PG_migrate_skip bits should be cleared */bool          compact_blockskip_flush;
#endifbool          contiguous;ZONE_PADDING(_pad3_)/* Zone statistics */atomic_long_t       vm_stat[NR_VM_ZONE_STAT_ITEMS];atomic_long_t        vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];ANDROID_KABI_RESERVE(1);ANDROID_KABI_RESERVE(2);ANDROID_KABI_RESERVE(3);ANDROID_KABI_RESERVE(4);
} ____cacheline_internodealigned_in_smp;

//include/linux/mmzone.h
enum zone_watermarks {WMARK_MIN,WMARK_LOW,WMARK_HIGH,NR_WMARK
};

​

User@Ubuntu-149-19:/proc$ cat vmstat | grep "pageout*"
pageoutrun 370693User@Ubuntu-149-19:/proc$ cat vmstat | grep "alloc*"
allocstall_dma 0
allocstall_dma32 0
allocstall_normal 2721
allocstall_movable 1534443

/** Try to keep at least this much lowmem free.  Do not allow normal* allocations below this point, only high priority ones. Automatically* tuned according to the amount of memory in the system.*/
//这是默认初始化的值，下面在init_per_zone_wmark_min函数里面会对其根据实际情况更新值
int min_free_kbytes = 1024; //系统的
int user_min_free_kbytes = -1; //用户设置的
/** Extra memory for the system to try freeing. Used to temporarily* free memory, to make space for new workloads. Anyone can allocate* down to the min watermarks controlled by min_free_kbytes above.*/
int extra_free_kbytes = 0;//默认值为0，用户通过系统调用进行配置，如果kernel版本有这个，可在/proc/sys/vm下面看到这个参数/*** nr_free_zone_pages - count number of pages beyond high watermark* @offset: The zone index of the highest zone** nr_free_zone_pages() counts the number of pages which are beyond the* high watermark within all zones at or below a given zone index.  For each* zone, the number of pages is calculated as:* 计算方式给定了,得到zone通过buddy管理的所有可用的页减去high watermark值后，剩余的可用pages*     nr_free_zone_pages = managed_pages - high_pages** Return: number of pages beyond high watermark.*/
static unsigned long nr_free_zone_pages(int offset)
{struct zoneref *z;struct zone *zone;/* Just pick one node, since fallback list is circular */unsigned long sum = 0;struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
//通过zonelist，遍历整个系统所有node的所有zone,根据最高的zone的索引offset（The zone index of the highest zone），得到所有符合要求的zone的nr_free_zone_pages值for_each_zone_zonelist(zone, z, zonelist, offset) {
//nr_free_zone_pages = managed_pages - high_pagesunsigned long size = zone_managed_pages(zone);//得到当前zone的托管的page数目managed_pagesunsigned long high = high_wmark_pages(zone);//得到当前zone的_watermark[WMARK_HIGH]+watermark_boost值
//如果size大于high，说明还有可分配的，所以累计进sum，当遍历完所有zone，得到所有zone的nr_free_zone_pages值if (size > high)sum += size - high;}return sum;
}/*** nr_free_buffer_pages - count number of pages beyond high watermark** nr_free_buffer_pages() counts the number of pages which are beyond the high* watermark within ZONE_DMA and ZONE_NORMAL.** Return: number of pages beyond high watermark within ZONE_DMA and* ZONE_NORMAL.*/
//函数返回累计内存域ZONE_DMA至ZONE_NORMAL之间所有zone，减去对应的_watermark[WMARK_HIGH]+watermark_boost后，可分配的free pgaes之和。
//函数comment之所以描述是ZONE_DMA和ZONE_NORMAL是默认只有这两个zone的情况
//nr_free_zone_pages上面有讲，gfp_zone(GFP_USER)得到的是一个enum zone_type的offest偏移量,从而知道我们要遍历哪些zone，得到实际的可分配的free pages
unsigned long nr_free_buffer_pages(void)
{return nr_free_zone_pages(gfp_zone(GFP_USER));
}
EXPORT_SYMBOL_GPL(nr_free_buffer_pages);/** Initialise min_free_kbytes.** For small machines we want it small (128k min).  For large machines* we want it large (256MB max).  But it is not linear, because network* bandwidth does not increase linearly with machine size.  We use**  min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:*   min_free_kbytes = sqrt(lowmem_kbytes * 16)** which yields* 实际物理内存对应min_free_kbytes的推荐值，最小128KB,最大256MB* 对于这个函数一般是min_free_kbytes发生变化，然后才会调用来更新watermark和watermark_boost等值* 16MB: 512k* 32MB: 724k* 64MB: 1024k* 128MB:   1448k* 256MB:   2048k* 512MB:   2896k* 1024MB:  4096k* 2048MB:  5792k* 4096MB:  8192k* 8192MB:  11584k* 16384MB:    16384k*/
int __meminit init_per_zone_wmark_min(void)
{unsigned long lowmem_kbytes;int new_min_free_kbytes;
//PAGE_SIZE为4096B,(PAGE_SIZE >> 10)得4KB，nr_free_buffer_pages函数获得当前系统符合要求的所有zone的free pages，最后得到lowmem_kbyteslowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
//计算min_free_kbytes = 4 * int_sqrt(lowmem_kbytes)new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);//如果大于user_min_free_kbytes的，会更新min_free_kbytes，同步进行最大最小值的限定，否则，不更新，以用户的user_min_free_kbytes为准if (new_min_free_kbytes > user_min_free_kbytes) {min_free_kbytes = new_min_free_kbytes;if (min_free_kbytes < 128)min_free_kbytes = 128;if (min_free_kbytes > 262144)min_free_kbytes = 262144;//256MB} else {pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",new_min_free_kbytes, user_min_free_kbytes);}setup_per_zone_wmarks();//开始在这里设置zone的三个watermark值，本节重点讲这个函数refresh_zone_stat_thresholds();setup_per_zone_lowmem_reserve();//这个函数后面章节会讲#ifdef CONFIG_NUMAsetup_min_unmapped_ratio();setup_min_slab_ratio();
#endifkhugepaged_min_free_kbytes_update();return 0;
}/**正主在这里呀......* setup_per_zone_wmarks - called when min_free_kbytes changes* or when memory is hot-{added|removed}** Ensures that the watermark[min,low,high] values for each zone are set* correctly with respect to min_free_kbytes.*/
void setup_per_zone_wmarks(void)
{static DEFINE_SPINLOCK(lock);spin_lock(&lock);__setup_per_zone_wmarks();//直接调用这个，当min_free_kbytes发生改变或者当内存是可插拔内存，发生插拔时，需要更新spin_unlock(&lock);
}static void __setup_per_zone_wmarks(void)
{
/**min_free_kbytes和extra_free_kbytes的官方介绍在内核代码的Documentation/admin-guide/sysctl/vm.rst下面有*min_free_kbytes不做多的介绍，大家比较熟悉*extra_free_kbytes是来自Red Hat的Rik van Riel于2011年提出的，这个值是添加在watermark min和watermark low之间，用户可修改*相当于在watermark min不变的基础上，增大了watermark low。通过增加这个，针对burst allocation*（如：在网络收发的时候，数据量可能突然增大，需要临时申请大量的内存）类型，可以避免快速达到watermark min，而导致触发direct reclaim，*影响系统性能，特别是对于低时延的内存分配请求。*不过这个功能在5.4上还有，但在5.15上已经没有了，这是因为这个功能跟下面讲的watermark_scale_factor参数功能重复，社区后面合并了，统一用watermark_scale_factor了*/unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);//PAGE_SHIFT=12，计算得到对应的pages_min值unsigned long pages_low = extra_free_kbytes >> (PAGE_SHIFT - 10);//计算得到对应的pages_low值unsigned long lowmem_pages = 0;struct zone *zone;unsigned long flags;/* Calculate total number of !ZONE_HIGHMEM pages */
//遍历系统所有online node的所有除ZONE_HIGHMEM外的zone，得到对应的zone的托管的页面（managed_pages），累计进lowmem_pagesfor_each_zone(zone) {if (!is_highmem(zone))lowmem_pages += zone_managed_pages(zone);}
//遍历系统所有online node的所有的zone，开始设置它们的watermarkfor_each_zone(zone) {u64 tmp, low;spin_lock_irqsave(&zone->lock, flags);tmp = (u64)pages_min * zone_managed_pages(zone);
/*do_div除法操作，tmp/lowmem_pages=(u64)pages_min * zone_managed_pages(zone)/lowmem_pages*从上面可知，lowmem_pages是很多zone的累计值，zone_managed_pages(zone)/lowmem_page，实际上从含义上就是得到一个比例值，*然后从总的min_free_kbytes分出对应比值的pages,作为这个zone的watermark min,最后将计算结果重新赋值给tmp*/do_div(tmp, lowmem_pages);low = (u64)pages_low * zone_managed_pages(zone);
/*GFP_HIGHUSER_MOVABLE：分配的内存不限定要被内核直接访问，且是可迁移*nr_free_zone_pages,前面有讲，根据offset，累加计算managed_pages - high_pages值*同上面一样，也是得到一个比例值，然后从总extra_free_kbytes分出对应比值的pages,作为这个zone的extra_free_kbytes*/do_div(low, nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)));if (is_highmem(zone)) {/*对于ZONE_HIGHMEM的watermark min要特殊处理，对于64位系统不用考虑* __GFP_HIGH and PF_MEMALLOC allocations usually don't* need highmem pages, so cap pages_min to a small* value here.** The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)* deltas control async page reclaim, and so should* not be capped for highmem.*/unsigned long min_pages;min_pages = zone_managed_pages(zone) / 1024;
//SWAP_CLUSTER_MAX=32UL，clamp函数，给定最大/小值，在范围内就是min_pages，若大于最大值，取最大值，若小于最小值，取最小值min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
//赋ZONE_HIGHMEM的watermark min是该zone的zone_managed_pages(zone) / 1024zone->_watermark[WMARK_MIN] = min_pages;} else {/** If it's a lowmem zone, reserve a number of pages* proportionate to the zone's size.*根据前面计算得到的该zone的watermark min值，赋值进_watermark[WMARK_MIN]里面*/zone->_watermark[WMARK_MIN] = tmp;}/** Set the kswapd watermarks distance according to the* scale factor in proportion to available memory, but* ensure a minimum size on small systems.*/
/**max_t(type,x,y)取x和y两者之间的最大值*mult_frac(x,numer,denom)=x*(numer/denom)*watermark_scale_factor，根据内核文档/Documentation/admin-guide/sysctl/vm.rst可知，*该值控制着kswapd的唤醒和睡眠时机，因为该值会影响zone的watermark low和watermark high，而这两个值分别影响kswapd唤醒和睡眠*watermark_scale_factor默认值是10，最大值1000（5.4还是当前值，在kernel 5.15中是3000），可在proc/sys/vm下面看到，表示low与min之间差值和high与low之间差值是总内存的[0.1%，10%]之间*/tmp = max_t(u64, tmp >> 2,mult_frac(zone_managed_pages(zone),watermark_scale_factor, 10000));zone->watermark_boost = 0;//这里把watermark_boost赋值为0，避免对下面的_watermark计算产生干扰
/**下面就是根据 该zone的watermark min，extra_free_kbytes（上面的low变量）和上面tmp值设置_watermark[WMARK_LOW]和_watermark[WMARK_HIGH]*这里理想化下，前面开始计算tmp的值是_watermark[WMARK_MIN]，如果max_t返回值tmp/4，即_watermark[WMARK_MIN]/4,在不考虑low的情况下*则_watermark[WMARK_LOW]=_watermark[WMARK_MIN]+_watermark[WMARK_MIN]/4=1.25*_watermark[WMARK_MIN]*_watermark[WMARK_HIGH]=_watermark[WMARK_MIN]+(_watermark[WMARK_MIN]/4)*2=1.5*_watermark[WMARK_MIN]*所以，三者的比例关系大致为：_watermark[WMARK_MIN]：_watermark[WMARK_LOW]：_watermark[WMARK_HIGH] = 4:5:6*/zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + low + tmp;zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + low + tmp * 2;spin_unlock_irqrestore(&zone->lock, flags);}/* update totalreserve_pages */
//对于这个函数后面会细讲，init_per_zone_wmark_min函数可能会修改min_free_kbytes，所以必须调用calculate_totalreserve_pages重新计算整个系统的totalreserve_pagescalculate_totalreserve_pages();
}

User@Ubuntu-149-19:/proc/sys/vm$ cat min_free_kbytes
4194304

//include/linux/mmzone.h
#define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost)
#define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost)
#define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
#define wmark_pages(z, i) (z->_watermark[i] + z->watermark_boost)//mm/page_alloc.c
static void __setup_per_zone_wmarks(void)
{
......zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + low + tmp;
zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + low + tmp * 2;......
}

static inline bool boost_watermark(struct zone *zone)
{unsigned long max_boost;//根据Documentation/admin-guide/sysctl/vm.rst描述，watermark_boost_factor为0表示关闭watermark_boost功能，所以这里直接返回false
//watermark_boost_factor默认值是15000if (!watermark_boost_factor)return false;/** Don't bother in zones that are unlikely to produce results.* On small machines, including kdump capture kernels running* in a small area, boosting the watermark can cause an out of* memory situation immediately.*/
//如果一个pageblock里面的page数目(pageblock_nr_pages)乘以4,就已经大于当前zone所托管的page数目了，就没必要去做提升，返回false
//主要针对小内存机器，现在大内存一般不会if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))return false;
//mult_frac(x,numer,denom)=x*(numer/denom)，根据默认值，max_boost是1.5*zone->_watermark[WMARK_HIGH]max_boost = mult_frac(zone->_watermark[WMARK_HIGH],watermark_boost_factor, 10000);/** high watermark may be uninitialised if fragmentation occurs* very early in boot so do not boost. We do not fall* through and boost by pageblock_nr_pages as failing* allocations that early means that reclaim is not going* to help and it may even be impossible to reclaim the* boosted watermark resulting in a hang.*/
//在系统启动的早期，high watermark可能还没初始化，为0，这是时候没必要boost,直接返回falseif (!max_boost)return false;
//max_boost和pageblock_nr_pages取最大者max_boost = max(pageblock_nr_pages, max_boost);
//取min,赋值给zone->watermark_boostzone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,max_boost);
//设置watermark_boost完成，返回truereturn true;
}

//kernel/bounds.c
// SPDX-License-Identifier: GPL-2.0
/** Generate definitions needed by the preprocessor.* This code generates raw asm output which is post-processed* to extract and format the required data.*/#define __GENERATING_BOUNDS_H
/* Include headers that define the enum constants of interest */#include <linux/page-flags.h>#include <linux/mmzone.h>#include <linux/kbuild.h>#include <linux/log2.h>#include <linux/spinlock_types.h>int main(void)
{/* The enum constants to put into include/generated/bounds.h */DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS);DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES);
#ifdef CONFIG_SMPDEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
#endifDEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));/* End of constants */return 0;
}/** We don't know if the memory that we're going to allocate will be* freeable or/and it will be released eventually, so to avoid totally* wasting several GB of ram we must reserve some of the lower zone* memory (otherwise we risk to run OOM on the lower zones despite* there being tons of freeable ram on the higher zones).  This array is* recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl* changes.*/
//从上面的定义中可以知道，MAX_NR_ZONES实际上就是__MAX_NR_ZONES，在enum zone_type中有说明，如果zone_type里面的config全开，MAX_NR_ZONES就是6long lowmem_reserve[MAX_NR_ZONES];

//mm/page_alloc.c
/** results with 256, 32 in the lowmem_reserve sysctl:* 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)*  1G machine -> (16M dma, 784M normal, 224M high)* NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA*  HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL*   HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA** TBD: should special case ZONE_DMA32 machines here - in those we normally* don't need any ZONE_NORMAL reservation*/
int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
//如果全部使能，则6个全有
#ifdef CONFIG_ZONE_DMA[ZONE_DMA] = 256,
#endif
#ifdef CONFIG_ZONE_DMA32[ZONE_DMA32] = 256,
#endif[ZONE_NORMAL] = 32,
#ifdef CONFIG_HIGHMEM[ZONE_HIGHMEM] = 0,
#endif[ZONE_MOVABLE] = 0,
};

​

//手机
User5613:/proc/sys/vm # cat lowmem_reserve_ratio
cat lowmem_reserve_ratio
32      0
//linux服务器
user:/$ cat /proc/sys/vm/lowmem_reserve_ratio
256     256     32      1user:/$ cat /proc/zoneinfo
cat /proc/zoninfo结果的部分截图
Node 0, zone      DMA
......pages free     3949min      126low      157high     188
......protection: (0, 1592, 64023, 64023, 64023)
......
Node 0, zone    DMA32
......pages free     79853min      12995low      16243high     19491protection: (0, 0, 62430, 62430, 62430)
......Node 0, zone   Normal
......pages free     3953716min      509331low      636663high     763995protection: (0, 0, 0, 0, 0)
......
Node 1, zone   Normal
......pages free     5216084min      526122low      657652high     789182protection: (0, 0, 0, 0, 0)
......

static inline long __zone_watermark_unusable_free(struct zone *z,unsigned int order, unsigned int alloc_flags)
{
//根据alloc_flags，判断是否有ALLOC_HARDER标志位或者ALLOC_OOM标志位
//（仅在MMU机制中才有OOM机制，一般系统都是默认MMU，就看alloc_flags带不带ALLOC_OOM去做判断）const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
//因为是要分配(1 << order)个page，相当于这部分page是提前预定了，那么在跟水线进行判断的时，我们需要free_pages先减去这部分，再进行比较。之所以是(1 << order) - 1，简单是因为从0开始计算。long unusable_free = (1 << order) - 1;/** If the caller does not have rights to ALLOC_HARDER then subtract* the high-atomic reserves. This will over-estimate the size of the* atomic reserve but it avoids a search.*/
//如果alloc_harder为false，说明ALLOC_HARDER标志位为0，则说明不允许使用页迁移中的MIGRATE_HIGHATOMIC保留的内存，所以要加上if (likely(!alloc_harder))unusable_free += z->nr_reserved_highatomic;#ifdef CONFIG_CMA/* If allocation can't use CMA areas don't use free CMA pages */
//在使能了CMA的情况，但是alloc_flags上的ALLOC_CMA标志位为0，则这部分给CMA用的内存也要保留，所以也要加上if (!(alloc_flags & ALLOC_CMA))unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
#endifreturn unusable_free;//最终返回实际的不可用freepage
}正主......
/** Return true if free base pages are above 'mark'. For high-order checks it* will return true of the order-0 watermark is reached and there is at least* one free page of a suitable size. Checking now avoids taking the zone lock* to check in the allocation paths if no pages are free.*/
bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,int highest_zoneidx, unsigned int alloc_flags,long free_pages)
{
/*根据函数调用向上回溯，发现比如在get_page_from_freelist函数里有对mark值来源介绍
mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); //z->_watermark[i] + z->watermark_boost
而wmark_pages在include/linux/mmzone.h中有定义，在前面有介绍这个函数。实际上，这四个函数的输出值都是给到mark值，根据不同需求调用不同函数
*/long min = mark;//这个mask根据代码实现可知，实际上就是对应的watermark min值int o;//ALLOC_HARDER:是否允许使用页迁移中的MIGRATE_HIGHATOMIC保留的内存
//ALLOC_OOM:内存不足时允许触发OOM,一般在MMU机制下使用
//ALLOC_HIGH：与__GFP_HIGH功能相同,请求分配非常紧急的内存，在内存分配方面比ALLOC_HARDER更激进
//关于GFP(gfp_mask)的介绍可以参考这篇文章：https://blog.csdn.net/farmwang/article/details/66975128
//关于alloc_flags的介绍可以参考这篇文章：https://zhuanlan.zhihu.com/p/579030406
/*
注意点：需要明确一点alloc_flags和gfp_mask之间的区别，
gfp_mask是使用alloc_pages申请内存时所传递的申请标记，而alloc_flags是在内存管理子系统内部使用的另一个标记，
二者是不同的，当然alloc_flags也是从gfp_mask经过计算得到的
*/
//根据alloc_flags（mm/internal.h），判断是否有ALLOC_HARDER标志位或者ALLOC_OOM标志位（仅在MMU机制中才有OOM机制）const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));/* free_pages may go negative - that's OK */
//从当前zone中的free pages刨去不可用于分配的pagefree_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);/*从前面的水线图可以看到当内存低于min时还会下降一小段，说明仍然可以分配内存。
但仅仅限于内核中的一些紧急的分配或是带有GFP_ATOMIC标志的分配请求，会放宽对watermark的检查，放宽多少，
具体就看我们这个函数了，这里就会对min进行放宽
*/
//根据alloc_flags，使能了ALLOC_HIGH标志位，说明此次内存分配非常紧急，出于满足这个内存分配的请求，放宽的水线，min值减一半if (alloc_flags & ALLOC_HIGH)min -= min / 2;
//如果alloc_harder为ture，这说面可能使能了ALLOC_HARDER或者ALLOC_OOM，那么它要在前面的基础上，继续放宽内存限制if (unlikely(alloc_harder)) {/** OOM victims can try even harder than normal ALLOC_HARDER* users on the grounds that it's definitely going to be in* the exit path shortly and free memory. Any allocation it* makes during the free path will be small and short-lived.*/
//根据上面描述可知，ALLOC_OOM情况比ALLOC_HARDER更紧急，所以它要放宽min/2，而ALLOC_HARDER是min/4if (alloc_flags & ALLOC_OOM)min -= min / 2;elsemin -= min / 4;}/** Check watermarks for an order-0 allocation request. If these* are not met, then a high-order request also cannot go ahead* even if a suitable page happened to be free.*/
/*
1、检查空闲page数目跟min+z->lowmem_reserve[highest_zoneidx]比较大小，相当于剔除掉min+z->lowmem_reserve[highest_zoneidx]，是否还有多余page，
如果不大于，则经过水线检查后，当前内存域z无法满足这次内存分配请求，返回false。
2、这里就用到了内存域的lowmem_reserve数组，之所以索引是highest_zoneidx，
从前面关于lowmem_reserve可知，lowmem_reserve[highest_zoneidx]是最大，
这里就是要判断在最大值的情况，free_pages是否满足要求。
3、到这里，我们知道，当我们判断时，并不单单只是看watermark值，还要加上watermark_boost和lowmem_reserve值
*/if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])return false;/* If this is an order-0 request then the watermark is fine */
//前面判断通过后，如果此次内存分配的order是0,那边这次判断后就可以了，直接返回true,if (!order)return true;/* For a high-order request, check at least one suitable page is free */
//下面开始对order>=1的情况进行处理。检查当前内存域z的free_area数组中是否有存在大于等于目标order的page，
//如果存在，返回true;如果遍历到MAX_ORDER之后，还不存在，退出循环，返回falsefor (o = order; o < MAX_ORDER; o++) {
//从order开始，依次遍历至MAX_ORDER-1,获取对应的free_area[o]struct free_area *area = &z->free_area[o];int mt;
//如果这个free_area中nr_free说明没有空闲page，直接跳过这次循环if (!area->nr_free)continue;
//如果nr_free不为空，则开始要遍历free_area->free_list，这个free_list数组长度是MIGRATE_TYPES，包含各种迁移类型
//这个图在Linux 物理内存管理涉及的三大结构体之struct page里面有for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
#ifdef CONFIG_CMA/** Note that this check is needed only* when MIGRATE_CMA < MIGRATE_PCPTYPES.*/
//在使能了CONFIG_CMA情况下，如果mt为MIGRATE_CMA，直接跳过，因为这部分内存是用于专门留给CMA用的，必须使能了对应config才能使用,下面代码会体现if (mt == MIGRATE_CMA)continue;
#endif
//free_area_empty判断对应的free_list[mt]是否是空链表，如果是，返回true，否则返回false，此时表明链表free_list[mt]上有足够的free pages，可以满足这次内存分配if (!free_area_empty(area, mt))return true;}//如果上面的遍历free_list，没有找到合适page分配,继续下面的操作
#ifdef CONFIG_CMA
//在使能了CONFIG_CMA情况下，如果alloc_flags的ALLOC_CMA标志位置位了，且通过free_area_empty判断free_list[MIGRATE_CMA]上有page，则返回trueif ((alloc_flags & ALLOC_CMA) &&!free_area_empty(area, MIGRATE_CMA)) {return true;}
#endif
//如果alloc_harder为ture，说面当前内存需求比较紧急，且判断free_list[MIGRATE_HIGHATOMIC]上有page，则返回true,会同意这次内存分配if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))return true;}
//最后在遍历了所有order(order~MAX_ORDER-1)后，没有找到合适的，说明当前内存域z的watermark不ok，无法满足内存分配要求return false;
}bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,int highest_zoneidx, unsigned int alloc_flags)
{return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,zone_page_state(z, NR_FREE_PAGES));
}
EXPORT_SYMBOL_GPL(zone_watermark_ok);

/** setup_per_zone_lowmem_reserve - called whenever* sysctl_lowmem_reserve_ratio changes.  Ensures that each zone*   has a correct pages reserved value, so an adequate number of*   pages are left in the zone after a successful __alloc_pages().*/
static void setup_per_zone_lowmem_reserve(void)
{struct pglist_data *pgdat;enum zone_type i, j;//for_each_online_pgdat遍历所有内存节点nodefor_each_online_pgdat(pgdat) {
//下面有两个for循环，因为zone本身使用自己的内存是不用reserve的，且fallback机制是高位zone向低位zone借用内存
//因此i小于MAX_NR_ZONES - 1且j从i+1开始。
//i代表的被借用方，j代表的借用方for (i = 0; i < MAX_NR_ZONES - 1; i++) {
//遍历每个node下，除MAX_NR_ZONES - 1对应的zone_type外，所有类型的zonestruct zone *zone = &pgdat->node_zones[i];
//得到被借用方对应zone_type的lowmem_reserve_ratio值int ratio = sysctl_lowmem_reserve_ratio[i];
//如果ratio为0（表示高位zone用它可以不用reserve）
//或者该zone被buddy system管理的page数(=zone的管理的所有物理页减去要预留的页，前面strcut zone结构体有讲)为0，则clear为truebool clear = !ratio || !zone_managed_pages(zone);unsigned long managed_pages = 0;for (j = i + 1; j < MAX_NR_ZONES; j++) {
//遍历同node下，zone_type比i大的zone，得到upper_zonestruct zone *upper_zone = &pgdat->node_zones[j];
//根据前面的计算公式可知，分子的值就是这个，当j>i+1,managed_pages会不断累加zone_managed_pages(upper_zone)managed_pages += zone_managed_pages(upper_zone);
//如果clear为0，则zone->lowmem_reserve[j]就为0if (clear)zone->lowmem_reserve[j] = 0;else
//否则，计算j借用i时，需要预留的page数目zone->lowmem_reserve[j] = managed_pages / ratio;}}}/* update totalreserve_pages */
//所有node下所有zones的high watermark加上lowmem_reserve的值calculate_totalreserve_pages();
}/**mm/page_alloc.c*totalreserve_pages是个全局变量，且是__read_mostly的，意味着该变量存放在.data.read_mostly段中，定义在arch/arm64/include/asm/cache.h中*这个的作用就是将经常需要被读取的数据定义为__read_mostly类型，这样Linux内核被加载时,该数据将自动被存放到Cache中,以提高整个系统的执行效率。*/
unsigned long totalreserve_pages __read_mostly;/** calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio*  or min_free_kbytes changes.*如描述，当sysctl_lowmem_reserve_ratio和min_free_kbytes发生变化时，这个函数就会被调用来更新totalreserve_pages值*/
static void calculate_totalreserve_pages(void)
{struct pglist_data *pgdat;unsigned long reserve_pages = 0;enum zone_type i, j;
//遍历系统所有node节点for_each_online_pgdat(pgdat) {
//先初始化为0，后面开始计算totalreserve_pagespgdat->totalreserve_pages = 0;
//第一个for循环，遍历每个node下面所有zone_type对应的zonefor (i = 0; i < MAX_NR_ZONES; i++) {struct zone *zone = pgdat->node_zones + i;long max = 0;
//该zone被buddy system管理的page数(=zone的管理的所有物理页减去要预留的页，前面strcut zone结构体有讲),实际上就是当前zone托管的page数目unsigned long managed_pages = zone_managed_pages(zone);/* Find valid and maximum lowmem_reserve in the zone */
//第二个for循环，如注释描述那样，遍历当前zone,找到lowmem_reserve数组里面的最大值，赋给max,
//这个地方特别在sysctl_lowmem_reserve_ratio发生变化时，很大概率会有变化for (j = i; j < MAX_NR_ZONES; j++) {if (zone->lowmem_reserve[j] > max)max = zone->lowmem_reserve[j];}/* we treat the high watermark as reserved pages. */
//根据前面已描述的函数high_wmark_pages(z)  (z->_watermark[WMARK_HIGH] + z->watermark_boost)，将高水线对应的值计算到max中，当做要reserve_pagesmax += high_wmark_pages(zone);//这里有限制，如果max大于当前zone所托管的页面，那么直接是max就等于managed_pagesif (max > managed_pages)max = managed_pages;
//将这个值累计计入到totalreserve_pages和reserve_pages里面，这里得到每个node的totalreserve_pagespgdat->totalreserve_pages += max;reserve_pages += max;}}
//最后，得到所有node的totalreserve_pagestotalreserve_pages = reserve_pages;
}

​

struct per_cpu_pages {int count;     /* number of pages in the list 当前per CPU page list中page数目*/int high;        /* high watermark, emptying needed 当前list中page的上限，超过了则还给当前zone*/int batch;       /* chunk size for buddy add/remove 如果list是空的，则从当前zone中一些找patch个page放到list里面*//* Lists of pages, one per migrate type stored on the pcp-lists */
//每个zone下面有不同order下，有不同的迁移类型，其中MIGRATE_UNMOVABLE,MIGRATE_MOVABLE,MIGRATE_RECLAIMABLE和MIGRATE_CMA（使能了CONFIG_CMA）迁移类型都有per CPU page list
//根据最新社区patch，已经支持page order<3的使用pcp技术，之前只支持order=0（即分配一个page时，per CPU可以直接list中取）的 struct list_head lists[MIGRATE_PCPTYPES];
};struct per_cpu_pageset {struct per_cpu_pages pcp;//per_cpu_pageset结构体的核心
//NUMA和SMP是二选一的，不可能同时使能，对于普通电脑和手机，基本是SMP（即UMA），一般是大型服务器现在用NUMA的多
//下面两个参数基本都是对等含义，重点讲下SMP里面的，这部分在kernel5.15里面，是在struct per_cpu_zonestat里面
#ifdef CONFIG_NUMAs8 expire;u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS];
#endif
#ifdef CONFIG_SMPs8 stat_threshold;//用于vm stat信息的阈值判断，在init_per_zone_wmark_min函数里面会调用refresh_zone_stat_thresholds对其进行更新s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];//保存per CPU的VM stat的统计信息，在后面计算zone当前更精确free pages时会用到
#endif
};

User@Ubuntu-149-19:/proc/sys/vm$ cat percpu_pagelist_fraction
0Node 0, zone      DMA
......pagesetscpu: 0count: 0high:  0batch: 1vm stats threshold: 14cpu: 1count: 0high:  0batch: 1vm stats threshold: 14cpu: 2count: 0high:  0batch: 1vm stats threshold: 14
......cpu: 70count: 0high:  0batch: 1vm stats threshold: 14cpu: 71count: 0high:  0batch: 1vm stats threshold: 14
......
Node 0, zone    DMA32
......pagesetscpu: 0count: 6high:  186batch: 31vm stats threshold: 70cpu: 1count: 0high:  186batch: 31vm stats threshold: 70cpu: 2count: 6high:  186batch: 31vm stats threshold: 70
......cpu: 70count: 146high:  186batch: 31vm stats threshold: 70cpu: 71count: 0high:  186batch: 31vm stats threshold: 70
......
Node 0, zone   Normal
......pagesetscpu: 0count: 180high:  186batch: 31vm stats threshold: 125cpu: 1count: 0high:  186batch: 31vm stats threshold: 125cpu: 2count: 131high:  186batch: 31vm stats threshold: 125
......cpu: 70count: 162high:  186batch: 31vm stats threshold: 125cpu: 71count: 105high:  186batch: 31vm stats threshold: 125
......
Node 1, zone   Normal
......pagesetscpu: 0count: 0high:  186batch: 31vm stats threshold: 125cpu: 1count: 98high:  186batch: 31vm stats threshold: 125cpu: 2count: 26high:  186batch: 31vm stats threshold: 125
......cpu: 70count: 0high:  186batch: 31vm stats threshold: 125cpu: 71count: 137high:  186batch: 31vm stats threshold: 125
......

#ifndef CONFIG_SPARSEMEM/** Flags for a pageblock_nr_pages block. See pageblock-flags.h.* In SPARSEMEM, this map is stored in struct mem_section*/unsigned long      *pageblock_flags;
#endif /* CONFIG_SPARSEMEM */

 /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */unsigned long     zone_start_pfn;

/*
spanned_pages is the total pages spanned by the zone, including
holes, which is calculated as:
spanned_pages = zone_end_pfn - zone_start_pfn;
*/unsigned long     spanned_pages;

/** present_pages is physical pages existing within the zone, which* is calculated as:*  present_pages = spanned_pages - absent_pages(pages in holes);* cma pages is present pages that are assigned for CMA use* (MIGRATE_CMA).** So present_pages may be used by memory hotplug or memory power* management logic to figure out unmanaged pages by checking* (present_pages - managed_pages). And managed_pages should be used* by page allocator and vm scanner to calculate all kinds of watermarks* and thresholds.*/unsigned long     present_pages;

/** managed_pages is present pages managed by the buddy system, which* is calculated as (reserved_pages includes pages allocated by the* bootmem allocator):*    managed_pages = present_pages - reserved_pages;*/atomic_long_t     managed_pages;

/** cma pages is present pages that are assigned for CMA use* (MIGRATE_CMA).*/
#ifdef CONFIG_CMAunsigned long      cma_pages;
#endif

static char * const zone_names[MAX_NR_ZONES] = {
#ifdef CONFIG_ZONE_DMA"DMA",
#endif
#ifdef CONFIG_ZONE_DMA32"DMA32",
#endif"Normal",
#ifdef CONFIG_HIGHMEM"HighMem",
#endif"Movable",
#ifdef CONFIG_ZONE_DEVICE"Device",
#endif
};

#ifdef CONFIG_MEMORY_ISOLATION/** Number of isolated pageblock. It is used to solve incorrect* freepage counting problem due to racy retrieving migratetype* of pageblock. Protected by zone->lock.*/unsigned long        nr_isolate_pageblock;
#endif

/** Locking rules:** zone_start_pfn and spanned_pages are protected by span_seqlock.* It is a seqlock because it has to be read outside of zone->lock,* and it is done in the main allocator path.  But, it is written* quite infrequently.** The span_seq lock is declared along with zone->lock because it is* frequently read in proximity to zone->lock.  It's good to* give them a chance of being in the same cacheline.*/
#ifdef CONFIG_MEMORY_HOTPLUG/* see spanned/present_pages for more description */seqlock_t       span_seqlock;
#endif

 int initialized;

//include/linux/mmzone.h
/** zone->lock and the zone lru_lock are two of the hottest locks in the kernel.* So add a wild amount of padding here to ensure that they fall into separate* cachelines.  There are very few zone structures in the machine, so space* consumption is not a concern here.*/
#if defined(CONFIG_SMP)
struct zone_padding {char x[0];
} ____cacheline_internodealigned_in_smp;
#define ZONE_PADDING(name)  struct zone_padding name;
#else
#define ZONE_PADDING(name)
#endif  /* Write-intensive fields used from the page allocator */ZONE_PADDING(_pad1_)

User@Ubuntu-149-19:/proc$ cat buddyinfo
Node 0, zone      DMA      1      0      1      1      0      1      1      0      1      1      3
Node 0, zone    DMA32   1029    499    228    207    401    629     50     88     29      3     26
Node 0, zone   Normal  12365   8292   4969   5681   7442   4774   2680    870     54      1      0
Node 1, zone   Normal  10712    965 168740  10128   3836   1735    316    727    644    390      3

//include/linux/mmzone.h
struct free_area {struct list_head  free_list[MIGRATE_TYPES];//相同order下，不同内存块迁移类型可能不一样，这里进行了分类，归属到不同的链表上unsigned long      nr_free; //注意：这是空闲内存块数目，而不是free page数目，因为对于order>0，一个内存块就不是一个page了
};  //这是struct zone的核心结构，实际上也是buddy system系统核心数据结构，其用于分配的page来自这里/* free areas of different sizes  MAX_ORDER=11*/struct free_area    free_area[MAX_ORDER];

   /* zone flags, see below */unsigned long        flags;//include/linux/mmzone.h
enum zone_flags {ZONE_BOOSTED_WATERMARK,        /* zone recently boosted watermarks.* Cleared when kswapd is woken.*/ZONE_RECLAIM_ACTIVE,       /* kswapd may be scanning the zone. 在5.4中没有，在5.15中有*/
};

   /* Primarily protects free_area */spinlock_t        lock;

   /* Write-intensive fields used by compaction and vmstats. */ZONE_PADDING(_pad2_)

   /** When free pages are below this point, additional steps are taken* when reading the number of free pages to avoid per-cpu counter* drift allowing watermarks to be breached*/unsigned long percpu_drift_mark;/** Refresh the thresholds for each zone.*/
void refresh_zone_stat_thresholds(void)
{struct pglist_data *pgdat;struct zone *zone;int cpu;int threshold;
....../** Only set percpu_drift_mark if there is a danger that* NR_FREE_PAGES reports the low watermark is ok when in fact* the min watermark could be breached by an allocation*/tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);max_drift = num_online_cpus() * threshold;
//如下对percpu_drift_mark进行设置if (max_drift > tolerate_drift)zone->percpu_drift_mark = high_wmark_pages(zone) +max_drift;}
}

bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,int highest_zoneidx, unsigned int alloc_flags)
{return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,zone_page_state(z, NR_FREE_PAGES));
}
EXPORT_SYMBOL_GPL(zone_watermark_ok);bool zone_watermark_ok_safe(struct zone *z, unsigned int order,unsigned long mark, int highest_zoneidx)
{long free_pages = zone_page_state(z, NR_FREE_PAGES);//这里多了一个步骤，当free pages小于percpu_drift_mark时，从zone_page_state_snapshot获取更精确的free pages,然后再调用__zone_watermark_okif (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,free_pages);
}
EXPORT_SYMBOL_GPL(zone_watermark_ok_safe);//下面就是zone_page_state函数和zone_page_state_snapshot的实现
static inline unsigned long zone_page_state(struct zone *zone,enum zone_stat_item item)
{long x = atomic_long_read(&zone->vm_stat[item]);
#ifdef CONFIG_SMPif (x < 0)x = 0;
#endifreturn x;
}/** More accurate version that also considers the currently pending* deltas. For that we need to loop over all cpus to find the current* deltas. There is no synchronization so the result cannot be* exactly accurate either.*/
static inline unsigned long zone_page_state_snapshot(struct zone *zone,enum zone_stat_item item)
{long x = atomic_long_read(&zone->vm_stat[item]);#ifdef CONFIG_SMPint cpu;
//为了得到更精确的free page，还要计算per cpu counter，不单单是读取vm_statfor_each_online_cpu(cpu)x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];if (x < 0)x = 0;
#endifreturn x;
}

​

#if defined CONFIG_COMPACTION || defined CONFIG_CMA/* pfn where compaction free scanner should start */
//记录上次内存规整结束后，对应的free pfn值（空闲页的页帧号）,在这次规整是非整个zone扫描时，会直接复用这个值继续这次规整unsigned long        compact_cached_free_pfn;/* pfn where compaction migration scanner should start */
//记录上次内存规整结束后，对应的migrate pfn值（可迁移页的页帧号），包括异步(0)和同步(1)的场景下的值，同样也是在这次规整是非整个zone扫描时，会直接复用这个值继续这次规整unsigned long      compact_cached_migrate_pfn[ASYNC_AND_SYNC];unsigned long        compact_init_migrate_pfn;//内存规整时迁移页框初始帧号，当是全zone扫描时，会用这个值unsigned long        compact_init_free_pfn;//内存规整时空闲页框初始帧号，当是全zone扫描时，会用这个值
#endif

#ifdef CONFIG_COMPACTION/** On compaction failure, 1<<compact_defer_shift compactions* are skipped before trying again. The number attempted since* last failure is tracked with compact_considered.* compact_order_failed is the minimum compaction failed order.*/unsigned int     compact_considered;//内存规整推迟的次数unsigned int      compact_defer_shift;//内存规整推迟次数的阀值int            compact_order_failed;//当前zone内存规整可能失败的最小order
#endif

/* Do not skip compaction more than 64 times */
#define COMPACT_MAX_DEFER_SHIFT 6 //规定了延迟的最大次数//判断是否推迟此次内存规整
/* Returns true if compaction should be skipped this time */
bool compaction_deferred(struct zone *zone, int order)
{
//得到内存规整推迟次数上限unsigned long defer_limit = 1UL << zone->compact_defer_shift;//如果当前order小于compact_order_failed，则返回false，表示可以在该zone上进行内存规整if (order < zone->compact_order_failed)return false;/* Avoid possible overflow */
//为了避免可能的溢出，如果内存规整推迟次数累计值前置自增后大于等于上限值，直接等于上限值，同时返回false，允许在该zone上进行内存规整if (++zone->compact_considered >= defer_limit) {zone->compact_considered = defer_limit;return false;}
//利用trace跟踪记录这些操作trace_mm_compaction_deferred(zone, order);
//否则，返回true，不允许在该zone上进行此次内存规整，推迟return true;
}/** Compaction is deferred when compaction fails to result in a page* allocation success. 1 << compact_defer_shift, compactions are skipped up* to a limit of 1 << COMPACT_MAX_DEFER_SHIFT*/
//当内存规整失败后，就会调用defer_compaction，更新上面三个参数
void defer_compaction(struct zone *zone, int order)
{zone->compact_considered = 0;//失败后，将推迟次数置位0，重新开始计数zone->compact_defer_shift++;//每次内存规整失败一次，compact_defer_shift就自增1//这里保留内存规整失败时的最小orderif (order < zone->compact_order_failed)zone->compact_order_failed = order;
//如果compact_defer_shift大于等于上限值，直接赋值为上限值COMPACT_MAX_DEFER_SHIFTif (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;trace_mm_compaction_defer_compaction(zone, order);
}/** Update defer tracking counters after successful compaction of given order,* which means an allocation either succeeded (alloc_success == true) or is* expected to succeed.*/
//如上面注释，当内存规整成功后（但分配是否成功得看alloc_success值），就会调用compaction_defer_reset，更新上面三个参数
void compaction_defer_reset(struct zone *zone, int order,bool alloc_success)
{
//alloc_success为true，意味着内存规整已经成功，且已经满足连续物理内存分配需求，会将compact_considered和compact_defer_shift置为0，重新开始下一轮
//如果enum compact_result是COMPACT_SUCCESS，只代表内存规整已经完成，但连续物理内存需求不一定能够得到满足，所以此时alloc_success还是false。此时不会置0if (alloc_success) {zone->compact_considered = 0;zone->compact_defer_shift = 0;}
//在规整规整成功或可能成功时，如果order是大于等于compact_order_failed的，
//那么将compact_order_failed置为order + 1，提升当前zone内存规整可能失败的最小orderif (order >= zone->compact_order_failed)zone->compact_order_failed = order + 1;trace_mm_compaction_defer_reset(zone, order);
}

#if defined CONFIG_COMPACTION || defined CONFIG_CMA/* Set to true when the PG_migrate_skip bits should be cleared */bool           compact_blockskip_flush;
#endif

bool           contiguous;//设置zone->contiguous
void set_zone_contiguous(struct zone *zone)
{unsigned long block_start_pfn = zone->zone_start_pfn;unsigned long block_end_pfn;block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);for (; block_start_pfn < zone_end_pfn(zone);block_start_pfn = block_end_pfn,block_end_pfn += pageblock_nr_pages) {block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
//如果__pageblock_pfn_to_page返回NULL，则说面zone里面有hole，直接return，重新调度if (!__pageblock_pfn_to_page(block_start_pfn,block_end_pfn, zone))return;cond_resched();}
//如果走到这里说面zone中没有内存空洞（hole），置为true/* We confirm that there is no hole */zone->contiguous = true;
}//清除zone->contiguous
void clear_zone_contiguous(struct zone *zone)
{zone->contiguous = false;
}

    ZONE_PADDING(_pad3_)

#ifdef CONFIG_NUMA
enum numa_stat_item {NUMA_HIT,      /* allocated in intended node 分配的page来自此node的次数*/NUMA_MISS,     /* allocated in non intended node 由于目的node内存不足，导致分配的page来自此node的次数，每次numa_miss都有对应的numa_foreign事件发生在其他node上*/NUMA_FOREIGN,        /* was intended here, hit elsewhere 分配的page本来自此node，但由于此node内存不足，实际分配了其他node的page，每个numa_foreign对应ruma_miss事件*/NUMA_INTERLEAVE_HIT,    /* interleaver preferred this zone interleave策略下分配的page来自此node的次数*/NUMA_LOCAL,      /* allocation from local node 此节点对应的CPU上的进程成功在此node分配到page次数*/NUMA_OTHER,       /* allocation from other node 其他节点对应的CPU上的进程成功在此node分配到page次数*/NR_VM_NUMA_STAT_ITEMS
};
#else
#define NR_VM_NUMA_STAT_ITEMS 0
#endifenum zone_stat_item {/* First 128 byte cacheline (assuming 64 bit words) */NR_FREE_PAGES, //空闲页数量NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry 用于LRU_BASE的统计,LRU链表是从LRU_BASE开始标记的*/NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,//inactive的匿名页数量NR_ZONE_ACTIVE_ANON, //active的匿名页数量NR_ZONE_INACTIVE_FILE, //inactive的文件页数量NR_ZONE_ACTIVE_FILE, //active的文件页数量NR_ZONE_UNEVICTABLE, //不可回收的页数量NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages 属于dirty，writeback和unstable的页的数量*/NR_MLOCK,       /* mlock()ed pages found and moved off LRU mlock锁住的页数量*/NR_PAGETABLE,       /* used for pagetables 用户页表的页数量*//* Second 128 byte cacheline */NR_BOUNCE, //用于Bounce buffer的页数量
#if IS_ENABLED(CONFIG_ZSMALLOC)NR_ZSPAGES,      /* allocated in zsmalloc 用zsmalloc机制分配的页数量*/
#endifNR_FREE_CMA_PAGES, //空闲CMA页数量NR_VM_ZONE_STAT_ITEMS };/* Zone statistics */atomic_long_t       vm_stat[NR_VM_ZONE_STAT_ITEMS];atomic_long_t        vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];

NUMA系统
Node 0, zone      DMA
......pages free     3949min      126low      157high     188
......nr_free_pages 3949nr_zone_inactive_anon 0nr_zone_active_anon 0nr_zone_inactive_file 0nr_zone_active_file 0nr_zone_unevictable 0nr_zone_write_pending 0nr_mlock     0nr_slab_reclaimable 0nr_slab_unreclaimable 25nr_page_table_pages 0nr_kernel_stack 0nr_bounce    0nr_zspages   0numa_hit     4numa_miss    0numa_foreign 0numa_interleave 0numa_local   4numa_other   0nr_free_cma  0protection: (0, 1592, 64023, 64023, 64023)
......
Node 0, zone    DMA32pages free     79853min      12995low      16243high     19491
......nr_free_pages 79853nr_zone_inactive_anon 2612nr_zone_active_anon 22602nr_zone_inactive_file 161381nr_zone_active_file 31110nr_zone_unevictable 0nr_zone_write_pending 0nr_mlock     0nr_slab_reclaimable 103905nr_slab_unreclaimable 1644nr_page_table_pages 4nr_kernel_stack 0nr_bounce    0nr_zspages   0numa_hit     2458074123numa_miss    202448086numa_foreign 0numa_interleave 0numa_local   2457937333numa_other   202584876nr_free_cma  0protection: (0, 0, 62430, 62430, 62430)
......
Node 0, zone   Normalpages free     3953716min      509331low      636663high     763995
......nr_free_pages 3953716nr_zone_inactive_anon 85175nr_zone_active_anon 108741nr_zone_inactive_file 4779186nr_zone_active_file 3893709nr_zone_unevictable 151nr_zone_write_pending 12nr_mlock     151nr_slab_reclaimable 2594461nr_slab_unreclaimable 310561nr_page_table_pages 5952nr_kernel_stack 11656nr_bounce    0nr_zspages   0numa_hit     293952634947numa_miss    11354089760numa_foreign 52237391395numa_interleave 43197numa_local   293950210030numa_other   11356514677nr_free_cma  0protection: (0, 0, 0, 0, 0)
......
Node 1, zone   Normalpages free     5216084min      526122low      657652high     789182
......nr_free_pages 5216084nr_zone_inactive_anon 36150nr_zone_active_anon 42170nr_zone_inactive_file 5507744nr_zone_active_file 4180727nr_zone_unevictable 764nr_zone_write_pending 8nr_mlock     764nr_slab_reclaimable 1102874nr_slab_unreclaimable 305895nr_page_table_pages 4806nr_kernel_stack 10744nr_bounce    0nr_zspages   0numa_hit     297066657309numa_miss    52237391395numa_foreign 11556537846numa_interleave 43138numa_local   297072362418numa_other   52231686286nr_free_cma  0protection: (0, 0, 0, 0, 0)
......UMA系统
Node 0, zone   Normal
......pages free     34010min      3409low      23933high     28593spanned  3144192present  2991568managed  2912800protection: (0, 0)nr_free_pages 34010nr_zone_inactive_anon 139310nr_zone_active_anon 281741nr_zone_inactive_file 1572082nr_zone_active_file 338922nr_zone_unevictable 44830nr_zone_write_pending 72nr_mlock     44829nr_page_table_pages 29103nr_kernel_stack 76704nr_shadow_call_stack_bytes 19636224nr_bounce    0nr_zspages   2235nr_free_cma  0nr_ioncache_pages 57426
......

//include/linux/android_kabi.h
//预留一些padding，以备后面可能需要使用到，添加在结构体的尾部ANDROID_KABI_RESERVE(1);ANDROID_KABI_RESERVE(2);ANDROID_KABI_RESERVE(3);ANDROID_KABI_RESERVE(4);//drivers/android/Kconfig
config ANDROID_KABI_RESERVEbool "Android KABI reserve padding"default yhelpThis option enables the padding that the Android GKI kernel addsto many different kernel structures to support an in-kernel stable ABIover the lifespan of support for the kernel.Only disable this option if you have a system that needs the Androidkernel drivers, but is NOT an Android GKI kernel image. If disabledit has the possibility to make the kernel static and runtime imageslightly smaller but will NOT be supported by the Google Androidkernel team.If even slightly unsure, say Y.//include/linux/android_kabi.h
#define _ANDROID_KABI_RESERVE(n)        u64 android_kabi_reserved##n/** Macros to use _before_ the ABI is frozen*//** ANDROID_KABI_RESERVE*   Reserve some "padding" in a structure for potential future use.*   This normally placed at the end of a structure.*   number: the "number" of the padding variable in the structure.  Start with*   1 and go up.*/
#ifdef CONFIG_ANDROID_KABI_RESERVE
#define ANDROID_KABI_RESERVE(number)    _ANDROID_KABI_RESERVE(number)
#else
#define ANDROID_KABI_RESERVE(number)
#endif

//include/linux/cache.h
#if !defined(____cacheline_internodealigned_in_smp)
#if defined(CONFIG_SMP)
#define ____cacheline_internodealigned_in_smp \__attribute__((__aligned__(1 << (INTERNODE_CACHE_SHIFT))))
#else
#define ____cacheline_internodealigned_in_smp
#endif
#endif