【Nginx 源码学习】平滑重启，源码追踪

文章目录

平滑重启难点
平滑重启流程
源码探秘

平滑重启难点

重启意味着新旧接替，在交接任务的过程中势必会存在新旧server并存的情形，因此，最主要的问题在于如何保证新旧server可以并存，如果重启前后的server端口一致，如何保证两者可以监听同一端口。

平滑重启流程

nginx reload流程
(1)向 master 进程发送 HUP 信号(reload命令)
(2)master 进程校验配置文件语法是否正确
(3)master 进程打开新的监听端口
(4)master 进程用新配置启动新的 worker 子进程
(5)master 进程向老 worker 子进程发送 QUIT 信号
(6)老 worker 进程关闭监听句柄，处理完当前连接后结束进程

reload 可以实现平滑重启、

原先我也不得其法，翻来翻去，后来想明白了一点：
拿重启和正常启动做比较不就好了吗？

源码探秘

先定位 “reload” 所在位置：

static ngx_int_t
ngx_get_options(int argc, char *const *argv)
{u_char     *p;ngx_int_t   i;for (i = 1; i < argc; i++) {p = (u_char *) argv[i];if (*p++ != '-') {ngx_log_stderr(0, "invalid option: \"%s\"", argv[i]);return NGX_ERROR;}while (*p) {switch (*p++) {...case 's':if (*p) {ngx_signal = (char *) p;} else if (argv[++i]) {ngx_signal = argv[i];} else {ngx_log_stderr(0, "option \"-s\" requires parameter");return NGX_ERROR;}if (ngx_strcmp(ngx_signal, "stop") == 0|| ngx_strcmp(ngx_signal, "quit") == 0|| ngx_strcmp(ngx_signal, "reopen") == 0|| ngx_strcmp(ngx_signal, "reload") == 0){ngx_process = NGX_PROCESS_SIGNALLER;goto next;}...}}next:continue;}return NGX_OK;
}

这里有正常启动的和重启的，区别在于重启的话ngx_process = NGX_PROCESS_SIGNALLER;

下一步追踪这个函数被调用的地方，运气很好，就被调用了一次：

在main里面：

if (ngx_get_options(argc, argv) != NGX_OK) {return 1;
}

没有什么价值。

再追踪 ngx_process 的位置，是一个全局变量，不过被调用的地方太多了。

于是我就换了个方向，追踪 NGX_PROCESS_SIGNALLER 被调用的位置，机智如我。

这个就好找多了，除了刚那个地方和声明的地方，只有一个地方被用到了：

具体流程就是：
如果有旧循环的话，先从旧的循环中将必需数据拷贝到新循环中，出现一个判断 ngx_process == NGX_PROCESS_SIGNALLER ，如果是，则直接返回新循环。不然就继续初始化下去。

就写在这里吧，代码太长了。

又追踪了 ngx_init_cycle 的被调用的地方，运气不错，也只有一个地方调用了，赋值给 ngx_cycle，拿去初始化 master 了。
在那个函数里面会处理一些后续信号。

ngx_cycle_t *
ngx_init_cycle(ngx_cycle_t *old_cycle)
{void                *rv;char               **senv;ngx_uint_t           i, n;ngx_log_t           *log;ngx_time_t          *tp;ngx_conf_t           conf;ngx_pool_t          *pool;ngx_cycle_t         *cycle, **old;ngx_shm_zone_t      *shm_zone, *oshm_zone;ngx_list_part_t     *part, *opart;ngx_open_file_t     *file;ngx_listening_t     *ls, *nls;ngx_core_conf_t     *ccf, *old_ccf;ngx_core_module_t   *module;char                 hostname[NGX_MAXHOSTNAMELEN];ngx_timezone_update();/* force localtime update with a new timezone */tp = ngx_timeofday();tp->sec = 0;ngx_time_update();log = old_cycle->log;pool = ngx_create_pool(NGX_CYCLE_POOL_SIZE, log);if (pool == NULL) {return NULL;}pool->log = log;cycle = ngx_pcalloc(pool, sizeof(ngx_cycle_t));if (cycle == NULL) {ngx_destroy_pool(pool);return NULL;}cycle->pool = pool;cycle->log = log;cycle->old_cycle = old_cycle;cycle->conf_prefix.len = old_cycle->conf_prefix.len;cycle->conf_prefix.data = ngx_pstrdup(pool, &old_cycle->conf_prefix);if (cycle->conf_prefix.data == NULL) {ngx_destroy_pool(pool);return NULL;}cycle->prefix.len = old_cycle->prefix.len;cycle->prefix.data = ngx_pstrdup(pool, &old_cycle->prefix);if (cycle->prefix.data == NULL) {ngx_destroy_pool(pool);return NULL;}cycle->conf_file.len = old_cycle->conf_file.len;cycle->conf_file.data = ngx_pnalloc(pool, old_cycle->conf_file.len + 1);if (cycle->conf_file.data == NULL) {ngx_destroy_pool(pool);return NULL;}ngx_cpystrn(cycle->conf_file.data, old_cycle->conf_file.data,old_cycle->conf_file.len + 1);cycle->conf_param.len = old_cycle->conf_param.len;cycle->conf_param.data = ngx_pstrdup(pool, &old_cycle->conf_param);if (cycle->conf_param.data == NULL) {ngx_destroy_pool(pool);return NULL;}n = old_cycle->paths.nelts ? old_cycle->paths.nelts : 10;if (ngx_array_init(&cycle->paths, pool, n, sizeof(ngx_path_t *))!= NGX_OK){ngx_destroy_pool(pool);return NULL;}ngx_memzero(cycle->paths.elts, n * sizeof(ngx_path_t *));if (ngx_array_init(&cycle->config_dump, pool, 1, sizeof(ngx_conf_dump_t))!= NGX_OK){ngx_destroy_pool(pool);return NULL;}ngx_rbtree_init(&cycle->config_dump_rbtree, &cycle->config_dump_sentinel,ngx_str_rbtree_insert_value);if (old_cycle->open_files.part.nelts) {n = old_cycle->open_files.part.nelts;for (part = old_cycle->open_files.part.next; part; part = part->next) {n += part->nelts;}} else {n = 20;}if (ngx_list_init(&cycle->open_files, pool, n, sizeof(ngx_open_file_t))!= NGX_OK){ngx_destroy_pool(pool);return NULL;}if (old_cycle->shared_memory.part.nelts) {n = old_cycle->shared_memory.part.nelts;for (part = old_cycle->shared_memory.part.next; part; part = part->next){n += part->nelts;}} else {n = 1;}if (ngx_list_init(&cycle->shared_memory, pool, n, sizeof(ngx_shm_zone_t))!= NGX_OK){ngx_destroy_pool(pool);return NULL;}n = old_cycle->listening.nelts ? old_cycle->listening.nelts : 10;if (ngx_array_init(&cycle->listening, pool, n, sizeof(ngx_listening_t))!= NGX_OK){ngx_destroy_pool(pool);return NULL;}ngx_memzero(cycle->listening.elts, n * sizeof(ngx_listening_t));ngx_queue_init(&cycle->reusable_connections_queue);cycle->conf_ctx = ngx_pcalloc(pool, ngx_max_module * sizeof(void *));if (cycle->conf_ctx == NULL) {ngx_destroy_pool(pool);return NULL;}if (gethostname(hostname, NGX_MAXHOSTNAMELEN) == -1) {ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "gethostname() failed");ngx_destroy_pool(pool);return NULL;}/* on Linux gethostname() silently truncates name that does not fit */hostname[NGX_MAXHOSTNAMELEN - 1] = '\0';cycle->hostname.len = ngx_strlen(hostname);cycle->hostname.data = ngx_pnalloc(pool, cycle->hostname.len);if (cycle->hostname.data == NULL) {ngx_destroy_pool(pool);return NULL;}ngx_strlow(cycle->hostname.data, (u_char *) hostname, cycle->hostname.len);if (ngx_cycle_modules(cycle) != NGX_OK) {ngx_destroy_pool(pool);return NULL;}for (i = 0; cycle->modules[i]; i++) {if (cycle->modules[i]->type != NGX_CORE_MODULE) {continue;}module = cycle->modules[i]->ctx;if (module->create_conf) {rv = module->create_conf(cycle);if (rv == NULL) {ngx_destroy_pool(pool);return NULL;}cycle->conf_ctx[cycle->modules[i]->index] = rv;}}senv = environ;ngx_memzero(&conf, sizeof(ngx_conf_t));/* STUB: init array ? */conf.args = ngx_array_create(pool, 10, sizeof(ngx_str_t));if (conf.args == NULL) {ngx_destroy_pool(pool);return NULL;}conf.temp_pool = ngx_create_pool(NGX_CYCLE_POOL_SIZE, log);if (conf.temp_pool == NULL) {ngx_destroy_pool(pool);return NULL;}conf.ctx = cycle->conf_ctx;conf.cycle = cycle;conf.pool = pool;conf.log = log;conf.module_type = NGX_CORE_MODULE;conf.cmd_type = NGX_MAIN_CONF;#if 0log->log_level = NGX_LOG_DEBUG_ALL;
#endifif (ngx_conf_param(&conf) != NGX_CONF_OK) {environ = senv;ngx_destroy_cycle_pools(&conf);return NULL;}if (ngx_conf_parse(&conf, &cycle->conf_file) != NGX_CONF_OK) {environ = senv;ngx_destroy_cycle_pools(&conf);return NULL;}if (ngx_test_config && !ngx_quiet_mode) {ngx_log_stderr(0, "the configuration file %s syntax is ok",cycle->conf_file.data);}for (i = 0; cycle->modules[i]; i++) {if (cycle->modules[i]->type != NGX_CORE_MODULE) {continue;}module = cycle->modules[i]->ctx;if (module->init_conf) {if (module->init_conf(cycle,cycle->conf_ctx[cycle->modules[i]->index])== NGX_CONF_ERROR){environ = senv;ngx_destroy_cycle_pools(&conf);return NULL;}}}if (ngx_process == NGX_PROCESS_SIGNALLER) {return cycle;}ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);if (ngx_test_config) {if (ngx_create_pidfile(&ccf->pid, log) != NGX_OK) {goto failed;}} else if (!ngx_is_init_cycle(old_cycle)) {/** we do not create the pid file in the first ngx_init_cycle() call* because we need to write the demonized process pid*/old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx,ngx_core_module);if (ccf->pid.len != old_ccf->pid.len|| ngx_strcmp(ccf->pid.data, old_ccf->pid.data) != 0){/* new pid file name */if (ngx_create_pidfile(&ccf->pid, log) != NGX_OK) {goto failed;}ngx_delete_pidfile(old_cycle);}}if (ngx_test_lockfile(cycle->lock_file.data, log) != NGX_OK) {goto failed;}if (ngx_create_paths(cycle, ccf->user) != NGX_OK) {goto failed;}if (ngx_log_open_default(cycle) != NGX_OK) {goto failed;}/* open the new files */part = &cycle->open_files.part;file = part->elts;for (i = 0; /* void */ ; i++) {if (i >= part->nelts) {if (part->next == NULL) {break;}part = part->next;file = part->elts;i = 0;}if (file[i].name.len == 0) {continue;}file[i].fd = ngx_open_file(file[i].name.data,NGX_FILE_APPEND,NGX_FILE_CREATE_OR_OPEN,NGX_FILE_DEFAULT_ACCESS);ngx_log_debug3(NGX_LOG_DEBUG_CORE, log, 0,"log: %p %d \"%s\"",&file[i], file[i].fd, file[i].name.data);if (file[i].fd == NGX_INVALID_FILE) {ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,ngx_open_file_n " \"%s\" failed",file[i].name.data);goto failed;}#if !(NGX_WIN32)if (fcntl(file[i].fd, F_SETFD, FD_CLOEXEC) == -1) {ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,"fcntl(FD_CLOEXEC) \"%s\" failed",file[i].name.data);goto failed;}
#endif}cycle->log = &cycle->new_log;pool->log = &cycle->new_log;/* create shared memory */part = &cycle->shared_memory.part;shm_zone = part->elts;for (i = 0; /* void */ ; i++) {if (i >= part->nelts) {if (part->next == NULL) {break;}part = part->next;shm_zone = part->elts;i = 0;}if (shm_zone[i].shm.size == 0) {ngx_log_error(NGX_LOG_EMERG, log, 0,"zero size shared memory zone \"%V\"",&shm_zone[i].shm.name);goto failed;}shm_zone[i].shm.log = cycle->log;opart = &old_cycle->shared_memory.part;oshm_zone = opart->elts;for (n = 0; /* void */ ; n++) {if (n >= opart->nelts) {if (opart->next == NULL) {break;}opart = opart->next;oshm_zone = opart->elts;n = 0;}if (shm_zone[i].shm.name.len != oshm_zone[n].shm.name.len) {continue;}if (ngx_strncmp(shm_zone[i].shm.name.data,oshm_zone[n].shm.name.data,shm_zone[i].shm.name.len)!= 0){continue;}if (shm_zone[i].tag == oshm_zone[n].tag&& shm_zone[i].shm.size == oshm_zone[n].shm.size&& !shm_zone[i].noreuse){shm_zone[i].shm.addr = oshm_zone[n].shm.addr;
#if (NGX_WIN32)shm_zone[i].shm.handle = oshm_zone[n].shm.handle;
#endifif (shm_zone[i].init(&shm_zone[i], oshm_zone[n].data)!= NGX_OK){goto failed;}goto shm_zone_found;}break;}if (ngx_shm_alloc(&shm_zone[i].shm) != NGX_OK) {goto failed;}if (ngx_init_zone_pool(cycle, &shm_zone[i]) != NGX_OK) {goto failed;}if (shm_zone[i].init(&shm_zone[i], NULL) != NGX_OK) {goto failed;}shm_zone_found:continue;}/* handle the listening sockets */if (old_cycle->listening.nelts) {ls = old_cycle->listening.elts;for (i = 0; i < old_cycle->listening.nelts; i++) {ls[i].remain = 0;}nls = cycle->listening.elts;for (n = 0; n < cycle->listening.nelts; n++) {for (i = 0; i < old_cycle->listening.nelts; i++) {if (ls[i].ignore) {continue;}if (ls[i].remain) {continue;}if (ls[i].type != nls[n].type) {continue;}if (ngx_cmp_sockaddr(nls[n].sockaddr, nls[n].socklen,ls[i].sockaddr, ls[i].socklen, 1)== NGX_OK){nls[n].fd = ls[i].fd;nls[n].previous = &ls[i];ls[i].remain = 1;if (ls[i].backlog != nls[n].backlog) {nls[n].listen = 1;}#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)/** FreeBSD, except the most recent versions,* could not remove accept filter*/nls[n].deferred_accept = ls[i].deferred_accept;if (ls[i].accept_filter && nls[n].accept_filter) {if (ngx_strcmp(ls[i].accept_filter,nls[n].accept_filter)!= 0){nls[n].delete_deferred = 1;nls[n].add_deferred = 1;}} else if (ls[i].accept_filter) {nls[n].delete_deferred = 1;} else if (nls[n].accept_filter) {nls[n].add_deferred = 1;}
#endif#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT)if (ls[i].deferred_accept && !nls[n].deferred_accept) {nls[n].delete_deferred = 1;} else if (ls[i].deferred_accept != nls[n].deferred_accept){nls[n].add_deferred = 1;}
#endif#if (NGX_HAVE_REUSEPORT)if (nls[n].reuseport && !ls[i].reuseport) {nls[n].add_reuseport = 1;}
#endifbreak;}}if (nls[n].fd == (ngx_socket_t) -1) {nls[n].open = 1;
#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)if (nls[n].accept_filter) {nls[n].add_deferred = 1;}
#endif
#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT)if (nls[n].deferred_accept) {nls[n].add_deferred = 1;}
#endif}}} else {ls = cycle->listening.elts;for (i = 0; i < cycle->listening.nelts; i++) {ls[i].open = 1;
#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)if (ls[i].accept_filter) {ls[i].add_deferred = 1;}
#endif
#if (NGX_HAVE_DEFERRED_ACCEPT && defined TCP_DEFER_ACCEPT)if (ls[i].deferred_accept) {ls[i].add_deferred = 1;}
#endif}}if (ngx_open_listening_sockets(cycle) != NGX_OK) {goto failed;}if (!ngx_test_config) {ngx_configure_listening_sockets(cycle);}/* commit the new cycle configuration */if (!ngx_use_stderr) {(void) ngx_log_redirect_stderr(cycle);}pool->log = cycle->log;if (ngx_init_modules(cycle) != NGX_OK) {/* fatal */exit(1);}/* close and delete stuff that lefts from an old cycle *//* free the unnecessary shared memory */opart = &old_cycle->shared_memory.part;oshm_zone = opart->elts;for (i = 0; /* void */ ; i++) {if (i >= opart->nelts) {if (opart->next == NULL) {goto old_shm_zone_done;}opart = opart->next;oshm_zone = opart->elts;i = 0;}part = &cycle->shared_memory.part;shm_zone = part->elts;for (n = 0; /* void */ ; n++) {if (n >= part->nelts) {if (part->next == NULL) {break;}part = part->next;shm_zone = part->elts;n = 0;}if (oshm_zone[i].shm.name.len != shm_zone[n].shm.name.len) {continue;}if (ngx_strncmp(oshm_zone[i].shm.name.data,shm_zone[n].shm.name.data,oshm_zone[i].shm.name.len)!= 0){continue;}if (oshm_zone[i].tag == shm_zone[n].tag&& oshm_zone[i].shm.size == shm_zone[n].shm.size&& !oshm_zone[i].noreuse){goto live_shm_zone;}break;}ngx_shm_free(&oshm_zone[i].shm);live_shm_zone:continue;}old_shm_zone_done:/* close the unnecessary listening sockets */ls = old_cycle->listening.elts;for (i = 0; i < old_cycle->listening.nelts; i++) {if (ls[i].remain || ls[i].fd == (ngx_socket_t) -1) {continue;}if (ngx_close_socket(ls[i].fd) == -1) {ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,ngx_close_socket_n " listening socket on %V failed",&ls[i].addr_text);}#if (NGX_HAVE_UNIX_DOMAIN)if (ls[i].sockaddr->sa_family == AF_UNIX) {u_char  *name;name = ls[i].addr_text.data + sizeof("unix:") - 1;ngx_log_error(NGX_LOG_WARN, cycle->log, 0,"deleting socket %s", name);if (ngx_delete_file(name) == NGX_FILE_ERROR) {ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,ngx_delete_file_n " %s failed", name);}}#endif}/* close the unnecessary open files */part = &old_cycle->open_files.part;file = part->elts;for (i = 0; /* void */ ; i++) {if (i >= part->nelts) {if (part->next == NULL) {break;}part = part->next;file = part->elts;i = 0;}if (file[i].fd == NGX_INVALID_FILE || file[i].fd == ngx_stderr) {continue;}if (ngx_close_file(file[i].fd) == NGX_FILE_ERROR) {ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,ngx_close_file_n " \"%s\" failed",file[i].name.data);}}ngx_destroy_pool(conf.temp_pool);if (ngx_process == NGX_PROCESS_MASTER || ngx_is_init_cycle(old_cycle)) {ngx_destroy_pool(old_cycle->pool);cycle->old_cycle = NULL;return cycle;}if (ngx_temp_pool == NULL) {ngx_temp_pool = ngx_create_pool(128, cycle->log);if (ngx_temp_pool == NULL) {ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,"could not create ngx_temp_pool");exit(1);}n = 10;if (ngx_array_init(&ngx_old_cycles, ngx_temp_pool, n,sizeof(ngx_cycle_t *))!= NGX_OK){exit(1);}ngx_memzero(ngx_old_cycles.elts, n * sizeof(ngx_cycle_t *));ngx_cleaner_event.handler = ngx_clean_old_cycles;ngx_cleaner_event.log = cycle->log;ngx_cleaner_event.data = &dumb;dumb.fd = (ngx_socket_t) -1;}ngx_temp_pool->log = cycle->log;old = ngx_array_push(&ngx_old_cycles);if (old == NULL) {exit(1);}*old = old_cycle;if (!ngx_cleaner_event.timer_set) {ngx_add_timer(&ngx_cleaner_event, 30000);ngx_cleaner_event.timer_set = 1;}return cycle;failed:if (!ngx_is_init_cycle(old_cycle)) {old_ccf = (ngx_core_conf_t *) ngx_get_conf(old_cycle->conf_ctx,ngx_core_module);if (old_ccf->environment) {environ = old_ccf->environment;}}/* rollback the new cycle configuration */part = &cycle->open_files.part;file = part->elts;for (i = 0; /* void */ ; i++) {if (i >= part->nelts) {if (part->next == NULL) {break;}part = part->next;file = part->elts;i = 0;}if (file[i].fd == NGX_INVALID_FILE || file[i].fd == ngx_stderr) {continue;}if (ngx_close_file(file[i].fd) == NGX_FILE_ERROR) {ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,ngx_close_file_n " \"%s\" failed",file[i].name.data);}}/* free the newly created shared memory */part = &cycle->shared_memory.part;shm_zone = part->elts;for (i = 0; /* void */ ; i++) {if (i >= part->nelts) {if (part->next == NULL) {break;}part = part->next;shm_zone = part->elts;i = 0;}if (shm_zone[i].shm.addr == NULL) {continue;}opart = &old_cycle->shared_memory.part;oshm_zone = opart->elts;for (n = 0; /* void */ ; n++) {if (n >= opart->nelts) {if (opart->next == NULL) {break;}opart = opart->next;oshm_zone = opart->elts;n = 0;}if (shm_zone[i].shm.name.len != oshm_zone[n].shm.name.len) {continue;}if (ngx_strncmp(shm_zone[i].shm.name.data,oshm_zone[n].shm.name.data,shm_zone[i].shm.name.len)!= 0){continue;}if (shm_zone[i].tag == oshm_zone[n].tag&& shm_zone[i].shm.size == oshm_zone[n].shm.size&& !shm_zone[i].noreuse){goto old_shm_zone_found;}break;}ngx_shm_free(&shm_zone[i].shm);old_shm_zone_found:continue;}if (ngx_test_config) {ngx_destroy_cycle_pools(&conf);return NULL;}ls = cycle->listening.elts;for (i = 0; i < cycle->listening.nelts; i++) {if (ls[i].fd == (ngx_socket_t) -1 || !ls[i].open) {continue;}if (ngx_close_socket(ls[i].fd) == -1) {ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,ngx_close_socket_n " %V failed",&ls[i].addr_text);}}ngx_destroy_cycle_pools(&conf);return NULL;
}