From d87d0f82caa26839bfc3f90866517b22ac15cc88 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Tue, 30 Aug 2022 01:52:51 +0300 Subject: [PATCH 01/30] Version bump. --- src/core/nginx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/nginx.h b/src/core/nginx.h index 428e3a0ec..a7571cdf3 100644 --- a/src/core/nginx.h +++ b/src/core/nginx.h @@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1023001 -#define NGINX_VERSION "1.23.1" +#define nginx_version 1023002 +#define NGINX_VERSION "1.23.2" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD From 4a4572ff4a18b76bdd039c623a5ca0cb56904d34 Mon Sep 17 00:00:00 2001 From: Murilo Andrade Date: Tue, 9 Aug 2022 17:13:46 -0300 Subject: [PATCH 02/30] SSL: logging level of "bad record type" errors. The SSL_R_BAD_RECORD_TYPE ("bad record type") errors are reported by OpenSSL 1.1.1 or newer when using TLSv1.3 if the client sends a record with unknown or unexpected type. These errors are now logged at the "info" level. --- src/event/ngx_event_openssl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 7b0417e4f..bbeb5f0c5 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3422,6 +3422,9 @@ ngx_ssl_connection_error(ngx_connection_t *c, int sslerr, ngx_err_t err, #endif #ifdef SSL_R_VERSION_TOO_LOW || n == SSL_R_VERSION_TOO_LOW /* 396 */ +#endif +#ifdef SSL_R_BAD_RECORD_TYPE + || n == SSL_R_BAD_RECORD_TYPE /* 443 */ #endif || n == 1000 /* SSL_R_SSLV3_ALERT_CLOSE_NOTIFY */ #ifdef SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE From f6a6758ca2afaf34fc2af0ab5d63bec9a0f4880c Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 7 Sep 2022 00:43:51 +0300 Subject: [PATCH 03/30] Events: fixed style and wrong error handling in the iocp module. --- src/event/modules/ngx_iocp_module.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/event/modules/ngx_iocp_module.c b/src/event/modules/ngx_iocp_module.c index b03944bb0..dca5cedff 100644 --- a/src/event/modules/ngx_iocp_module.c +++ b/src/event/modules/ngx_iocp_module.c @@ -231,9 +231,8 @@ ngx_iocp_del_connection(ngx_connection_t *c, ngx_uint_t flags) } -static -ngx_int_t ngx_iocp_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, - ngx_uint_t flags) +static ngx_int_t +ngx_iocp_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) { int rc; u_int key; @@ -356,7 +355,7 @@ ngx_iocp_create_conf(ngx_cycle_t *cycle) cf = ngx_palloc(cycle->pool, sizeof(ngx_iocp_conf_t)); if (cf == NULL) { - return NGX_CONF_ERROR; + return NULL; } cf->threads = NGX_CONF_UNSET; From 116a744799cf46916d1add01ad764718b7c53e9b Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 7 Sep 2022 00:44:10 +0300 Subject: [PATCH 04/30] SSL: fixed incorrect usage of #if instead of #ifdef. In 2014ed60f17f, "#if SSL_CTRL_SET_ECDH_AUTO" test was incorrectly used instead of "#ifdef SSL_CTRL_SET_ECDH_AUTO". There is no practical difference, since SSL_CTRL_SET_ECDH_AUTO evaluates to a non-zero numeric value when defined, but anyway it's better to correctly test if the value is defined. --- src/event/ngx_event_openssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index bbeb5f0c5..a95eec05d 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1426,7 +1426,7 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name) SSL_CTX_set_options(ssl->ctx, SSL_OP_SINGLE_ECDH_USE); -#if SSL_CTRL_SET_ECDH_AUTO +#ifdef SSL_CTRL_SET_ECDH_AUTO /* not needed in OpenSSL 1.1.0+ */ SSL_CTX_set_ecdh_auto(ssl->ctx, 1); #endif From a3016da23f1f05961c77cae05b9731931d01ebe4 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 7 Sep 2022 00:47:02 +0300 Subject: [PATCH 05/30] Win32: removed misleading comment about warnings being disabled. Warnings being disabled are not only from the "-W4" level since e4590dfd97ff. --- src/os/win32/ngx_win32_config.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/os/win32/ngx_win32_config.h b/src/os/win32/ngx_win32_config.h index 96156870d..dcd45fb25 100644 --- a/src/os/win32/ngx_win32_config.h +++ b/src/os/win32/ngx_win32_config.h @@ -80,8 +80,6 @@ typedef long time_t; #pragma warning(default:4201) -/* disable some "-W4" level warnings */ - /* 'type cast': from function pointer to data pointer */ #pragma warning(disable:4054) From 90f1b5fd08e9aca01d568d6653da772980033e43 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 7 Sep 2022 00:47:07 +0300 Subject: [PATCH 06/30] Win32: disabled C4306 warnings with MSVC. Multiple C4306 warnings (conversion from 'type1' to 'type2' of greater size) appear during 64-bit compilation with MSVC 2010 (and older) due to extensively used constructs like "(void *) -1", so they were disabled. In newer MSVC versions C4306 warnings were replaced with C4312 ones, and these are not generated for such trivial type casts. --- src/os/win32/ngx_win32_config.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/os/win32/ngx_win32_config.h b/src/os/win32/ngx_win32_config.h index dcd45fb25..406003a78 100644 --- a/src/os/win32/ngx_win32_config.h +++ b/src/os/win32/ngx_win32_config.h @@ -104,6 +104,9 @@ typedef long time_t; /* array is too small to include a terminating null character */ #pragma warning(disable:4295) +/* conversion from 'type1' to 'type2' of greater size */ +#pragma warning(disable:4306) + #endif From 589f78dd885f52b4ac8216f54d374ef6ef034a8f Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 7 Sep 2022 00:47:17 +0300 Subject: [PATCH 07/30] Win32: fixed build on Windows with OpenSSL 3.0.x (ticket #2379). SSL_sendfile() expects integer file descriptor as an argument, but nginx uses OS file handles (HANDLE) to work with files on Windows, and passing HANDLE instead of an integer correctly results in build failure. Since SSL_sendfile() is not expected to work on Windows anyway, the code is now disabled on Windows with appropriate compile-time checks. --- src/event/ngx_event_openssl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index a95eec05d..66f418a58 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1769,7 +1769,7 @@ ngx_ssl_handshake(ngx_connection_t *c) #endif #endif -#ifdef BIO_get_ktls_send +#if (defined BIO_get_ktls_send && !NGX_WIN32) if (BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection)) == 1) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -1914,7 +1914,7 @@ ngx_ssl_try_early_data(ngx_connection_t *c) c->read->ready = 1; c->write->ready = 1; -#ifdef BIO_get_ktls_send +#if (defined BIO_get_ktls_send && !NGX_WIN32) if (BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection)) == 1) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, @@ -2943,7 +2943,7 @@ ngx_ssl_write_early(ngx_connection_t *c, u_char *data, size_t size) static ssize_t ngx_ssl_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size) { -#ifdef BIO_get_ktls_send +#if (defined BIO_get_ktls_send && !NGX_WIN32) int sslerr, flags; ssize_t n; From fb0890aee646ec4bb2b5c57f113f5bed433be7cc Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 7 Sep 2022 00:47:31 +0300 Subject: [PATCH 08/30] Win32: disabled threads support in OpenSSL builds. Threads are disabled during UNIX builds (see b329c0ab1a48), and also not needed for Windows builds. This used to be the default before OpenSSL 1.1.0. --- auto/lib/openssl/makefile.msvc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto/lib/openssl/makefile.msvc b/auto/lib/openssl/makefile.msvc index 5b90dcb25..a30b28669 100644 --- a/auto/lib/openssl/makefile.msvc +++ b/auto/lib/openssl/makefile.msvc @@ -6,7 +6,7 @@ all: cd $(OPENSSL) - perl Configure VC-WIN32 no-shared \ + perl Configure VC-WIN32 no-shared no-threads \ --prefix="%cd%/openssl" \ --openssldir="%cd%/openssl/ssl" \ $(OPENSSL_OPT) From 5071bc0bcf18c2eade9d452b27d92bee341dd053 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 8 Sep 2022 13:53:49 +0400 Subject: [PATCH 09/30] SSL: silenced GCC warnings when building with BoringSSL. BoringSSL uses macro stub for SSL_CTX_set_ecdh_auto that expands to 1, which triggers -Wunused-value "statement with no effect" warnings. --- src/event/ngx_event_openssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 66f418a58..085ec5530 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1428,7 +1428,7 @@ ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name) #ifdef SSL_CTRL_SET_ECDH_AUTO /* not needed in OpenSSL 1.1.0+ */ - SSL_CTX_set_ecdh_auto(ssl->ctx, 1); + (void) SSL_CTX_set_ecdh_auto(ssl->ctx, 1); #endif if (ngx_strcmp(name->data, "auto") == 0) { From 68119b43620c4da4ce0269a2f860a3df7c4dc0b5 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Mon, 10 Oct 2022 13:57:31 +0400 Subject: [PATCH 10/30] Log only the first line of user input on PROXY protocol v1 error. Previously, all received user input was logged. If a multi-line text was received from client and logged, it could reduce log readability and also make it harder to parse nginx log by scripts. The change brings to PROXY protocol the same behavior that exists for HTTP request line in ngx_http_log_error_handler(). --- src/core/ngx_proxy_protocol.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/ngx_proxy_protocol.c b/src/core/ngx_proxy_protocol.c index 7a9e7f9d1..1f59f1ff4 100644 --- a/src/core/ngx_proxy_protocol.c +++ b/src/core/ngx_proxy_protocol.c @@ -139,8 +139,14 @@ skip: invalid: + for (p = buf; p < last; p++) { + if (*p == CR || *p == LF) { + break; + } + } + ngx_log_error(NGX_LOG_ERR, c->log, 0, - "broken header: \"%*s\"", (size_t) (last - buf), buf); + "broken header: \"%*s\"", (size_t) (p - buf), buf); return NULL; } From 50e3ff8a006100feaa0666cf5e4f9fd5fdcfb721 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 12 Oct 2022 16:58:16 +0400 Subject: [PATCH 11/30] PROXY protocol v2 TLV variables. The variables have prefix $proxy_protocol_tlv_ and are accessible by name and by type. Examples are: $proxy_protocol_tlv_0x01, $proxy_protocol_tlv_alpn. --- src/core/ngx_proxy_protocol.c | 186 +++++++++++++++++++++++++++++- src/core/ngx_proxy_protocol.h | 3 + src/http/ngx_http_variables.c | 39 +++++++ src/stream/ngx_stream_variables.c | 39 +++++++ 4 files changed, 265 insertions(+), 2 deletions(-) diff --git a/src/core/ngx_proxy_protocol.c b/src/core/ngx_proxy_protocol.c index 1f59f1ff4..dfbd04bc9 100644 --- a/src/core/ngx_proxy_protocol.c +++ b/src/core/ngx_proxy_protocol.c @@ -15,6 +15,12 @@ #define ngx_proxy_protocol_parse_uint16(p) ((p)[0] << 8 | (p)[1]) +#define ngx_proxy_protocol_parse_uint32(p) \ + ( ((uint32_t) (p)[0] << 24) \ + + ( (p)[1] << 16) \ + + ( (p)[2] << 8) \ + + ( (p)[3]) ) + typedef struct { u_char signature[12]; @@ -40,12 +46,52 @@ typedef struct { } ngx_proxy_protocol_inet6_addrs_t; +typedef struct { + u_char type; + u_char len[2]; +} ngx_proxy_protocol_tlv_t; + + +typedef struct { + u_char client; + u_char verify[4]; +} ngx_proxy_protocol_tlv_ssl_t; + + +typedef struct { + ngx_str_t name; + ngx_uint_t type; +} ngx_proxy_protocol_tlv_entry_t; + + static u_char *ngx_proxy_protocol_read_addr(ngx_connection_t *c, u_char *p, u_char *last, ngx_str_t *addr); static u_char *ngx_proxy_protocol_read_port(u_char *p, u_char *last, in_port_t *port, u_char sep); static u_char *ngx_proxy_protocol_v2_read(ngx_connection_t *c, u_char *buf, u_char *last); +static ngx_int_t ngx_proxy_protocol_lookup_tlv(ngx_connection_t *c, + ngx_str_t *tlvs, ngx_uint_t type, ngx_str_t *value); + + +static ngx_proxy_protocol_tlv_entry_t ngx_proxy_protocol_tlv_entries[] = { + { ngx_string("alpn"), 0x01 }, + { ngx_string("authority"), 0x02 }, + { ngx_string("unique_id"), 0x05 }, + { ngx_string("ssl"), 0x20 }, + { ngx_string("netns"), 0x30 }, + { ngx_null_string, 0x00 } +}; + + +static ngx_proxy_protocol_tlv_entry_t ngx_proxy_protocol_tlv_ssl_entries[] = { + { ngx_string("version"), 0x21 }, + { ngx_string("cn"), 0x22 }, + { ngx_string("cipher"), 0x23 }, + { ngx_string("sig_alg"), 0x24 }, + { ngx_string("key_alg"), 0x25 }, + { ngx_null_string, 0x00 } +}; u_char * @@ -418,11 +464,147 @@ ngx_proxy_protocol_v2_read(ngx_connection_t *c, u_char *buf, u_char *last) &pp->src_addr, pp->src_port, &pp->dst_addr, pp->dst_port); if (buf < end) { - ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, - "PROXY protocol v2 %z bytes of tlv ignored", end - buf); + pp->tlvs.data = ngx_pnalloc(c->pool, end - buf); + if (pp->tlvs.data == NULL) { + return NULL; + } + + ngx_memcpy(pp->tlvs.data, buf, end - buf); + pp->tlvs.len = end - buf; } c->proxy_protocol = pp; return end; } + + +ngx_int_t +ngx_proxy_protocol_get_tlv(ngx_connection_t *c, ngx_str_t *name, + ngx_str_t *value) +{ + u_char *p; + size_t n; + uint32_t verify; + ngx_str_t ssl, *tlvs; + ngx_int_t rc, type; + ngx_proxy_protocol_tlv_ssl_t *tlv_ssl; + ngx_proxy_protocol_tlv_entry_t *te; + + if (c->proxy_protocol == NULL) { + return NGX_DECLINED; + } + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, + "PROXY protocol v2 get tlv \"%V\"", name); + + te = ngx_proxy_protocol_tlv_entries; + tlvs = &c->proxy_protocol->tlvs; + + p = name->data; + n = name->len; + + if (n >= 4 && p[0] == 's' && p[1] == 's' && p[2] == 'l' && p[3] == '_') { + + rc = ngx_proxy_protocol_lookup_tlv(c, tlvs, 0x20, &ssl); + if (rc != NGX_OK) { + return rc; + } + + if (ssl.len < sizeof(ngx_proxy_protocol_tlv_ssl_t)) { + return NGX_ERROR; + } + + p += 4; + n -= 4; + + if (n == 6 && ngx_strncmp(p, "verify", 6) == 0) { + + tlv_ssl = (ngx_proxy_protocol_tlv_ssl_t *) ssl.data; + verify = ngx_proxy_protocol_parse_uint32(tlv_ssl->verify); + + value->data = ngx_pnalloc(c->pool, NGX_INT32_LEN); + if (value->data == NULL) { + return NGX_ERROR; + } + + value->len = ngx_sprintf(value->data, "%uD", verify) + - value->data; + return NGX_OK; + } + + ssl.data += sizeof(ngx_proxy_protocol_tlv_ssl_t); + ssl.len -= sizeof(ngx_proxy_protocol_tlv_ssl_t); + + te = ngx_proxy_protocol_tlv_ssl_entries; + tlvs = &ssl; + } + + if (n >= 2 && p[0] == '0' && p[1] == 'x') { + + type = ngx_hextoi(p + 2, n - 2); + if (type == NGX_ERROR) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "invalid PROXY protocol TLV \"%V\"", name); + return NGX_ERROR; + } + + return ngx_proxy_protocol_lookup_tlv(c, tlvs, type, value); + } + + for ( /* void */ ; te->type; te++) { + if (te->name.len == n && ngx_strncmp(te->name.data, p, n) == 0) { + return ngx_proxy_protocol_lookup_tlv(c, tlvs, te->type, value); + } + } + + ngx_log_error(NGX_LOG_ERR, c->log, 0, + "unknown PROXY protocol TLV \"%V\"", name); + + return NGX_DECLINED; +} + + +static ngx_int_t +ngx_proxy_protocol_lookup_tlv(ngx_connection_t *c, ngx_str_t *tlvs, + ngx_uint_t type, ngx_str_t *value) +{ + u_char *p; + size_t n, len; + ngx_proxy_protocol_tlv_t *tlv; + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, + "PROXY protocol v2 lookup tlv:%02xi", type); + + p = tlvs->data; + n = tlvs->len; + + while (n) { + if (n < sizeof(ngx_proxy_protocol_tlv_t)) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "broken PROXY protocol TLV"); + return NGX_ERROR; + } + + tlv = (ngx_proxy_protocol_tlv_t *) p; + len = ngx_proxy_protocol_parse_uint16(tlv->len); + + p += sizeof(ngx_proxy_protocol_tlv_t); + n -= sizeof(ngx_proxy_protocol_tlv_t); + + if (n < len) { + ngx_log_error(NGX_LOG_ERR, c->log, 0, "broken PROXY protocol TLV"); + return NGX_ERROR; + } + + if (tlv->type == type) { + value->data = p; + value->len = len; + return NGX_OK; + } + + p += len; + n -= len; + } + + return NGX_DECLINED; +} diff --git a/src/core/ngx_proxy_protocol.h b/src/core/ngx_proxy_protocol.h index b71622094..7d9d3eb70 100644 --- a/src/core/ngx_proxy_protocol.h +++ b/src/core/ngx_proxy_protocol.h @@ -21,6 +21,7 @@ struct ngx_proxy_protocol_s { ngx_str_t dst_addr; in_port_t src_port; in_port_t dst_port; + ngx_str_t tlvs; }; @@ -28,6 +29,8 @@ u_char *ngx_proxy_protocol_read(ngx_connection_t *c, u_char *buf, u_char *last); u_char *ngx_proxy_protocol_write(ngx_connection_t *c, u_char *buf, u_char *last); +ngx_int_t ngx_proxy_protocol_get_tlv(ngx_connection_t *c, ngx_str_t *name, + ngx_str_t *value); #endif /* _NGX_PROXY_PROTOCOL_H_INCLUDED_ */ diff --git a/src/http/ngx_http_variables.c b/src/http/ngx_http_variables.c index f17e90803..16ffda3fe 100644 --- a/src/http/ngx_http_variables.c +++ b/src/http/ngx_http_variables.c @@ -61,6 +61,8 @@ static ngx_int_t ngx_http_variable_proxy_protocol_addr(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_proxy_protocol_port(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_variable_proxy_protocol_tlv(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_server_addr(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_variable_server_port(ngx_http_request_t *r, @@ -214,6 +216,10 @@ static ngx_http_variable_t ngx_http_core_variables[] = { ngx_http_variable_proxy_protocol_port, offsetof(ngx_proxy_protocol_t, dst_port), 0, 0 }, + { ngx_string("proxy_protocol_tlv_"), NULL, + ngx_http_variable_proxy_protocol_tlv, + 0, NGX_HTTP_VAR_PREFIX, 0 }, + { ngx_string("server_addr"), NULL, ngx_http_variable_server_addr, 0, 0, 0 }, { ngx_string("server_port"), NULL, ngx_http_variable_server_port, 0, 0, 0 }, @@ -1386,6 +1392,39 @@ ngx_http_variable_proxy_protocol_port(ngx_http_request_t *r, } +static ngx_int_t +ngx_http_variable_proxy_protocol_tlv(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + ngx_str_t *name = (ngx_str_t *) data; + + ngx_int_t rc; + ngx_str_t tlv, value; + + tlv.len = name->len - (sizeof("proxy_protocol_tlv_") - 1); + tlv.data = name->data + sizeof("proxy_protocol_tlv_") - 1; + + rc = ngx_proxy_protocol_get_tlv(r->connection, &tlv, &value); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_DECLINED) { + v->not_found = 1; + return NGX_OK; + } + + v->len = value.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = value.data; + + return NGX_OK; +} + + static ngx_int_t ngx_http_variable_server_addr(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) diff --git a/src/stream/ngx_stream_variables.c b/src/stream/ngx_stream_variables.c index 8b596683f..812689098 100644 --- a/src/stream/ngx_stream_variables.c +++ b/src/stream/ngx_stream_variables.c @@ -23,6 +23,8 @@ static ngx_int_t ngx_stream_variable_proxy_protocol_addr( ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_variable_proxy_protocol_port( ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_stream_variable_proxy_protocol_tlv( + ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_variable_server_addr(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_stream_variable_server_port(ngx_stream_session_t *s, @@ -79,6 +81,10 @@ static ngx_stream_variable_t ngx_stream_core_variables[] = { ngx_stream_variable_proxy_protocol_port, offsetof(ngx_proxy_protocol_t, dst_port), 0, 0 }, + { ngx_string("proxy_protocol_tlv_"), NULL, + ngx_stream_variable_proxy_protocol_tlv, + 0, NGX_STREAM_VAR_PREFIX, 0 }, + { ngx_string("server_addr"), NULL, ngx_stream_variable_server_addr, 0, 0, 0 }, @@ -621,6 +627,39 @@ ngx_stream_variable_proxy_protocol_port(ngx_stream_session_t *s, } +static ngx_int_t +ngx_stream_variable_proxy_protocol_tlv(ngx_stream_session_t *s, + ngx_stream_variable_value_t *v, uintptr_t data) +{ + ngx_str_t *name = (ngx_str_t *) data; + + ngx_int_t rc; + ngx_str_t tlv, value; + + tlv.len = name->len - (sizeof("proxy_protocol_tlv_") - 1); + tlv.data = name->data + sizeof("proxy_protocol_tlv_") - 1; + + rc = ngx_proxy_protocol_get_tlv(s->connection, &tlv, &value); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + if (rc == NGX_DECLINED) { + v->not_found = 1; + return NGX_OK; + } + + v->len = value.len; + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + v->data = value.data; + + return NGX_OK; +} + + static ngx_int_t ngx_stream_variable_server_addr(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, uintptr_t data) From f27af85016f357048cd097a8779a22517fd62741 Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Tue, 27 Sep 2022 11:31:16 +0400 Subject: [PATCH 12/30] Added type cast to ngx_proxy_protocol_parse_uint16(). The cast is added to make ngx_proxy_protocol_parse_uint16() similar to ngx_proxy_protocol_parse_uint32(). --- src/core/ngx_proxy_protocol.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/ngx_proxy_protocol.c b/src/core/ngx_proxy_protocol.c index dfbd04bc9..2d9c095b1 100644 --- a/src/core/ngx_proxy_protocol.c +++ b/src/core/ngx_proxy_protocol.c @@ -13,7 +13,9 @@ #define NGX_PROXY_PROTOCOL_AF_INET6 2 -#define ngx_proxy_protocol_parse_uint16(p) ((p)[0] << 8 | (p)[1]) +#define ngx_proxy_protocol_parse_uint16(p) \ + ( ((uint16_t) (p)[0] << 8) \ + + ( (p)[1]) ) #define ngx_proxy_protocol_parse_uint32(p) \ ( ((uint32_t) (p)[0] << 24) \ From 23ce9baf0e9802571eae59f29c83eee18628e568 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:34 +0300 Subject: [PATCH 13/30] SSL: disabled saving tickets to session cache. OpenSSL tries to save TLSv1.3 sessions into session cache even when using tickets for stateless session resumption, "because some applications just want to know about the creation of a session". To avoid trashing session cache with useless data, we do not save such sessions now. --- src/event/ngx_event_openssl.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 085ec5530..a80f30253 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3818,6 +3818,23 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) ngx_ssl_session_cache_t *cache; u_char buf[NGX_SSL_MAX_SESSION_SIZE]; +#ifdef TLS1_3_VERSION + + /* + * OpenSSL tries to save TLSv1.3 sessions into session cache + * even when using tickets for stateless session resumption, + * "because some applications just want to know about the creation + * of a session"; do not cache such sessions + */ + + if (SSL_version(ssl_conn) == TLS1_3_VERSION + && (SSL_get_options(ssl_conn) & SSL_OP_NO_TICKET) == 0) + { + return 0; + } + +#endif + len = i2d_SSL_SESSION(sess, NULL); /* do not cache too big session */ From 4ea6baf7bc64cd9936f9e8684667e04471b038f4 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:36 +0300 Subject: [PATCH 14/30] SSL: reduced logging of session cache failures (ticket #621). Session cache allocations might fail as long as the new session is different in size from the one least recently used (and freed when the first allocation fails). In particular, it might not be possible to allocate space for sessions with client certificates, since they are noticeably bigger than normal sessions. To ensure such allocation failures won't clutter logs, logging level changed to "warn", and logging is now limited to at most one warning per second. --- src/event/ngx_event_openssl.c | 9 +++++++-- src/event/ngx_event_openssl.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index a80f30253..5cd0c5bc7 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3770,6 +3770,8 @@ ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data) ngx_queue_init(&cache->expire_queue); + cache->fail_time = 0; + len = sizeof(" in SSL session shared cache \"\"") + shm_zone->shm.name.len; shpool->log_ctx = ngx_slab_alloc(shpool, len); @@ -3953,8 +3955,11 @@ failed: ngx_shmtx_unlock(&shpool->mutex); - ngx_log_error(NGX_LOG_ALERT, c->log, 0, - "could not allocate new session%s", shpool->log_ctx); + if (cache->fail_time != ngx_time()) { + cache->fail_time = ngx_time(); + ngx_log_error(NGX_LOG_WARN, c->log, 0, + "could not allocate new session%s", shpool->log_ctx); + } return 0; } diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index c9e86d9c0..4d005ed75 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -150,6 +150,7 @@ typedef struct { ngx_rbtree_t session_rbtree; ngx_rbtree_node_t sentinel; ngx_queue_t expire_queue; + time_t fail_time; } ngx_ssl_session_cache_t; From aeae4c78453c11adc609abe9c4f8d4117ba8963b Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:37 +0300 Subject: [PATCH 15/30] SSL: updated comment about session sizes. Previous numbers are somewhat outdated, typical ASN1 representations of sessions are slightly bigger now. --- src/event/ngx_event_openssl.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 5cd0c5bc7..e2b0b3a26 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3790,16 +3790,16 @@ ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data) /* * The length of the session id is 16 bytes for SSLv2 sessions and - * between 1 and 32 bytes for SSLv3/TLSv1, typically 32 bytes. - * It seems that the typical length of the external ASN1 representation - * of a session is 118 or 119 bytes for SSLv3/TSLv1. + * between 1 and 32 bytes for SSLv3 and TLS, typically 32 bytes. + * Typical length of the external ASN1 representation of a session + * is about 150 bytes plus SNI server name. * - * Thus on 32-bit platforms we allocate separately an rbtree node, + * On 32-bit platforms we allocate separately an rbtree node, * a session id, and an ASN1 representation, they take accordingly - * 64, 32, and 128 bytes. + * 64, 32, and 256 bytes. * * On 64-bit platforms we allocate separately an rbtree node + session_id, - * and an ASN1 representation, they take accordingly 128 and 128 bytes. + * and an ASN1 representation, they take accordingly 128 and 256 bytes. * * OpenSSL's i2d_SSL_SESSION() and d2i_SSL_SESSION are slow, * so they are outside the code locked by shared pool mutex From 5c5c449ba0ba12708d2c9d121d46426eeaf911da Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:39 +0300 Subject: [PATCH 16/30] SSL: explicit session id length checking. Session ids are not expected to be longer than 32 bytes, but this is theoretically possible with TLSv1.3, where session ids are essentially arbitrary and sent as session tickets. Since on 64-bit platforms we use fixed 32-byte buffer for session ids, added an explicit length check to make sure the buffer is large enough. --- src/event/ngx_event_openssl.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index e2b0b3a26..c6d9af4cd 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3848,6 +3848,14 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) p = buf; i2d_SSL_SESSION(sess, &p); + session_id = (u_char *) SSL_SESSION_get_id(sess, &session_id_length); + + /* do not cache sessions with too long session id */ + + if (session_id_length > 32) { + return 0; + } + c = ngx_ssl_get_connection(ssl_conn); ssl_ctx = c->ssl->session_ctx; @@ -3892,8 +3900,6 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) } } - session_id = (u_char *) SSL_SESSION_get_id(sess, &session_id_length); - #if (NGX_PTR_SIZE == 8) id = sess_id->sess_id; From 76876c160f6f66a1f906a74649f1242844d2b910 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:40 +0300 Subject: [PATCH 17/30] SSL: single allocation in session cache on 32-bit platforms. Given the present typical SSL session sizes, on 32-bit platforms it is now beneficial to store all data in a single allocation, since rbtree node + session id + ASN1 representation of a session takes 256 bytes of shared memory (36 + 32 + 150 = about 218 bytes plus SNI server name). Storing all data in a single allocation is beneficial for SNI names up to about 40 characters long and makes it possible to store about 4000 sessions in one megabyte (instead of about 3000 sessions now). This also slightly simplifies the code. --- src/event/ngx_event_openssl.c | 65 +++++++++++------------------------ src/event/ngx_event_openssl.h | 8 ++--- 2 files changed, 25 insertions(+), 48 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index c6d9af4cd..24b17f985 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3794,9 +3794,9 @@ ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data) * Typical length of the external ASN1 representation of a session * is about 150 bytes plus SNI server name. * - * On 32-bit platforms we allocate separately an rbtree node, - * a session id, and an ASN1 representation, they take accordingly - * 64, 32, and 256 bytes. + * On 32-bit platforms we allocate an rbtree node, a session id, and + * an ASN1 representation in a single allocation, it typically takes + * 256 bytes. * * On 64-bit platforms we allocate separately an rbtree node + session_id, * and an ASN1 representation, they take accordingly 128 and 256 bytes. @@ -3809,7 +3809,8 @@ static int ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) { int len; - u_char *p, *id, *cached_sess, *session_id; + u_char *p, *session_id; + size_t n; uint32_t hash; SSL_CTX *ssl_ctx; unsigned int session_id_length; @@ -3869,23 +3870,13 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) /* drop one or two expired sessions */ ngx_ssl_expire_sessions(cache, shpool, 1); - cached_sess = ngx_slab_alloc_locked(shpool, len); +#if (NGX_PTR_SIZE == 8) + n = sizeof(ngx_ssl_sess_id_t); +#else + n = offsetof(ngx_ssl_sess_id_t, session) + len; +#endif - if (cached_sess == NULL) { - - /* drop the oldest non-expired session and try once more */ - - ngx_ssl_expire_sessions(cache, shpool, 0); - - cached_sess = ngx_slab_alloc_locked(shpool, len); - - if (cached_sess == NULL) { - sess_id = NULL; - goto failed; - } - } - - sess_id = ngx_slab_alloc_locked(shpool, sizeof(ngx_ssl_sess_id_t)); + sess_id = ngx_slab_alloc_locked(shpool, n); if (sess_id == NULL) { @@ -3893,7 +3884,7 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) ngx_ssl_expire_sessions(cache, shpool, 0); - sess_id = ngx_slab_alloc_locked(shpool, sizeof(ngx_ssl_sess_id_t)); + sess_id = ngx_slab_alloc_locked(shpool, n); if (sess_id == NULL) { goto failed; @@ -3902,30 +3893,25 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) #if (NGX_PTR_SIZE == 8) - id = sess_id->sess_id; + sess_id->session = ngx_slab_alloc_locked(shpool, len); -#else - - id = ngx_slab_alloc_locked(shpool, session_id_length); - - if (id == NULL) { + if (sess_id->session == NULL) { /* drop the oldest non-expired session and try once more */ ngx_ssl_expire_sessions(cache, shpool, 0); - id = ngx_slab_alloc_locked(shpool, session_id_length); + sess_id->session = ngx_slab_alloc_locked(shpool, len); - if (id == NULL) { + if (sess_id->session == NULL) { goto failed; } } #endif - ngx_memcpy(cached_sess, buf, len); - - ngx_memcpy(id, session_id, session_id_length); + ngx_memcpy(sess_id->session, buf, len); + ngx_memcpy(sess_id->id, session_id, session_id_length); hash = ngx_crc32_short(session_id, session_id_length); @@ -3935,9 +3921,7 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) sess_id->node.key = hash; sess_id->node.data = (u_char) session_id_length; - sess_id->id = id; sess_id->len = len; - sess_id->session = cached_sess; sess_id->expire = ngx_time() + SSL_CTX_get_timeout(ssl_ctx); @@ -3951,10 +3935,6 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) failed: - if (cached_sess) { - ngx_slab_free_locked(shpool, cached_sess); - } - if (sess_id) { ngx_slab_free_locked(shpool, sess_id); } @@ -4051,9 +4031,8 @@ ngx_ssl_get_cached_session(ngx_ssl_conn_t *ssl_conn, ngx_rbtree_delete(&cache->session_rbtree, node); +#if (NGX_PTR_SIZE == 8) ngx_slab_free_locked(shpool, sess_id->session); -#if (NGX_PTR_SIZE == 4) - ngx_slab_free_locked(shpool, sess_id->id); #endif ngx_slab_free_locked(shpool, sess_id); @@ -4141,9 +4120,8 @@ ngx_ssl_remove_session(SSL_CTX *ssl, ngx_ssl_session_t *sess) ngx_rbtree_delete(&cache->session_rbtree, node); +#if (NGX_PTR_SIZE == 8) ngx_slab_free_locked(shpool, sess_id->session); -#if (NGX_PTR_SIZE == 4) - ngx_slab_free_locked(shpool, sess_id->id); #endif ngx_slab_free_locked(shpool, sess_id); @@ -4190,9 +4168,8 @@ ngx_ssl_expire_sessions(ngx_ssl_session_cache_t *cache, ngx_rbtree_delete(&cache->session_rbtree, &sess_id->node); +#if (NGX_PTR_SIZE == 8) ngx_slab_free_locked(shpool, sess_id->session); -#if (NGX_PTR_SIZE == 4) - ngx_slab_free_locked(shpool, sess_id->id); #endif ngx_slab_free_locked(shpool, sess_id); } diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 4d005ed75..2f12aab90 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -134,14 +134,14 @@ typedef struct ngx_ssl_sess_id_s ngx_ssl_sess_id_t; struct ngx_ssl_sess_id_s { ngx_rbtree_node_t node; - u_char *id; size_t len; - u_char *session; ngx_queue_t queue; time_t expire; + u_char id[32]; #if (NGX_PTR_SIZE == 8) - void *stub; - u_char sess_id[32]; + u_char *session; +#else + u_char session[1]; #endif }; From 3057e6e9ad3ccb0be5cd1546697871cfec4d1fa3 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:43 +0300 Subject: [PATCH 18/30] SSL: explicit clearing of expired sessions. This reduces lifetime of session keying material in server's memory, and therefore can be beneficial from forward secrecy point of view. --- src/event/ngx_event_openssl.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 24b17f985..ec4aa8f59 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -4031,6 +4031,8 @@ ngx_ssl_get_cached_session(ngx_ssl_conn_t *ssl_conn, ngx_rbtree_delete(&cache->session_rbtree, node); + ngx_explicit_memzero(sess_id->session, sess_id->len); + #if (NGX_PTR_SIZE == 8) ngx_slab_free_locked(shpool, sess_id->session); #endif @@ -4120,6 +4122,8 @@ ngx_ssl_remove_session(SSL_CTX *ssl, ngx_ssl_session_t *sess) ngx_rbtree_delete(&cache->session_rbtree, node); + ngx_explicit_memzero(sess_id->session, sess_id->len); + #if (NGX_PTR_SIZE == 8) ngx_slab_free_locked(shpool, sess_id->session); #endif @@ -4168,6 +4172,8 @@ ngx_ssl_expire_sessions(ngx_ssl_session_cache_t *cache, ngx_rbtree_delete(&cache->session_rbtree, &sess_id->node); + ngx_explicit_memzero(sess_id->session, sess_id->len); + #if (NGX_PTR_SIZE == 8) ngx_slab_free_locked(shpool, sess_id->session); #endif From a5b768c5365d1b3b286c9690ef33ae3f54e08321 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:45 +0300 Subject: [PATCH 19/30] SSL: style. Runtime OCSP functions separated from configuration ones. --- src/event/ngx_event_openssl.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 2f12aab90..4de996e57 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -205,10 +205,12 @@ ngx_int_t ngx_ssl_ocsp(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *responder, ngx_uint_t depth, ngx_shm_zone_t *shm_zone); ngx_int_t ngx_ssl_ocsp_resolver(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_resolver_t *resolver, ngx_msec_t resolver_timeout); + ngx_int_t ngx_ssl_ocsp_validate(ngx_connection_t *c); ngx_int_t ngx_ssl_ocsp_get_status(ngx_connection_t *c, const char **s); void ngx_ssl_ocsp_cleanup(ngx_connection_t *c); ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data); + ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file); ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf, ngx_array_t *passwords); From 6e5120a01d09478342af1bf37efeaaede7833f86 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:47 +0300 Subject: [PATCH 20/30] SSL: renamed session ticket key type. The ngx_ssl_session_ticket_key_t is way too long, renamed to ngx_ssl_ticket_key_t to simplify writing code. --- src/event/ngx_event_openssl.c | 40 +++++++++++++++++------------------ src/event/ngx_event_openssl.h | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index ec4aa8f59..5d5cb2abb 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -4229,23 +4229,23 @@ ngx_ssl_session_rbtree_insert_value(ngx_rbtree_node_t *temp, ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) { - u_char buf[80]; - size_t size; - ssize_t n; - ngx_str_t *path; - ngx_file_t file; - ngx_uint_t i; - ngx_array_t *keys; - ngx_file_info_t fi; - ngx_pool_cleanup_t *cln; - ngx_ssl_session_ticket_key_t *key; + u_char buf[80]; + size_t size; + ssize_t n; + ngx_str_t *path; + ngx_file_t file; + ngx_uint_t i; + ngx_array_t *keys; + ngx_file_info_t fi; + ngx_pool_cleanup_t *cln; + ngx_ssl_ticket_key_t *key; if (paths == NULL) { return NGX_OK; } keys = ngx_array_create(cf->pool, paths->nelts, - sizeof(ngx_ssl_session_ticket_key_t)); + sizeof(ngx_ssl_ticket_key_t)); if (keys == NULL) { return NGX_ERROR; } @@ -4372,14 +4372,14 @@ ngx_ssl_session_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) { - size_t size; - SSL_CTX *ssl_ctx; - ngx_uint_t i; - ngx_array_t *keys; - ngx_connection_t *c; - ngx_ssl_session_ticket_key_t *key; - const EVP_MD *digest; - const EVP_CIPHER *cipher; + size_t size; + SSL_CTX *ssl_ctx; + ngx_uint_t i; + ngx_array_t *keys; + ngx_connection_t *c; + ngx_ssl_ticket_key_t *key; + const EVP_MD *digest; + const EVP_CIPHER *cipher; c = ngx_ssl_get_connection(ssl_conn); ssl_ctx = c->ssl->session_ctx; @@ -4508,7 +4508,7 @@ ngx_ssl_session_ticket_keys_cleanup(void *data) ngx_array_t *keys = data; ngx_explicit_memzero(keys->elts, - keys->nelts * sizeof(ngx_ssl_session_ticket_key_t)); + keys->nelts * sizeof(ngx_ssl_ticket_key_t)); } #else diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 4de996e57..f98971246 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -161,7 +161,7 @@ typedef struct { u_char name[16]; u_char hmac_key[32]; u_char aes_key[32]; -} ngx_ssl_session_ticket_key_t; +} ngx_ssl_ticket_key_t; #endif From 2a8e56b1e2addb0fdf9fbd0d33b2c1371409176b Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:49 +0300 Subject: [PATCH 21/30] SSL: renamed session ticket key functions and data index. Previously used names are way too long, renamed to simplify writing code. --- src/event/ngx_event_openssl.c | 27 ++++++++++++--------------- src/event/ngx_event_openssl.h | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 5d5cb2abb..a549ba167 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -71,10 +71,10 @@ static void ngx_ssl_session_rbtree_insert_value(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); #ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB -static int ngx_ssl_session_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, +static int ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc); -static void ngx_ssl_session_ticket_keys_cleanup(void *data); +static void ngx_ssl_ticket_keys_cleanup(void *data); #endif #ifndef X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT @@ -131,7 +131,7 @@ ngx_module_t ngx_openssl_module = { int ngx_ssl_connection_index; int ngx_ssl_server_conf_index; int ngx_ssl_session_cache_index; -int ngx_ssl_session_ticket_keys_index; +int ngx_ssl_ticket_keys_index; int ngx_ssl_ocsp_index; int ngx_ssl_certificate_index; int ngx_ssl_next_certificate_index; @@ -208,9 +208,9 @@ ngx_ssl_init(ngx_log_t *log) return NGX_ERROR; } - ngx_ssl_session_ticket_keys_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, - NULL, NULL); - if (ngx_ssl_session_ticket_keys_index == -1) { + ngx_ssl_ticket_keys_index = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, + NULL); + if (ngx_ssl_ticket_keys_index == -1) { ngx_ssl_error(NGX_LOG_ALERT, log, 0, "SSL_CTX_get_ex_new_index() failed"); return NGX_ERROR; @@ -4255,7 +4255,7 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) return NGX_ERROR; } - cln->handler = ngx_ssl_session_ticket_keys_cleanup; + cln->handler = ngx_ssl_ticket_keys_cleanup; cln->data = keys; path = paths->elts; @@ -4333,16 +4333,13 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) ngx_explicit_memzero(&buf, 80); } - if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_session_ticket_keys_index, keys) - == 0) - { + if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_ticket_keys_index, keys) == 0) { ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, "SSL_CTX_set_ex_data() failed"); return NGX_ERROR; } - if (SSL_CTX_set_tlsext_ticket_key_cb(ssl->ctx, - ngx_ssl_session_ticket_key_callback) + if (SSL_CTX_set_tlsext_ticket_key_cb(ssl->ctx, ngx_ssl_ticket_key_callback) == 0) { ngx_log_error(NGX_LOG_WARN, cf->log, 0, @@ -4368,7 +4365,7 @@ failed: static int -ngx_ssl_session_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, +ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) { @@ -4390,7 +4387,7 @@ ngx_ssl_session_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, digest = EVP_sha256(); #endif - keys = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_session_ticket_keys_index); + keys = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_ticket_keys_index); if (keys == NULL) { return -1; } @@ -4503,7 +4500,7 @@ ngx_ssl_session_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, static void -ngx_ssl_session_ticket_keys_cleanup(void *data) +ngx_ssl_ticket_keys_cleanup(void *data) { ngx_array_t *keys = data; diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index f98971246..936662127 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -317,7 +317,7 @@ void ngx_ssl_cleanup_ctx(void *data); extern int ngx_ssl_connection_index; extern int ngx_ssl_server_conf_index; extern int ngx_ssl_session_cache_index; -extern int ngx_ssl_session_ticket_keys_index; +extern int ngx_ssl_ticket_keys_index; extern int ngx_ssl_ocsp_index; extern int ngx_ssl_certificate_index; extern int ngx_ssl_next_certificate_index; From 3b127da11169c63c04d2ea41a79023b030010a1e Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:51 +0300 Subject: [PATCH 22/30] SSL: shorter debug messages about session tickets. --- src/event/ngx_event_openssl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index a549ba167..c128e96d8 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -4398,7 +4398,7 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, /* encrypt session ticket */ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "ssl session ticket encrypt, key: \"%*xs\" (%s session)", + "ssl ticket encrypt, key: \"%*xs\" (%s session)", (size_t) 16, key[0].name, SSL_session_reused(ssl_conn) ? "reused" : "new"); @@ -4445,7 +4445,7 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, } ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "ssl session ticket decrypt, key: \"%*xs\" not found", + "ssl ticket decrypt, key: \"%*xs\" not found", (size_t) 16, name); return 0; @@ -4453,7 +4453,7 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, found: ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, - "ssl session ticket decrypt, key: \"%*xs\"%s", + "ssl ticket decrypt, key: \"%*xs\"%s", (size_t) 16, key[i].name, (i == 0) ? " (default)" : ""); if (key[i].size == 48) { From 1d572e359a210dcb27e5e073c016c1768c435263 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:53 +0300 Subject: [PATCH 23/30] SSL: automatic rotation of session ticket keys. As long as ssl_session_cache in shared memory is configured, session ticket keys are now automatically generated in shared memory, and rotated periodically. This can be beneficial from forward secrecy point of view, and also avoids increased CPU usage after configuration reloads. This also helps BoringSSL to properly resume sessions in configurations with multiple worker processes and no ssl_session_ticket_key directives, as BoringSSL tries to automatically rotate session ticket keys and does this independently in different worker processes, thus breaking session resumption between worker processes. --- src/event/ngx_event_openssl.c | 167 ++++++++++++++++++++++++++++++---- src/event/ngx_event_openssl.h | 23 +++-- 2 files changed, 160 insertions(+), 30 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index c128e96d8..ba66119b0 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -74,6 +74,7 @@ static void ngx_ssl_session_rbtree_insert_value(ngx_rbtree_node_t *temp, static int ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc); +static ngx_int_t ngx_ssl_rotate_ticket_keys(SSL_CTX *ssl_ctx, ngx_log_t *log); static void ngx_ssl_ticket_keys_cleanup(void *data); #endif @@ -3770,6 +3771,9 @@ ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data) ngx_queue_init(&cache->expire_queue); + cache->ticket_keys[0].expire = 0; + cache->ticket_keys[1].expire = 0; + cache->fail_time = 0; len = sizeof(" in SSL session shared cache \"\"") + shm_zone->shm.name.len; @@ -4240,11 +4244,13 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) ngx_pool_cleanup_t *cln; ngx_ssl_ticket_key_t *key; - if (paths == NULL) { + if (paths == NULL + && SSL_CTX_get_ex_data(ssl->ctx, ngx_ssl_session_cache_index) == NULL) + { return NGX_OK; } - keys = ngx_array_create(cf->pool, paths->nelts, + keys = ngx_array_create(cf->pool, paths ? paths->nelts : 2, sizeof(ngx_ssl_ticket_key_t)); if (keys == NULL) { return NGX_ERROR; @@ -4258,6 +4264,34 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) cln->handler = ngx_ssl_ticket_keys_cleanup; cln->data = keys; + if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_ticket_keys_index, keys) == 0) { + ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, + "SSL_CTX_set_ex_data() failed"); + return NGX_ERROR; + } + + if (SSL_CTX_set_tlsext_ticket_key_cb(ssl->ctx, ngx_ssl_ticket_key_callback) + == 0) + { + ngx_log_error(NGX_LOG_WARN, cf->log, 0, + "nginx was built with Session Tickets support, however, " + "now it is linked dynamically to an OpenSSL library " + "which has no tlsext support, therefore Session Tickets " + "are not available"); + return NGX_OK; + } + + if (paths == NULL) { + + /* placeholder for keys in shared memory */ + + key = ngx_array_push_n(keys, 2); + key[0].shared = 1; + key[1].shared = 1; + + return NGX_OK; + } + path = paths->elts; for (i = 0; i < paths->nelts; i++) { @@ -4312,6 +4346,8 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) goto failed; } + key->shared = 0; + if (size == 48) { key->size = 48; ngx_memcpy(key->name, buf, 16); @@ -4333,22 +4369,6 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) ngx_explicit_memzero(&buf, 80); } - if (SSL_CTX_set_ex_data(ssl->ctx, ngx_ssl_ticket_keys_index, keys) == 0) { - ngx_ssl_error(NGX_LOG_EMERG, ssl->log, 0, - "SSL_CTX_set_ex_data() failed"); - return NGX_ERROR; - } - - if (SSL_CTX_set_tlsext_ticket_key_cb(ssl->ctx, ngx_ssl_ticket_key_callback) - == 0) - { - ngx_log_error(NGX_LOG_WARN, cf->log, 0, - "nginx was built with Session Tickets support, however, " - "now it is linked dynamically to an OpenSSL library " - "which has no tlsext support, therefore Session Tickets " - "are not available"); - } - return NGX_OK; failed: @@ -4381,6 +4401,10 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, c = ngx_ssl_get_connection(ssl_conn); ssl_ctx = c->ssl->session_ctx; + if (ngx_ssl_rotate_ticket_keys(ssl_ctx, c->log) != NGX_OK) { + return -1; + } + #ifdef OPENSSL_NO_SHA256 digest = EVP_sha1(); #else @@ -4499,6 +4523,113 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, } +static ngx_int_t +ngx_ssl_rotate_ticket_keys(SSL_CTX *ssl_ctx, ngx_log_t *log) +{ + time_t now, expire; + ngx_array_t *keys; + ngx_shm_zone_t *shm_zone; + ngx_slab_pool_t *shpool; + ngx_ssl_ticket_key_t *key; + ngx_ssl_session_cache_t *cache; + u_char buf[80]; + + keys = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_ticket_keys_index); + if (keys == NULL) { + return NGX_OK; + } + + key = keys->elts; + + if (!key[0].shared) { + return NGX_OK; + } + + now = ngx_time(); + expire = now + SSL_CTX_get_timeout(ssl_ctx); + + shm_zone = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_session_cache_index); + + cache = shm_zone->data; + shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; + + ngx_shmtx_lock(&shpool->mutex); + + key = cache->ticket_keys; + + if (key[0].expire == 0) { + + /* initialize the current key */ + + if (RAND_bytes(buf, 80) != 1) { + ngx_ssl_error(NGX_LOG_ALERT, log, 0, "RAND_bytes() failed"); + ngx_shmtx_unlock(&shpool->mutex); + return NGX_ERROR; + } + + key->shared = 1; + key->expire = expire; + key->size = 80; + ngx_memcpy(key->name, buf, 16); + ngx_memcpy(key->hmac_key, buf + 16, 32); + ngx_memcpy(key->aes_key, buf + 48, 32); + + ngx_explicit_memzero(&buf, 80); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0, + "ssl ticket key: \"%*xs\"", + (size_t) 16, key->name); + } + + if (key[1].expire < now) { + + /* + * if the previous key is no longer needed (or not initialized), + * replace it with the current key and generate new current key + */ + + key[1] = key[0]; + + if (RAND_bytes(buf, 80) != 1) { + ngx_ssl_error(NGX_LOG_ALERT, log, 0, "RAND_bytes() failed"); + ngx_shmtx_unlock(&shpool->mutex); + return NGX_ERROR; + } + + key->shared = 1; + key->expire = expire; + key->size = 80; + ngx_memcpy(key->name, buf, 16); + ngx_memcpy(key->hmac_key, buf + 16, 32); + ngx_memcpy(key->aes_key, buf + 48, 32); + + ngx_explicit_memzero(&buf, 80); + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0, + "ssl ticket key: \"%*xs\"", + (size_t) 16, key->name); + } + + /* + * update expiration of the current key: it is going to be needed + * at least till the session being created expires + */ + + if (expire > key[0].expire) { + key[0].expire = expire; + } + + /* sync keys to the worker process memory */ + + ngx_memcpy(keys->elts, cache->ticket_keys, + 2 * sizeof(ngx_ssl_ticket_key_t)); + + ngx_shmtx_unlock(&shpool->mutex); + + return NGX_OK; +} + + static void ngx_ssl_ticket_keys_cleanup(void *data) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 936662127..45d8b78a7 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -147,23 +147,22 @@ struct ngx_ssl_sess_id_s { typedef struct { - ngx_rbtree_t session_rbtree; - ngx_rbtree_node_t sentinel; - ngx_queue_t expire_queue; - time_t fail_time; -} ngx_ssl_session_cache_t; - - -#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB - -typedef struct { - size_t size; u_char name[16]; u_char hmac_key[32]; u_char aes_key[32]; + time_t expire; + unsigned size:8; + unsigned shared:1; } ngx_ssl_ticket_key_t; -#endif + +typedef struct { + ngx_rbtree_t session_rbtree; + ngx_rbtree_node_t sentinel; + ngx_queue_t expire_queue; + ngx_ssl_ticket_key_t ticket_keys[2]; + time_t fail_time; +} ngx_ssl_session_cache_t; #define NGX_SSL_SSLv2 0x0002 From 02314f0c3c70bad88ae3554eb66439a261898f24 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:55 +0300 Subject: [PATCH 24/30] SSL: optimized rotation of session ticket keys. Instead of syncing keys with shared memory on each ticket operation, the code now does this only when the worker is going to change expiration of the current key, or going to switch to a new key: that is, usually at most once per second. To do so without races, the code maintains 3 keys: current, previous, and next. If a worker will switch to the next key earlier, other workers will still be able to decrypt new tickets, since they will be encrypted with the next key. --- src/event/ngx_event_openssl.c | 64 +++++++++++++++++++++++++---------- src/event/ngx_event_openssl.h | 2 +- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index ba66119b0..a7d1080ee 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3773,6 +3773,7 @@ ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data) cache->ticket_keys[0].expire = 0; cache->ticket_keys[1].expire = 0; + cache->ticket_keys[2].expire = 0; cache->fail_time = 0; @@ -4250,7 +4251,7 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) return NGX_OK; } - keys = ngx_array_create(cf->pool, paths ? paths->nelts : 2, + keys = ngx_array_create(cf->pool, paths ? paths->nelts : 3, sizeof(ngx_ssl_ticket_key_t)); if (keys == NULL) { return NGX_ERROR; @@ -4285,9 +4286,13 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) /* placeholder for keys in shared memory */ - key = ngx_array_push_n(keys, 2); + key = ngx_array_push_n(keys, 3); key[0].shared = 1; + key[0].expire = 0; key[1].shared = 1; + key[1].expire = 0; + key[2].shared = 1; + key[2].expire = 0; return NGX_OK; } @@ -4347,6 +4352,7 @@ ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *paths) } key->shared = 0; + key->expire = 1; if (size == 48) { key->size = 48; @@ -4514,7 +4520,7 @@ ngx_ssl_ticket_key_callback(ngx_ssl_conn_t *ssl_conn, /* renew if non-default key */ - if (i != 0) { + if (i != 0 && key[i].expire) { return 2; } @@ -4545,9 +4551,21 @@ ngx_ssl_rotate_ticket_keys(SSL_CTX *ssl_ctx, ngx_log_t *log) return NGX_OK; } + /* + * if we don't need to update expiration of the current key + * and the previous key is still needed, don't sync with shared + * memory to save some work; in the worst case other worker process + * will switch to the next key, but this process will still be able + * to decrypt tickets encrypted with it + */ + now = ngx_time(); expire = now + SSL_CTX_get_timeout(ssl_ctx); + if (key[0].expire >= expire && key[1].expire >= now) { + return NGX_OK; + } + shm_zone = SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_session_cache_index); cache = shm_zone->data; @@ -4567,28 +4585,38 @@ ngx_ssl_rotate_ticket_keys(SSL_CTX *ssl_ctx, ngx_log_t *log) return NGX_ERROR; } - key->shared = 1; - key->expire = expire; - key->size = 80; - ngx_memcpy(key->name, buf, 16); - ngx_memcpy(key->hmac_key, buf + 16, 32); - ngx_memcpy(key->aes_key, buf + 48, 32); + key[0].shared = 1; + key[0].expire = expire; + key[0].size = 80; + ngx_memcpy(key[0].name, buf, 16); + ngx_memcpy(key[0].hmac_key, buf + 16, 32); + ngx_memcpy(key[0].aes_key, buf + 48, 32); ngx_explicit_memzero(&buf, 80); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0, "ssl ticket key: \"%*xs\"", - (size_t) 16, key->name); + (size_t) 16, key[0].name); + + /* + * copy the current key to the next key, as initialization of + * the previous key will replace the current key with the next + * key + */ + + key[2] = key[0]; } if (key[1].expire < now) { /* * if the previous key is no longer needed (or not initialized), - * replace it with the current key and generate new current key + * replace it with the current key, replace the current key with + * the next key, and generate new next key */ key[1] = key[0]; + key[0] = key[2]; if (RAND_bytes(buf, 80) != 1) { ngx_ssl_error(NGX_LOG_ALERT, log, 0, "RAND_bytes() failed"); @@ -4596,18 +4624,18 @@ ngx_ssl_rotate_ticket_keys(SSL_CTX *ssl_ctx, ngx_log_t *log) return NGX_ERROR; } - key->shared = 1; - key->expire = expire; - key->size = 80; - ngx_memcpy(key->name, buf, 16); - ngx_memcpy(key->hmac_key, buf + 16, 32); - ngx_memcpy(key->aes_key, buf + 48, 32); + key[2].shared = 1; + key[2].expire = 0; + key[2].size = 80; + ngx_memcpy(key[2].name, buf, 16); + ngx_memcpy(key[2].hmac_key, buf + 16, 32); + ngx_memcpy(key[2].aes_key, buf + 48, 32); ngx_explicit_memzero(&buf, 80); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0, "ssl ticket key: \"%*xs\"", - (size_t) 16, key->name); + (size_t) 16, key[2].name); } /* diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 45d8b78a7..88f91382d 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -160,7 +160,7 @@ typedef struct { ngx_rbtree_t session_rbtree; ngx_rbtree_node_t sentinel; ngx_queue_t expire_queue; - ngx_ssl_ticket_key_t ticket_keys[2]; + ngx_ssl_ticket_key_t ticket_keys[3]; time_t fail_time; } ngx_ssl_session_cache_t; From 4d61d59ae9ecc8f1a5f0f505bf0310035b0d9ce9 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 12 Oct 2022 20:14:57 +0300 Subject: [PATCH 25/30] SSL: workaround for session timeout handling with TLSv1.3. OpenSSL with TLSv1.3 updates the session creation time on session resumption and keeps the session timeout unmodified, making it possible to maintain the session forever, bypassing client certificate expiration and revocation. To make sure session timeouts are actually used, we now update the session creation time and reduce the session timeout accordingly. BoringSSL with TLSv1.3 ignores configured session timeouts and uses a hardcoded timeout instead, 7 days. So we update session timeout to the configured value as soon as a session is created. --- src/event/ngx_event_openssl.c | 47 +++++++++++++++++++++++++++++++++++ src/event/ngx_event_openssl.h | 1 + 2 files changed, 48 insertions(+) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index a7d1080ee..6e6129734 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -1084,6 +1084,53 @@ ngx_ssl_info_callback(const ngx_ssl_conn_t *ssl_conn, int where, int ret) } } +#endif + +#ifdef TLS1_3_VERSION + + if ((where & SSL_CB_ACCEPT_LOOP) == SSL_CB_ACCEPT_LOOP + && SSL_version(ssl_conn) == TLS1_3_VERSION) + { + time_t now, time, timeout, conf_timeout; + SSL_SESSION *sess; + + /* + * OpenSSL with TLSv1.3 updates the session creation time on + * session resumption and keeps the session timeout unmodified, + * making it possible to maintain the session forever, bypassing + * client certificate expiration and revocation. To make sure + * session timeouts are actually used, we now update the session + * creation time and reduce the session timeout accordingly. + * + * BoringSSL with TLSv1.3 ignores configured session timeouts + * and uses a hardcoded timeout instead, 7 days. So we update + * session timeout to the configured value as soon as a session + * is created. + */ + + c = ngx_ssl_get_connection((ngx_ssl_conn_t *) ssl_conn); + sess = SSL_get0_session(ssl_conn); + + if (!c->ssl->session_timeout_set && sess) { + c->ssl->session_timeout_set = 1; + + now = ngx_time(); + time = SSL_SESSION_get_time(sess); + timeout = SSL_SESSION_get_timeout(sess); + conf_timeout = SSL_CTX_get_timeout(c->ssl->session_ctx); + + timeout = ngx_min(timeout, conf_timeout); + + if (now - time >= timeout) { + SSL_SESSION_set1_id_context(sess, (unsigned char *) "", 0); + + } else { + SSL_SESSION_set_time(sess, now); + SSL_SESSION_set_timeout(sess, timeout - (now - time)); + } + } + } + #endif if ((where & SSL_CB_ACCEPT_LOOP) == SSL_CB_ACCEPT_LOOP) { diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h index 88f91382d..860ea26dd 100644 --- a/src/event/ngx_event_openssl.h +++ b/src/event/ngx_event_openssl.h @@ -114,6 +114,7 @@ struct ngx_ssl_connection_s { unsigned no_send_shutdown:1; unsigned shutdown_without_free:1; unsigned handshake_buffer_set:1; + unsigned session_timeout_set:1; unsigned try_early_data:1; unsigned in_early:1; unsigned in_ocsp:1; From 5b23fe690f0ddd0fe4af1ed01d7a4df95aa54d10 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 13 Oct 2022 16:18:56 +0400 Subject: [PATCH 26/30] SSL: removed cast not needed after 5ffd76a9ccf3. --- src/event/ngx_event_openssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index 6e6129734..3be64b6c2 100644 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3894,7 +3894,7 @@ ngx_ssl_new_session(ngx_ssl_conn_t *ssl_conn, ngx_ssl_session_t *sess) /* do not cache too big session */ - if (len > (int) NGX_SSL_MAX_SESSION_SIZE) { + if (len > NGX_SSL_MAX_SESSION_SIZE) { return 0; } From 35fce42269bf1c84eadef6660021cefa08a960d7 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Mon, 17 Oct 2022 16:24:53 +0400 Subject: [PATCH 27/30] SSL: improved validation of ssl_session_cache and ssl_ocsp_cache. Now it properly detects invalid shared zone configuration with omitted size. Previously it used to read outside of the buffer boundary. Found with AddressSanitizer. --- src/http/modules/ngx_http_ssl_module.c | 4 ++-- src/mail/ngx_mail_ssl_module.c | 2 +- src/stream/ngx_stream_ssl_module.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c index d74d46094..6fe5463df 100644 --- a/src/http/modules/ngx_http_ssl_module.c +++ b/src/http/modules/ngx_http_ssl_module.c @@ -1093,7 +1093,7 @@ ngx_http_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) len++; } - if (len == 0) { + if (len == 0 || j == value[i].len) { goto invalid; } @@ -1183,7 +1183,7 @@ ngx_http_ssl_ocsp_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) len++; } - if (len == 0) { + if (len == 0 || j == value[1].len) { goto invalid; } diff --git a/src/mail/ngx_mail_ssl_module.c b/src/mail/ngx_mail_ssl_module.c index 2a1043e66..b80a0ca80 100644 --- a/src/mail/ngx_mail_ssl_module.c +++ b/src/mail/ngx_mail_ssl_module.c @@ -682,7 +682,7 @@ ngx_mail_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) len++; } - if (len == 0) { + if (len == 0 || j == value[i].len) { goto invalid; } diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c index c5308322a..105aa1178 100644 --- a/src/stream/ngx_stream_ssl_module.c +++ b/src/stream/ngx_stream_ssl_module.c @@ -1073,7 +1073,7 @@ ngx_stream_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) len++; } - if (len == 0) { + if (len == 0 || j == value[i].len) { goto invalid; } From 0d23105373e6d8a720b9826079c077b9b4be919d Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Wed, 19 Oct 2022 10:53:17 +0300 Subject: [PATCH 28/30] Mp4: disabled duplicate atoms. Most atoms should not appear more than once in a container. Previously, this was not enforced by the module, which could result in worker process crash, memory corruption and disclosure. --- src/http/modules/ngx_http_mp4_module.c | 147 +++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index 5721efbe6..75a7315f9 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -1121,6 +1121,12 @@ ngx_http_mp4_read_ftyp_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) return NGX_ERROR; } + if (mp4->ftyp_atom.buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 ftyp atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom_size = sizeof(ngx_mp4_atom_header_t) + (size_t) atom_data_size; ftyp_atom = ngx_palloc(mp4->request->pool, atom_size); @@ -1179,6 +1185,12 @@ ngx_http_mp4_read_moov_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) return NGX_DECLINED; } + if (mp4->moov_atom.buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 moov atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + conf = ngx_http_get_module_loc_conf(mp4->request, ngx_http_mp4_module); if (atom_data_size > mp4->buffer_size) { @@ -1246,6 +1258,12 @@ ngx_http_mp4_read_mdat_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "mp4 mdat atom"); + if (mp4->mdat_atom.buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 mdat atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + data = &mp4->mdat_data_buf; data->file = &mp4->file; data->in_file = 1; @@ -1372,6 +1390,12 @@ ngx_http_mp4_read_mvhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) ngx_log_debug0(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "mp4 mvhd atom"); + if (mp4->mvhd_atom.buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 mvhd atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom_header = ngx_mp4_atom_header(mp4); mvhd_atom = (ngx_mp4_mvhd_atom_t *) atom_header; mvhd64_atom = (ngx_mp4_mvhd64_atom_t *) atom_header; @@ -1637,6 +1661,13 @@ ngx_http_mp4_read_tkhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) atom_size = sizeof(ngx_mp4_atom_header_t) + (size_t) atom_data_size; trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_TKHD_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 tkhd atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->tkhd_size = atom_size; trak->movie_duration = duration; @@ -1676,6 +1707,12 @@ ngx_http_mp4_read_mdia_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_MDIA_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 mdia atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->mdia_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -1799,6 +1836,13 @@ ngx_http_mp4_read_mdhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) atom_size = sizeof(ngx_mp4_atom_header_t) + (size_t) atom_data_size; trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_MDHD_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 mdhd atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->mdhd_size = atom_size; trak->timescale = timescale; trak->duration = duration; @@ -1862,6 +1906,12 @@ ngx_http_mp4_read_hdlr_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 hdlr atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->hdlr_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -1890,6 +1940,12 @@ ngx_http_mp4_read_minf_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_MINF_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 minf atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->minf_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -1933,6 +1989,15 @@ ngx_http_mp4_read_vmhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_VMHD_ATOM].buf + || trak->out[NGX_HTTP_MP4_SMHD_ATOM].buf) + { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 vmhd/smhd atom in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->vmhd_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -1964,6 +2029,15 @@ ngx_http_mp4_read_smhd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_VMHD_ATOM].buf + || trak->out[NGX_HTTP_MP4_SMHD_ATOM].buf) + { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 vmhd/smhd atom in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->smhd_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -1995,6 +2069,12 @@ ngx_http_mp4_read_dinf_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_DINF_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 dinf atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->dinf_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -2023,6 +2103,12 @@ ngx_http_mp4_read_stbl_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_STBL_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stbl atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->stbl_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -2144,6 +2230,12 @@ ngx_http_mp4_read_stsd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); + if (trak->out[NGX_HTTP_MP4_STSD_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stsd atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + atom = &trak->stsd_atom_buf; atom->temporary = 1; atom->pos = atom_header; @@ -2212,6 +2304,13 @@ ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) atom_end = atom_table + entries * sizeof(ngx_mp4_stts_entry_t); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_STTS_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stts atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->time_to_sample_entries = entries; atom = &trak->stts_atom_buf; @@ -2480,6 +2579,13 @@ ngx_http_mp4_read_stss_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) "sync sample entries:%uD", entries); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_STSS_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stss atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->sync_samples_entries = entries; atom_table = atom_header + sizeof(ngx_http_mp4_stss_atom_t); @@ -2678,6 +2784,13 @@ ngx_http_mp4_read_ctts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) "composition offset entries:%uD", entries); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_CTTS_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 ctts atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->composition_offset_entries = entries; atom_table = atom_header + sizeof(ngx_mp4_ctts_atom_t); @@ -2881,6 +2994,13 @@ ngx_http_mp4_read_stsc_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) atom_end = atom_table + entries * sizeof(ngx_mp4_stsc_entry_t); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_STSC_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stsc atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->sample_to_chunk_entries = entries; atom = &trak->stsc_atom_buf; @@ -3213,6 +3333,13 @@ ngx_http_mp4_read_stsz_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) "sample uniform size:%uD, entries:%uD", size, entries); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_STSZ_ATOM].buf) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stsz atom in \"%s\"", mp4->file.name.data); + return NGX_ERROR; + } + trak->sample_sizes_entries = entries; atom_table = atom_header + sizeof(ngx_mp4_stsz_atom_t); @@ -3396,6 +3523,16 @@ ngx_http_mp4_read_stco_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) atom_end = atom_table + entries * sizeof(uint32_t); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_STCO_ATOM].buf + || trak->out[NGX_HTTP_MP4_CO64_ATOM].buf) + { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stco/co64 atom in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + trak->chunks = entries; atom = &trak->stco_atom_buf; @@ -3602,6 +3739,16 @@ ngx_http_mp4_read_co64_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) atom_end = atom_table + entries * sizeof(uint64_t); trak = ngx_mp4_last_trak(mp4); + + if (trak->out[NGX_HTTP_MP4_STCO_ATOM].buf + || trak->out[NGX_HTTP_MP4_CO64_ATOM].buf) + { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "duplicate mp4 stco/co64 atom in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + trak->chunks = entries; atom = &trak->co64_atom_buf; From 569f417a7f87ce37ee1d778439a4cd54d3d85cb7 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 19 Oct 2022 10:56:20 +0300 Subject: [PATCH 29/30] nginx-1.23.2-RELEASE --- docs/xml/nginx/changes.xml | 114 +++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/docs/xml/nginx/changes.xml b/docs/xml/nginx/changes.xml index d0381625e..4e2e9a486 100644 --- a/docs/xml/nginx/changes.xml +++ b/docs/xml/nginx/changes.xml @@ -5,6 +5,120 @@ + + + + +обработка специально созданного mp4-файла модулем ngx_http_mp4_module +могла приводить к падению рабочего процесса, +отправке клиенту части содержимого памяти рабочего процесса, +а также потенциально могла иметь другие последствия +(CVE-2022-41741, CVE-2022-41742). + + +processing of a specially crafted mp4 file by the ngx_http_mp4_module +might cause a worker process crash, +worker process memory disclosure, +or might have potential other impact +(CVE-2022-41741, CVE-2022-41742). + + + + + +переменные "$proxy_protocol_tlv_...". + + +the "$proxy_protocol_tlv_..." variables. + + + + + +ключи шифрования TLS session tickets теперь автоматически меняются +при использовании разделяемой памяти в ssl_session_cache. + + +TLS session tickets encryption keys are now automatically rotated +when using shared memory in the "ssl_session_cache" directive. + + + + + +уровень логгирования ошибок SSL "bad record type" +понижен с уровня crit до info.
+Спасибо Murilo Andrade. + + +the logging level of the "bad record type" SSL errors +has been lowered from "crit" to "info".
+Thanks to Murilo Andrade. + + + + + +теперь при использовании разделяемой памяти в ssl_session_cache +сообщения "could not allocate new session" +логгируются на уровне warn вместо alert +и не чаще одного раза в секунду. + + +now when using shared memory in the "ssl_session_cache" directive +the "could not allocate new session" errors +are logged at the "warn" level instead of "alert" +and not more often than once per second. + + + + + +nginx/Windows не собирался с OpenSSL 3.0.x. + + +nginx/Windows could not be built with OpenSSL 3.0.x. + + + + + +в логгировании ошибок протокола PROXY.
+Спасибо Сергею Брестеру. + + +in logging of the PROXY protocol errors.
+Thanks to Sergey Brester. + + + + + +при использовании TLSv1.3 с OpenSSL +разделяемая память из ssl_session_cache расходовалась +в том числе на сессии, использующие TLS session tickets. + + +shared memory from the "ssl_session_cache" directive +was spent on sessions using TLS session tickets +when using TLSv1.3 with OpenSSL. + + + + + +таймаут, заданный с помощью директивы ssl_session_timeout, +не работал при использовании TLSv1.3 с OpenSSL или BoringSSL. + + +timeout specified with the "ssl_session_timeout" directive +did not work when using TLSv1.3 with OpenSSL or BoringSSL. + + + + + + From 09d5305592c3ff8ff30e54dc3e9ce3dbac72cf59 Mon Sep 17 00:00:00 2001 From: Maxim Dounin Date: Wed, 19 Oct 2022 10:56:21 +0300 Subject: [PATCH 30/30] release-1.23.2 tag --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index fdfbe2439..a9108b3ec 100644 --- a/.hgtags +++ b/.hgtags @@ -469,3 +469,4 @@ d986378168fd4d70e0121cabac274c560cca9bdf release-1.21.5 714eb4b2c09e712fb2572a2164ce2bf67638ccac release-1.21.6 5da2c0902e8e2aa4534008a582a60c61c135960e release-1.23.0 a63d0a70afea96813ba6667997bc7d68b5863f0d release-1.23.1 +aa901551a7ebad1e8b0f8c11cb44e3424ba29707 release-1.23.2