diff --git a/fw/connection.h b/fw/connection.h index 0cc44c654..54d002d18 100644 --- a/fw/connection.h +++ b/fw/connection.h @@ -438,10 +438,16 @@ tfw_connection_live(TfwConn *conn) #define tfw_srv_conn_live(c) tfw_connection_live((TfwConn *)(c)) +static inline void +tfw_connection_get_many(TfwConn *conn, int cnt) +{ + atomic_add(cnt, &conn->refcnt); +} + static inline void tfw_connection_get(TfwConn *conn) { - atomic_inc(&conn->refcnt); + tfw_connection_get_many(conn, 1); } #define TFW_CONNETION_GET_IF(name, cond) \ diff --git a/fw/http.c b/fw/http.c index 10d267e20..d960f5e48 100644 --- a/fw/http.c +++ b/fw/http.c @@ -1146,49 +1146,50 @@ tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status, return 0; } +static inline bool +tfw_http_resp_check_skb_head_owner(TfwHttpResp *resp) +{ + return (TFW_SKB_CB(resp->msg.skb_head)->opaque_data == + CLIENT_MEM_FROM_CONN(resp->req->conn)) + && ss_skb_has_dflt_destructor(resp->msg.skb_head); +} + void tfw_h2_resp_fwd(TfwHttpResp *resp) { - bool resp_in_xmit = !!TFW_SKB_CB(resp->msg.skb_head)->stream_id; TfwHttpReq *req = resp->req; TfwConn *conn = req->conn; int status = READ_ONCE(resp->status); - bool need_extra_put = false; - tfw_connection_get(conn); + /* + * `tfw_h2_stream_init_for_xmit` should be called for + * response before send it to the client. + * + */ + if (WARN_ON(!TFW_SKB_CB(resp->msg.skb_head)->stream_id + || !tfw_http_resp_check_skb_head_owner(resp))) + return; + /* * We need this extra get, because if send fails, connection * will be put during freeing skbs of sending response (in * skb destructor). */ - if (resp_in_xmit) { - TfwClientMem *owner = - TFW_SKB_CB(resp->msg.skb_head)->opaque_data; - - WARN_ON(owner != CLIENT_MEM_FROM_CONN(resp->req->conn)); - TFW_SKB_CB(resp->msg.skb_head)->opaque_data = resp; - TFW_SKB_CB(resp->msg.skb_head)->destructor = - tfw_h2_stream_skb_destructor; - need_extra_put = true; - tfw_connection_get(conn); - } + tfw_connection_get_many(conn, 2); + __ss_skb_set_owner(resp->msg.skb_head, tfw_h2_stream_skb_destructor, + resp); do_access_log(resp); if (tfw_cli_conn_send((TfwCliConn *)conn, (TfwMsg *)resp)) { T_DBG("%s: cannot send data to client via HTTP/2\n", __func__); TFW_INC_STAT_BH(serv.msgs_otherr); - /* We can't send response, so we should free it here. */ tfw_connection_close(conn, true); - resp_in_xmit = !resp_in_xmit || !resp->msg.skb_head; } else { TFW_INC_STAT_BH(serv.msgs_forwarded); tfw_inc_global_hm_stats(status); } - if (!resp_in_xmit) - tfw_http_resp_pair_free_and_put_conn(resp); - if (need_extra_put) - tfw_connection_put(conn); + tfw_connection_put(conn); } /* @@ -1338,13 +1339,23 @@ tfw_http_req_init_ss_flags(TfwSrvConn *srv_conn, TfwHttpReq *req) ((TfwMsg *)req)->ss_flags |= SS_F_KEEP_SKB; } -static inline void -tfw_http_resp_init_ss_flags(TfwHttpResp *resp) +void +tfw_http_req_skb_destructor(struct sk_buff *skb) { - if (test_bit(TFW_HTTP_B_CONN_CLOSE, resp->req->flags)) - resp->msg.ss_flags |= SS_F_CONN_CLOSE; - if (test_bit(TFW_HTTP_B_CONN_CLOSE_FORCE, resp->req->flags)) - resp->msg.ss_flags |= __SS_F_FORCE; + TfwHttpReq *req = (TfwHttpReq *)TFW_SKB_CB(skb)->opaque_data; + + /* + * We can safely access `req` pointer here, because + * we remove change this destructor to default in + * `tfw_http_req_on_tcp_entail`. + * If server connection will be dropped before `on_tcp_entail` + * callback is called, this destructor will be called during + * server connection release. + * request never destroed on the client side until the + * response for current request will be fully received. + */ + TFW_SKB_CB(skb)->opaque_data = CLIENT_MEM_FROM_CONN(req->conn); + ss_skb_dflt_destructor(skb); } /* @@ -1357,18 +1368,6 @@ tfw_http_conn_nip_reset(TfwSrvConn *srv_conn) clear_bit(TFW_CONN_B_HASNIP, &srv_conn->flags); } -/* - * Put @req on the list of non-idempotent requests in @srv_conn. - * Raise the flag saying that @srv_conn has non-idempotent requests. - */ -static inline void -tfw_http_req_nip_enlist(TfwSrvConn *srv_conn, TfwHttpReq *req) -{ - BUG_ON(!list_empty(&req->nip_list)); - list_add_tail(&req->nip_list, &srv_conn->nip_queue); - set_bit(TFW_CONN_B_HASNIP, &srv_conn->flags); -} - /* * Remove @req from the list of non-idempotent requests in @srv_conn. * If it is the last request on the list, then clear the flag saying @@ -1385,6 +1384,138 @@ tfw_http_req_nip_delist(TfwSrvConn *srv_conn, TfwHttpReq *req) } } +/** + * Remove @req from the server connection's forwarding queue. + * Caller must care about @srv_conn->last_msg_sent on it's own to keep the + * queue state consistent. + */ +static inline void +tfw_http_req_delist(TfwSrvConn *srv_conn, TfwHttpReq *req) +{ + tfw_http_req_nip_delist(srv_conn, req); + list_del_init(&req->fwd_list); + srv_conn->qsize--; +} + +/* + * Get the request that is previous to @msg. + */ +static inline TfwMsg * +__tfw_http_conn_msg_sent_prev(TfwSrvConn *srv_conn, TfwMsg *msg) +{ + TfwHttpReq *req_sent = (TfwHttpReq *)msg; + + /* + * There is list_is_last() function in the Linux kernel, + * but there is no list_is_first(). The condition below + * is an implementation of list_is_first(). + */ + return (srv_conn->fwd_queue.next == &req_sent->fwd_list) ? + NULL : (TfwMsg *)list_prev_entry(req_sent, fwd_list); +} + +static inline TfwMsg * +__tfw_http_conn_curr_msg_sent_prev(TfwSrvConn *srv_conn) +{ + if (unlikely(!srv_conn->curr_msg_sent)) + return NULL; + + return __tfw_http_conn_msg_sent_prev(srv_conn, + srv_conn->curr_msg_sent); +} + +static inline TfwMsg * +__tfw_http_conn_last_msg_sent_prev(TfwSrvConn *srv_conn) +{ + if (unlikely(!srv_conn->last_msg_sent)) + return NULL; + + return __tfw_http_conn_msg_sent_prev(srv_conn, + srv_conn->last_msg_sent); +} + +static inline bool +tfw_http_req_is_valid(TfwHttpReq *req) +{ + return ((TFW_MSG_H2(req) && req->stream) + || (!TFW_MSG_H2(req) + && !test_bit(TFW_HTTP_B_REQ_DROP, req->flags))); +} + +static int +tfw_http_req_on_tcp_entail(void *conn, struct sk_buff *skb) +{ + TfwSrvConn *srv_conn = (TfwSrvConn *)conn; + TfwHttpReq *req = (TfwHttpReq *)TFW_SKB_CB(skb)->opaque_data; + + /* + * Should be called at the beginning of this function, + * before request deleting to prevent memory corruption + * from `tfw_http_req_skb_destructor` when we will free + * skbs after this callback fails. + */ + __ss_skb_set_owner(skb, ss_skb_dflt_destructor, + CLIENT_MEM_FROM_CONN(req->conn)); + + if (unlikely(!tfw_http_req_is_valid(req))) { + spin_lock_bh(&srv_conn->fwd_qlock); + /* + * Should be called before removing request from the + * server connection queue. + */ + if ((TfwMsg *)req == srv_conn->last_msg_sent) { + srv_conn->last_msg_sent = + __tfw_http_conn_last_msg_sent_prev(srv_conn); + } + if ((TfwMsg *)req == srv_conn->curr_msg_sent) { + srv_conn->curr_msg_sent = + __tfw_http_conn_curr_msg_sent_prev(srv_conn); + } + tfw_http_req_delist(srv_conn, req); + tfw_http_conn_msg_free((TfwHttpMsg *)req); + + spin_unlock_bh(&srv_conn->fwd_qlock); + + return -EINVAL; + } + + return 0; +} + +static inline void +tfw_http_req_init_for_send(TfwSrvConn *srv_conn, TfwHttpReq *req) +{ + tfw_http_req_init_ss_flags(srv_conn, req); + if (likely(test_bit(TFW_HTTP_B_HMONITOR, req->flags))) + return; + + TFW_SKB_CB(req->msg.skb_head)->on_tcp_entail = + tfw_http_req_on_tcp_entail; + __ss_skb_set_owner(req->msg.skb_head, tfw_http_req_skb_destructor, + req); +} + +static inline void +tfw_http_resp_init_ss_flags(TfwHttpResp *resp) +{ + if (test_bit(TFW_HTTP_B_CONN_CLOSE, resp->req->flags)) + resp->msg.ss_flags |= SS_F_CONN_CLOSE; + if (test_bit(TFW_HTTP_B_CONN_CLOSE_FORCE, resp->req->flags)) + resp->msg.ss_flags |= __SS_F_FORCE; +} + +/* + * Put @req on the list of non-idempotent requests in @srv_conn. + * Raise the flag saying that @srv_conn has non-idempotent requests. + */ +static inline void +tfw_http_req_nip_enlist(TfwSrvConn *srv_conn, TfwHttpReq *req) +{ + BUG_ON(!list_empty(&req->nip_list)); + list_add_tail(&req->nip_list, &srv_conn->nip_queue); + set_bit(TFW_CONN_B_HASNIP, &srv_conn->flags); +} + /* * Remove idempotent requests from the list of non-idempotent requests * in @srv_conn. A non-idempotent request may become idempotent when @@ -1453,23 +1584,6 @@ tfw_http_conn_need_fwd(TfwSrvConn *srv_conn) && !tfw_http_conn_drained(srv_conn)); } -/* - * Get the request that is previous to @srv_conn->last_msg_sent. - */ -static inline TfwMsg * -__tfw_http_conn_msg_sent_prev(TfwSrvConn *srv_conn) -{ - TfwHttpReq *req_sent = (TfwHttpReq *)srv_conn->last_msg_sent; - - /* - * There is list_is_last() function in the Linux kernel, - * but there is no list_is_first(). The condition below - * is an implementation of list_is_first(). - */ - return (srv_conn->fwd_queue.next == &req_sent->fwd_list) ? - NULL : (TfwMsg *)list_prev_entry(req_sent, fwd_list); -} - /* * Reset server connection's @fwd_queue and move all requests * to @dst list. @@ -1496,19 +1610,6 @@ tfw_http_req_enlist(TfwSrvConn *srv_conn, TfwHttpReq *req) tfw_http_req_nip_enlist(srv_conn, req); } -/** - * Remove @req from the server connection's forwarding queue. - * Caller must care about @srv_conn->last_msg_sent on it's own to keep the - * queue state consistent. - */ -static inline void -tfw_http_req_delist(TfwSrvConn *srv_conn, TfwHttpReq *req) -{ - tfw_http_req_nip_delist(srv_conn, req); - list_del_init(&req->fwd_list); - srv_conn->qsize--; -} - /* * Common actions in case of an error while forwarding requests. * Erroneous requests are removed from the forwarding queue and placed @@ -1953,14 +2054,10 @@ tfw_http_req_zap_error(struct list_head *eq) list_for_each_entry_safe(req, tmp, eq, fwd_list) { list_del_init(&req->fwd_list); - if ((TFW_MSG_H2(req) && req->stream) - || (!TFW_MSG_H2(req) - && !test_bit(TFW_HTTP_B_REQ_DROP, req->flags))) - { + if (tfw_http_req_is_valid(req)) { tfw_http_send_err_resp(req, req->httperr.status, req->httperr.reason); - } - else { + } else { tfw_http_conn_msg_free((TfwHttpMsg *)req); } @@ -2075,7 +2172,7 @@ tfw_http_req_fwd_send(TfwSrvConn *srv_conn, TfwServer *srv, TfwHttpReq *req, int r; req->jtxtstamp = jiffies; - tfw_http_req_init_ss_flags(srv_conn, req); + tfw_http_req_init_for_send(srv_conn, req); /* * We set TFW_CONN_B_UNSCHED on server connection. New requests must @@ -2287,8 +2384,16 @@ tfw_http_conn_treatnip(TfwSrvConn *srv_conn, struct list_head *eq) && !(srv->sg->flags & TFW_SRV_RETRY_NIP)) { BUG_ON(list_empty(&req_sent->nip_list)); + /* + * This function is called during connection repair, + * so `srv_conn->curr_msg_sent` is set to NULL. + * If it is not NULL and we don't update it here + * (when we update `last_msg_sent`), we catch BUG_ON + * later during requests rescheduling. + */ + BUG_ON(srv_conn->curr_msg_sent); srv_conn->last_msg_sent = - __tfw_http_conn_msg_sent_prev(srv_conn); + __tfw_http_conn_last_msg_sent_prev(srv_conn); tfw_http_nip_req_resched_err(srv_conn, req_sent, eq); } } @@ -2620,6 +2725,12 @@ tfw_http_conn_shrink_fwdq(TfwSrvConn *srv_conn) return; } + /* + * This function is called during connection repairing, + * `curr_msg_sent` was set to NULL. If it is not NULL + * and we don't update it during updating `last_msg_sent` + * we catch BUG_ON. + */ BUG_ON(srv_conn->curr_msg_sent); /* @@ -2644,7 +2755,7 @@ tfw_http_conn_shrink_fwdq(TfwSrvConn *srv_conn) * reassign @srv_conn->last_msg_sent in case it is evicted. * @req is now the same as @srv_conn->last_msg_sent. */ - msg_sent_prev = __tfw_http_conn_msg_sent_prev(srv_conn); + msg_sent_prev = __tfw_http_conn_last_msg_sent_prev(srv_conn); if (tfw_http_req_evict_stale_req(srv_conn, srv, req, &eq)) srv_conn->last_msg_sent = msg_sent_prev; } @@ -4626,7 +4737,6 @@ tfw_http_resp_get_conn_flags(TfwHttpResp *resp) static int tfw_http_resp_set_empty_skb_head(TfwHttpResp *resp, TfwHttpMsgCleanup *cleanup) { - void *opaque_data = TFW_SKB_CB(resp->msg.skb_head)->opaque_data; TfwMsgIter *iter = &resp->iter; struct sk_buff *nskb; @@ -4634,8 +4744,7 @@ tfw_http_resp_set_empty_skb_head(TfwHttpResp *resp, TfwHttpMsgCleanup *cleanup) if (unlikely(!nskb)) return -ENOMEM; - ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - opaque_data, nskb->truesize); + ss_skb_copy_owner(nskb, resp->msg.skb_head, nskb->truesize); nskb->mark = resp->msg.skb_head->mark; cleanup->skb_head = resp->msg.skb_head; resp->msg.skb_head = NULL; @@ -5486,9 +5595,8 @@ tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head) if (WARN_ON(stream->xmit.skb_head || stream->xmit.resp)) return -EINVAL; - TFW_SKB_CB(*skb_head)->opaque_data = - CLIENT_MEM_FROM_CONN(resp->req->conn); - TFW_SKB_CB(*skb_head)->destructor = ss_skb_dflt_destructor; + __ss_skb_set_owner(*skb_head, ss_skb_dflt_destructor, + CLIENT_MEM_FROM_CONN(resp->req->conn)); stream->xmit.resp = resp; if (test_bit(TFW_HTTP_B_CLOSE_ERROR_RESPONSE, stream->xmit.resp->flags)) @@ -7459,11 +7567,7 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb, /* `cli_conn` is equal to zero for health monitor requests. */ if (likely(cli_conn)) { - if (TFW_FSM_TYPE(cli_conn->proto.type) == TFW_FSM_H2) - conn_stop = !hmresp->req->stream; - else - conn_stop = test_bit(TFW_HTTP_B_REQ_DROP, - hmresp->req->flags); + conn_stop = !tfw_http_req_is_valid(hmresp->req); ss_skb_set_owner(skb, ss_skb_dflt_destructor, CLIENT_MEM_FROM_CONN(cli_conn), skb->truesize); diff --git a/fw/http.h b/fw/http.h index c77100f83..d147a5f0e 100644 --- a/fw/http.h +++ b/fw/http.h @@ -815,5 +815,6 @@ void tfw_http_extract_request_authority(TfwHttpReq *req); bool tfw_http_mark_is_in_whitlist(unsigned int mark); char *tfw_http_resp_status_line(int status, size_t *len); int tfw_h2_on_send_resp(void *conn, struct sk_buff **skb_head); +void tfw_http_req_skb_destructor(struct sk_buff *skb); #endif /* __TFW_HTTP_H__ */ diff --git a/fw/http2.c b/fw/http2.c index aedd48780..2ff04e9a8 100644 --- a/fw/http2.c +++ b/fw/http2.c @@ -721,7 +721,16 @@ tfw_h2_entail_stream_skb(struct sock *sk, TfwStream *stream, struct sk_buff *skb, *split; int r = 0; - BUG_ON(!TFW_SKB_CB(stream->xmit.skb_head)->is_head); + /* + * Currently we use this function only for sending HEADERS, DATA + * or TRAILER frames for http2 response. `on_tcp_entail` callback + * is not implemented for responses. For it's implementation + * we should rework this function. + */ + if (WARN_ON(!TFW_SKB_CB(stream->xmit.skb_head)->is_head + || TFW_SKB_CB(stream->xmit.skb_head)->on_tcp_entail)) + return -EINVAL; + while (*len) { skb = ss_skb_dequeue(&stream->xmit.skb_head); BUG_ON(!skb); diff --git a/fw/http_frame.c b/fw/http_frame.c index 1988bc583..09c02c102 100644 --- a/fw/http_frame.c +++ b/fw/http_frame.c @@ -227,13 +227,15 @@ ctx_new_settings_flags[] = { [HTTP2_SETTINGS_MAX_HDR_LIST_SIZE] = 0x20 }; -static void +static int tfw_h2_on_tcp_entail_ack(void *conn, struct sk_buff *skb_head) { TfwH2Ctx *ctx = tfw_h2_context_unsafe((TfwConn *)conn); if (test_bit(HTTP2_SETTINGS_NEED_TO_APPLY, ctx->settings_to_apply)) tfw_h2_apply_new_settings(ctx); + + return 0; } static int diff --git a/fw/sock.c b/fw/sock.c index 43e860ac5..819717970 100644 --- a/fw/sock.c +++ b/fw/sock.c @@ -467,7 +467,6 @@ ss_skb_tcp_entail(struct sock *sk, struct sk_buff *skb, unsigned int mark, { struct tcp_sock *tp = tcp_sk(sk); - ss_skb_on_tcp_entail(sk->sk_user_data, skb); T_DBG3("[%d]: %s: entail sk=%pK skb=%pK data_len=%u len=%u" " truesize=%u mark=%u tls_type=%x\n", smp_processor_id(), __func__, sk, skb, skb->data_len, @@ -490,9 +489,9 @@ int ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, unsigned int mss_now, unsigned long *snd_wnd) { - struct sk_buff *tail, *next, *to_destroy; unsigned char tls_type = 0; unsigned int mark = 0; + bool skip_list = false; int r; while ((*snd_wnd = tfw_tcp_calc_snd_wnd(sk, mss_now))) { @@ -508,16 +507,23 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, * belongs. */ if (TFW_SKB_CB(skb)->is_head) { + skip_list = false; tls_type = skb_tfw_tls_type(skb); mark = skb->mark; - tail = tcp_write_queue_tail(sk); } + + if (!skip_list + && ss_skb_on_tcp_entail(sk->sk_user_data, skb)) + skip_list = true; + /* * Zero-sized SKBs may appear when the message headers (or any * other contents) are modified or deleted by Tempesta. Drop * these SKBs. + * If `on_tcp_entail` callback fails we also free all skbs from + * the list. */ - if (!skb->len) { + if (skip_list || !skb->len) { T_DBG3("[%d]: %s: drop skb=%pK data_len=%u len=%u\n", smp_processor_id(), __func__, skb, skb->data_len, skb->len); @@ -527,8 +533,9 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, r = ss_skb_realloc_headroom(skb); if (unlikely(r)) { - ss_skb_queue_head(skb_head, skb); - goto restore_sk_write_queue; + printk(KERN_ALERT "BADBAD!!!!\n"); + ss_kfree_skb(skb); + return r; } ss_skb_tcp_entail(sk, skb, mark, tls_type); @@ -538,19 +545,6 @@ ss_skb_tcp_entail_list(struct sock *sk, struct sk_buff **skb_head, ss_skb_setup_head_of_list(*skb_head, mark, tls_type); return 0; - -restore_sk_write_queue: - to_destroy = tail ? tail->next : tcp_send_head(sk); - if (to_destroy) { - tcp_for_write_queue_from_safe(to_destroy, next, sk) { - tcp_unlink_write_queue(to_destroy, sk); - tcp_wmem_free_skb(sk, to_destroy); - } - } - if (*skb_head && !TFW_SKB_CB(*skb_head)->is_head) - ss_skb_setup_head_of_list(*skb_head, mark, tls_type); - - return r; } /** @@ -676,9 +670,7 @@ ss_send(struct sock *sk, struct sk_buff **skb_head, int flags) SKB_DATA_ALIGN(sizeof(struct sk_buff)) + SKB_DATA_ALIGN(head_data + sizeof(struct skb_shared_info)); - ss_skb_set_owner(twin_skb, ss_skb_dflt_destructor, - TFW_SKB_CB(skb)->opaque_data, - copied_truesize); + ss_skb_copy_cb(twin_skb, skb, copied_truesize); ss_skb_queue_tail(&sw.skb_head, twin_skb); skb = skb->next; } while (skb != *skb_head); diff --git a/fw/ss_skb.c b/fw/ss_skb.c index b6999a180..836dcda60 100644 --- a/fw/ss_skb.c +++ b/fw/ss_skb.c @@ -220,11 +220,8 @@ __extend_pgfrags(struct sk_buff *skb_head, struct sk_buff *skb, int from, int n) if (nskb == NULL) return -ENOMEM; - if (!skb_tfw_is_in_socket_write_queue(skb)) { - ss_skb_set_owner(nskb, ss_skb_dflt_destructor, - TFW_SKB_CB(skb)->opaque_data, - nskb->truesize); - } + if (!skb_tfw_is_in_socket_write_queue(skb)) + ss_skb_copy_owner(nskb, skb, nskb->truesize); skb_shinfo(nskb)->flags = skb_shinfo(skb)->flags; ss_skb_insert_after(skb, nskb); skb_shinfo(nskb)->nr_frags = n_excess; @@ -1725,20 +1722,24 @@ int ss_skb_realloc_headroom(struct sk_buff *skb) { int delta = MAX_TCP_HEADER - skb_headroom(skb); + bool skb_has_owner = !!TFW_SKB_CB(skb)->opaque_data; unsigned int old_truesize; int r; if (likely(delta <= 0)) return 0; - if (TFW_SKB_CB(skb)->opaque_data) + if (WARN_ON(skb_has_owner && !ss_skb_has_dflt_destructor(skb))) + return -EINVAL; + + if (skb_has_owner) old_truesize = skb->truesize; r = pskb_expand_head(skb, SKB_DATA_ALIGN(delta), 0, GFP_ATOMIC); if (unlikely(r)) return r; - if (TFW_SKB_CB(skb)->opaque_data) + if (skb_has_owner) ss_skb_adjust_client_mem(skb, skb->truesize - old_truesize); return 0; @@ -1770,20 +1771,29 @@ ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head) sock_set_flag(((TfwConn *)conn)->sk, SOCK_TEMPESTA_HAS_DATA); } +static inline void +__ss_skb_adjust_client_mem(struct sk_buff *skb, TfwClientMem *cli_mem, + int delta) +{ + TFW_SKB_CB(skb)->mem += delta; + WARN_ON(TFW_SKB_CB(skb)->mem < 0); + tfw_client_adjust_mem(cli_mem, delta); +} + void ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), TfwClientMem *owner, unsigned int mem) { - TfwClientMem *cli_mem = (TfwClientMem *)owner; + if (WARN_ON(skb_tfw_is_in_socket_write_queue(skb))) + return; - if (!cli_mem || !tfw_client_mem_get(cli_mem)) + if (!owner || !tfw_client_mem_get(owner)) return; - WARN_ON(TFW_SKB_CB(skb)->opaque_data); WARN_ON(TFW_SKB_CB(skb)->mem != 0); - TFW_SKB_CB(skb)->opaque_data = cli_mem; - TFW_SKB_CB(skb)->destructor = destructor; - ss_skb_adjust_client_mem(skb, mem); + WARN_ON(TFW_SKB_CB(skb)->destructor || TFW_SKB_CB(skb)->opaque_data); + __ss_skb_set_owner(skb, destructor, owner); + __ss_skb_adjust_client_mem(skb, owner, mem); } void @@ -1795,9 +1805,61 @@ ss_skb_adjust_client_mem(struct sk_buff *skb, int delta) return; cli_mem = (TfwClientMem *)TFW_SKB_CB(skb)->opaque_data; - if (cli_mem) { - TFW_SKB_CB(skb)->mem += delta; - WARN_ON(TFW_SKB_CB(skb)->mem < 0); - tfw_client_adjust_mem(cli_mem, delta); + if (cli_mem) + __ss_skb_adjust_client_mem(skb, cli_mem, delta); +} + +void +ss_skb_copy_owner(struct sk_buff *to, struct sk_buff *from, + unsigned int delta) +{ + void (*destructor)(struct sk_buff *) = TFW_SKB_CB(from)->destructor; + TfwClientMem *cli_mem; + void *owner; + + if (!destructor) + return; + + owner = TFW_SKB_CB(from)->opaque_data; + /* + * If skb has not default destructor, it means that + * owner is not a TfwClientMem structure. In this + * case we should get TfwClientMem structure from + * the real owner and set to source skb default + * destructor with default owner, because only + * TfwClientMem structure allow to be owned by + * several skbs. + */ + if (destructor == tfw_h2_stream_skb_destructor) { + TfwHttpResp *resp = (TfwHttpResp *)owner; + + cli_mem = CLIENT_MEM_FROM_CONN(resp->req->conn); + __ss_skb_set_owner(from, ss_skb_dflt_destructor, cli_mem); + } else if (destructor == tfw_http_req_skb_destructor) { + TfwHttpReq *req = (TfwHttpReq *)owner; + + cli_mem = CLIENT_MEM_FROM_CONN(req->conn); + __ss_skb_set_owner(from, ss_skb_dflt_destructor, cli_mem); + } else { + cli_mem = owner; } + + if (!tfw_client_mem_get(cli_mem)) + return; + __ss_skb_set_owner(to, destructor, owner); + __ss_skb_adjust_client_mem(to, cli_mem, delta); +} + +void +ss_skb_copy_cb(struct sk_buff *to, struct sk_buff *from, + unsigned int delta) +{ + /* + * Do not copy `mem` (it should be set during copy owner). + */ + ss_skb_copy_owner(to, from, delta); + TFW_SKB_CB(to)->on_send = TFW_SKB_CB(from)->on_send; + TFW_SKB_CB(to)->on_tcp_entail = TFW_SKB_CB(from)->on_tcp_entail; + TFW_SKB_CB(to)->stream_id = TFW_SKB_CB(from)->stream_id; + TFW_SKB_CB(to)->is_head = TFW_SKB_CB(from)->is_head; } diff --git a/fw/ss_skb.h b/fw/ss_skb.h index a98e80a68..e80c2fd24 100644 --- a/fw/ss_skb.h +++ b/fw/ss_skb.h @@ -28,7 +28,7 @@ #include "str.h" typedef int (*on_send_cb_t)(void *conn, struct sk_buff **skb_head); -typedef void (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head); +typedef int (*on_tcp_entail_t)(void *conn, struct sk_buff *skb_head); typedef void (*on_send_fail_cb_t)(void *conn, struct sk_buff *skb_head); typedef struct tfw_client_mem_t TfwClientMem; @@ -64,6 +64,24 @@ void ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), void ss_skb_adjust_client_mem(struct sk_buff *skb, int delta); void ss_skb_dflt_destructor(struct sk_buff *skb); void ss_skb_on_send_dflt(void *conn, struct sk_buff **skb_head); +void ss_skb_copy_owner(struct sk_buff *to, struct sk_buff *from, + unsigned int delta); +void ss_skb_copy_cb(struct sk_buff *to, struct sk_buff *from, + unsigned int delta); + +static inline bool +ss_skb_has_dflt_destructor(struct sk_buff *skb) +{ + return TFW_SKB_CB(skb)->destructor == ss_skb_dflt_destructor; +} + +static inline void +__ss_skb_set_owner(struct sk_buff *skb, void (*destructor)(struct sk_buff *), + void *owner) +{ + TFW_SKB_CB(skb)->destructor = destructor; + TFW_SKB_CB(skb)->opaque_data = owner; +} static inline bool ss_skb_is_within_fragment(char *begin_fragment, char *position, @@ -99,13 +117,14 @@ ss_skb_on_send(void *conn, struct sk_buff **skb_head) return r; } -static inline void +static inline int ss_skb_on_tcp_entail(void *conn, struct sk_buff *skb_head) { on_tcp_entail_t on_tcp_entail = TFW_SKB_CB(skb_head)->on_tcp_entail; if (on_tcp_entail) - on_tcp_entail(conn, skb_head); + return on_tcp_entail(conn, skb_head); + return 0; } typedef int ss_skb_actor_t(void *conn, unsigned char *data, unsigned int len,