From dc83ccc4addcc1199648dca872699463af9a061b Mon Sep 17 00:00:00 2001 From: Nils Goroll Date: Wed, 27 Dec 2023 12:39:47 +0100 Subject: [PATCH] Generalize the VDP API This commit is to prepare use of the VDP API also for the backend side to filter bereq.body through bereq.filters by putting all pointers _intended_ to be used by a VDP init function into vdp_ctx. For background, see #4035 --- bin/varnishd/cache/cache_deliver_proc.c | 38 ++++++++++++++---- bin/varnishd/cache/cache_esi_deliver.c | 49 +++++++++++++----------- bin/varnishd/cache/cache_filter.h | 12 ++++-- bin/varnishd/cache/cache_gzip.c | 24 ++++++------ bin/varnishd/cache/cache_range.c | 31 +++++++++------ bin/varnishd/cache/cache_req_fsm.c | 4 +- bin/varnishd/cache/cache_varnishd.h | 5 ++- bin/varnishd/cache/cache_vrt_filter.c | 21 ++++++++-- bin/varnishd/http2/cache_http2_deliver.c | 5 +-- vmod/vmod_debug.c | 39 ++++++++++--------- 10 files changed, 140 insertions(+), 88 deletions(-) diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 04ff3bbec57..db658d17b9f 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -71,18 +71,35 @@ VDP_Fini(const struct vdp_ctx *vdc) void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, - struct req *req) + struct req *req, struct busyobj *bo, intmax_t *clen) { AN(vdc); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(vsl); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + CHECK_OBJ_ORNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + AN(clen); + + assert((req ? 1 : 0) ^ (bo ? 1 : 0)); + + AN(clen); + assert(*clen >= -1); INIT_OBJ(vdc, VDP_CTX_MAGIC); VTAILQ_INIT(&vdc->vdp); vdc->wrk = wrk; vdc->vsl = vsl; - vdc->req = req; + vdc->clen = clen; + + if (req != NULL) { + vdc->oc = req->objcore; + vdc->hp = req->resp; + } + else { + vdc->oc = bo->bereq_body; + vdc->hp = bo->bereq; + } } /* VDP_bytes @@ -152,6 +169,10 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + assert(*vdc->clen >= -1); AN(ws); AN(vdp); AN(vdp->name); @@ -175,10 +196,9 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp, vdc->nxt = VTAILQ_FIRST(&vdc->vdp); AZ(vdc->retval); - if (vdpe->vdp->init != NULL) { - vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv, - vdpe == vdc->nxt ? vdc->req->objcore : NULL); - } + if (vdpe->vdp->init != NULL) + vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv); + vdc->oc = NULL; if (vdc->retval > 0) { VTAILQ_REMOVE(&vdc->vdp, vdpe, list); @@ -254,7 +274,9 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC); AN(vdc->vsl); - vdc->req = NULL; + AZ(vdc->oc); + vdc->hp = NULL; + vdc->clen = NULL; final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterate, final); if (r < 0) diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 23cdfe2421e..ba6b0412ddf 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -261,35 +261,41 @@ ved_decode_len(struct vsl_log *vsl, const uint8_t **pp) */ static int v_matchproto_(vdp_init_f) -ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct ecx *ecx; - struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_ORNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); - if (oc == NULL || !ObjHasAttr(vdc->wrk, oc, OA_ESIDATA)) - return (1); - - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); AN(priv); + AZ(*priv); + if (vdc->oc == NULL || !ObjHasAttr(vdc->wrk, vdc->oc, OA_ESIDATA)) + return (1); + + if (ctx->req == NULL) { + VSLb(vdc->vsl, SLT_Error, + "esi can only be used on the client side"); + return (1); + } ALLOC_OBJ(ecx, ECX_MAGIC); AN(ecx); assert(sizeof gzip_hdr == 10); - ecx->preq = req; + ecx->preq = ctx->req; *priv = ecx; - RFC2616_Weaken_Etag(req->resp); - - req->res_mode |= RES_ESI; - if (req->resp_len != 0) - req->resp_len = -1; - if (req->esi_level > 0) { - assert(req->transport == &VED_transport); - CAST_OBJ_NOTNULL(ecx->pecx, req->transport_priv, ECX_MAGIC); + RFC2616_Weaken_Etag(vdc->hp); + + ctx->req->res_mode |= RES_ESI; + if (*vdc->clen != 0) + *vdc->clen = -1; + if (ctx->req->esi_level > 0) { + assert(ctx->req->transport == &VED_transport); + CAST_OBJ_NOTNULL(ecx->pecx, ctx->req->transport_priv, ECX_MAGIC); if (!ecx->pecx->isgzip) ecx->pecx = NULL; } @@ -604,18 +610,16 @@ struct ved_foo { }; static int v_matchproto_(vdp_init_f) -ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { ssize_t l; const char *p; struct ved_foo *foo; - struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - (void)oc; - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(priv); + CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); CHECK_OBJ_NOTNULL(foo->objcore, OBJCORE_MAGIC); @@ -925,7 +929,6 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) foo->ecx = ecx; foo->objcore = req->objcore; i = VDP_Push(ctx, req->vdc, req->ws, &ved_gzgz, foo); - } else if (ecx->isgzip && !i) { /* Non-Gzip'ed include in gzip'ed parent */ i = VDP_Push(ctx, req->vdc, req->ws, &ved_pretend_gz, ecx); diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index d06540092ff..329ab9982c5 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -90,7 +90,7 @@ struct vfp_ctx { struct http *req; struct http *resp; struct worker *wrk; - struct objcore *oc; + struct objcore *oc; // Only first filter, if at all struct vfp_entry_s vfp; struct vfp_entry *vfp_nxt; @@ -112,8 +112,8 @@ enum vdp_action { VDP_END, /* Last buffer or after, implies VDP_FLUSH */ }; -typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv, - struct objcore *); + +typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv); /* * Return value: * negative: Error - abandon delivery @@ -155,7 +155,11 @@ struct vdp_ctx { struct vdp_entry *nxt; struct worker *wrk; struct vsl_log *vsl; - struct req *req; + // NULL'ed after the first filter has been pushed + struct objcore *oc; + // NULL'ed for delivery + struct http *hp; + intmax_t *clen; }; int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *, ssize_t); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 472ac2e5b9f..a7402ca569e 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -288,11 +288,10 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags) */ static int v_matchproto_(vdp_init_f) -vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct vgz *vg; struct boc *boc; - struct req *req; enum boc_state_e bos; const char *p; ssize_t dl; @@ -300,9 +299,10 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); vg = VGZ_NewGunzip(vdc->vsl, "U D -"); AN(vg); @@ -314,27 +314,27 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) VGZ_Obuf(vg, vg->m_buf, vg->m_sz); *priv = vg; - http_Unset(req->resp, H_Content_Encoding); + http_Unset(vdc->hp, H_Content_Encoding); - req->resp_len = -1; + *vdc->clen = -1; - if (oc == NULL) + if (vdc->oc == NULL) return (0); - boc = HSH_RefBoc(oc); + boc = HSH_RefBoc(vdc->oc); if (boc != NULL) { CHECK_OBJ(boc, BOC_MAGIC); bos = boc->state; - HSH_DerefBoc(vdc->wrk, oc); + HSH_DerefBoc(vdc->wrk, vdc->oc); if (bos < BOS_FINISHED) return (0); /* OA_GZIPBITS is not stable yet */ } - p = ObjGetAttr(vdc->wrk, oc, OA_GZIPBITS, &dl); + p = ObjGetAttr(vdc->wrk, vdc->oc, OA_GZIPBITS, &dl); if (p != NULL && dl == 32) { u = vbe64dec(p + 24); if (u != 0) - req->resp_len = u; + *vdc->clen = u; } return (0); } diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index d7398f33f1e..a9a86abc2f3 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -196,33 +196,40 @@ vrg_ifrange(struct req *req) } static int v_matchproto_(vdp_init_f) -vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { const char *err; - struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_ORNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - (void)oc; - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (!vrg_ifrange(req)) // rfc7233,l,455,456 + AN(priv); + + if (ctx->req == NULL) { + VSLb(vdc->vsl, SLT_Error, + "range can only be used on the client side"); + return (1); + } + + // not using vdc->{hd,cl}, because range needs req anyway for Req_Fail() + + if (!vrg_ifrange(ctx->req)) // rfc7233,l,455,456 return (1); - err = vrg_dorange(req, priv); + err = vrg_dorange(ctx->req, priv); if (err == NULL) return (*priv == NULL ? 1 : 0); VSLb(vdc->vsl, SLT_Debug, "RANGE_FAIL %s", err); - if (req->resp_len >= 0) - http_PrintfHeader(req->resp, + if (ctx->req->resp_len >= 0) + http_PrintfHeader(ctx->req->resp, "Content-Range: bytes */%jd", - (intmax_t)req->resp_len); - http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); + (intmax_t)ctx->req->resp_len); + http_PutResponse(ctx->req->resp, "HTTP/1.1", 416, NULL); /* * XXX: We ought to produce a body explaining things. * XXX: That really calls for us to hit vcl_synth{} */ - req->resp_len = 0; + ctx->req->resp_len = 0; return (1); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index ed97a01107a..80111633f2d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -460,11 +460,11 @@ cnt_transmit(struct worker *wrk, struct req *req) sendbody = 1; } - VDP_Init(req->vdc, req->wrk, req->vsl, req); + VDP_Init(req->vdc, req->wrk, req->vsl, req, NULL, &req->resp_len); if (req->vdp_filter_list == NULL) req->vdp_filter_list = resp_Get_Filter_List(req); if (req->vdp_filter_list == NULL || - VCL_StackVDP(req, req->vcl, req->vdp_filter_list)) { + VCL_StackVDP(req->vdc, req->vcl, req->vdp_filter_list, req, NULL)) { VSLb(req->vsl, SLT_Error, "Failure to push processors"); req->doclose = SC_OVERLOAD; req->acct.resp_bodybytes += diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 016516bcd9d..8049614938c 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -193,7 +193,7 @@ void VDI_Init(void); /* cache_deliver_proc.c */ void VDP_Fini(const struct vdp_ctx *vdc); void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, - struct req *req); + struct req *req, struct busyobj *bo, intmax_t *cl); uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *); void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); int VDP_Push(VRT_CTX, struct vdp_ctx *, struct ws *, const struct vdp *, @@ -511,7 +511,8 @@ void pan_privs(struct vsb *, const struct vrt_privs *); /* cache_vrt_filter.c */ int VCL_StackVFP(struct vfp_ctx *, const struct vcl *, const char *); -int VCL_StackVDP(struct req *, const struct vcl *, const char *); +int VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl, + struct req *req, struct busyobj *bo); const char *resp_Get_Filter_List(struct req *req); void VCL_VRT_Init(void); diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index e0ca1832162..702026a4737 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -254,15 +254,28 @@ VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl) } int -VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl) +VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl, + struct req *req, struct busyobj *bo) { const struct vfilter *vp; struct vrt_ctx ctx[1]; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + AN(vcl); AN(fl); - VSLbs(req->vsl, SLT_Filters, TOSTRAND(fl)); + + CHECK_OBJ_ORNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + + assert((req ? 1 : 0) ^ (bo ? 1 : 0)); + + VSLbs(vdc->vsl, SLT_Filters, TOSTRAND(fl)); INIT_OBJ(ctx, VRT_CTX_MAGIC); - VCL_Req2Ctx(ctx, req); + + if (req) + VCL_Req2Ctx(ctx, req); + else + VCL_Bo2Ctx(ctx, bo); while (1) { vp = vcl_filter_list_iter(0, &vrt_filters, &vcl->filters, &fl); @@ -273,7 +286,7 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl) "Filter '...%s' not found", fl); return (-1); } - if (VDP_Push(ctx, req->vdc, req->ws, vp->vdp, NULL)) + if (VDP_Push(ctx, vdc, ctx->ws, vp->vdp, NULL)) return (-1); } } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index cdd356d7dc6..5de2a7fc4ce 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -73,7 +73,7 @@ V2D_Init(void) /**********************************************************************/ static int v_matchproto_(vdp_init_f) -h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct h2_req *r2; @@ -82,7 +82,6 @@ h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) AN(priv); CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); (void)r2; - (void)oc; return (0); } @@ -348,7 +347,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) if (sendbody) { INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); - if (!VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2)) + if (! VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2)) (void)VDP_DeliverObj(req->vdc, req->objcore); } diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index cea472a09e9..30d936cbc03 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -105,16 +105,21 @@ static const struct vfp xyzzy_vfp_rot13 = { #define ROT13_BUFSZ 8 static int v_matchproto_(vdp_init_f) -xyzzy_vdp_rot13_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +xyzzy_vdp_rot13_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { - (void)vdc; - (void)oc; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); + *priv = malloc(ROT13_BUFSZ); if (*priv == NULL) return (-1); + return (0); } @@ -182,20 +187,16 @@ static const struct vdp xyzzy_vdp_rot13 = { */ static int v_matchproto_(vdp_init_f) -xyzzy_vdp_chunked_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +xyzzy_vdp_chunked_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { - struct http *hp; - - (void)vdc; - (void)oc; - (void)priv; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); - hp = vdc->req->resp; - CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - http_Unset(hp, H_Content_Length); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(priv); + + http_Unset(vdc->hp, H_Content_Length); return (1); } @@ -247,15 +248,18 @@ static const struct vmod_priv_methods priv_pedantic_methods[1] = {{ }}; static int v_matchproto_(vdp_init_f) -xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, - struct objcore *oc) +xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct vdp_state_s *vdps; struct vmod_priv *p; - (void)oc; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); + WS_TASK_ALLOC_OBJ(ctx, vdps, VDP_STATE_MAGIC); if (vdps == NULL) return (-1); @@ -267,7 +271,6 @@ xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, p->priv = vdps; p->methods = priv_pedantic_methods; - AN(priv); *priv = vdps; vdps->state = VDPS_INIT;