Generalize the VDP API

This commit is to prepare for use of the VDP API also for the backend
side to filter bereq.body through bereq.filters:

The req member of struct vdp_ctx is removed, because it should only be
used for initialization and, consequently, is already nulled in
VDP_DeliverObj().

vdp_init_f() and its caller VDP_Push() gain arguments:

- req is only present if the VDP is used on the client side.

- hd is a pointer to the outgoing headers corresponding to the body
  being worked on: (struct req).resp on the client side and (struct
  busyobj).bereq on the backend side (tbd).

- cl is a pointer to the content-length. -1 indicates 'unknown'.

VDPs should aim for compatibility with both the client and backend use
case by using only hd and cl instead of req. If they depend on req,
they should return an error if req is NULL.

Note that none of the new arguments would be necessary, they could all
(including the existing oc argument) be derived from the VRT_CTX. The
reasons for having these arguments are code clarity and avoidance of
code duplication, e.g. for repeatedly figuring out the correct header
and length pointer on the client vs. backend side.

bin/varnishd/cache/cache_deliver_proc.c

@@ -62,19 +62,16 @@ VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc)
 
 
 void
-VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl,
-    struct req *req)
+VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl)
 {
 	AN(vdc);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(vsl);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	INIT_OBJ(vdc, VDP_CTX_MAGIC);
 	VTAILQ_INIT(&vdc->vdp);
 	vdc->wrk = wrk;
 	vdc->vsl = vsl;
-	vdc->req = req;
 }
 
 /* VDP_bytes
@@ -136,9 +133,18 @@ VDP_bytes(struct vdp_ctx *vdc, enum vdp_action act,
 	return (vdc->retval);
 }
 
+/*
+ * oc: optional, only passed to the first VDP
+ * req: optional, for client-side only VDPs
+ * hd: headers to modify (request or response)
+ * cl: known content-length or -1, settable
+ */
+
 int
-VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp,
-    void *priv)
+VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws,
+     const struct vdp *vdp, void *priv,
+     struct objcore *oc, struct req *req,
+     struct http *hd, intmax_t *cl)
 {
 	struct vdp_entry *vdpe;
 
@@ -148,6 +154,11 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp,
 	AN(vdp);
 	AN(vdp->name);
 	AN(vdp->bytes);
+	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(hd, HTTP_MAGIC);
+	AN(cl);
+	assert(*cl >= -1);
 
 	if (vdc->retval)
 		return (vdc->retval);
@@ -172,7 +183,7 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp,
 		return (vdc->retval);
 
 	vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv,
-	    vdpe == vdc->nxt ? vdc->req->objcore : NULL);
+	    vdpe == vdc->nxt ? oc : NULL, req, hd, cl);
 
 	if (vdc->retval > 0) {
 		VTAILQ_REMOVE(&vdc->vdp, vdpe, list);
@@ -246,7 +257,6 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC);
 	AN(vdc->vsl);
-	vdc->req = NULL;
 	final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0;
 	r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterate, final);
 	if (r < 0)

bin/varnishd/cache/cache_esi_deliver.c

@@ -261,32 +261,40 @@ ved_decode_len(struct vsl_log *vsl, const uint8_t **pp)
  */
 
 static int v_matchproto_(vdp_init_f)
-ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv,
+    struct objcore *oc, struct req *req,
+    struct http *hd, intmax_t *cl)
 {
 	struct ecx *ecx;
-	struct req *req;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
+	AN(priv);
 	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(hd, HTTP_MAGIC);
+	AN(cl);
+
+	AZ(*priv);
 	if (oc == NULL || !ObjHasAttr(vdc->wrk, oc, OA_ESIDATA))
 		return (1);
 
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AN(priv);
-	AZ(*priv);
+	if (req == NULL) {
+		VSLb(vdc->vsl, SLT_Error,
+		     "esi can only be used on the client side");
+		return (1);
+	}
 
 	ALLOC_OBJ(ecx, ECX_MAGIC);
 	AN(ecx);
 	assert(sizeof gzip_hdr == 10);
 	ecx->preq = req;
 	*priv = ecx;
-	RFC2616_Weaken_Etag(req->resp);
+	RFC2616_Weaken_Etag(hd);
 
 	req->res_mode |= RES_ESI;
-	if (req->resp_len != 0)
-		req->resp_len = -1;
+	if (*cl != 0)
+		*cl = -1;
 	if (req->esi_level > 0) {
 		assert(req->transport == &VED_transport);
 		CAST_OBJ_NOTNULL(ecx->pecx, req->transport_priv, ECX_MAGIC);
@@ -604,18 +612,22 @@ struct ved_foo {
 };
 
 static int v_matchproto_(vdp_init_f)
-ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv,
+    struct objcore *oc, struct req *req,
+    struct http *hd, intmax_t *cl)
 {
 	ssize_t l;
 	const char *p;
 	struct ved_foo *foo;
-	struct req *req;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
-	(void)oc;
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AN(priv);
+	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(hd, HTTP_MAGIC);
+	AN(cl);
+
 	CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC);
 	CHECK_OBJ_NOTNULL(foo->objcore, OBJCORE_MAGIC);
 
@@ -906,14 +918,16 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody)
 		INIT_OBJ(foo, VED_FOO_MAGIC);
 		foo->ecx = ecx;
 		foo->objcore = req->objcore;
-		i = VDP_Push(ctx, req->vdc, req->ws, &ved_gzgz, foo);
-
+		i = VDP_Push(ctx, req->vdc, req->ws, &ved_gzgz, foo,
+		    NULL, req, req->resp, &req->resp_len);
 	} else if (ecx->isgzip && !i) {
 		/* Non-Gzip'ed include in gzip'ed parent */
-		i = VDP_Push(ctx, req->vdc, req->ws, &ved_pretend_gz, ecx);
+		i = VDP_Push(ctx, req->vdc, req->ws, &ved_pretend_gz, ecx,
+		    NULL, req, req->resp, &req->resp_len);
 	} else {
 		/* Anything else goes straight through */
-		i = VDP_Push(ctx, req->vdc, req->ws, &ved_ved, ecx);
+		i = VDP_Push(ctx, req->vdc, req->ws, &ved_ved, ecx,
+		    NULL, req, req->resp, &req->resp_len);
 	}
 
 	if (i == 0) {

bin/varnishd/cache/cache_filter.h

@@ -104,8 +104,16 @@ enum vdp_action {
 	VDP_END,		/* Last buffer or after, implies VDP_FLUSH */
 };
 
+
+/*
+ * oc: optional, only passed to the first VDP
+ * req: optional, for client-side only VDPs
+ * hd: headers to modify (request or response)
+ * cl: known content-length or -1, settable
+ */
 typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv,
-    struct objcore *);
+    struct objcore *oc, struct req *req,
+    struct http *hd, intmax_t *cl);
 /*
  * Return value:
  *	negative:	Error - abandon delivery
@@ -147,7 +155,6 @@ struct vdp_ctx {
 	struct vdp_entry	*nxt;
 	struct worker		*wrk;
 	struct vsl_log		*vsl;
-	struct req		*req;
 };
 
 int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *, ssize_t);

bin/varnishd/cache/cache_gzip.c

@@ -288,21 +288,24 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
  */
 
 static int v_matchproto_(vdp_init_f)
-vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv,
+    struct objcore *oc, struct req *req,
+    struct http *hd, intmax_t *cl)
 {
 	struct vgz *vg;
 	struct boc *boc;
-	struct req *req;
 	enum boc_state_e bos;
 	const char *p;
 	ssize_t dl;
 	uint64_t u;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
+	AN(priv);
 	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(hd, HTTP_MAGIC);
+	AN(cl);
 
 	vg = VGZ_NewGunzip(vdc->vsl, "U D -");
 	AN(vg);
@@ -314,9 +317,9 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 	VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 	*priv = vg;
 
-	http_Unset(req->resp, H_Content_Encoding);
+	http_Unset(hd, H_Content_Encoding);
 
-	req->resp_len = -1;
+	*cl = -1;
 
 	if (oc == NULL)
 		return (0);
@@ -334,7 +337,7 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 	if (p != NULL && dl == 32) {
 		u = vbe64dec(p + 24);
 		if (u != 0)
-			req->resp_len = u;
+			*cl = u;
 	}
 	return (0);
 }

bin/varnishd/cache/cache_range.c

@@ -196,16 +196,28 @@ vrg_ifrange(struct req *req)
 }
 
 static int v_matchproto_(vdp_init_f)
-vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv,
+    struct objcore *oc, struct req *req,
+    struct http *hd, intmax_t *cl)
 {
 	const char *err;
-	struct req *req;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
-	(void)oc;
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AN(priv);
+	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
+
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	if (req == NULL) {
+		VSLb(vdc->vsl, SLT_Error,
+		     "range can only be used on the client side");
+		return (1);
+	}
+
+	// not using hd/cl, because range needs req anyway for Req_Fail()
+	CHECK_OBJ_NOTNULL(hd, HTTP_MAGIC);
+	AN(cl);
+
 	if (!vrg_ifrange(req))		// rfc7233,l,455,456
 		return (1);
 	err = vrg_dorange(req, priv);

bin/varnishd/cache/cache_req_fsm.c

@@ -460,7 +460,7 @@ cnt_transmit(struct worker *wrk, struct req *req)
 		sendbody = 1;
 	}
 
-	VDP_Init(req->vdc, req->wrk, req->vsl, req);
+	VDP_Init(req->vdc, req->wrk, req->vsl);
 	if (req->vdp_filter_list == NULL)
 		req->vdp_filter_list = resp_Get_Filter_List(req);
 	if (req->vdp_filter_list == NULL ||

bin/varnishd/cache/cache_varnishd.h

@@ -189,12 +189,15 @@ void VDI_Event(const struct director *d, enum vcl_event_e ev);
 void VDI_Init(void);
 
 /* cache_deliver_proc.c */
-void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl,
-    struct req *req);
+void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl);
 uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *);
 void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc);
-int VDP_Push(VRT_CTX, struct vdp_ctx *, struct ws *, const struct vdp *,
-    void *priv);
+
+int
+VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws,
+     const struct vdp *vdp, void *priv,
+     struct objcore *oc, struct req *req,
+     struct http *hd, intmax_t *cl);
 int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc);
 extern const struct vdp VDP_gunzip;
 extern const struct vdp VDP_esi;

bin/varnishd/cache/cache_vrt_filter.c

@@ -258,12 +258,15 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl)
 {
 	const struct vfilter *vp;
 	struct vrt_ctx ctx[1];
+	struct objcore *oc;
 
 	AN(fl);
 	VSLbs(req->vsl, SLT_Filters, TOSTRAND(fl));
 	INIT_OBJ(ctx, VRT_CTX_MAGIC);
 	VCL_Req2Ctx(ctx, req);
 
+	oc = req->objcore;
+
 	while (1) {
 		vp = vcl_filter_list_iter(0, &vrt_filters, &vcl->filters, &fl);
 		if (vp == NULL)
@@ -273,7 +276,8 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl)
 			    "Filter '...%s' not found", fl);
 			return (-1);
 		}
-		if (VDP_Push(ctx, req->vdc, req->ws, vp->vdp, NULL))
+		if (VDP_Push(ctx, req->vdc, req->ws, vp->vdp, NULL,
+		    oc, req, req->resp, &req->resp_len))
 			return (-1);
 	}
 }

bin/varnishd/http1/cache_http1_deliver.c

@@ -97,7 +97,8 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 		}
 		INIT_OBJ(ctx, VRT_CTX_MAGIC);
 		VCL_Req2Ctx(ctx, req);
-		if (VDP_Push(ctx, req->vdc, req->ws, VDP_v1l, NULL)) {
+		if (VDP_Push(ctx, req->vdc, req->ws, VDP_v1l, NULL,
+		    NULL, req, req->resp, &req->resp_len)) {
 			v1d_error(req, "Failure to push v1d processor");
 			return;
 		}

bin/varnishd/http2/cache_http2_deliver.c

@@ -73,7 +73,9 @@ V2D_Init(void)
 /**********************************************************************/
 
 static int v_matchproto_(vdp_init_f)
-h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv,
+    struct objcore *oc, struct req *req,
+    struct http *hd, intmax_t *cl)
 {
 	struct h2_req *r2;
 
@@ -82,7 +84,10 @@ h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 	AN(priv);
 	CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC);
 	(void)r2;
-	(void)oc;
+	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);	// only client side
+	AN(hd);
+	AN(cl);
 	return (0);
 }
 
@@ -348,7 +353,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	if (sendbody) {
 		INIT_OBJ(ctx, VRT_CTX_MAGIC);
 		VCL_Req2Ctx(ctx, req);
-		if (!VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2))
+		if (! VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2,
+		    NULL, req, req->resp, &req->resp_len))
 			(void)VDP_DeliverObj(req->vdc, req->objcore);
 	}