File: inbound_grpc_to_outbound_http.go

package info (click to toggle)
gitlab-agent 16.1.3-2
  • links: PTS, VCS
  • area: contrib
  • in suites: forky, sid, trixie
  • size: 6,324 kB
  • sloc: makefile: 175; sh: 52; ruby: 3
file content (321 lines) | stat: -rw-r--r-- 11,090 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
package grpctool

import (
	"bufio"
	"context"
	"errors"
	"io"
	"net"
	"net/http"

	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/logz"
	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/memz"
	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/prototool"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

type InboundGrpcToOutboundHttpStream interface {
	Send(*HttpResponse) error
	grpc.ServerStream
}

type HandleProcessingErrorFunc func(msg string, err error)
type HandleIoErrorFunc func(msg string, err error) error

type DoResponse struct {
	// Resp is the server's response to a request.
	Resp *http.Response
	// UpgradeConn is the underlying network connection to the server.
	// May be nil if request was not an Upgrade request or if server decided not to switch protocols
	// (non-101 response status code).
	UpgradeConn net.Conn
	// ConnReader is a buffered reader, wrapping UpgradeConn. Is set when UpgradeConn is set.
	// Must be used for reading as it may contain buffered bytes that are no longer available directly via UpgradeConn.
	ConnReader *bufio.Reader
}

// HttpDo makes an HTTP request and returns a response. If an HTTP upgrade was requested, the underlying network
// connection is also returned. Implementations that don't support Upgrade should return an error.
type HttpDo func(ctx context.Context, header *HttpRequest_Header, body io.Reader) (DoResponse, error)

type InboundGrpcToOutboundHttp struct {
	Log                   *zap.Logger
	HandleProcessingError HandleProcessingErrorFunc
	HandleIoError         HandleIoErrorFunc
	HttpDo                HttpDo
}

func (x *InboundGrpcToOutboundHttp) Pipe(inbound InboundGrpcToOutboundHttpStream) (retErr error) {
	var upgradeConn net.Conn
	defer func() {
		if upgradeConn != nil {
			err := upgradeConn.Close()
			if retErr == nil {
				retErr = x.maybeHandleIoError("error closing connection", err)
			}
		}
	}()

	ctx := inbound.Context()

	pr, pw := io.Pipe()
	headerC := make(chan *HttpRequest_Header)
	// buffered to not block the sender as receiver might encounter an error and exit before even trying to receive.
	respC := make(chan DoResponse, 1)
	s := InboundStreamToOutboundStream{
		// Pipe gRPC request -> HTTP request
		PipeInboundToOutbound: func() error {
			// unblock the PipeOutboundToInbound goroutine if we exited before sending the header due to an error.
			defer close(headerC)
			return x.pipeInboundToOutbound(inbound, headerC, respC, pw)
		},
		// Pipe HTTP response -> gRPC response
		PipeOutboundToInbound: func() error {
			// Make sure the writer is unblocked if we exit abruptly
			// The error is ignored because it will always occur if things go normally - the pipe will have been
			// closed already when this code is reached (and that's an error).
			defer pr.Close() // nolint: errcheck
			// unblock the PipeInboundToOutbound goroutine if we exited before sending the response object due to an error.
			defer close(respC)
			select {
			case <-ctx.Done():
				return ctx.Err()
			case header, ok := <-headerC:
				if !ok {
					// Something went wrong in the PipeInboundToOutbound goroutine, exit.
					return nil
				}

				var body io.Reader
				if header.IsRequestWithoutBody() {
					// NOTE: The golang standard library will add a `Transfer-Encoding: chunked` to the request
					// for bodies with unknown size - which upgrade requests are,
					// see https://github.com/golang/go/blob/39ca989b883b913287d282365510a9152a3f80e6/src/net/http/transfer.go#L95
					// This leads to a zero-sized chunked HTTP body (`0 CR LF CR LF`) during upgrade requests which may
					// not be consumed by certain HTTP servers before hijacking the connection and switching
					// to "raw" TCP mode, namely the spdy upgrade logic in the Kubernetes apimachinery pkg (used in CRIs), see
					// https://github.com/kubernetes/kubernetes/blob/f51dad586ddc1a02b4fcc4e3974092ad78b630a7/staging/src/k8s.io/apimachinery/pkg/util/httpstream/spdy/upgrade.go#LL86C9-L86C9
					// However, we suspect that there is another bug on the Kubernetes stack to sometimes consumes
					// these additionally bytes in the body and forwards a correct request to destination (e.g. CRI).
					// See https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/issues/393
					body = http.NoBody
				} else {
					body = pr
				}
				r, err := x.HttpDo(ctx, header, body)
				if err != nil {
					return err
				}
				respC <- r
				// this store is not synchronized and that's ok because PipeOutboundToInbound is executed
				// on the caller's goroutine.
				upgradeConn = r.UpgradeConn
				return x.pipeOutboundToInbound(inbound, r, header.Request.IsUpgrade())
			}
		},
	}
	err := s.Pipe()
	switch {
	case err == nil:
	case IsStatusError(err):
		// A gRPC status already
	case errors.Is(err, context.Canceled):
		x.Log.Debug("gRPC -> HTTP", logz.Error(err))
		err = status.Errorf(codes.Canceled, "gRPC -> HTTP: %v", err)
	case errors.Is(err, context.DeadlineExceeded):
		x.Log.Debug("gRPC -> HTTP", logz.Error(err))
		err = status.Errorf(codes.DeadlineExceeded, "gRPC -> HTTP: %v", err)
	default:
		x.HandleProcessingError("gRPC -> HTTP", err)
		err = status.Errorf(codes.Unavailable, "gRPC -> HTTP: %v", err)
	}
	return err
}

func (x *InboundGrpcToOutboundHttp) pipeInboundToOutbound(inbound InboundGrpcToOutboundHttpStream,
	headerC chan<- *HttpRequest_Header, respC <-chan DoResponse, pw *io.PipeWriter) error {
	var isUpgrade bool
	var notExpectingBody bool
	var upgradeConn net.Conn
	return HttpRequestStreamVisitor.Get().Visit(inbound,
		WithCallback(HttpRequestHeaderFieldNumber, func(header *HttpRequest_Header) error {
			x.logRequest(header)
			isUpgrade = header.Request.IsUpgrade()
			notExpectingBody = header.IsRequestWithoutBody()
			ctx := inbound.Context()
			select {
			case <-ctx.Done():
				return ctx.Err()
			case headerC <- header:
				return nil
			}
		}),
		WithCallback(HttpRequestDataFieldNumber, func(data *HttpRequest_Data) error {
			if notExpectingBody {
				return status.Errorf(codes.Internal, "unexpected HttpRequest_Data message received")
			}
			_, err := pw.Write(data.Data)
			return x.maybeHandleIoError("request body write", err)
		}),
		WithCallback(HttpRequestTrailerFieldNumber, func(trailer *HttpRequest_Trailer) error {
			if isUpgrade {
				// Nothing more to send, close the write end of the pipe
				err := pw.Close()
				return x.maybeHandleIoError("request body close", err)
			}
			// Nothing to do
			return nil
		}),
		WithCallback(HttpRequestUpgradeDataFieldNumber, func(data *HttpRequest_UpgradeData) error {
			if !isUpgrade {
				// Inbound client didn't request a connection upgrade but sent an upgrade data frame.
				return status.Error(codes.Internal, "unexpected HttpRequest_UpgradeData message for non-upgrade request")
			}
			if upgradeConn == nil {
				r, ok := <-respC
				if !ok {
					// error in the other goroutine, abort.
					return context.Canceled
				}
				if r.Resp.StatusCode != http.StatusSwitchingProtocols {
					// Outbound server doesn't want to switch protocols but inbound client sent an upgrade data frame.
					return status.Errorf(codes.Internal, "unexpected HttpRequest_UpgradeData message for HTTP status code %d", r.Resp.StatusCode)
				}
				upgradeConn = r.UpgradeConn
			}
			_, err := upgradeConn.Write(data.Data)
			return x.maybeHandleIoError("upgrade request write", err)
		}),
		WithEOFCallback(func() error {
			if !isUpgrade {
				// Nothing more to send, close the write end of the pipe
				err := pw.Close()
				return x.maybeHandleIoError("request body close", err)
			}
			return nil
		}),
	)
}

func (x *InboundGrpcToOutboundHttp) logRequest(header *HttpRequest_Header) {
	if !x.Log.Core().Enabled(zap.DebugLevel) {
		return
	}
	req := header.Request
	sugar := x.Log.Sugar()
	if len(req.Query) > 0 {
		sugar.Debugf("Handling %s %s?%s", req.Method, req.UrlPath, req.UrlQuery().Encode())
	} else {
		sugar.Debugf("Handling %s %s", req.Method, req.UrlPath)
	}
}

func (x *InboundGrpcToOutboundHttp) pipeOutboundToInbound(inbound InboundGrpcToOutboundHttpStream, r DoResponse, isUpgrade bool) error {
	err := x.sendResponseHeaderAndBody(inbound, r.Resp)
	if err != nil {
		return err
	}

	err = inbound.Send(&HttpResponse{
		Message: &HttpResponse_Trailer_{
			Trailer: &HttpResponse_Trailer{},
		},
	})
	if err != nil {
		return x.handleIoError("SendMsg(HttpResponse_Trailer) failed", err)
	}
	if isUpgrade && r.Resp.StatusCode == http.StatusSwitchingProtocols {
		// Only stream if upgrade was requested AND outbound server is switching protocols.
		return x.sendUpgradeResponseStream(inbound, r.ConnReader)
	}
	return nil
}

func (x *InboundGrpcToOutboundHttp) sendResponseHeaderAndBody(inbound InboundGrpcToOutboundHttpStream, resp *http.Response) (retErr error) {
	defer func() {
		err := resp.Body.Close()
		if retErr == nil {
			retErr = x.maybeHandleIoError("response body close", err)
		}
	}()
	err := inbound.Send(&HttpResponse{
		Message: &HttpResponse_Header_{
			Header: &HttpResponse_Header{
				Response: &prototool.HttpResponse{
					StatusCode: int32(resp.StatusCode),
					Status:     resp.Status,
					Header:     prototool.HttpHeaderToValuesMap(resp.Header),
				},
			},
		},
	})
	if err != nil {
		return x.handleIoError("SendMsg(HttpResponse_Header) failed", err)
	}

	buffer := memz.Get32k()
	defer memz.Put32k(buffer)
	for {
		n, readErr := resp.Body.Read(buffer)
		if n > 0 { // handle n>0 before readErr != nil to ensure any consumed data gets forwarded
			sendErr := inbound.Send(&HttpResponse{
				Message: &HttpResponse_Data_{
					Data: &HttpResponse_Data{
						Data: buffer[:n],
					},
				},
			})
			if sendErr != nil {
				return x.handleIoError("SendMsg(HttpResponse_Data) failed", sendErr)
			}
		}
		if readErr != nil {
			if readErr == io.EOF { // nolint:errorlint
				break
			}
			return x.handleIoError("read HTTP response body", readErr)
		}
	}
	return nil
}

func (x *InboundGrpcToOutboundHttp) sendUpgradeResponseStream(inbound InboundGrpcToOutboundHttpStream, upgradeConn *bufio.Reader) error {
	buffer := memz.Get32k()
	defer memz.Put32k(buffer)
	for {
		n, readErr := upgradeConn.Read(buffer)
		if n > 0 { // handle n>0 before readErr != nil to ensure any consumed data gets forwarded
			sendErr := inbound.Send(&HttpResponse{
				Message: &HttpResponse_UpgradeData_{
					UpgradeData: &HttpResponse_UpgradeData{
						Data: buffer[:n],
					},
				},
			})
			if sendErr != nil {
				return x.handleIoError("SendMsg(HttpResponse_UpgradeData) failed", sendErr)
			}
		}
		if readErr != nil {
			if readErr == io.EOF {
				break
			}
			return x.handleIoError("read upgrade response body", readErr)
		}
	}
	return nil
}

func (x *InboundGrpcToOutboundHttp) maybeHandleIoError(msg string, err error) error {
	if err != nil {
		return x.handleIoError(msg, err)
	}
	return nil
}

func (x *InboundGrpcToOutboundHttp) handleIoError(msg string, err error) error {
	return x.HandleIoError("gRPC -> HTTP: "+msg, err)
}