File: inbound_http_to_outbound_grpc.go

package info (click to toggle)
gitlab-agent 16.1.3-2
  • links: PTS, VCS
  • area: contrib
  • in suites: forky, sid, trixie
  • size: 6,324 kB
  • sloc: makefile: 175; sh: 52; ruby: 3
file content (435 lines) | stat: -rw-r--r-- 15,116 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
package grpctool

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"time"

	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/httpz"
	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/logz"
	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/memz"
	"gitlab.com/gitlab-org/cluster-integration/gitlab-agent/v16/internal/tool/prototool"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/anypb"
)

var (
	// See https://httpwg.org/http-core/draft-ietf-httpbis-semantics-latest.html#field.connection
	// See https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1
	// See https://github.com/golang/go/blob/81ea89adf38b90c3c3a8c4eed9e6c093a8634d59/src/net/http/httputil/reverseproxy.go#L169-L184
	// Must be in canonical form.
	hopHeaders = []string{
		httpz.ConnectionHeader,
		httpz.ProxyConnectionHeader,
		httpz.KeepAliveHeader,
		httpz.ProxyAuthenticateHeader,
		httpz.ProxyAuthorizationHeader,
		httpz.TeHeader,
		httpz.TrailerHeader,
		httpz.TransferEncodingHeader,
		httpz.UpgradeHeader,
	}

	// earlyExitError is a sentinel error value to make stream visitor exit early.
	earlyExitError = errors.New("")
)

type HttpRequestClient interface {
	Send(*HttpRequest) error
	Recv() (*HttpResponse, error)
	grpc.ClientStream
}

type MergeHeadersFunc func(outboundResponse, inboundResponse http.Header)
type WriteErrorResponse func(w http.ResponseWriter, r *http.Request, eResp *ErrResp)

type ErrResp struct {
	StatusCode int32
	Msg        string
	// Err can be nil.
	Err error
}

type InboundHttpToOutboundGrpc struct {
	Log                   *zap.Logger
	HandleProcessingError HandleProcessingErrorFunc
	WriteErrorResponse    WriteErrorResponse
	MergeHeaders          MergeHeadersFunc
}

func (x *InboundHttpToOutboundGrpc) Pipe(outboundClient HttpRequestClient, w http.ResponseWriter, r *http.Request, headerExtra proto.Message) {
	// headerExtra can be nil.
	headerWritten, eResp := x.pipe(outboundClient, w, r, headerExtra)
	if eResp != nil {
		if headerWritten {
			// HTTP status has been written already as part of the normal response flow.
			// But then something went wrong and an error happened. To let the client know that something isn't right
			// we have only one thing we can do - abruptly close the connection. To do that we panic with a special
			// error value that the "http" package provides. See its description.
			// If we try to write the status again here, http package would log a warning, which is not nice.
			panic(http.ErrAbortHandler)
		} else {
			x.WriteErrorResponse(w, r, eResp)
		}
	}
}

func (x *InboundHttpToOutboundGrpc) pipe(outboundClient HttpRequestClient, w http.ResponseWriter, r *http.Request,
	headerExtra proto.Message) (bool /* headerWritten */, *ErrResp) {
	// 0. Check if connection upgrade is requested and if connection can be hijacked.
	var hijacker http.Hijacker
	isUpgrade := len(r.Header[httpz.UpgradeHeader]) > 0
	if isUpgrade {
		// Connection upgrade requested. For that ResponseWriter must support hijacking.
		var ok bool
		hijacker, ok = w.(http.Hijacker)
		if !ok {
			return false, x.handleInternalError("unable to upgrade connection", fmt.Errorf("unable to hijack response: %T does not implement http.Hijacker", w))
		}
	}
	// http.ResponseWriter does not support concurrent request body reads and response writes so
	// consume the request body first and then write the response from remote.
	// See https://github.com/golang/go/issues/15527
	// See https://github.com/golang/go/blob/go1.17.2/src/net/http/server.go#L118-L139

	// 1. Pipe client -> remote
	eResp := x.pipeInboundToOutbound(outboundClient, r, headerExtra)
	if eResp != nil {
		return false, eResp
	}
	if !isUpgrade { // Close outbound connection for writes if it's not an upgraded connection
		eResp = x.sendCloseSend(outboundClient)
		if eResp != nil {
			return false, eResp
		}
	}
	// 2. Pipe remote -> client
	headerWritten, responseStatusCode, eResp := x.pipeOutboundToInbound(outboundClient, w, isUpgrade)
	if eResp != nil {
		return headerWritten, eResp
	}
	// 3. Pipe client <-> remote if connection upgrade is requested
	if !isUpgrade { // nothing to do
		return true, nil
	}
	if responseStatusCode != http.StatusSwitchingProtocols {
		// Remote doesn't want to upgrade the connection
		return true, x.sendCloseSend(outboundClient)
	}
	return true, x.pipeUpgradedConnection(outboundClient, hijacker)
}

func (x *InboundHttpToOutboundGrpc) pipeOutboundToInbound(outboundClient HttpRequestClient, w http.ResponseWriter, isUpgrade bool) (bool, int32, *ErrResp) {
	writeFailed := false
	headerWritten := false
	var responseStatusCode int32
	flush := x.flush(w)
	err := HttpResponseStreamVisitor.Get().Visit(outboundClient,
		WithCallback(HttpResponseHeaderFieldNumber, func(header *HttpResponse_Header) error {
			responseStatusCode = header.Response.StatusCode
			outboundResponse := header.Response.HttpHeader()
			cleanHeader(outboundResponse)
			x.MergeHeaders(outboundResponse, w.Header())
			w.WriteHeader(int(header.Response.StatusCode))
			// NOTE: the HTTP standard library doesn't no-op for a flush when WriteHeader() was already called with a 1xx status code
			// and already flushed. This leads to the response being sent twice once with the correct status code and once with `200 OK`.
			// Thus, we avoid flushing manually for all 1xx responses.
			// This seems to be a regression in Go 1.19, introduced with https://go-review.googlesource.com/c/go/+/269997
			if header.Response.StatusCode >= 200 {
				flush()
			}
			headerWritten = true
			return nil
		}),
		WithCallback(HttpResponseDataFieldNumber, func(data *HttpResponse_Data) error {
			_, err := w.Write(data.Data)
			if err != nil {
				writeFailed = true
				return err
			}
			flush()
			return nil
		}),
		WithCallback(HttpResponseTrailerFieldNumber, func(trailer *HttpResponse_Trailer) error {
			if isUpgrade && responseStatusCode == http.StatusSwitchingProtocols {
				// Successful upgrade.
				return earlyExitError
			}
			return nil
		}),
		// if it's a successful upgrade, then this field is unreachable because of the early exit above.
		// otherwise, (unsuccessful upgrade or not an upgrade) the remote must not send this field.
		WithNotExpectingToGet(codes.Internal, HttpResponseUpgradeDataFieldNumber),
	)
	if err != nil && err != earlyExitError { // nolint: errorlint
		if writeFailed {
			// there is likely a connection problem so the client will likely not receive this
			return headerWritten, responseStatusCode, x.handleIoError("failed to write HTTP response", err)
		}
		return headerWritten, responseStatusCode, x.handleIoError("failed to read gRPC response", err)
	}
	return headerWritten, responseStatusCode, nil
}

func (x *InboundHttpToOutboundGrpc) flush(w http.ResponseWriter) func() {
	// ResponseWriter buffers headers and response body writes and that may break use cases like long polling or streaming.
	// Flusher is used so that when HTTP headers and response body chunks are received from the outbound connection,
	// they are flushed to the inbound stream ASAP.
	flusher, ok := w.(http.Flusher)
	if !ok {
		x.Log.Sugar().Warnf("HTTP->gRPC: %T does not implement http.Flusher, cannot flush data to client", w)
		return func() {}
	}
	return flusher.Flush
}

func (x *InboundHttpToOutboundGrpc) pipeInboundToOutbound(outboundClient HttpRequestClient, r *http.Request, headerExtra proto.Message) *ErrResp {
	var extra *anypb.Any
	if headerExtra != nil {
		var err error
		extra, err = anypb.New(headerExtra)
		if err != nil {
			return x.handleInternalError("failed to marshal header extra proto", err)
		}
	}
	eResp := x.send(outboundClient, "failed to send request header", &HttpRequest{
		Message: &HttpRequest_Header_{
			Header: &HttpRequest_Header{
				Request: &prototool.HttpRequest{
					Method:  r.Method,
					Header:  headerFromHttpRequestHeader(r.Header),
					UrlPath: r.URL.Path,
					Query:   prototool.UrlValuesToValuesMap(r.URL.Query()),
				},
				Extra:         extra,
				ContentLength: &r.ContentLength,
			},
		},
	})
	if eResp != nil {
		return eResp
	}

	eResp = x.sendRequestBody(outboundClient, r.Body)
	if eResp != nil {
		return eResp
	}
	return x.send(outboundClient, "failed to send trailer", &HttpRequest{
		Message: &HttpRequest_Trailer_{
			Trailer: &HttpRequest_Trailer{},
		},
	})
}

func (x *InboundHttpToOutboundGrpc) sendRequestBody(outboundClient HttpRequestClient, body io.Reader) *ErrResp {
	buffer := memz.Get32k()
	defer memz.Put32k(buffer)
	for {
		n, readErr := body.Read(buffer)
		if n > 0 { // handle n>0 before readErr != nil to ensure any consumed data gets forwarded
			eResp := x.send(outboundClient, "failed to send request data", &HttpRequest{
				Message: &HttpRequest_Data_{
					Data: &HttpRequest_Data{
						Data: buffer[:n],
					},
				},
			})
			if eResp != nil {
				return eResp
			}
		}
		if readErr != nil {
			if readErr == io.EOF { // nolint:errorlint
				break
			}
			// There is likely a connection problem so the client will likely not receive this
			return x.handleIoError("failed to read request body", readErr)
		}
	}
	return nil
}

func (x *InboundHttpToOutboundGrpc) sendCloseSend(outboundClient HttpRequestClient) *ErrResp {
	err := outboundClient.CloseSend()
	if err != nil {
		return x.handleIoError("failed to send close frame", err)
	}
	return nil
}

func (x *InboundHttpToOutboundGrpc) send(client HttpRequestClient, errMsg string, msg *HttpRequest) *ErrResp {
	err := client.Send(msg)
	if err != nil {
		if err == io.EOF { // nolint:errorlint
			_, err = client.Recv()
		}
		return x.handleIoError(errMsg, err)
	}
	return nil
}

func (x *InboundHttpToOutboundGrpc) handleIoError(msg string, err error) *ErrResp {
	msg = "HTTP->gRPC: " + msg
	x.Log.Debug(msg, logz.Error(err))
	return &ErrResp{
		// See https://datatracker.ietf.org/doc/html/rfc7231#section-6.6.3
		StatusCode: http.StatusBadGateway,
		Msg:        msg,
		Err:        err,
	}
}

func (x *InboundHttpToOutboundGrpc) handleInternalError(msg string, err error) *ErrResp {
	msg = "HTTP->gRPC: " + msg
	x.HandleProcessingError(msg, err)
	return &ErrResp{
		// See https://datatracker.ietf.org/doc/html/rfc7231#section-6.6.1
		StatusCode: http.StatusInternalServerError,
		Msg:        msg,
		Err:        err,
	}
}

func (x *InboundHttpToOutboundGrpc) pipeUpgradedConnection(outboundClient HttpRequestClient, hijacker http.Hijacker) (errRet *ErrResp) {
	conn, bufrw, err := hijacker.Hijack()
	if err != nil {
		return x.handleInternalError("unable to upgrade connection: error hijacking response", err)
	}
	defer func() {
		err = conn.Close()
		if err != nil && errRet == nil {
			errRet = x.handleIoError("failed to close upgraded connection", err)
		}
	}()
	// Hijack() docs say we are responsible for managing connection deadlines and a deadline may be set already.
	// We clear the read deadline here because we don't know if the client will be sending any data to us soon.
	err = conn.SetReadDeadline(time.Time{})
	if err != nil {
		return x.handleIoError("failed to clear connection read deadline", err)
	}
	// We don't care if a write deadline is set already, we just wrap the connection in a wrapper that
	// will each time set a new deadline before performing an actual write.
	conn = &httpz.WriteTimeoutConn{
		Conn:    conn,
		Timeout: 20 * time.Second,
	}
	r, err := decoupleReader(bufrw.Reader, conn)
	if err != nil {
		return x.handleIoError("failed to read buffered data", err)
	}
	p := InboundStreamToOutboundStream{
		PipeInboundToOutbound: func() error {
			return x.pipeInboundToOutboundUpgraded(outboundClient, r)
		},
		PipeOutboundToInbound: func() error {
			return x.pipeOutboundToInboundUpgraded(outboundClient, conn)
		},
	}
	err = p.Pipe()
	if err != nil {
		return x.handleIoError("failed to pipe upgraded connection streams", err)
	}
	return nil
}

func (x *InboundHttpToOutboundGrpc) pipeInboundToOutboundUpgraded(outboundClient HttpRequestClient, inboundStream io.Reader) error {
	buffer := memz.Get32k()
	defer memz.Put32k(buffer)
	for {
		n, readErr := inboundStream.Read(buffer)
		if n > 0 { // handle n>0 before readErr != nil to ensure any consumed data gets forwarded
			sendErr := outboundClient.Send(&HttpRequest{
				Message: &HttpRequest_UpgradeData_{
					UpgradeData: &HttpRequest_UpgradeData{
						Data: buffer[:n],
					},
				},
			})
			if sendErr != nil {
				if readErr == io.EOF {
					return nil // the other goroutine will receive the error in RecvMsg()
				}
				return fmt.Errorf("Send(HttpRequest_UpgradeData): %w", sendErr)
			}
		}
		if readErr != nil {
			if readErr == io.EOF {
				break
			}
			// There is likely a connection problem so the client will likely not receive this
			return fmt.Errorf("read failed: %w", readErr)
		}
	}
	err := outboundClient.CloseSend()
	if err != nil {
		return fmt.Errorf("failed to send close frame: %w", err)
	}
	return nil
}

func (x *InboundHttpToOutboundGrpc) pipeOutboundToInboundUpgraded(outboundClient HttpRequestClient, inboundStream io.Writer) error {
	var writeFailed bool
	err := HttpResponseStreamVisitor.Get().Visit(outboundClient,
		WithStartState(HttpResponseTrailerFieldNumber),
		WithCallback(HttpResponseUpgradeDataFieldNumber, func(data *HttpResponse_UpgradeData) error {
			_, err := inboundStream.Write(data.Data)
			if err != nil {
				writeFailed = true
			}
			return err
		}),
	)
	if err != nil {
		if writeFailed {
			// there is likely a connection problem so the client will likely not receive this
			return fmt.Errorf("failed to write upgraded HTTP response: %w", err)
		}
		return fmt.Errorf("failed to read upgraded gRPC response: %w", err)
	}
	return nil
}

// decoupleReader returns an io.Reader that is decoupled from the buffered reader, returned by Hijack.
// This is a workaround for https://github.com/golang/go/issues/32314.
func decoupleReader(r *bufio.Reader, conn net.Conn) (io.Reader, error) {
	buffered := r.Buffered()
	if buffered == 0 {
		return conn, nil
	}
	b, err := r.Peek(buffered)
	if err != nil {
		return nil, err
	}
	return io.MultiReader(bytes.NewReader(b), conn), nil
}

func headerFromHttpRequestHeader(header http.Header) map[string]*prototool.Values {
	header = header.Clone()
	delete(header, httpz.HostHeader) // Use the destination host name
	cleanHeader(header)
	return prototool.HttpHeaderToValuesMap(header)
}

func cleanHeader(header http.Header) {
	upgrade := header[httpz.UpgradeHeader]

	// 1. Remove hop-by-hop headers listed in the Connection header. See https://datatracker.ietf.org/doc/html/rfc7230#section-6.1
	httpz.RemoveConnectionHeaders(header)
	// 2. Remove well-known hop-by-hop headers
	for _, name := range hopHeaders {
		delete(header, name)
	}
	// 3. Fix up Connection and Upgrade headers if upgrade is requested/confirmed
	if len(upgrade) > 0 {
		header[httpz.UpgradeHeader] = upgrade                // put it back
		header[httpz.ConnectionHeader] = []string{"upgrade"} // this discards any other connection options if they were there
	}
}