File: checkmgr.go

package info (click to toggle)
golang-github-circonus-labs-circonus-gometrics 2.3.1-4
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, forky, sid, trixie
  • size: 824 kB
  • sloc: makefile: 2
file content (507 lines) | stat: -rw-r--r-- 13,526 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package checkmgr provides a check management interface to circonus-gometrics
package checkmgr

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-gometrics/api"
	"github.com/pkg/errors"
	"github.com/tv42/httpunix"
)

// Check management offers:
//
// Create a check if one cannot be found matching specific criteria
// Manage metrics in the supplied check (enabling new metrics as they are submitted)
//
// To disable check management, leave Config.Api.Token.Key blank
//
// use cases:
// configure without api token - check management disabled
//  - configuration parameters other than Check.SubmissionUrl, Debug and Log are ignored
//  - note: SubmissionUrl is **required** in this case as there is no way to derive w/o api
// configure with api token - check management enabled
//  - all other configuration parameters affect how the trap url is obtained
//    1. provided (Check.SubmissionUrl)
//    2. via check lookup (CheckConfig.Id)
//    3. via a search using CheckConfig.InstanceId + CheckConfig.SearchTag
//    4. a new check is created

const (
	defaultCheckType             = "httptrap"
	defaultTrapMaxURLAge         = "60s"   // 60 seconds
	defaultBrokerMaxResponseTime = "500ms" // 500 milliseconds
	defaultForceMetricActivation = "false"
	statusActive                 = "active"
)

// CheckConfig options for check
type CheckConfig struct {
	// a specific submission url
	SubmissionURL string
	// a specific check id (not check bundle id)
	ID string
	// unique instance id string
	// used to search for a check to use
	// used as check.target when creating a check
	InstanceID string
	// explicitly set check.target (default: instance id)
	TargetHost string
	// a custom display name for the check (as viewed in UI Checks)
	// default: instance id
	DisplayName string
	// unique check searching tag (or tags)
	// used to search for a check to use (combined with instanceid)
	// used as a regular tag when creating a check
	SearchTag string
	// httptrap check secret (for creating a check)
	Secret string
	// additional tags to add to a check (when creating a check)
	// these tags will not be added to an existing check
	Tags string
	// max amount of time to to hold on to a submission url
	// when a given submission fails (due to retries) if the
	// time the url was last updated is > than this, the trap
	// url will be refreshed (e.g. if the broker is changed
	// in the UI) **only relevant when check management is enabled**
	// e.g. 5m, 30m, 1h, etc.
	MaxURLAge string
	// force metric activation - if a metric has been disabled via the UI
	// the default behavior is to *not* re-activate the metric; this setting
	// overrides the behavior and will re-activate the metric when it is
	// encountered. "(true|false)", default "false"
	ForceMetricActivation string
	// Type of check to use (default: httptrap)
	Type string
	// Custom check config fields (default: none)
	CustomConfigFields map[string]string
}

// BrokerConfig options for broker
type BrokerConfig struct {
	// a specific broker id (numeric portion of cid)
	ID string
	// one or more tags used to select 1-n brokers from which to select
	// when creating a new check (e.g. datacenter:abc or loc:dfw,dc:abc)
	SelectTag string
	// for a broker to be considered viable it must respond to a
	// connection attempt within this amount of time e.g. 200ms, 2s, 1m
	MaxResponseTime string
	// TLS configuration to use when communicating within broker
	TLSConfig *tls.Config
}

// Config options
type Config struct {
	Log   *log.Logger
	Debug bool

	// Circonus API config
	API api.Config
	// Check specific configuration options
	Check CheckConfig
	// Broker specific configuration options
	Broker BrokerConfig
}

// CheckTypeType check type
type CheckTypeType string

// CheckInstanceIDType check instance id
type CheckInstanceIDType string

// CheckTargetType check target/host
type CheckTargetType string

// CheckSecretType check secret
type CheckSecretType string

// CheckTagsType check tags
type CheckTagsType string

// CheckDisplayNameType check display name
type CheckDisplayNameType string

// BrokerCNType broker common name
type BrokerCNType string

// CheckManager settings
type CheckManager struct {
	enabled bool
	Log     *log.Logger
	Debug   bool
	apih    *api.API

	initialized   bool
	initializedmu sync.RWMutex

	// check
	checkType             CheckTypeType
	checkID               api.IDType
	checkInstanceID       CheckInstanceIDType
	checkTarget           CheckTargetType
	checkSearchTag        api.TagType
	checkSecret           CheckSecretType
	checkTags             api.TagType
	customConfigFields    map[string]string
	checkSubmissionURL    api.URLType
	checkDisplayName      CheckDisplayNameType
	forceMetricActivation bool
	forceCheckUpdate      bool

	// metric tags
	metricTags map[string][]string
	mtmu       sync.Mutex

	// broker
	brokerID              api.IDType
	brokerSelectTag       api.TagType
	brokerMaxResponseTime time.Duration
	brokerTLS             *tls.Config

	// state
	checkBundle        *api.CheckBundle
	cbmu               sync.Mutex
	availableMetrics   map[string]bool
	availableMetricsmu sync.Mutex
	trapURL            api.URLType
	trapCN             BrokerCNType
	trapLastUpdate     time.Time
	trapMaxURLAge      time.Duration
	trapmu             sync.Mutex
	certPool           *x509.CertPool
	sockRx             *regexp.Regexp
}

// Trap config
type Trap struct {
	URL           *url.URL
	TLS           *tls.Config
	IsSocket      bool
	SockTransport *httpunix.Transport
}

// NewCheckManager returns a new check manager
func NewCheckManager(cfg *Config) (*CheckManager, error) {
	return New(cfg)
}

// New returns a new check manager
func New(cfg *Config) (*CheckManager, error) {

	if cfg == nil {
		return nil, errors.New("invalid Check Manager configuration (nil)")
	}

	cm := &CheckManager{enabled: true, initialized: false}

	// Setup logging for check manager
	cm.Debug = cfg.Debug
	cm.Log = cfg.Log
	if cm.Debug && cm.Log == nil {
		cm.Log = log.New(os.Stderr, "", log.LstdFlags)
	}
	if cm.Log == nil {
		cm.Log = log.New(ioutil.Discard, "", log.LstdFlags)
	}

	{
		rx, err := regexp.Compile(`^http\+unix://(?P<sockfile>.+)/write/(?P<id>.+)$`)
		if err != nil {
			return nil, errors.Wrap(err, "compiling socket regex")
		}
		cm.sockRx = rx
	}

	if cfg.Check.SubmissionURL != "" {
		cm.checkSubmissionURL = api.URLType(cfg.Check.SubmissionURL)
	}

	// Blank API Token *disables* check management
	if cfg.API.TokenKey == "" {
		cm.enabled = false
	}

	if !cm.enabled && cm.checkSubmissionURL == "" {
		return nil, errors.New("invalid check manager configuration (no API token AND no submission url)")
	}

	if cm.enabled {
		// initialize api handle
		cfg.API.Debug = cm.Debug
		cfg.API.Log = cm.Log
		apih, err := api.New(&cfg.API)
		if err != nil {
			return nil, errors.Wrap(err, "initializing api client")
		}
		cm.apih = apih
	}

	// initialize check related data
	if cfg.Check.Type != "" {
		cm.checkType = CheckTypeType(cfg.Check.Type)
	} else {
		cm.checkType = defaultCheckType
	}

	idSetting := "0"
	if cfg.Check.ID != "" {
		idSetting = cfg.Check.ID
	}
	id, err := strconv.Atoi(idSetting)
	if err != nil {
		return nil, errors.Wrap(err, "converting check id")
	}
	cm.checkID = api.IDType(id)

	cm.checkInstanceID = CheckInstanceIDType(cfg.Check.InstanceID)
	cm.checkTarget = CheckTargetType(cfg.Check.TargetHost)
	cm.checkDisplayName = CheckDisplayNameType(cfg.Check.DisplayName)
	cm.checkSecret = CheckSecretType(cfg.Check.Secret)

	fma := defaultForceMetricActivation
	if cfg.Check.ForceMetricActivation != "" {
		fma = cfg.Check.ForceMetricActivation
	}
	fm, err := strconv.ParseBool(fma)
	if err != nil {
		return nil, errors.Wrap(err, "parsing force metric activation")
	}
	cm.forceMetricActivation = fm

	_, an := path.Split(os.Args[0])
	hn, err := os.Hostname()
	if err != nil {
		hn = "unknown"
	}
	if cm.checkInstanceID == "" {
		cm.checkInstanceID = CheckInstanceIDType(fmt.Sprintf("%s:%s", hn, an))
	}
	if cm.checkDisplayName == "" {
		cm.checkDisplayName = CheckDisplayNameType(cm.checkInstanceID)
	}
	if cm.checkTarget == "" {
		cm.checkTarget = CheckTargetType(cm.checkInstanceID)
	}

	if cfg.Check.SearchTag == "" {
		cm.checkSearchTag = []string{fmt.Sprintf("service:%s", an)}
	} else {
		cm.checkSearchTag = strings.Split(strings.Replace(cfg.Check.SearchTag, " ", "", -1), ",")
	}

	if cfg.Check.Tags != "" {
		cm.checkTags = strings.Split(strings.Replace(cfg.Check.Tags, " ", "", -1), ",")
	}

	cm.customConfigFields = make(map[string]string)
	if len(cfg.Check.CustomConfigFields) > 0 {
		for fld, val := range cfg.Check.CustomConfigFields {
			cm.customConfigFields[fld] = val
		}
	}

	dur := cfg.Check.MaxURLAge
	if dur == "" {
		dur = defaultTrapMaxURLAge
	}
	maxDur, err := time.ParseDuration(dur)
	if err != nil {
		return nil, errors.Wrap(err, "parsing max url age")
	}
	cm.trapMaxURLAge = maxDur

	// setup broker
	idSetting = "0"
	if cfg.Broker.ID != "" {
		idSetting = cfg.Broker.ID
	}
	id, err = strconv.Atoi(idSetting)
	if err != nil {
		return nil, errors.Wrap(err, "parsing broker id")
	}
	cm.brokerID = api.IDType(id)

	if cfg.Broker.SelectTag != "" {
		cm.brokerSelectTag = strings.Split(strings.Replace(cfg.Broker.SelectTag, " ", "", -1), ",")
	}

	dur = cfg.Broker.MaxResponseTime
	if dur == "" {
		dur = defaultBrokerMaxResponseTime
	}
	maxDur, err = time.ParseDuration(dur)
	if err != nil {
		return nil, errors.Wrap(err, "parsing broker max response time")
	}
	cm.brokerMaxResponseTime = maxDur

	// add user specified tls config for broker if provided
	cm.brokerTLS = cfg.Broker.TLSConfig

	// metrics
	cm.availableMetrics = make(map[string]bool)
	cm.metricTags = make(map[string][]string)

	return cm, nil
}

// Initialize for sending metrics
func (cm *CheckManager) Initialize() {

	// if not managing the check, quicker initialization
	if !cm.enabled {
		err := cm.initializeTrapURL()
		if err == nil {
			cm.initializedmu.Lock()
			cm.initialized = true
			cm.initializedmu.Unlock()
		} else {
			cm.Log.Printf("[WARN] error initializing trap %s", err.Error())
		}
		return
	}

	// background initialization when we have to reach out to the api
	go func() {
		cm.apih.EnableExponentialBackoff()
		err := cm.initializeTrapURL()
		if err == nil {
			cm.initializedmu.Lock()
			cm.initialized = true
			cm.initializedmu.Unlock()
		} else {
			cm.Log.Printf("[WARN] error initializing trap %s", err.Error())
		}
		cm.apih.DisableExponentialBackoff()
	}()
}

// IsReady reflects if the check has been initialied and metrics can be sent to Circonus
func (cm *CheckManager) IsReady() bool {
	cm.initializedmu.RLock()
	defer cm.initializedmu.RUnlock()
	return cm.initialized
}

// GetSubmissionURL returns submission url for circonus
func (cm *CheckManager) GetSubmissionURL() (*Trap, error) {
	if cm.trapURL == "" {
		return nil, errors.Errorf("get submission url - submission url unavailable")
	}

	trap := &Trap{}

	u, err := url.Parse(string(cm.trapURL))
	if err != nil {
		return nil, errors.Wrap(err, "get submission url")
	}
	trap.URL = u

	if u.Scheme == "http+unix" {
		service := "circonus-agent"
		sockPath := ""
		metricID := ""

		subNames := cm.sockRx.SubexpNames()
		matches := cm.sockRx.FindAllStringSubmatch(string(cm.trapURL), -1)
		for _, match := range matches {
			for idx, val := range match {
				switch subNames[idx] {
				case "sockfile":
					sockPath = val
				case "id":
					metricID = val
				}
			}
		}

		if sockPath == "" || metricID == "" {
			return nil, errors.Errorf("get submission url - invalid socket url (%s)", cm.trapURL)
		}

		u, err = url.Parse(fmt.Sprintf("http+unix://%s/write/%s", service, metricID))
		if err != nil {
			return nil, errors.Wrap(err, "get submission url")
		}
		trap.URL = u

		trap.SockTransport = &httpunix.Transport{
			DialTimeout:           100 * time.Millisecond,
			RequestTimeout:        1 * time.Second,
			ResponseHeaderTimeout: 1 * time.Second,
		}
		trap.SockTransport.RegisterLocation(service, sockPath)
		trap.IsSocket = true
	}

	if u.Scheme == "https" {
		// preference user-supplied TLS configuration
		if cm.brokerTLS != nil {
			trap.TLS = cm.brokerTLS
			return trap, nil
		}

		// api.circonus.com uses a public CA signed certificate
		// trap.noit.circonus.net uses Circonus CA private certificate
		// enterprise brokers use private CA certificate
		if trap.URL.Hostname() == "api.circonus.com" {
			return trap, nil
		}

		if cm.certPool == nil {
			if err := cm.loadCACert(); err != nil {
				return nil, errors.Wrap(err, "get submission url")
			}
		}
		t := &tls.Config{
			RootCAs: cm.certPool,
		}
		if cm.trapCN != "" {
			t.ServerName = string(cm.trapCN)
		}
		trap.TLS = t
	}

	return trap, nil
}

// ResetTrap URL, force request to the API for the submission URL and broker ca cert
func (cm *CheckManager) ResetTrap() error {
	if cm.trapURL == "" {
		return nil
	}

	cm.trapURL = ""
	cm.certPool = nil // force re-fetching CA cert (if custom TLS config not supplied)
	return cm.initializeTrapURL()
}

// RefreshTrap check when the last time the URL was reset, reset if needed
func (cm *CheckManager) RefreshTrap() error {
	if cm.trapURL == "" {
		return nil
	}

	if time.Since(cm.trapLastUpdate) >= cm.trapMaxURLAge {
		return cm.ResetTrap()
	}

	return nil
}