{"id":127,"date":"2020-08-18T19:37:33","date_gmt":"2020-08-18T11:37:33","guid":{"rendered":"https:\/\/chenqinghe.com\/?p=127"},"modified":"2020-08-19T15:14:57","modified_gmt":"2020-08-19T07:14:57","slug":"prometheus%e5%91%8a%e8%ad%a6%e6%ba%90%e7%a0%81%e5%88%86%e6%9e%90","status":"publish","type":"post","link":"https:\/\/chenqinghe.com\/?p=127","title":{"rendered":"prometheus\u544a\u8b66\u6e90\u7801\u5206\u6790\uff08\u4e00\uff09\u2014\u2014\u6267\u884c\u6d41\u7a0b\u5206\u6790"},"content":{"rendered":"
\n

\u672c\u6587\u6839\u636eprometheus master\u5206\u652f\u6700\u65b0\u4ee3\u7801\u6240\u5199\uff0ccommit id\uff1adca84112a97ea7a31f2ddb2ce7cfb4f7cae91f86<\/p>\n<\/blockquote>\n

\u544a\u8b66\u662fprometheus\u7684\u4e00\u4e2a\u91cd\u8981\u529f\u80fd\uff0c\u63a5\u4e0b\u6765\u4ece\u6e90\u7801\u7684\u89d2\u5ea6\u6765\u5206\u6790\u4e0b\u544a\u8b66\u7684\u6267\u884c\u6d41\u7a0b\u3002<\/p>\n

\u6574\u4f53\u7684\u5927\u81f4\u6d41\u7a0b\u8bf7\u89c1\u4e0b\u65b9\u6d41\u7a0b\u56fe\uff1a<\/p>\n

\"\u300aprometheus\u544a\u8b66\u6e90\u7801\u5206\u6790\uff08\u4e00\uff09\u2014\u2014\u6267\u884c\u6d41\u7a0b\u5206\u6790\u300b\"<\/figure>\n<\/p>\n

prometheus\u7684\u544a\u8b66\u90e8\u5206\u7684\u6e90\u7801\u4e3b\u8981\u5728github.com\/prometheus\/prometheus\/rules\u6587\u4ef6\u5939\u4e0b\u9762\uff0c\u4e3b\u8981\u7ed3\u6784\u5982\u4e0b\uff1a<\/p>\n

\/\/ The Manager manages recording and alerting rules.\ntype Manager struct {\n\topts     *ManagerOptions   \/\/ \u521d\u59cb\u5316\u65f6\u7684option\n\tgroups   map[string]*Group  \/\/ \u89c4\u5219group\n\tmtx      sync.RWMutex  \n\tblock    chan struct{}  \/\/ \u66f4\u65b0\u89c4\u5219\u7ec4\u540e\u963b\u585e\uff0c\u76f4\u5230storage\u51c6\u5907\u597d\u4e86\u540e\u518d\u6267\u884c\n\tdone     chan struct{}\n\trestored bool\n\tlogger log.Logger\n}<\/pre>\n
Manager\u8d1f\u8d23\u7ba1\u7406\u89c4\u5219\uff0c\u5b9a\u65f6\u626b\u63cf\u89c4\u5219\u6587\u4ef6\u3001\u52a0\u8f7d\u65b0\u7684\u89c4\u5219\u3001\u6267\u884c\u89c4\u5219\u3001\u53d1\u9001\u544a\u8b66\u4fe1\u606f\u7b49\u3002\u9996\u5148\u662f\u521b\u5efaManager\uff1a<\/p>\n
var (\n\tlocalStorage  = &readyStorage{}\n\tremoteStorage = remote.NewStorage(log.With(logger, \"component\", \"remote\"), prometheus.DefaultRegisterer, localStorage.StartTime, cfg.localStoragePath, time.Duration(cfg.RemoteFlushDeadline))\n\tfanoutStorage = storage.NewFanout(logger, localStorage, remoteStorage)\n)\n\nruleManager = rules.NewManager(&rules.ManagerOptions{\n\tAppendable:      fanoutStorage,\n\tQueryable:       localStorage,\n\tQueryFunc:       rules.EngineQueryFunc(queryEngine, fanoutStorage),\n\tNotifyFunc:      sendAlerts(notifierManager, cfg.web.ExternalURL.String()),\n\tContext:         ctxRule,\n\tExternalURL:     cfg.web.ExternalURL,\n\tRegisterer:      prometheus.DefaultRegisterer,\n\tLogger:          log.With(logger, \"component\", \"rule manager\"),\n\tOutageTolerance: time.Duration(cfg.outageTolerance),\n\tForGracePeriod:  time.Duration(cfg.forGracePeriod),\n\tResendDelay:     time.Duration(cfg.resendDelay),\n})<\/pre>\n
\u51e0\u4e2a\u91cd\u8981\u7684\u914d\u7f6e\uff1a<\/p>\n
    \n
  • Appendable\uff1a\u8d1f\u8d23\u4e34\u65f6\u5b58\u50a8\u6ee1\u8db3\u544a\u8b66\u89c4\u5219\u7684\u6570\u636e\u3002<\/li>\n
  • Queryable\uff1a\u8d1f\u8d23\u67e5\u8be2Appendable\u4e2d\u5b58\u50a8\u7684\u6570\u636e\u3002<\/li>\n
  • QueryFunc\uff1a\u7528\u4e8e\u67e5\u8be2prometheus\u6570\u636e\u3002<\/li>\n
  • NotifyFunc\uff1a\u7528\u4e8e\u6ee1\u8db3\u544a\u8b66\u6761\u4ef6\u65f6\u53d1\u9001\u901a\u77e5\u3002<\/li>\n
  • OutageTolerance\uff1a\uff1f\uff1f\uff1f\uff1f\uff1f<\/li>\n
  • ForGracePeriod\uff1a\uff1f\uff1f\uff1f\uff1f\uff1f<\/li>\n
  • ResendDelay\uff1a\u53d1\u9001\u544a\u8b66\u901a\u77e5\u540e\uff0c\u5728\u8fd9\u6bb5\u65f6\u95f4\u540e\u4e0d\u518d\u91cd\u590d\u53d1\u9001\u901a\u77e5\u3002<\/li>\n<\/ul>\n
    \u4e00\u3001\u8bfb\u53d6\u89c4\u5219\u914d\u7f6e<\/h3>\n
    \u7a0b\u5e8f\u542f\u52a8\u65f6\u4f1a\u521b\u5efa\u4e00\u7cfb\u5217\u7684reloader\uff0c\u5176\u4e2d\u6709\u4e00\u4e2a\u662f\u8bfb\u53d6\u89c4\u5219\u6587\u4ef6\uff0c\u66f4\u65b0Manager\uff1a<\/p>\n
    reloaders := []reloader{\n\t\/\/ \u6b64\u5904\u7701\u7565\u5176\u4ed6reloader\uff0c\u611f\u5174\u8da3\u7684\u53ef\u4ee5\u81ea\u884c\u67e5\u770b\u6e90\u7801............\n\n\t{\n\t\tname: \"rules\",\n\t\treloader: func(cfg *config.Config) error {\n\t\t\t\/\/ Get all rule files matching the configuration paths.\n\t\t\tvar files []string\n\t\t\tfor _, pat := range cfg.RuleFiles {\n\t\t\t\tfs, err := filepath.Glob(pat)\n\t\t\t\tif err != nil {\n\t\t\t\t\t\/\/ The only error can be a bad pattern.\n\t\t\t\t\treturn errors.Wrapf(err, \"error retrieving rule files for %s\", pat)\n\t\t\t\t}\n\t\t\t\tfiles = append(files, fs...)\n\t\t\t}\n\t\t\treturn ruleManager.Update(\n\t\t\t\ttime.Duration(cfg.GlobalConfig.EvaluationInterval),\n\t\t\t\tfiles,\n\t\t\t\tcfg.GlobalConfig.ExternalLabels,\n\t\t\t)\n\t\t},\n\t},\n}<\/pre>\n
    \u8fd9\u91cc\u4f1a\u4f9d\u6b21\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6\u4e2drule_files\u4e0b\u7684\u914d\u7f6e\u9879\uff0c\u626b\u63cf\u6ee1\u8db3\u6761\u4ef6\u7684\u914d\u7f6e\u6587\u4ef6\uff0c\u7136\u540e\u66f4\u65b0\u89c4\u5219\u3002<\/p>\n
    \u8be6\u7ec6\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n
    func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error {\n\tm.mtx.Lock()\n\tdefer m.mtx.Unlock()\n\n\t\/\/ \u52a0\u8f7d\u914d\u7f6e\u7684\u89c4\u5219\u7ec4\n\tgroups, errs := m.LoadGroups(interval, externalLabels, files...)\n\tif errs != nil {\n\t\tfor _, e := range errs {\n\t\t\tlevel.Error(m.logger).Log(\"msg\", \"loading groups failed\", \"err\", e)\n\t\t}\n\t\treturn errors.New(\"error loading rules, previous rule set restored\")\n\t}\n\tm.restored = true \/\/ \u6807\u8bb0\u89c4\u5219\u5df2\u7ecf\u91cd\u65b0\u52a0\u8f7d\u8fc7\u4e86\n\n\tvar wg sync.WaitGroup\n\tfor _, newg := range groups {\n\t\t\/\/ \u5224\u65ad\u6587\u4ef6\u914d\u7f6e\u4e2d\u7684\u89c4\u5219\u662f\u5426\u5df2\u7ecf\u5b58\u5728\uff0c\u5982\u679c\u5df2\u7ecf\u5b58\u5728\u5e76\u4e14\u672a\u7ecf\u4fee\u6539\uff0c\u5c31\u7565\u8fc7\n\t\t\/\/ \u5426\u5219\uff0c\u505c\u6b62\u539f\u6765\u7684\u89c4\u5219\u5e76\u4ece\u5f53\u524d\u89c4\u5219\u7ec4\u5217\u8868\u4e2d\u5220\u9664\uff0c\u7136\u540e\u590d\u5236\u65e7\u89c4\u5219\u7ec4\u7684\u72b6\u6001\u81f3\u540c\u540d\u7684\u65b0\u7684\u89c4\u5219\u7ec4\n\t\tgn := groupKey(newg.file, newg.name)\n\t\toldg, ok := m.groups[gn]\n\t\tdelete(m.groups, gn)\n\n\t\tif ok && oldg.Equals(newg) {\n\t\t\tgroups[gn] = oldg\n\t\t\tcontinue\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func(newg *Group) {\n\t\t\tif ok {\n\t\t\t\toldg.stop()\n\t\t\t\tnewg.CopyState(oldg)\n\t\t\t}\n\t\t\tgo func() {\n\t\t\t\t\/\/ \u5f00\u59cb\u8fd0\u884c\u65b0\u7684\u89c4\u5219\u7ec4\n\t\t\t\t<-m.block\n\t\t\t\tnewg.run(m.opts.Context)\n\t\t\t}()\n\t\t\twg.Done()\n\t\t}(newg)\n\t}\n\n\t\/\/ \u505c\u6b62\u6240\u6709\u65e7\u7684\u7ec4\u548c\u89c4\u5219\n\t\/\/ \u5f00\u59cb\u8fd0\u884c\u65b0\u7684\u89c4\u5219\u7ec4\n\t\/\/ \u8fd9\u91cc\u8981\u963b\u585e\uff0c\u76f4\u5230\u8c03\u7528ruleManager.Start()\u65b9\u6cd5\u540e\u624d\u53ef\u4ee5\u7ee7\u7eed\u6267\u884c\n\t\/\/ \u56e0\u4e3a\u6267\u884c\u89c4\u5219\u65f6\u8981\u67e5\u8be2\u6570\u636e\uff0c\u4f46\u662f\u67e5\u8be2\u6570\u636e\u7684\u65f6\u5019\u6570\u636e\u6e90\u53ef\u80fd\u8fd8\u672a\u51c6\u5907\u597d\n\t\/\/ \u56e0\u6b64\u8981\u7b49\u6570\u636e\u6e90\u521d\u59cb\u5316\u5e76\u4e14\u4e0a\u5c42\u8c03\u7528Start\u65b9\u6cd5\u540e\u624d\u53ef\u4ee5\u7ee7\u7eed\u6267\u884c\n\twg.Add(len(m.groups))\n\tfor n, oldg := range m.groups {\n\t\tgo func(n string, g *Group) {\n\t\t\tg.markStale = true\n\t\t\tg.stop()\n\t\t\tif m := g.metrics; m != nil {\n\t\t\t\tm.evalTotal.DeleteLabelValues(n)\n\t\t\t\tm.evalFailures.DeleteLabelValues(n)\n\t\t\t\tm.groupInterval.DeleteLabelValues(n)\n\t\t\t\tm.groupLastEvalTime.DeleteLabelValues(n)\n\t\t\t\tm.groupLastDuration.DeleteLabelValues(n)\n\t\t\t\tm.groupRules.DeleteLabelValues(n)\n\t\t\t}\n\t\t\twg.Done()\n\t\t}(n, oldg)\n\t}\n\n\twg.Wait()\n\tm.groups = groups\n\n\treturn nil\n}<\/pre>\n
    \u8fd9\u91cc\u52a0\u8f7d\u914d\u7f6e\u7684\u89c4\u5219\u7ec4\u6ca1\u4ec0\u4e48\u597d\u8bf4\u7684\uff0c\u4e3b\u8981\u5c31\u662f\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6\u5e76\u4e14\u89e3\u6790\u3002\u4e0b\u9762\u5177\u4f53\u770b\u4e0bnewg.run(m.opts.Context)\uff0c\u65b0\u89c4\u5219\u7684\u5177\u4f53\u6267\u884c\u5185\u5bb9\u3002<\/p>\n
    func (g *Group) run(ctx context.Context) {\n\t\/\/ .....\n\n\titer := func() {\n\t\tg.metrics.iterationsScheduled.Inc()\n\n\t\tstart := time.Now()\n                \/\/ \u5177\u4f53\u6267\u884c\n\t\tg.Eval(ctx, evalTimestamp)\n\t\ttimeSinceStart := time.Since(start)\n\n                \/\/ \u6267\u884c\u8fc7\u7a0b\u7684\u6570\u636e\u7edf\u8ba1\n\t\tg.metrics.iterationDuration.Observe(timeSinceStart.Seconds())\n\t\tg.setEvaluationDuration(timeSinceStart)\n\t\tg.setEvaluationTimestamp(start)\n\t}\n        \/\/ .....\n\n\t\/\/ \u4e0b\u9762\u5b9a\u65f6\u5faa\u73af\u6267\u884c\u89c4\u5219\n\tfor {\n\t\tselect {\n\t\tcase <-g.done:\n\t\t\treturn\n\t\tdefault:\n\t\t\tselect {\n\t\t\tcase <-g.done:\n\t\t\t\treturn\n\t\t\tcase <-tick.C:\n\t\t\t\tmissed := (time.Since(evalTimestamp) \/ g.interval) - 1\n\t\t\t\tif missed > 0 {\n\t\t\t\t\tg.metrics.iterationsMissed.Add(float64(missed))\n\t\t\t\t\tg.metrics.iterationsScheduled.Add(float64(missed))\n\t\t\t\t}\n\t\t\t\tevalTimestamp = evalTimestamp.Add((missed + 1) * g.interval)\n\t\t\t\titer()\n\t\t\t}\n\t\t}\n\t}\n}<\/pre>\n
    \u89c4\u5219\u7ec4\u7684\u5177\u4f53\u6267\u884c\u6d41\u7a0b\uff1a<\/p>\n
    for i, rule := range g.rules {\n\t\t\/\/ \u904d\u5386\u89c4\u5219\u7ec4\u4e2d\u7684\u6240\u6709\u89c4\u5219\uff0c\u4f9d\u6b21\u6267\u884c\n\t\tfunc(i int, rule Rule) {\n\t\t\t\/\/ .......\n\t\t\tvector, err := rule.Eval(ctx, ts, g.opts.QueryFunc, g.opts.ExternalURL)\n\t\t\tif err != nil {\n\t\t\t\t\/\/ Canceled queries are intentional termination of queries. This normally\n\t\t\t\t\/\/ happens on shutdown and thus we skip logging of any errors here.\n\t\t\t\tif _, ok := err.(promql.ErrQueryCanceled); !ok {\n\t\t\t\t\tlevel.Warn(g.logger).Log(\"msg\", \"Evaluating rule failed\", \"rule\", rule, \"err\", err)\n\t\t\t\t}\n\t\t\t\t\/\/ .......\n\t\t\t\treturn\n\t\t\t}\n\t\t\t\n\t\t\t\/\/ \u5224\u65ad\u89c4\u5219\u662f\u5426\u662falert\u89c4\u5219\uff0c\u5982\u679c\u662f\u5219\u53d1\u9001\u544a\u8b66\u4fe1\u606f\uff08\u5177\u4f53\u662f\u5426\u771f\u6b63\u53d1\u9001\u7531ar.sendAlerts\u4e2d\u7684\u903b\u8f91\u5224\u65ad\uff09\n\t\t\tif ar, ok := rule.(*AlertingRule); ok {\n\t\t\t\tar.sendAlerts(ctx, ts, g.opts.ResendDelay, g.interval, g.opts.NotifyFunc)\n\t\t\t}\n\t\t}\n\t\t\/\/ ........\n\n}<\/pre>\n
    \u7136\u540e\u5c31\u662f\u89c4\u5219\u7684\u5177\u4f53\u6267\u884c\u4e86\uff0c\u6211\u4eec\u8fd9\u91cc\u5148\u53ea\u770bAlertingRule\u7684\u6d41\u7a0b\u3002\u9996\u5148\u770b\u4e0bAlertingRule\u7684\u7ed3\u6784\uff1a<\/p>\n
    \/\/ An AlertingRule generates alerts from its vector expression.\ntype AlertingRule struct {\n\t\/\/ The name of the alert.\n\tname string\n\t\/\/ The vector expression from which to generate alerts.\n\tvector parser.Expr\n\t\/\/ The duration for which a labelset needs to persist in the expression\n\t\/\/ output vector before an alert transitions from Pending to Firing state.\n\tholdDuration time.Duration\n\t\/\/ Extra labels to attach to the resulting alert sample vectors.\n\tlabels labels.Labels\n\t\/\/ Non-identifying key\/value pairs.\n\tannotations labels.Labels\n\t\/\/ External labels from the global config.\n\texternalLabels map[string]string\n\t\/\/ true if old state has been restored. We start persisting samples for ALERT_FOR_STATE\n\t\/\/ only after the restoration.\n\trestored bool\n\t\/\/ Protects the below.\n\tmtx sync.Mutex\n\t\/\/ Time in seconds taken to evaluate rule.\n\tevaluationDuration time.Duration\n\t\/\/ Timestamp of last evaluation of rule.\n\tevaluationTimestamp time.Time\n\t\/\/ The health of the alerting rule.\n\thealth RuleHealth\n\t\/\/ The last error seen by the alerting rule.\n\tlastError error\n\t\/\/ A map of alerts which are currently active (Pending or Firing), keyed by\n\t\/\/ the fingerprint of the labelset they correspond to.\n\tactive map[uint64]*Alert\n\n\tlogger log.Logger\n}<\/pre>\n
    \u8fd9\u91cc\u6bd4\u8f83\u91cd\u8981\u7684\u5c31\u662factive\u5b57\u6bb5\u4e86\uff0c\u5b83\u4fdd\u5b58\u4e86\u6267\u884c\u89c4\u5219\u540e\u9700\u8981\u8fdb\u884c\u544a\u8b66\u7684\u8d44\u6e90\uff0c\u5177\u4f53\u662f\u5426\u544a\u8b66\u8fd8\u8981\u6267\u884c\u4e00\u7cfb\u5217\u7684\u903b\u8f91\u6765\u5224\u65ad\u662f\u5426\u6ee1\u8db3\u544a\u8b66\u6761\u4ef6\u3002\u5177\u4f53\u6267\u884c\u7684\u903b\u8f91\u5982\u4e0b\uff1a<\/p>\n
    func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, externalURL *url.URL) (promql.Vector, error) {\n\tres, err := query(ctx, r.vector.String(), ts)\n\tif err != nil {\n\t\tr.SetHealth(HealthBad)\n\t\tr.SetLastError(err)\n\t\treturn nil, err\n\t}\n\n\t\/\/ ......\n}<\/pre>\n
    \u8fd9\u4e00\u6b65\u901a\u8fc7\u521b\u5efaManager\u65f6\u4f20\u5165\u7684QueryFunc\u51fd\u6570\u6267\u884c\u89c4\u5219\u914d\u7f6e\u4e2d\u7684expr\u8868\u8fbe\u5f0f\uff0c\u7136\u540e\u5f97\u5230\u8fd4\u56de\u7684\u7ed3\u679c\uff0c\u8fd9\u91cc\u7684\u7ed3\u679c\u662f\u6ee1\u8db3\u8868\u8fbe\u5f0f\u7684\u6307\u6807\u7684\u96c6\u5408\u3002<\/p>\n
    \u6bd4\u5982\u914d\u7f6e\u7684\u89c4\u5219\u4e3a\uff1a<\/p>\n
    cpu_usage > 80<\/pre>\n
    \u90a3\u4e48\u67e5\u51fa\u6765\u7684\u7ed3\u679c\u53ef\u80fd\u662f<\/p>\n
    cpu_usage{instance=\"192.168.0.100\"} 85\ncpu_usage{instance=\"192.168.0.101\"} 92<\/pre>\n
    \u7136\u540e\u904d\u5386\u67e5\u8be2\u5230\u7684\u7ed3\u679c\uff0c\u6839\u636e\u6307\u6807\u7684\u6807\u7b7e\u751f\u6210\u4e00\u4e2ahash\u503c\uff0c\u7136\u540e\u5224\u65ad\u8fd9\u4e2ahash\u503c\u662f\u5426\u4e4b\u524d\u5df2\u7ecf\u5b58\u5728\uff08\u5373\u4e4b\u524d\u662f\u5426\u5df2\u7ecf\u6709\u76f8\u540c\u7684\u6307\u6807\u6570\u636e\u8fd4\u56de\uff09\uff0c\u5982\u679c\u662f\uff0c\u5219\u66f4\u65b0\u4e0a\u6b21\u7684value\u53caannotations\uff0c\u5982\u679c\u4e0d\u662f\uff0c\u5219\u521b\u5efa\u4e00\u4e2a\u65b0\u7684alert\u5e76\u4fdd\u5b58\u81f3\u8be5\u89c4\u5219\u4e0b\u7684active alert\u5217\u8868\u4e2d\u3002<\/p>\n
    \u7136\u540e\u904d\u5386\u89c4\u5219\u7684active alert\u5217\u8868\uff0c\u6839\u636e\u89c4\u5219\u7684\u6301\u7eed\u65f6\u957f\u914d\u7f6e\u3001alert\u7684\u4e0a\u6b21\u89e6\u53d1\u65f6\u95f4\u3001alert\u7684\u5f53\u524d\u72b6\u6001\u3001\u672c\u6b21\u67e5\u8be2alert\u662f\u5426\u4f9d\u7136\u5b58\u5728\u7b49\u4fe1\u606f\u6765\u4fee\u6539alert\u7684\u72b6\u6001\u3002\u5177\u4f53\u89c4\u5219\u5982\u4e0b\uff1a<\/p>\n
      \n
    1. \u5982\u679calert\u4e4b\u524d\u5b58\u5728\uff0c\u4f46\u672c\u6b21\u6267\u884c\u65f6\u4e0d\u5b58\u5728\n
        \n
      1. \u72b6\u6001\u662fStatePending\u6216\u8005\u672c\u6b21\u68c0\u67e5\u65f6\u95f4\u8ddd\u79bb\u4e0a\u6b21\u89e6\u53d1\u65f6\u95f4\u8d85\u8fc715\u5206\u949f\uff0815\u5206\u949f\u4e3a\u5199\u6b7b\u7684\u5e38\u91cf\uff09\uff0c\u5219\u5c06\u8be5alert\u4eceactive\u5217\u8868\u4e2d\u5220\u9664<\/li>\n
      2. \u72b6\u6001\u4e0d\u4e3aStateInactive\u7684alert\u4fee\u6539\u4e3aStateInactive<\/li>\n<\/ol>\n<\/li>\n
      3. \u5982\u679calert\u4e4b\u524d\u5b58\u5728\u5e76\u4e14\u672c\u6b21\u6267\u884c\u4ecd\u7136\u5b58\u5728\n
          \n
        1. alert\u7684\u72b6\u6001\u662fStatePending\u5e76\u4e14\u672c\u6b21\u68c0\u67e5\u8ddd\u79bb\u4e0a\u6b21\u89e6\u53d1\u65f6\u95f4\u8d85\u8fc7\u914d\u7f6e\u7684for\u6301\u7eed\u65f6\u957f\uff0c\u90a3\u4e48\u72b6\u6001\u4fee\u6539\u4e3aStateFiring<\/li>\n<\/ol>\n<\/li>\n
        2. \u5176\u4f59\u60c5\u51b5\u4fee\u6539alert\u7684\u72b6\u6001\u4e3aStatePending<\/li>\n<\/ol>\n
          \u4e0a\u9762\u90a3\u4e00\u6b65\u53ea\u662f\u4fee\u6539\u4e86alert\u7684\u72b6\u6001\uff0c\u4f46\u662f\u5e76\u6ca1\u6709\u771f\u6b63\u6267\u884c\u53d1\u9001\u544a\u8b66\u64cd\u4f5c\u3002\u4e0b\u9762\u624d\u662f\u771f\u6b63\u8981\u6267\u884c\u544a\u8b66\u64cd\u4f5c\uff1a<\/p>\n
          \/\/ \u5224\u65ad\u89c4\u5219\u662f\u5426\u662falert\u89c4\u5219\uff0c\u5982\u679c\u662f\u5219\u53d1\u9001\u544a\u8b66\u4fe1\u606f\uff08\u5177\u4f53\u662f\u5426\u771f\u6b63\u53d1\u9001\u7531ar.sendAlerts\u4e2d\u7684\u903b\u8f91\u5224\u65ad\uff09\nif ar, ok := rule.(*AlertingRule); ok {\n\tar.sendAlerts(ctx, ts, g.opts.ResendDelay, g.interval, g.opts.NotifyFunc)\n}\n\n\/\/ .......\n\nfunc (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay time.Duration, interval time.Duration, notifyFunc NotifyFunc) {\n\talerts := []*Alert{}\n\tr.ForEachActiveAlert(func(alert *Alert) {\n\t\tif alert.needsSending(ts, resendDelay) {\n\t\t\talert.LastSentAt = ts\n\t\t\t\/\/ Allow for two Eval or Alertmanager send failures.\n\t\t\tdelta := resendDelay\n\t\t\tif interval > resendDelay {\n\t\t\t\tdelta = interval\n\t\t\t}\n\t\t\talert.ValidUntil = ts.Add(4 * delta)\n\t\t\tanew := *alert\n\t\t\talerts = append(alerts, &anew)\n\t\t}\n\t})\n\tnotifyFunc(ctx, r.vector.String(), alerts...)\n}\n\nfunc (a *Alert) needsSending(ts time.Time, resendDelay time.Duration) bool {\n\tif a.State == StatePending {\n\t\treturn false\n\t}\n\n\t\/\/ if an alert has been resolved since the last send, resend it\n\tif a.ResolvedAt.After(a.LastSentAt) {\n\t\treturn true\n\t}\n\n\treturn a.LastSentAt.Add(resendDelay).Before(ts)\n}<\/pre>\n
          \u6982\u62ec\u4e00\u4e0b\u4ee5\u4e0a\u903b\u8f91\u5c31\u662f\uff1a<\/p>\n
            \n
          1. \u5982\u679calert\u7684\u72b6\u6001\u662fStatePending\uff0c\u5219\u4e0d\u53d1\u9001\u544a\u8b66<\/li>\n
          2. \u5982\u679calert\u7684\u5df2\u7ecf\u88ab\u89e3\u51b3\uff0c\u90a3\u4e48\u518d\u6b21\u53d1\u9001\u544a\u8b66\u6807\u8bc6\u8be5\u6761\u4fe1\u606f\u5df2\u7ecf\u88ab\u89e3\u51b3<\/li>\n
          3. \u5982\u679c\u5f53\u524d\u65f6\u95f4\u8ddd\u79bb\u4e0a\u6b21\u53d1\u9001\u544a\u8b66\u7684\u65f6\u95f4\u5927\u4e8e\u914d\u7f6e\u7684\u91cd\u65b0\u53d1\u9001\u5ef6\u65f6\u65f6\u95f4\uff08ResendDelay\uff09\uff0c\u5219\u53d1\u9001\u544a\u8b66\uff0c\u5426\u5219\u4e0d\u53d1\u9001<\/li>\n<\/ol>\n
            \n
            \u4ee5\u4e0a\u5927\u81f4\u4ecb\u7ecd\u4e86\u4ece\u52a0\u8f7d\u544a\u8b66\u89c4\u5219\u914d\u7f6e\u5230\u6700\u540e\u6267\u884c\u544a\u8b66\u7684\u5927\u81f4\u903b\u8f91\uff0c\u540e\u7eed\u4f1a\u8be6\u7ec6\u4ecb\u7ecd\u5177\u4f53\u7684\u4e00\u4e9b\u7ec6\u8282\uff0c\u656c\u8bf7\u5173\u6ce8\uff01<\/p>\n","protected":false},"excerpt":{"rendered":"
            \u672c\u6587\u6839\u636eprometheus master\u5206\u652f\u6700\u65b0\u4ee3\u7801\u6240\u5199\uff0ccommit id\uff1adca84112a97ea7a31f2ddb2ce7cfb4f7cae91f86 \u544a\u8b66\u662fprometheus\u7684\u4e00\u4e2a\u91cd\u8981\u529f\u80fd\uff0c\u63a5\u4e0b\u6765\u4ece\u6e90\u7801\u7684\u89d2\u5ea6\u6765\u5206\u6790\u4e0b\u544a\u8b66\u7684\u6267\u884c\u6d41\u7a0b\u3002 \u6574\u4f53\u7684\u5927\u81f4\u6d41\u7a0b\u8bf7\u89c1\u4e0b\u65b9\u6d41\u7a0b\u56fe\uff1a prometheus\u7684\u544a\u8b66\u90e8\u5206\u7684\u6e90\u7801\u4e3b\u8981\u5728github.co\u2026\u2026<\/p>\n","protected":false},"author":1,"featured_media":130,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[18],"tags":[19,20],"_links":{"self":[{"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/posts\/127"}],"collection":[{"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=127"}],"version-history":[{"count":1,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/posts\/127\/revisions"}],"predecessor-version":[{"id":136,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/posts\/127\/revisions\/136"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=\/wp\/v2\/media\/130"}],"wp:attachment":[{"href":"https:\/\/chenqinghe.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=127"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=127"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/chenqinghe.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=127"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}