远程存储 - 应用实时监控服务ARMS

Prometheus监控支持作为远程数据库存储开源Prometheus监控数据。本文介绍如何使用Remote Write功能将开源Prometheus接入Prometheus监控，从而构建监控数据的高效存储方案。

前提条件

已开通ARMS。具体操作，请参见开通和升级ARMS。

步骤一：创建RemoteWrite并获取读写URL

登录ARMS控制台。
在左侧导航栏单击Prometheus监控。
在Prometheus监控页面的顶部菜单栏，选择地域，然后单击接入Prometheus监控。
在接入Prometheus监控页面单击Remote Write Prometheus。
在配置向导对话框，完成以下操作：
1. 在第1步页签下，输入实例名称，然后单击下一步。
2. 在第2步页签下，复制并保存Remote Write地址和Remote Read地址。

步骤二：配置Prometheus

安装Prometheus，安装方法可以参考官方文档。

打开Prometheus.yaml配置文件，并在文件末尾增加以下内容，将remote_write和remote_read链接替换为步骤一：创建RemoteWrite并获取读写URL中获取的URL，然后保存文件。

global:
  scrape_interval:     15s
  evaluation_interval: 15s
scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ['localhost:9090']
remote_write:
  - url: "http://${regionId}-intranet.arms.aliyuncs.com/prometheus/${password}/${userId}/${promCluster}/${regionId}/api/v3/write"
    basic_auth:   
      //username和password需要遵循AK/SK的限制，且AK/SK需要能够调用GetPrometheusApiToken。
      username: access-key-id
      password: access-key-secret
remote_read:
  - url: "http://${regionId}-intranet.arms.aliyuncs.com/prometheus/${password}/${userId}/${promCluster}/${regionId}/api/v3/read"
    read_recent: true

使用远程存储对接OpenTelemetry

业务代码进行OpenTelemetry埋点。[Demo]

package stat

import (
    "context"
    "fmt"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/metric"
    "time"
)

var buyCounter metric.Int64Counter

func init() {
    fmt.Println(time.Now(), " - initMetrics start......")
    meter := otel.GetMeterProvider().Meter("github.com/liguozhong/prometheus-arms-aliyun-go-demo")
    buyCounter = metric.Must(meter).NewInt64Counter(
        "buy_total",
        metric.WithDescription("Measures  buy"),
    )
}

func DoBuy() (string, error) {
    buyCounter.Add(context.Background(), 1)
    return "buy success", nil
}

OpenTelemetry对Meter进行初始化并和连接Prometheus。[Demo]

package stat

import (
    "context"
    prometheusPushExporter "go.opentelemetry.io/contrib/exporters/metric/cortex"
    prometheusExporter "go.opentelemetry.io/otel/exporters/metric/prometheus"

    "errors"
    "fmt"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp"
    "go.opentelemetry.io/otel/label"
    "go.opentelemetry.io/otel/sdk/metric/controller/pull"
    "go.opentelemetry.io/otel/sdk/metric/controller/push"
    "go.opentelemetry.io/otel/sdk/metric/processor/basic"
    "go.opentelemetry.io/otel/sdk/metric/selector/simple"
    "go.opentelemetry.io/otel/sdk/resource"
    "net/http"
    "time"

    _ "net/http/pprof"
)

func InitMeter(app string, push bool) error {
    fmt.Println(time.Now(), " - initMeter start......")
    if push {
        fmt.Println(time.Now(), " - initMeter opentelemetry push......")
        remoteUrl := "http://region.arms.aliyuncs.com/prometheus/../../../../api/v3/write"
        ak := "ak"
        sk := "sk"
        return initPushMeter(app, remoteUrl, ak, sk)
    }
    fmt.Println(time.Now(), " - initMeter opentelemetry pull......")
    return initPullMeter(app)
}

func initPushMeter(regionId string, remoteWriteUrl string, ak string, sk string) error {
    fmt.Println(time.Now(), " - initPushMeter start......")
    var validatedStandardConfig = prometheusPushExporter.Config{
        Endpoint:      remoteWriteUrl,
        Name:          "AliyunConfig",
        RemoteTimeout: 30 * time.Second,
        PushInterval:  10 * time.Second,
        Quantiles:     []float64{0.5, 0.9, 0.95, 0.99},
        BasicAuth: map[string]string{
            "username": ak,
            "password": sk,
        },
    }
    if validatedStandardConfig.Endpoint == "" {
        return errors.New(" validatedStandardConfig.Endpoint==empty.regionId:" + regionId)
    }
    fmt.Println("Success: Created Config struct")
    r, err := resource.New(context.Background(),
        resource.WithAttributes(
            label.String("cluster", "test-otel"),
            label.String("app", "buy")))
    if err != nil {
        fmt.Println("resource Error:", err)
    }
    pusher, err := prometheusPushExporter.InstallNewPipeline(validatedStandardConfig,
        push.WithPeriod(30*time.Second), push.WithResource(r))
    if err != nil {
        fmt.Println("InstallNewPipeline Error:", err)
    }
    otel.SetMeterProvider(pusher.MeterProvider())
    return nil
}

func initPullMeter(app string) error {
    fmt.Println(time.Now(), " - initPullMeter start......")
    r, err := resource.New(context.Background(),
        resource.WithAttributes(
            label.String("cluster", "test-otel"),
            label.String("app", app)))
    if err != nil {
        fmt.Println("resource Error:", err)
    }
    exporter, err := prometheusExporter.NewExportPipeline(
        prometheusExporter.Config{
            DefaultHistogramBoundaries: []float64{-0.5, 1},
        },
        pull.WithCachePeriod(0),
        pull.WithResource(r),
    )
    if err != nil {
        return err
    }
    http.HandleFunc("/opentelemetry", exporter.ServeHTTP)
    otel.SetMeterProvider(exporter.MeterProvider())
    return nil
}

func initOtlpProvider(regionId string) (*push.Controller, error) {
    exporter, err := otlp.NewExporter(
        context.Background(),
        otlp.WithInsecure(),
        otlp.WithAddress(regionId+"-intranet.arms.aliyuncs.com:8000"),
    )
    if err != nil {
        return nil, err
    }

    pusher := push.New(
        basic.New(
            simple.NewWithExactDistribution(),
            exporter,
        ),
        exporter,
        push.WithPeriod(30*time.Second),
    )

    otel.SetMeterProvider(pusher.MeterProvider())
    pusher.Start()

    return pusher, err
}

业务入口初始化，并启动OpenTelemetry功能。[Demo]

package pkg

import (
    "fmt"
    stat "github.com/liguozhong/prometheus-arms-aliyun-go-demo/pkg/opentelemetry"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "io"
    "net/http"
    "strconv"
)

type Server struct {
    port int
}

func NewServer(port int) *Server {
    return &Server{
        port: port,
    }
}

func (s *Server) Run() error {
    port := ":" + strconv.Itoa(s.port)
    path := "/metrics"
    service := "/buy"
    http.Handle(path, promhttp.Handler()) //初始一个http handler
    http.HandleFunc(service, func(writer http.ResponseWriter, request *http.Request) {
        content, err := stat.DoBuy()
        if err != nil {
            io.WriteString(writer, err.Error())
            return
        }
        io.WriteString(writer, content)
    })
    stat.InitMeter("buy2", true)
    fmt.Println("http.url: http://localhost" + port + path)
    fmt.Println("service.url: http://localhost" + port + service)
    err := http.ListenAndServe(port, nil)
    if err != nil {
        return err
    }
    return nil
}

查看Go module的依赖列表。[Demo]

module github.com/liguozhong/prometheus-arms-aliyun-go-demo

go 1.12

require (
    github.com/go-kit/kit v0.9.0
    github.com/prometheus/client_golang v1.7.1
    go.opentelemetry.io/contrib/exporters/metric/cortex v0.15.0
    go.opentelemetry.io/otel v0.15.0
    go.opentelemetry.io/otel/exporters/metric/prometheus v0.15.0
    go.opentelemetry.io/otel/exporters/otlp v0.15.0
    go.opentelemetry.io/otel/sdk v0.15.0
    golang.org/x/text v0.3.3 // indirect
)

在Grafana中查看数据。