远程存储 - 应用实时监控服务ARMS

阿里云Prometheus的RemoteWrite功能支持作为远程数据库存储Prometheus监控数据。本文将介绍如何使用阿里云Prometheus的RemoteWrite对接自建Prometheus，构建监控数据的高效存储方案。

（可选）步骤一：为RAM用户授予GetPrometheusApiToken接口调用权限

自建Prometheus写入阿里云Prometheus监控时需要调用GetPrometheusApiToken接口。阿里云账号（主账号）默认支持调用GetPrometheusApiToken接口。如果您是RAM用户（子账号），则需要先使用阿里云账号为RAM用户授予GetPrometheusApiToken接口的调用权限。

使用阿里云账号（主账号）登录RAM控制台。
在左侧导航栏选择权限管理 > 权限策略。
在权限策略页面单击创建权限策略。

在新建自定义权限策略页面自定义策略名称，设置配置模式为脚本配置，然后在策略内容区域输入以下内容。

{
    "Version": "1",
    "Statement": [
        {
            "Action": [
                "arms:GetPrometheusApiToken"
            ],
            "Resource": [
                "*"
            ],
            "Effect": "Allow"
        }
    ]
}

设置完成后单击确定。
在左侧导航栏选择身份管理 > 用户。
在用户页面，单击目标RAM用户右侧的添加权限。
在添加权限面板的选择权限区域，单击自定义策略，然后通过搜索查找步骤4创建的权限策略，单击权限策略名称将权限策略添加至右侧已选择区域，然后单击确定。

步骤二：创建RemoteWrite并获取读写URL

登录ARMS控制台。
在左侧导航栏单击Prometheus监控。
在页面左上角选择目标地域，在Prometheus监控页面单击接入Prometheus监控。
在接入Prometheus监控页面单击Remote Write Prometheus。
在第1步向导页输入RemoteWrite 名字，然后单击下一步。
在步骤2向导页复制并保存生成的Prometheus Remote Write Url，然后单击下一步。
在步骤3向导页复制并保存生成的Prometheus Remote Read Url，然后关闭该向导页。

步骤三：配置Prometheus

安装Prometheus，安装方法可以参考官方文档。

打开Prometheus.yaml配置文件，并在文件末尾增加以下内容，将remote_write和remote_read链接替换为步骤二：创建RemoteWrite并获取读写URL中获取的URL，然后保存文件。

global:
  scrape_interval:     15s
  evaluation_interval: 15s
scrape_configs:
  - job_name: 'prometheus'
    static_configs:
    - targets: ['localhost:9090']
remote_write:
  - url: "http://ts-xxxxxxxxxxxx.hitsdb.rds.aliyuncs.com:3242/api/prom_write"
    basic_auth:   
      //Username和Password需要遵循AK/SK的限制，且AK/SK需要能够调用GetPrometheusApiToken。
      username: access-key-id
      password: access-key-secret
remote_read:
  - url: "http://ts-xxxxxxxxxxxx.hitsdb.rds.aliyuncs.com:3242/api/prom_read"
    read_recent: true

说明自建Prometheus写入阿里云Prometheus监控时需要调用GetPrometheusApiToken接口。阿里云账号（主账号）默认支持调用GetPrometheusApiToken接口。如果您是RAM用户（子账号），则需要先使用阿里云账号为RAM用户授予GetPrometheusApiToken接口的调用权限。具体操作，请参见步骤一。

使用远程存储对接OpenTelemetry

业务代码进行OpenTelemetry埋点。[Demo]

package stat

import (
    "context"
    "fmt"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/metric"
    "time"
)

var buyCounter metric.Int64Counter

func init() {
    fmt.Println(time.Now(), " - initMetrics start......")
    meter := otel.GetMeterProvider().Meter("github.com/liguozhong/prometheus-arms-aliyun-go-demo")
    buyCounter = metric.Must(meter).NewInt64Counter(
        "buy_total",
        metric.WithDescription("Measures  buy"),
    )
}

func DoBuy() (string, error) {
    buyCounter.Add(context.Background(), 1)
    return "buy success", nil
}

OpenTelemetry对Meter进行初始化并和连接Prometheus。[Demo]

package stat

import (
    "context"
    prometheusPushExporter "go.opentelemetry.io/contrib/exporters/metric/cortex"
    prometheusExporter "go.opentelemetry.io/otel/exporters/metric/prometheus"

    "errors"
    "fmt"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp"
    "go.opentelemetry.io/otel/label"
    "go.opentelemetry.io/otel/sdk/metric/controller/pull"
    "go.opentelemetry.io/otel/sdk/metric/controller/push"
    "go.opentelemetry.io/otel/sdk/metric/processor/basic"
    "go.opentelemetry.io/otel/sdk/metric/selector/simple"
    "go.opentelemetry.io/otel/sdk/resource"
    "net/http"
    "time"

    _ "net/http/pprof"
)

func InitMeter(app string, push bool) error {
    fmt.Println(time.Now(), " - initMeter start......")
    if push {
        fmt.Println(time.Now(), " - initMeter opentelemetry push......")
        remoteUrl := "http://region.arms.aliyuncs.com/prometheus/../../../../api/v3/write"
        ak := "ak"
        sk := "sk"
        return initPushMeter(app, remoteUrl, ak, sk)
    }
    fmt.Println(time.Now(), " - initMeter opentelemetry pull......")
    return initPullMeter(app)
}

func initPushMeter(regionId string, remoteWriteUrl string, ak string, sk string) error {
    fmt.Println(time.Now(), " - initPushMeter start......")
    var validatedStandardConfig = prometheusPushExporter.Config{
        Endpoint:      remoteWriteUrl,
        Name:          "AliyunConfig",
        RemoteTimeout: 30 * time.Second,
        PushInterval:  10 * time.Second,
        Quantiles:     []float64{0.5, 0.9, 0.95, 0.99},
        BasicAuth: map[string]string{
            "username": ak,
            "password": sk,
        },
    }
    if validatedStandardConfig.Endpoint == "" {
        return errors.New(" validatedStandardConfig.Endpoint==empty.regionId:" + regionId)
    }
    fmt.Println("Success: Created Config struct")
    r, err := resource.New(context.Background(),
        resource.WithAttributes(
            label.String("cluster", "test-otel"),
            label.String("app", "buy")))
    if err != nil {
        fmt.Println("resource Error:", err)
    }
    pusher, err := prometheusPushExporter.InstallNewPipeline(validatedStandardConfig,
        push.WithPeriod(30*time.Second), push.WithResource(r))
    if err != nil {
        fmt.Println("InstallNewPipeline Error:", err)
    }
    otel.SetMeterProvider(pusher.MeterProvider())
    return nil
}

func initPullMeter(app string) error {
    fmt.Println(time.Now(), " - initPullMeter start......")
    r, err := resource.New(context.Background(),
        resource.WithAttributes(
            label.String("cluster", "test-otel"),
            label.String("app", app)))
    if err != nil {
        fmt.Println("resource Error:", err)
    }
    exporter, err := prometheusExporter.NewExportPipeline(
        prometheusExporter.Config{
            DefaultHistogramBoundaries: []float64{-0.5, 1},
        },
        pull.WithCachePeriod(0),
        pull.WithResource(r),
    )
    if err != nil {
        return err
    }
    http.HandleFunc("/opentelemetry", exporter.ServeHTTP)
    otel.SetMeterProvider(exporter.MeterProvider())
    return nil
}

func initOtlpProvider(regionId string) (*push.Controller, error) {
    exporter, err := otlp.NewExporter(
        context.Background(),
        otlp.WithInsecure(),
        otlp.WithAddress(regionId+"-intranet.arms.aliyuncs.com:8000"),
    )
    if err != nil {
        return nil, err
    }

    pusher := push.New(
        basic.New(
            simple.NewWithExactDistribution(),
            exporter,
        ),
        exporter,
        push.WithPeriod(30*time.Second),
    )

    otel.SetMeterProvider(pusher.MeterProvider())
    pusher.Start()

    return pusher, err
}

业务入口初始化，并启动OpenTelemetry功能。[Demo]

package pkg

import (
    "fmt"
    stat "github.com/liguozhong/prometheus-arms-aliyun-go-demo/pkg/opentelemetry"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "io"
    "net/http"
    "strconv"
)

type Server struct {
    port int
}

func NewServer(port int) *Server {
    return &Server{
        port: port,
    }
}

func (s *Server) Run() error {
    port := ":" + strconv.Itoa(s.port)
    path := "/metrics"
    service := "/buy"
    http.Handle(path, promhttp.Handler()) //初始一个http handler
    http.HandleFunc(service, func(writer http.ResponseWriter, request *http.Request) {
        content, err := stat.DoBuy()
        if err != nil {
            io.WriteString(writer, err.Error())
            return
        }
        io.WriteString(writer, content)
    })
    stat.InitMeter("buy2", true)
    fmt.Println("http.url: http://localhost" + port + path)
    fmt.Println("service.url: http://localhost" + port + service)
    err := http.ListenAndServe(port, nil)
    if err != nil {
        return err
    }
    return nil
}

查看Go module的依赖列表。[Demo]

module github.com/liguozhong/prometheus-arms-aliyun-go-demo

go 1.12

require (
    github.com/go-kit/kit v0.9.0
    github.com/prometheus/client_golang v1.7.1
    go.opentelemetry.io/contrib/exporters/metric/cortex v0.15.0
    go.opentelemetry.io/otel v0.15.0
    go.opentelemetry.io/otel/exporters/metric/prometheus v0.15.0
    go.opentelemetry.io/otel/exporters/otlp v0.15.0
    go.opentelemetry.io/otel/sdk v0.15.0
    golang.org/x/text v0.3.3 // indirect
)

在Grafana中查看数据。