Initial commit

2020-04-06 19:57:19 +08:00
commit 2fc86fd0a1
42 changed files with 4056 additions and 0 deletions
--- a/metrics/example1/example1.go
+++ b/metrics/example1/example1.go
@@ -0,0 +1,197 @@
+package main
+
+import (
+	"flag"
+	"globalfintech/golib/metrics"
+	"globalfintech/golib/metrics/prometheus"
+	"globalfintech/golib/tlog"
+	"math/rand"
+	"os"
+	"os/signal"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/context"
+)
+
+func main() {
+	// parse the flags
+	flag.Parse()
+
+	processMetrics, err := metrics.NewProcessMetrics("test", "INDIA", "testProject")
+	if err != nil {
+		tlog.Error(err)
+		return
+	}
+	workPort := processMetrics.GetMetricsPort()
+	// init Exporter
+	tlog.Info("Beginning to serve on port :", workPort)
+	processMetrics.InitExporter()
+
+	// You can registry metrics any where as you want
+	// Now we registry a local gauge in function main
+	workersGauge := prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+		Namespace: "worker",
+		Subsystem: "jobs",
+		Name:      "worker_number",
+		Help:      "Number of workers.",
+	}, []string{"server"})
+
+	host, _ := os.Hostname()
+	workersGauge.With("server", host).Set(float64(workers))
+
+	// create a channel with a 1000 Job buffer
+	jobChannel := make(chan *Job, 1000)
+	// start the job processor
+	go startJobProcessor(jobChannel)
+	// start a Goroutine to create some mock jobs
+	go createJobs(jobChannel)
+
+	exit := make(chan os.Signal, 1)
+
+	// kill (no param) default send syscanll.SIGTERM
+	// kill -2 is syscall.SIGINT
+	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it
+	signal.Notify(exit, syscall.SIGTERM, syscall.SIGINT)
+	<-exit
+
+	tlog.Info("Graceful shutdown Server in 10 seconds...")
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// set workers to 0 , then stop all workers and job creator
+	workers = 0
+
+	select {
+	case <-ctx.Done():
+		tlog.Info("timeout of 10 seconds.")
+	}
+
+	// init pusher
+	// pusher usually uses at DAG/Cron job processor
+	pusher := metrics.InitPusher("http://ops-dev.a.mobimagic.com:9091", "MetricsExample1")
+	if err := pusher.Collector(workerTotalCounter.Cv).Add(); err != nil {
+		tlog.Error("Could not push to Pushgateway:", err)
+	}
+	// time is up
+	close(jobChannel)
+	tlog.Infof("Jobs left: %d", len(jobChannel))
+	for leftJob := range jobChannel {
+		tlog.Info(leftJob)
+	}
+
+	tlog.Info("Server exit")
+
+}
+
+var (
+	modes   = []string{"SMS", "WeChat", "WhatsApp", "GooglePush", "ApplePush"}
+	workers = 0
+
+	// Two ways to define name
+	// Name : fullname
+	// or Namespace / Subsystem / Name to combine a full name
+	workerTotalCounter = prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+		Name: "worker_jobs_processed_total",
+		Help: "Total number of jobs processed by the workers.",
+	}, []string{"worker_id", "mode"})
+
+	inqueueGauge = prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+		Namespace: "worker",
+		Subsystem: "jobs",
+		Name:      "inqueue",
+		Help:      "Number of jobs in queue.",
+	}, []string{"mode"})
+
+	processingTimeHistogram = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+		Name:    "worker_jobs_process_time_seconds",
+		Help:    "Amount of time spent processing jobs",
+		Buckets: []float64{1, 2, 3, 4, 5}, //if no set, then use DefBuckets : []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
+	}, []string{"worker_id", "mode"})
+)
+
+type Job struct {
+	Mode  string
+	Sleep time.Duration
+}
+
+func init() {
+	flag.IntVar(&workers, "workers", 10, "Number of workers to use")
+}
+
+func getMode() string {
+	return modes[rand.Int()%len(modes)]
+}
+
+// makeJob creates a new job with a random sleep time between 0.5s and 5s
+func makeJob() *Job {
+	return &Job{
+		Mode:  getMode(),
+		Sleep: time.Duration(rand.Int()%4500+500) * time.Millisecond,
+	}
+}
+
+func createJobs(jobs chan<- *Job) {
+	for {
+		if workers == 0 {
+			tlog.Info("Stop create jobs.")
+			return
+		}
+
+		// create a random job
+		job := makeJob()
+		// track the job in the inqueue tracker
+		inqueueGauge.With("mode", job.Mode).Add(1)
+		// send the job down the channel
+		jobs <- job
+		// don't pile up too quickly
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// creates a worker that pulls jobs from the job channel
+func startWorker(workerID int, jobs <-chan *Job) {
+	for {
+		select {
+		// read from the job channel
+		case job := <-jobs:
+			startTime := time.Now()
+			// decrement the in queue tracker
+			inqueueGauge.With("mode", job.Mode).Sub(1)
+
+			// fake processing the request
+			time.Sleep(job.Sleep)
+			tlog.Infof("[%d][%s] Processed job in %0.3f seconds", workerID, job.Mode, time.Now().Sub(startTime).Seconds())
+			// track the total number of jobs processed by the worker
+			workerTotalCounter.With("worker_id", strconv.FormatInt(int64(workerID), 10), "mode", job.Mode).Inc()
+			processingTimeHistogram.With("worker_id", strconv.FormatInt(int64(workerID), 10), "mode", job.Mode).Observe(time.Now().Sub(startTime).Seconds())
+
+			if workers == 0 {
+				tlog.Infof("worker[%d] quit", workerID)
+				return
+			}
+
+		}
+	}
+}
+
+func startJobProcessor(jobs <-chan *Job) {
+	tlog.Infof("[INFO] starting %d workers\n", workers)
+	wait := sync.WaitGroup{}
+	// notify the sync group we need to wait for 10 goroutines
+	wait.Add(workers)
+
+	// start 10 works
+	for i := 0; i < workers; i++ {
+		go func(workerID int) {
+			// start the worker
+			startWorker(workerID, jobs)
+			wait.Done()
+		}(i)
+	}
+
+	wait.Wait()
+}
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -0,0 +1,190 @@
+package metrics
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"globalfintech/golib/tlog"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	consulapi "github.com/hashicorp/consul/api"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/client_golang/prometheus/push"
+)
+
+const (
+	defaultAddr string = "9166"
+)
+
+// Counter describes a metric that accumulates values monotonically.
+// An example of a counter is the number of received HTTP requests.
+type Counter interface {
+	With(labelValues ...string) Counter
+	Add(delta float64)
+	Inc()
+}
+
+// Gauge describes a metric that takes specific values over time.
+// An example of a gauge is the current depth of a job queue.
+type Gauge interface {
+	With(labelValues ...string) Gauge
+	Set(value float64)
+	Add(delta float64)
+	Sub(delta float64)
+}
+
+// Histogram describes a metric that takes repeated observations of the same
+// kind of thing, and produces a statistical summary of those observations,
+// typically expressed as quantiles or buckets. An example of a histogram is
+// HTTP request latencies.
+type Histogram interface {
+	With(labelValues ...string) Histogram
+	Observe(value float64)
+}
+
+// LabelValues is a type alias that provides validation on its With method.
+// Metrics may include it as a member to help them satisfy With semantics and
+// save some code duplication.
+type LabelValues []string
+
+// With validates the input, and returns a new aggregate labelValues.
+func (lvs LabelValues) With(labelValues ...string) LabelValues {
+	if len(labelValues)%2 != 0 {
+		labelValues = append(labelValues, "unknown")
+	}
+	return append(lvs, labelValues...)
+}
+
+// new a *ProcessMetrics
+// productName,serviceRegion,appName all those params must be passed
+// port if not be pass,will use defaultAddr 9166
+func NewProcessMetrics(productName, serviceRegion, appName string, port ...string) (*ProcessMetrics, error) {
+	if len(productName) == 0 || len(serviceRegion) == 0 || len(appName) == 0 {
+		err := errors.New("productName,serviceRegion,appName all those params must be passed")
+		tlog.Error(err)
+		return nil, err
+	}
+	addr := defaultAddr
+	if len(port) > 0 {
+		addr = port[0]
+	}
+	var processMetrics ProcessMetrics
+	processMetrics.Port = addr
+	processMetrics.ProductName = productName
+	processMetrics.ServiceRegion = serviceRegion
+	processMetrics.Appname = appName
+	return &processMetrics, nil
+}
+
+type ProcessMetrics struct {
+	Port          string // port default 9166
+	ProductName   string
+	ServiceRegion string
+	Appname       string
+}
+
+// get real work port
+func (s *ProcessMetrics) GetMetricsPort() string {
+	if len(s.Port) > 0 {
+		return s.Port
+	}
+	return defaultAddr
+}
+
+// Init Export on GF default port 9166
+// If you want custom listen port, just pass your port to MetricsParams struct
+// return exporter http server
+func (s *ProcessMetrics) InitExporter() *http.Server {
+	h := http.NewServeMux()
+	h.Handle("/metrics", promhttp.Handler())
+	h.HandleFunc("/health_checker", s.healthChecker)
+	addr := defaultAddr
+	if len(s.Port) > 0 {
+		addr = s.Port
+	} else {
+		s.Port = defaultAddr
+	}
+	server := &http.Server{
+		Addr:              ":" + addr,
+		Handler:           h,
+		ReadTimeout:       30 * time.Second,
+		ReadHeaderTimeout: 30 * time.Second,
+		IdleTimeout:       30 * time.Second,
+		WriteTimeout:      10 * time.Second,
+	}
+
+	{
+		var tags []string
+
+		tags = append(tags, fmt.Sprintf("app=%s", s.ProductName))
+		tags = append(tags, fmt.Sprintf("country=%s", s.ServiceRegion))
+		s.initExporterOnExistHttpServer(addr, "/health_checker", tags, s.Appname)
+	}
+	go func() {
+		if err := server.ListenAndServe(); err != nil {
+			tlog.Fatal(err)
+		}
+	}()
+	return server
+}
+
+func (s *ProcessMetrics) healthChecker(resp http.ResponseWriter, req *http.Request) {
+	data := map[string]interface{}{
+		"code":        0,
+		"message":     "ok",
+		"server_time": time.Now().Unix(),
+	}
+	res, _ := json.Marshal(data)
+	resp.Write(res)
+}
+
+// Init Pusher
+// return prometheus pusher
+func InitPusher(gateway string, job string) *push.Pusher {
+	host, _ := os.Hostname()
+	return push.New(gateway, job).Grouping("instance", host)
+}
+
+// InitExporterOnExistHttpServer Init http exporter
+// Why we init exporter on exist http server:
+// to avoid new custom port
+// which should add healthCheckRouter - like /healthChecker func
+// must export a route: /metrics router for prometheus
+// promhttp.Handler().ServeHTTP(c.Ctx.ResponseWriter, c.Ctx.Request)
+func (s *ProcessMetrics) initExporterOnExistHttpServer(port string, healthCheckRouter string, tags []string, name string, id ...string) {
+	config := consulapi.DefaultConfig()
+	client, err := consulapi.NewClient(config)
+	if err != nil {
+		tlog.Error("consul client error : ", err)
+		return
+	}
+	registration := new(consulapi.AgentServiceRegistration)
+	registration.Name = name
+	intPort, _ := strconv.Atoi(port)
+	registration.Port = intPort
+	hostName, _ := os.Hostname()
+	if len(id) > 0 {
+		registration.ID = registration.Name + ":" + id[0]
+	} else {
+		registration.ID = registration.Name + ":" + hostName
+	}
+
+	registration.Tags = append(tags, "lang=go")
+
+	registration.Check = new(consulapi.AgentServiceCheck)
+	registration.Check.HTTP = fmt.Sprintf("http://%s:%s%s",
+		"127.0.0.1", port, healthCheckRouter)
+	registration.Check.Interval = "5s"
+	registration.Check.Timeout = "3s"
+	// 为兼容现有的 平滑逻辑
+	//err = client.Agent().ServiceDeregister(registration.ID)
+
+	err = client.Agent().ServiceRegister(registration)
+
+	if err != nil {
+		tlog.Error(err)
+	}
+}
--- a/metrics/prometheus/prometheus.go
+++ b/metrics/prometheus/prometheus.go
@@ -0,0 +1,169 @@
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"globalfintech/golib/metrics"
+)
+
+// Counter implements Counter, via a Prometheus CounterVec.
+type Counter struct {
+	Cv  *prometheus.CounterVec
+	lvs metrics.LabelValues
+}
+
+// NewCounterFrom constructs and registers a Prometheus CounterVec,
+// and returns a usable Counter object.
+func NewCounterFrom(opts prometheus.CounterOpts, labelNames []string) *Counter {
+	cv := prometheus.NewCounterVec(opts, labelNames)
+	prometheus.MustRegister(cv)
+	return NewCounter(cv)
+}
+
+// NewCounter wraps the CounterVec and returns a usable Counter object.
+func NewCounter(cv *prometheus.CounterVec) *Counter {
+	return &Counter{
+		Cv: cv,
+	}
+}
+
+// With implements Counter.
+func (c *Counter) With(labelValues ...string) metrics.Counter {
+	return &Counter{
+		Cv:  c.Cv,
+		lvs: c.lvs.With(labelValues...),
+	}
+}
+
+// Add implements Counter.
+func (c *Counter) Add(delta float64) {
+	c.Cv.With(makeLabels(c.lvs...)).Add(delta)
+}
+
+// Inc implement Counter.
+func (c *Counter) Inc() {
+	c.Cv.With(makeLabels(c.lvs...)).Inc()
+}
+
+// Gauge implements Gauge, via a Prometheus GaugeVec.
+type Gauge struct {
+	gv  *prometheus.GaugeVec
+	lvs metrics.LabelValues
+}
+
+// NewGaugeFrom construts and registers a Prometheus GaugeVec,
+// and returns a usable Gauge object.
+func NewGaugeFrom(opts prometheus.GaugeOpts, labelNames []string) *Gauge {
+	gv := prometheus.NewGaugeVec(opts, labelNames)
+	prometheus.MustRegister(gv)
+	return NewGauge(gv)
+}
+
+// NewGauge wraps the GaugeVec and returns a usable Gauge object.
+func NewGauge(gv *prometheus.GaugeVec) *Gauge {
+	return &Gauge{
+		gv: gv,
+	}
+}
+
+// With implements Gauge.
+func (g *Gauge) With(labelValues ...string) metrics.Gauge {
+	return &Gauge{
+		gv:  g.gv,
+		lvs: g.lvs.With(labelValues...),
+	}
+}
+
+// Set implements Gauge.
+func (g *Gauge) Set(value float64) {
+	g.gv.With(makeLabels(g.lvs...)).Set(value)
+}
+
+// Add is supported by Prometheus GaugeVecs.
+func (g *Gauge) Add(delta float64) {
+	g.gv.With(makeLabels(g.lvs...)).Add(delta)
+}
+
+// Sub is supported by Prometheus GaugeVecs.
+func (g *Gauge) Sub(delta float64) {
+	g.gv.With(makeLabels(g.lvs...)).Sub(delta)
+}
+
+// Summary implements Histogram, via a Prometheus SummaryVec. The difference
+// between a Summary and a Histogram is that Summaries don't require predefined
+// quantile buckets, but cannot be statistically aggregated.
+type Summary struct {
+	sv  *prometheus.SummaryVec
+	lvs metrics.LabelValues
+}
+
+// NewSummaryFrom constructs and registers a Prometheus SummaryVec,
+// and returns a usable Summary object.
+func NewSummaryFrom(opts prometheus.SummaryOpts, labelNames []string) *Summary {
+	sv := prometheus.NewSummaryVec(opts, labelNames)
+	prometheus.MustRegister(sv)
+	return NewSummary(sv)
+}
+
+// NewSummary wraps the SummaryVec and returns a usable Summary object.
+func NewSummary(sv *prometheus.SummaryVec) *Summary {
+	return &Summary{
+		sv: sv,
+	}
+}
+
+// With implements Histogram.
+func (s *Summary) With(labelValues ...string) metrics.Histogram {
+	return &Summary{
+		sv:  s.sv,
+		lvs: s.lvs.With(labelValues...),
+	}
+}
+
+// Observe implements Histogram.
+func (s *Summary) Observe(value float64) {
+	s.sv.With(makeLabels(s.lvs...)).Observe(value)
+}
+
+// Histogram implements Histogram via a Prometheus HistogramVec. The difference
+// between a Histogram and a Summary is that Histograms require predefined
+// quantile buckets, and can be statistically aggregated.
+type Histogram struct {
+	hv  *prometheus.HistogramVec
+	lvs metrics.LabelValues
+}
+
+// NewHistogramFrom constructs and registers a Prometheus HistogramVec,
+// and returns a usable Histogram object.
+func NewHistogramFrom(opts prometheus.HistogramOpts, labelNames []string) *Histogram {
+	hv := prometheus.NewHistogramVec(opts, labelNames)
+	prometheus.MustRegister(hv)
+	return NewHistogram(hv)
+}
+
+// NewHistogram wraps the HistogramVec and returns a usable Histogram object.
+func NewHistogram(hv *prometheus.HistogramVec) *Histogram {
+	return &Histogram{
+		hv: hv,
+	}
+}
+
+// With implements Histogram.
+func (h *Histogram) With(labelValues ...string) metrics.Histogram {
+	return &Histogram{
+		hv:  h.hv,
+		lvs: h.lvs.With(labelValues...),
+	}
+}
+
+// Observe implements Histogram.
+func (h *Histogram) Observe(value float64) {
+	h.hv.With(makeLabels(h.lvs...)).Observe(value)
+}
+
+func makeLabels(labelValues ...string) prometheus.Labels {
+	labels := prometheus.Labels{}
+	for i := 0; i < len(labelValues); i += 2 {
+		labels[labelValues[i]] = labelValues[i+1]
+	}
+	return labels
+}
--- a/metrics/prometheus/prometheus_test.go
+++ b/metrics/prometheus/prometheus_test.go
@@ -0,0 +1,61 @@
+package prometheus
+
+import (
+	"fmt"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"globalfintech/golib/metrics"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"strconv"
+	"testing"
+)
+
+func TestCounter(t *testing.T) {
+	s := httptest.NewServer(promhttp.Handler())
+	defer s.Close()
+
+	scrape := func() string {
+		resp, _ := http.Get(s.URL)
+		buf, _ := ioutil.ReadAll(resp.Body)
+		return string(buf)
+	}
+
+	namespace, subsystem, name := "ns", "ss", "foo"
+	re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{alpha="alpha-value",beta="beta-value"} ([0-9\.]+)`)
+
+	counter := NewCounterFrom(stdprometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      name,
+		Help:      "This is the help string.",
+	}, []string{"alpha", "beta"}).With("beta", "beta-value", "alpha", "alpha-value") // order shouldn't matter
+
+	value := func() float64 {
+		matches := re.FindStringSubmatch(scrape())
+		f, _ := strconv.ParseFloat(matches[1], 64)
+		return f
+	}
+	want := FillCounter(counter)
+	if have := value(); want != have {
+		t.Fatal(fmt.Errorf("want %f, have %f", want, have))
+	}
+
+}
+
+// FillCounter puts some deltas through the counter and returns the total value.
+func FillCounter(counter metrics.Counter) float64 {
+	a := rand.Perm(100)
+	n := rand.Intn(len(a))
+
+	var want float64
+	for i := 0; i < n; i++ {
+		f := float64(a[i])
+		counter.Add(f)
+		want += f
+	}
+	return want
+}