commit 2fc86fd0a1c858c70a85004dc973d60ac2b41448 Author: Cruise Zhao Date: Mon Apr 6 19:57:19 2020 +0800 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f579cd0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,44 @@ +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex +.DS_Store +*~ +*.swp +*.swo +*.dSYM +*.bak + +# IDE project +*.project +*.buildpath +.idea/* +.idea + +# Xcode +## gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore +## Build generated +build/ +DerivedData +## Various settings +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata +## Other +*.xccheckout +*.moved-aside +*.xcuserstate +*.xcscmblueprint +## Playgrounds +timeline.xctimeline +playground.xcworkspace +.build/ \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..940989b --- /dev/null +++ b/Readme.md @@ -0,0 +1,24 @@ +#Go lib + +## tlog + +## trace + +## metrics + +## cache + +## lrucache + +## db +mysql +## kv +redis +## kafka + +## aws + +## geo +maxmind geoip2 +## qgrpc + diff --git a/aws/aws.go b/aws/aws.go new file mode 100644 index 0000000..c343d87 --- /dev/null +++ b/aws/aws.go @@ -0,0 +1,126 @@ +package aws + +import ( + "io" + "io/ioutil" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +type Config struct { + AwsAccessKeyId string `toml:"access_key_id" json:"access_key_id"` + AwsSecretAccessKey string `toml:"secret_access_key" json:"secret_access_id"` + AwsBuckets string `toml:"buckets" json:"buckets"` + AwsRegion string `toml:"region" json:"region"` +} + +func NewS3Session(c *Config) *session.Session { + cfg := aws.NewConfig().WithRegion(c.AwsRegion).WithCredentials( + credentials.NewStaticCredentials( + c.AwsAccessKeyId, + c.AwsSecretAccessKey, + ""), + ) + s, _ := session.NewSession(cfg) + return s +} + +func Put(c Config, key, contentType string, input io.ReadSeeker) error { + s := NewS3Session(&c) + client := s3.New(s) + _, err := client.PutObject( + &s3.PutObjectInput{ + Bucket: aws.String(c.AwsBuckets), + Key: aws.String(key), + ContentType: aws.String(contentType), + Body: input, + CacheControl: aws.String("max-age=2592000"), + }, + ) + return err +} + +func Del(c Config, key string) error { + s := NewS3Session(&c) + client := s3.New(s) + + arg := &s3.DeleteObjectInput{ + Bucket: aws.String(c.AwsBuckets), + Key: aws.String(key), + } + + _, err := client.DeleteObject(arg) + return err +} + +func Get(c Config, key string) (string, []byte, error) { + s := NewS3Session(&c) + client := s3.New(s) + + arg := &s3.GetObjectInput{ + Bucket: aws.String(c.AwsBuckets), + Key: aws.String(key), + } + + ret, err := client.GetObject(arg) + if err != nil { + return "", nil, err + } + + if ret != nil { + defer ret.Body.Close() + } + + contentType := "" + if ret != nil && ret.ContentType != nil { + contentType = *ret.ContentType + } + + b, err := ioutil.ReadAll(ret.Body) + + return contentType, b, err +} + +func List(c Config, prefix string, file string, count int64) ([]string, error) { + s := NewS3Session(&c) + client := s3.New(s) + + input := &s3.ListObjectsInput{ + Bucket: aws.String(c.AwsBuckets), + Prefix: aws.String(prefix), + Marker: aws.String(file), + MaxKeys: aws.Int64(count), + } + + ret, err := client.ListObjects(input) + if err != nil { + return nil, err + } + + keys := []string{} + + for _, obj := range ret.Contents { + if strings.HasSuffix(*obj.Key, "/") { + continue + } + keys = append(keys, *obj.Key) + } + return keys, nil +} + +func Copy(c Config, source, key string) error { + s := NewS3Session(&c) + client := s3.New(s) + + _, err := client.CopyObject(&s3.CopyObjectInput{ + Bucket: aws.String(c.AwsBuckets), + CopySource: aws.String(source), + Key: aws.String(key), + }) + return err + +} diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..2a9744c --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,122 @@ +package cache + +import ( + "errors" + "github.com/creachadair/cityhash" + "globalfintech/golib/tlog" + "sync" + "time" +) + +//API +var Nil error = errors.New("cache: nil") + +func Set(key string, value interface{}) error { + return SetEx(key, value, 86400) +} + +func SetEx(key string, value interface{}, exp int) error { + return setEx(key, value, exp) +} + +func Get(key string) (interface{}, error) { + return get(key) +} + +func Del(key string) error { + return del(key) +} + +//internal +var dbs map[byte]*db + +type db struct { + m map[string]*item + t time.Duration + sync.RWMutex +} + +type item struct { + t int + d interface{} +} + +func init() { + if dbs == nil { + dbs = make(map[byte]*db, 256) + for i := 0; i <= 255; i++ { + dbs[byte(i)] = newdb(time.Duration(3600+i) * time.Second) + } + } +} + +func newdb(t time.Duration) *db { + ret := &db{ + t: t, + m: make(map[string]*item), + } + go ret.flush() + return ret +} + +//定期扫描过期的key +func (this *db) flush() { + for range time.NewTicker(this.t).C { + this.doflush() + } +} + +func (this *db) doflush() { + start := time.Now() + tt := int(start.Unix()) + var i, j int + this.Lock() + for key, item := range this.m { + i++ + if item.t != 0 && item.t < tt { + j++ + delete(this.m, key) + } + } + this.Unlock() + tlog.Infof("FlushCacheKey||Totoal=%d||Expired=%d||t=%s", i, j, time.Since(start)) +} + +func setEx(key string, value interface{}, exp int) error { + if exp <= 0 { + return errors.New("args error, expTime must be > 0") + } + db, _ := dbs[hash(key)] + db.Lock() + db.m[key] = &item{ + d: value, + t: int(time.Now().Unix()) + exp} + db.Unlock() + return nil +} + +func get(key string) (interface{}, error) { + db, _ := dbs[hash(key)] + db.RLock() + item, ok := db.m[key] + db.RUnlock() + if ok { + if item.t >= int(time.Now().Unix()) { + return item.d, nil + } + del(key) + } + return nil, Nil +} + +func del(key string) error { + db, _ := dbs[hash(key)] + db.Lock() + delete(db.m, key) + db.Unlock() + return nil +} + +func hash(key string) byte { + return byte(cityhash.Hash32([]byte(key))) +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..797c08c --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,88 @@ +package cache + +import ( + "fmt" + "testing" + "time" +) + +type T struct { + A int + B string +} + +func TestSet(t *testing.T) { + err := Set("test", "test") + if err != nil { + fmt.Println(err) + return + } + value, err := Get("test") + if err != nil { + fmt.Println(err) + return + } + fmt.Println(value.(string)) +} + +func TestStruct(t *testing.T) { + err := Set("struct", T{A: 1, B: "testtest"}) + if err != nil { + fmt.Println(err) + return + } + value1, err := Get("struct") + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("%+v\n", value1.(T)) +} + +func TestStruct1(t *testing.T) { + err := Set("struct", &T{A: 1, B: "testtest"}) + if err != nil { + fmt.Println(err) + return + } + value1, err := Get("struct") + if err != nil { + fmt.Println(err) + return + } + fmt.Printf("%+v\n", value1.(*T)) +} + +func TestDel(t *testing.T) { + err := Del("test") + if err != nil { + fmt.Println(err) + return + } + value, err := Get("test") + if err != nil { + fmt.Println(err) + return + } + fmt.Println(value.(string)) +} + +func TestSetEx(t *testing.T) { + err := SetEx("testex", []byte("testex"), 2) + if err != nil { + fmt.Println(err) + } + valueEx, err := Get("testex") + if err != nil { + fmt.Println(err) + return + } + fmt.Println(valueEx.([]byte)) + time.Sleep(3 * time.Second) + valueEx, err = Get("testex") + if err != nil { + fmt.Println(err) + return + } + fmt.Println(valueEx.([]byte)) +} diff --git a/db/db.go b/db/db.go new file mode 100644 index 0000000..25b870e --- /dev/null +++ b/db/db.go @@ -0,0 +1,67 @@ +package db + +import ( + "database/sql" + "errors" + "fmt" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +var x map[string]*sql.DB = make(map[string]*sql.DB) + +var ErrNoDb = errors.New("no db") + +type Config struct { + Addr string `toml:"addr" json:"addr"` + User string `toml:"user" json:"user"` + Pwd string `toml:"pwd" json:"pwd"` + Db string `toml:"db" json:"db"` + Options string `toml:"options" json:"options"` + MaxOpenConns int `toml:"max_open_conns" json:"max_open_conns"` + MaxIdleConns int `toml:"max_idle_conns" json:"max_idle_conns"` + MaxConnLifeTime int `toml:"max_conn_life_time" json:"max_conn_life_time"` +} + +func (c Config) Dsn() string { + if c.Options == "" { + format := "%s:%s@tcp(%s)/%s?charset=utf8mb4&timeout=5s" + return fmt.Sprintf(format, c.User, c.Pwd, c.Addr, c.Db) + } else { + format := "%s:%s@tcp(%s)/%s?charset=utf8mb4&timeout=5s&%s" + return fmt.Sprintf(format, c.User, c.Pwd, c.Addr, c.Db, c.Options) + } +} + +func Add(name string, c Config) error { + db, err := sql.Open("mysql", c.Dsn()) + if err != nil { + return err + } + if err := db.Ping(); err != nil { + return err + } + db.SetMaxIdleConns(c.MaxIdleConns) + db.SetMaxOpenConns(c.MaxOpenConns) + + if c.MaxConnLifeTime <= 0 { + c.MaxConnLifeTime = 600 // Default 10 minutes + } + db.SetConnMaxLifetime(time.Duration(c.MaxConnLifeTime) * time.Second) + + _, ok := x[name] + if ok { + return errors.New("db exists") + } + x[name] = db + return nil +} + +func GetDB(name string) (*sql.DB, error) { + db, ok := x[name] + if ok { + return db, nil + } + return nil, ErrNoDb +} diff --git a/geo/geo.go b/geo/geo.go new file mode 100644 index 0000000..e95a278 --- /dev/null +++ b/geo/geo.go @@ -0,0 +1,92 @@ +package geo + +import ( + "net" + + iplib "github.com/oschwald/geoip2-golang" +) + +var db *iplib.Reader + +func Init(file string) (err error) { + db, err = iplib.Open(file) + return +} + +func Close() { + if db != nil { + db.Close() + } +} + +func City(ip string) (*iplib.City, error) { + return db.City(net.ParseIP(ip)) +} + +func GetCity(ip string) (string, error) { + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return "", err + } + return r.City.Names["en"], nil +} + +func GetCountry(ip string) (string, error) { + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return "", err + } + return r.Country.Names["en"], nil +} + +func GetProvince(ip string) (string, error) { + defer func() { recover() }() + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return "", err + } + return r.Subdivisions[0].Names["en"], nil +} + +//同时获取国家，省份，简写, 城市名称 +func GetLoc(ip string) (string, string, string, error) { + defer func() { recover() }() + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return "", "", "", err + } + + return r.Country.IsoCode, r.Subdivisions[0].IsoCode, r.City.Names["en"], nil +} + +// 获取 iso_code, name +func GetContinent(ip string) (string, string, error) { + defer func() { recover() }() + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return "", "", err + } + + return r.Continent.Code, r.Continent.Names["en"], nil +} + +// Return latitude and longitude +func GetCoordinate(ip string) (float64, float64, error) { + defer func() { recover() }() + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return 0, 0, err + } + + return r.Location.Latitude, r.Location.Longitude, nil +} + +func GetTimezone(ip string) (string, error) { + defer func() { recover() }() + r, err := db.City(net.ParseIP(ip)) + if err != nil { + return "", err + } + + return r.Location.TimeZone, nil +} diff --git a/gfetcd/README.md b/gfetcd/README.md new file mode 100644 index 0000000..e7951aa --- /dev/null +++ b/gfetcd/README.md @@ -0,0 +1,79 @@ +## 代码结构 + +``` shell +├── README.md +├── client.go # grpc客户端调用 +├── config.go # 配置结构 +├── resolver.go # 地址解析 +└── server.go # grpc服务端调用 +``` + +## 数据结构 + +### 基本结构 + +``` golang +type BasicConfig struct { + DialTimeout int `toml:"dial_timeout"` // 连接超时时间，默认5秒 + EndPoints []string `toml:"endpoints"` // etcd的地址 + Prefix string `toml:"prefix"` // etcd服务的路径前缀，区分环境用 + Debug bool `toml:"debug"` // 使用debug模式，打印详细的etcd交互日志，很多，非调试不建议打开 +} +``` + +### grpc服务端用的配置 + +``` golang +type NameConfig struct { + BasicConfig + Servers []string `toml:"servers"` // 注册的服务名称，可以是多个 + Addr string `toml:"addr"` // 服务地址，可以是IP:PORT或者:PORT格式 + TTL int64 `toml:"ttl"` // 租约超时时间，默认5秒 +} +``` + +### grpc客户端用的配置 + +``` golang +type ResolverConfig struct { + BasicConfig + WatchServers []string `toml:"watch_servers"` // 依赖的服务名称 +} +``` + +## 使用方法 + +### grpc服务端 + +``` golang + // demo + &config.EtcdName = { + EndPoints:[]string{"127.0.0.1:2379"}, + Prefix: "/ng/test/okash", + Servers: []string{"orders.Orders", "product.Product"}, + Addr: ":8701", + } + etcdName, err := gfetcd.NameRegist(&config.EtcdName) + if err != nil { + panic(err) + } + defer etcdName.Unregister() +``` + +### grpc客户端 + +```golang + // demo + &config.EtcdReslv{ + EndPoints:[]string{"127.0.0.1:2379"}, + Prefix: "/ng/test/okash", + WatchServers: []string{"product.Product"}, + } + + // 连接etcd + EtcdReslv, err = gfetcd.NewResolver(&config.EtcdReslv) + + // 调用方法 + err := EtcdReslv.Call(context.Background(), &XXXReq{}, &XXXResp{}) +``` + diff --git a/gfetcd/client.go b/gfetcd/client.go new file mode 100644 index 0000000..008a0a0 --- /dev/null +++ b/gfetcd/client.go @@ -0,0 +1,91 @@ +package gfetcd + +import ( + "context" + "fmt" + "globalfintech/golib/tlog" + "os" + "strings" + "time" + + "github.com/coreos/etcd/clientv3" + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/keepalive" +) + +// 客户端配置 +type ResolverConfig struct { + BasicConfig + WatchServers []string `toml:"watch_servers"` // 依赖的服务名称 +} + +type Resolver struct { + userCfg *ResolverConfig + etcdCli *clientv3.Client + rpcServers map[string]*grpc.ClientConn +} + +func NewResolver(cfg *ResolverConfig) (resolver *Resolver, err error) { + + if cfg.Debug { + clientv3.SetLogger(grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 4)) + } + + cli, err := etcdDial(&cfg.BasicConfig) + if err != nil { + return nil, err + } + + resolver = &Resolver{ + userCfg: cfg, + etcdCli: cli, + rpcServers: make(map[string]*grpc.ClientConn), + } + + for _, serviceName := range cfg.WatchServers { + + service := ServiceKey(cfg.Prefix, serviceName) + tlog.Debugf("ETCD RESOLVER||service: %s", service) + + //r := &etcdnaming.GRPCResolver{Client: cli} // todo: 全部更新为新版后，改为这种方式 + r := &GFgrpcResolver{Client: cli} + b := grpc.RoundRobin(r) + + resolver.rpcServers[service], err = grpc.Dial(service, grpc.WithInsecure(), + grpc.WithTimeout(time.Second*5), + grpc.WithKeepaliveParams(keepalive.ClientParameters{Time: time.Duration(2) * time.Second}), + grpc.WithBalancer(b), + grpc.WithBlock()) + if err != nil { + tlog.Errorf("ETCD RESOLVER||service: %s ||grpc dial err:%s", service, err) + return nil, fmt.Errorf("grpc dial err") + } + } + + return resolver, nil +} + +func (self *Resolver) Call(ctx context.Context, method string, args, reply interface{}, opts ...grpc.CallOption) error { + // method format: // + // eg. "/case_expire.CaseExpire/ExpireCase" + + t := strings.Split(method, "/") + if len(t) < 3 { + tlog.Errorf("ETCD RESOLVER||method name: %s incorrect format", method) + return fmt.Errorf("invalid method format") + } + + serviceKey := ServiceKey(self.userCfg.Prefix, t[1], "") + + tlog.Debugf("ETCD RESOLVER||Call(serviceKey: %s, method: %s)", + serviceKey, method) + + conn, ok := self.rpcServers[serviceKey] + if !ok { + tlog.Errorf("ETCD RESOLVER||method: %s||no grpc client available", method) + return fmt.Errorf("no grpc client available") + } + + return grpc.Invoke(ctx, method, args, reply, conn, opts...) +} diff --git a/gfetcd/config.go b/gfetcd/config.go new file mode 100644 index 0000000..f78e096 --- /dev/null +++ b/gfetcd/config.go @@ -0,0 +1,39 @@ +package gfetcd + +import ( + "path" + "time" + + "github.com/coreos/etcd/clientv3" +) + +const ( + DEFAULT_DIAL_TIMEOUT = time.Second * 5 + DEFAULT_LEASE_TTL = 5 +) + +// 基础配置，公共字段 +type BasicConfig struct { + DialTimeout int `toml:"dial_timeout"` // 连接超时时间 + EndPoints []string `toml:"endpoints"` // etcd的地址 + Prefix string `toml:"prefix"` // etcd服务的路径前缀，区分环境用 + Debug bool `toml:"debug"` // 使用debug模式 +} + +func etcdDial(cfg *BasicConfig) (*clientv3.Client, error) { + dialTimeout := DEFAULT_DIAL_TIMEOUT + if cfg.DialTimeout > 0 { + dialTimeout = time.Duration(cfg.DialTimeout) * time.Second + } + + return clientv3.New(clientv3.Config{ + Endpoints: cfg.EndPoints, + DialTimeout: dialTimeout, + DialKeepAliveTime: 2 * time.Second, + DialKeepAliveTimeout: 2 * time.Second, + }) +} + +func ServiceKey(opts ...string) string { + return path.Join(opts...) +} diff --git a/gfetcd/resolver.go b/gfetcd/resolver.go new file mode 100644 index 0000000..69c8722 --- /dev/null +++ b/gfetcd/resolver.go @@ -0,0 +1,159 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gfetcd + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + etcd "github.com/coreos/etcd/clientv3" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/naming" + "google.golang.org/grpc/status" +) + +var ErrWatcherClosed = fmt.Errorf("naming: watch closed") + +// GRPCResolver creates a grpc.Watcher for a target to track its resolution changes. +type GFgrpcResolver struct { + // Client is an initialized etcd client. + Client *etcd.Client +} + +/* +func (gr *GRPCResolver) Update(ctx context.Context, target string, nm naming.Update, opts ...etcd.OpOption) (err error) { + switch nm.Op { + case naming.Add: + var v []byte + if v, err = json.Marshal(nm); err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + _, err = gr.Client.KV.Put(ctx, target+"/"+nm.Addr, string(v), opts...) + case naming.Delete: + _, err = gr.Client.Delete(ctx, target+"/"+nm.Addr, opts...) + default: + return status.Error(codes.InvalidArgument, "naming: bad naming op") + } + return err +} +*/ + +func (gr *GFgrpcResolver) Resolve(target string) (naming.Watcher, error) { + ctx, cancel := context.WithCancel(context.Background()) + w := &GFgRPCWatcher{c: gr.Client, target: target + "/", ctx: ctx, cancel: cancel} + return w, nil +} + +type GFgRPCWatcher struct { + c *etcd.Client + target string + ctx context.Context + cancel context.CancelFunc + wch etcd.WatchChan + err error +} + +// Next gets the next set of updates from the etcd resolver. +// Calls to Next should be serialized; concurrent calls are not safe since +// there is no way to reconcile the update ordering. +func (gw *GFgRPCWatcher) Next() ([]*naming.Update, error) { + if gw.wch == nil { + // first Next() returns all addresses + return gw.firstNext() + } + if gw.err != nil { + return nil, gw.err + } + + // process new events on target/* + wr, ok := <-gw.wch + if !ok { + gw.err = status.Error(codes.Unavailable, ErrWatcherClosed.Error()) + return nil, gw.err + } + if gw.err = wr.Err(); gw.err != nil { + return nil, gw.err + } + + updates := make([]*naming.Update, 0, len(wr.Events)) + for _, e := range wr.Events { + var jupdate naming.Update + var err error + var jsonVal bool + + switch e.Type { + case etcd.EventTypePut: + if strings.HasPrefix(string(e.Kv.Value), "{") { + err = json.Unmarshal(e.Kv.Value, &jupdate) + jsonVal = true + } else { + jupdate.Addr = string(e.Kv.Value) + } + jupdate.Op = naming.Add + + case etcd.EventTypeDelete: + if strings.HasPrefix(string(e.PrevKv.Value), "{") { + err = json.Unmarshal(e.PrevKv.Value, &jupdate) + } else { + jupdate.Addr = string(e.PrevKv.Value) + } + jupdate.Op = naming.Delete + + default: + continue + } + + if jsonVal { + if err == nil { + updates = append(updates, &jupdate) + } + } else { + updates = append(updates, &jupdate) + } + } + return updates, nil +} + +func (gw *GFgRPCWatcher) firstNext() ([]*naming.Update, error) { + // Use serialized request so resolution still works if the target etcd + // server is partitioned away from the quorum. + resp, err := gw.c.Get(gw.ctx, gw.target, etcd.WithPrefix(), etcd.WithSerializable()) + if gw.err = err; err != nil { + return nil, err + } + + updates := make([]*naming.Update, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + var jupdate naming.Update + + if strings.HasPrefix(string(kv.Value), "{") { + if err := json.Unmarshal(kv.Value, &jupdate); err != nil { + continue + } + } else { + jupdate.Addr = string(kv.Value) + } + updates = append(updates, &jupdate) + } + + opts := []etcd.OpOption{etcd.WithRev(resp.Header.Revision + 1), etcd.WithPrefix(), etcd.WithPrevKV()} + gw.wch = gw.c.Watch(gw.ctx, gw.target, opts...) + return updates, nil +} + +func (gw *GFgRPCWatcher) Close() { gw.cancel() } diff --git a/gfetcd/server.go b/gfetcd/server.go new file mode 100644 index 0000000..72a8e5b --- /dev/null +++ b/gfetcd/server.go @@ -0,0 +1,142 @@ +package gfetcd + +import ( + "context" + "fmt" + "net" + "strings" + "time" + + "globalfintech/golib/tlog" + + "github.com/coreos/etcd/clientv3" + etcdnaming "github.com/coreos/etcd/clientv3/naming" + "github.com/siddontang/go/log" + "google.golang.org/grpc/naming" +) + +// 服务端配置 +type NameConfig struct { + BasicConfig + Servers []string `toml:"servers"` // 注册的服务名称，可以是多个 + Addr string `toml:"addr"` // 服务地址 + TTL int64 `toml:"ttl"` // lease, default 5 + UseNew bool `toml:"use_new"` // 使用新版地址格式 +} + +func (self *NameConfig) GetAddr() string { + if strings.HasPrefix(self.Addr, ":") { + return fmt.Sprintf("%s%s", getLocalIP(), self.Addr) + } + return self.Addr +} + +//获取本机的内网Ip, 如果发现对方的ip 和自己的ip 相同，用127.0.0.1 替代 +func getLocalIP() string { + ifaces, _ := net.Interfaces() + for _, i := range ifaces { + addrs, _ := i.Addrs() + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + ipAddr := ip.String() + if strings.HasPrefix(ipAddr, "172.") || strings.HasPrefix(ipAddr, "192.") || strings.HasPrefix(ipAddr, "10.") { + return ipAddr + } + } + } + return "127.0.0.1" +} + +type Name struct { + userCfg *NameConfig + etcdCli *clientv3.Client + stopSignal chan bool +} + +func NameRegist(cfg *NameConfig) (name *Name, err error) { + + cli, err := etcdDial(&cfg.BasicConfig) + if err != nil { + tlog.Errorf("ETCD NAME||endpoints: %#v||err: %s", cfg.EndPoints, err) + return nil, err + } + + name = &Name{ + userCfg: cfg, + etcdCli: cli, + stopSignal: make(chan bool, 1), + } + + if cfg.TTL == 0 { + cfg.TTL = DEFAULT_LEASE_TTL + } + + go func() { + + ticker := time.NewTicker(time.Second * time.Duration(cfg.TTL/2)) + serviceAddr := cfg.GetAddr() + resolver := &etcdnaming.GRPCResolver{Client: cli} + + for { + + lease, err := cli.Grant(context.Background(), cfg.TTL) + if err != nil { + tlog.Errorf("ETCD NAME||endpoints: %#v||grant err: %s", + cfg.EndPoints, err) + continue + } + + for _, serviceName := range cfg.Servers { + + serviceKey := ServiceKey("/new", cfg.Prefix, serviceName) + + if err = resolver.Update(context.Background(), serviceKey, + naming.Update{Op: naming.Add, Addr: serviceAddr}, + clientv3.WithLease(lease.ID)); err != nil { + + tlog.Errorf("ETCD NAME||endpoints: %#v service: %s addr: %s||update err: %s", + cfg.EndPoints, serviceKey, serviceAddr, err) + } + + if !cfg.UseNew { + serviceKey = ServiceKey(cfg.Prefix, serviceName, serviceAddr) + if _, err = cli.Put(context.Background(), serviceKey, serviceAddr, clientv3.WithLease(lease.ID)); err != nil { + tlog.Errorf("ETCD NAME||endpoints: %#v put key: %s value: %s||err: %s", + cfg.EndPoints, serviceKey, serviceAddr, err) + } + } + } + + select { + case <-name.stopSignal: + tlog.Info("ETCD NAME||received stop signal") + return + case <-ticker.C: + } + } + }() + + return name, nil +} + +func (self *Name) Unregister() { + self.stopSignal <- true + //self.stopSignal = make(chan bool, 1) // just a hack to avoid multi unregister deadlock + + serviceAddr := self.userCfg.GetAddr() + for _, serviceName := range self.userCfg.Servers { + serviceKey := ServiceKey(self.userCfg.Prefix, serviceName, serviceAddr) + + if _, err := self.etcdCli.Delete(context.Background(), serviceKey); err != nil { + tlog.Errorf("ETCD NAME||unregister key: %s||err: %s", serviceKey, err) + } + } + + log.Info("ETCD SERVER||unregister done") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5c40311 --- /dev/null +++ b/go.mod @@ -0,0 +1,52 @@ +module globalfintech/golib + +go 1.12 + +require ( + github.com/Shopify/sarama v1.22.0 + github.com/aws/aws-sdk-go v1.19.15 + github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 + github.com/bsm/sarama-cluster v2.1.15+incompatible + github.com/coreos/bbolt v1.3.2 // indirect + github.com/coreos/etcd v3.3.15+incompatible + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/creachadair/cityhash v0.1.0 + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/go-redis/redis v6.15.2+incompatible + github.com/go-sql-driver/mysql v1.4.1 + github.com/gogo/protobuf v1.2.1 // indirect + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect + github.com/golang/protobuf v1.3.1 + github.com/google/btree v1.0.0 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/gorilla/websocket v1.4.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect + github.com/hashicorp/consul/api v1.2.0 + github.com/hashicorp/golang-lru v0.5.1 + github.com/jonboulle/clockwork v0.1.0 // indirect + github.com/json-iterator/go v1.1.7 // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.5.0 // indirect + github.com/oschwald/geoip2-golang v1.2.1 + github.com/oschwald/maxminddb-golang v1.3.0 // indirect + github.com/prometheus/client_golang v0.9.2 + github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect + github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 + github.com/sirupsen/logrus v1.4.1 // indirect + github.com/soheilhy/cmux v0.1.4 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.2 // indirect + go.uber.org/atomic v1.3.2 // indirect + go.uber.org/multierr v1.1.0 // indirect + go.uber.org/zap v1.9.1 // indirect + golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + google.golang.org/grpc v1.20.1 + sigs.k8s.io/yaml v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7e97dda --- /dev/null +++ b/go.sum @@ -0,0 +1,265 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14= +github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Shopify/sarama v1.22.0 h1:rtiODsvY4jW6nUV6n3K+0gx/8WlAwVt+Ixt6RIvpYyo= +github.com/Shopify/sarama v1.22.0/go.mod h1:lm3THZ8reqBDBQKQyb5HB3sY1lKp3grEbQ81aWSgPp4= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.19.15 h1:Ov8zxPunHxZFxrXBarqzkZBSZugJRzPgb4mfq/SyVgA= +github.com/aws/aws-sdk-go v1.19.15/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK+hNk4tyD+nuGjpVLPEHuJSFXMw11/HPA= +github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/bsm/sarama-cluster v2.1.15+incompatible h1:RkV6WiNRnqEEbp81druK8zYhmnIgdOjqSVi0+9Cnl2A= +github.com/bsm/sarama-cluster v2.1.15+incompatible/go.mod h1:r7ao+4tTNXvWm+VRpRJchr2kQhqxgmAp2iEX5W96gMM= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= +github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= +github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= +github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.2.0 h1:oPsuzLp2uk7I7rojPKuncWbZ+m5TMoD4Ivs+2Rkeh4Y= +github.com/hashicorp/consul/api v1.2.0/go.mod h1:1SIkFYi2ZTXUE5Kgt179+4hH33djo11+0Eo2XgTAtkw= +github.com/hashicorp/consul/sdk v0.2.0 h1:GWFYFmry/k4b1hEoy7kSkmU8e30GAyI4VZHk0fRxeL4= +github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/oschwald/geoip2-golang v1.2.1 h1:3iz+jmeJc6fuCyWeKgtXSXu7+zvkxJbHFXkMT5FVebU= +github.com/oschwald/geoip2-golang v1.2.1/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE= +github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE= +github.com/oschwald/maxminddb-golang v1.3.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 h1:GeinFsrjWz97fAxVUEd748aV0cYL+I6k44gFJTCVvpU= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/kafka/README.md b/kafka/README.md new file mode 100644 index 0000000..e093ff5 --- /dev/null +++ b/kafka/README.md @@ -0,0 +1,55 @@ + +# kafka cluster封装 + +> 目前只封装了consumer，单个consumer支持同时消费一个topic +> 如果指定了group，且group名称开头不为'test_consumer_'则认为是正式订阅，会在处理完成后标记消息已经被消费 +> 否则会生成随机的group名称，用于测试 + +## 示例代码如下 + +```go +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "golib/kafka" +) + +// 回调的消息处理函数,用户自己实现的消息处理逻辑 +func processor(msg []byte) { + fmt.Println("recv msg:", string(msg)) +} + +func main() { + + // 配置，可以从文件加载 + cfg := kafka.Config{ + Brokers: []string{"test.kafka.o-kash.com:9092"}, + Topic: "charge_success", + Workers: 5, // 并发的回调函数数量，不是partition的数量 + } + + if consumer := kafka.NewConsumer(&cfg, processor); consumer != nil { + consumer.Run() + + // 监控退出信号通知consumer退出 + ch := make(chan os.Signal) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + <-ch + + consumer.Stop() + } +} +``` + +## 产生测试用的消息 + +使用官方工具 + +```shell +./kafka-console-producer.sh --broker-list test.kafka.o-kash.com:9092 --topic charge_success +``` diff --git a/kafka/kafka.go b/kafka/kafka.go new file mode 100644 index 0000000..2d8d0f1 --- /dev/null +++ b/kafka/kafka.go @@ -0,0 +1,123 @@ +package kafka + +import ( + "fmt" + "math/rand" + "strings" + "sync" + + "globalfintech/golib/tlog" + + "github.com/Shopify/sarama" + cluster "github.com/bsm/sarama-cluster" +) + +type Config struct { + Brokers []string `toml:"brokers"` + Topic string `toml:"topic"` + Group string `toml:"group"` + Workers int `toml:"workers"` + Oldest bool `toml:"oldest"` +} + +type Consumer struct { + test bool + consumer *cluster.Consumer + workers int + handler func([]byte) + stop chan bool + wg *sync.WaitGroup +} + +const ( + TEST_PREFIX = "test_consumer_" +) + +func NewConsumer(cfg *Config, hdl func([]byte)) *Consumer { + + var ( + csm *cluster.Consumer + err error + group string + ) + + config := cluster.NewConfig() + config.Consumer.Return.Errors = true + config.Group.Return.Notifications = true + + if cfg.Oldest { + config.Consumer.Offsets.Initial = sarama.OffsetOldest + } else { + config.Consumer.Offsets.Initial = sarama.OffsetNewest + } + + group = cfg.Group + if group == "" { + group = fmt.Sprintf("%s%.8x", TEST_PREFIX, rand.Int63()) + } + + tlog.Infof("NewConsumer() brokers=%#v, topic=%s, workers=%d, group=%s", cfg.Brokers, cfg.Topic, cfg.Workers, group) + + if csm, err = cluster.NewConsumer(cfg.Brokers, group, []string{cfg.Topic}, config); err != nil { + tlog.Errorf("NewConsumer() create fail: %s", err) + return nil + } + + tlog.Infof("NewConsumer() Created OK! %#v", csm) + + go func() { + for err := range csm.Errors() { + tlog.Errorf("KafkaConsumer Error: %s", cfg.Brokers, group, err.Error()) + } + }() + + go func() { + for ntf := range csm.Notifications() { + tlog.Infof("KafkaConsumer Notice: %#v", cfg.Brokers, group, ntf) + } + }() + + return &Consumer{ + test: strings.HasPrefix(group, TEST_PREFIX), + consumer: csm, + workers: cfg.Workers, + handler: hdl, + stop: make(chan bool), + wg: &sync.WaitGroup{}, + } +} + +func (self *Consumer) Run() { + + for i := 0; i < self.workers; i++ { + go self.Worker() + } +} + +func (self *Consumer) Stop() { + close(self.stop) + self.wg.Wait() + self.consumer.Close() +} + +func (self *Consumer) Worker() { + self.wg.Add(1) + defer self.wg.Done() + for { + select { + case msg, ok := <-self.consumer.Messages(): + if ok { + //tlog.Debugf("topic: %s, partition: %d, offset: %d, msg: %s", msg.Topic, msg.Partition, msg.Offset, string(msg.Value)) + self.handler(msg.Value) + + // 测试用的group不标记处理完毕，防止破坏集群数据 + if !self.test { + self.consumer.MarkOffset(msg, "done") + } + } + case <-self.stop: + tlog.Info("KafkaConsumer stop signal received!") + return + } + } +} diff --git a/kv/kv.go b/kv/kv.go new file mode 100644 index 0000000..a7d9d5b --- /dev/null +++ b/kv/kv.go @@ -0,0 +1,98 @@ +package kv + +import ( + "errors" + "github.com/go-redis/redis" + "net" + "time" +) + +var x map[string]*redis.Client = make(map[string]*redis.Client) + +type Config struct { + Addrs []string `toml:"addrs" json:"addrs"` + Pwd string `toml:"pwd" json:"pwd"` + PoolSize int `toml:"pool_size" json:"pool_size"` + ReadTimeout int `toml:"read_timeout" json:"read_timeout"` +} + +type UnixConfig struct { + Addr string `toml:"addr" json:"addr"` + PoolSize int `toml:"pool_size" json:"pool_size"` +} + +func AddByUnix(name string, c *UnixConfig) error { + p := redis.NewClient(&redis.Options{ + Network: "unix", + Addr: c.Addr, + PoolSize: c.PoolSize, + }) + + x[name] = p + return nil +} + +func Add(name string, c *Config) error { + p, err := NewRedisClientWithConfig(c) + if err != nil { + return err + } + x[name] = p + return nil +} + +func GetRedisConn(name string) (*redis.Client, error) { + conn, ok := x[name] + if ok { + return conn, nil + } + return nil, errors.New("no redis conn") +} + +func (c *Config) Parse() (*redis.Options, error) { + redisNum := len(c.Addrs) + if redisNum < 1 { + return nil, errors.New("redis addrs is empty") + } + ch := make(chan []string, redisNum) + for i := 0; i < redisNum; i++ { + list := make([]string, redisNum) + for j := 0; j < redisNum; j++ { + list[j] = c.Addrs[(i+j)%redisNum] + } + ch <- list + } + + if c.ReadTimeout < 1 { + c.ReadTimeout = 1 + } + + options := &redis.Options{ + Password: c.Pwd, + PoolSize: c.PoolSize, + ReadTimeout: time.Duration(c.ReadTimeout) * time.Second, + WriteTimeout: time.Second, + PoolTimeout: time.Second, + IdleTimeout: 60 * time.Second, + Dialer: func() (net.Conn, error) { + list := <-ch + ch <- list + for _, addr := range list { + c, err := net.DialTimeout("tcp", addr, 1000*time.Millisecond) + if err == nil { + return c, nil + } + } + return nil, errors.New("all redis down") + }, + } + return options, nil +} + +func NewRedisClientWithConfig(c *Config) (*redis.Client, error) { + options, err := c.Parse() + if err != nil { + return nil, err + } + return redis.NewClient(options), nil +} diff --git a/kv/mkv.go b/kv/mkv.go new file mode 100644 index 0000000..cc1935f --- /dev/null +++ b/kv/mkv.go @@ -0,0 +1,61 @@ +package kv + +import ( + "errors" + "github.com/bradfitz/gomemcache/memcache" +) + +var ErrNoClient error = errors.New("no memcache client") +var mClient *memcache.Client + +//memcache 的配置 +type MConfig struct { + Addrs []string `toml:"addrs" json:"addrs"` + PoolSize int `toml:"pool_size" json:"pool_size"` +} + +func (c *MConfig) GetAddrs() []string { + if c != nil { + return c.Addrs + } + return []string{} +} + +func (c *MConfig) GetPoolSize() int { + if c != nil { + return c.PoolSize + } + return 5 +} + +func InitMcClient(c *MConfig) { + mClient = memcache.New(c.GetAddrs()...) + mClient.MaxIdleConns = c.GetPoolSize() + return +} + +func GetMcClient() *memcache.Client { + return mClient +} + +func Get(key string) ([]byte, error) { + client := GetMcClient() + if client == nil { + return nil, ErrNoClient + } + + item, err := client.Get(key) + if err != nil { + return nil, err + } + return item.Value, nil +} + +func Del(key string) error { + if client := GetMcClient(); client == nil { + return ErrNoClient + } else { + return client.Delete(key) + } + return nil +} diff --git a/lrucache/lrucache.go b/lrucache/lrucache.go new file mode 100644 index 0000000..7f943d9 --- /dev/null +++ b/lrucache/lrucache.go @@ -0,0 +1,108 @@ +package lrucache + +import ( + "errors" + "github.com/creachadair/cityhash" + "github.com/hashicorp/golang-lru" + "globalfintech/golib/tlog" + "time" +) + +// API +var ( + Nil error = errors.New("cache: nil") + ErrExpTime error = errors.New("args error, expTime must be > 0") + dbs map[int]*lru.Cache +) + +func Set(key string, value interface{}) error { + return SetEx(key, value, 86400) +} + +func SetEx(key string, value interface{}, exp int) error { + return setEx(key, value, exp) +} + +func Get(key string) (interface{}, error) { + return get(key) +} + +func Del(key string) error { + return del(key) +} + +// internal + +func init() { + if dbs == nil { + dbs = make(map[int]*lru.Cache, 1024) + for i := 0; i < 1024; i++ { + dbs[i], _ = lru.New(3000) + } + } + go flush() +} + +func flush() { + i := 0 + for range time.NewTicker(time.Minute).C { + start := time.Now() + shard := i % 1024 + db, _ := dbs[shard] + keys := db.Keys() + for _, key := range keys { + if keyS, ok := key.(string); ok { + getByDb(keyS, db) + } + } + i++ + tlog.Infof("FlushKey||time=%s||shard=%d||old=%d||new=%d", time.Since(start), shard, len(keys), db.Len()) + } +} + +type item struct { + d interface{} + t int +} + +func setEx(key string, value interface{}, exp int) error { + if exp < 1 { + return ErrExpTime + } + + db, _ := dbs[hash(key)] + db.Add(key, &item{d: value, t: int(time.Now().Unix()) + exp}) + return nil +} + +func get(key string) (interface{}, error) { + db, _ := dbs[hash(key)] + return getByDb(key, db) +} + +func getByDb(key string, db *lru.Cache) (interface{}, error) { + if value, ok := db.Get(key); ok { + if item, ok := value.(*item); ok { + if item.t >= int(time.Now().Unix()) { + return item.d, nil + } else { + delByDb(key, db) + } + } + } + return nil, Nil +} + +func del(key string) error { + db, _ := dbs[hash(key)] + return delByDb(key, db) +} + +func delByDb(key string, db *lru.Cache) error { + db.Remove(key) + return nil +} + +func hash(key string) int { + return int(cityhash.Hash32([]byte(key)) % 1024) +} diff --git a/metric/alert.go b/metric/alert.go new file mode 100644 index 0000000..14ac0cb --- /dev/null +++ b/metric/alert.go @@ -0,0 +1,14 @@ +package metric + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +var singletonCounter prometheus.Counter +var once sync.Once + +func SingletonCounter() { + +} diff --git a/metric/examples/exporter_demo.go b/metric/examples/exporter_demo.go new file mode 100644 index 0000000..73b7b8f --- /dev/null +++ b/metric/examples/exporter_demo.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "globalfintech/golib/metric" +) + +func main() { + cfg := metric.ExporterCfg{ + Listener: nil, + RegisterConsul: false, + ProductName: "ProductName_001", + ServiceRegion: "ServiceRegion_001", + AppName: "AppName_001", + } + + go func() { + if err := metric.InitExporter(cfg); err != nil { + fmt.Println("prometheus exporter stopped,err: ", err) + } + }() + fmt.Println("prometheus exporter started") + + select {} +} diff --git a/metric/examples/main.go b/metric/examples/main.go new file mode 100644 index 0000000..34990c5 --- /dev/null +++ b/metric/examples/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "github.com/prometheus/client_golang/prometheus" + "globalfintech/golib/metric" + "time" +) + +var callCounter = metric.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "crm", + Name: "call_total_count2", + Help: "Total Request", + }, + []string{"privilege", "uri"}, +) + +func test2() { + pusher := metric.InitPusher("http://ops-dev.a.mobimagic.com:9091", "MetricsExample3", prometheus.Labels{}) + + pusher.Collector(callCounter).Add() + pusher.Push() + //pusher.Push(callCounter) +} + +func main() { + //test2() + + var labels = prometheus.Labels{"app": "app2", + "country": "INDIA", + "service": "test2", + } + + var pushGateway, job = "http://ops-dev.a.mobimagic.com:9091", "app2" + + //fmt.Println("labels:", labels) + metric.InitDefaultPusher(pushGateway, job, labels) + + for { + callCounter.With(prometheus.Labels{"privilege": "test", "uri": "xxxxx/sswwe"}).Inc() + time.Sleep(time.Second * 1) + fmt.Println("Inc 1") + + } +} diff --git a/metric/exporter.go b/metric/exporter.go new file mode 100644 index 0000000..54a85c7 --- /dev/null +++ b/metric/exporter.go @@ -0,0 +1,113 @@ +package metric + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "strconv" + "time" + + consulapi "github.com/hashicorp/consul/api" + "github.com/prometheus/client_golang/prometheus/promhttp" + "globalfintech/golib/tlog" +) + +const ( + DefaultListenerAddr = "0.0.0.0:9166" +) + +type ExporterCfg struct { + Listener net.Listener + RegisterConsul bool + ProductName string + ServiceRegion string + AppName string +} + +// Exporter init function must be executed asynchronously +func InitExporter(cfg ExporterCfg) (err error) { + lis := cfg.Listener + if cfg.Listener == nil { + if lis, err = net.Listen("tcp", DefaultListenerAddr); err != nil { + tlog.Error("fail to create listener with default address") + return err + } + } + + _, port, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + tlog.Error("fail to get port from listener, err: ", err) + return err + } + + h := http.NewServeMux() + h.Handle("/metrics", promhttp.Handler()) + h.HandleFunc("/health_checker", healthChecker) + server := &http.Server{ + Handler: h, + ReadTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + IdleTimeout: 30 * time.Second, + WriteTimeout: 10 * time.Second, + } + + if cfg.RegisterConsul { + var tags []string + tags = append(tags, fmt.Sprintf("app=%s", cfg.ProductName)) + tags = append(tags, fmt.Sprintf("country=%s", cfg.ServiceRegion)) + err = consulRegister(port, "", tags, cfg.AppName) + if err != nil { + tlog.Error("consul register failed, err: ", err) + return err + } + } + + if err = server.Serve(lis); err != nil { + tlog.Error("fail to init prometheus exporter, err: ", err) + return err + } + + return nil +} + +func consulRegister(checkPort string, regAddr string, tags []string, name string) (err error) { + config := consulapi.DefaultConfig() + if regAddr != "" { + config.Address = regAddr + } + + client, err := consulapi.NewClient(config) + if err != nil { + tlog.Error("fail to new consul client, err: ", err) + return err + } + + intPort, _ := strconv.Atoi(checkPort) + hostName, _ := os.Hostname() + + registration := &consulapi.AgentServiceRegistration{ + Port: intPort, + Name: name, + ID: name + ":" + hostName, + Tags: append(tags, "lang=go"), + Check: &consulapi.AgentServiceCheck{ + HTTP: fmt.Sprintf("http://%s:%s%s", "127.0.0.1", checkPort, "/health_checker"), + Interval: "10s", + Timeout: "10s", + }, + } + + return client.Agent().ServiceRegister(registration) +} + +func healthChecker(resp http.ResponseWriter, req *http.Request) { + data := map[string]interface{}{ + "code": 0, + "message": "ok", + "server_time": time.Now().Unix(), + } + res, _ := json.Marshal(data) + resp.Write(res) +} diff --git a/metric/metric.go b/metric/metric.go new file mode 100644 index 0000000..55cc608 --- /dev/null +++ b/metric/metric.go @@ -0,0 +1,160 @@ +package metric + +import ( + "github.com/prometheus/client_golang/prometheus" + "sync" +) + +const missVal = "MissLabel" + +// CounterVecDecorator Add safe decorator +type CounterVecDecorator struct { + *prometheus.CounterVec + secureLabels []string + once *sync.Once +} + +func NewCounterVec(opts prometheus.CounterOpts, labelNames []string) *CounterVecDecorator { + v := &CounterVecDecorator{prometheus.NewCounterVec(opts, labelNames), labelNames, &sync.Once{}} + + return v +} + +// With is secure wrap for with of prometheus.CounterVec +// there is no panic, and always calculate to metrics , if miss values, will calculate to MissLabel +// It's secure +func (v *CounterVecDecorator) With(labels prometheus.Labels) prometheus.Counter { + // To avoid initialization sequence, so must delay register to with func + v.once.Do(func() { + Register(v) + }) + + safeLabels := labelAlignment(v.secureLabels, labels) + + // + c, err := v.GetMetricWith(safeLabels) + + if err != nil { + // should never happen , but should still add handler to record alert + // TODO singleton alert Gauge and return an alert counter + + } + return c +} + +// GaugeVecDecorator Add safe decorator +type GaugeVecDecorator struct { + *prometheus.GaugeVec + secureLabels []string + once *sync.Once +} + +func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *GaugeVecDecorator { + v := &GaugeVecDecorator{prometheus.NewGaugeVec(opts, labelNames), labelNames, &sync.Once{}} + + return v +} + +// With is secure wrap for with of prometheus.GaugeVec +// there is no panic, and always calculate to metrics , if miss values, will calculate to MissLabel +// It's secure +func (v *GaugeVecDecorator) With(labels prometheus.Labels) prometheus.Gauge { + // To avoid initialization sequence, so must delay register to with func + v.once.Do(func() { + Register(v) + }) + + safeLabels := labelAlignment(v.secureLabels, labels) + // + c, err := v.GetMetricWith(safeLabels) + + if err != nil { + // should never happen , but should still add handler to record alert + // TODO singleton alert Gauge and return + } + return c +} + +// HistogramVecDecorator Add safe +type HistogramVecDecorator struct { + *prometheus.HistogramVec + secureLabels []string + once *sync.Once +} + +func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *HistogramVecDecorator { + v := &HistogramVecDecorator{prometheus.NewHistogramVec(opts, labelNames), labelNames, &sync.Once{}} + + return v +} + +// With is secure wrap for with of prometheus.HistogramVec +// there is no panic, and always calculate to metrics , if miss values, will calculate to MissLabel +// It's secure +func (v *HistogramVecDecorator) With(labels prometheus.Labels) prometheus.Observer { + // To avoid initialization sequence, so must delay register to with func + v.once.Do(func() { + Register(v) + }) + + safeLabels := labelAlignment(v.secureLabels, labels) + // + c, err := v.GetMetricWith(safeLabels) + + if err != nil { + // should never happen , but should still add handler to record alert + // TODO singleton alert Histogram and return + } + return c +} + +// SummaryVecDecorator Add safe +type SummaryVecDecorator struct { + *prometheus.SummaryVec + secureLabels []string + once *sync.Once +} + +func NewSummaryVec(opts prometheus.SummaryOpts, labelNames []string) *SummaryVecDecorator { + v := &SummaryVecDecorator{prometheus.NewSummaryVec(opts, labelNames), labelNames, &sync.Once{}} + //if len(autoReg) == 0 || autoReg[0] == true { + // Register(v) + //} + return v +} + +// With is secure wrap for with of prometheus.SummaryVec +// there is no panic, and always calculate to metrics , if miss values, will calculate to MissLabel +// It's secure +func (v *SummaryVecDecorator) With(labels prometheus.Labels) prometheus.Observer { + // To avoid initialization sequence, so must delay register to with func + v.once.Do(func() { + Register(v) + }) + + safeLabels := labelAlignment(v.secureLabels, labels) + // + c, err := v.GetMetricWith(safeLabels) + + if err != nil { + // should never happen , but should still add handler to record alert + // TODO singleton alert Gauge and return + } + //prometheus.NewCounter(v.SummaryVec.) + return c +} + +// 标签对齐 +func labelAlignment(labelNames []string, labels prometheus.Labels) prometheus.Labels { + + newLabels := make(prometheus.Labels, len(labelNames)) + + for _, val := range labelNames { + if v, ok := labels[val]; ok { + newLabels[val] = v + } else { + newLabels[val] = missVal + } + } + return newLabels +} diff --git a/metric/metric_test.go b/metric/metric_test.go new file mode 100644 index 0000000..6336e81 --- /dev/null +++ b/metric/metric_test.go @@ -0,0 +1,48 @@ +package metric + +import ( + "github.com/prometheus/client_golang/prometheus" + "reflect" + "testing" +) + +func Test_labelAlignment(t *testing.T) { + tds := []struct { + in1 []string + in2 prometheus.Labels + in2New prometheus.Labels + }{ + { + []string{"label1", "label2"}, + map[string]string{"label1": "1"}, + map[string]string{"label1": "1", "label2": missVal}, + }, + { + []string{"label1", "label3"}, + map[string]string{"label1": "1", "label2": "2"}, + map[string]string{"label1": "1", "label3": missVal}, + }, + { + []string{}, + map[string]string{"label1": "1", "label2": "2"}, + map[string]string{}, + }, + { + []string{"label1", "label3", "label4"}, + map[string]string{"label1": "1", "label2": "2"}, + map[string]string{"label1": "1", "label3": missVal, "label4": missVal}, + }, + } + for _, d := range tds { + //var org map[string]string + //copy(org, d.in2) + result := labelAlignment(d.in1, d.in2) + t.Log(result, d.in2New) + + if !reflect.DeepEqual(result, d.in2New) { + + t.Errorf("Input: %#v, Expected is not same as , get ", d) + } + + } +} diff --git a/metric/push.go b/metric/push.go new file mode 100644 index 0000000..141a9b6 --- /dev/null +++ b/metric/push.go @@ -0,0 +1,69 @@ +package metric + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/push" + "os" + "time" +) + +var defaultPusher *push.Pusher + +const ( + defaultPushIntervalSecond = 5 +) + +func InitDefaultPusher(gateway string, job string, labels prometheus.Labels, pushIntervalSecond ...int) { + defaultPusher = InitPusher(gateway, job, labels) + + // for default pusher , we should add process collector and go collector + // then we can monitor all base monitor on our job + defaultPusher. + Collector(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})). + Collector(prometheus.NewGoCollector()). + Add() + + // TODO implement init + // and push + //defaultPusher = InitPusher("") + go func() { + var pushInterval = defaultPushIntervalSecond * time.Second + if len(pushIntervalSecond) > 0 { + pushInterval = time.Second * time.Duration(pushIntervalSecond[0]) + } + for { + defaultPusher.Push() + time.Sleep(pushInterval) + } + }() +} + +// ExitTriggerLastPush +// TODO +func ExitTriggerLastPush() { + + defaultPusher.Push() + +} + +//var MustLabel prometheus.Labels +// +//type RequireLabelConf struct { +// Service string `json:"service";yaml:` +// App string `` +// Country string +//} + +// Init Pusher +// return prometheus pusher +// TODO add labels +// need lan ,app,country, service labels to implement +func InitPusher(gateway string, job string, labels prometheus.Labels) *push.Pusher { + host, _ := os.Hostname() + + p := push.New(gateway, job).Grouping("instance", host).Grouping("lang","go") + for name, value := range labels { + p.Grouping(name, value) + } + return p +} diff --git a/metric/register.go b/metric/register.go new file mode 100644 index 0000000..47f8fd0 --- /dev/null +++ b/metric/register.go @@ -0,0 +1,27 @@ +package metric + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// Register Here, we must decide pusher/exporter which we use +// it's very important, note: var/init/func init sort +// keep, when a git rep, not only a service or job +// collector will mixed with Push And Registered for pull +// So register with pull and add for push collector must control in one place +// then we can use conf to switch +// avoid collector duplicate +// we don't use must register, to avoid panic +func Register(cs ...prometheus.Collector) { + if defaultPusher == nil { + // register by exporter + for _, mv := range cs { + prometheus.Register(mv) + } + } else { + // register by pusher + for _, mv := range cs { + defaultPusher.Collector(mv).Add() + } + } +} diff --git a/metrics/example1/example1.go b/metrics/example1/example1.go new file mode 100644 index 0000000..dbb4aed --- /dev/null +++ b/metrics/example1/example1.go @@ -0,0 +1,197 @@ +package main + +import ( + "flag" + "globalfintech/golib/metrics" + "globalfintech/golib/metrics/prometheus" + "globalfintech/golib/tlog" + "math/rand" + "os" + "os/signal" + "strconv" + "sync" + "syscall" + "time" + + stdprometheus "github.com/prometheus/client_golang/prometheus" + "golang.org/x/net/context" +) + +func main() { + // parse the flags + flag.Parse() + + processMetrics, err := metrics.NewProcessMetrics("test", "INDIA", "testProject") + if err != nil { + tlog.Error(err) + return + } + workPort := processMetrics.GetMetricsPort() + // init Exporter + tlog.Info("Beginning to serve on port :", workPort) + processMetrics.InitExporter() + + // You can registry metrics any where as you want + // Now we registry a local gauge in function main + workersGauge := prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: "worker", + Subsystem: "jobs", + Name: "worker_number", + Help: "Number of workers.", + }, []string{"server"}) + + host, _ := os.Hostname() + workersGauge.With("server", host).Set(float64(workers)) + + // create a channel with a 1000 Job buffer + jobChannel := make(chan *Job, 1000) + // start the job processor + go startJobProcessor(jobChannel) + // start a Goroutine to create some mock jobs + go createJobs(jobChannel) + + exit := make(chan os.Signal, 1) + + // kill (no param) default send syscanll.SIGTERM + // kill -2 is syscall.SIGINT + // kill -9 is syscall.SIGKILL but can't be catch, so don't need add it + signal.Notify(exit, syscall.SIGTERM, syscall.SIGINT) + <-exit + + tlog.Info("Graceful shutdown Server in 10 seconds...") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // set workers to 0 , then stop all workers and job creator + workers = 0 + + select { + case <-ctx.Done(): + tlog.Info("timeout of 10 seconds.") + } + + // init pusher + // pusher usually uses at DAG/Cron job processor + pusher := metrics.InitPusher("http://ops-dev.a.mobimagic.com:9091", "MetricsExample1") + if err := pusher.Collector(workerTotalCounter.Cv).Add(); err != nil { + tlog.Error("Could not push to Pushgateway:", err) + } + // time is up + close(jobChannel) + tlog.Infof("Jobs left: %d", len(jobChannel)) + for leftJob := range jobChannel { + tlog.Info(leftJob) + } + + tlog.Info("Server exit") + +} + +var ( + modes = []string{"SMS", "WeChat", "WhatsApp", "GooglePush", "ApplePush"} + workers = 0 + + // Two ways to define name + // Name : fullname + // or Namespace / Subsystem / Name to combine a full name + workerTotalCounter = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ + Name: "worker_jobs_processed_total", + Help: "Total number of jobs processed by the workers.", + }, []string{"worker_id", "mode"}) + + inqueueGauge = prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: "worker", + Subsystem: "jobs", + Name: "inqueue", + Help: "Number of jobs in queue.", + }, []string{"mode"}) + + processingTimeHistogram = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ + Name: "worker_jobs_process_time_seconds", + Help: "Amount of time spent processing jobs", + Buckets: []float64{1, 2, 3, 4, 5}, //if no set, then use DefBuckets : []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} + }, []string{"worker_id", "mode"}) +) + +type Job struct { + Mode string + Sleep time.Duration +} + +func init() { + flag.IntVar(&workers, "workers", 10, "Number of workers to use") +} + +func getMode() string { + return modes[rand.Int()%len(modes)] +} + +// makeJob creates a new job with a random sleep time between 0.5s and 5s +func makeJob() *Job { + return &Job{ + Mode: getMode(), + Sleep: time.Duration(rand.Int()%4500+500) * time.Millisecond, + } +} + +func createJobs(jobs chan<- *Job) { + for { + if workers == 0 { + tlog.Info("Stop create jobs.") + return + } + + // create a random job + job := makeJob() + // track the job in the inqueue tracker + inqueueGauge.With("mode", job.Mode).Add(1) + // send the job down the channel + jobs <- job + // don't pile up too quickly + time.Sleep(200 * time.Millisecond) + } +} + +// creates a worker that pulls jobs from the job channel +func startWorker(workerID int, jobs <-chan *Job) { + for { + select { + // read from the job channel + case job := <-jobs: + startTime := time.Now() + // decrement the in queue tracker + inqueueGauge.With("mode", job.Mode).Sub(1) + + // fake processing the request + time.Sleep(job.Sleep) + tlog.Infof("[%d][%s] Processed job in %0.3f seconds", workerID, job.Mode, time.Now().Sub(startTime).Seconds()) + // track the total number of jobs processed by the worker + workerTotalCounter.With("worker_id", strconv.FormatInt(int64(workerID), 10), "mode", job.Mode).Inc() + processingTimeHistogram.With("worker_id", strconv.FormatInt(int64(workerID), 10), "mode", job.Mode).Observe(time.Now().Sub(startTime).Seconds()) + + if workers == 0 { + tlog.Infof("worker[%d] quit", workerID) + return + } + + } + } +} + +func startJobProcessor(jobs <-chan *Job) { + tlog.Infof("[INFO] starting %d workers\n", workers) + wait := sync.WaitGroup{} + // notify the sync group we need to wait for 10 goroutines + wait.Add(workers) + + // start 10 works + for i := 0; i < workers; i++ { + go func(workerID int) { + // start the worker + startWorker(workerID, jobs) + wait.Done() + }(i) + } + + wait.Wait() +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..5aa636e --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,190 @@ +package metrics + +import ( + "encoding/json" + "errors" + "fmt" + "globalfintech/golib/tlog" + "net/http" + "os" + "strconv" + "time" + + consulapi "github.com/hashicorp/consul/api" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/client_golang/prometheus/push" +) + +const ( + defaultAddr string = "9166" +) + +// Counter describes a metric that accumulates values monotonically. +// An example of a counter is the number of received HTTP requests. +type Counter interface { + With(labelValues ...string) Counter + Add(delta float64) + Inc() +} + +// Gauge describes a metric that takes specific values over time. +// An example of a gauge is the current depth of a job queue. +type Gauge interface { + With(labelValues ...string) Gauge + Set(value float64) + Add(delta float64) + Sub(delta float64) +} + +// Histogram describes a metric that takes repeated observations of the same +// kind of thing, and produces a statistical summary of those observations, +// typically expressed as quantiles or buckets. An example of a histogram is +// HTTP request latencies. +type Histogram interface { + With(labelValues ...string) Histogram + Observe(value float64) +} + +// LabelValues is a type alias that provides validation on its With method. +// Metrics may include it as a member to help them satisfy With semantics and +// save some code duplication. +type LabelValues []string + +// With validates the input, and returns a new aggregate labelValues. +func (lvs LabelValues) With(labelValues ...string) LabelValues { + if len(labelValues)%2 != 0 { + labelValues = append(labelValues, "unknown") + } + return append(lvs, labelValues...) +} + +// new a *ProcessMetrics +// productName,serviceRegion,appName all those params must be passed +// port if not be pass,will use defaultAddr 9166 +func NewProcessMetrics(productName, serviceRegion, appName string, port ...string) (*ProcessMetrics, error) { + if len(productName) == 0 || len(serviceRegion) == 0 || len(appName) == 0 { + err := errors.New("productName,serviceRegion,appName all those params must be passed") + tlog.Error(err) + return nil, err + } + addr := defaultAddr + if len(port) > 0 { + addr = port[0] + } + var processMetrics ProcessMetrics + processMetrics.Port = addr + processMetrics.ProductName = productName + processMetrics.ServiceRegion = serviceRegion + processMetrics.Appname = appName + return &processMetrics, nil +} + +type ProcessMetrics struct { + Port string // port default 9166 + ProductName string + ServiceRegion string + Appname string +} + +// get real work port +func (s *ProcessMetrics) GetMetricsPort() string { + if len(s.Port) > 0 { + return s.Port + } + return defaultAddr +} + +// Init Export on GF default port 9166 +// If you want custom listen port, just pass your port to MetricsParams struct +// return exporter http server +func (s *ProcessMetrics) InitExporter() *http.Server { + h := http.NewServeMux() + h.Handle("/metrics", promhttp.Handler()) + h.HandleFunc("/health_checker", s.healthChecker) + addr := defaultAddr + if len(s.Port) > 0 { + addr = s.Port + } else { + s.Port = defaultAddr + } + server := &http.Server{ + Addr: ":" + addr, + Handler: h, + ReadTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + IdleTimeout: 30 * time.Second, + WriteTimeout: 10 * time.Second, + } + + { + var tags []string + + tags = append(tags, fmt.Sprintf("app=%s", s.ProductName)) + tags = append(tags, fmt.Sprintf("country=%s", s.ServiceRegion)) + s.initExporterOnExistHttpServer(addr, "/health_checker", tags, s.Appname) + } + go func() { + if err := server.ListenAndServe(); err != nil { + tlog.Fatal(err) + } + }() + return server +} + +func (s *ProcessMetrics) healthChecker(resp http.ResponseWriter, req *http.Request) { + data := map[string]interface{}{ + "code": 0, + "message": "ok", + "server_time": time.Now().Unix(), + } + res, _ := json.Marshal(data) + resp.Write(res) +} + +// Init Pusher +// return prometheus pusher +func InitPusher(gateway string, job string) *push.Pusher { + host, _ := os.Hostname() + return push.New(gateway, job).Grouping("instance", host) +} + +// InitExporterOnExistHttpServer Init http exporter +// Why we init exporter on exist http server: +// to avoid new custom port +// which should add healthCheckRouter - like /healthChecker func +// must export a route: /metrics router for prometheus +// promhttp.Handler().ServeHTTP(c.Ctx.ResponseWriter, c.Ctx.Request) +func (s *ProcessMetrics) initExporterOnExistHttpServer(port string, healthCheckRouter string, tags []string, name string, id ...string) { + config := consulapi.DefaultConfig() + client, err := consulapi.NewClient(config) + if err != nil { + tlog.Error("consul client error : ", err) + return + } + registration := new(consulapi.AgentServiceRegistration) + registration.Name = name + intPort, _ := strconv.Atoi(port) + registration.Port = intPort + hostName, _ := os.Hostname() + if len(id) > 0 { + registration.ID = registration.Name + ":" + id[0] + } else { + registration.ID = registration.Name + ":" + hostName + } + + registration.Tags = append(tags, "lang=go") + + registration.Check = new(consulapi.AgentServiceCheck) + registration.Check.HTTP = fmt.Sprintf("http://%s:%s%s", + "127.0.0.1", port, healthCheckRouter) + registration.Check.Interval = "5s" + registration.Check.Timeout = "3s" + // 为兼容现有的 平滑逻辑 + //err = client.Agent().ServiceDeregister(registration.ID) + + err = client.Agent().ServiceRegister(registration) + + if err != nil { + tlog.Error(err) + } +} diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go new file mode 100644 index 0000000..3622d08 --- /dev/null +++ b/metrics/prometheus/prometheus.go @@ -0,0 +1,169 @@ +package prometheus + +import ( + "github.com/prometheus/client_golang/prometheus" + "globalfintech/golib/metrics" +) + +// Counter implements Counter, via a Prometheus CounterVec. +type Counter struct { + Cv *prometheus.CounterVec + lvs metrics.LabelValues +} + +// NewCounterFrom constructs and registers a Prometheus CounterVec, +// and returns a usable Counter object. +func NewCounterFrom(opts prometheus.CounterOpts, labelNames []string) *Counter { + cv := prometheus.NewCounterVec(opts, labelNames) + prometheus.MustRegister(cv) + return NewCounter(cv) +} + +// NewCounter wraps the CounterVec and returns a usable Counter object. +func NewCounter(cv *prometheus.CounterVec) *Counter { + return &Counter{ + Cv: cv, + } +} + +// With implements Counter. +func (c *Counter) With(labelValues ...string) metrics.Counter { + return &Counter{ + Cv: c.Cv, + lvs: c.lvs.With(labelValues...), + } +} + +// Add implements Counter. +func (c *Counter) Add(delta float64) { + c.Cv.With(makeLabels(c.lvs...)).Add(delta) +} + +// Inc implement Counter. +func (c *Counter) Inc() { + c.Cv.With(makeLabels(c.lvs...)).Inc() +} + +// Gauge implements Gauge, via a Prometheus GaugeVec. +type Gauge struct { + gv *prometheus.GaugeVec + lvs metrics.LabelValues +} + +// NewGaugeFrom construts and registers a Prometheus GaugeVec, +// and returns a usable Gauge object. +func NewGaugeFrom(opts prometheus.GaugeOpts, labelNames []string) *Gauge { + gv := prometheus.NewGaugeVec(opts, labelNames) + prometheus.MustRegister(gv) + return NewGauge(gv) +} + +// NewGauge wraps the GaugeVec and returns a usable Gauge object. +func NewGauge(gv *prometheus.GaugeVec) *Gauge { + return &Gauge{ + gv: gv, + } +} + +// With implements Gauge. +func (g *Gauge) With(labelValues ...string) metrics.Gauge { + return &Gauge{ + gv: g.gv, + lvs: g.lvs.With(labelValues...), + } +} + +// Set implements Gauge. +func (g *Gauge) Set(value float64) { + g.gv.With(makeLabels(g.lvs...)).Set(value) +} + +// Add is supported by Prometheus GaugeVecs. +func (g *Gauge) Add(delta float64) { + g.gv.With(makeLabels(g.lvs...)).Add(delta) +} + +// Sub is supported by Prometheus GaugeVecs. +func (g *Gauge) Sub(delta float64) { + g.gv.With(makeLabels(g.lvs...)).Sub(delta) +} + +// Summary implements Histogram, via a Prometheus SummaryVec. The difference +// between a Summary and a Histogram is that Summaries don't require predefined +// quantile buckets, but cannot be statistically aggregated. +type Summary struct { + sv *prometheus.SummaryVec + lvs metrics.LabelValues +} + +// NewSummaryFrom constructs and registers a Prometheus SummaryVec, +// and returns a usable Summary object. +func NewSummaryFrom(opts prometheus.SummaryOpts, labelNames []string) *Summary { + sv := prometheus.NewSummaryVec(opts, labelNames) + prometheus.MustRegister(sv) + return NewSummary(sv) +} + +// NewSummary wraps the SummaryVec and returns a usable Summary object. +func NewSummary(sv *prometheus.SummaryVec) *Summary { + return &Summary{ + sv: sv, + } +} + +// With implements Histogram. +func (s *Summary) With(labelValues ...string) metrics.Histogram { + return &Summary{ + sv: s.sv, + lvs: s.lvs.With(labelValues...), + } +} + +// Observe implements Histogram. +func (s *Summary) Observe(value float64) { + s.sv.With(makeLabels(s.lvs...)).Observe(value) +} + +// Histogram implements Histogram via a Prometheus HistogramVec. The difference +// between a Histogram and a Summary is that Histograms require predefined +// quantile buckets, and can be statistically aggregated. +type Histogram struct { + hv *prometheus.HistogramVec + lvs metrics.LabelValues +} + +// NewHistogramFrom constructs and registers a Prometheus HistogramVec, +// and returns a usable Histogram object. +func NewHistogramFrom(opts prometheus.HistogramOpts, labelNames []string) *Histogram { + hv := prometheus.NewHistogramVec(opts, labelNames) + prometheus.MustRegister(hv) + return NewHistogram(hv) +} + +// NewHistogram wraps the HistogramVec and returns a usable Histogram object. +func NewHistogram(hv *prometheus.HistogramVec) *Histogram { + return &Histogram{ + hv: hv, + } +} + +// With implements Histogram. +func (h *Histogram) With(labelValues ...string) metrics.Histogram { + return &Histogram{ + hv: h.hv, + lvs: h.lvs.With(labelValues...), + } +} + +// Observe implements Histogram. +func (h *Histogram) Observe(value float64) { + h.hv.With(makeLabels(h.lvs...)).Observe(value) +} + +func makeLabels(labelValues ...string) prometheus.Labels { + labels := prometheus.Labels{} + for i := 0; i < len(labelValues); i += 2 { + labels[labelValues[i]] = labelValues[i+1] + } + return labels +} diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go new file mode 100644 index 0000000..f0b2a23 --- /dev/null +++ b/metrics/prometheus/prometheus_test.go @@ -0,0 +1,61 @@ +package prometheus + +import ( + "fmt" + stdprometheus "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "globalfintech/golib/metrics" + "io/ioutil" + "math/rand" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "testing" +) + +func TestCounter(t *testing.T) { + s := httptest.NewServer(promhttp.Handler()) + defer s.Close() + + scrape := func() string { + resp, _ := http.Get(s.URL) + buf, _ := ioutil.ReadAll(resp.Body) + return string(buf) + } + + namespace, subsystem, name := "ns", "ss", "foo" + re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{alpha="alpha-value",beta="beta-value"} ([0-9\.]+)`) + + counter := NewCounterFrom(stdprometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: "This is the help string.", + }, []string{"alpha", "beta"}).With("beta", "beta-value", "alpha", "alpha-value") // order shouldn't matter + + value := func() float64 { + matches := re.FindStringSubmatch(scrape()) + f, _ := strconv.ParseFloat(matches[1], 64) + return f + } + want := FillCounter(counter) + if have := value(); want != have { + t.Fatal(fmt.Errorf("want %f, have %f", want, have)) + } + +} + +// FillCounter puts some deltas through the counter and returns the total value. +func FillCounter(counter metrics.Counter) float64 { + a := rand.Perm(100) + n := rand.Intn(len(a)) + + var want float64 + for i := 0; i < n; i++ { + f := float64(a[i]) + counter.Add(f) + want += f + } + return want +} diff --git a/qgrpc/config.go b/qgrpc/config.go new file mode 100644 index 0000000..c0b4348 --- /dev/null +++ b/qgrpc/config.go @@ -0,0 +1,66 @@ +package qgrpc + +import ( + "fmt" + "net" + "os" + "strings" +) + +// 基础配置，公共字段 +type BasicConfig struct { + DialTimeout int `toml:"dial_timeout"` // 连接超时时间 + KeepAlive int `toml:"keep_alive"` // 保活时间 + EndPoints []string `toml:"endpoints"` // etcd的地址 + Prefix string `toml:"prefix"` // etcd服务的路径前缀，区分环境用 +} + +// 服务端配置 +type ServerConfig struct { + BasicConfig + Servers []string `toml:"servers"` // 注册的服务名称，可以是多个 + Addr string `toml:"addr"` // 服务地址 +} + +func (self *ServerConfig) GetAddr() string { + if strings.HasPrefix(self.Addr, ":") { + localIP := getLocalIP() + return fmt.Sprintf("%s%s", localIP, self.Addr) + } + return self.Addr +} + +// 客户端配置 +type ClientConfig struct { + BasicConfig + WatchServers []string `toml:"watch_servers"` // 依赖的服务名称，多个服务用','分割；每个服务名后增加'|ip:port'表示默认连接的服务地址 +} + +func getHostName() string { + if h, err := os.Hostname(); err == nil { + return h + } + return getLocalIP() +} + +//获取本机的内网Ip, 如果发现对方的ip 和自己的ip 相同，用127.0.0.1 替代 +func getLocalIP() string { + ifaces, _ := net.Interfaces() + for _, i := range ifaces { + addrs, _ := i.Addrs() + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + ipAddr := ip.String() + if strings.HasPrefix(ipAddr, "172.") || strings.HasPrefix(ipAddr, "192.") || strings.HasPrefix(ipAddr, "10.") { + return ipAddr + } + } + } + return "127.0.0.1" +} diff --git a/qgrpc/op.go b/qgrpc/op.go new file mode 100644 index 0000000..758e3c9 --- /dev/null +++ b/qgrpc/op.go @@ -0,0 +1,141 @@ +package qgrpc + +import ( + "context" + "fmt" + "github.com/coreos/etcd/clientv3" + "globalfintech/golib/qgrpc/ping" + "globalfintech/golib/tlog" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" + "strings" + "time" +) + +type Server struct { + userCfg *ServerConfig + etcdCli *clientv3.Client + etcdId clientv3.LeaseID +} + +type Client struct { + userCfg *ClientConfig + etcdCli *clientv3.Client + etcdId clientv3.LeaseID + rpcServers map[string]*grpc.ClientConn +} + +type RpcServer struct { + Name string //服务的名字 + Addrs []string //默认的地址 +} + +const ( + DFT_DIAL_TIMEOUT = 3 +) + +func etcdDialWrapper(cfg *BasicConfig) (*clientv3.Client, error) { + dialTimeout := DFT_DIAL_TIMEOUT + if cfg.DialTimeout > 0 { + dialTimeout = cfg.DialTimeout + } + return clientv3.New(clientv3.Config{ + Endpoints: cfg.EndPoints, + DialTimeout: time.Duration(dialTimeout) * time.Second, + }) +} + +func ServerRegist(cfg *ServerConfig) (server *Server, err error) { + server = &Server{userCfg: cfg} + + if server.etcdCli, err = etcdDialWrapper(&cfg.BasicConfig); err != nil { + return nil, err + } + + server.register() + + return server, nil +} + +func NewClient(cfg *ClientConfig) (client *Client, err error) { + client = &Client{userCfg: cfg} + + if client.etcdCli, err = etcdDialWrapper(&cfg.BasicConfig); err != nil { + return nil, err + } + + client.rpcServers = make(map[string]*grpc.ClientConn) + + keepAlive := 3 + if cfg.KeepAlive > 0 { + keepAlive = cfg.KeepAlive + } + + for _, watchServer := range cfg.WatchServers { + rpcServer := ParseServerName(watchServer) + resolver := newresolver(client, rpcServer.Name, rpcServer.Addrs) + rrBalancer := grpc.RoundRobin(resolver) + + client.rpcServers[rpcServer.Name], err = grpc.Dial("", grpc.WithInsecure(), + grpc.WithTimeout(time.Second), + grpc.WithKeepaliveParams(keepalive.ClientParameters{Time: time.Duration(keepAlive) * time.Second}), + grpc.WithBalancer(rrBalancer)) + } + + go client.ping() + + return +} + +//user.User|127.0.0.1:9011 -> Server{Name: user.User Addrs: []string{127.0.0.1:9011}} +func ParseServerName(serverName string) *RpcServer { + ss := strings.Split(serverName, "|") + s := &RpcServer{Name: ss[0]} + + //没有配置默认地址 + if len(ss) < 2 { + return s + } + + s.Addrs = strings.Split(ss[1], ",") + return s +} + +//根据server-name 获得对应的连接池 +func (self *Client) GetConn(serverName string) *grpc.ClientConn { + return self.rpcServers[serverName] +} + +//从etcd注销 +func (self *Server) Revoke() error { + _, err := self.etcdCli.Revoke(context.Background(), self.etcdId) + return err +} + +func (self *Client) Call(ctx context.Context, method string, args, reply interface{}, opts ...grpc.CallOption) (err error) { + ss := strings.Split(method, "/") + fmt.Println("ss:", ss, len(ss)) + if len(ss) < 3 { + tlog.Errorf("no grpc client available:%s", method) + return fmt.Errorf("no grpc client available:%s", method) + } + conn, ok := self.rpcServers[ss[1]] + fmt.Println("conn:", conn) + if !ok { + tlog.Errorf("no grpc client available:%s", method) + return fmt.Errorf("no grpc client available:%s", method) + } + err = grpc.Invoke(ctx, method, args, reply, conn, opts...) + return +} + +func (self *Client) ping() { + for range time.NewTicker(time.Second).C { + for server, conn := range self.rpcServers { + req, resp := &ping.NoArgs{}, &ping.NoArgs{} + if err := grpc.Invoke(context.Background(), strings.Join([]string{server, "Ping"}, "/"), req, resp, conn); err != nil { + tlog.Errorf("qgrpc ping||server=%s||err=%s", server, err) + } + } + } +} diff --git a/qgrpc/ping/ping.pb.go b/qgrpc/ping/ping.pb.go new file mode 100644 index 0000000..aa2a06d --- /dev/null +++ b/qgrpc/ping/ping.pb.go @@ -0,0 +1,51 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: ping.proto + +/* +Package ping is a generated protocol buffer package. + +It is generated from these files: + ping.proto + +It has these top-level messages: + NoArgs +*/ +package ping + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type NoArgs struct { +} + +func (m *NoArgs) Reset() { *m = NoArgs{} } +func (m *NoArgs) String() string { return proto.CompactTextString(m) } +func (*NoArgs) ProtoMessage() {} +func (*NoArgs) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func init() { + proto.RegisterType((*NoArgs)(nil), "ping.NoArgs") +} + +func init() { proto.RegisterFile("ping.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 66 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2a, 0xc8, 0xcc, 0x4b, + 0xd7, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x01, 0xb1, 0x95, 0xd8, 0xb9, 0x58, 0x5d, 0x73, + 0x0b, 0x4a, 0x2a, 0x9d, 0x04, 0xa2, 0xf8, 0x72, 0x32, 0x93, 0xf4, 0xc1, 0x72, 0xfa, 0x20, 0xa9, + 0x24, 0x36, 0x30, 0xdb, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0xb5, 0x65, 0x5a, 0x5a, 0x35, 0x00, + 0x00, 0x00, +} diff --git a/qgrpc/register.go b/qgrpc/register.go new file mode 100644 index 0000000..c1038d7 --- /dev/null +++ b/qgrpc/register.go @@ -0,0 +1,60 @@ +package qgrpc + +import ( + "context" + "github.com/coreos/etcd/clientv3" + "globalfintech/golib/tlog" + "path" + "time" +) + +func etcdGrantWrapper(cli *clientv3.Client) (clientv3.LeaseID, error) { + resp, err := cli.Grant(context.Background(), 5) + if err != nil { + return 0, err + } + + return resp.ID, nil +} + +func (self *Server) register() (err error) { + + if len(self.userCfg.Prefix) < 1 || len(self.userCfg.Addr) < 1 { + tlog.Info("qgrpc server||register||prefix or addr less than 1 char") + return nil + } + + if self.etcdId, err = etcdGrantWrapper(self.etcdCli); err != nil { + return err + } + + for _, server := range self.userCfg.Servers { + addr := self.userCfg.GetAddr() + etcdKey := path.Join(self.userCfg.Prefix, server, addr) + if _, err = self.etcdCli.Put(context.Background(), etcdKey, addr, clientv3.WithLease(self.etcdId)); err != nil { + return err + } + } + + ch, err := self.etcdCli.KeepAlive(context.Background(), self.etcdId) + if err != nil { + tlog.Error(err) + } + + go self.keepalive(ch) + + return err +} + +func (self *Server) keepalive(ch <-chan *clientv3.LeaseKeepAliveResponse) { + for { + resp := <-ch + if resp == nil { + tlog.Error("qgrpc server||keepalive||nil response") + self.Revoke() + time.Sleep(time.Second) + self.register() + return + } + } +} diff --git a/qgrpc/watch.go b/qgrpc/watch.go new file mode 100644 index 0000000..450b5c1 --- /dev/null +++ b/qgrpc/watch.go @@ -0,0 +1,148 @@ +package qgrpc + +import ( + "context" + "fmt" + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/mvcc/mvccpb" + "google.golang.org/grpc/naming" + "path" + "strings" + "time" +) + +type watcher struct { + prefix string + addrs []string + c *clientv3.Client + hasStart bool +} + +type resolver struct { + client *Client + name string + addrs []string +} + +func newresolver(client *Client, name string, addrs []string) *resolver { + return &resolver{client: client, name: name, addrs: addrs} +} + +func (self *resolver) Resolve(name string) (naming.Watcher, error) { + return self.client.newwatcher(self.name, self.addrs), nil +} + +func (self *Client) newwatcher(name string, addrs []string) naming.Watcher { + prefix := path.Join(self.userCfg.Prefix, name) + return &watcher{prefix: prefix, addrs: addrs, c: self.etcdCli} +} + +func (w *watcher) Close() {} + +func (w *watcher) Next() ([]*naming.Update, error) { + //第一次启动从etcd上获取所有的地址同时加上默认的地址 + if !w.hasStart { + updates := []*naming.Update{} + + //etcd 的连接正常 + if w.c != nil { + resp, err := w.c.Get(context.Background(), w.prefix, clientv3.WithPrefix()) + if err == nil { + addrs := extractAddrs(resp) + + for _, addr := range addrs { + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: addr}) + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(addr, fmt.Sprintf("%s:", getLocalIP()), "127.0.0.1:", -1)}) + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(addr, fmt.Sprintf("%s:", getLocalIP()), "localhost:", -1)}) + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(addr, fmt.Sprintf("%s:", getLocalIP()), getHostName()+":", -1)}) + } + } + } + + //将默认的地址加进入 + for _, addr := range w.addrs { + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: addr}) + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(addr, fmt.Sprintf("%s:", getLocalIP()), "127.0.0.1:", -1)}) + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(addr, fmt.Sprintf("%s:", getLocalIP()), "localhost:", -1)}) + updates = append(updates, &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(addr, fmt.Sprintf("%s:", getLocalIP()), getHostName()+":", -1)}) + } + + w.hasStart = true + return updates, nil + + } + + if w.c == nil { + time.Sleep(time.Second) + return nil, nil + } + + rch := w.c.Watch(context.Background(), w.prefix, clientv3.WithPrefix()) + for wresp := range rch { + for _, ev := range wresp.Events { + switch ev.Type { + case mvccpb.PUT: + _, value := string(ev.Kv.Key), string(ev.Kv.Value) + //fmt.Println(time.Now(), "[PUT]", key, value) + return []*naming.Update{ + &naming.Update{ + Op: naming.Add, + Addr: value}, + &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(value, fmt.Sprintf("%s:", getLocalIP()), "127.0.0.1:", -1)}, + &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(value, fmt.Sprintf("%s:", getLocalIP()), "localhost:", -1)}, + &naming.Update{ + Op: naming.Add, + Addr: strings.Replace(value, fmt.Sprintf("%s:", getLocalIP()), getHostName()+":", -1)}, + }, nil + + case mvccpb.DELETE: + _, value := string(ev.Kv.Key), string(ev.Kv.Value) + //fmt.Println(time.Now(), "[DELETE]", key) + return []*naming.Update{ + &naming.Update{ + Op: naming.Delete, + Addr: value, + }, + }, nil + + } + } + } + + return nil, nil +} + +func extractAddrs(resp *clientv3.GetResponse) []string { + addrs := []string{} + + if resp == nil || resp.Kvs == nil { + return addrs + } + + for _, v := range resp.Kvs { + if v.Value != nil { + addrs = append(addrs, string(v.Value)) + } + } + return addrs +} diff --git a/tlog/clog.go b/tlog/clog.go new file mode 100644 index 0000000..2ede242 --- /dev/null +++ b/tlog/clog.go @@ -0,0 +1,81 @@ +package tlog + +import ( + "context" + "fmt" + + "google.golang.org/grpc/metadata" +) + +func Cdebug(ctx context.Context, args ...interface{}) { + id := fmt.Sprintf("trace_id=%s", getTraceIdFromCtx(ctx)) + data := append([]interface{}{id}, args...) + l.p(DEBUG, data...) +} + +func Cdebugf(ctx context.Context, format string, args ...interface{}) { + f := "trace_id=%s||" + format + data := append([]interface{}{getTraceIdFromCtx(ctx)}, args...) + l.pf(DEBUG, f, data...) +} + +func Cinfo(ctx context.Context, args ...interface{}) { + id := fmt.Sprintf("trace_id=%s", getTraceIdFromCtx(ctx)) + data := append([]interface{}{id}, args...) + l.p(INFO, data...) +} + +func Cinfof(ctx context.Context, format string, args ...interface{}) { + f := "trace_id=%s||" + format + data := append([]interface{}{getTraceIdFromCtx(ctx)}, args...) + l.pf(INFO, f, data...) +} + +func Cerror(ctx context.Context, args ...interface{}) { + id := fmt.Sprintf("trace_id=%s", getTraceIdFromCtx(ctx)) + data := append([]interface{}{id}, args...) + l.p(ERROR, data...) +} + +func Cerrorf(ctx context.Context, format string, args ...interface{}) { + f := "trace_id=%s||" + format + data := append([]interface{}{getTraceIdFromCtx(ctx)}, args...) + l.pf(ERROR, f, data...) +} + +func Cwarning(ctx context.Context, args ...interface{}) { + id := fmt.Sprintf("trace_id=%s", getTraceIdFromCtx(ctx)) + data := append([]interface{}{id}, args...) + l.p(WARNING, data...) +} + +func Cwarningf(ctx context.Context, format string, args ...interface{}) { + f := "trace_id=%s||" + format + data := append([]interface{}{getTraceIdFromCtx(ctx)}, args...) + l.pf(WARNING, f, data...) +} + +func Cfatal(ctx context.Context, args ...interface{}) { + id := fmt.Sprintf("trace_id=%s", getTraceIdFromCtx(ctx)) + data := append([]interface{}{id}, args...) + l.p(FATAL, data...) +} + +func Cfatalf(ctx context.Context, format string, args ...interface{}) { + f := "trace_id=%s||" + format + data := append([]interface{}{getTraceIdFromCtx(ctx)}, args...) + l.pf(FATAL, f, data...) +} + +func getTraceIdFromCtx(ctx context.Context) string { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "0" + } + v, ok := md["trace_id"] + if !ok || len(v) < 1 { + return "0" + } + + return v[0] +} diff --git a/tlog/level.go b/tlog/level.go new file mode 100644 index 0000000..6b22189 --- /dev/null +++ b/tlog/level.go @@ -0,0 +1,38 @@ +package tlog + +type LEVEL byte + +const ( + ALL LEVEL = iota + DEBUG + INFO + WARNING + ERROR + FATAL +) + +var levelText = map[LEVEL]string{ + ALL: "ALL", + DEBUG: "DEBUG", + INFO: "INFO", + WARNING: "WARNING", + ERROR: "ERROR", + FATAL: "FATAL", +} + +func getLevel(level string) LEVEL { + switch level { + case "DEBUG": + return DEBUG + case "INFO": + return INFO + case "WARNING": + return WARNING + case "ERROR": + return ERROR + case "FATAL": + return FATAL + default: + return ALL + } +} diff --git a/tlog/logger.go b/tlog/logger.go new file mode 100644 index 0000000..fe1b108 --- /dev/null +++ b/tlog/logger.go @@ -0,0 +1,334 @@ +package tlog + +import ( + "bufio" + "bytes" + //"encoding/json" + "fmt" + //"github.com/getsentry/raven-go" + "log/syslog" + "os" + "os/exec" + "path" + "runtime" + "strings" + "sync" + "time" +) + +var l *Logger +var mu sync.Mutex + +type Logger struct { + fileSize int64 + fileNum int + fileName string + host string + debug bool + tag string + level LEVEL + dir string + ch chan *Atom + f *os.File + w *bufio.Writer + useSyslog bool + bytePool *sync.Pool + syslogW *syslog.Writer + //jsonEncoder *json.Encoder + //sentryCh chan *raven.Packet + //client *raven.Client +} + +type Atom struct { + //isJson bool + line int + file string + format string + level LEVEL + args []interface{} + data map[string]interface{} +} + +func newLogger(config Config) { + l = &Logger{ + dir: config.Dir, + fileSize: int64(config.FileSize * 1024 * 1024), + fileNum: config.FileNum, + fileName: path.Join(config.Dir, config.FileName+".log"), + tag: config.Tag, + debug: config.Debug, + level: getLevel(config.Level), + ch: make(chan *Atom, 102400), + bytePool: &sync.Pool{New: func() interface{} { return new(bytes.Buffer) }}, + } + host, _ := os.Hostname() + ss := strings.Split(host, "-") + if len(ss) < 2 { + l.host = host + } else { + l.host = ss[len(ss)-2] + ss[len(ss)-1] + } + + if l.debug { + return + } + + os.MkdirAll(l.dir, 0755) + l.f, _ = os.OpenFile(l.fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + l.w = bufio.NewWriterSize(l.f, 1024*1024) + /* + l.jsonEncoder = json.NewEncoder(l.w) + if config.SentryUrl != "" { + l.sentryCh = make(chan *raven.Packet, 1024) + l.client, _ = raven.New(config.SentryUrl) + } + */ + if config.UseSyslog { + var err error + l.syslogW, err = syslog.New(syslog.LOG_LOCAL3|syslog.LOG_INFO, config.SyslogTag) + if err == nil { + l.useSyslog = true + } + } +} + +func (l *Logger) start() { + /* + if l.client != nil && l.sentryCh != nil { + go l.sentry() + } + */ + for { + a := <-l.ch + if a == nil { + l.w.Flush() + fileInfo, err := os.Stat(l.fileName) + if err != nil && os.IsNotExist(err) { + l.f.Close() + l.f, _ = os.OpenFile(l.fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + l.w.Reset(l.f) + } + if fileInfo.Size() > l.fileSize { + l.f.Close() + os.Rename(l.fileName, l.logname()) + l.f, _ = os.OpenFile(l.fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + l.w.Reset(l.f) + l.rm() + } + continue + } + b := l.bytes(a) + l.w.Write(b) + if l.useSyslog { + l.syslogW.Write(b) + } + //发送sentry + //l.toSentryCh(a) + } +} + +/* +func (l *Logger) sentry() { + for { + p := <-l.sentryCh + _, ch := l.client.Capture(p, nil) + if ch != nil { + if err := <-ch; err != nil { + } + close(ch) + } + } +} +*/ + +/* +func (l *Logger) toSentryCh(a *Atom) { + if a.level >= ERROR { + packet := l.formatSentryPacket(a) + select { + case l.sentryCh <- packet: + default: + } + } +} + +func (l *Logger) formatSentryPacket(a *Atom) *raven.Packet { + packet := &raven.Packet{Message: l.fileName} + if a.level == ERROR { + packet.Level = raven.ERROR + } + if a.level == FATAL { + packet.Level = raven.FATAL + } + if a.isJson { + packet.Extra = a.data + packet.Extra["fileloc"] = fmt.Sprintf("%s:%d", a.file, a.line) + return packet + } + packet.Extra = map[string]interface{}{"fileloc": fmt.Sprintf("%s:%d", a.file, a.line)} + if a.format == "" { + packet.Culprit = fmt.Sprint(a.args...) + } else { + packet.Culprit = fmt.Sprintf(a.format, a.args...) + } + return packet +} +*/ + +func (l *Logger) run() { + if l.debug { + return + } + go l.flush() + go l.start() +} + +func (l *Logger) stop() { + if l != nil && l.w != nil { + l.w.Flush() + } +} + +var bytePool *sync.Pool = &sync.Pool{New: func() interface{} { return new(bytes.Buffer) }} + +func (l *Logger) bytes(a *Atom) []byte { + w := l.bytePool.Get().(*bytes.Buffer) + defer func() { + recover() + w.Reset() + l.bytePool.Put(w) + }() + w.Write(l.genTime()) + fmt.Fprintf(w, "%s %s %s %s:%d ", l.host, l.tag, levelText[a.level], a.file, a.line) + if len(a.format) < 1 { + for _, arg := range a.args { + w.WriteByte(' ') + fmt.Fprint(w, arg) + } + } else { + fmt.Fprintf(w, a.format, a.args...) + } + w.WriteByte(10) + b := make([]byte, w.Len()) + copy(b, w.Bytes()) + return b +} + +func (l *Logger) rm() { + if out, err := exec.Command("ls", l.dir).Output(); err == nil { + files := bytes.Split(out, []byte("\n")) + totol, idx := len(files)-1, 0 + for i := totol; i >= 0; i-- { + file := path.Join(l.dir, string(files[i])) + if strings.HasPrefix(file, l.fileName) && file != l.fileName { + idx++ + if idx > l.fileNum { + os.Remove(file) + } + } + } + } +} + +func (l *Logger) flush() { + for range time.NewTicker(time.Second).C { + l.ch <- nil + } +} + +func (l *Logger) logname() string { + t := fmt.Sprintf("%s", time.Now())[:19] + tt := strings.Replace( + strings.Replace( + strings.Replace(t, "-", "", -1), + " ", "", -1), + ":", "", -1) + return fmt.Sprintf("%s.%s", l.fileName, tt) +} + +func (l *Logger) genTime() []byte { + now := time.Now() + _, month, day := now.Date() + hour, minute, second := now.Clock() + return []byte{byte(month/10) + 48, byte(month%10) + 48, '-', byte(day/10) + 48, byte(day%10) + 48, ' ', + byte(hour/10) + 48, byte(hour%10) + 48, ':', byte(minute/10) + 48, byte(minute%10) + 48, ':', + byte(second/10) + 48, byte(second%10) + 48, ' '} +} + +func (l *Logger) getTag() string { + if l != nil { + return l.tag + } + return "" +} + +func (l *Logger) p(level LEVEL, args ...interface{}) { + file, line := l.getFileNameAndLine() + if l == nil || l.debug { + mu.Lock() + defer mu.Unlock() + fmt.Printf("%s%s %s %s:%d ", l.genTime(), l.getTag(), levelText[level], file, line) + fmt.Println(args...) + return + } + if level >= l.level { + select { + case l.ch <- &Atom{file: file, line: line, level: level, args: args}: + default: + } + } +} + +func (l *Logger) pf(level LEVEL, format string, args ...interface{}) { + file, line := l.getFileNameAndLine() + if l == nil || l.debug { + mu.Lock() + defer mu.Unlock() + fmt.Printf("%s%s %s %s:%d ", l.genTime(), l.getTag(), levelText[level], file, line) + fmt.Printf(format, args...) + fmt.Println() + return + } + if level >= l.level { + select { + case l.ch <- &Atom{file: file, line: line, format: format, level: level, args: args}: + default: + } + } +} + +//暂不支持json +func (l *Logger) pj(level LEVEL, m map[string]interface{}) { + return +} + +/* + file, line := l.getFileNameAndLine() + if l == nil || l.debug { + mu.Lock() + defer mu.Unlock() + m["time"] = string(l.genTime()) + m["fileloc"] = fmt.Sprintf("%s:%d", file, line) + m["level"] = levelText[level] + json.NewEncoder(os.Stdout).Encode(m) + return + } + if level >= l.level { + select { + case l.ch <- &Atom{file: file, line: line, data: m, level: level, isJson: true}: + default: + } + } +*/ + +func (l *Logger) getFileNameAndLine() (string, int) { + _, file, line, ok := runtime.Caller(3) + if !ok { + return "???", 1 + } + dirs := strings.Split(file, "/") + if len(dirs) >= 2 { + return dirs[len(dirs)-2] + "/" + dirs[len(dirs)-1], line + } + return file, line +} diff --git a/tlog/tlog.go b/tlog/tlog.go new file mode 100644 index 0000000..007311d --- /dev/null +++ b/tlog/tlog.go @@ -0,0 +1,101 @@ +package tlog + +type Config struct { + FileSize int `toml:"filesize" json:"filesize"` + FileNum int `toml:"filenum" json:"filenum"` + FileName string `toml:"filename" json:"filename"` + Tag string `toml:"tag" json:"tag"` + Level string `toml:"level" json:"level"` + Debug bool `toml:"debug" json:"debug"` + Dir string `toml:"dir" json:"dir"` + SentryUrl string `toml:"sentry_url" json:"sentry_url"` + UseSyslog bool `toml:"use_syslog" json:"use_syslog"` + SyslogTag string `toml:"syslog_tag" json:"syslog_tag"` +} + +func (c *Config) check() { + if c.FileSize == 0 { + c.FileSize = 128 + } + if c.FileNum == 0 { + c.FileNum = 10 + } + if c.FileName == "" { + c.FileName = "INFO" + } + if c.Dir == "" { + c.Dir = "./logs" + } + if c.Level == "" { + c.Level = "DEBUG" + } +} + +func Init(c Config) { + c.check() + newLogger(c) + l.run() +} + +func Close() { + l.stop() +} + +func Debug(args ...interface{}) { + l.p(DEBUG, args...) +} + +func Debugf(format string, args ...interface{}) { + l.pf(DEBUG, format, args...) +} + +func DebugJson(m map[string]interface{}) { + l.pj(DEBUG, m) +} + +func Info(args ...interface{}) { + l.p(INFO, args...) +} + +func Infof(format string, args ...interface{}) { + l.pf(INFO, format, args...) +} +func InfoJson(m map[string]interface{}) { + l.pj(INFO, m) +} + +func Warning(args ...interface{}) { + l.p(WARNING, args...) +} + +func Warningf(format string, args ...interface{}) { + l.pf(WARNING, format, args...) +} + +func WarningJson(m map[string]interface{}) { + l.pj(WARNING, m) +} + +func Error(args ...interface{}) { + l.p(ERROR, args...) +} + +func Errorf(format string, args ...interface{}) { + l.pf(ERROR, format, args...) +} + +func ErrorJson(m map[string]interface{}) { + l.pj(ERROR, m) +} + +func Fatal(args ...interface{}) { + l.p(FATAL, args...) +} + +func Fatalf(format string, args ...interface{}) { + l.pf(FATAL, format, args...) +} + +func FatalJson(m map[string]interface{}) { + l.pj(FATAL, m) +} diff --git a/tlog/tlog_test.go b/tlog/tlog_test.go new file mode 100644 index 0000000..6b6b80c --- /dev/null +++ b/tlog/tlog_test.go @@ -0,0 +1,30 @@ +package tlog + +import ( + "testing" + "time" +) + +func TestTlog(t *testing.T) { + Init(Config{ + FileSize: 128, + FileNum: 5, + FileName: "test", + Level: "DEBUG", + Dir: "./logs", + Tag: "test", + //Debug: true, + //SentryUrl: "http://d25c96785a184d6f8d051712a209a3a8:7f4a22c48897423caa9a0aec6bb93416@172.20.0.84:9000/5", + UseSyslog: true, + SyslogTag: "test", + }) + for i := 0; i < 10; i++ { + Info("xxxxxxxxxxasfsadjflasjfdlasjdfsajdfsadfjasjfdafjsfsa") + Infof("%s", "xxxxxxxxxxasfsadjflasjfdlasjdfsajdfsadfjasjfdafjsfsa") + InfoJson(map[string]interface{}{"test": time.Now().UnixNano()}) + Error("xxxxxxxxxxasfsadjflasjfdlasjdfsajdfsadfjasjfdafjsfsa") + Errorf("%s", "xxxxxxxxxxasfsadjflasjfdlasjdfsajdfsadfjasjfdafjsfsa") + ErrorJson(map[string]interface{}{"test": time.Now().UnixNano()}) + } + time.Sleep(1 * time.Second) +} diff --git a/trace/trace.go b/trace/trace.go new file mode 100644 index 0000000..200f6a4 --- /dev/null +++ b/trace/trace.go @@ -0,0 +1,52 @@ +package trace + +import "time" + +var t *Trace + +type Trace struct { + queue chan int64 + tick <-chan time.Time + ts int64 + serverId int64 +} + +func NewTrace(serverId int64) { + if t != nil { + return + } + + t = &Trace{ + queue: make(chan int64, 1000), + tick: time.Tick(time.Second), + serverId: serverId % 65536, + } + + t.ts = t.serverId<<47 | time.Now().Unix()<<16 + go func() { + var i int64 = 0 + for { + t.queue <- i % 65536 + i++ + } + }() + + go func() { + for now := range t.tick { + t.ts = t.serverId<<47 | now.Unix()<<16 + } + }() +} + +func GenId() int64 { + if t == nil { + return 0 + } + + select { + case seq := <-t.queue: + return seq | t.ts + default: + return 0 + } +}