openbilibili/app/interface/openplatform/monitor-end/service/monitor.go

package service

import (
	"context"
	"regexp"
	"strconv"
	"strings"

	"go-common/app/interface/openplatform/monitor-end/model"
	"go-common/app/interface/openplatform/monitor-end/model/monitor"
	"go-common/app/interface/openplatform/monitor-end/model/prom"
	"go-common/library/log"

	"github.com/json-iterator/go"
)

const _regex = `.*\d{2,}`

var json = jsoniter.ConfigCompatibleWithStandardLibrary

const (
	_typeAPP       = "app"
	_typeWeb       = "web/h5"
	_typeAPPH      = "app_h"
	_eventPage     = "page"
	_eventAPI      = "api"
	_eventResource = "resource"
	_levelInfo     = "info"
	_levelWarning  = "warning"
	_levelError    = "error"
	_logTypeOne    = "1"
	_logTypeTwo    = "2"
)

// Report .
func (s *Service) Report(c context.Context, params *model.LogParams, mid int64, ip string, buvid string, userAgent string) (err error) {
	var l *monitor.Log
	log.Info("get log report(%+v)", params)
	if params.IsAPP == 1 {
		_, err = s.nativeLog(c, params.Log)
	} else {
		l, err = s.frontendLog(c, params.Source, params.Log, mid, ip, buvid, userAgent)
		go s.promsFE(context.TODO(), l)
	}
	return
}
func induceSubEvent(s string) string {
	if strings.Contains(s, "?") {
		s = s[:strings.Index(s, "?")]
	}
	if strings.Contains(s, "://") {
		s = s[(strings.Index(s, "://") + 3):]
	}
	if len(s) > 512 {
		s = s[:512]
	}
	var (
		ok  bool
		err error
	)
	for {
		if ok, err = regexp.Match(_regex, []byte(s)); err != nil {
			log.Error("s.Report.regexp error(%+v), data(%s)", err, s)
			return s
		}
		if !ok {
			break
		}
		if !strings.Contains(s, "/") {
			return s
		}
		s = s[:strings.LastIndex(s, "/")]
	}
	return s
}

func (s *Service) promsFE(c context.Context, l *monitor.Log) {
	if !s.c.Prom.Promed || l == nil {
		return
	}
	var (
		ok  bool
		err error
	)
	l.SubEvent = induceSubEvent(l.SubEvent)
	if l.Event == _eventResource {
		regex := `.*i\d{1}.hdslb.com`
		if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
			log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
			return
		}
		if ok {
			l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
		}
		prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
		return
	}
	prom.AddHTTPCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
	prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
	if l.Details == nil {
		var cost int64
		if l.Duration == "" {
			return
		}
		if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
			log.Warn("s.Info.ParseInt can not convert duration(%s) to int64", l.Duration)
			return
		}
		prom.AddCommonLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
	} else {
		prom.AddDetailedLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.Details)
	}
}

func (s *Service) nativeLog(c context.Context, data string) (l *monitor.Log, err error) {
	l = &monitor.Log{}
	if err = json.Unmarshal([]byte(data), l); err != nil {
		log.Error("s.nativeLog.unmarshal error(%+v), data(%s)", err, data)
		return
	}
	l.Type = _typeAPPH
	s.handleLog(c, l)
	return
}

// HandleMsg .
func (s *Service) HandleMsg(msg []byte) {
	l := monitor.LogFromBytes(msg)
	l.Type = _typeAPP
	s.handleLog(context.TODO(), l)
}

func (s *Service) handleLog(c context.Context, l *monitor.Log) {
	var (
		appID   string
		kv      []log.D
		err     error
		logtype int
		isInt   bool
		cost    int64
		ok      bool
		app     = "android"
	)
	l.TraceidSvr = l.Traceid
	l.Traceid = ""
	l.CalCode()
	if appID, _, kv, err = l.LogData(); err != nil {
		log.Error("s.nativeLog.LogData error(%+v), data(%+v)", err, l)
		return
	}
	if l.Result == "1" || l.Result == "" {
		s.mh.Info(c, appID, kv...)
	} else {
		s.mh.Error(c, appID, kv...)
	}
	if l.SubProduct == "" {
		l.SubProduct = l.Product
	}
	if strings.Contains(l.RequestURI, "ios") {
		app = "ios"
		isInt = true
	}
	if logtype, err = bStrToInt(l.LogType, isInt); err != nil {
		return
	}
	if s.c.Prom.IgnoreNA && !s.checkProduct(l) {
		return
	}
	//	丢弃老版本的network日志
	if l.Event == "network" && l.Codes == "" {
		return
	}
	l.SubEvent = induceSubEvent(l.SubEvent)
	regex := `.*i\d{1}.hdslb.com`
	if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
		log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
		return
	}
	if ok {
		l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
	}
	if (logtype & 1) == 1 {
		// 性能日志
		if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
			log.Warn("s.handleLog.ParseInt can not convert duration(%s) to int64", l.Duration)
			return
		}
		prom.AddCommonLog(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
	}
	if (logtype >> 2 & 1) == 1 {
		// 成功/失败日志
		if l.Event == "network" {
			// 网络类型 上报业务码和http状态
			prom.AddHTTPCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
			prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
		} else {
			// 其他类型 只上报失败成功 result=1 表示成功
			res := "999"
			if l.Result == "1" {
				res = "0"
			}
			prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, res)
		}
	}
}

func (s *Service) frontendLog(c context.Context, source string, data string, mid int64, ip string, buvid string, userAgent string) (l *monitor.Log, err error) {
	var (
		ms                []interface{}
		kv                []log.D
		logtype, loglevel string
		url, query        string
	)
	if err = json.Unmarshal([]byte(data), &ms); err != nil {
		log.Error("s.frontendLog.unmarshal error(%+v), data(%s)", err, data)
		return
	}
	for _, m := range ms {
		switch m := m.(type) {
		case map[string]interface{}:
			logtype = stringValueByKey(m, "1", "logtype")
			loglevel = stringValueByKey(m, "info", "level")
			tmp := stringValueByKey(m, "", "url")
			if strings.Contains(tmp, "?") {
				url = strings.Split(tmp, "?")[0]
				query = strings.Split(tmp, "?")[1]
			} else {
				url = tmp
			}
			switch logtype {
			case _logTypeOne:
				l = &monitor.Log{
					LogType:      _logTypeOne,
					Type:         _typeWeb,
					Product:      source,
					IP:           ip,
					Buvid:        buvid,
					UserAgent:    userAgent,
					Event:        _eventAPI,
					Mid:          strconv.FormatInt(mid, 10),
					SubEvent:     url,
					Query:        query,
					Duration:     stringValueByKey(m, "", "cost"),
					TraceidSvr:   stringValueByKey(m, "", "traceid_svr"),
					TraceidEnd:   stringValueByKey(m, "", "traceid_end"),
					HTTPCode:     stringValueByKey(m, "200", "status"),
					BusinessCode: stringValueByKey(m, "0", "errno", "code"),
					SubProduct:   stringValueByKey(m, source, "sub_product"),
					Result:       "0",
				}
				srcURL := stringValueByKey(m, "", "srcUrl")
				if srcURL != "" {
					l.Event = _eventResource
					l.SubEvent = srcURL
					l.BusinessCode = "-999"
				}
				if loglevel == _levelInfo {
					l.Result = "1"
				}
				if _, _, kv, err = l.LogData(); err != nil {
					return
				}
				kv = append(kv, extKV(m)...)
			case _logTypeTwo:
				var msg []byte
				l = &monitor.Log{
					LogType:      _logTypeTwo,
					Type:         _typeWeb,
					Product:      source,
					Event:        _eventPage,
					IP:           ip,
					Buvid:        buvid,
					UserAgent:    userAgent,
					Mid:          strconv.FormatInt(mid, 10),
					SubEvent:     url,
					Query:        query,
					HTTPCode:     "200",
					Details:      details(m),
					Result:       "1",
					BusinessCode: "0",
					SubProduct:   stringValueByKey(m, source, "sub_product"),
				}
				if msg, err = json.Marshal(m); err != nil {
					log.Error("s.frontendLog.Marshal error(%v), data(%v)", err, m)
					continue
				}
				l.Message = string(msg)
				if _, _, kv, err = l.LogData(); err != nil {
					return
				}
			default:
				log.Error("s.frontendLog error logype(%v)", logtype)
				return
			}
			switch loglevel {
			case _levelInfo:
				s.mh.Info(c, source, kv...)
			case _levelWarning:
				s.mh.Warn(c, source, kv...)
			case _levelError:
				s.collectFE(c, l)
				s.mh.Error(c, source, kv...)
			default:
				s.mh.Info(c, source, kv...)
			}
		default:
			log.Error("s.frontendLog log data is not json type: " + data)
		}
	}
	return
}

func (s *Service) collectFE(c context.Context, l *monitor.Log) {
	if s.c.CollectFE {
		regex := `.*i\d{1}.hdslb.com`
		if ok, err := regexp.Match(regex, []byte(l.SubEvent)); err != nil {
			log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
			return
		} else if ok {
			l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
		}
		go s.Collect(context.TODO(), l)
	}
}

func stringValueByKey(m map[string]interface{}, defValue string, keys ...string) (r string) {
	r = defValue
	if m == nil {
		return
	}
	var (
		res interface{}
		ok  bool
	)
	for _, key := range keys {
		if res, ok = m[key]; ok {
			break
		}
		return
	}
	if res == nil {
		return
	}
	switch d := res.(type) {
	case int:
		r = strconv.Itoa(d)
	case float64:
		r = strconv.FormatFloat(d, 'f', -1, 64)
	case int64:
		r = strconv.FormatInt(d, 10)
	case string:
		r = d
	default:
		log.Warn("s.stringValueByKey unexcept type of value(%v)", d)
	}
	return
}

func extKV(m map[string]interface{}) (kv []log.D) {
	if m == nil {
		return
	}
	var keys = []string{"level", "logtype", "url", "cost", "traceid_svr", "traceid_end", "status", "code", "errno", "sub_product"}
	for _, key := range keys {
		delete(m, key)
	}
	for k, v := range m {
		kv = append(kv, log.KV(k, v))
	}
	return
}

func details(m map[string]interface{}) (res map[string]int64) {
	res = make(map[string]int64)
	if m == nil {
		return
	}
	delete(m, "level")
	delete(m, "logtype")
	delete(m, "url")
	for k, v := range m {
		var d int64
		if v == nil {
			res[k] = 0
			continue
		}
		switch v := v.(type) {
		case int64:
			d = v
		case float64:
			d = int64(v)
		case int:
			d = int64(v)
		case int32:
			d = int64(v)
		case string:
			d, _ = strconv.ParseInt(v, 10, 64)
		default:
			log.Warn("s.details unexcept type of value(%v)", v)
		}
		res[k] = d
	}
	return
}

func bStrToInt(str string, isInt bool) (r int, err error) {
	if isInt {
		if r, err = strconv.Atoi(str); err != nil {
			log.Warn("s.bStrToInt error(%+v), string(%s)", err, str)
		}
		return
	}
	for _, i := range str {
		if i == '1' {
			r = r*2 + 1
		} else {
			r *= 2
		}
	}
	return
}

func (s *Service) checkProduct(l *monitor.Log) bool {
	if s.naProducts == nil {
		return true
	}
	if s.naProducts[l.Product] {
		return true
	}
	return false
}