openbilibili/app/job/main/passport-game-data/service/parse_diff_log.go

package service

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"

	"go-common/app/job/main/passport-game-data/model"
	"go-common/library/log"
)

const (
	_cloudJobGoroutineNum = 32
)

// ParseDiffLog parse diff log printed by compare proc.
func ParseDiffLog(src, dst string) (err error) {
	f, err := os.Open(src)
	if err != nil {
		log.Error("failed to open file %s, error(%v)", src, err)
		return
	}
	defer f.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		log.Error("failed to open file %s, error(%v)", dst, err)
		return
	}
	defer dstFile.Close()

	var (
		line         string
		skippedCount = 0
		res          = make([]*model.CompareRes, 0)
		rd           = bufio.NewReader(f)
	)
	for {
		line, err = rd.ReadString('\n')

		if err != nil || io.EOF == err {
			break
		}

		idx := strings.LastIndex(line, "]")
		if idx == -1 {
			log.Error("failed to parse log, expected have ] in string but not")
			skippedCount++
			continue
		}

		logJSON := line[idx+1:]
		l := new(model.Log)
		if err = json.Unmarshal([]byte(logJSON), &l); err != nil {
			log.Error("failed to parse log, json.Unmarshal(%s) error(%v), skip", logJSON, err)
			skippedCount++
			continue
		}

		var cRes *model.CompareRes
		if cRes, err = diffLog2CompareRes(l.Log); err != nil {
			log.Error("diffLog2CompareRes(%s) error(%v), skip", l.Log, err)
			skippedCount++
			continue
		}

		// compare local encrypted and cloud, parse diff flags
		flags := diff(cRes.Cloud, cRes.LocalEncrypted)

		if flags == _diffTypeNon {
			continue
		}

		cRes.Flags = flags
		cRes.FlagsDesc = formatFlags(flags)
		cRes.Seq = cRes.Local.Mid % _cloudJobGoroutineNum
		res = append(res, cRes)
	}

	percentMap := make(map[uint8]*model.CountAndPercent)
	seqMap := make(map[int64]*model.SeqCountAndPercent)

	for _, v := range res {
		percent, ok := percentMap[v.Flags]
		if !ok {
			percent = &model.CountAndPercent{
				DiffType: v.FlagsDesc,
			}
			percentMap[v.Flags] = percent
		}
		percent.Count++

		seq, ok := seqMap[v.Seq]
		if !ok {
			seq = &model.SeqCountAndPercent{
				Seq: v.Seq,
			}
			seqMap[v.Seq] = seq
		}
		seq.Count++
	}

	sort.Slice(res, func(i, j int) bool {
		return res[i].Cloud.Mtime.After(res[j].Cloud.Mtime)
	})

	percentList := make([]*model.CountAndPercent, 0)
	for _, v := range percentMap {
		v.Percent = fmt.Sprintf("%0.2f", 100*float64(v.Count)/float64(len(res))) + "%"
		percentList = append(percentList, v)
	}
	sort.Slice(percentList, func(i, j int) bool {
		return percentList[i].Count > percentList[j].Count
	})

	seqList := make([]*model.SeqCountAndPercent, 0)
	for _, v := range seqMap {
		v.Percent = fmt.Sprintf("%0.2f", 100*float64(v.Count)/float64(_cloudJobGoroutineNum)) + "%"
		seqList = append(seqList, v)
	}
	sort.Slice(seqList, func(i, j int) bool {
		return seqList[i].Count > seqList[j].Count
	})

	stat := &model.DiffParseResp{
		Total:            len(res),
		SeqAndPercents:   seqList,
		CompareResList:   res,
		CountAndPercents: percentList,
	}

	str, _ := json.Marshal(stat)
	_, err = dstFile.WriteString(string(str))
	if err != nil {
		log.Info("failed to write parse diff log result to file %s, error(%v)", dst, err)
	}

	log.Info("len res: %d, write ok", len(res))
	return
}

func diffLog2CompareRes(str string) (*model.CompareRes, error) {
	idx := strings.Index(str, "local")
	if idx == -1 {
		return nil, fmt.Errorf("failed to parse diff log, expected have local in string but not")
	}
	res := replace(str[idx:])
	cRes := new(model.CompareRes)
	err := json.Unmarshal([]byte(res), &cRes)
	return cRes, err
}

// parse string like "local({\"mid\":1}) local_encrypted({\"mid\":1}) cloud({\"mid\":1})" to json string {"local":{},"local_encrypted":{},"cloud":{}}
func replace(str string) string {
	res := strings.Replace(str, "local(", `{"local":`, -1)

	res = strings.Replace(res, "local_encrypted(", `"local_encrypted":`, -1)

	res = strings.Replace(res, "cloud(", `"cloud":`, -1)

	res = strings.Replace(res, ")", ",", -1)

	res = strings.Replace(res, "\\", "", -1)

	if strings.HasSuffix(res, ",") {
		res = res[:len(res)-1]
	}

	res = res + "}"
	return res
}

const (
	_diffTypeNon            = uint8(0)  // 0x00000000
	_diffTypePwd            = uint8(1)  // 0x00000001
	_diffTypeEmail          = uint8(2)  // 0x00000010
	_diffTypeTel            = uint8(4)  // 0x00000100
	_diffTypeCountryID      = uint8(16) // 0x00001000
	_diffTypeMobileVerified = uint8(32) // 0x00010000
	_diffTypeIsLeak         = uint8(64) // 0x00100000
)

func formatFlags(flags uint8) string {
	fs := make([]string, 0)

	if flags&_diffTypePwd > 0 {
		fs = append(fs, "pwd")
	}

	if flags&_diffTypeEmail > 0 {
		fs = append(fs, "email")
	}

	if flags&_diffTypeTel > 0 {
		fs = append(fs, "tel")
	}

	if flags&_diffTypeCountryID > 0 {
		fs = append(fs, "country_id")
	}

	if flags&_diffTypeMobileVerified > 0 {
		fs = append(fs, "mobile_verified")
	}

	if flags&_diffTypeIsLeak > 0 {
		fs = append(fs, "is_leak")
	}

	if len(fs) == 0 {
		return "non"
	}
	return strings.Join(fs, ",")
}

func diff(cloud, localEncrypted *model.AsoAccount) uint8 {
	if localEncrypted == cloud {
		return _diffTypeNon
	}
	if localEncrypted == nil || cloud == nil {
		return _diffTypePwd | _diffTypeEmail | _diffTypeTel
	}

	res := _diffTypeNon

	if cloud.Salt != localEncrypted.Salt || cloud.Pwd != localEncrypted.Pwd {
		res = res | _diffTypePwd
	}

	if cloud.Email != localEncrypted.Email {
		res = res | _diffTypeEmail
	}

	if cloud.Tel != localEncrypted.Tel {
		res = res | _diffTypeTel
	}

	if cloud.CountryID != localEncrypted.CountryID {
		res = res | _diffTypeCountryID
	}

	if cloud.MobileVerified != localEncrypted.MobileVerified {
		res = res | _diffTypeMobileVerified
	}

	if cloud.Isleak != localEncrypted.Isleak {
		res = res | _diffTypeIsLeak
	}

	return res
}