openbilibili/app/admin/main/apm/service/ut.go

package service

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"go-common/app/admin/main/apm/conf"
	"go-common/app/admin/main/apm/model/ut"
	"go-common/library/log"
	xtime "go-common/library/time"

	"github.com/jinzhu/gorm"
)

const (
	_utMerge  = "ut_merge"
	_utCommit = "ut_commit"
	_utpkg    = "ut_pkganls"
)

// UtList return ut_list by merge_id or username
func (s *Service) UtList(c context.Context, arg *ut.MergeReq) (mrInfs []*ut.Merge, count int, err error) {
	var (
		mtime = time.Now().Add(-time.Hour * 24 * 90)
		hql   = fmt.Sprintf("mtime>'%s'", mtime)
	)
	if arg.MergeID != 0 {
		hql += fmt.Sprintf(" and merge_id=%d ", arg.MergeID)
	}
	if arg.UserName != "" {
		hql += fmt.Sprintf(" and username='%s' ", arg.UserName)
	}
	if arg.IsMerged != 0 {
		hql += fmt.Sprintf(" and is_merged='%d' ", arg.IsMerged)
	}
	if err = s.DB.Table(_utMerge).Where(hql).Count(&count).Error; err != nil {
		log.Error("service.UtList error(%v)", err)
		return
	}
	if err = s.DB.Where(hql).Offset((arg.Pn - 1) * arg.Ps).Limit(arg.Ps).Order("mtime desc").Find(&mrInfs).Error; err != nil {
		log.Error("service.UtList error(%v)", err)
		return
	}
	for _, mr := range mrInfs {
		var commit = &ut.Commit{}
		if err = s.DB.Where("merge_id=?", mr.MergeID).Order("id desc").First(commit).Error; err != nil {
			log.Error("service.UtList mergeID(%d) error(%v)", mr.MergeID, err)
			return
		}
		if err = s.DB.Select(`max(commit_id) as commit_id, substring_index(pkg,"/",5) as pkg, ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, MAX(mtime) as mtime`).
			Where(`commit_id=? and (pkg!=substring_index(pkg, "/", 5) or pkg like "go-common/library/%")`, commit.CommitID).
			Group(`substring_index(pkg, "/", 5)`).Find(&commit.PkgAnls).Error; err != nil {
			log.Error("service.UtList commitID(%s) error(%v)", commit.CommitID, err)
			return
		}
		mr.Commit = commit
	}
	return
}

// UtDetailList get ut pkganls by commit_id&pkg
func (s *Service) UtDetailList(c context.Context, arg *ut.DetailReq) (utpkgs []*ut.PkgAnls, err error) {
	var (
		mtime = time.Now().Add(-time.Hour * 24 * 90)
		hql   = fmt.Sprintf("mtime>'%s'", mtime)
	)
	if arg.CommitID != "" {
		hql += fmt.Sprintf(" and commit_id='%s'", arg.CommitID)
	}
	if arg.PKG != "" {
		hql += fmt.Sprintf(" and substring_index(pkg,'/',5)='%s' and (pkg!=substring_index(pkg,'/',5) or pkg like 'go-common/library/%%')", arg.PKG)
	}
	if err = s.dao.DB.Select("id, commit_id, merge_id, pkg, ROUND(coverage/100,2) as coverage, ROUND(passed/assertions*100,2) as pass_rate,panics,failures,skipped,passed,assertions,html_url,report_url,mtime,ctime").Where(hql).Find(&utpkgs).Error; err != nil {
		log.Error("service.UTDetilList commitID(%s) project(%s) error(%v)", arg.CommitID, arg.PKG, err)
		return
	}
	return
}

// UtHistoryCommit get commits history list by merge_id
func (s *Service) UtHistoryCommit(c context.Context, arg *ut.HistoryCommitReq) (utcmts []*ut.Commit, count int, err error) {
	var (
		mtime = time.Now().Add(-time.Hour * 24 * 90)
		hql   = fmt.Sprintf("mtime>'%s'", mtime)
	)
	hql += fmt.Sprintf(" and merge_id=%d", arg.MergeID)
	if err = s.DB.Table(_utCommit).Where(hql).Count(&count).Error; err != nil {
		log.Error("service.UTHistoryCommit mergeID(%d) error(%v)", arg.MergeID, err)
		return
	}
	if err = s.dao.DB.Where(hql).Offset((arg.Pn - 1) * arg.Ps).Limit(arg.Ps).Order("mtime desc").Find(&utcmts).Error; err != nil {
		log.Error("service.UTHistoryCommit mergeID(%d) error(%v)", arg.MergeID, err)
		return
	}
	for _, commit := range utcmts {
		if err = s.dao.DB.Select(`max(commit_id) as commit_id, substring_index(pkg,"/",5) as pkg, ROUND(AVG(coverage/100),2) as coverage, SUM(assertions) as assertions, SUM(panics) as panics, SUM(passed) as passed, ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate, SUM(failures) as failures, MAX(mtime) as mtime`).
			Where("commit_id=? and (pkg!=substring_index(pkg,'/',5) or pkg like 'go-common/library/%')", commit.CommitID).Group(`substring_index(pkg, "/", 5)`).
			Order(`substring_index(pkg, "/", 5)`).Find(&commit.PkgAnls).Error; err != nil {
			log.Error("service.UTHistoryCommit commitID(%s) error(%v)", commit.CommitID, err)
			return
		}
	}
	for i := 0; i < len(utcmts)-1; i++ {
		for _, curPkg := range utcmts[i].PkgAnls {
			if utcmts[i+1] == nil || len(utcmts[i+1].PkgAnls) == 0 {
				continue
			}
			for _, prePkg := range utcmts[i+1].PkgAnls {
				if prePkg.PKG == curPkg.PKG {
					curPkg.CovChange, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", curPkg.Coverage-prePkg.Coverage), 64)
				}
			}
		}
	}
	return
}

// AddUT store ut data with a transaction
func (s *Service) AddUT(c context.Context, pkg *ut.PkgAnls, files []*ut.File, request *ut.UploadRes, dataURL, reportURL, HTMLURL string) (err error) {
	var (
		tx = s.DB.Begin()
		// pkgCoverage float64
	)
	if err = s.AddUTMerge(c, tx, request); err != nil {
		tx.Rollback()
		return
	}
	if err = s.AddUTCommit(c, tx, request); err != nil {
		tx.Rollback()
		return
	}
	if err = s.AddUTPKG(c, tx, pkg, request, reportURL, HTMLURL, dataURL); err != nil {
		tx.Rollback()
		return
	}
	if err = s.AddUTFiles(c, tx, files); err != nil {
		tx.Rollback()
		return
	}
	tx.Commit()
	return
}

// AddUTMerge insert merge if not exist
func (s *Service) AddUTMerge(c context.Context, tx *gorm.DB, request *ut.UploadRes) (err error) {
	var (
		uts   = make([]*ut.Merge, 0)
		merge = &ut.Merge{
			UserName: request.UserName,
			MergeID:  request.MergeID,
		}
	)
	if err = tx.Where("merge_id=?", merge.MergeID).Order("mtime desc").Find(&uts).Error; err != nil {
		log.Error("service.CreateUtList err(%v)", err)
		return
	}
	if len(uts) > 0 {
		return
	}
	if err = tx.Create(merge).Error; err != nil {
		log.Error("service.CreateUtInfo err (%v)", err)
	}
	return
}

// AddUTPKG if not exists pkg.commit_id then insert,otherwise,update
func (s *Service) AddUTPKG(c context.Context, tx *gorm.DB, pkg *ut.PkgAnls, request *ut.UploadRes, reportURL, HTMLURL, dataURL string) (err error) {
	var (
		pkgs = make([]*ut.PkgAnls, 0)
		in   = map[string]interface{}{
			"merge_id":   request.MergeID,
			"failures":   pkg.Failures,
			"passed":     pkg.Passed,
			"assertions": pkg.Assertions,
			"panics":     pkg.Panics,
			"skipped":    pkg.Skipped,
			"coverage":   pkg.Coverage,
			"html_url":   HTMLURL,
			"report_url": reportURL,
			"data_url":   dataURL,
			"mtime":      xtime.Time(time.Now().Unix())}
	)
	pkg.MergeID = request.MergeID
	pkg.PKG = request.PKG
	pkg.CommitID = request.CommitID
	pkg.HTMLURL = HTMLURL
	pkg.ReportURL = reportURL
	pkg.DataURL = dataURL
	if err = tx.Where("commit_id=? and pkg=?", pkg.CommitID, pkg.PKG).Find(&pkgs).Error; err != nil {
		log.Error("service.CreateUtPKG query error(%v)", err)
		return
	}
	if len(pkgs) > 0 {
		log.Info("s.CreateUtPKG commit_id(%s) pkg(%+v)", pkg.CommitID, pkg)
		err = tx.Table(_utpkg).Where("commit_id=? and pkg=?", pkg.CommitID, pkg.PKG).Update(in).Error
	} else {
		err = tx.Create(pkg).Error
	}
	if err != nil {
		log.Error("service.CreateUtPKG create error(%v)", err)
	}
	return
}

// AddUTFiles add files in table ut_file
func (s *Service) AddUTFiles(c context.Context, tx *gorm.DB, files []*ut.File) (err error) {
	var (
		recordFiles []*ut.File
	)
	if len(files) == 0 {
		err = fmt.Errorf("length of files is 0")
		log.Error("apmSvc.AddUTFiles Error(%v)", err)
		return
	}
	if err = tx.Where("commit_id=? and pkg=?", files[0].CommitID, files[0].PKG).Find(&recordFiles).Error; err != nil {
		log.Error("apmSvc.AddUTFiles Error(%v)", err)
		return
	}
	if len(recordFiles) > 0 {
		return
	}
	for _, file := range files {
		if err = tx.Create(file).Error; err != nil {
			log.Error("service.AddUTFiles create error(%v)", err)
			return
		}
	}
	return
}

// AddUTCommit insert commit if not exist
func (s *Service) AddUTCommit(c context.Context, tx *gorm.DB, request *ut.UploadRes) (err error) {
	var (
		cts = make([]*ut.Commit, 0)
		ct  = &ut.Commit{
			CommitID: request.CommitID,
			MergeID:  request.MergeID,
		}
	)
	if request.Author == "" || request.Author == "null" {
		ct.UserName = request.UserName
	} else {
		ct.UserName = request.Author
	}
	if err = tx.Where("commit_id=?", ct.CommitID).Find(&cts).Error; err != nil {
		log.Error("s.CreateUtCommit query error(%v)", err)
		return
	}
	if len(cts) > 0 {
		log.Info("s.AddUTCommit(%d) update merge request", ct.MergeID)
		err = tx.Table(_utCommit).Where("commit_id=?", ct.CommitID).Update("merge_id", ct.MergeID).Error
	} else {
		err = tx.Create(ct).Error
	}
	if err != nil {
		log.Error("s.CreateUtCommit(%+v) error(%v)", ct, err)
	}
	return
}

// SetMerged set is_merged = 1 in ut_merge
func (s *Service) SetMerged(c context.Context, mrid int64) (err error) {
	if err = s.dao.DB.Table(_utMerge).Where("merge_id=?", mrid).Update("is_merged", 1).Error; err != nil {
		log.Error("s.SetMerged error(%v)", err)
		return
	}
	return
}

// CheckUT .coverage>=30% && pass_rate=100% && increase >=0
func (s *Service) CheckUT(c context.Context, cid string) (t *ut.Tyrant, err error) {
	var (
		pkgs = make([]*ut.PkgAnls, 0)
	)
	if err = s.DB.Where("commit_id=? and (pkg!=substring_index(pkg,'/',5) or pkg like 'go-common/library/%')", cid).Find(&pkgs).Error; err != nil {
		log.Error("s.Tyrant query by commit_id error(%v)", err)
		return
	}
	if len(pkgs) == 0 {
		err = fmt.Errorf("pkgs is none")
		log.Error("s.Tyrant query by commit_id error(%v)", err)
		return
	}
	for _, pkg := range pkgs {
		if t, err = s.tyrant(pkg); err != nil {
			return
		}
		if t.Tyrant {
			break
		}
	}
	return
}

// tyrant determine whether the standard is up to standard
func (s *Service) tyrant(pkg *ut.PkgAnls) (t *ut.Tyrant, err error) {
	var (
		pkgs     = make([]*ut.PkgAnls, 0)
		curCover = pkg.Coverage / 100
		preCover float64
	)
	if err = s.DB.Select("commit_id, coverage").Where("pkg=?", pkg.PKG).Order("coverage desc").Find(&pkgs).Error; err != nil {
		log.Error("s.Tyrant query by pkg error(%v)", err)
		return
	}
	if len(pkgs) == 0 {
		return
	} else if len(pkgs) == 1 || pkg.CommitID != pkgs[0].CommitID {
		preCover = pkgs[0].Coverage / 100
	} else if pkg.CommitID == pkgs[0].CommitID {
		preCover = pkgs[1].Coverage / 100
	}
	t = &ut.Tyrant{
		Package:  pkg.PKG,
		Coverage: curCover,
		PassRate: float64(pkg.Passed) / float64(pkg.Assertions) * 100,
		Tyrant:   curCover < float64(s.c.UTBaseLine.Coverage) || pkg.Passed != pkg.Assertions,
		Standard: s.c.UTBaseLine.Coverage,
		Increase: curCover - preCover,
		LastCID:  pkgs[0].CommitID,
	}
	return
}

//QATrend give single user's coverage ,passrate and score data
func (s *Service) QATrend(c context.Context, arg *ut.QATrendReq) (trend *ut.QATrendResp, err error) {
	var (
		date, group, order string
		details            []*ut.PkgAnls
		where              = "1=1 and (ut_pkganls.pkg!=substring_index(ut_pkganls.pkg,'/',5) or ut_pkganls.pkg like 'go-common/library/%')"
	)
	if arg.User != "" {
		where += fmt.Sprintf(" and ut_commit.username = '%s'", arg.User)
	}
	if arg.StartTime != 0 && arg.EndTime != 0 {
		where += fmt.Sprintf(" and ut_pkganls.mtime >= '%s' and ut_pkganls.mtime <= '%s'", time.Unix(arg.StartTime, 0).Format("2006-01-02 15:04:05"), time.Unix(arg.EndTime, 0).Format("2006-01-02 15:04:05"))
	} else {
		where += fmt.Sprintf(" and ut_pkganls.mtime >= '%s'", time.Now().AddDate(0, 0, -arg.LastTime))
	}
	if arg.Period == "hour" {
		order += "SUBSTRING(ut_pkganls.mtime,11,2) asc"
	} else {
		order += "mtime asc"
	}
	if arg.Period == "day" {
		group += "LEFT(ut_pkganls.mtime,10)"
	} else {
		group += fmt.Sprintf("%s(ut_pkganls.mtime)", arg.Period)
	}
	if err = s.DB.Table(_utpkg).Joins("join ut_commit on ut_pkganls.commit_id = ut_commit.commit_id").
		Select(`ROUND(AVG(coverage/100),2) AS coverage,ROUND(SUM(passed)/SUM(assertions)*100,2) AS pass_rate, ROUND((AVG(coverage)/100*(SUM(passed)/SUM(assertions))),2) AS score, MAX(ut_pkganls.mtime) AS mtime,GROUP_CONCAT(Distinct ut_commit.commit_id) AS cids`).
		Where(where).Group(group).Order(order).Find(&details).Error; err != nil {
		log.Error("service.QATrend find detail error(%v)", err)
		return
	}
	trend = &ut.QATrendResp{BaseLine: conf.Conf.UTBaseLine.Coverage}
	for _, detail := range details {
		switch arg.Period {
		case "hour":
			date = fmt.Sprintf("%d时", detail.MTime.Time().Hour())
		case "month":
			date = detail.MTime.Time().Month().String()
		default:
			date = detail.MTime.Time().Format("01-02")
		}
		trend.Dates = append(trend.Dates, date)
		trend.Coverages = append(trend.Coverages, detail.Coverage)
		trend.PassRates = append(trend.PassRates, detail.PassRate)
		trend.Scores = append(trend.Scores, detail.Score)
		trend.CommitIDs = append(trend.CommitIDs, detail.Cids)
	}
	return
}

//UTGernalCommit get singe or all users' general commit infos
func (s *Service) UTGernalCommit(c context.Context, commits string) (cmInfos []*ut.CommitInfo, err error) {
	if err = s.DB.Table(_utpkg).Joins("join ut_commit on ut_pkganls.commit_id = ut_commit.commit_id").
		Select("ROUND(AVG(coverage/100),2) as coverage,ROUND(SUM(passed)/SUM(assertions)*100,2) as pass_rate,ut_pkganls.merge_id,ut_pkganls.commit_id,ut_pkganls.mtime").
		Where("ut_pkganls.commit_id in (?) and (pkg!=substring_index(ut_pkganls.pkg,'/',5)", strings.Split(commits, ",")).Group("commit_id").Order("mtime DESC").Find(&cmInfos).Error; err != nil {
		log.Error("service.UTGernalCommit get cmInfos error(%v)", err)
		return
	}
	for i, cmInfo := range cmInfos {
		cmInfos[i].GitlabCommit, _ = s.dao.GitLabCommits(context.Background(), cmInfo.CommitID)
	}
	return
}

// CommentOnMR create or modify comment on relavent merge request
func (s *Service) CommentOnMR(c context.Context, projID, mrID int, msg string) (err error) {
	var (
		mr     = &ut.Merge{}
		noteID int
	)
	if err = s.DB.Where("merge_id=?", mrID).First(mr).Error; err != nil {
		log.Error("apmSvc.GitReport error search ut_merge error(%v)", err)
		return
	}
	if mr.NoteID == 0 {
		if noteID, err = s.gitlab.CreateMRNote(projID, mrID, msg); err != nil {
			log.Error("apmSvc.GitReport error CreateMRNote error(%v)", err)
			return
		}
		if err = s.DB.Table(_utMerge).Where("merge_id=?", mrID).Update("note_id", noteID).Error; err != nil {
			log.Error("apmSvc.GitReport error update ut_merge error(%v)", err)
			return
		}
	} else {
		if err = s.gitlab.UpdateMRNote(projID, mrID, mr.NoteID, msg); err != nil {
			log.Error("apmSvc.GitReport error CreateMRNote error(%v)", err)
			return
		}
	}
	return
}

// CommitHistory .
func (s *Service) CommitHistory(c context.Context, username string, times int64) (pkgs []*ut.PkgAnls, err error) {
	pkgs = make([]*ut.PkgAnls, 0)
	if err = s.DB.Table("ut_pkganls p").
		Select("p.merge_id, p.commit_id, group_concat( p.pkg ) AS pkg,group_concat( ROUND( p.coverage / 100, 2 ) ) AS coverages,group_concat( ROUND( p.passed / p.assertions * 100, 2 ) ) AS pass_rates,p.mtime").
		Joins("left join ut_commit c on c.commit_id=p.commit_id").Where(" c.username=? AND (p.pkg!=substring_index(p.pkg,'/',5) or p.pkg like 'go-common/library/%')", username).
		Group("p.commit_id").Order("p.mtime desc").Limit(times).Find(&pkgs).Error; err != nil {
		log.Error("s.CommitHistory query error(%v)", err)
	}
	return
}