openbilibili/app/admin/ep/saga/service/data.go

package service

import (
	"context"
	"encoding/json"
	"strconv"
	"strings"
	"time"

	"go-common/app/admin/ep/saga/model"
	"go-common/app/admin/ep/saga/service/utils"
	"go-common/library/log"

	"github.com/xanzy/go-gitlab"
)

/*-------------------------------------- sync issue ----------------------------------------*/

// SyncAllIssues ...
func (s *Service) SyncAllIssues(projectID int) (totalPage, totalNum int, err error) {
	var (
		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
		syncAllTime = false
		issues      []*gitlab.Issue
		resp        *gitlab.Response
		since       *time.Time
		until       *time.Time
	)
	if !syncAllTime {
		since, until = utils.CalSyncTime()
	}

	for page := 1; ; page++ {
		totalPage++
		if issues, resp, err = s.gitlab.ListProjectIssues(projectID, page, since, until); err != nil {
			return
		}
		for _, issue := range issues {
			var (
				issueAssignees string
				issueLabels    string
				issueTimeStats string
				milestoneID    int
			)
			if issue.Milestone != nil {
				milestoneID = issue.Milestone.ID
			}
			issueAssigneesByte, _ := json.Marshal(issue.Assignees)
			issueAssignees = string(issueAssigneesByte)
			issueLabelsByte, _ := json.Marshal(issue.Labels)
			issueLabels = string(issueLabelsByte)
			issueTimeStatsByte, _ := json.Marshal(issue.TimeStats)
			issueTimeStats = string(issueTimeStatsByte)
			issueDB := &model.StatisticsIssues{
				ProjectID:        projectID,
				IssueID:          issue.ID,
				IssueIID:         issue.IID,
				MilestoneID:      milestoneID,
				Description:      issue.Description,
				State:            issue.State,
				Assignees:        issueAssignees,
				Upvotes:          issue.Upvotes,
				Downvotes:        issue.Downvotes,
				Labels:           issueLabels,
				Title:            issue.Title,
				UpdatedAt:        issue.UpdatedAt,
				CreatedAt:        issue.CreatedAt,
				ClosedAt:         issue.ClosedAt,
				Subscribed:       issue.Subscribed,
				UserNotesCount:   issue.UserNotesCount,
				DueDate:          issue.DueDate,
				WebURL:           issue.WebURL,
				TimeStats:        issueTimeStats,
				Confidential:     issue.Confidential,
				Weight:           issue.Weight,
				DiscussionLocked: issue.DiscussionLocked,
				IssueLinkID:      issue.IssueLinkID,
			}

			if issue.Author != nil {
				issueDB.AuthorID = issue.Author.ID
				issueDB.AuthorName = issue.Author.Name
			}
			if issue.Assignee != nil {
				issueDB.AssigneeID = issue.Assignee.ID
				issueDB.AssigneeName = issue.Assignee.Name
			}

			if len(issueDB.Description) > model.MessageMaxLen {
				issueDB.Description = issueDB.Description[0 : model.MessageMaxLen-1]
			}

			if err = s.SaveDatabaseIssue(issueDB); err != nil {
				log.Error("issue Save Database err: projectID(%d), IssueID(%d)", projectID, issue.ID)
				err = nil
				continue
			}
			totalNum++
		}

		if resp.NextPage == 0 {
			break
		}
	}
	return
}

// SaveDatabaseIssue ...
func (s *Service) SaveDatabaseIssue(issueDB *model.StatisticsIssues) (err error) {
	var total int

	if total, err = s.dao.HasIssue(issueDB.ProjectID, issueDB.IssueID); err != nil {
		log.Error("SaveDatabaseIssue HasIssue(%+v)", err)
		return
	}

	// update found row
	if total == 1 {
		if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil {
			if strings.Contains(err.Error(), model.DatabaseErrorText) {
				issueDB.Title = strconv.QuoteToASCII(issueDB.Title)
				issueDB.Description = strconv.QuoteToASCII(issueDB.Description)
				issueDB.Title = utils.Unicode2Chinese(issueDB.Title)
				issueDB.Description = utils.Unicode2Chinese(issueDB.Description)
			}
			if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil {
				log.Error("SaveDatabaseIssue UpdateIssue(%+v)", err)
				return
			}
		}
		return
	} else if total > 1 {
		// found repeated row, this situation will not exist under normal
		log.Warn("SaveDatabaseIssue issue has more rows(%d)", total)
		return
	}

	// insert row now
	if err = s.dao.CreateIssue(issueDB); err != nil {
		if strings.Contains(err.Error(), model.DatabaseErrorText) {
			issueDB.Title = strconv.QuoteToASCII(issueDB.Title)
			issueDB.Description = strconv.QuoteToASCII(issueDB.Description)
			issueDB.Title = utils.Unicode2Chinese(issueDB.Title)
			issueDB.Description = utils.Unicode2Chinese(issueDB.Description)
		}
		if err = s.dao.CreateIssue(issueDB); err != nil {
			log.Error("SaveDatabaseIssue CreateIssue(%+v)", err)
			return
		}
	}

	return
}

/*-------------------------------------- sync note ----------------------------------------*/

// SyncProjectNotes ...
func (s *Service) SyncProjectNotes(c context.Context, projectID int) (totalPage, totalNum int, err error) {
	var (
		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
		syncAllTime = false
		resp        *gitlab.Response
		projectInfo *model.ProjectInfo
		mrs         []*model.StatisticsMrs
		notes       []*gitlab.Note
		since       *time.Time
		until       *time.Time
	)

	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
		return
	}

	if !syncAllTime {
		since, until = utils.CalSyncTime()
	}

	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
		return
	}

	for _, mr := range mrs {
		for page := 1; ; page++ {
			totalPage++
			if notes, resp, err = s.gitlab.ListMRNotes(c, projectID, mr.MRIID, page); err != nil {
				return
			}
			for _, note := range notes {
				var (
					notePosition     string
					notePositionByte []byte
				)
				if notePositionByte, err = json.Marshal(note.Position); err != nil {
					return
				}
				notePosition = string(notePositionByte)
				noteDB := &model.StatisticsNotes{
					ProjectID:      projectID,
					ProjectName:    projectInfo.Name,
					MrIID:          mr.MRIID,
					NoteID:         note.ID,
					Body:           note.Body,
					Attachment:     note.Attachment,
					Title:          note.Title,
					FileName:       note.FileName,
					AuthorID:       note.Author.ID,
					AuthorName:     note.Author.Name,
					System:         note.System,
					ExpiresAt:      note.ExpiresAt,
					UpdatedAt:      note.UpdatedAt,
					CreatedAt:      note.CreatedAt,
					NoteableID:     note.NoteableID,
					NoteableType:   note.NoteableType,
					Position:       notePosition,
					Resolvable:     note.Resolvable,
					Resolved:       note.Resolved,
					ResolvedByID:   note.ResolvedBy.ID,
					ResolvedByName: note.ResolvedBy.Username,
					NoteableIID:    note.NoteableIID,
				}
				if err = s.SaveDatabaseNote(c, noteDB); err != nil {
					log.Error("note Save Database err: projectID(%d), NoteID(%d)", projectID, note.ID)
					err = nil
					continue
				}
				totalNum++
			}
			if resp.NextPage == 0 {
				break
			}
		}
	}
	return
}

// HandleNoteDBError ...
func (s *Service) HandleNoteDBError(c context.Context, noteDB *model.StatisticsNotes, errText string) (note *model.StatisticsNotes) {
	if strings.Contains(errText, model.DatabaseErrorText) {
		noteDB.Title = strconv.QuoteToASCII(noteDB.Title)
		noteDB.Body = strconv.QuoteToASCII(noteDB.Body)
		noteDB.Title = utils.Unicode2Chinese(noteDB.Title)
		noteDB.Body = utils.Unicode2Chinese(noteDB.Body)
		if len(noteDB.Body) > model.MessageMaxLen {
			noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1]
		}
	} else if strings.Contains(errText, model.DatabaseMaxLenthErrorText) {
		noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1]
	}
	return noteDB
}

// SaveDatabaseNote ...
func (s *Service) SaveDatabaseNote(c context.Context, noteDB *model.StatisticsNotes) (err error) {
	var total int

	if total, err = s.dao.HasNote(c, noteDB.ProjectID, noteDB.NoteID); err != nil {
		log.Error("SaveDatabaseNote HasNote(%+v)", err)
		return
	}

	// update found row
	if total == 1 {
		if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil {
			noteDB = s.HandleNoteDBError(c, noteDB, err.Error())
			if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil {
				log.Error("SaveDatabaseNote UpdateNote(%+v)", err)
				return
			}
		}
		return
	} else if total > 1 {
		// found repeated row, this situation will not exist under normal
		log.Warn("SaveDatabaseNote Note has more rows(%d)", total)
		return
	}

	// insert row now
	if err = s.dao.CreateNote(c, noteDB); err != nil {
		noteDB = s.HandleNoteDBError(c, noteDB, err.Error())
		if err = s.dao.CreateNote(c, noteDB); err != nil {
			log.Error("SaveDatabaseNote CreateNote(%+v)", err)
			return
		}
	}

	return
}

/*-------------------------------------- sync emoji ----------------------------------------*/

// SyncProjectAwardEmoji ...
func (s *Service) SyncProjectAwardEmoji(c context.Context, projectID int) (totalPage, totalNum int, err error) {

	var (
		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
		syncAllTime = false
		resp        *gitlab.Response
		projectInfo *model.ProjectInfo
		mrs         []*model.StatisticsMrs
		awardEmojis []*gitlab.AwardEmoji
		since       *time.Time
		until       *time.Time
	)

	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
		return
	}

	if !syncAllTime {
		since, until = utils.CalSyncTime()
	}

	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
		return
	}

	for _, mr := range mrs {
		for page := 1; ; page++ {
			totalPage++
			if awardEmojis, resp, err = s.gitlab.ListMRAwardEmoji(projectID, mr.MRIID, page); err != nil {
				return
			}
			for _, awardEmoji := range awardEmojis {
				awardEmojiDB := &model.StatisticsMRAwardEmojis{
					ProjectID:     projectID,
					ProjectName:   projectInfo.Name,
					MrIID:         mr.MRIID,
					AwardEmojiID:  awardEmoji.ID,
					Name:          awardEmoji.Name,
					UserID:        awardEmoji.User.ID,
					UserName:      awardEmoji.User.Name,
					CreatedAt:     awardEmoji.CreatedAt,
					UpdatedAt:     awardEmoji.UpdatedAt,
					AwardableID:   awardEmoji.AwardableID,
					AwardableType: awardEmoji.AwardableType,
				}
				if err = s.SaveDatabaseAwardEmoji(c, awardEmojiDB); err != nil {
					log.Error("awardEmoji Save Database err: projectID(%d), AwardEmojiID(%d)", projectID, awardEmoji.ID)
					err = nil
					continue
				}
				totalNum++
			}
			if resp.NextPage == 0 {
				break
			}
		}
	}
	return
}

// SaveDatabaseAwardEmoji ...
func (s *Service) SaveDatabaseAwardEmoji(c context.Context, awardEmojiDB *model.StatisticsMRAwardEmojis) (err error) {
	var total int

	if total, err = s.dao.HasMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID); err != nil {
		log.Error("SaveDatabaseAwardEmoji HasAwardEmoji(%+v)", err)
		return
	}

	// update found row
	if total == 1 {
		if err = s.dao.UpdateMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID, awardEmojiDB); err != nil {
			log.Error("SaveDatabaseAwardEmoji UpdateAwardEmoji(%+v)", err)
		}
		return
	} else if total > 1 {
		// found repeated row, this situation will not exist under normal
		log.Warn("SaveDatabaseAwardEmoji Note has more rows(%d)", total)
		return
	}

	// insert row now
	if err = s.dao.CreateMRAwardEmoji(c, awardEmojiDB); err != nil {
		log.Error("SaveDatabaseAwardEmoji CreateAwardEmoji(%+v)", err)
		return

	}

	return
}

/*-------------------------------------- sync Discussion ----------------------------------------*/

// SyncProjectDiscussion ...
func (s *Service) SyncProjectDiscussion(c context.Context, projectID int) (totalPage, totalNum int, err error) {

	var (
		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
		syncAllTime = false
		resp        *gitlab.Response
		projectInfo *model.ProjectInfo
		mrs         []*model.StatisticsMrs
		discussions []*gitlab.Discussion
		since       *time.Time
		until       *time.Time
	)

	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
		return
	}

	if !syncAllTime {
		since, until = utils.CalSyncTime()
	}

	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
		return
	}

	for _, mr := range mrs {
		for page := 1; ; page++ {
			totalPage++
			if discussions, resp, err = s.gitlab.ListMRDiscussions(projectID, mr.MRIID, page); err != nil {
				return
			}
			for _, discussion := range discussions {
				var (
					notesArray []int
					notesByte  []byte
					notes      string
				)
				if discussion.Notes != nil {
					for _, note := range discussion.Notes {
						notesArray = append(notesArray, note.ID)
					}
				}
				if notesByte, err = json.Marshal(notesArray); err != nil {
					notes = model.JsonMarshalErrorText
				} else {
					notes = string(notesByte)
				}
				discussionDB := &model.StatisticsDiscussions{
					ProjectID:      projectID,
					ProjectName:    projectInfo.Name,
					MrIID:          mr.MRIID,
					DiscussionID:   discussion.ID,
					IndividualNote: discussion.IndividualNote,
					Notes:          notes,
				}
				if err = s.SaveDatabaseDiscussion(c, discussionDB); err != nil {
					log.Error("discussion Save Database err: projectID(%d), DiscussionID(%s)", projectID, discussion.ID)
					err = nil
					continue
				}
				totalNum++
			}
			if resp.NextPage == 0 {
				break
			}
		}
	}
	return
}

// SaveDatabaseDiscussion ...
func (s *Service) SaveDatabaseDiscussion(c context.Context, discussionDB *model.StatisticsDiscussions) (err error) {
	var total int

	if total, err = s.dao.HasDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID); err != nil {
		log.Error("SaveDatabaseDiscussion HasDiscussion(%+v)", err)
		return
	}

	// update found row
	if total == 1 {
		if err = s.dao.UpdateDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID, discussionDB); err != nil {
			log.Error("SaveDatabaseDiscussion UpdateDiscussion(%+v)", err)
		}
		return
	} else if total > 1 {
		// found repeated row, this situation will not exist under normal
		log.Warn("SaveDatabaseDiscussion Note has more rows(%d)", total)
		return
	}

	// insert row now
	if err = s.dao.CreateDiscussion(c, discussionDB); err != nil {
		log.Error("SaveDatabaseDiscussion CreateDiscussion(%+v)", err)
		return

	}

	return
}