openbilibili/app/job/main/passport-game-cloud/service/bin_log.go

package service

import (
	"context"
	"encoding/json"
	"strings"
	"time"

	"go-common/app/job/main/passport-game-cloud/model"
	"go-common/library/log"
	"go-common/library/queue/databus"
)

func (s *Service) binlogconsumeproc() {
	mergeRoutineNum := int64(s.c.Group.BinLog.Num)
	for {
		msg, ok := <-s.binLogDataBus.Messages()
		if !ok {
			log.Error("binlogconsumeproc closed")
			return
		}
		// marked head to first commit
		m := &message{data: msg}
		s.mu.Lock()
		if s.head == nil {
			s.head = m
			s.last = m
		} else {
			s.last.next = m
			s.last = m
		}
		s.mu.Unlock()
		bmsg := new(model.BMsg)
		if err := json.Unmarshal(msg.Value, bmsg); err != nil {
			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
			continue
		}
		mid := int64(0)
		if bmsg.Table == _asoAccountTable {
			t := new(model.OriginAsoAccount)
			if err := json.Unmarshal(bmsg.New, t); err != nil {
				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
				continue
			}
			mid = t.Mid
			bmsg.MTS = s.asoAccountInterval.MTS(context.TODO(), t.Mtime)
		} else if strings.HasPrefix(bmsg.Table, _tokenTablePrefix) {
			t := new(model.OriginPerm)
			if err := json.Unmarshal(bmsg.New, t); err != nil {
				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
				continue
			}
			mid = t.Mid
			bmsg.MTS = s.tokenInterval.MTS(context.TODO(), t.Mtime)
		} else if strings.HasPrefix(bmsg.Table, _memberTablePrefix) {
			t := new(model.OriginMember)
			if err := json.Unmarshal(bmsg.New, t); err != nil {
				log.Error("json.Unmarshal(%s) error(%v)", string(bmsg.New), err)
				continue
			}
			mid = t.Mid
			bmsg.MTS = s.memberInterval.MTS(context.TODO(), t.Mtime)
		}
		m.object = bmsg
		// use specify goroutine to merge messages
		s.binLogMergeChans[mid%mergeRoutineNum] <- m
		log.Info("binlogconsumeproc table:%s key:%s partition:%d offset:%d", bmsg.Table, msg.Key, msg.Partition, msg.Offset)
	}
}

func (s *Service) binlogcommitproc() {
	commits := make(map[int32]*databus.Message, s.c.Group.BinLog.Size)
	for {
		done := <-s.binLogDoneChan
		// merge partitions to commit offset
		for _, d := range done {
			d.done = true
		}
		s.mu.Lock()
		for ; s.head != nil && s.head.done; s.head = s.head.next {
			commits[s.head.data.Partition] = s.head.data
		}
		s.mu.Unlock()
		for k, m := range commits {
			log.Info("binlogcommitproc committed, key:%s partition:%d offset:%d", m.Key, m.Partition, m.Offset)
			m.Commit()
			delete(commits, k)
		}
	}
}

func (s *Service) binlogmergeproc(c chan *message) {
	var (
		max    = s.c.Group.BinLog.Size
		merges = make([]*model.BMsg, 0, max)
		marked = make([]*message, 0, max)
		ticker = time.NewTicker(time.Duration(s.c.Group.BinLog.Ticker))
	)
	for {
		select {
		case msg, ok := <-c:
			if !ok {
				log.Error("binlogmergeproc closed")
				return
			}
			p, assertOk := msg.object.(*model.BMsg)
			if assertOk {
				merges = append(merges, p)
			}
			marked = append(marked, msg)
			if len(marked) < max && len(merges) < max {
				continue
			}
		case <-ticker.C:
		}
		if len(merges) > 0 {
			s.process(merges)
			merges = make([]*model.BMsg, 0, max)
		}
		if len(marked) > 0 {
			s.binLogDoneChan <- marked
			marked = make([]*message, 0, max)
		}
	}
}

func (s *Service) process(bmsgs []*model.BMsg) {
	for _, bmsg := range bmsgs {
		if bmsg.Table == _asoAccountTable {
			s.processUserInfo(bmsg)
		} else if strings.HasPrefix(bmsg.Table, _tokenTablePrefix) {
			s.processToken(bmsg)
		} else if strings.HasPrefix(bmsg.Table, _memberTablePrefix) {
			s.processMemberInfo(bmsg)
		}
	}
}