Initial commit: web_hghl_server

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

web/service/srv_search/content_tag.go

@@ -0,0 +1,13 @@
+package srv_search
+
+import "strings"
+
+func contentTagNamesString(tagIds []int64, tagMap map[int64]string) string {
+	names := make([]string, 0, len(tagIds))
+	for _, tagId := range tagIds {
+		if name := tagMap[tagId]; name != "" {
+			names = append(names, name)
+		}
+	}
+	return strings.Join(names, ",")
+}

web/service/srv_search/define.go

@@ -0,0 +1,19 @@
+package srv_search
+
+import (
+	"videoweb/pub/mongos"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// Param 引擎参数
+type Param struct {
+	//数据库
+	Coll *mongos.Collection
+	//过滤器
+	CollFilter bson.M
+	//跟新字段
+	CollFields []string
+	//es索引
+	EsIndex string
+}

web/service/srv_search/es.go

@@ -0,0 +1,187 @@
+package srv_search
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+	"videoweb/pub/common/log"
+	"videoweb/pub/common/srvdef"
+	"videoweb/pub/common/utils"
+	"videoweb/pub/esmanager"
+	"videoweb/pub/models/esmod"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var syncIsRuining = false
+
+func StartES() {
+	log.Info("srv_search:Start...")
+	defer log.Info("srv_search:Start!!!")
+	startSyncEs()
+	_, _ = skdCron.AddFunc("0 0/17 * * * ?", func() {
+		startSyncEs()
+	})
+	skdCron.Start()
+}
+
+func startSyncEs() {
+	ctx := context.Background()
+	updateEsData(ctx, esmod.GetVideoIndex(), getVideoEsData)
+	updateEsData(ctx, esmod.GetPlayletIndex(), getPlayletEsData)
+	updateEsData(ctx, esmod.GetActressIndex(), getActressEsData)
+}
+
+func StopES() {
+	log.Info("srv_search:Stop...")
+	defer log.Info("srv_search:Stop!!!")
+}
+
+func updateEsData(ctx context.Context, esIndex string, fn func(ctx context.Context, startTime, endTime *time.Time, esIndex string) (esmanager.M, *srvdef.SrvErr)) {
+	log.Info("srv_search:do...", log.Any("ESIndex", esIndex))
+	defer log.Info("srv_search:do end", log.Any("EsIndex", esIndex))
+	if fn == nil {
+		return
+	}
+	esRecord, serr := esmod.GetRecordTimeByESIndex(ctx, esIndex)
+	if serr != nil {
+		log.Error(fmt.Sprintf("srv_search:startEngine GetRecordByESIndex failed! Index:%v,err:%+v:", esIndex, serr))
+		return
+	}
+	isNew := false
+	if esRecord == nil {
+		isNew = true
+		esRecord = &esmod.ESRecord{
+			Id:         primitive.NewObjectID(),
+			EsIndex:    esIndex,
+			RecordTime: time.Time{},
+			Timestamp:  esmod.ESTimestamp{},
+			UpdatedAt:  time.Time{},
+			CreatedAt:  time.Now(),
+		}
+	}
+	var now = time.Now()
+	var esData esmanager.M
+	// 跨天凌晨5点重置所有数据
+	if isNew || ((now.Hour() >= 5 && esRecord.UpdatedAt.Local().Hour() < 5) || esRecord.UpdatedAt.Local().Day() < now.Day()) {
+		log.Info("srv_search:updateEsData 全量同步", log.Any("index", esIndex), log.Any("isNew", isNew))
+		esData, serr = fn(ctx, nil, nil, esIndex)
+		if serr != nil {
+			log.Warn("updateEsData fn return err", log.Any("index", esIndex), log.E(serr))
+			return
+		}
+		log.Info("srv_search:updateEsData 全量数据拉取完成", log.Any("index", esIndex), log.Any("count", len(esData)))
+		if len(esData) == 0 {
+			if err := esmod.SetOneEsRecordTime(nil, esIndex, now); err != nil {
+				return
+			}
+			return
+		}
+		isUpdate, err := commit(esData, esIndex, time.Now())
+		if err != nil {
+			log.Error("Elastic sync video data RunSync commit err! ", log.E(err))
+			return
+		}
+		log.Info("srv_search:updateEsData 全量写入ES完成", log.Any("index", esIndex), log.Any("isUpdate", isUpdate))
+		if isUpdate {
+			if err := esmod.SetOneEsRecordTime(nil, esIndex, now); err != nil {
+				return
+			}
+		}
+	} else {
+		timestamp := esRecord.RecordTime
+		var recentMinute = utils.RecentMinute(now, esmod.FiveMinuteScale)
+		log.Info("srv_search:updateEsData 增量同步", log.Any("index", esIndex),
+			log.Any("from", timestamp.Format("2006-01-02 15:04:05")),
+			log.Any("to", recentMinute.Format("2006-01-02 15:04:05")))
+		totalIncr := 0
+		for recentMinute.After(timestamp) {
+			var end = timestamp.Add(esmod.FiveMinuteScale * time.Minute)
+			if end.After(recentMinute) {
+				end = recentMinute
+			}
+			esData, serr = fn(ctx, &timestamp, &end, esIndex)
+			if serr != nil {
+				log.Warn("updateEsData fn return err", log.Any("index", esIndex), log.E(serr))
+				return
+			}
+			if esData != nil && len(esData) > 0 {
+				totalIncr += len(esData)
+				log.Info("srv_search:updateEsData 增量批次",
+					log.Any("index", esIndex),
+					log.Any("window_start", timestamp.Format("2006-01-02 15:04:05")),
+					log.Any("window_end", end.Format("2006-01-02 15:04:05")),
+					log.Any("count", len(esData)))
+				isUpdate, err := commit(esData, esIndex, time.Now())
+				if err != nil {
+					log.Error("Elastic sync video data RunSync commit err! ", log.E(err))
+					return
+				}
+				if isUpdate {
+					if err := esmod.SetOneEsRecordTime(nil, esIndex, end); err != nil {
+						return
+					}
+				}
+			} else {
+				if err := esmod.SetOneEsRecordTime(nil, esIndex, end); err != nil {
+					return
+				}
+			}
+			timestamp = end
+		}
+		log.Info("srv_search:updateEsData 增量同步完成", log.Any("index", esIndex), log.Any("totalCount", totalIncr))
+	}
+}
+
+func commit(data esmanager.M, esIndex string, syncTime time.Time) (bool, error) {
+	bulkData := handleBulkCountLimit(data)
+	isHaveSuccess := false
+	failedBatches := make([]int, 0)
+	log.Info("srv_search:commit 开始写入ES", log.Any("index", esIndex),
+		log.Any("totalDocs", len(data)), log.Any("batches", len(bulkData)))
+	if bulkData != nil && len(bulkData) > 0 {
+		for i, doc := range bulkData {
+			if len(doc) == 0 {
+				continue
+			}
+			if err := esmanager.Instance.Bulk(esIndex, doc); err != nil {
+				log.Warn(fmt.Sprintf("Elastic sync commit video data SyncDataToES failed!, err:%+v\n", err),
+					log.Any("index", esIndex), log.Any("batch", i), log.Any("batchSize", len(doc)))
+				failedBatches = append(failedBatches, i)
+			} else {
+				isHaveSuccess = true
+				log.Info("srv_search:commit 批次写入成功", log.Any("index", esIndex),
+					log.Any("batch", i), log.Any("batchSize", len(doc)))
+			}
+		}
+	}
+	if len(failedBatches) > 0 {
+		return isHaveSuccess, fmt.Errorf("elastic sync commit failed batches: %v", failedBatches)
+	}
+	if isHaveSuccess {
+		if err := esmanager.Instance.Refresh(esIndex); err != nil {
+			return isHaveSuccess, err
+		}
+		return isHaveSuccess, nil
+	} else {
+		return isHaveSuccess, errors.New("Elastic sync commit video data UpsertOneTrans failed!")
+	}
+}
+
+func handleBulkCountLimit(data esmanager.M) []esmanager.M {
+	if data == nil || len(data) == 0 {
+		return make([]esmanager.M, 0)
+	}
+	rsl := make([]esmanager.M, 0)
+	rsl = append(rsl, make(esmanager.M, batchSize))
+	index := 0
+	for k, v := range data {
+		rsl[index][k] = v
+		if len(rsl[index]) >= batchSize {
+			index++
+			rsl = append(rsl, make(esmanager.M, batchSize))
+		}
+	}
+	return rsl
+}

web/service/srv_search/es_playlet.go

@@ -0,0 +1,378 @@
+package srv_search
+
+import (
+	"context"
+	"strconv"
+	"time"
+	"videoweb/pub/common/log"
+	"videoweb/pub/common/srvdef"
+	"videoweb/pub/esmanager"
+	"videoweb/pub/models/contenttagmod"
+	"videoweb/pub/models/esmod"
+	"videoweb/pub/models/playletcategorymod"
+	"videoweb/pub/models/playletmod"
+	"videoweb/pub/models/playletthememod"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+const playletESPageSize int64 = 1000
+
+var playletESProjection = bson.D{
+	{Key: playletmod.FN.Playlet.Id, Value: 1},
+	{Key: playletmod.FN.Playlet.Status, Value: 1},
+	{Key: playletmod.FN.Playlet.IsFeatured, Value: 1},
+	{Key: playletmod.FN.Playlet.IsRecommend, Value: 1},
+	{Key: playletmod.FN.Playlet.ShowOnBanner, Value: 1},
+	{Key: playletmod.FN.Playlet.Banner, Value: 1},
+	{Key: playletmod.FN.Playlet.OnlineAt, Value: 1},
+	{Key: playletmod.FN.Playlet.CategoryIds, Value: 1},
+	{Key: playletmod.FN.Playlet.ThemeIds, Value: 1},
+	{Key: playletmod.FN.Playlet.Title, Value: 1},
+	{Key: playletmod.FN.Playlet.Desc, Value: 1},
+	{Key: playletmod.FN.Playlet.Anchor, Value: 1},
+	{Key: playletmod.FN.Playlet.Cover, Value: 1},
+	{Key: playletmod.FN.Playlet.CompressedCover, Value: 1},
+	{Key: playletmod.FN.Playlet.TagIds, Value: 1},
+	{Key: playletmod.FN.Playlet.TotalEpisode, Value: 1},
+	{Key: playletmod.FN.Playlet.UpdateStatus, Value: 1},
+	{Key: playletmod.FN.Playlet.CountBrowse, Value: 1},
+	{Key: playletmod.FN.Playlet.VirtualCountBrowse, Value: 1},
+	{Key: playletmod.FN.Playlet.HeatValue, Value: 1},
+	{Key: playletmod.FN.Playlet.VirtualHeatValue, Value: 1},
+	{Key: playletmod.FN.Playlet.CountCollect, Value: 1},
+	{Key: playletmod.FN.Playlet.CountLike, Value: 1},
+	{Key: playletmod.FN.Playlet.VirtualCountLike, Value: 1},
+	{Key: playletmod.FN.Playlet.SeoInfo.Title, Value: 1},
+	{Key: playletmod.FN.Playlet.SeoInfo.Description, Value: 1},
+	{Key: playletmod.FN.Playlet.SeoInfo.Keyworld, Value: 1},
+	{Key: playletmod.FN.Playlet.CreatedAt, Value: 1},
+	{Key: playletmod.FN.Playlet.UpdatedAt, Value: 1},
+}
+
+// getPlayletEsData 查询短剧增量数据并转换为ES批量写入数据
+func getPlayletEsData(ctx context.Context, minTime, endTime *time.Time, esIndex string) (esmanager.M, *srvdef.SrvErr) {
+	list, err1 := getPlayletData(ctx, minTime, endTime)
+	if err1 != nil {
+		return nil, err1
+	}
+	return playletToEsData(ctx, list)
+}
+
+// SyncPlayletsToESByLatestStatus 根据短剧最新上下架状态同步ES搜索数据
+func SyncPlayletsToESByLatestStatus(ctx context.Context, ids []int64) *srvdef.SrvErr {
+	ids = dedupePlayletESIds(ids)
+	if len(ids) == 0 {
+		return nil
+	}
+	list, err1 := playletmod.FindManyPlaylet(ctx, bson.M{
+		playletmod.FN.Playlet.Id: bson.M{"$in": ids},
+	}, options.Find().SetProjection(bson.D{
+		{Key: playletmod.FN.Playlet.Id, Value: 1},
+		{Key: playletmod.FN.Playlet.Status, Value: 1},
+	}))
+	if err1 != nil {
+		return err1
+	}
+	foundIds := make(map[int64]struct{}, len(list))
+	upIds := make([]int64, 0, len(list))
+	downIds := make([]int64, 0, len(ids))
+	for _, item := range list {
+		if item == nil {
+			continue
+		}
+		foundIds[item.Id] = struct{}{}
+		if item.Status == playletmod.StatusUp {
+			upIds = append(upIds, item.Id)
+		} else {
+			downIds = append(downIds, item.Id)
+		}
+	}
+	for _, id := range ids {
+		if _, ok := foundIds[id]; !ok {
+			downIds = append(downIds, id)
+		}
+	}
+	if err1 := SyncPlayletStatusToES(ctx, upIds, playletmod.StatusUp); err1 != nil {
+		return err1
+	}
+	if err1 := SyncPlayletStatusToES(ctx, downIds, playletmod.StatusDown); err1 != nil {
+		return err1
+	}
+	return nil
+}
+
+// SyncPlayletStatusToES 根据短剧上下架状态同步ES搜索数据
+func SyncPlayletStatusToES(ctx context.Context, ids []int64, status playletmod.Status) *srvdef.SrvErr {
+	ids = dedupePlayletESIds(ids)
+	if len(ids) == 0 {
+		return nil
+	}
+	if esmanager.Instance == nil {
+		log.Error("短剧ES同步失败，ES未初始化", log.Any("短剧ID", ids), log.Any("状态", status))
+		return srvdef.New(srvdef.ErrElsOperError, "", "")
+	}
+	switch status {
+	case playletmod.StatusUp:
+		return upsertPlayletsToES(ctx, ids)
+	case playletmod.StatusDown:
+		return deletePlayletsFromES(ids)
+	default:
+		return nil
+	}
+}
+
+func getPlayletData(ctx context.Context, minTime, endTime *time.Time) (playletmod.PlayletList, *srvdef.SrvErr) {
+	pageNum := int64(1)
+	result := make(playletmod.PlayletList, 0)
+	filter := bson.M{
+		playletmod.FN.Playlet.Status: playletmod.StatusUp,
+	}
+	if minTime != nil && endTime != nil {
+		filter[playletmod.FN.Playlet.UpdatedAt] = bson.M{
+			"$gte": minTime,
+			"$lte": endTime,
+		}
+	}
+	for {
+		list, err1 := playletmod.FindManyPlaylet(ctx, filter, options.Find().
+			SetProjection(playletESProjection).
+			SetSort(bson.D{{Key: playletmod.FN.Playlet.UpdatedAt, Value: -1}}).
+			SetSkip((pageNum-1)*playletESPageSize).
+			SetLimit(playletESPageSize))
+		if err1 != nil {
+			return nil, err1
+		}
+		result = append(result, list...)
+		if int64(len(list)) < playletESPageSize {
+			break
+		}
+		pageNum++
+	}
+	return result, nil
+}
+
+func playletToEsData(ctx context.Context, list playletmod.PlayletList) (esmanager.M, *srvdef.SrvErr) {
+	categoryNameMap, err1 := buildPlayletESCategoryNameMap(ctx, list)
+	if err1 != nil {
+		return nil, err1
+	}
+	themeNameMap, err1 := buildPlayletESThemeNameMap(ctx, list)
+	if err1 != nil {
+		return nil, err1
+	}
+	tagNameMap, err1 := buildPlayletESTagNameMap(ctx, list)
+	if err1 != nil {
+		return nil, err1
+	}
+	result := make(esmanager.M, len(list))
+	for _, item := range list {
+		if item == nil {
+			continue
+		}
+		result[strconv.FormatInt(item.Id, 10)] = playletToESDoc(item, categoryNameMap, themeNameMap, tagNameMap)
+	}
+	return result, nil
+}
+
+// upsertPlayletsToES 批量写入已上架短剧ES文档
+func upsertPlayletsToES(ctx context.Context, ids []int64) *srvdef.SrvErr {
+	list, err1 := playletmod.FindManyPlaylet(ctx, bson.M{
+		playletmod.FN.Playlet.Id:     bson.M{"$in": ids},
+		playletmod.FN.Playlet.Status: playletmod.StatusUp,
+	}, options.Find().SetProjection(playletESProjection))
+	if err1 != nil {
+		return err1
+	}
+	if len(list) == 0 {
+		return nil
+	}
+	esData, err1 := playletToEsData(ctx, list)
+	if err1 != nil {
+		return err1
+	}
+	if len(esData) == 0 {
+		return nil
+	}
+	if err1 = AddMultiDataToES(esmod.GetPlayletIndex(), esData); err1 != nil {
+		log.Error("批量写入短剧ES失败", log.Any("短剧ID", ids), log.E(err1))
+		return err1
+	}
+	log.Info("批量写入短剧ES成功", log.Any("数量", len(esData)), log.Any("短剧ID", ids))
+	return nil
+}
+
+// deletePlayletsFromES 批量删除短剧ES文档
+func deletePlayletsFromES(ids []int64) *srvdef.SrvErr {
+	strIds := make([]string, 0, len(ids))
+	for _, id := range ids {
+		if id <= 0 {
+			continue
+		}
+		strIds = append(strIds, strconv.FormatInt(id, 10))
+	}
+	if len(strIds) == 0 {
+		return nil
+	}
+	if err1 := batchDelFromES(esmod.GetPlayletIndex(), strIds); err1 != nil {
+		log.Error("批量删除短剧ES失败", log.Any("短剧ID", ids), log.E(err1))
+		return err1
+	}
+	log.Info("批量删除短剧ES成功", log.Any("数量", len(strIds)), log.Any("短剧ID", ids))
+	return nil
+}
+
+// playletToESDoc 转换短剧Mongo数据为ES文档
+func playletToESDoc(item *playletmod.Playlet, categoryNameMap map[int64]string, themeNameMap map[int64]string, tagNameMap map[int64]string) *esmod.ESPlaylet {
+	return &esmod.ESPlaylet{
+		Id:            item.Id,
+		Status:        item.Status,
+		IsFeatured:    item.IsFeatured,
+		IsRecommend:   item.IsRecommend,
+		ShowOnBanner:  item.ShowOnBanner,
+		Banner:        item.Banner,
+		OnlineAt:      item.OnlineAt,
+		CategoryIds:   item.CategoryIds,
+		CategoryNames: playletESNamesByIds(item.CategoryIds, categoryNameMap),
+		ThemeIds:      item.ThemeIds,
+		ThemeNames:    playletESNamesByIds(item.ThemeIds, themeNameMap),
+		Title:         item.Title,
+		Desc:          item.Desc,
+		Anchor:        item.Anchor,
+		Cover:         item.DisplayCover(),
+		TagIds:        item.TagIds,
+		TagNames:      playletESNamesByIds(item.TagIds, tagNameMap),
+		TotalEpisode:  item.TotalEpisode,
+		UpdateStatus:  item.UpdateStatus,
+		CountBrowse:   item.DisplayCountBrowse(),
+		HeatValue:     item.DisplayHeatValue(),
+		CountCollect:  item.CountCollect,
+		CountLike:     item.CountLike + item.VirtualCountLike,
+		SeoInfo:       item.SeoInfo,
+		CreatedAt:     item.CreatedAt,
+		UpdatedAt:     item.UpdatedAt,
+	}
+}
+
+// buildPlayletESCategoryNameMap 批量查询短剧分类名称
+func buildPlayletESCategoryNameMap(ctx context.Context, list playletmod.PlayletList) (map[int64]string, *srvdef.SrvErr) {
+	ids := collectPlayletESIds(list, func(item *playletmod.Playlet) []int64 {
+		return item.CategoryIds
+	})
+	result := make(map[int64]string, len(ids))
+	if len(ids) == 0 {
+		return result, nil
+	}
+	categories, err1 := playletcategorymod.FindManyPlayletCategory(ctx, bson.M{
+		playletcategorymod.FN.PlayletCategory.Id: bson.M{"$in": ids},
+	}, options.Find().SetProjection(bson.D{
+		{Key: playletcategorymod.FN.PlayletCategory.Id, Value: 1},
+		{Key: playletcategorymod.FN.PlayletCategory.Name, Value: 1},
+	}))
+	if err1 != nil {
+		return nil, err1
+	}
+	for _, category := range categories {
+		if category != nil {
+			result[category.Id] = category.Name
+		}
+	}
+	return result, nil
+}
+
+// buildPlayletESThemeNameMap 批量查询短剧题材名称
+func buildPlayletESThemeNameMap(ctx context.Context, list playletmod.PlayletList) (map[int64]string, *srvdef.SrvErr) {
+	ids := collectPlayletESIds(list, func(item *playletmod.Playlet) []int64 {
+		return item.ThemeIds
+	})
+	result := make(map[int64]string, len(ids))
+	if len(ids) == 0 {
+		return result, nil
+	}
+	themes, err1 := playletthememod.FindManyPlayletTheme(ctx, bson.M{
+		playletthememod.FN.PlayletTheme.Id: bson.M{"$in": ids},
+	}, options.Find().SetProjection(bson.D{
+		{Key: playletthememod.FN.PlayletTheme.Id, Value: 1},
+		{Key: playletthememod.FN.PlayletTheme.Name, Value: 1},
+	}))
+	if err1 != nil {
+		return nil, err1
+	}
+	for _, theme := range themes {
+		if theme != nil {
+			result[theme.Id] = theme.Name
+		}
+	}
+	return result, nil
+}
+
+// buildPlayletESTagNameMap 批量查询短剧标签名称
+func buildPlayletESTagNameMap(ctx context.Context, list playletmod.PlayletList) (map[int64]string, *srvdef.SrvErr) {
+	ids := collectPlayletESIds(list, func(item *playletmod.Playlet) []int64 {
+		return item.TagIds
+	})
+	result := make(map[int64]string, len(ids))
+	if len(ids) == 0 {
+		return result, nil
+	}
+	tags, err1 := contenttagmod.GetMapTags(ctx, ids)
+	if err1 != nil {
+		return nil, err1
+	}
+	for _, tag := range tags {
+		if tag != nil {
+			result[tag.Id] = tag.Name
+		}
+	}
+	return result, nil
+}
+
+// collectPlayletESIds 汇总并去重短剧关联ID
+func collectPlayletESIds(list playletmod.PlayletList, pick func(item *playletmod.Playlet) []int64) []int64 {
+	result := make([]int64, 0)
+	seen := make(map[int64]struct{})
+	for _, item := range list {
+		if item == nil {
+			continue
+		}
+		for _, id := range pick(item) {
+			if id <= 0 {
+				continue
+			}
+			if _, ok := seen[id]; ok {
+				continue
+			}
+			seen[id] = struct{}{}
+			result = append(result, id)
+		}
+	}
+	return result
+}
+
+func dedupePlayletESIds(ids []int64) []int64 {
+	result := make([]int64, 0, len(ids))
+	seen := make(map[int64]struct{}, len(ids))
+	for _, id := range ids {
+		if id <= 0 {
+			continue
+		}
+		if _, ok := seen[id]; ok {
+			continue
+		}
+		seen[id] = struct{}{}
+		result = append(result, id)
+	}
+	return result
+}
+
+// playletESNamesByIds 按ID顺序返回ES文档名称集合
+func playletESNamesByIds(ids []int64, nameMap map[int64]string) []string {
+	result := make([]string, 0, len(ids))
+	for _, id := range ids {
+		if name := nameMap[id]; name != "" {
+			result = append(result, name)
+		}
+	}
+	return result
+}

web/service/srv_search/esactress.go

@@ -0,0 +1,77 @@
+package srv_search
+
+import (
+	"context"
+	"strconv"
+	"time"
+	"videoweb/pub/common/srvdef"
+	"videoweb/pub/esmanager"
+	"videoweb/pub/models/actress"
+	"videoweb/pub/models/esmod"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func getActressEsData(ctx context.Context, minTime, endTime *time.Time, esIndex string) (esmanager.M, *srvdef.SrvErr) {
+	data, serr := getActressData(ctx, minTime, endTime)
+	if serr != nil {
+		return nil, serr
+	}
+	return actressToEsData(data), nil
+}
+func getActressData(ctx context.Context, minTime, endTime *time.Time) ([]*actress.Actress, *srvdef.SrvErr) {
+	pageNumber := 1
+	pageSize := 1000
+	rsl := make([]*actress.Actress, 0)
+	filter := bson.M{}
+	if minTime != nil && endTime != nil {
+		filter[actress.FN.Actress.UpdatedAt] = bson.M{
+			"$gte": minTime,
+			"$lte": endTime,
+		}
+	}
+	for {
+		opts := &options.FindOptions{}
+		opts.SetSort(bson.M{actress.FN.Actress.UpdatedAt: -1})
+		opts.SetSkip(int64(pageSize * (pageNumber - 1)))
+		opts.SetLimit(int64(pageSize))
+		data, serr := actress.FindManyActresss(ctx, filter, opts)
+		if serr != nil {
+			return nil, serr
+		}
+		rsl = append(rsl, data...)
+		if len(data) < pageSize {
+			break
+		}
+		pageNumber++
+	}
+	return rsl, nil
+}
+func actressToEsData(data []*actress.Actress) esmanager.M {
+	rsl := make(map[string]interface{}, len(data))
+	for _, v := range data {
+		esInfo := esmod.ESActress{
+			Id:                    v.Id,
+			Alias:                 v.Alias,
+			Avatar:                v.Avatar,
+			Meas:                  v.Meas,
+			Desc:                  v.Desc,
+			Office:                v.Office,
+			DebutDate:             v.DebutDate,
+			BornDate:              v.BornDate,
+			Region:                v.Region,
+			Score:                 v.Score,
+			Works:                 v.Works,
+			HotValue:              v.HotValue,
+			Real:                  v.Real,
+			Status:                v.Status,
+			CreatedAt:             v.CreatedAt,
+			UpdatedAt:             v.UpdatedAt,
+			TranslationCategoryId: v.TranslationCategoryId,
+			Rank:                  v.Rank,
+		}
+		rsl[strconv.FormatInt(v.Id, 10)] = esInfo
+	}
+	return rsl
+}

web/service/srv_search/esmedia.go

@@ -0,0 +1,92 @@
+package srv_search
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"time"
+	"videoweb/pub/common/srvdef"
+	"videoweb/pub/esmanager"
+	"videoweb/pub/models/esmod"
+	"videoweb/pub/models/videoinformationmod"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func getVideoEsData(ctx context.Context, minTime, endTime *time.Time, esIndex string) (esmanager.M, *srvdef.SrvErr) {
+	data, serr := getVideoData(ctx, minTime, endTime)
+	if serr != nil {
+		return nil, serr
+	}
+	return videoToEsData(data), nil
+}
+func getVideoData(ctx context.Context, minTime, endTime *time.Time) ([]*videoinformationmod.VideoInformation, *srvdef.SrvErr) {
+	pageNumber := 1
+	pageSize := 1000
+	rsl := make([]*videoinformationmod.VideoInformation, 0)
+	filter := bson.M{}
+	if minTime != nil && endTime != nil {
+		filter[videoinformationmod.FN.VideoInformation.UpdatedAt] = bson.M{
+			"$gte": minTime,
+			"$lte": endTime,
+		}
+	}
+	for {
+		opts := &options.FindOptions{}
+		opts.SetSort(bson.M{videoinformationmod.FN.VideoInformation.UpdatedAt: -1})
+		opts.SetSkip(int64(pageSize * (pageNumber - 1)))
+		opts.SetLimit(int64(pageSize))
+		data, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
+		if serr != nil {
+			return nil, serr
+		}
+		rsl = append(rsl, data...)
+		if len(data) < pageSize {
+			break
+		}
+		pageNumber++
+	}
+	return rsl, nil
+}
+func videoToEsData(data []*videoinformationmod.VideoInformation) esmanager.M {
+	rsl := make(map[string]interface{}, len(data))
+	for _, v := range data {
+		esInfo := esmod.ESVideo{
+			Id:          v.Id,
+			Title:       v.Title,
+			MediaType:   v.MediaType,
+			AddedTime:   v.AddedTime,
+			CreatorName: v.CreatorName,
+			Keyworlds:   "",
+			Desc:        v.Desc,
+			Remark:      v.Remark,
+			Status:      v.Status,
+			IsHot:       v.IsHot,
+			Sort:        v.Sort,
+			WatchTimes:  v.Real.WatchTimes,
+			ShowTimes:   v.Real.ShowTimes,
+			Comments:    v.Real.Comments,
+			UpdatedAt:   v.UpdatedAt,
+			CreatedAt:   v.CreatedAt,
+			Keyworld:    v.Keyworld,
+			CreatorId:   v.CreatorId,
+			FirstTag:    v.FirstTag,
+			SecondTags:  v.SecondTags,
+			CoverImg:    v.CoverImg,
+			Nodes:       v.Nodes,
+			Video:       v.Video,
+			Tags:        v.Tags,
+			Real:        v.Real,
+			VideoCount:  v.VideoCount,
+			IssuedBrand: v.IssuedBrand,
+			Bango:       v.Bango,
+			IssuedAt:    v.IssuedAt,
+		}
+		if v.Keyworld != nil && len(v.Keyworld) > 0 {
+			esInfo.Keyworlds = strings.Join(v.Keyworld, " ")
+		}
+		rsl[strconv.FormatInt(v.Id, 10)] = esInfo
+	}
+	return rsl
+}

web/service/srv_search/search.go

@@ -0,0 +1,27 @@
+package srv_search
+
+import (
+	"github.com/robfig/cron/v3"
+	"videoweb/pub/common/log"
+)
+
+var (
+	skdCron   *cron.Cron
+	batchSize int   = 100
+	opMaxDiff int32 = 10
+)
+
+func Start() {
+	skdCron = cron.New(cron.WithSeconds())
+	log.Info("srv_search:Start...")
+	defer log.Info("srv_search:Start!!!")
+	go func() {
+		StartES()
+	}()
+}
+
+func Stop() {
+	log.Info("srv_search:Stop...")
+	defer log.Info("srv_search:Stop!!!")
+	StopES()
+}

web/service/srv_search/updater.go

@@ -0,0 +1,229 @@
+package srv_search
+
+import (
+	"context"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"strconv"
+	"videoweb/pub/common/log"
+	"videoweb/pub/common/srvdef"
+	"videoweb/pub/common/utils"
+	"videoweb/pub/esmanager"
+	"videoweb/pub/models/esmod"
+	"videoweb/pub/mongos"
+)
+
+// resetAllDocsToES 重设所有文档到数据库
+func resetAllDocsToES(ctx context.Context, param *Param) *srvdef.SrvErr {
+	log.Info("srv_search:resetAllDocsToES...", log.Any("ESIndex", param.EsIndex))
+	defer log.Info("srv_search:startActress!!!")
+	//删除所有记录
+	delAllFromES(param.EsIndex)
+
+	var pageNum int64 = 1
+	var pageSize int64 = 200
+	updSize := 0
+	for {
+		findOpt := constructFindOption(pageNum, pageSize)
+		docs, serr := findManyFromDb(ctx, param, findOpt)
+		if serr != nil {
+			return serr
+		}
+		if len(docs) == 0 {
+			break
+		}
+		updSize = updSize + len(docs)
+		updateESData, serr := sliceMapToJsonMap(docs, param)
+		log.Info("srv_search:resetAllDocsToES", log.Any("updateESData", updateESData))
+		if serr = batchAddToES(param.EsIndex, updateESData); serr != nil {
+			return serr
+		}
+		pageNum++
+		log.Info("resetAllToES>>>>>>>>>>>>>>>>>>", log.Any("updateSize:", updSize))
+	}
+	if updSize > 0 {
+		if err := esmanager.Instance.Refresh(param.EsIndex); err != nil {
+			log.Warn("srv_search:resetAllDocsToES Refresh failed", log.Any("ESIndex", param.EsIndex), log.E(err))
+			return srvdef.New(srvdef.ErrElsOperError, "", "")
+		}
+	}
+	return nil
+}
+
+// constructMatch 构建match
+func constructMatch(param *Param) bson.M {
+	updateFilter := bson.A{}
+	for _, fieldName := range param.CollFields {
+		filter := bson.M{"updateDescription.updatedFields." + fieldName: bson.M{"$exists": true}}
+		updateFilter = append(updateFilter, filter)
+	}
+	match := bson.M{"$or": bson.A{
+		bson.M{"operationType": bson.M{"$in": []string{"delete", "insert", "replace"}}},
+		bson.M{"$and": bson.A{
+			bson.M{"operationType": bson.M{"$in": []string{"update"}}},
+			bson.M{"$or": updateFilter}},
+		}}}
+	return match
+}
+
+// constructActtressOption 查找选项
+func constructFindOption(pageNum int64, pageSize int64) *options.FindOptions {
+	opt := &options.FindOptions{}
+	opt.SetSort(bson.M{"createdAt": -1})
+	skip := (pageNum - 1) * pageSize
+	opt.SetSkip(skip)
+	opt.SetLimit(pageSize)
+	return opt
+}
+
+// AddMultiDataToES 添加一批Ex
+func AddMultiDataToES(esIndex string, esvs esmanager.M) *srvdef.SrvErr {
+	if err := esmanager.Instance.Bulk(esIndex, esvs); err != nil {
+		info := fmt.Sprintf("(srv_search:AddMultiDataToES) ES Bulk failed! err:%+v:", err)
+		log.Warn(info)
+		return srvdef.New(srvdef.ErrElsOperError, "", "")
+	}
+	return nil
+}
+
+// findManyFromDb 查找一批数据
+func findManyFromDb(ctx context.Context, param *Param,
+	opts ...*options.FindOptions) ([]bson.M, *srvdef.SrvErr) {
+	docs := make([]bson.M, 0)
+	if err := param.Coll.Find(ctx, &docs, param.CollFilter, opts...); err != nil {
+		log.Warn("srv_search:findManyFromDb db.Find failed", log.E(err), log.Any("filter", param.CollFilter))
+		return nil, srvdef.New(srvdef.ErrDbQueryError, "", "")
+	}
+	return docs, nil
+}
+
+// sliceMapToJsonMap 从map切片构建Map对象
+func sliceMapToJsonMap(docs []bson.M, param *Param) (esmanager.M, *srvdef.SrvErr) {
+	esMap := make(esmanager.M)
+	for _, doc := range docs {
+		fieldMap := make(esmanager.M)
+		for key, value := range doc {
+			if utils.IncludeString(param.CollFields, key) {
+				fieldMap[key] = value
+			} else if key == "_id" {
+				fieldMap["id"] = value
+			}
+		}
+		if id, ok := doc["_id"].(int64); ok {
+			strId := strconv.FormatInt(id, 10)
+			esMap[strId] = fieldMap
+		} else {
+			log.Warn("srv_search:sliceMapToJsonMap Key doesn't exist", log.Any("keyId", "_id"))
+		}
+	}
+	return esMap, nil
+}
+
+// getInfoFromChangeData 从改变流中获取数据信息
+func getInfoFromChangeData(changeData bson.M, param *Param) (
+	string, *esmod.ESTimestamp, string, esmanager.M, *srvdef.SrvErr) {
+	operationType, ok := changeData["operationType"].(string)
+	if ok == false {
+		log.Error("srv_search:getInfoFromChangeData operationType type error")
+		return "", nil, "", nil, srvdef.New(srvdef.ErrData, "", "")
+	}
+	ptimestamp, ok := changeData["clusterTime"].(primitive.Timestamp)
+	if ok == false {
+		log.Error("srv_search:getInfoFromChangeData clusterTime type error")
+		return "", nil, "", nil, srvdef.New(srvdef.ErrData, "", "")
+	}
+	estimestamp := &esmod.ESTimestamp{T: ptimestamp.T, I: ptimestamp.I}
+	documentKey, ok := changeData["documentKey"].(primitive.M)
+	if ok == false {
+		log.Error("srv_search:getInfoFromChangeData documentKey type error")
+		return "", nil, "", nil, srvdef.New(srvdef.ErrData, "", "")
+	}
+	docKey, ok := documentKey["_id"].(int64)
+	if ok == false {
+		log.Error("srv_search:getInfoFromChangeData documentKey[_id] type error")
+		return "", nil, "", nil, srvdef.New(srvdef.ErrData, "", "")
+	}
+	strId := strconv.FormatInt(docKey, 10)
+	esDocMap := make(esmanager.M)
+	if changeData["fullDocument"] != nil {
+		docMap, ok := changeData["fullDocument"].(primitive.M)
+		if ok == false {
+			log.Error("srv_search:getInfoFromChangeData fullDocument type error")
+			return "", nil, "", nil, srvdef.New(srvdef.ErrData, "", "")
+		}
+
+		for key, value := range docMap {
+			if utils.IncludeString(param.CollFields, key) {
+				esDocMap[key] = value
+			} else if key == "_id" {
+				esDocMap["id"] = value
+			}
+		}
+	}
+	return operationType, estimestamp, strId, esDocMap, nil
+}
+
+// delAllFromES 删除所有数据
+func delAllFromES(esIndex string) *srvdef.SrvErr {
+	query := esmanager.M{
+		"query": esmanager.M{
+			"match_all": esmanager.M{},
+		},
+	}
+	if err := esmanager.Instance.DelMany([]string{esIndex}, query); err != nil {
+		log.Warn(fmt.Sprintf("(srv_search:DelAllFromEs) Es DelMany failed! err:%+v:", err),
+			log.Any("query", query))
+		return srvdef.New(srvdef.ErrElsOperError, "", "")
+	}
+	return nil
+}
+
+// batchAddToES 添加一批数据到ES
+func batchAddToES(esIndex string, esvs esmanager.M) *srvdef.SrvErr {
+	log.Info("srv_search:resetAllDocsToES", log.Any("esIndex", esIndex), log.Any("esvs", esvs))
+	if err := esmanager.Instance.Bulk(esIndex, esvs); err != nil {
+		info := fmt.Sprintf("(srv_search:batchAddToES) ES Bulk failed! err:%+v:", err)
+		log.Warn(info)
+		return srvdef.New(srvdef.ErrElsOperError, "", "")
+	}
+	return nil
+}
+
+// batchDelFromES 删除一批数据从ES
+func batchDelFromES(esIndex string, ids []string) *srvdef.SrvErr {
+	query := esmanager.M{
+		"query": esmanager.M{
+			"bool": esmanager.M{
+				"must": esmanager.A{
+					{
+						"ids": esmanager.M{
+							"values": ids,
+						},
+					},
+				},
+			},
+		},
+	}
+	if err := esmanager.Instance.DelMany([]string{esIndex}, query); err != nil {
+		info := fmt.Sprintf(
+			"(srv_search:batchDelFromES) Es DelMany failed! err:%+v:", err)
+		log.Warn(info)
+		return srvdef.New(srvdef.ErrElsOperError, "", "")
+	}
+	return nil
+}
+
+// watchOpLog 监控表变化
+func watchOpLog(ctx context.Context, db *mongos.Collection, pipeline []bson.M,
+	opt *options.ChangeStreamOptions) (*mongo.ChangeStream, *srvdef.SrvErr) {
+	changeStream, err := db.WatchOpLog(ctx, pipeline, opt)
+	if err != nil {
+		log.Warn("srv_search:watchOpLog db.WatchOpLog failed",
+			log.E(err), log.Any("pipeline", pipeline))
+		return nil, srvdef.New(srvdef.ErrDbQueryError, "", "")
+	}
+	return changeStream, nil
+}