Initial commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
谢宇宁
2026-10-22 14:47:17 +07:00
co-authored by Claude Opus 5.5
commit 3639d2184a
574 changed files with 61690 additions and 0 deletions
+54
View File
@@ -0,0 +1,54 @@
package esmod
import (
"context"
"time"
"videoweb/pub/common/srvdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const (
FiveMinuteScale = 5 //对齐时间(分钟)
)
var (
videoIndex = "webmdgw_video_index_v3"
)
func GetVideoIndex() string {
return videoIndex
}
func GetRecordByESIndex(ctx context.Context, esIndex string) (
*ESTimestamp, *srvdef.SrvErr) {
filter := bson.M{FN.ESRecord.EsIndex: esIndex}
esRecord, serr := FindOneESRecord(ctx, filter)
if serr != nil {
if serr.Code != srvdef.ErrNoData {
return nil, serr
}
return nil, nil
}
return &esRecord.Timestamp, nil
}
func SetOneEsRecord(ctx context.Context, esIndex string,
timestamp *ESTimestamp) *srvdef.SrvErr {
filter := bson.M{FN.ESRecord.EsIndex: esIndex}
upsertDoc := bson.M{
"$set": bson.M{
FN.ESRecord.Timestamp.T: timestamp.T,
FN.ESRecord.Timestamp.I: timestamp.I,
FN.ESRecord.UpdatedAt: time.Now(),
},
"$setOnInsert": bson.M{
FN.ESRecord.Id: primitive.NewObjectID(),
FN.ESRecord.CreatedAt: time.Now(),
},
}
serr := UpsertOneESRecord(ctx, filter, upsertDoc)
return serr
}
+124
View File
@@ -0,0 +1,124 @@
package esmod
import (
"context"
"fmt"
"videoweb/pub/esmanager"
"videoweb/pub/mongos"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
var db *mongos.Collection
// Init 初始化数据
func Init(ctx context.Context) {
initDBIndex(ctx)
initIndex()
return
}
// initIndex 初始化索引
func initIndex() {
if esmanager.Instance == nil {
return
}
initVideoIndex(videoIndex)
}
// initDBIndex 初始化数据库索引
func initDBIndex(ctx context.Context) {
//获取数据库表
db = mongos.Shop.Coll(TN.ESRecord)
// 获取数据库表
indexes := []mongo.IndexModel{
{
Keys: bson.D{
{Key: FN.ESRecord.EsIndex, Value: 1},
},
Options: options.Index().SetUnique(true),
},
}
if _, err := db.CreateIndex(ctx, indexes); err != nil {
panic(fmt.Sprintf("(esmod:initDBIndex) Create es_record Index failed! err:%+v:", err))
}
}
func initVideoIndex(esIndex string) {
var setting = esmanager.M{
"settings": esmanager.M{
"number_of_shards": esmanager.NumberOfShards,
"number_of_replicas": esmanager.NumberOfReplicas,
"analysis": esmanager.M{
"analyzer": esmanager.M{
"ik": esmanager.M{
"tokenizer": esmanager.AnalyzerIkSmart,
},
},
},
},
"mappings": esmanager.M{
"properties": esmanager.M{
FN.ESVideo.Title: esmanager.M{
"type": "text",
"analyzer": esmanager.AnalyzerIkMaxWord,
"search_analyzer": esmanager.AnalyzerIkSmart,
},
FN.ESVideo.MediaType: esmanager.M{
"type": "integer",
},
FN.ESVideo.Remark: esmanager.M{
"type": "text",
"analyzer": esmanager.AnalyzerIkSmart,
"search_analyzer": esmanager.AnalyzerIkSmart,
},
FN.ESVideo.Desc: esmanager.M{
"type": "text",
"analyzer": esmanager.AnalyzerIkSmart,
"search_analyzer": esmanager.AnalyzerIkSmart,
},
FN.ESVideo.Keyworld: esmanager.M{
"type": "text",
"analyzer": esmanager.AnalyzerIkSmart,
"search_analyzer": esmanager.AnalyzerIkSmart,
},
FN.ESVideo.CreatorName: esmanager.M{
"type": "text",
"analyzer": esmanager.AnalyzerIkSmart,
"search_analyzer": esmanager.AnalyzerIkSmart,
},
FN.ESVideo.Status: esmanager.M{
"type": "integer",
},
//FN.ESVideo.IsHot: esmanager.M{
// "type": "boolean",
//},
//FN.ESVideo.Sort: esmanager.M{
// "type": "integer",
//},
FN.ESVideo.WatchTimes: esmanager.M{
"type": "integer",
},
FN.ESVideo.ShowTimes: esmanager.M{
"type": "integer",
},
FN.ESVideo.Comments: esmanager.M{
"type": "integer",
},
FN.ESVideo.AddedTime: esmanager.M{
"type": "date",
"format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss",
},
//FN.ESVideo.CreatedAt: esmanager.M{
// "type": "date",
// "format": "strict_date_optional_time||epoch_millis||yyyy-MM-dd HH:mm:ss",
//},
},
},
}
if err := esmanager.Instance.InitIndices(esIndex, setting); err != nil {
panic(fmt.Sprintf("ElasticSearch Index %s initMediaIndex error", esIndex))
}
}
+144
View File
@@ -0,0 +1,144 @@
package esmod
import (
"context"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
"videoweb/pub/common/log"
"videoweb/pub/common/srvdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// InsertOneESRecord 插入ES操作记录
func InsertOneESRecord(ctx context.Context, esRecord *ESRecord) *srvdef.SrvErr {
if _, err := db.InsertOne(ctx, esRecord); err != nil {
log.Warn("esmod:InsertOneESRecord db.InsertOne failed", log.E(err))
return srvdef.New(srvdef.ErrDbInsertError, "", "")
}
return nil
}
// DelOneESRecord 删除一个ES操作记录
func DelOneESRecord(ctx context.Context, filter bson.M) (
int64, *srvdef.SrvErr) {
count, err := db.DeleteOne(ctx, filter)
if err != nil {
log.Warn("esmod:DelOnAppTag db.DeleteOne failed",
log.E(err), log.Any("filter", filter))
return 0, srvdef.New(srvdef.ErrDbDeleteError, "", "")
}
return count, nil
}
// FindOneESRecord 查找一个ES操作记录
func FindOneESRecord(ctx context.Context, filter bson.M) (
*ESRecord, *srvdef.SrvErr) {
var esRecord ESRecord
if err := db.FindOneEx(
ctx, &esRecord, filter); err != nil {
if err != mongo.ErrNoDocuments {
log.Info("esmod:FindOnESRecord db.FindOne failed",
log.E(err), log.Any("filter", filter))
return nil, srvdef.New(srvdef.ErrDbQueryError, "", "")
}
return nil, srvdef.New(srvdef.ErrNoData, "", "")
}
return &esRecord, nil
}
// CountESRecords 统计ES操作记录
func CountESRecords(ctx context.Context, filter bson.M) (
int64, *srvdef.SrvErr) {
count, err := db.Count(ctx, filter)
if err != nil {
log.Warn("esmod:CountMediaCats db.Count failed",
log.E(err), log.Any("filter", filter))
return 0, srvdef.New(srvdef.ErrDbQueryError, "", "")
}
return count, nil
}
// UpdateOneESRecord 更新ES操作记录
func UpdateOneESRecord(ctx context.Context, filter bson.M, update bson.M) *srvdef.SrvErr {
result, err := db.UpdateOne(ctx, filter, update)
if err != nil {
log.Warn("esmod:UpdateOnESRecord db.UpdateOne failed",
log.E(err), log.Any("filter", filter))
return srvdef.New(srvdef.ErrDbUpdateError, "", "")
}
if result.MatchedCount != 1 {
log.Warn("esmod:UpdateOnESRecord db.UpdateOne failed",
log.E(err), log.Any("filter", filter))
return srvdef.New(srvdef.ErrMediaRecordNotExsit, "", "")
}
return nil
}
// UpsertOneESRecord 插入或更新一条ES操作记录
func UpsertOneESRecord(ctx context.Context,
filter bson.M, update bson.M) *srvdef.SrvErr {
if _, err := db.UpsertOne(ctx, filter, update); err != nil {
log.Info("esmod:UpsertOneESRecord db.InsertOne failed",
log.E(err), log.Any("filter", filter))
return srvdef.New(srvdef.ErrDbUpdateError, "", "")
}
return nil
}
// UpdateManyESRecord 更新一批ES操作记录
func UpdateManyESRecord(ctx context.Context, filter bson.M, update bson.M) *srvdef.SrvErr {
result, err := db.UpdateMMany(ctx, filter, update)
if err != nil {
log.Warn("esmod:UpdateManyMediaCat db.UpdateMMany failed",
log.E(err), log.Any("filter", filter))
return srvdef.New(srvdef.ErrDbUpdateError, "", "")
}
if result.MatchedCount <= 0 {
log.Warn("esmod:UpdateManyMediaCat db.UpdateMMany failed",
log.E(err), log.Any("filter", filter))
//return srvdef.New(srvdef.ErrMediaTagNotExsit, "", "")
}
return nil
}
// FindManyESRecord 查找一批ES操作记录
func FindManyESRecord(ctx context.Context, filter bson.M,
opts ...*options.FindOptions) (ESRecordList, *srvdef.SrvErr) {
data := make(ESRecordList, 0)
if err := db.Find(ctx, &data, filter, opts...); err != nil {
log.Warn("esmod:FindManyESRecord db.Find failed",
log.E(err), log.Any("filter", filter))
return nil, srvdef.New(srvdef.ErrDbQueryError, "", "")
}
return data, nil
}
func GetRecordTimeByESIndex(ctx context.Context, esIndex string) (*ESRecord, *srvdef.SrvErr) {
filter := bson.M{FN.ESRecord.EsIndex: esIndex}
esRecord, serr := FindOneESRecord(ctx, filter)
if serr != nil {
if serr.Code != srvdef.ErrNoData {
return nil, serr
}
return nil, nil
}
return esRecord, nil
}
func SetOneEsRecordTime(ctx context.Context, esIndex string, recordTime time.Time) *srvdef.SrvErr {
filter := bson.M{FN.ESRecord.EsIndex: esIndex}
upsertDoc := bson.M{
"$set": bson.M{
FN.ESRecord.RecordTime: recordTime,
FN.ESRecord.UpdatedAt: time.Now(),
},
"$setOnInsert": bson.M{
FN.ESRecord.Id: primitive.NewObjectID(),
FN.ESRecord.CreatedAt: time.Now(),
},
}
serr := UpsertOneESRecord(ctx, filter, upsertDoc)
return serr
}
+90
View File
@@ -0,0 +1,90 @@
package esmod
type TagESRecord struct {
CreatedAt string
EsIndex string
Id string
RecordTime string
Timestamp struct {
I string
T string
}
UpdatedAt string
}
type TagESVideo struct {
Id string
Title string
MediaType string
AddedTime string
CreatorName string
Keyworlds string
Desc string
Remark string
Status string
IsHot string
Sort string
WatchTimes string
ShowTimes string
Comments string
UpdatedAt string
CreatedAt string
Keyworld string
CreatorId string
FirstTag string
SecondTags string
CoverImg string
Nodes string
Video string
CategoryIds string
Tags string
Real string
}
var FN struct {
ESRecord TagESRecord
ESVideo TagESVideo
}
var TN struct {
ESRecord string
ESVideo string
}
func init() {
FN.ESRecord = TagESRecord{
CreatedAt: "createdAt",
EsIndex: "esIndex",
Id: "_id",
RecordTime: "recordTime",
Timestamp: struct {
I string
T string
}{
I: "i",
T: "t",
},
UpdatedAt: "updatedAt",
}
FN.ESVideo = TagESVideo{
Id: "_id",
Title: "title",
Desc: "desc",
Status: "status",
AddedTime: "addedTime",
MediaType: "mediaType",
CreatorName: "creatorName",
Keyworlds: "keyworlds",
Remark: "remark",
IsHot: "isHot",
Sort: "sort",
UpdatedAt: "updatedAt",
CreatedAt: "createdAt",
WatchTimes: "watchTimes",
ShowTimes: "showTimes",
Comments: "comments",
}
TN.ESRecord = "es_record"
TN.ESVideo = "es_video"
}
+85
View File
@@ -0,0 +1,85 @@
//go:generate gen_bson_tag -m esmod
//esmod essearch数据结构定义
package esmod
import (
"time"
"videoweb/pub/models/videoinformationmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ESTimestamp struct {
T uint32 `json:"t" bson:"t"`
I uint32 `json:"i" bson:"i"`
}
func (v *ESTimestamp) IsZero() bool {
if v == nil {
return true
}
return v.T == 0 && v.I == 0
}
// ESOperator es操作记录
// @table:es_record
type ESRecord struct {
Id primitive.ObjectID `json:"id" bson:"_id"`
EsIndex string `json:"esIndex" bson:"esIndex"` //es索引名称
Timestamp ESTimestamp `json:"timestamp" bson:"timestamp"` //操作时间
RecordTime time.Time `json:"recordTime" bson:"recordTime"` // 操作记录时间
UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"` //更新时间
CreatedAt time.Time `json:"createdAt" bson:"createdAt"` //创建时间
}
type ESRecordList []*ESRecord
// ESVideo es中存储的media数据
// @table:es_video
type ESVideo struct {
Id int64 `json:"id" bson:"_id,omitempty"`
Title string `json:"title" bson:"title"` // 帖子标题
MediaType videoinformationmod.MediaType `json:"mediaType" bson:"mediaType"` // 资源类型
AddedTime time.Time `json:"addedTime" bson:"addedTime"` // 帖子上架时间
CreatorName string `json:"creatorName" bson:"creatorName"` // 作者名称
Keyworlds string `bson:"keyworlds,omitempty" json:"keyworlds"` // 关键字
Desc string `bson:"desc,omitempty" json:"desc"` // 详细描述
Remark string `bson:"remark,omitempty" json:"remark"` // 备注
Status videoinformationmod.InformationSatus `json:"status" bson:"status"` // 帖子上架状态
IsHot bool `json:"isHot" bson:"isHot"` // 是否热门
Sort int64 `json:"sort" bson:"sort"` // 排序
WatchTimes int64 `json:"watchTimes" bson:"watchTimes"` // 观看次数
ShowTimes int64 `json:"showTimes" bson:"showTimes"` // 展示次数
Comments int64 `json:"comments" bson:"comments"` // 评论条数
UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"` // 更新时间
CreatedAt time.Time `json:"createdAt" bson:"createdAt"` // 创建时间
Keyworld []string `bson:"keyworld,omitempty" json:"keyworld"` // 关键字
CreatorId int64 `json:"creatorId" bson:"creatorId"` // 作者Id
FirstTag string `bson:"firstTag,omitempty" json:"firstTag"` // 一级标签
SecondTags []string `bson:"secondTags,omitempty" json:"secondTags"` // 二级标签标签
CoverImg string `bson:"coverImg,omitempty" json:"coverImg"` // 封面
Nodes []videoinformationmod.Node `json:"nodes" bson:"nodes"` // 节点
Video videoinformationmod.VideoInfo `json:"video" bson:"video"` // 视频
CategoryIds []int64 `json:"categoryIds" bson:"categoryIds"` // 分类Id
Tags []int64 `json:"tags" bson:"tags"` // 标签
Real videoinformationmod.Flow `json:"real" bson:"real"` // 真实流量
VideoCount int64 `json:"videoCount" bson:"videoCount"` // 合集视频数量
}
type ESVideoList []*ESVideo
type ESTotal struct {
Value int64 `json:"value"`
}
type ESVideoSource struct {
ID string `json:"_id"`
Score float32 `json:"_score"`
Source *ESVideo `json:"_source"`
}
// ESVideoHits search media hits
type ESVideoHits struct {
Total ESTotal `json:"total"`
Hits []*ESVideoSource `json:"hits"`
}
+135
View File
@@ -0,0 +1,135 @@
package esmod
import (
"fmt"
"videoweb/app/api/proto"
"videoweb/pub/common/log"
"videoweb/pub/common/srvdef"
"videoweb/pub/esmanager"
"videoweb/pub/models/videoinformationmod"
)
// SearchVideo 根据关键字搜索媒体
func SearchVideo(content string, from int64, size int64, sort string) (int64, []*videoinformationmod.VideoInformation, *srvdef.SrvErr) {
fields := []string{FN.ESVideo.Title, FN.ESVideo.CreatorName, FN.ESVideo.Keyworld, FN.ESVideo.Desc, FN.ESVideo.Remark}
must := esmanager.M{
"bool": esmanager.M{
"must": esmanager.A{
{"multi_match": esmanager.M{
"query": content,
"type": "best_fields",
"fields": fields},
},
{"term": esmanager.M{FN.ESVideo.Status: videoinformationmod.InformationSatusVerifyOK}},
},
},
}
switch sort {
case string(proto.SearchSortTypeHot):
must = esmanager.M{
"function_score": esmanager.M{
"query": esmanager.M{
"bool": esmanager.M{
"must": esmanager.A{
{"multi_match": esmanager.M{
"query": content,
"type": "best_fields",
"fields": fields,
}},
{"term": esmanager.M{FN.ESVideo.Status: videoinformationmod.InformationSatusVerifyOK}},
},
},
},
"functions": esmanager.A{
esmanager.M{
"field_value_factor": esmanager.M{
"field": FN.ESVideo.WatchTimes,
"factor": 1.0,
"modifier": "none",
},
},
},
"score_mode": "sum",
"boost_mode": "multiply",
},
}
case string(proto.SearchSortTypeNew):
must = esmanager.M{
"bool": esmanager.M{
"must": esmanager.A{
{"multi_match": esmanager.M{
"query": content,
"type": "best_fields",
"fields": fields},
},
{"term": esmanager.M{FN.ESVideo.Status: videoinformationmod.InformationSatusVerifyOK}},
},
"should": esmanager.A{
{"range": esmanager.M{FN.ESVideo.AddedTime: esmanager.M{"gte": "now-30d/d", "boost": 2.0}}}, //7天内提升4倍权重
{"range": esmanager.M{FN.ESVideo.AddedTime: esmanager.M{"gte": "now-180d/d", "boost": 1.5}}}, // 30天内提升3倍权重
{"range": esmanager.M{FN.ESVideo.AddedTime: esmanager.M{"gte": "now-365d/d", "boost": 1.1}}}, // 90天内提升1.5倍权重
},
"minimum_should_match": 0, // 允许不匹配任何should条件
},
}
}
query := esmanager.M{
"query": must,
"sort": esmanager.A{
esmanager.M{"_score": esmanager.M{"order": "desc"}},
},
"from": from,
"size": size,
}
videos := &ESVideoHits{}
err := esmanager.Instance.Search(videoIndex, videos, query)
if err != nil {
info := fmt.Sprintf(
"(esmod:SearchVideo) Search failed! err:%+v:", err)
log.Warn(info,
log.Any("keywords", content),
log.Any("from", from),
log.Any("size", size))
return 0, nil, srvdef.New(srvdef.ErrElsOperError, "", "")
}
total := int64(0)
rslData := make([]*videoinformationmod.VideoInformation, 0)
if videos != nil && videos.Hits != nil && len(videos.Hits) > 0 {
total = videos.Total.Value
for _, video := range videos.Hits {
rslData = append(rslData, EsVideoInfoToData(video.Source))
}
}
return total, rslData, nil
}
func EsVideoInfoToData(esInfo *ESVideo) *videoinformationmod.VideoInformation {
info := &videoinformationmod.VideoInformation{
Id: esInfo.Id,
Title: esInfo.Title,
MediaType: esInfo.MediaType,
AddedTime: esInfo.AddedTime,
CreatorId: esInfo.CreatorId,
CreatorName: esInfo.CreatorName,
Keyworld: esInfo.Keyworld,
FirstTag: esInfo.FirstTag,
SecondTags: esInfo.SecondTags,
CoverImg: esInfo.CoverImg,
Desc: esInfo.Desc,
Remark: esInfo.Remark,
Nodes: esInfo.Nodes,
Status: esInfo.Status,
Video: esInfo.Video,
CategoryIds: esInfo.CategoryIds,
Tags: esInfo.Tags,
IsHot: esInfo.IsHot,
Sort: esInfo.Sort,
Real: esInfo.Real,
VideoCount: esInfo.VideoCount,
UpdatedAt: esInfo.UpdatedAt,
CreatedAt: esInfo.CreatedAt,
}
return info
}