root/huangguo_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
root
2026-09-15 13:57:10 +08:00
co-authored by Claude Opus 5
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skd/job/elastic/esaudiobook/method.go
+27
View File
@@ -0,0 +1,27 @@
package esaudiobook
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/audiobookmod"
)
// 同步用户数据
func SyncDataToES(data []audiobookmod.AudioBookBase) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for k := range data {
var tmp = data[k]
source[data[k].ID.Hex()] = tmp
}
log.Info("Elastic sync audiobook data committing...")
if err := es.Bulk(models.ESInfoAudioBookTable, source); err != nil {
log.Info("Elastic sync audiobook data commit failed.")
return err
}
log.Info("Elastic sync audiobook data committed.")
return nil
}
skd/job/elastic/esaudiobook/sync.go
+58
View File
@@ -0,0 +1,58 @@
package esaudiobook
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/audiobookmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search audiobook sync start...")
defer log.Info("Elastic Search audiobook sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncAudioBookJob, defaultDate)
if err != nil {
log.Error("Elastic sync audiobook data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now().Add(-time.Minute * 20)
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := audiobookmod.GetListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync audiobook data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []audiobookmod.AudioBookBase, syncTime time.Time) error {
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit audiobook data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncAudioBookJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit audiobook data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/eskeyword/method.go
+26
View File
@@ -0,0 +1,26 @@
package eskeyword
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/s/kwstatmod"
)
// 同步数据到ES
func SyncDataToES(data []kwstatmod.Keyword) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
source[v.ID.Hex()] = kwstatmod.ESKeywordSrc(v)
}
log.Info("Elastic sync keyword committing...")
if err := es.Bulk(models.ESStatKeywordTable, source); err != nil {
log.Info("Elastic sync keyword commit failed.")
return err
}
log.Info("Elastic sync keyword committed.")
return nil
}
skd/job/elastic/eskeyword/sync.go
+58
View File
@@ -0,0 +1,58 @@
package eskeyword
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/kwstatmod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search sync keyword start...")
defer log.Info("Elastic Search tag sync keyword end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncKeywordJob, defaultDate)
if err != nil {
log.Error("Elastic sync keyword data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := kwstatmod.GetKeywordsByTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync keyword data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []kwstatmod.Keyword, syncTime time.Time) error {
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit keyword data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncKeywordJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit keyword data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/esmedia/method.go
+65
View File
@@ -0,0 +1,65 @@
package esmedia
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/mediamod"
)
// SyncDataToES 同步数据到ES
func SyncDataToES(data []*mediamod.Media) error {
if len(data) == 0 {
return nil
}
source := buildSource(data)
log.Info("Elastic sync ACG media data committing...")
if err := es.BulkChecked(models.ESMediaTable, source); err != nil {
log.Info("Elastic sync ACG media data commit failed.")
return err
}
log.Info("Elastic sync ACG media data committed.")
return nil
}
func buildSource(data []*mediamod.Media) elastic.M {
var source = elastic.M{}
for _, v := range data {
source[v.ID.Hex()] = mediamod.ESMedia{
ID: v.ID,
Title: v.Title,
HorizontalCover: v.HorizontalCover,
VerticalCover: v.VerticalCover,
Tags: v.Tags,
Summary: v.Summary,
TotalEpisode: v.TotalEpisode,
UpdateStatus: v.UpdateStatus,
MediaType: v.MediaType,
Kind: v.Kind,
PermissionIconHide: v.PermissionIconHide,
Permission: v.Permission,
Price: v.Price,
Direction: v.Direction,
FreeTime: v.FreeTime,
CountLike: v.CountLike,
CountBrowse: v.CountBrowse,
CountCollect: v.CountCollect,
MID: v.MID,
ModuleName: v.ModuleName,
SectionName: v.SectionName,
SID: v.SID,
SectionSort: v.SectionSort,
Choice: v.Choice,
ChoiceSort: v.ChoiceSort,
Number: v.Number,
SortCode: v.SortCode,
Status: v.Status,
IsDelete: v.IsDelete,
UpdatedAct: v.UpdatedAct,
CreatedAt: v.CreatedAt,
UpdateTime: v.UpdateTime,
}
}
return source
}
skd/job/elastic/esmedia/method_test.go
+25
View File
@@ -0,0 +1,25 @@
package esmedia
import (
"testing"
"91porn-server/models/v/mediamod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func TestBuildSourceIncludesCountCollect(t *testing.T) {
id := primitive.NewObjectID()
source := buildSource([]*mediamod.Media{{
ID: id,
CountCollect: 12,
}})
media, ok := source[id.Hex()].(mediamod.ESMedia)
if !ok {
t.Fatalf("source type = %T, want mediamod.ESMedia", source[id.Hex()])
}
if media.CountCollect != 12 {
t.Fatalf("countCollect = %d, want 12", media.CountCollect)
}
}
skd/job/elastic/esmedia/sync.go
+80
View File
@@ -0,0 +1,80 @@
package esmedia
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/mediamod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson/primitive"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search ACG media sync start...")
defer log.Info("Elastic Search ACG media sync end.")
// MongoDB 日期精度为毫秒；本轮开始后的修改必须留给下一轮再次同步。
startedAt := time.Now().Truncate(time.Millisecond)
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncMediaJob, defaultDate)
if err != nil {
log.Error("Elastic sync ACG media get last record time failed", log.E(err))
return
}
if !startedAt.After(lastSyncAt) {
return
}
// 固定 ID 上界，避免持续新增数据使本轮无法结束。
maxID, err := mediamod.GetLatestIDForESSync()
if err != nil {
log.Error("Elastic sync ACG media get latest ID failed", log.E(err))
return
}
sync := mediaSync{
read: mediamod.GetListForESSync,
write: SyncDataToES,
checkpoint: func(at time.Time) error {
return statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncMediaJob, at)
},
}
if err = sync.run(lastSyncAt, startedAt, maxID); err != nil {
log.Error("Elastic sync ACG media failed; checkpoint not confirmed", log.E(err))
}
}
type mediaSync struct {
read func(time.Time, primitive.ObjectID, primitive.ObjectID, int) ([]*mediamod.Media, error)
write func([]*mediamod.Media) error
checkpoint func(time.Time) error
}
func (s mediaSync) run(since, startedAt time.Time, maxID primitive.ObjectID) error {
const size = 1000
var after primitive.ObjectID
for !maxID.IsZero() {
data, err := s.read(since, after, maxID, size)
if err != nil {
return fmt.Errorf("read media batch: %w", err)
}
if len(data) == 0 {
break
}
if err = s.write(data); err != nil {
return fmt.Errorf("write media batch: %w", err)
}
after = data[len(data)-1].ID
if len(data) < size || after == maxID {
break
}
}
// 所有批次及每条 ES 写入均成功后才推进；失败时下一轮从原进度幂等重试。
if err := s.checkpoint(startedAt); err != nil {
return fmt.Errorf("save media checkpoint: %w", err)
}
return nil
}
skd/job/elastic/essearchlog/method.go
+26
View File
@@ -0,0 +1,26 @@
package essearchlog
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/l/searchlogmod"
)
// 同步数据到ES
func SyncDataToES(data []searchlogmod.SearchLog) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
source[v.ID.Hex()] = searchlogmod.ESSearchLogSource(v)
}
log.Info("Elastic sync search log data committing...")
if err := es.Bulk(models.ESLogSearchLogTable, source); err != nil {
log.Info("Elastic sync search log data commit failed.")
return err
}
log.Info("Elastic sync search log data committed.")
return nil
}
skd/job/elastic/essearchlog/sync.go
+58
View File
@@ -0,0 +1,58 @@
package essearchlog
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/l/searchlogmod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search sync search log start...")
defer log.Info("Elastic Search tag sync search log end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncSearchLogJob, defaultDate)
if err != nil {
log.Error("Elastic sync search log data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := searchlogmod.GetSearchLogByTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync search log data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []searchlogmod.SearchLog, syncTime time.Time) error {
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit search log data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncSearchLogJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit search log data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/estag/method.go
+44
View File
@@ -0,0 +1,44 @@
package estag
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/tagmod"
)
// 同步数据到ES
func SyncDataToES(data []tagmod.Tag) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
var tmp = tagmod.ESTag{
ID: v.ID,
TagName: v.TagName,
SortCode: v.SortCode,
VPlayCount: v.VPlayCount,
TPlayCount: v.TPlayCount,
FakePlayCount: v.FakePlayCount,
PlayRating: v.PlayRating,
CollCount: v.CollCount,
LikeCount: v.LikeCount,
FakeCollCount: v.FakeCollCount,
VideoCount: v.VideoCount,
IsActive: v.IsActive,
IsWonder: v.IsWonder,
WonderSortCode: v.WonderSortCode,
CreatedAt: v.CreatedAt,
UpdatedAt: v.UpdatedAt,
}
source[v.ID.Hex()] = tmp
}
log.Info("Elastic sync tag data committing...")
if err := es.Bulk(models.ESInfoTagTable, source); err != nil {
log.Info("Elastic sync tag data commit failed.")
return err
}
log.Info("Elastic sync tag data committed.")
return nil
}
skd/job/elastic/estag/sync.go
+61
View File
@@ -0,0 +1,61 @@
package estag
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/tagmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search tag sync start...")
defer log.Info("Elastic Search tag sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncTagJob, defaultDate)
if err != nil {
log.Error("Elastic sync tag data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := tagmod.GetTagListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync tag data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []tagmod.Tag, syncTime time.Time) error {
if len(data) <= 0 {
return nil
}
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit tag data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncTagJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit tag data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/esuser/method.go
+38
View File
@@ -0,0 +1,38 @@
package esuser
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/usermod"
)
// 同步用户数据
func SyncDataToES(data []usermod.User) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
var tmp = usermod.ESUser{
ID: v.ID,
UID: v.UID,
Gender: v.Gender,
Name: v.Name,
Summary: v.Summary,
Region: v.Region,
CreatedAt: v.CreatedAt,
UpdatedAt: v.UpdatedAt,
HasLocked: v.HasLocked,
HasBanned: v.HasBanned,
}
source[v.ID.Hex()] = tmp
}
log.Info("Elastic sync user data committing...")
if err := es.Bulk(models.ESInfoUserTable, source); err != nil {
log.Info("Elastic sync user data commit failed.")
return err
}
log.Info("Elastic sync user data committed.")
return nil
}
skd/job/elastic/esuser/sync.go
+72
View File
@@ -0,0 +1,72 @@
package esuser
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/usermod"
"91porn-server/skd/skdg"
"fmt"
"time"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search user sync start...")
defer log.Info("Elastic Search user sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncUserJob, defaultDate)
if err != nil {
log.Error("Elastic sync user data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []usermod.User
for hasNext {
data, hasNext, err = usermod.GetUserListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Elastic sync user data GetUserListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToES(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync user data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []usermod.User, syncTime time.Time) error {
err := SyncDataToES(data)
if err != nil {
return fmt.Errorf("Elastic sync commit user data SyncDataToES failed!, err:%+v\n", err)
}
err = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncUserJob, syncTime)
if err != nil {
return fmt.Errorf("Elastic sync commit user data UpsertOneTrans failed!, err:%+v\n", err)
}
return err
}
skd/job/elastic/esvideo/method.go
+108
View File
@@ -0,0 +1,108 @@
package esvideo
import (
"math"
"time"
"91porn-server/common/elastic"
"91porn-server/common/localcache"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/tagmod"
"91porn-server/models/v/vidmod"
"91porn-server/models/v/vidpopmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const vidPopCfgKey = "vidPopCfgKey"
// SyncDataToES 同步数据到ES
func SyncDataToES(data []*vidmod.VideoModel) error {
if len(data) == 0 {
return nil
}
// 获取标签名字
var tagsID []primitive.ObjectID
for _, v := range data {
tagsID = append(tagsID, v.Tags...)
}
tags, err := tagmod.FindTagsByIDS(tagsID)
if err != nil {
return err
}
var tagsNameMap = make(map[primitive.ObjectID]string)
for _, v := range tags {
tagsNameMap[v.ID] = v.TagName
}
var source = elastic.M{}
for _, v := range data {
var tagsName []string
for _, v1 := range v.Tags {
tagsName = append(tagsName, tagsNameMap[v1])
}
source[v.ID.Hex()] = vidmod.ESVideo{
ID: v.ID,
PublisherID: v.PublisherID,
Title: v.Title,
NewsType: v.NewsType,
Tags: v.Tags,
TagsName: tagsName,
Filename: v.Filename,
PlayCount: v.PlayCount,
EffectivePlayCount: v.EffectivePlayCount,
PurchaseCount: v.PurchaseCount,
LikeCount: v.LikeCount,
CommentCount: v.CommentCount,
CollectCount: v.CollectCount,
ShareCount: v.ShareCount,
FakeLikeCount: v.FakeLikeCount,
FakeShareCount: v.FakeShareCount,
FakePlayCount: v.FakePlayCount,
Status: v.Status,
Location: v.Location,
CreatedAt: v.CreatedAt,
UpdatedAt: v.UpdatedAt,
ReviewedAt: v.ReviewAt,
Hot: calcHot(v),
PlayTime: v.PlayTime,
Coins: v.Coins,
}
}
log.Info("Elastic sync video data committing...")
if err = es.Bulk(models.ESInfoVideoTable, source); err != nil {
log.Info("Elastic sync video data commit failed.")
return err
}
log.Info("Elastic sync video data committed.")
return nil
}
func calcHot(video *vidmod.VideoModel) float64 {
if video.Status == vidmod.CheckPass || video.Status == vidmod.Free {
cfg, err := getVideoPopularityConfig()
if err != nil {
log.Error("Elastic sync video getVideoPopularityConfig", log.E(err))
return 0
}
qualityScore := float64(video.PlayCount + cfg.EffectivePlayCountMultiplier*video.EffectivePlayCount +
cfg.LikeCountMultiplier*video.LikeCount)
hot := (qualityScore + float64(cfg.InitialPopularity)) / math.Pow(1+time.Since(video.ReviewAt).Hours(),
cfg.ReviewTimePower)
return hot
}
return 0
}
func getVideoPopularityConfig() (*vidpopmod.VideoPopularityConfig, error) {
cfg, ok := localcache.C.Get(vidPopCfgKey)
if ok {
return cfg.(*vidpopmod.VideoPopularityConfig), nil
}
res, err := vidpopmod.FindOne()
if err != nil {
return nil, err
}
localcache.C.Set(vidPopCfgKey, &res, 5*time.Minute)
return &res, nil
}
skd/job/elastic/esvideo/sync.go
+74
View File
@@ -0,0 +1,74 @@
package esvideo
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search video sync start...")
defer log.Info("Elastic Search video sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncVideoJob, defaultDate)
if err != nil {
log.Error("Elastic sync video data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []*vidmod.VideoModel
for hasNext {
data, hasNext, err = vidmod.GetVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Elastic sync video data GetVideoListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToES(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync video data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
if len(data) <= 0 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime)
return nil
}
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit video data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit video data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/rdvideo/method.go
+27
View File
@@ -0,0 +1,27 @@
package rdvideo
import (
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/models/v/vidmod"
)
// SyncDataToRedis 同步数据到redis
func SyncDataToRedis(data []*vidmod.VideoModel) error {
if len(data) == 0 {
return nil
}
for _, v := range data {
if v.NewsType != vidmod.SHORT || (v.Status != vidmod.CheckPass && v.Status != vidmod.Free) {
// 去除
_, _ = redis.Handler.SRem(redisconst.ShortVideosKey, v.ID.Hex())
} else {
_, _ = redis.Handler.SAdd(redisconst.ShortVideosKey, v.ID.Hex())
}
}
log.Info("Redis sync video data completed.")
return nil
}
skd/job/elastic/rdvideo/sync.go
+77
View File
@@ -0,0 +1,77 @@
package rdvideo
import (
"fmt"
"time"
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/vidmod"
)
func RunSync(defaultDate time.Time) {
log.Info("Redis short video sync start...")
defer log.Info("Redis short video sync end.")
// 如果短视频缓存存在的数据量过低，表示缓存被清理过，需要重新同步
c, err := redis.Handler.SCard(redisconst.ShortVideosKey)
if err == nil && c < 1000 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, defaultDate)
}
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.RedisSyncShortVideoJob, defaultDate)
if err != nil {
log.Error("Redis sync short video data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.OneDayScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.OneDayScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []*vidmod.VideoModel
for hasNext {
data, hasNext, err = vidmod.GetShortVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Redis sync short video data GetVideoListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToRedis(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Redis sync short video data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
if len(data) <= 0 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime)
return nil
}
if err := SyncDataToRedis(data); err != nil {
return fmt.Errorf("Redis sync commit short video data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime); err != nil {
return fmt.Errorf("Redis sync commit short video data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}