root/huangguo_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
root
2026-09-15 13:57:10 +08:00
co-authored by Claude Opus 5
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skd/job/cache/rankcache/run.go
Vendored
+51
View File
@@ -0,0 +1,51 @@
package rankcache
import (
"91porn-server/app/service/rankser"
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/models/v/usermod"
"91porn-server/skd/skdg"
)
type UserRank = rankser.UserRank
type User = rankser.User
type UBaseInfo = usermod.BaseInfo
const (
UserInviteRankType = rankser.UserInviteRankType //推广达人
UserIncomeRankType = rankser.UserIncomeRankType //收益达人
UserUploadRankType = rankser.UserUploadRankType //上传达人
)
func Update() {
saveUserIncomeRank()
saveUserUploadRank()
}
func saveUserIncomeRank() {
rank, err := rankser.GetUserIncomeRank()
if err != nil {
log.Error("rankcach GetUserIncomeRank error", log.E(err))
return
}
redisKey := redisconst.RankKey(UserIncomeRankType)
if err = skdg.Redis.Set(redisKey, rank, redisconst.RankExpireMax); err != nil {
log.Warn("rankcach UserIncomeRankType redis set error", log.E(err))
}
}
func saveUserUploadRank() {
rank, err := rankser.GetUserUploadRank()
if err != nil {
log.Error("rankcach GetUserUploadRank error", log.E(err))
return
}
redisKey := redisconst.RankKey(UserUploadRankType)
_ = skdg.Redis.Set(redisKey, rank, redisconst.RankExpireMax)
if err != nil {
log.Warn("rankcach UserUploadRankType redis set error", log.E(err))
}
}
skd/job/cache/tonecache/run.go
Vendored
+51
View File
@@ -0,0 +1,51 @@
package tonecache
import (
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/models/v/tonemod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
)
const MaxVideoCount = 500
func Update() {
officialRecomPlayCount, err := vidmod.SumFakePlayCountSortByToneOfficialRecom(MaxVideoCount)
if err != nil {
log.Error("tonecach OfficialRecomPlayCount SumFakePlayCountSortByToneOfficialRecom faild", log.E(err))
}
latestUploadPlayCount, err := vidmod.SumFakePlayCountSortByCreatedAt(MaxVideoCount)
if err != nil {
log.Error("tonecach LatestUploadPlayCount SumFakePlayCountSortByCreatedAt faild", log.E(err))
}
mostLikesPlayCount, err := vidmod.SumFakePlayCountSortByFakeLikeCount(MaxVideoCount)
if err != nil {
log.Error("tonecach MostLikesPlayCount SumFakePlayCountSortByFakeLikeCount faild", log.E(err))
}
playAtMostPlayCount, err := vidmod.SumFakePlayCountSortByFakePlayCount(MaxVideoCount)
if err != nil {
log.Error("tonecach PlayAtMostPlayCount SumFakePlayCountSortByFakePlayCount faild", log.E(err))
}
mostCommentsPlayCount, err := vidmod.SumFakePlayCountSortByFakeCommentCount(MaxVideoCount)
if err != nil {
log.Error("tonecach MostCommentsPlayCount SumFakePlayCountSortByFakeCommentCount faild", log.E(err))
}
freeAreaPlayCount, err := vidmod.SumFakePlayCountSortByFreeArea(MaxVideoCount)
if err != nil {
log.Error("tonecach MostCommentsPlayCount SumFakePlayCountSortByFakeCommentCount faild", log.E(err))
}
toneValue := tonemod.ToneValue{
OfficialRecomPlayCount: officialRecomPlayCount,
GoldCoinAreaPayCount: 0,
LatestUploadPlayCount: latestUploadPlayCount,
MostLikesPlayCount: mostLikesPlayCount,
PlayAtMostPlayCount: playAtMostPlayCount,
MostCommentsPlayCount: mostCommentsPlayCount,
FreeAreaPlayCount: freeAreaPlayCount,
}
redisKey := redisconst.ToneKey()
if err = skdg.Redis.Set(redisKey, toneValue, redisconst.ToneExpireMax); err != nil {
log.Warn("tonecach Update redis set error", log.E(err))
}
}
skd/job/check/daichongcallback.go
+149
View File
@@ -0,0 +1,149 @@
package check
import (
"context"
"time"
"91porn-server/common/daichong"
"91porn-server/common/log"
"91porn-server/common/pageopt"
"91porn-server/common/stderr"
"91porn-server/models/commod"
"91porn-server/models/v/rchgordmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson/primitive"
)
var pageSize int64 = 20
// 通知第三方代充订单结果
func DaiChongOrderCallBack() {
log.Info("DaiChongOrderCallBack start")
sort := rchgordmod.Sort{{Key: "createdAt", Value: 1}}
daichongType := string(rchgordmod.Rech_DaiChong)
matList := []pageopt.Matcher{
(&rchgordmod.RechargeTypeMatch{RechargeType: &daichongType}).New(),
(&rchgordmod.NotifyStatusMatch{NotifyStatus: []int{rchgordmod.Notify_DEFAULT, rchgordmod.Notify_FAILURE}}).New(),
(&rchgordmod.StatusInMatch{Status: []int{rchgordmod.SUCCESS, rchgordmod.FAILURE}}).New(),
}
var pageNumber int64 = 1 //从第一页开始
for {
var skip = (pageNumber - 1) * pageSize
//订单列表
rchgorderList, err := rchgordmod.List(sort, &skip, &pageSize, matList...)
if err != nil {
return
}
if len(rchgorderList) == 0 {
log.Info("DaiChongOrderCallBack end")
return
}
for _, v := range rchgorderList {
if oid, err := primitive.ObjectIDFromHex(v.OID); err == nil {
if !oid.IsZero() {
continue
}
}
handOrderBack(v)
}
pageNumber++
}
}
// 将超时支付的订单设置为失败订单,并通知给对方
func HandleProcessingOrder() {
log.Info("HandleProcessingOrder start")
//创建时间小于三天前的订单
threeDayBefore := time.Now().AddDate(0, 0, -3)
processing := rchgordmod.Processing
sort := rchgordmod.Sort{{Key: "createdAt", Value: 1}}
daichongType := string(rchgordmod.Rech_DaiChong)
matList := []pageopt.Matcher{
(&rchgordmod.RechargeTypeMatch{RechargeType: &daichongType}).New(),
(&rchgordmod.CreatedAtLTMatch{LT: &threeDayBefore}).New(),
(&rchgordmod.StatusMatch{Status: &processing}).New(),
}
var pageNumber int64 = 1 //从第一页开始
for {
var skip = (pageNumber - 1) * pageSize
//订单列表
rchgorderList, err := rchgordmod.List(sort, &skip, &pageSize, matList...)
if err != nil {
return
}
if len(rchgorderList) == 0 {
log.Info("HandleProcessingOrder end")
return
}
for _, v := range rchgorderList {
v.Status = rchgordmod.FAILURE
v.StatusDesc = rchgordmod.Status(rchgordmod.FAILURE).Desc()
v.Remark = "该订单超时三天未处理,失败"
v.FailureAt = time.Now()
handOrderBack(v)
}
pageNumber++
}
}
func handOrderBack(v *rchgordmod.RechargeOrder) {
var orderStatus int
switch v.Status {
case rchgordmod.SUCCESS:
orderStatus = daichong.Success
case rchgordmod.FAILURE:
orderStatus = daichong.Failed
}
if orderStatus == 0 {
return
}
var timestamp = v.NotifyTime
if v.NotifyTime == 0 {
timestamp = time.Now().UTC().Unix()
}
resp, code := orderCallBack(daichong.OrderCallSign{
OrderId: v.OID,
Time: timestamp,
Status: orderStatus,
})
selector := rchgordmod.ReplaceEditSelector{
Status: &v.Status,
StatusDesc: &v.StatusDesc,
Remark: &v.Remark,
FailureAt: &v.FailureAt,
SuccessAt: &v.SuccessAt,
}
if code == stderr.Success {
notifySuccess := rchgordmod.Notify_SUCCESS
selector.NotifyStatus = &notifySuccess
log.Info("daichong order call back send http request successfully",
log.Any("orderStatus", orderStatus),
log.Any("orderId", v.OID),
log.Any("time", time.Now().Format(time.RFC3339)))
} else {
notifyFailure := rchgordmod.FAILURE
selector.NotifyStatus = &notifyFailure
log.Warn("daichong order call back send http request failed", log.Any("responseCode", code), log.Any("msg", resp.Msg))
}
_ = rchgordmod.UpdateBaseOnOid(nil, v.OID, selector)
}
// 上分通知 将订单的处理结果通知给代充平台
func orderCallBack(req daichong.OrderCallSign) (commod.Resp, stderr.Code) {
sign, err := daichong.Convert2Sign(req, skdg.Conf.DaiChong.AppSecret)
if err != nil {
log.Warn("create sign data wrong", log.Any("warn", err))
return commod.Resp{}, stderr.Failure
}
orderReq := daichong.CommonReq{
AppId: skdg.Conf.DaiChong.AppID,
Data: sign,
}
resp, err := daichong.OrderCallBack(context.TODO(), orderReq, skdg.Conf.DaiChong.Domain)
if err != nil {
log.Warn("GetOrderInfo send http wrong", log.Any("warn", err))
return commod.Resp{}, stderr.Failure
}
return resp, resp.Code
}
skd/job/checkin/checkin.go
+70
View File
@@ -0,0 +1,70 @@
package checkin
import (
"91porn-server/common/log"
"91porn-server/models/v/checkinconfigmod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/service"
"go.mongodb.org/mongo-driver/bson"
)
var videoCheckRunning bool
// VideoCheckRS 定期检查签到奖励视频的切片状态
func VideoCheckRS() {
if videoCheckRunning {
return
}
videoCheckRunning = true
defer func() {
videoCheckRunning = false
}()
cfg, err := checkinconfigmod.FindOne(bson.M{})
if err != nil || cfg == nil {
return
}
if len(cfg.RewardBgVideos) == 0 {
return
}
updated := false
for i, v := range cfg.RewardBgVideos {
if v.FsResourceId == "" {
continue
}
// 已完成的跳过
if v.FsResourceStatus == vidmod.Completed {
continue
}
fileInfo, err := service.PullFileInfo(v.FsResourceId)
if err != nil {
log.Error("checkin VideoCheckRS PullFileInfo fail",
log.Any("fsResourceId", v.FsResourceId), log.E(err))
continue
}
newStatus := fileInfo.Data.Status
if newStatus == "" {
continue
}
if newStatus != v.FsResourceStatus {
cfg.RewardBgVideos[i].FsResourceStatus = newStatus
updated = true
log.Info("checkin VideoCheckRS status changed",
log.Any("fsResourceId", v.FsResourceId),
log.Any("oldStatus", v.FsResourceStatus),
log.Any("newStatus", newStatus))
}
}
if updated {
err := checkinconfigmod.UpdateOne(
bson.M{"_id": cfg.ID},
bson.M{"$set": bson.M{"rewardBgVideos": cfg.RewardBgVideos}},
)
if err != nil {
log.Error("checkin VideoCheckRS update config fail", log.E(err))
}
}
}
skd/job/dailyAdverCalc/job.go
+205
View File
@@ -0,0 +1,205 @@
package dailyAdverCalc
import (
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/models/commod"
"91porn-server/models/s/dailyretentionmod"
"91porn-server/models/s/useradverstatmod"
"context"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// retentionKey 用于内存中按 (RegDay, AdGroup) 分组
type retentionKey struct {
RegDay time.Time
AdGroup commod.AdGroup
}
// retentionAccum 内存中的累计数据
type retentionAccum struct {
Retained int64
AdClick int64
TotalClick int64
PayCount int64
PayTotal int64
}
// CalcRetention 计算留存数据
// targetDate: 活跃日期（要统计哪一天的活跃数据）
// 例如：targetDate=15号，会计算所有在15号活跃的用户，按其注册日分组，计算各自的第N天留存
func CalcRetention(ctx context.Context, targetDate time.Time) error {
targetDate = common.NormalizeDate(targetDate)
startDate := targetDate.AddDate(0, 0, -dailyretentionmod.MaxRetentionDays)
accumMap := make(map[retentionKey]*retentionAccum)
// 1. 在 Mongo 中按 (regDay, adGroup) 聚合，避免把当天明细全部拉回内存
stats, err := findUserAdStatSummaryByDate(ctx, targetDate, startDate)
if err != nil {
return fmt.Errorf("query UserAdStat failed: %w", err)
}
for _, stat := range stats {
key := retentionKey{
RegDay: common.NormalizeDate(stat.ID.RegDay),
AdGroup: stat.ID.AdGroup,
}
if accumMap[key] == nil {
accumMap[key] = &retentionAccum{}
}
accumMap[key].Retained = stat.Retained
accumMap[key].AdClick = stat.AdClick
accumMap[key].TotalClick = stat.TotalClick
accumMap[key].PayCount = stat.PayCount
accumMap[key].PayTotal = stat.PayTotal
}
log.Info("calc retention", log.Any("stats.total", len(stats)), log.Any("stats", stats))
// 2. 批量写入 DailyRetention
now := time.Now()
writes := make([]mongo.WriteModel, 0, len(accumMap))
for key, acc := range accumMap {
// 计算是注册后第几天
dayN := int(targetDate.Sub(key.RegDay).Hours() / 24)
if dayN < 0 || dayN > dailyretentionmod.MaxRetentionDays {
log.Info("calc retention check dayN", log.Any("dayN", dayN))
continue // 超出范围，跳过
}
retained := acc.Retained
// 获取已有记录（用于计算留存率和累计值）
existingItem, _ := getDailyRetention(ctx, key.RegDay, key.AdGroup)
cohortSize := retained
if dayN > 0 && existingItem != nil {
cohortSize = existingItem.NewUsers
}
rate := calculateRetentionRate(retained, cohortSize)
// 计算累计值：前一天累计 + 当天值
prevAdClick, prevTotalClick, prevPayCount, prevPayTotal := getPrevAccum(existingItem, dayN)
adClickAcc := prevAdClick + acc.AdClick
totalClickAcc := prevTotalClick + acc.TotalClick
payCountAcc := prevPayCount + acc.PayCount
payTotalAcc := prevPayTotal + acc.PayTotal
writes = append(writes, buildDailyRetentionUpsertModel(
key.RegDay, key.AdGroup, dayN, retained, rate,
adClickAcc, totalClickAcc, payCountAcc, payTotalAcc, now,
))
}
log.Info("calc retention ", log.Any("writes", writes))
if err := dailyretentionmod.BulkWrite(writes, options.BulkWrite().SetOrdered(false)); err != nil {
return fmt.Errorf("bulk upsert daily retention failed: %w", err)
}
return nil
}
// getDailyRetention 获取某个 (RegDay, AdGroup) 的 DailyRetention 记录
func getDailyRetention(ctx context.Context, regDay time.Time, adGroup commod.AdGroup) (*dailyretentionmod.DailyRetention, error) {
filter := bson.M{
"date": regDay,
"adGroup": adGroup,
}
return dailyretentionmod.QueryOne(filter)
}
// buildDailyRetentionUpsertModel 构造留存数据 upsert 模型
func buildDailyRetentionUpsertModel(regDay time.Time, adGroup commod.AdGroup, dayN int, retained int64, rate float64, adClick, totalClick, payCount, payTotal int64, now time.Time) mongo.WriteModel {
filter, update := buildDailyRetentionUpsertData(regDay, adGroup, dayN, retained, rate, adClick, totalClick, payCount, payTotal, now)
return mongo.NewUpdateOneModel().
SetFilter(filter).
SetUpdate(update).
SetUpsert(true)
}
func buildDailyRetentionUpsertData(regDay time.Time, adGroup commod.AdGroup, dayN int, retained int64, rate float64, adClick, totalClick, payCount, payTotal int64, now time.Time) (bson.M, bson.M) {
regDay = common.NormalizeDate(regDay)
filter := bson.M{
"date": regDay,
"adGroup": adGroup,
}
// 使用点号表示法更新 map 中的特定 key
retainedKey := fmt.Sprintf("%s.%d", "userRetained", dayN)
ratesKey := fmt.Sprintf("%s.%d", "userRates", dayN)
adClickKey := fmt.Sprintf("%s.%d", "adClickAcc", dayN)
totalClickKey := fmt.Sprintf("%s.%d", "totalClickAcc", dayN)
payCountKey := fmt.Sprintf("%s.%d", "payCountAcc", dayN)
payTotalKey := fmt.Sprintf("%s.%d", "payTotalAcc", dayN)
update := bson.M{
"$setOnInsert": bson.M{
"date": regDay,
"adGroup": adGroup,
"createdAt": now,
},
"$set": bson.M{
"updatedAt": now,
retainedKey: retained,
ratesKey: rate,
adClickKey: adClick,
totalClickKey: totalClick,
payCountKey: payCount,
payTotalKey: payTotal,
},
}
// 如果是 day0，同时更新 NewUsers
if dayN == 0 {
update["$set"].(bson.M)["newUsers"] = retained
}
return filter, update
}
func calculateRetentionRate(retained, cohortSize int64) float64 {
if cohortSize <= 0 {
return 0
}
return float64(retained) / float64(cohortSize)
}
// getPrevAccum 获取前一天的累计值
func getPrevAccum(item *dailyretentionmod.DailyRetention, dayN int) (adClick, totalClick, payCount, payTotal int64) {
if item == nil || dayN <= 0 {
return 0, 0, 0, 0
}
prevDay := dayN - 1
if item.AdClickAcc != nil {
adClick = item.AdClickAcc[prevDay]
}
if item.TotalClickAcc != nil {
totalClick = item.TotalClickAcc[prevDay]
}
if item.PayCountAcc != nil {
payCount = item.PayCountAcc[prevDay]
}
if item.PayTotalAcc != nil {
payTotal = item.PayTotalAcc[prevDay]
}
return
}
// findDailyRetentionByDateRange 查询指定日期范围内的 DailyRetention 记录
func findDailyRetentionByDateRange(ctx context.Context, startDate, endDate time.Time) ([]*dailyretentionmod.DailyRetention, error) {
filter := bson.M{
"date": bson.M{
"$gte": startDate,
"$lte": endDate,
},
}
return dailyretentionmod.QueryAllList(filter)
}
// findUserAdStatSummaryByDate 查询指定日期在留存窗口内的聚合数据
func findUserAdStatSummaryByDate(ctx context.Context, date, startDate time.Time) ([]*useradverstatmod.DailyRetentionSummary, error) {
local, _ := time.LoadLocation("Asia/Shanghai")
date = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, local)
startDate = common.NormalizeDate(startDate)
return useradverstatmod.AggregateDailyRetention(date, startDate, date)
}
skd/job/dailyAdverCalc/job_test.go
+105
View File
@@ -0,0 +1,105 @@
package dailyAdverCalc
import (
"91porn-server/common"
"91porn-server/models/commod"
"91porn-server/models/s/dailyretentionmod"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
)
func TestCalculateRetentionRate(t *testing.T) {
t.Run("zero cohort size returns zero", func(t *testing.T) {
if got := calculateRetentionRate(10, 0); got != 0 {
t.Fatalf("expected zero rate, got %v", got)
}
})
t.Run("uses retained divided by cohort size", func(t *testing.T) {
got := calculateRetentionRate(25, 100)
want := 0.25
if got != want {
t.Fatalf("expected %v, got %v", want, got)
}
})
}
func TestGetPrevAccum(t *testing.T) {
t.Run("nil item or day zero returns zeros", func(t *testing.T) {
adClick, totalClick, payCount, payTotal := getPrevAccum(nil, 0)
if adClick != 0 || totalClick != 0 || payCount != 0 || payTotal != 0 {
t.Fatalf("expected all zero values, got %d %d %d %d", adClick, totalClick, payCount, payTotal)
}
})
t.Run("reads previous day accumulators", func(t *testing.T) {
item := &dailyretentionmod.DailyRetention{
AdClickAcc: map[int]int64{1: 3, 2: 8},
TotalClickAcc: map[int]int64{1: 5, 2: 13},
PayCountAcc: map[int]int64{1: 1, 2: 2},
PayTotalAcc: map[int]int64{1: 100, 2: 300},
}
adClick, totalClick, payCount, payTotal := getPrevAccum(item, 3)
if adClick != 8 || totalClick != 13 || payCount != 2 || payTotal != 300 {
t.Fatalf("unexpected previous accum values: %d %d %d %d", adClick, totalClick, payCount, payTotal)
}
})
}
func TestBuildDailyRetentionUpsertData(t *testing.T) {
loc, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
t.Fatalf("load location failed: %v", err)
}
regDay := time.Date(2026, 3, 18, 14, 25, 0, 0, loc)
now := time.Date(2026, 3, 19, 9, 30, 0, 0, loc)
t.Run("day zero writes new users and normalizes date", func(t *testing.T) {
filter, update := buildDailyRetentionUpsertData(regDay, commod.AdGroupA, 0, 12, 1, 4, 8, 2, 600, now)
wantDate := common.NormalizeDate(regDay)
if got := filter["date"].(time.Time); !got.Equal(wantDate) {
t.Fatalf("expected normalized date %v, got %v", wantDate, got)
}
if got := filter["adGroup"]; got != commod.AdGroupA {
t.Fatalf("expected ad group %v, got %v", commod.AdGroupA, got)
}
setOnInsert := update["$setOnInsert"].(bson.M)
if got := setOnInsert["date"].(time.Time); !got.Equal(wantDate) {
t.Fatalf("expected setOnInsert date %v, got %v", wantDate, got)
}
if got := setOnInsert["createdAt"].(time.Time); !got.Equal(now) {
t.Fatalf("expected createdAt %v, got %v", now, got)
}
setFields := update["$set"].(bson.M)
if got := setFields["userRetained.0"]; got != int64(12) {
t.Fatalf("expected userRetained.0 to be 12, got %v", got)
}
if got := setFields["userRates.0"]; got != float64(1) {
t.Fatalf("expected userRates.0 to be 1, got %v", got)
}
if got := setFields["newUsers"]; got != int64(12) {
t.Fatalf("expected newUsers to be 12, got %v", got)
}
})
t.Run("non day zero does not overwrite new users", func(t *testing.T) {
_, update := buildDailyRetentionUpsertData(regDay, commod.AdGroupB, 2, 6, 0.3, 9, 11, 3, 800, now)
setFields := update["$set"].(bson.M)
if got := setFields["userRetained.2"]; got != int64(6) {
t.Fatalf("expected userRetained.2 to be 6, got %v", got)
}
if got := setFields["userRates.2"]; got != 0.3 {
t.Fatalf("expected userRates.2 to be 0.3, got %v", got)
}
if _, ok := setFields["newUsers"]; ok {
t.Fatalf("did not expect newUsers on non-day-zero update")
}
})
}
skd/job/dataclean/dataclean.go
+29
View File
@@ -0,0 +1,29 @@
package dataclean
import (
"time"
"91porn-server/common/timeutil"
)
func RunSync() {
today := timeutil.BeginningOfDay(time.Now())
// log库清理
go clearVisitLog(today, 7)
go clearPlayLog(today)
go clearSearchLog(today)
go clearLoginTime(today)
go clearUserRegisterLog(today)
// stat库清理
go clearKeywordStat(today)
go clearNewsHot(today)
go clearKwRankStat(today)
go clearVideoStat(today)
// 主库清理
go clearNoticeFmt(today)
go clearUserAct(today)
go clearGuestUser(today)
}
skd/job/dataclean/method.go
+94
View File
@@ -0,0 +1,94 @@
package dataclean
import (
"91porn-server/models/v/usermod"
"time"
"91porn-server/common/log"
"91porn-server/models/l/loginlgmod"
"91porn-server/models/l/playlgmod"
"91porn-server/models/l/registermod"
"91porn-server/models/l/searchlogmod"
"91porn-server/models/l/visitlogmod"
"91porn-server/models/s/hotstatmod"
"91porn-server/models/s/kwrankmod"
"91porn-server/models/s/kwstatmod"
"91porn-server/models/s/statvidmod"
"91porn-server/models/v/noticefmtmod"
"91porn-server/models/v/useractmod"
)
func clearVisitLog(today time.Time, day int) {
count, err := visitlogmod.DataClean(today.AddDate(0, 0, -day))
if err != nil {
log.Error("skd dataclean clearVisitLog error", log.E(err))
return
}
log.Info("skd dataclean clearVisitLog cleaned", log.Any("count", count))
}
func clearPlayLog(today time.Time) {
if err := playlgmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -3)); err != nil {
log.Error("skd dataclean clearPlayLog error", log.E(err))
}
}
func clearSearchLog(today time.Time) {
if err := searchlogmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -3)); err != nil {
log.Error("skd dataclean clearSearchLog error", log.E(err))
}
}
func clearKeywordStat(today time.Time) {
if err := kwstatmod.DeleteBeforeSumDate(nil, today.AddDate(0, 0, -30)); err != nil {
log.Error("skd dataclean clearKeywordStat error", log.E(err))
}
}
func clearNewsHot(today time.Time) {
if err := hotstatmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -32)); err != nil {
log.Error("skd dataclean clearNewsHot error", log.E(err))
}
}
func clearLoginTime(today time.Time) {
if err := loginlgmod.DeleteBeforeLoginTime(nil, today.AddDate(0, 0, -5)); err != nil {
log.Error("skd dataclean clearLoginTime error", log.E(err))
}
}
func clearUserRegisterLog(today time.Time) {
if err := registermod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -7)); err != nil {
log.Error("skd dataclean clearUserRegisterLog error", log.E(err))
}
}
func clearKwRankStat(today time.Time) {
if err := kwrankmod.DeleteBeforeSumDate(nil, today.AddDate(0, 0, -30)); err != nil {
log.Error("skd dataclean clearKwRankStat error", log.E(err))
}
}
func clearVideoStat(today time.Time) {
if err := statvidmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -7)); err != nil {
log.Error("skd dataclean clearVideoStat error", log.E(err))
}
}
func clearNoticeFmt(today time.Time) {
if err := noticefmtmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -5)); err != nil {
log.Error("skd dataclean clearNoticeFmt error", log.E(err))
}
}
func clearUserAct(today time.Time) {
if err := useractmod.DeleteBeforeDailyDate(nil, today.AddDate(0, 0, -2)); err != nil {
log.Error("skd dataclean clearUserAct error", log.E(err))
}
}
func clearGuestUser(today time.Time) {
if err := usermod.DeleteGuestBeforeDate(nil, today.AddDate(0, 0, -7)); err != nil {
log.Error("skd dataclean clearGuestUser error", log.E(err))
}
}
skd/job/elastic/esaudiobook/method.go
+27
View File
@@ -0,0 +1,27 @@
package esaudiobook
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/audiobookmod"
)
// 同步用户数据
func SyncDataToES(data []audiobookmod.AudioBookBase) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for k := range data {
var tmp = data[k]
source[data[k].ID.Hex()] = tmp
}
log.Info("Elastic sync audiobook data committing...")
if err := es.Bulk(models.ESInfoAudioBookTable, source); err != nil {
log.Info("Elastic sync audiobook data commit failed.")
return err
}
log.Info("Elastic sync audiobook data committed.")
return nil
}
skd/job/elastic/esaudiobook/sync.go
+58
View File
@@ -0,0 +1,58 @@
package esaudiobook
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/audiobookmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search audiobook sync start...")
defer log.Info("Elastic Search audiobook sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncAudioBookJob, defaultDate)
if err != nil {
log.Error("Elastic sync audiobook data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now().Add(-time.Minute * 20)
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := audiobookmod.GetListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync audiobook data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []audiobookmod.AudioBookBase, syncTime time.Time) error {
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit audiobook data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncAudioBookJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit audiobook data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/eskeyword/method.go
+26
View File
@@ -0,0 +1,26 @@
package eskeyword
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/s/kwstatmod"
)
// 同步数据到ES
func SyncDataToES(data []kwstatmod.Keyword) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
source[v.ID.Hex()] = kwstatmod.ESKeywordSrc(v)
}
log.Info("Elastic sync keyword committing...")
if err := es.Bulk(models.ESStatKeywordTable, source); err != nil {
log.Info("Elastic sync keyword commit failed.")
return err
}
log.Info("Elastic sync keyword committed.")
return nil
}
skd/job/elastic/eskeyword/sync.go
+58
View File
@@ -0,0 +1,58 @@
package eskeyword
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/kwstatmod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search sync keyword start...")
defer log.Info("Elastic Search tag sync keyword end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncKeywordJob, defaultDate)
if err != nil {
log.Error("Elastic sync keyword data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := kwstatmod.GetKeywordsByTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync keyword data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []kwstatmod.Keyword, syncTime time.Time) error {
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit keyword data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncKeywordJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit keyword data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/esmedia/method.go
+65
View File
@@ -0,0 +1,65 @@
package esmedia
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/mediamod"
)
// SyncDataToES 同步数据到ES
func SyncDataToES(data []*mediamod.Media) error {
if len(data) == 0 {
return nil
}
source := buildSource(data)
log.Info("Elastic sync ACG media data committing...")
if err := es.BulkChecked(models.ESMediaTable, source); err != nil {
log.Info("Elastic sync ACG media data commit failed.")
return err
}
log.Info("Elastic sync ACG media data committed.")
return nil
}
func buildSource(data []*mediamod.Media) elastic.M {
var source = elastic.M{}
for _, v := range data {
source[v.ID.Hex()] = mediamod.ESMedia{
ID: v.ID,
Title: v.Title,
HorizontalCover: v.HorizontalCover,
VerticalCover: v.VerticalCover,
Tags: v.Tags,
Summary: v.Summary,
TotalEpisode: v.TotalEpisode,
UpdateStatus: v.UpdateStatus,
MediaType: v.MediaType,
Kind: v.Kind,
PermissionIconHide: v.PermissionIconHide,
Permission: v.Permission,
Price: v.Price,
Direction: v.Direction,
FreeTime: v.FreeTime,
CountLike: v.CountLike,
CountBrowse: v.CountBrowse,
CountCollect: v.CountCollect,
MID: v.MID,
ModuleName: v.ModuleName,
SectionName: v.SectionName,
SID: v.SID,
SectionSort: v.SectionSort,
Choice: v.Choice,
ChoiceSort: v.ChoiceSort,
Number: v.Number,
SortCode: v.SortCode,
Status: v.Status,
IsDelete: v.IsDelete,
UpdatedAct: v.UpdatedAct,
CreatedAt: v.CreatedAt,
UpdateTime: v.UpdateTime,
}
}
return source
}
skd/job/elastic/esmedia/method_test.go
+25
View File
@@ -0,0 +1,25 @@
package esmedia
import (
"testing"
"91porn-server/models/v/mediamod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func TestBuildSourceIncludesCountCollect(t *testing.T) {
id := primitive.NewObjectID()
source := buildSource([]*mediamod.Media{{
ID: id,
CountCollect: 12,
}})
media, ok := source[id.Hex()].(mediamod.ESMedia)
if !ok {
t.Fatalf("source type = %T, want mediamod.ESMedia", source[id.Hex()])
}
if media.CountCollect != 12 {
t.Fatalf("countCollect = %d, want 12", media.CountCollect)
}
}
skd/job/elastic/esmedia/sync.go
+80
View File
@@ -0,0 +1,80 @@
package esmedia
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/mediamod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson/primitive"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search ACG media sync start...")
defer log.Info("Elastic Search ACG media sync end.")
// MongoDB 日期精度为毫秒；本轮开始后的修改必须留给下一轮再次同步。
startedAt := time.Now().Truncate(time.Millisecond)
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncMediaJob, defaultDate)
if err != nil {
log.Error("Elastic sync ACG media get last record time failed", log.E(err))
return
}
if !startedAt.After(lastSyncAt) {
return
}
// 固定 ID 上界，避免持续新增数据使本轮无法结束。
maxID, err := mediamod.GetLatestIDForESSync()
if err != nil {
log.Error("Elastic sync ACG media get latest ID failed", log.E(err))
return
}
sync := mediaSync{
read: mediamod.GetListForESSync,
write: SyncDataToES,
checkpoint: func(at time.Time) error {
return statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncMediaJob, at)
},
}
if err = sync.run(lastSyncAt, startedAt, maxID); err != nil {
log.Error("Elastic sync ACG media failed; checkpoint not confirmed", log.E(err))
}
}
type mediaSync struct {
read func(time.Time, primitive.ObjectID, primitive.ObjectID, int) ([]*mediamod.Media, error)
write func([]*mediamod.Media) error
checkpoint func(time.Time) error
}
func (s mediaSync) run(since, startedAt time.Time, maxID primitive.ObjectID) error {
const size = 1000
var after primitive.ObjectID
for !maxID.IsZero() {
data, err := s.read(since, after, maxID, size)
if err != nil {
return fmt.Errorf("read media batch: %w", err)
}
if len(data) == 0 {
break
}
if err = s.write(data); err != nil {
return fmt.Errorf("write media batch: %w", err)
}
after = data[len(data)-1].ID
if len(data) < size || after == maxID {
break
}
}
// 所有批次及每条 ES 写入均成功后才推进；失败时下一轮从原进度幂等重试。
if err := s.checkpoint(startedAt); err != nil {
return fmt.Errorf("save media checkpoint: %w", err)
}
return nil
}
skd/job/elastic/essearchlog/method.go
+26
View File
@@ -0,0 +1,26 @@
package essearchlog
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/l/searchlogmod"
)
// 同步数据到ES
func SyncDataToES(data []searchlogmod.SearchLog) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
source[v.ID.Hex()] = searchlogmod.ESSearchLogSource(v)
}
log.Info("Elastic sync search log data committing...")
if err := es.Bulk(models.ESLogSearchLogTable, source); err != nil {
log.Info("Elastic sync search log data commit failed.")
return err
}
log.Info("Elastic sync search log data committed.")
return nil
}
skd/job/elastic/essearchlog/sync.go
+58
View File
@@ -0,0 +1,58 @@
package essearchlog
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/l/searchlogmod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search sync search log start...")
defer log.Info("Elastic Search tag sync search log end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncSearchLogJob, defaultDate)
if err != nil {
log.Error("Elastic sync search log data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := searchlogmod.GetSearchLogByTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync search log data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []searchlogmod.SearchLog, syncTime time.Time) error {
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit search log data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncSearchLogJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit search log data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/estag/method.go
+44
View File
@@ -0,0 +1,44 @@
package estag
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/tagmod"
)
// 同步数据到ES
func SyncDataToES(data []tagmod.Tag) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
var tmp = tagmod.ESTag{
ID: v.ID,
TagName: v.TagName,
SortCode: v.SortCode,
VPlayCount: v.VPlayCount,
TPlayCount: v.TPlayCount,
FakePlayCount: v.FakePlayCount,
PlayRating: v.PlayRating,
CollCount: v.CollCount,
LikeCount: v.LikeCount,
FakeCollCount: v.FakeCollCount,
VideoCount: v.VideoCount,
IsActive: v.IsActive,
IsWonder: v.IsWonder,
WonderSortCode: v.WonderSortCode,
CreatedAt: v.CreatedAt,
UpdatedAt: v.UpdatedAt,
}
source[v.ID.Hex()] = tmp
}
log.Info("Elastic sync tag data committing...")
if err := es.Bulk(models.ESInfoTagTable, source); err != nil {
log.Info("Elastic sync tag data commit failed.")
return err
}
log.Info("Elastic sync tag data committed.")
return nil
}
skd/job/elastic/estag/sync.go
+61
View File
@@ -0,0 +1,61 @@
package estag
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/tagmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search tag sync start...")
defer log.Info("Elastic Search tag sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncTagJob, defaultDate)
if err != nil {
log.Error("Elastic sync tag data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := tagmod.GetTagListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync tag data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(data []tagmod.Tag, syncTime time.Time) error {
if len(data) <= 0 {
return nil
}
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit tag data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncTagJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit tag data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/esuser/method.go
+38
View File
@@ -0,0 +1,38 @@
package esuser
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/usermod"
)
// 同步用户数据
func SyncDataToES(data []usermod.User) error {
if len(data) == 0 {
return nil
}
var source = elastic.M{}
for _, v := range data {
var tmp = usermod.ESUser{
ID: v.ID,
UID: v.UID,
Gender: v.Gender,
Name: v.Name,
Summary: v.Summary,
Region: v.Region,
CreatedAt: v.CreatedAt,
UpdatedAt: v.UpdatedAt,
HasLocked: v.HasLocked,
HasBanned: v.HasBanned,
}
source[v.ID.Hex()] = tmp
}
log.Info("Elastic sync user data committing...")
if err := es.Bulk(models.ESInfoUserTable, source); err != nil {
log.Info("Elastic sync user data commit failed.")
return err
}
log.Info("Elastic sync user data committed.")
return nil
}
skd/job/elastic/esuser/sync.go
+72
View File
@@ -0,0 +1,72 @@
package esuser
import (
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/usermod"
"91porn-server/skd/skdg"
"fmt"
"time"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search user sync start...")
defer log.Info("Elastic Search user sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncUserJob, defaultDate)
if err != nil {
log.Error("Elastic sync user data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []usermod.User
for hasNext {
data, hasNext, err = usermod.GetUserListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Elastic sync user data GetUserListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToES(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync user data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []usermod.User, syncTime time.Time) error {
err := SyncDataToES(data)
if err != nil {
return fmt.Errorf("Elastic sync commit user data SyncDataToES failed!, err:%+v\n", err)
}
err = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncUserJob, syncTime)
if err != nil {
return fmt.Errorf("Elastic sync commit user data UpsertOneTrans failed!, err:%+v\n", err)
}
return err
}
skd/job/elastic/esvideo/method.go
+108
View File
@@ -0,0 +1,108 @@
package esvideo
import (
"math"
"time"
"91porn-server/common/elastic"
"91porn-server/common/localcache"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/tagmod"
"91porn-server/models/v/vidmod"
"91porn-server/models/v/vidpopmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const vidPopCfgKey = "vidPopCfgKey"
// SyncDataToES 同步数据到ES
func SyncDataToES(data []*vidmod.VideoModel) error {
if len(data) == 0 {
return nil
}
// 获取标签名字
var tagsID []primitive.ObjectID
for _, v := range data {
tagsID = append(tagsID, v.Tags...)
}
tags, err := tagmod.FindTagsByIDS(tagsID)
if err != nil {
return err
}
var tagsNameMap = make(map[primitive.ObjectID]string)
for _, v := range tags {
tagsNameMap[v.ID] = v.TagName
}
var source = elastic.M{}
for _, v := range data {
var tagsName []string
for _, v1 := range v.Tags {
tagsName = append(tagsName, tagsNameMap[v1])
}
source[v.ID.Hex()] = vidmod.ESVideo{
ID: v.ID,
PublisherID: v.PublisherID,
Title: v.Title,
NewsType: v.NewsType,
Tags: v.Tags,
TagsName: tagsName,
Filename: v.Filename,
PlayCount: v.PlayCount,
EffectivePlayCount: v.EffectivePlayCount,
PurchaseCount: v.PurchaseCount,
LikeCount: v.LikeCount,
CommentCount: v.CommentCount,
CollectCount: v.CollectCount,
ShareCount: v.ShareCount,
FakeLikeCount: v.FakeLikeCount,
FakeShareCount: v.FakeShareCount,
FakePlayCount: v.FakePlayCount,
Status: v.Status,
Location: v.Location,
CreatedAt: v.CreatedAt,
UpdatedAt: v.UpdatedAt,
ReviewedAt: v.ReviewAt,
Hot: calcHot(v),
PlayTime: v.PlayTime,
Coins: v.Coins,
}
}
log.Info("Elastic sync video data committing...")
if err = es.Bulk(models.ESInfoVideoTable, source); err != nil {
log.Info("Elastic sync video data commit failed.")
return err
}
log.Info("Elastic sync video data committed.")
return nil
}
func calcHot(video *vidmod.VideoModel) float64 {
if video.Status == vidmod.CheckPass || video.Status == vidmod.Free {
cfg, err := getVideoPopularityConfig()
if err != nil {
log.Error("Elastic sync video getVideoPopularityConfig", log.E(err))
return 0
}
qualityScore := float64(video.PlayCount + cfg.EffectivePlayCountMultiplier*video.EffectivePlayCount +
cfg.LikeCountMultiplier*video.LikeCount)
hot := (qualityScore + float64(cfg.InitialPopularity)) / math.Pow(1+time.Since(video.ReviewAt).Hours(),
cfg.ReviewTimePower)
return hot
}
return 0
}
func getVideoPopularityConfig() (*vidpopmod.VideoPopularityConfig, error) {
cfg, ok := localcache.C.Get(vidPopCfgKey)
if ok {
return cfg.(*vidpopmod.VideoPopularityConfig), nil
}
res, err := vidpopmod.FindOne()
if err != nil {
return nil, err
}
localcache.C.Set(vidPopCfgKey, &res, 5*time.Minute)
return &res, nil
}
skd/job/elastic/esvideo/sync.go
+74
View File
@@ -0,0 +1,74 @@
package esvideo
import (
"fmt"
"time"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
)
var es *elastic.Client
func RunSync(defaultDate time.Time) {
es = skdg.VideoES
log.Info("Elastic Search video sync start...")
defer log.Info("Elastic Search video sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncVideoJob, defaultDate)
if err != nil {
log.Error("Elastic sync video data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []*vidmod.VideoModel
for hasNext {
data, hasNext, err = vidmod.GetVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Elastic sync video data GetVideoListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToES(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Elastic sync video data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
if len(data) <= 0 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime)
return nil
}
if err := SyncDataToES(data); err != nil {
return fmt.Errorf("Elastic sync commit video data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime); err != nil {
return fmt.Errorf("Elastic sync commit video data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/elastic/rdvideo/method.go
+27
View File
@@ -0,0 +1,27 @@
package rdvideo
import (
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/models/v/vidmod"
)
// SyncDataToRedis 同步数据到redis
func SyncDataToRedis(data []*vidmod.VideoModel) error {
if len(data) == 0 {
return nil
}
for _, v := range data {
if v.NewsType != vidmod.SHORT || (v.Status != vidmod.CheckPass && v.Status != vidmod.Free) {
// 去除
_, _ = redis.Handler.SRem(redisconst.ShortVideosKey, v.ID.Hex())
} else {
_, _ = redis.Handler.SAdd(redisconst.ShortVideosKey, v.ID.Hex())
}
}
log.Info("Redis sync video data completed.")
return nil
}
skd/job/elastic/rdvideo/sync.go
+77
View File
@@ -0,0 +1,77 @@
package rdvideo
import (
"fmt"
"time"
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/vidmod"
)
func RunSync(defaultDate time.Time) {
log.Info("Redis short video sync start...")
defer log.Info("Redis short video sync end.")
// 如果短视频缓存存在的数据量过低，表示缓存被清理过，需要重新同步
c, err := redis.Handler.SCard(redisconst.ShortVideosKey)
if err == nil && c < 1000 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, defaultDate)
}
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.RedisSyncShortVideoJob, defaultDate)
if err != nil {
log.Error("Redis sync short video data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.OneDayScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.OneDayScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []*vidmod.VideoModel
for hasNext {
data, hasNext, err = vidmod.GetShortVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Redis sync short video data GetVideoListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToRedis(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Redis sync short video data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
if len(data) <= 0 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime)
return nil
}
if err := SyncDataToRedis(data); err != nil {
return fmt.Errorf("Redis sync commit short video data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime); err != nil {
return fmt.Errorf("Redis sync commit short video data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/hevc_transcode.go
+555
View File
@@ -0,0 +1,555 @@
package job
import (
"context"
"errors"
"fmt"
"net/url"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"91porn-server/common/constant"
"91porn-server/common/constant/redisconst"
"91porn-server/common/hevcpull"
"91porn-server/common/laosiji"
"91porn-server/common/log"
"91porn-server/models/v/hevctaskmod"
"91porn-server/models/v/moduleconfmod"
"91porn-server/models/v/modulesectionmod"
"91porn-server/models/v/modulevidmod"
"91porn-server/models/v/sysconfmod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
h265DefaultMaxRunning = 400 // 配置缺失或无效时，最多允许 400 个视频处于云端转码中
h265CheckBatchSize = 100 // 每轮最多轮询 100 个已提交的 H265 任务
h265HomeCandidateSize = 400 // 首页每个模块、每种排序最多保障 400 个视频
h265PendingTimeout = 240 * time.Hour
h265PullURLTTL = 480 * time.Hour
)
var (
h265SubmitRunning int32
h265CheckRunning int32
h265ScanRunning int32
h265MissingCredentialsLogOnce sync.Once
h265MissingAppURLLogOnce sync.Once
h265InvalidPullSecretLogOnce sync.Once
)
// h265TranscodingJobEnabled 默认只允许生产环境执行真实云转码。
// 测试环境必须通过 hevc.enableTranscode 显式开启，避免误提交云端任务。
func h265TranscodingJobEnabled() bool {
if skdg.Conf == nil {
return false
}
log.Info("h265TranscodingJobEnabled ", log.Any("skdg.Conf.Base.Env", skdg.Conf.Base.Env))
log.Info("h265TranscodingJobEnabled ", log.Any("skdg.Conf.Hevc.EnableTranscode", skdg.Conf.Hevc.EnableTranscode))
if skdg.Conf.Hevc.EnableTranscode != nil && !*skdg.Conf.Hevc.EnableTranscode {
return false
}
if skdg.Conf.Base.Env != constant.ProdEnv &&
(skdg.Conf.Hevc.EnableTranscode == nil || !*skdg.Conf.Hevc.EnableTranscode) {
return false
}
if !laosiji.Configured() {
h265MissingCredentialsLogOnce.Do(func() {
log.Warn("H265 transcoding jobs disabled: laosiji credentials are incomplete")
})
return false
}
if _, err := hevcpull.SignURL(
"https://configuration-check.invalid/source.m3u8",
skdg.Conf.Hevc.PullSecret,
time.Now().Add(h265PullURLTTL),
); err != nil {
h265InvalidPullSecretLogOnce.Do(func() {
log.Warn("H265 transcoding jobs disabled: hevc.pullSecret is missing or invalid", log.E(err))
})
return false
}
if buildH265AppTranscodeM3u8URL(
skdg.Conf.Url.AppApiUrl,
"health-check.m3u8",
skdg.Conf.Hevc.PullSecret,
time.Now().Add(h265PullURLTTL),
) == "" {
h265MissingAppURLLogOnce.Do(func() {
log.Warn("H265 transcoding jobs disabled: url.appApiUrl is missing or invalid", log.Any("skdg.Conf.Url.AppApiUrl", skdg.Conf.Url.AppApiUrl))
})
return false
}
return true
}
// SubmitH265TranscodingQueue 把本地等待队列中的任务异步提交到云转码。
func SubmitH265TranscodingQueue() {
if !h265TranscodingJobEnabled() {
return
}
if !atomic.CompareAndSwapInt32(&h265SubmitRunning, 0, 1) {
log.Info("SubmitH265TranscodingQueue skip: previous run still running")
return
}
defer atomic.StoreInt32(&h265SubmitRunning, 0)
ctx := context.Background()
maxRunning := h265MaxRunning()
pendingCount, err := vidmod.CountH265Pending()
if err != nil {
return
}
available := int64(maxRunning) - pendingCount
if available <= 0 {
return
}
domains, err := laosiji.SystemDomains(ctx)
if err != nil {
log.Warn("SubmitH265TranscodingQueue skip: laosiji domains failed", log.E(err))
return
}
videos, err := vidmod.GetQueuedH265Videos(available)
if err != nil {
return
}
for _, video := range videos {
if err = submitH265Video(ctx, domains, video, maxRunning); err != nil {
log.Warn("SubmitH265TranscodingQueue submit failed", log.Any("videoId", video.ID), log.E(err))
if errors.Is(err, laosiji.ErrTranscodeQueueFull) {
return
}
}
}
}
// CheckH265TranscodingResults 轮询云端任务并回写 h265Url。
func CheckH265TranscodingResults() {
if !h265TranscodingJobEnabled() {
return
}
if !atomic.CompareAndSwapInt32(&h265CheckRunning, 0, 1) {
log.Info("CheckH265TranscodingResults skip: previous run still running")
return
}
defer atomic.StoreInt32(&h265CheckRunning, 0)
ctx := context.Background()
domains, err := laosiji.SystemDomains(ctx)
if err != nil {
log.Warn("CheckH265TranscodingResults skip: laosiji domains failed", log.E(err))
return
}
videos, err := vidmod.GetPendingH265Videos(h265CheckBatchSize)
if err != nil || len(videos) == 0 {
return
}
now := time.Now()
for _, video := range videos {
if video == nil {
continue
}
if h265PendingTimedOut(video, now) {
timeoutMsg := fmt.Sprintf("H265 transcode pending timeout after %.0fh", h265PendingTimeout.Hours())
marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
if markErr != nil {
log.Warn("CheckH265TranscodingResults mark timeout failed", log.Any("videoId", video.ID), log.E(markErr))
} else if marked {
upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusFailed, timeoutMsg)
} else {
log.Info("CheckH265TranscodingResults ignored stale timeout", log.Any("videoId", video.ID))
}
continue
}
result, found, queryErr := queryH265Transcode(ctx, domains, video)
if queryErr != nil {
log.Warn("CheckH265TranscodingResults query failed", log.Any("videoId", video.ID), log.E(queryErr))
// 保留原 processing 审计状态。查询返回和日志已经足够诊断，
// 此处不落库可避免并发成功后被旧查询错误覆盖。
continue
}
if !found {
const notFoundMsg = "laosiji transcode task not found"
log.Warn("CheckH265TranscodingResults task not found, retry as failed", log.Any("videoId", video.ID))
marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
if markErr != nil {
log.Warn("CheckH265TranscodingResults mark not-found failed", log.Any("videoId", video.ID), log.E(markErr))
} else if marked {
// Failed 状态仍会进入等待队列，但会累计失败次数，避免云端永久查无任务时无限重提。
upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusFailed, notFoundMsg)
} else {
log.Info("CheckH265TranscodingResults ignored stale not-found result", log.Any("videoId", video.ID))
}
continue
}
applyH265Result(video, result)
}
}
func h265PendingTimedOut(video *vidmod.VideoModel, now time.Time) bool {
if video == nil ||
video.H265Status != vidmod.H265StatusPending ||
video.H265PendingAt.IsZero() ||
video.H265PendingAt.After(now) {
return false
}
return now.Sub(video.H265PendingAt) >= h265PendingTimeout
}
// ScanHomeH265TranscodingQueue 将首页各启用视频模块的重点 SP 视频加入等待队列。
func ScanHomeH265TranscodingQueue() {
if !h265TranscodingJobEnabled() {
return
}
if !atomic.CompareAndSwapInt32(&h265ScanRunning, 0, 1) {
log.Info("ScanHomeH265TranscodingQueue skip: previous run still running")
return
}
defer atomic.StoreInt32(&h265ScanRunning, 0)
moduleIDs, moduleIDHexes := h265HomeModuleIDs()
if len(moduleIDs) == 0 {
return
}
videoIDs := make([]primitive.ObjectID, 0)
seen := make(map[primitive.ObjectID]struct{})
for _, moduleIDHex := range moduleIDHexes {
for _, sortType := range []int{1, 2, 3, 7, 9} {
videos, err := vidmod.GetVideoListByCond(homeH265Filter(moduleIDHex, sortType), homeH265Options(sortType))
if err != nil {
log.Warn("ScanHomeH265TranscodingQueue video query failed",
log.Any("moduleId", moduleIDHex), log.Any("sortType", sortType), log.E(err))
continue
}
appendUniqueH265VideoIDs(&videoIDs, seen, videos)
}
}
for _, moduleID := range moduleIDs {
sections, err := modulesectionmod.GetAllBySubModuleID(moduleID)
if err != nil {
log.Warn("ScanHomeH265TranscodingQueue section query failed", log.Any("moduleId", moduleID), log.E(err))
continue
}
for _, section := range sections {
sectionVideos, err := modulevidmod.SectionVideosBySectionID(section.ID, options.Find().
SetLimit(h265HomeCandidateSize).
SetSort(bson.D{{Key: "sortCode", Value: -1}, {Key: "videoReviewedAt", Value: -1}}))
if err != nil {
log.Warn("ScanHomeH265TranscodingQueue section videos failed", log.Any("sectionId", section.ID), log.E(err))
continue
}
for _, sectionVideo := range sectionVideos {
if _, exists := seen[sectionVideo.VideoID]; exists {
continue
}
seen[sectionVideo.VideoID] = struct{}{}
videoIDs = append(videoIDs, sectionVideo.VideoID)
}
}
}
modified, err := vidmod.QueueH265Transcode(videoIDs)
if err != nil {
log.Warn("ScanHomeH265TranscodingQueue enqueue failed", log.E(err))
return
}
if modified > 0 {
log.Info("ScanHomeH265TranscodingQueue queued videos", log.Any("count", modified))
}
}
func submitH265Video(ctx context.Context, domains laosiji.SystemDomainsResp, video *vidmod.VideoModel, maxRunning int) error {
if video == nil || strings.TrimSpace(video.SourceURL) == "" {
return nil
}
claimedAt, claimed, err := vidmod.ClaimH265Pending(video.ID)
if err != nil || !claimed {
return err
}
video.H265PendingAt = claimedAt
result, err := submitH265Transcode(ctx, domains, video, maxRunning)
if err != nil {
if errors.Is(err, laosiji.ErrTranscodeQueueFull) {
marked, markErr := vidmod.MarkH265Queued(video.ID, video.SourceURL, video.H265PendingAt)
if markErr != nil {
log.Warn("submitH265Video restore queue failed", log.Any("videoId", video.ID), log.E(markErr))
} else if marked {
upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusQueued, err.Error())
}
return err
}
marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
if markErr != nil {
log.Warn("submitH265Video mark failed", log.Any("videoId", video.ID), log.E(markErr))
} else if marked {
upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusFailed, err.Error())
}
return err
}
applyH265Result(video, result)
return nil
}
func submitH265Transcode(ctx context.Context, domains laosiji.SystemDomainsResp, video *vidmod.VideoModel, maxRunning int) (laosiji.TranscodeResult, error) {
fileURL := hevcAppTranscodeM3u8URL(video.SourceURL, video.H265PendingAt)
fileID := h265CloudFileID(video)
if fileURL == "" || fileID == "" {
return laosiji.TranscodeResult{}, fmt.Errorf("empty signed app transcode m3u8 url, videoId:%s", video.ID.Hex())
}
return laosiji.SubmitH264ToH265Task(ctx, fileID, fileURL, domains, maxRunning)
}
func queryH265Transcode(ctx context.Context, domains laosiji.SystemDomainsResp, video *vidmod.VideoModel) (laosiji.TranscodeResult, bool, error) {
fileURL := hevcAppTranscodeM3u8URL(video.SourceURL, video.H265PendingAt)
fileID := h265CloudFileID(video)
if fileURL == "" || fileID == "" {
return laosiji.TranscodeResult{}, false, fmt.Errorf("empty signed app transcode m3u8 url, videoId:%s", video.ID.Hex())
}
return laosiji.QueryH264ToH265Task(ctx, fileID, fileURL, domains)
}
// hevcAppTranscodeM3u8URL 使用 claim 时间生成稳定、限时且绑定源路径的
// HMAC 拉流 URL。云端 file_id 独立计算，签名或域名变化不会影响轮询。
func hevcAppTranscodeM3u8URL(source string, claimedAt time.Time) string {
if skdg.Conf == nil {
return ""
}
if claimedAt.IsZero() {
return ""
}
return buildH265AppTranscodeM3u8URL(
skdg.Conf.Url.AppApiUrl,
source,
skdg.Conf.Hevc.PullSecret,
claimedAt.UTC().Truncate(time.Second).Add(h265PullURLTTL),
)
}
func buildH265AppTranscodeM3u8URL(baseURL, source, pullSecret string, expiresAt time.Time) string {
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
normalizedSource, sourceErr := hevcpull.NormalizeSource(source)
if baseURL == "" || sourceErr != nil {
return ""
}
parsed, err := url.Parse(baseURL)
if err != nil ||
//parsed.Scheme != "https" ||
parsed.Host == "" ||
parsed.User != nil ||
(parsed.Path != "" && parsed.Path != "/") ||
parsed.RawPath != "" ||
parsed.RawQuery != "" ||
parsed.Fragment != "" {
return ""
}
rawURL := baseURL + "/api/app/vid/transcode/m3u8/" + normalizedSource
signedURL, err := hevcpull.SignURL(rawURL, pullSecret, expiresAt)
if err != nil {
return ""
}
return signedURL
}
// h265CloudFileID is independent from the temporary signed fetch URL. It
// changes when the source or retry attempt changes, but not when App domains,
// expiry timestamps, or signing keys rotate during an in-flight attempt.
func h265CloudFileID(video *vidmod.VideoModel) string {
if video == nil || video.ID.IsZero() {
return ""
}
source, err := hevcpull.NormalizeSource(video.SourceURL)
if err != nil {
return ""
}
identity := fmt.Sprintf(
"91porn:h265:v1\x00%s\x00%s\x00%d",
video.ID.Hex(),
source,
video.H265FailCount,
)
return laosiji.TranscodeFileID(identity)
}
func applyH265Result(video *vidmod.VideoModel, result laosiji.TranscodeResult) {
h265URL := ""
taskStatus := hevctaskmod.H265TaskStatusProcessing
switch {
case result.Done:
h265URL = h265PlayableURL(video, result)
if h265URL == "" {
marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
if markErr != nil {
log.Warn("applyH265Result mark empty result failed", log.Any("videoId", video.ID), log.E(markErr))
} else if marked {
_ = hevctaskmod.UpsertWithError(video.ID, video.Title, result.FileURL, result.FileID, "", hevctaskmod.H265TaskStatusFailed, "empty H265 URL in completed task")
}
return
}
updated, err := vidmod.MarkPendingH265Success(video.ID, video.SourceURL, video.H265PendingAt, h265URL)
if err != nil {
log.Warn("applyH265Result MarkPendingH265Success failed", log.Any("videoId", video.ID), log.E(err))
return
}
if !updated {
// 当前视频可能已经被老司机回填 H.265，或 sourceURL 已经变化。
// 对仍处于 pending 且缺少地址的新源重新排队；已有成功地址时该操作自动跳过。
_, _ = vidmod.MarkH265Queued(video.ID, video.SourceURL, video.H265PendingAt)
// 不更新 hevc_task：并发轮询的另一个实例可能已经成功落库，
// 此处覆盖会把正确的 success 审计状态错误回退成 failed。
log.Info("applyH265Result ignored stale result", log.Any("videoId", video.ID))
return
}
clearH265VideoCache(video.ID)
taskStatus = hevctaskmod.H265TaskStatusSuccess
case result.Failed:
marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
if markErr != nil {
log.Warn("applyH265Result MarkH265Failed failed", log.Any("videoId", video.ID), log.E(markErr))
return
}
if !marked {
log.Info("applyH265Result ignored stale failure", log.Any("videoId", video.ID))
return
}
taskStatus = hevctaskmod.H265TaskStatusFailed
default:
// processing 结果不改变视频状态，也没有可用于跨集合原子写入的
// ModifiedCount。保留现有审计记录，避免旧轮询覆盖并发成功结果。
return
}
if err := hevctaskmod.UpsertWithError(video.ID, video.Title, result.FileURL, result.FileID, h265URL, taskStatus, result.ErrorMsg); err != nil {
log.Warn("applyH265Result upsert task failed", log.Any("videoId", video.ID), log.E(err))
}
}
// h265PlayableURL 区分云端“无需转码”和真正生成了新 H265 文件的结果。
// 无需转码时源地址本身已经可直接使用，不能把我方 App 拉流 URL 错存成老司机资源路径。
func h265PlayableURL(video *vidmod.VideoModel, result laosiji.TranscodeResult) string {
if result.NoNeed {
if video == nil {
return ""
}
return strings.TrimSpace(video.SourceURL)
}
return laosiji.MovieM3u8SourcePath(result.HevcURL)
}
func clearH265VideoCache(videoID primitive.ObjectID) {
if skdg.Redis == nil || videoID.IsZero() {
return
}
key := fmt.Sprintf(redisconst.VideoInfoKey(), videoID.Hex())
if _, err := skdg.Redis.Del(key); err != nil {
log.Warn("clearH265VideoCache failed", log.Any("videoId", videoID), log.E(err))
}
}
func upsertH265TaskWithError(video *vidmod.VideoModel, domains laosiji.SystemDomainsResp, h265URL string, status hevctaskmod.H265TaskStatus, errorMsg string) {
if video == nil {
return
}
submitURL, fileID := h265TaskSubmitInfo(video, domains)
if err := hevctaskmod.UpsertWithError(video.ID, video.Title, submitURL, fileID, h265URL, status, errorMsg); err != nil {
log.Warn("upsertH265TaskWithError failed", log.Any("videoId", video.ID), log.E(err))
}
}
func h265TaskSubmitInfo(video *vidmod.VideoModel, _ laosiji.SystemDomainsResp) (string, string) {
if video == nil {
return "", ""
}
submitURL := strings.TrimSpace(hevcAppTranscodeM3u8URL(video.SourceURL, video.H265PendingAt))
if submitURL == "" {
submitURL = strings.TrimSpace(video.SourceURL)
}
fileID := h265CloudFileID(video)
// 按运维要求持久化“可直接访问”的签名 URL，便于从 hevc_task 复制出来核验。
// 代价：库中会保存有效期内可用的签名拉流地址（含 HMAC），务必收紧 DB 访问权限。
return submitURL, fileID
}
func h265MaxRunning() int {
cfg, err := sysconfmod.GetByVCode(sysconfmod.VCodeH265MaxRunning)
if err != nil {
return h265DefaultMaxRunning
}
maxRunning, err := strconv.Atoi(cfg.Value)
if err != nil || maxRunning <= 0 {
return h265DefaultMaxRunning
}
return maxRunning
}
func h265HomeModuleIDs() ([]primitive.ObjectID, []string) {
modules, err := moduleconfmod.GetModuleConfByType(moduleconfmod.HomePage)
if err != nil {
log.Warn("h265HomeModuleIDs query homepage modules failed", log.E(err))
return nil, nil
}
moduleIDs := make([]primitive.ObjectID, 0, len(modules))
moduleIDHexes := make([]string, 0, len(modules))
for _, module := range modules {
if module.Status != 1 || module.ID.IsZero() {
continue
}
moduleIDs = append(moduleIDs, module.ID)
moduleIDHexes = append(moduleIDHexes, module.ID.Hex())
}
return moduleIDs, moduleIDHexes
}
func appendUniqueH265VideoIDs(ids *[]primitive.ObjectID, seen map[primitive.ObjectID]struct{}, videos []*vidmod.VideoModel) {
for _, video := range videos {
if video == nil {
continue
}
if _, exists := seen[video.ID]; exists {
continue
}
seen[video.ID] = struct{}{}
*ids = append(*ids, video.ID)
}
}
func homeH265Filter(moduleID string, sortType int) primitive.M {
return bson.M{
"status": vidmod.CheckPass,
"newsType": vidmod.SP,
"mId": moduleID,
"sourceURL": bson.M{"$exists": true, "$ne": ""},
"deleteAt": bson.M{"$exists": false},
}
}
func homeH265Options(sortType int) *options.FindOptions {
var sort bson.D
switch sortType {
case 2:
sort = bson.D{
{Key: "liaoBaTopSort", Value: -1},
{Key: "likeCount", Value: -1},
{Key: "reviewAt", Value: -1},
}
case 3:
sort = bson.D{{Key: "playCount", Value: -1}, {Key: "reviewAt", Value: -1}}
case 7:
sort = bson.D{{Key: "collectCount", Value: -1}, {Key: "reviewAt", Value: -1}}
case 9:
sort = bson.D{{Key: "hot", Value: -1}, {Key: "reviewAt", Value: -1}}
default:
sort = bson.D{{Key: "reviewAt", Value: -1}}
}
return options.Find().SetLimit(h265HomeCandidateSize).SetSort(sort)
}
skd/job/hevc_transcode_test.go
+330
View File
@@ -0,0 +1,330 @@
package job
import (
"net/url"
"reflect"
"strings"
"testing"
"time"
"91porn-server/common/hevcpull"
"91porn-server/common/laosiji"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
func TestBuildH265AppTranscodeM3u8URL(t *testing.T) {
const pullSecret = "test-pull-secret-strong-32-bytes!!"
now := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
expiresAt := now.Add(h265PullURLTTL)
tests := []struct {
name string
base string
source string
wantPath string
}{
{
name: "normalizes slashes",
base: " https://app.example.com/ ",
source: " /video/demo/index.m3u8 ",
wantPath: "/api/app/vid/transcode/m3u8/video/demo/index.m3u8",
},
{name: "empty base", source: "video.m3u8"},
{name: "empty source", base: "https://app.example.com"},
{name: "base without scheme", base: "app.example.com", source: "video.m3u8"},
{name: "plain http is forbidden", base: "http://app.example.com", source: "video.m3u8"},
{name: "unsupported scheme", base: "ftp://app.example.com", source: "video.m3u8"},
{name: "base with path", base: "https://app.example.com/prefix", source: "video.m3u8"},
{name: "base with query", base: "https://app.example.com?token=x", source: "video.m3u8"},
{name: "source traversal", base: "https://app.example.com", source: "video/../index.m3u8"},
{name: "absolute source", base: "https://app.example.com", source: "https://cdn.example/index.m3u8"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := buildH265AppTranscodeM3u8URL(tt.base, tt.source, pullSecret, expiresAt)
if tt.wantPath == "" {
if got != "" {
t.Fatalf("buildH265AppTranscodeM3u8URL() = %q, want empty", got)
}
return
}
parsed, err := url.Parse(got)
if err != nil {
t.Fatalf("parse signed URL: %v", err)
}
if parsed.Path != tt.wantPath {
t.Fatalf("signed URL path = %q, want %q", parsed.Path, tt.wantPath)
}
if err = hevcpull.VerifyURL(parsed, pullSecret, now); err != nil {
t.Fatalf("signed URL did not verify: %v", err)
}
})
}
}
func TestHevcAppTranscodeM3u8URLUsesStableClaimTime(t *testing.T) {
oldConf := skdg.Conf
t.Cleanup(func() {
skdg.Conf = oldConf
})
claimedAt := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
skdg.Conf = nil
if got := hevcAppTranscodeM3u8URL("video.m3u8", claimedAt); got != "" {
t.Fatalf("nil config returned %q", got)
}
skdg.Conf = &skdg.GlobalConfig{}
skdg.Conf.Url.AppApiUrl = "https://app.example.com"
skdg.Conf.Hevc.PullSecret = "test-pull-secret-strong-32-bytes!!"
first := hevcAppTranscodeM3u8URL("video.m3u8", claimedAt)
second := hevcAppTranscodeM3u8URL("video.m3u8", claimedAt)
if first == "" || first != second {
t.Fatalf("signed URL must be stable for one claim: first=%q second=%q", first, second)
}
parsed, err := url.Parse(first)
if err != nil {
t.Fatalf("parse signed URL: %v", err)
}
if !strings.HasPrefix(parsed.Path, "/api/app/vid/transcode/m3u8/") {
t.Fatalf("unexpected signed URL path: %s", parsed.Path)
}
if err = hevcpull.VerifyURL(parsed, skdg.Conf.Hevc.PullSecret, claimedAt); err != nil {
t.Fatalf("signed URL did not verify: %v", err)
}
}
func TestH265CloudFileIDIsIndependentFromSignedPullURL(t *testing.T) {
video := &vidmod.VideoModel{
ID: primitive.NewObjectID(),
SourceURL: " /sp/movie/index.m3u8 ",
H265FailCount: 1,
H265PendingAt: time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC),
}
first := h265CloudFileID(video)
if first == "" {
t.Fatal("empty cloud file ID")
}
video.H265PendingAt = video.H265PendingAt.Add(time.Hour)
video.SourceURL = "/sp/movie/index.m3u8"
if got := h265CloudFileID(video); got != first {
t.Fatalf("equivalent source or claim time changed cloud file ID: %q != %q", got, first)
}
video.SourceURL = "/sp/other/index.m3u8"
if got := h265CloudFileID(video); got == first {
t.Fatal("different source reused cloud file ID")
}
video.SourceURL = "/sp/movie/index.m3u8"
video.H265FailCount++
if got := h265CloudFileID(video); got == first {
t.Fatal("new retry attempt reused cloud file ID")
}
}
func TestH265TranscodingJobEnabledRequiresEnvironmentSwitchAndConfig(t *testing.T) {
oldConf := skdg.Conf
oldLSJConfig := laosiji.Config{
AppID: laosiji.Appid,
APIKey: laosiji.APIKey,
APIUrl: laosiji.APIUrl,
ImageYuan: laosiji.IMAGEYUAN,
NoticeURL: laosiji.NoticeURL,
}
t.Cleanup(func() {
skdg.Conf = oldConf
laosiji.InitConfig(oldLSJConfig)
})
skdg.Conf = &skdg.GlobalConfig{}
skdg.Conf.Base.Env = "prod"
skdg.Conf.Url.AppApiUrl = "https://app.example.com"
laosiji.InitConfig(laosiji.Config{})
if h265TranscodingJobEnabled() {
t.Fatal("job enabled without laosiji credentials")
}
laosiji.InitConfig(laosiji.Config{
AppID: "test-app",
APIKey: "test-key",
APIUrl: "https://laosiji.example.com",
})
skdg.Conf.Base.Env = "test"
if h265TranscodingJobEnabled() {
t.Fatal("job enabled in test without explicit switch")
}
enabled := true
skdg.Conf.Hevc.EnableTranscode = &enabled
if h265TranscodingJobEnabled() {
t.Fatal("job enabled in test without hevc.pullSecret")
}
skdg.Conf.Hevc.PullSecret = "test-pull-secret-strong-32-bytes!!"
skdg.Conf.Url.AppApiUrl = ""
if h265TranscodingJobEnabled() {
t.Fatal("job enabled without appApiUrl")
}
skdg.Conf.Url.AppApiUrl = "https://app.example.com"
if !h265TranscodingJobEnabled() {
t.Fatal("job disabled in test with explicit switch and complete configuration")
}
skdg.Conf.Base.Env = "prod"
skdg.Conf.Hevc.EnableTranscode = nil
if !h265TranscodingJobEnabled() {
t.Fatal("job disabled with complete production configuration")
}
disabled := false
skdg.Conf.Hevc.EnableTranscode = &disabled
if h265TranscodingJobEnabled() {
t.Fatal("job enabled after explicit production kill switch")
}
}
func TestH265PlayableURL(t *testing.T) {
video := &vidmod.VideoModel{SourceURL: "sp/source/index.m3u8"}
if got := h265PlayableURL(video, laosiji.TranscodeResult{
Done: true,
NoNeed: true,
HevcURL: "https://app.example.com/api/app/vid/h5/light/m3u8/sp/source/index.m3u8",
}); got != video.SourceURL {
t.Fatalf("no-need result returned %q, want original source %q", got, video.SourceURL)
}
if got := h265PlayableURL(video, laosiji.TranscodeResult{
Done: true,
HevcURL: "https://cdn.example.com/m3m/transcoded/index.m3u8",
}); got != "laosiji/m3m/transcoded/index.m3u8" {
t.Fatalf("transcoded result returned %q", got)
}
if got := h265PlayableURL(nil, laosiji.TranscodeResult{Done: true, NoNeed: true}); got != "" {
t.Fatalf("nil video no-need result returned %q", got)
}
}
func TestH265PendingTimedOut(t *testing.T) {
now := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
tests := []struct {
name string
video *vidmod.VideoModel
want bool
}{
{name: "nil video"},
{
name: "not pending",
video: &vidmod.VideoModel{
H265Status: vidmod.H265StatusQueued,
H265PendingAt: now.Add(-25 * time.Hour),
},
},
{
name: "missing claim time",
video: &vidmod.VideoModel{
H265Status: vidmod.H265StatusPending,
},
},
{
name: "still within timeout",
video: &vidmod.VideoModel{
H265Status: vidmod.H265StatusPending,
H265PendingAt: now.Add(-h265PendingTimeout + time.Second),
},
},
{
name: "at timeout",
video: &vidmod.VideoModel{
H265Status: vidmod.H265StatusPending,
H265PendingAt: now.Add(-h265PendingTimeout),
},
want: true,
},
{
name: "past timeout",
video: &vidmod.VideoModel{
H265Status: vidmod.H265StatusPending,
H265PendingAt: now.Add(-h265PendingTimeout - time.Second),
},
want: true,
},
{
name: "future timestamp",
video: &vidmod.VideoModel{
H265Status: vidmod.H265StatusPending,
H265PendingAt: now.Add(time.Second),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := h265PendingTimedOut(tt.video, now); got != tt.want {
t.Fatalf("h265PendingTimedOut() = %v, want %v", got, tt.want)
}
})
}
}
func TestHomeH265QueryMatchesAppHomepageSorts(t *testing.T) {
for _, sortType := range []int{1, 2, 3, 7, 9} {
filter := homeH265Filter("module-id", sortType)
if _, exists := filter["reviewAt"]; exists {
t.Fatalf("sort %d unexpectedly limits reviewAt: %#v", sortType, filter["reviewAt"])
}
}
tests := []struct {
sortType int
want bson.D
}{
{
sortType: 1,
want: bson.D{{Key: "reviewAt", Value: -1}},
},
{
sortType: 2,
want: bson.D{
{Key: "liaoBaTopSort", Value: -1},
{Key: "likeCount", Value: -1},
{Key: "reviewAt", Value: -1},
},
},
{
sortType: 3,
want: bson.D{
{Key: "playCount", Value: -1},
{Key: "reviewAt", Value: -1},
},
},
{
sortType: 7,
want: bson.D{
{Key: "collectCount", Value: -1},
{Key: "reviewAt", Value: -1},
},
},
{
sortType: 9,
want: bson.D{
{Key: "hot", Value: -1},
{Key: "reviewAt", Value: -1},
},
},
}
for _, tt := range tests {
got := homeH265Options(tt.sortType).Sort
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("sort %d = %#v, want %#v", tt.sortType, got, tt.want)
}
}
}
skd/job/hotvalue/video.go
+125
View File
@@ -0,0 +1,125 @@
package hotvalue
import (
"91porn-server/common/log"
"91porn-server/models/v/cmtmod"
"91porn-server/models/v/mediamod"
"91porn-server/models/v/vidmod"
"github.com/golang-module/carbon/v2"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"math"
"time"
)
type updateFn func(models []mongo.WriteModel) error
var (
actions = map[string]updateFn{
cmtmod.OTypeVideo: vidmod.Bulk,
cmtmod.OTypeCartoon: mediamod.Bulk,
}
)
// ResetHot 重置热度值
func ResetHot() {
start := carbon.NewCarbon().SubDays(30).StdTime()
end := time.Now()
writes := make([]mongo.WriteModel, 0)
for objType, fn := range actions {
skip := 0
limit := 100
for {
// 获取30天内有评论的objID
objIds, hasNext, err := cmtmod.GetObjIdsBetweenTime(objType, start, end, skip, limit)
if err != nil {
log.Error("Job ResetHot cmtmod.GetObjIdsBetweenTime fail", log.E(err))
return
}
for _, objId := range objIds {
hotData := resetObjectHot(objId, objType, start, end)
if len(hotData) == 0 {
continue
}
update := mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": objId}).SetUpdate(hotData).SetUpsert(true)
writes = append(writes, update)
}
if len(writes) > 0 {
err = fn(writes)
if err != nil {
log.Error("Job ResetHot fail", log.E(err),
log.Any("objType", objType),
log.Any("skip", skip))
}
}
writes = writes[0:0]
skip = skip + limit
if !hasNext {
break
}
}
}
}
// 热度=Z历史所有评论数 × 30% × e^（−{评论天数}×0.1）
//+Z近7天评论数 × 70%
func resetObjectHot(objId primitive.ObjectID, objType string, start, end time.Time) bson.M {
// 暂时只支持视频和ACG的评论
if objType != cmtmod.OTypeVideo && objType != cmtmod.OTypeCartoon {
return nil
}
// 获取该对性一段时间内，按天分组，每天的评论总数
list, err := cmtmod.GetCountGroupByDay(objId, start, end)
if err != nil {
log.Error("Job ResetHot resetObjectHot cmtmod.GetCountGroupByDay fail", log.E(err))
return nil
}
var historyHot float64
var sevenDayCount float64
// 计算出该对象的热度值
for _, v := range list {
// 获取跟今天相差的天数
day := float64(carbon.Now().DiffAbsInDays(carbon.ParseByFormat(v.Time, "Y-m-d")))
// 计算历史评论热度
hot := calculateCommentScore(v.Count, day)
historyHot = historyHot + hot
// 累加近七天的总数
if day <= 7 {
sevenDayCount = sevenDayCount + float64(v.Count)
}
}
currentHot := historyHot + sevenDayCount*0.7
log.Info("Job ResetHot resetObjectHot result", log.Any("hot", currentHot))
data := bson.M{"hot": currentHot}
if objType == cmtmod.OTypeVideo {
data["updatedAt"] = time.Now()
} else {
data["updateTime"] = time.Now()
}
return bson.M{"$set": data}
}
func calculateCommentScore(totalComments int64, daysSinceComment float64) float64 {
// Ensure we don't have negative days
if daysSinceComment < 0 {
daysSinceComment = 0
}
// Calculate the decay factor: e^(-days * 0.1)
decayFactor := math.Exp(-daysSinceComment * 0.1)
// Calculate the final score: totalComments * 0.3 * decayFactor
score := float64(totalComments) * 0.3 * decayFactor
return score
}
skd/job/im_ad_notify.go
+150
View File
@@ -0,0 +1,150 @@
package job
import (
"fmt"
"strconv"
"time"
"91porn-server/common/enum/imad"
"91porn-server/common/log"
"91porn-server/common/stderr"
"91porn-server/models/cache/sysconfdata"
"91porn-server/models/v/sysconfmod"
"91porn-server/skd/skdg"
"91porn-server/services/srv_im"
)
const (
imAdNotifyLockKey = "skd:im_ad_notify:online:lock"
imAdNotifyLastRunKey = "skd:im_ad_notify:online:last_run"
imAdNotifyLockTTL = 10 * time.Minute
)
type imAdNotifyConfig struct {
Enable bool
IntervalMinutes int64
Positions []string
MaxUsers int
}
// IMAdNotifyOnline 按 sysconf.imConfig 定时向当前 IM 在线用户透传广告刷新通知。
func IMAdNotifyOnline() {
if err := imAdNotifyOnline(); err != nil {
log.Error("IMAdNotifyOnline failed", log.E(err))
}
}
func imAdNotifyOnline() error {
conf := loadIMAdNotifyConfig()
now := time.Now()
lastRunAt, err := loadIMAdNotifyLastRunAt()
if err != nil {
return err
}
if !shouldRunIMAdNotify(conf, lastRunAt, now) {
return nil
}
acquired, err := skdg.Redis.Setnx_NewOK(imAdNotifyLockKey, 1, imAdNotifyLockTTL)
if err != nil {
return err
}
if !acquired {
log.Info("IMAdNotifyOnline skipped: lock held")
return nil
}
defer func() {
if _, err := skdg.Redis.Del(imAdNotifyLockKey); err != nil {
log.Warn("IMAdNotifyOnline release lock failed", log.E(err))
}
}()
lastRunAt, err = loadIMAdNotifyLastRunAt()
if err != nil {
return err
}
if !shouldRunIMAdNotify(conf, lastRunAt, now) {
return nil
}
resp, code, msg := srv_im.SendAdNotify(srv_im.AdNotifyReq{
Target: "online",
Positions: conf.Positions,
MaxUsers: conf.MaxUsers,
})
if code != stderr.Success {
return fmt.Errorf("im ad notify failed: code=%d msg=%s", code, msg)
}
log.Info("IMAdNotifyOnline done", log.Any("resp", resp))
return saveIMAdNotifyLastRunAt(now)
}
func loadIMAdNotifyConfig() imAdNotifyConfig {
conf := imAdNotifyConfig{IntervalMinutes: 60, MaxUsers: 1000}
all, err := sysconfdata.GetAllFromCache()
if err != nil {
log.Warn("load im sysconf failed", log.E(err))
return conf
}
conf.Enable = all.GetBool(sysconfmod.VCodeIMAdEnable)
if v := all.GetInt(sysconfmod.VCodeIMAdIntervalMinutes); v > 0 {
conf.IntervalMinutes = v
}
if v := all.GetInt(sysconfmod.VCodeIMAdMaxUsers); v > 0 {
conf.MaxUsers = int(v)
}
// 每个广告位独立开关，开了就加进去
conf.Positions = collectEnabledIMAdPositions(all)
return conf
}
// collectEnabledIMAdPositions 把启用了的广告位 code 收集成列表
func collectEnabledIMAdPositions(all sysconfmod.ConfMap) []string {
positions := make([]string, 0, 4)
if all.GetBool(sysconfmod.VCodeIMAdPosChatNotificationBar) {
positions = append(positions, imad.ChatNotificationBar)
}
if all.GetBool(sysconfmod.VCodeIMAdPosChatFloatingGifBall) {
positions = append(positions, imad.ChatFloatingGifBall)
}
if all.GetBool(sysconfmod.VCodeIMAdPosListFeedNative) {
positions = append(positions, imad.ListFeedNative)
}
if all.GetBool(sysconfmod.VCodeIMAdPosListBanner) {
positions = append(positions, imad.ListBanner)
}
return positions
}
func shouldRunIMAdNotify(conf imAdNotifyConfig, lastRunAt time.Time, now time.Time) bool {
if !conf.Enable || len(conf.Positions) == 0 {
return false
}
if lastRunAt.IsZero() {
return true
}
return !now.Before(lastRunAt.Add(normalizeIMAdNotifyInterval(conf.IntervalMinutes)))
}
func normalizeIMAdNotifyInterval(intervalMinutes int64) time.Duration {
if intervalMinutes <= 0 {
intervalMinutes = 60
}
return time.Duration(intervalMinutes) * time.Minute
}
func loadIMAdNotifyLastRunAt() (time.Time, error) {
val, err := skdg.Redis.Get(imAdNotifyLastRunKey)
if err != nil || val == nil || *val == "" {
return time.Time{}, err
}
unixSec, err := strconv.ParseInt(*val, 10, 64)
if err != nil || unixSec <= 0 {
return time.Time{}, nil
}
return time.Unix(unixSec, 0), nil
}
func saveIMAdNotifyLastRunAt(now time.Time) error {
return skdg.Redis.Set(imAdNotifyLastRunKey, strconv.FormatInt(now.Unix(), 10), 0)
}
skd/job/job.go
+162
View File
@@ -0,0 +1,162 @@
package job
import (
"91porn-server/common"
"91porn-server/common/aiService"
"91porn-server/common/log"
mediaService "91porn-server/common/media"
"91porn-server/common/mediaCenter"
"91porn-server/models/commod"
"91porn-server/skd/job/checkin"
"91porn-server/skd/job/dailyAdverCalc"
"91porn-server/skd/job/dataclean"
"91porn-server/skd/job/hotvalue"
"91porn-server/skd/job/shortrecommend"
"91porn-server/skd/job/user"
"91porn-server/skd/service"
"91porn-server/skd/service/contentreviewser"
"91porn-server/skd/service/export_task"
"91porn-server/skd/skdg"
"context"
"time"
)
// start 定时任务 开启
func start() {
_, _ = statCron.AddFunc("0 0 0 * * ?", func() { //每天0点执行一次
// 重置预售卡预付权益
user.ResetLimit()
})
if skdg.Conf.ShortRecommendEnabled() {
// 定时检查队列完整性；当天健康时只做一次Redis脚本，异常时自动重建。
if _, err := statCron.AddFunc(skdg.Conf.ShortRecommendCron(), shortrecommend.Run); err != nil {
log.Error("register short recommend cron failed",
log.E(err), log.Any("cron", skdg.Conf.ShortRecommendCron()))
}
// 启动时补建当日队列；分布式锁保证多实例只执行一次。
common.Go(shortrecommend.Run)
}
_, _ = statCron.AddFunc("0 0/5 * * * ?", func() { //每5分钟执行一次统计任务
runStat()
})
_, _ = statCron.AddFunc("@every 5m", func() { //每5分钟执行一次数据同步任务
runSync()
})
_, _ = statCron.AddFunc("0 0/5 * * * ?", func() { //每5分钟执行一次sql折叠任务
runSyncSqlcollaspe()
})
// _, _ = statCron.AddFunc("0 0/5 * * * ?", func() { //每5分钟刷新一下排行榜
// runRankStat()
// })
// _, _ = statCron.AddFunc("0 0/1 * * * ?", func() { //每1分钟执行一次统计任务
// runPromoteStat()
// })
// _, _ = statCron.AddFunc("0 0 0/1 * * ?", func() { //每1个小时增加内部人员推广人数
// runInnerFakePromote()
// })
// _, _ = statCron.AddFunc("0 0/2 * * * ?", func() { //每2分钟生成备用推广码
// runMakePromotionCode()
// })
// _, _ = statCron.AddFunc("0 0 0,1,3,6,7,9,10,12,15,20,21,23 * * ?", func() { //每天11，15，18，23点执行一次消息任务
// service.GoldEggLuckyRunSync()
// })
_, _ = statCron.AddFunc("0 0 4 * * ?", func() { //每天凌晨4点执行一次
dataclean.RunSync()
})
_, _ = statCron.AddFunc("0/10 * * * * ?", func() { //每10秒执行一次
service.VidTimeOnline()
})
common.Go(func() {
hotvalue.ResetHot()
})
_, _ = statCron.AddFunc("0 0 6 * * ?", func() { //每天凌晨6点执行一次
hotvalue.ResetHot()
})
// 每10秒检测需要导出的任务列表
_, _ = statCron.AddFunc("0/10 * * * * ?", func() { //每10秒执行一次
export_task.ExecExportTask()
})
// 每3秒检测有无待执行的内容审查任务（同一时刻仅 1 个任务运行）
_, _ = statCron.AddFunc("0/3 * * * * ?", func() {
contentreviewser.TryRunPendingTask()
})
_, _ = statCron.AddFunc("0 * * * * ?", func() { // 每分钟检查是否需要发送 IM 广告通知
IMAdNotifyOnline()
})
// 每 5 秒检查 web 端是否发起了 IM 用户同步请求
_, _ = statCron.AddFunc("0/5 * * * * ?", func() {
RunPendingSyncIMUsers()
})
// 同步acg订阅的动漫
_, _ = statCron.AddFunc("0 0/30 * * * ?", func() { //每30分钟执行一次
ms := mediaService.NewMediaSync(skdg.Redis, mediaCenter.NewMediaCenterService(mediaCenter.Config{
Url: skdg.Conf.MediaResourcesCfg.UploadMediaUrl,
ProId: skdg.Conf.MediaResourcesCfg.ProId,
Secret: skdg.Conf.MediaResourcesCfg.AppSecret,
}))
ms.SyncAll("系统")
})
common.Go(func() {
err := aiService.NewAiService(
aiService.AppId(int(commod.KFK_APPID)),
aiService.Url(skdg.Conf.Url.AiServer),
aiService.Redis(skdg.Redis),
).Sync()
if err != nil {
log.Error("Sync AI Template fail")
}
})
// 定期同步ai服务里的模版
_, _ = statCron.AddFunc("0 0/10 * * * ?", func() { //每10分钟执行一次
err := aiService.NewAiService(
aiService.AppId(int(commod.KFK_APPID)),
aiService.Url(skdg.Conf.Url.AiServer),
aiService.Redis(skdg.Redis),
).Sync()
if err != nil {
log.Error("Sync AI Template fail")
}
})
_, _ = statCron.AddFunc("0/2 * * * * ?", func() { //每2秒检查签到奖励视频切片状态
checkin.VideoCheckRS()
})
_, _ = statCron.AddFunc("0 0/1 * * * ?", func() { // 每分钟提交 H265 等待队列
SubmitH265TranscodingQueue()
})
_, _ = statCron.AddFunc("0 0/1 * * * ?", func() { // 每分钟轮询 H265 转码结果
CheckH265TranscodingResults()
})
_, _ = statCron.AddFunc("0 0 0/1 * * ?", func() { // 每小时保障首页重点视频的 H265 资源
ScanHomeH265TranscodingQueue()
})
common.Go(func() {
ScanHomeH265TranscodingQueue()
SubmitH265TranscodingQueue()
CheckH265TranscodingResults()
})
spec := "0 0/30 * * * ?"
if skdg.Conf.Base.Env == "test" {
spec = "0 0/1 * * * ?" // 测试环境3分钟刷新一次
}
_, _ = statCron.AddFunc(spec, func() { //每30分钟执行一次
now := time.Now()
if now.Hour() == 0 || now.Hour() == 1 || now.Hour() == 2 {
yesterday := now.AddDate(0, 0, -1)
if err := dailyAdverCalc.CalcRetention(context.Background(), yesterday); err != nil {
log.Error("CalcRetention failed", log.Any("error", err))
}
}
// 当天的数据，每小时计算一次，确保及时更新看板
if err := dailyAdverCalc.CalcRetention(context.Background(), now); err != nil {
log.Error("CalcRetention failed", log.Any("error", err))
}
})
statCron.Start()
}
skd/job/schedule.go
+34
View File
@@ -0,0 +1,34 @@
package job
import (
"context"
"time"
"91porn-server/common/log"
"github.com/robfig/cron/v3"
)
var statCron *cron.Cron
func init() {
statCron = cron.New(cron.WithSeconds()) //从秒开始的定时任务
}
// 开启定时任务
func Start() {
log.Info("start job...", log.Any("beginAt", time.Now()))
start()
}
func Stop() {
ctx := statCron.Stop()
<-ctx.Done()
err := ctx.Err()
//statCron.Stop()后，若job处于静默可能立即退出，再次执行<-ctx.Done()产生Canceled错误，此处忽略该错误
if err != nil && err != context.Canceled {
log.Error("Skd Job Stop error", log.E(err))
return
}
log.Info("end job", log.Any("endAt", time.Now()))
}
skd/job/shortrecommend/job.go
+231
View File
@@ -0,0 +1,231 @@
package shortrecommend
import (
"context"
"fmt"
"time"
"91porn-server/common/log"
redisutil "91porn-server/common/redis"
recommendqueue "91porn-server/common/shortrecommend"
"91porn-server/models/v/moduleconfmod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const (
buildLockTTL = 10 * time.Minute
buildLockRenewInterval = 3 * time.Minute
buildTimeout = 30 * time.Minute
healthCheckTimeout = 5 * time.Second
)
const renewBuildLockScriptSource = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then
return 0
end
return redis.call('EXPIRE', KEYS[1], ARGV[2])
`
const releaseBuildLockScriptSource = `
if redis.call('GET', KEYS[1]) ~= ARGV[1] then
return 0
end
return redis.call('DEL', KEYS[1])
`
var (
renewBuildLockScript = redisutil.NewScript(renewBuildLockScriptSource)
releaseBuildLockScript = redisutil.NewScript(releaseBuildLockScriptSource)
)
func evalLockScript(
ctx context.Context,
script *redisutil.Script,
key, token string,
args ...interface{},
) (bool, error) {
values := append([]interface{}{token}, args...)
raw, err := skdg.Redis.RunScriptContext(ctx, script, []string{key}, values...)
if err != nil {
return false, err
}
return fmt.Sprint(raw) == "1", nil
}
func renewBuildLock(ctx context.Context, key, token string) (bool, error) {
return evalLockScript(
ctx,
renewBuildLockScript,
key,
token,
int64(buildLockTTL/time.Second),
)
}
func releaseBuildLock(ctx context.Context, key, token string) error {
_, err := evalLockScript(ctx, releaseBuildLockScript, key, token)
return err
}
func keepBuildLock(ctx context.Context, cancel context.CancelFunc, key, token, version string) {
ticker := time.NewTicker(buildLockRenewInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
renewed, err := renewBuildLock(ctx, key, token)
if err != nil || !renewed {
log.Error("short recommend renew build lock failed",
log.E(err), log.Any("version", version))
cancel()
return
}
}
}
}
func buildQueueVersion(now time.Time, buildToken string) string {
return recommendqueue.VersionWithRevision(now, buildToken)
}
// Run 刷新真实互动分并发布当日全局短视频队列。
func Run() {
now := time.Now()
dateVersion := recommendqueue.VersionAt(now)
healthCtx, healthCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
health, err := recommendqueue.HealthContext(healthCtx, skdg.Redis, dateVersion)
healthCancel()
if err != nil {
log.Error("short recommend queue health check failed",
log.E(err), log.Any("dateVersion", dateVersion))
return
}
if health.Healthy {
return
}
log.Warn("short recommend queue unhealthy, start rebuild",
log.Any("dateVersion", dateVersion),
log.Any("currentVersion", health.Version),
log.Any("healthStatus", health.Status),
log.Any("queueLength", health.QueueLength),
log.Any("metadataLength", health.MetadataLength),
log.Any("queueTTLSeconds", int64(health.QueueTTL/time.Second)),
log.Any("metadataTTLSeconds", int64(health.MetadataTTL/time.Second)))
lockKey := recommendqueue.LockKey(dateVersion)
lockToken := primitive.NewObjectID().Hex()
lockCtx, lockCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
locked, err := skdg.Redis.SetNXContext(lockCtx, lockKey, lockToken, buildLockTTL)
lockCancel()
if err != nil {
log.Error("short recommend acquire build lock failed",
log.E(err), log.Any("dateVersion", dateVersion))
return
}
if !locked {
return
}
buildCtx, cancel := context.WithTimeout(context.Background(), buildTimeout)
lockKeeperDone := make(chan struct{})
go func() {
defer close(lockKeeperDone)
keepBuildLock(buildCtx, cancel, lockKey, lockToken, dateVersion)
}()
defer func() {
cancel()
<-lockKeeperDone
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
defer releaseCancel()
if err := releaseBuildLock(releaseCtx, lockKey, lockToken); err != nil {
log.Error("short recommend release build lock failed",
log.E(err), log.Any("dateVersion", dateVersion))
}
}()
// 首次健康检查与加锁之间可能已有其他实例完成发布；持锁后复检避免重复全量扫描。
recheckCtx, recheckCancel := context.WithTimeout(buildCtx, healthCheckTimeout)
rechecked, recheckErr := recommendqueue.HealthContext(recheckCtx, skdg.Redis, dateVersion)
recheckCancel()
if recheckErr != nil {
log.Error("short recommend queue health recheck failed",
log.E(recheckErr), log.Any("dateVersion", dateVersion))
return
}
if rechecked.Healthy {
return
}
queueVersion := buildQueueVersion(now, lockToken)
buildStartedAt := time.Now()
excludedModuleIDs, err := moduleconfmod.ExcludedVideoModuleIDs(now, true)
if err != nil {
log.Error("short recommend load excluded modules failed",
log.E(err), log.Any("version", queueVersion))
return
}
refreshStartedAt := time.Now()
initializationLimit := skdg.Conf.ShortRecommendInitializationLimit()
candidates, err := vidmod.RefreshRecommendScoresWithInitializationLimit(
buildCtx, now, initializationLimit, excludedModuleIDs...,
)
refreshDuration := time.Since(refreshStartedAt)
if err != nil {
log.Error("short recommend refresh scores failed",
log.E(err), log.Any("version", queueVersion),
log.Any("initializationWriteLimit", initializationLimit),
log.Any("refreshDurationMs", refreshDuration.Milliseconds()))
return
}
assembleStartedAt := time.Now()
ids, assemblyStats := recommendqueue.AssembleWithStats(candidates, now)
assembleDuration := time.Since(assembleStartedAt)
if len(ids) == 0 {
log.Warn("short recommend candidates empty, keep previous queue",
log.Any("version", queueVersion))
return
}
if err := buildCtx.Err(); err != nil {
log.Error("short recommend build lock lost before publish",
log.E(err), log.Any("version", queueVersion))
return
}
if currentVersion := recommendqueue.VersionAt(time.Now()); currentVersion != dateVersion {
log.Warn("short recommend build crossed version boundary, skip publish",
log.Any("version", queueVersion), log.Any("currentDateVersion", currentVersion))
return
}
renewed, err := renewBuildLock(buildCtx, lockKey, lockToken)
if err != nil || !renewed {
log.Error("short recommend verify build lock failed",
log.E(err), log.Any("version", queueVersion))
return
}
publishStartedAt := time.Now()
err = recommendqueue.Publish(buildCtx, skdg.Redis, recommendqueue.QueueMeta{
Version: queueVersion, Length: len(ids), GeneratedAt: now, StartOffset: 0,
}, ids, lockKey, lockToken)
publishDuration := time.Since(publishStartedAt)
if err != nil {
log.Error("short recommend publish queue failed",
log.E(err), log.Any("version", queueVersion),
log.Any("publishDurationMs", publishDuration.Milliseconds()))
return
}
log.Info("short recommend queue published",
log.Any("version", queueVersion),
log.Any("dateVersion", dateVersion),
log.Any("candidateCount", len(candidates)),
log.Any("excludedModuleCount", len(excludedModuleIDs)),
log.Any("queueLength", len(ids)),
log.Any("highScoreCount", assemblyStats.HighCount),
log.Any("newVideoCount", assemblyStats.NewCount),
log.Any("blockCount", assemblyStats.BlockCount),
log.Any("refreshDurationMs", refreshDuration.Milliseconds()),
log.Any("assembleDurationMs", assembleDuration.Milliseconds()),
log.Any("publishDurationMs", publishDuration.Milliseconds()),
log.Any("totalDurationMs", time.Since(buildStartedAt).Milliseconds()))
}
skd/job/shortrecommend/job_test.go
+18
View File
@@ -0,0 +1,18 @@
package shortrecommend
import (
"testing"
"time"
)
func TestBuildQueueVersionIsUniqueWithinSameDay(t *testing.T) {
now := time.Date(2026, 7, 31, 2, 0, 0, 0, time.UTC)
first := buildQueueVersion(now, "build-a")
second := buildQueueVersion(now, "build-b")
if first != "20260731-build-a" || second != "20260731-build-b" {
t.Fatalf("versions = %q, %q", first, second)
}
if first == second {
t.Fatal("same-day rebuilds must not reuse a queue version")
}
}
skd/job/sqlcollapser/shordvideo/method.go
+27
View File
@@ -0,0 +1,27 @@
package shortvideo
import (
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/models/v/vidmod"
)
// SyncDataToRedis 同步数据到redis
func SyncDataToRedis(data []*vidmod.VideoModel) error {
if len(data) == 0 {
return nil
}
for _, v := range data {
if v.NewsType != vidmod.SHORT || (v.Status != vidmod.CheckPass && v.Status != vidmod.Free) {
// 去除
_, _ = redis.Handler.SRem(redisconst.ShortVideosKey, v.ID.Hex())
} else {
_, _ = redis.Handler.SAdd(redisconst.ShortVideosKey, v.ID.Hex())
}
}
log.Info("Redis sync video data completed.")
return nil
}
skd/job/sqlcollapser/shordvideo/sync.go
+89
View File
@@ -0,0 +1,89 @@
package shortvideo
import (
"91porn-server/common/constant/redisconst"
"91porn-server/common/redis"
"fmt"
"time"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/vidmod"
)
func RunSync(defaultDate time.Time) {
log.Info("Redis short video sync start...")
defer log.Info("Redis short video sync end.")
// 如果短视频缓存存在的数据量过低，表示缓存被清理过，需要重新同步
c, err := redis.Handler.SCard(redisconst.ShortVideosKey)
if err == nil && c < 1000 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, defaultDate)
}
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.RedisSyncShortVideoJob, defaultDate)
if err != nil {
log.Error("Redis sync short video data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.OneDayScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.OneDayScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
//// 获取MID
//ret, err := moduleconfmod.GetModuleConfByType(moduleconfmod.ShortPage)
//if err != nil {
// log.Error("Redis sync short video data RunSync get module conf failed", log.E(err))
// return
//}
//midMap := make(map[string]any)
//for _, v := range ret {
// midMap[v.ID.Hex()] = nil
//}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
var page, size = 1, 1000
hasNext := true
var data []*vidmod.VideoModel
for hasNext {
data, hasNext, err = vidmod.GetShortVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
if err != nil {
log.Error("Redis sync short video data GetVideoListByUpdateTimeRange err! ", log.E(err))
return
}
if hasNext && len(data) > 0 {
_ = SyncDataToRedis(data)
page++
} else {
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Redis sync short video data RunSync commit err! ", log.E(err))
return
}
}
}
lastSyncAt = end
}
}
func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
if len(data) <= 0 {
_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime)
return nil
}
if err := SyncDataToRedis(data); err != nil {
return fmt.Errorf("Redis sync commit short video data SyncDataToES failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime); err != nil {
return fmt.Errorf("Redis sync commit short video data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/sqlcollapser/user/method.go
+44
View File
@@ -0,0 +1,44 @@
package user
import (
"91porn-server/models/v/usermod"
"91porn-server/models/v/vidmod"
"time"
)
func doSyncWorkCount(start, end time.Time) error {
mats := []vidmod.Matcher{
(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
(&vidmod.UpdatedAtGTEAndLTMatch{GTE: &start, LT: &end}).New(),
}
publisherIDs, err := vidmod.PublisherIDList(mats...)
if err != nil {
return err
}
// 分批操作
for len(publisherIDs) > 0 {
var uids []uint64
if len(publisherIDs) > 100 {
uids = publisherIDs[:100]
publisherIDs = publisherIDs[100:]
} else {
uids = publisherIDs
publisherIDs = nil
}
mats = []vidmod.Matcher{
(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
(&vidmod.PublisherIDInMatch{PublisherIDList: uids}).New(),
}
m, err := vidmod.CountMapByPublisherID(mats...)
if err != nil {
return err
}
err = usermod.UpdateUserCountWorks(m)
if err != nil {
return err
}
}
return nil
}
skd/job/sqlcollapser/user/runSync.go
+54
View File
@@ -0,0 +1,54 @@
package user
import (
"fmt"
"time"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
)
func RunSync(defaultDate time.Time) {
log.Info("SqlCollasper sync")
defer log.Info("SqlCollasper sync user total work end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SqlCollasperUserTotalWork, defaultDate)
if err != nil {
log.Error("SqlCollasper RunSync user total work get last record time failed", log.E(err))
return
}
var now = time.Now()
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end time.Time
if lastSyncAt.Add(time.Hour * 24).Before(now) {
end = lastSyncAt.Add(time.Hour * 24)
} else {
end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
if err = commit(subTimeRange); err != nil {
log.Error("SqlCollasper sync audiobook data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(t timerange.TimeRange) error {
if err := doSyncWorkCount(t.Head, t.Tail); err != nil {
return fmt.Errorf("SqlCollasper sync commit incPageView failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SqlCollasperUserTotalWork, t.Tail); err != nil {
return fmt.Errorf("SqlCollasper sync commit UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/sqlcollapser/vid/method.go
+51
View File
@@ -0,0 +1,51 @@
package vid
import (
"time"
"91porn-server/common/constant/redisconst"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
)
func incPageView(syncTime time.Time) error {
now := syncTime
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
key := redisconst.VidPageViewListKey(recentMinute.Unix())
count, err := skdg.Redis.LCount(key)
if err != nil {
return err
}
if count == 0 {
return nil
}
strList, err := skdg.Redis.LRange(key, 0, count-1)
if err != nil {
return err
}
dmap := make(map[string]int64)
for i := range strList {
dmap[strList[i]]++
}
models := make([]mongo.WriteModel, 0, len(dmap))
for k, v := range dmap {
_id, err1 := primitive.ObjectIDFromHex(k)
if err1 != nil {
continue
}
filter := bson.M{"_id": _id}
update := bson.M{"$inc": bson.M{"pageViewCount": v}}
models = append(models, mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update))
}
if err = vidmod.Bulk(models); err != nil {
return err
}
_, _ = skdg.Redis.Del(key)
return nil
}
skd/job/sqlcollapser/vid/runSync.go
+48
View File
@@ -0,0 +1,48 @@
package vid
import (
"fmt"
"time"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
)
func RunSync(defaultDate time.Time) {
log.Info("SqlCollasper sync start...")
defer log.Info("SqlCollasper sync end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SqlCollasperVidPageView, defaultDate)
if err != nil {
log.Error("SqlCollasper RunSync get last record time failed", log.E(err))
return
}
var now = time.Now().Add(-time.Minute * 5)
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
if err = commit(subTimeRange.Tail); err != nil {
log.Error("SqlCollasper sync audiobook data RunSync commit err! ", log.E(err))
return
}
lastSyncAt = end
}
}
func commit(syncTime time.Time) error {
if err := incPageView(syncTime); err != nil {
return fmt.Errorf("SqlCollasper sync commit incPageView failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SqlCollasperVidPageView, syncTime); err != nil {
return fmt.Errorf("SqlCollasper sync commit UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/sqlcollasper.go
+50
View File
@@ -0,0 +1,50 @@
package job
import (
"91porn-server/common"
"91porn-server/common/log"
shortvideo "91porn-server/skd/job/sqlcollapser/shordvideo"
"91porn-server/skd/job/sqlcollapser/user"
"91porn-server/skd/job/sqlcollapser/vid"
"sync"
)
var syncSqlcollaspeIsRuining = false
// start 定时任务 开启
func runSyncSqlcollaspe() {
//防止重入
if syncSqlcollaspeIsRuining {
return
}
syncSqlcollaspeIsRuining = true
defer func() {
syncSqlcollaspeIsRuining = false
}()
doSyncSqlcollasper() //做折叠业务
}
// 做折叠业务
func doSyncSqlcollasper() {
log.Info("SqlCollasper doSyncSqlcollasper start...")
defaultSyncDate, err := getDefaultSyncDate()
if err != nil {
log.Error("defaultStatDate invalid", log.E(err))
return
}
wg := sync.WaitGroup{}
wg.Add(3)
common.Go(func() {
defer wg.Done()
vid.RunSync(defaultSyncDate)
})
common.Go(func() {
defer wg.Done()
user.RunSync(defaultSyncDate)
})
common.Go(func() {
defer wg.Done()
shortvideo.RunSync(defaultSyncDate)
})
wg.Wait()
}
skd/job/stat.go
+104
View File
@@ -0,0 +1,104 @@
package job
import (
"sync"
"time"
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/skd/job/cache/tonecache"
"91porn-server/skd/job/stat/keyword"
"91porn-server/skd/job/stat/kwrank"
"91porn-server/skd/job/stat/rank"
"91porn-server/skd/job/stat/rechorder"
"91porn-server/skd/job/stat/statuser"
"91porn-server/skd/job/stat/tagplay"
"91porn-server/skd/job/stat/video"
"91porn-server/skd/job/stat/vidtotal"
"91porn-server/skd/job/stat/withorder"
"91porn-server/skd/skdg"
)
var (
statIsRunning = false //true:统计任务运行中，反之停止，
statRankIsRunning = false
)
// 运行统计任务
func runStat() {
//防止重入
if statIsRunning {
return
}
statIsRunning = true
defer func() {
statIsRunning = false
}()
doStat() //做统计业务
}
// 做统计业务
func doStat() {
defaultStatDate, err := getDefaultStatDate()
if err != nil {
log.Error("defaultStatDate invalid", log.E(err))
return
}
methodM := []func(){
//主要用于app
func() { //"关键字"
keyword.RunStat(defaultStatDate)
kwrank.RunStat(defaultStatDate) //有执行顺序
},
func() { //"标签播放"
tagplay.RunStat(defaultStatDate)
},
func() { //"用户数据"
statuser.RunStat(defaultStatDate)
},
// func() { //"ranks"
//rankcache.Update()
// },
func() { //"tone"
tonecache.Update()
},
func() { //"视屏"
video.RunStat(defaultStatDate)
vidtotal.RunStat(defaultStatDate) //有执行顺序
},
func() { //"充值订单"
rechorder.RunStat(defaultStatDate)
},
func() { //"兑换订单"
withorder.RunStat(defaultStatDate)
},
}
time.Sleep(time.Duration(30) * time.Second)
wg := sync.WaitGroup{}
wg.Add(len(methodM))
for _, method := range methodM {
_method := method
common.Go(func() {
defer wg.Done()
_method()
})
}
wg.Wait()
}
func getDefaultStatDate() (time.Time, error) {
return time.ParseInLocation("2006-01-02", skdg.Conf.StartDate, time.Local)
}
// 运行排行榜
func runRankStat() {
//防止重入
if statRankIsRunning {
return
}
statRankIsRunning = true
defer func() {
statRankIsRunning = false
}()
rank.RunStat()
}
skd/job/stat/keyword/method.go
+59
View File
@@ -0,0 +1,59 @@
package keyword
import (
"91porn-server/common/constant"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/l/searchlogmod"
"91porn-server/models/s/kwstatmod"
)
type TimeRange = timerange.TimeRange
func toKeywordIncDoc(realm constant.RealmType, data map[string]int64) []kwstatmod.KeywordIncDoc {
list := make([]kwstatmod.KeywordIncDoc, len(data))
i := 0
for k, v := range data {
word := k
count := v
list[i] = kwstatmod.KeywordIncDoc{
Realm: realm,
Word: word,
Count: count,
}
i++
}
return list
}
func GetIncDocList(timeRange TimeRange) ([]kwstatmod.KeywordIncDoc, error) {
incDocList := []kwstatmod.KeywordIncDoc{}
//综合关键字搜索数
complexKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchComplex)
if err != nil {
return nil, err
}
complexIncDocList := toKeywordIncDoc(constant.SearchComplex, complexKeywordCountMap)
incDocList = append(incDocList, complexIncDocList...)
//视屏关键字搜索数
vidKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchSP)
if err != nil {
return nil, err
}
videoIncDocList := toKeywordIncDoc(constant.SearchSP, vidKeywordCountMap)
incDocList = append(incDocList, videoIncDocList...)
//标签关键字搜索数
tagKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchTag)
if err != nil {
return nil, err
}
tagIncDocList := toKeywordIncDoc(constant.SearchTag, tagKeywordCountMap)
incDocList = append(incDocList, tagIncDocList...)
//用户关键字搜索数
userKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchUser)
if err != nil {
return nil, err
}
userIncDocList := toKeywordIncDoc(constant.SearchUser, userKeywordCountMap)
incDocList = append(incDocList, userIncDocList...)
return incDocList, nil
}
skd/job/stat/keyword/stat.go
+72
View File
@@ -0,0 +1,72 @@
package keyword
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/kwstatmod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
// RunStat
func RunStat(defaultStatDate time.Time) {
log.Info("keyword stat running...")
defer log.Info("keyword stat finished...")
now := time.Now()
//从历史提交记录中获取上次记录时间
lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.KeywordStatJob, defaultStatDate) //第一次从默认记录时间开始统计
if err != nil {
log.Error("Keyword RunStat lastRecordTime faild!", log.E(err))
return
}
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
//拆分时间区间并依次提交,均衡数据库读写压力
for recentMinute.After(lastStatAt) {
end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
//此时的零日点
sumDate := timerange.LocDayRange(subTimeRange.Head).Head
//装配基础统计数据
incDocList, err := GetIncDocList(subTimeRange)
if err != nil {
log.Error("Keyword RunStat GetIncDocList faild!", log.E(err))
return
}
//提交数据
log.Info("keywordStat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
if err = commit(sumDate, subTimeRange.Tail, incDocList); err != nil {
log.Error("Keyword RunStat commit faild!", log.E(err))
return
}
log.Info("keywordStat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
lastStatAt = end
}
}
// commit 提交统计数据
func commit(sumDate time.Time, recordTime time.Time, incDocList []kwstatmod.KeywordIncDoc) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) (err error) {
if len(incDocList) != 0 {
if err = kwstatmod.ChangeStatTrans(trans, sumDate, recordTime, incDocList); err != nil {
return fmt.Errorf(" keyword commit ChangeStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err = statrecordmod.UpsertOneTrans(trans, statrecordmod.KeywordStatJob, recordTime); err != nil {
return fmt.Errorf(" keyword commit UpsertOneTrans faild!, err:%+v\n", err)
}
return
}, opt)
}
skd/job/stat/kwrank/method.go
+29
View File
@@ -0,0 +1,29 @@
package kwrank
import (
"time"
"91porn-server/models/s/kwrankmod"
"91porn-server/models/s/kwstatmod"
)
type RankSetDoc = kwrankmod.RankSetDoc
func GetStatSetDocList(sumDate time.Time) ([]RankSetDoc, error) {
keywordCountMap, err := kwstatmod.KeywordCountMapBySumDate(sumDate)
if err != nil {
return nil, err
}
rankDocList := make([]RankSetDoc, len(keywordCountMap))
i := 0
for w, c := range keywordCountMap {
word := w
count := c
rankDocList[i] = RankSetDoc{
Word: &word,
Count: &count,
}
i++
}
return rankDocList, nil
}
skd/job/stat/kwrank/stat.go
+52
View File
@@ -0,0 +1,52 @@
package kwrank
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/kwrankmod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
// RunStat
func RunStat(defaultStatDate time.Time) {
log.Info("keyword rank stat running...")
defer log.Info("keyword rank stat finished...")
now := time.Now()
sumDate := timerange.LocDayRange(now).Head //获取零日点
//装配基础统计数据
setList, err := GetStatSetDocList(sumDate)
if err != nil {
log.Error("kwrank RunStat fillStatSetDoc faild!", log.E(err))
return
}
//提交数据
log.Info("keyword rank stat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
if err = commit(sumDate, recentMinute, setList); err != nil {
log.Error("kwrank RunStat commit faild!", log.E(err))
return
}
log.Info("keyword rank stat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
}
// commit 提交统计数据
func commit(sumDate time.Time, recordTime time.Time, setDocList []RankSetDoc) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) (err error) {
if len(setDocList) != 0 {
if err = kwrankmod.ChangeStatTrans(trans, sumDate, recordTime, setDocList); err != nil {
return fmt.Errorf("kwrank commit ChangeStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err = statrecordmod.UpsertOneTrans(trans, statrecordmod.KeywordRankStatJob, recordTime); err != nil {
return fmt.Errorf("kwrank commit UpsertOneTrans faild!, err:%+v\n", err)
}
return
}, opt)
}
skd/job/stat/rank/stat.go
+53
View File
@@ -0,0 +1,53 @@
package rank
import (
"time"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/hotstatmod"
"91porn-server/models/v/vidmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// RunStat 统计实时排行榜
func RunStat() {
log.Info("rank stat running...")
defer log.Info("rank stat finished...")
now := time.Now()
sumDate := timerange.LocDayRange(now).Head
infos, err := vidmod.GetTotalNewsList()
if err != nil {
return
}
mNow := calNowHot(infos)
mYesterday, err := hotstatmod.GetYesterdayHotValue(sumDate)
if err != nil {
return
}
mLastWeek, err := hotstatmod.GetLastWeekHotValue(sumDate)
if err != nil {
return
}
mLastMonth, _ := hotstatmod.GetLastMonthHotValue(sumDate)
mDayli := make(map[primitive.ObjectID]int)
mWeek := make(map[primitive.ObjectID]int)
mMonth := make(map[primitive.ObjectID]int)
for vid, v := range mNow {
mDayli[vid] = v - mYesterday[vid]
mWeek[vid] = v - mLastWeek[vid]
mMonth[vid] = v - mLastMonth[vid]
}
//入库
_ = hotstatmod.UpdateInsertHotValue(sumDate, mDayli, mWeek, mMonth, mNow)
}
func calNowHot(infos []*vidmod.VideoModel) map[primitive.ObjectID]int {
mNow := make(map[primitive.ObjectID]int)
for _, v := range infos {
value := 1*v.FakePlayCount + 4*v.FakeLikeCount + 2*v.FakeCommentCount + v.FakeShareCount
mNow[v.ID] = value
}
return mNow
}
skd/job/stat/rechorder/method.go
+138
View File
@@ -0,0 +1,138 @@
package rechorder
import (
"time"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statordermod"
"91porn-server/models/v/rchgordmod"
)
type RechargeOrderStateDoc = statordermod.RechargeOrderStateDoc
type RecOrderStateMap = statordermod.RecOrderStateMap
func fillRecOrderStatDocMap(sumDate time.Time, timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
//订单数 OrderCount
if err := fillRecOrderCount(timeRange, docMap); err != nil {
return err
}
//完成订单数 CompleteOrderCount
if err := fillRecCompleteOrderCount(timeRange, docMap); err != nil {
return err
}
//订单金额 OrderAmount
if err := fillRecOrderAmount(timeRange, docMap); err != nil {
return err
}
if err := fillRecCompleteOrderPayMoney(timeRange, docMap); err != nil {
return err
}
//用户已支付订单数
if err := fillRecPaidOrderCount(timeRange, docMap); err != nil {
return err
}
//用户已支付金额
return fillRecPaidMoney(timeRange, docMap)
}
// fillRecOrderCount fillRecOrderCount
func fillRecOrderCount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
orderCountMap, err := rchgordmod.ChannelOrderCountMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range orderCountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &RechargeOrderStateDoc{}
}
val := v
docM[channel].OrderCount = &val
}
return nil
}
// fillRecCompleteOrderCount fillRecCompleteOrderCount}
func fillRecCompleteOrderCount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
completeOrderCountMap, err := rchgordmod.ChannelCompleteOrderCountMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range completeOrderCountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &RechargeOrderStateDoc{}
}
val := v
docM[channel].CompleteOrderCount = &val
}
return nil
}
// fillRecOrderAmount fillRecOrderAmount
func fillRecOrderAmount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
orderAmountMap, err := rchgordmod.ChannelOrderMoneyMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range orderAmountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &RechargeOrderStateDoc{}
}
val := v
docM[channel].OrderAmount = &val
}
return nil
}
// fillRecCompleteOrderPayMoney fillRecCompleteOrderPayMoney
func fillRecCompleteOrderPayMoney(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
payMoneyMap, err := rchgordmod.ChannelCompleteOrderPayMoneyMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range payMoneyMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &RechargeOrderStateDoc{}
}
val := v
docM[channel].CompleteOrderAmount = &val
}
return nil
}
// fillRecPaidOrderCount fillRecPaidOrderCount
func fillRecPaidOrderCount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
paiedOrderCountMap, err := rchgordmod.ChannelPaymentOrderCountMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range paiedOrderCountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &RechargeOrderStateDoc{}
}
val := v
docM[channel].PaidOrderCount = &val
}
return nil
}
// fillRecPaidMoney fillRecPaidMoney
func fillRecPaidMoney(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
paiedMoneyMap, err := rchgordmod.ChannelPaymentMoneyMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range paiedMoneyMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &RechargeOrderStateDoc{}
}
val := v
docM[channel].PaidAmount = &val
}
return nil
}
skd/job/stat/rechorder/stat.go
+97
View File
@@ -0,0 +1,97 @@
package rechorder
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statordermod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
func RunStat(defaultStatDate time.Time) {
log.Info("rechorderStat stat running...")
defer log.Info("rechorderStat stat finished...")
now := time.Now()
//从历史提交记录中获取上次记录时间
lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.RechargeOrderStatJob, defaultStatDate) //第一次从默认记录时间开始统计
if err != nil {
log.Error("rechorder RunStat lastRecordTime faild!", log.E(err))
return
}
//过去几天的数据按一日为单位统计，帮助数据恢复
dayHead := timerange.LocDayRange(now).Head
for dayHead.After(lastStatAt) {
end := lastStatAt.AddDate(0, 0, 1)
//防止时间超出范围
if end.After(dayHead) {
end = dayHead
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err := statByTimeRange(subTimeRange); err != nil {
log.Error("rechorder RunStat statByTimeRange faild!", log.E(err))
return
}
lastStatAt = end
}
//当天的数据按照5分钟统计
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
//拆分时间区间并依次提交,控制数据库读写压力
for recentMinute.After(lastStatAt) {
end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err := statByTimeRange(subTimeRange); err != nil {
log.Error("rechorder RunStat statByTimeRange faild!", log.E(err))
return
}
lastStatAt = end
}
}
func statByTimeRange(timeRange timerange.TimeRange) error {
sumDate := timerange.LocDayRange(timeRange.Head).Head //获取此时的零日点
//装配渠道统计数据
recOrderStateMap := make(RecOrderStateMap)
if err := fillRecOrderStatDocMap(sumDate, timeRange, &recOrderStateMap); err != nil {
log.Error("rechorder RunStat fillRecOrderStatDocMap faild!", log.E(err))
return err
}
//提交渠道统计数据
log.Info("rechorderStat recharge committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
if err := commitRec(sumDate, timeRange.Tail, recOrderStateMap); err != nil {
log.Error("rechorderStat RunStat commitRec faild!", log.E(err))
return err
}
log.Info("rechorderStat recharge committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
return nil
}
// commit 提交订单统计数据
func commitRec(sumDate time.Time, recordTime time.Time, docMap RecOrderStateMap) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(docMap) != 0 {
if err := statordermod.ChangeRechargeStatTrans(trans, sumDate, recordTime, docMap); err != nil {
return fmt.Errorf("stat commitRec ChangeRechargeStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.RechargeOrderStatJob, recordTime); err != nil {
return fmt.Errorf("stat commitRec UpsertOneTrans faild!, err:%+v\n", err)
}
return nil
}, opt)
}
skd/job/stat/statuser/method.go
+91
View File
@@ -0,0 +1,91 @@
package statuser
import (
"time"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/l/payvidlgmod"
"91porn-server/models/s/statusermod"
"91porn-server/models/v/vidmod"
)
type TimeRange = timerange.TimeRange
type IncDoc = statusermod.IncDoc
type SetDoc = statusermod.SetDoc
type Itemtype = statusermod.Itemtype
const (
SetType string = "set"
IncType string = "inc"
)
type Method struct {
MethodType string
GetIncDocListMethod func(timeRange TimeRange) ([]IncDoc, error)
GetSetDocListMethod func(timeRange TimeRange) ([]SetDoc, error)
IncCommitMethod func(incDocList []IncDoc, name string, recordAt time.Time) error
SetCommitMethod func(setDocList []SetDoc, name string, recordAt time.Time) error
}
var methodMap = map[Itemtype]Method{
//视频收益
statusermod.VidIncome: {MethodType: IncType, GetIncDocListMethod: getIncomneDocList, IncCommitMethod: commitIncDoc},
//上传数量
statusermod.UploadCount: {MethodType: SetType, GetSetDocListMethod: getUploadCountDocList, SetCommitMethod: commitSetDoc},
}
func getIncomneDocList(timeRange TimeRange) ([]IncDoc, error) {
m, err := payvidlgmod.PublisherIncomeMapByTime(timeRange.Head, timeRange.Tail)
if err != nil {
return nil, err
}
list := make([]IncDoc, len(m))
i := 0
for uid, income := range m {
list[i] = IncDoc{
UID: uid,
Item: statusermod.VidIncome,
Count: income,
RecordAt: timeRange.Tail,
}
i++
}
return list, nil
}
func getUploadCountDocList(timeRange TimeRange) ([]SetDoc, error) {
mats := []vidmod.Matcher{
(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
(&vidmod.ReviewAtGTEAndLTMatch{GTE: &timeRange.Head, LT: &timeRange.Tail}).New(),
}
publisherIDs, err := vidmod.PublisherIDList(mats...)
if err != nil {
return nil, err
}
if len(publisherIDs) == 0 {
return nil, nil
}
mats = []vidmod.Matcher{
(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
(&vidmod.PublisherIDInMatch{PublisherIDList: publisherIDs}).New(),
}
m, err := vidmod.CountMapByPublisherID(mats...)
if err != nil {
return nil, err
}
list := make([]SetDoc, len(m))
i := 0
for uid, uploadCount := range m {
list[i] = SetDoc{
UID: uid,
Item: statusermod.UploadCount,
Count: uploadCount,
RecordAt: timeRange.Tail,
}
i++
}
return list, nil
}
skd/job/stat/statuser/stat.go
+143
View File
@@ -0,0 +1,143 @@
package statuser
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/s/statusermod"
"91porn-server/skd/skdg"
)
func RunStat(defaultStatDate time.Time) {
log.Info("user stat running...")
defer log.Info("user stat finished...")
defaultTimeMap := map[string]time.Time{
string(statusermod.VidIncome): defaultStatDate,
string(statusermod.UploadCount): defaultStatDate,
}
//从历史提交记录中获取上次记录时间
lastStatAtMap, err := statrecordmod.LastRecordTimeMap(statrecordmod.UserStatJob, defaultTimeMap) //第一次从默认记录时间开始统计
if err != nil {
log.Error("user stat statuser LastRecordTimeMap faild!", log.E(err))
return
}
now := time.Now()
dayHead := timerange.LocDayRange(now).Head
for item, lastStatAt := range lastStatAtMap {
_lastStatAt := lastStatAt
//过去几天的数据按一日为单位统计，帮助数据恢复
for dayHead.After(_lastStatAt) {
end := _lastStatAt.AddDate(0, 0, 1)
//防止时间超出范围
if end.After(dayHead) {
end = dayHead
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: _lastStatAt,
Tail: end,
}
if err = statByTimeRange(item, subTimeRange); err != nil {
log.Error("user stat statuser statByTimeRange 1 faild!", log.E(err))
return
}
_lastStatAt = end
}
lastStatAtMap[item] = _lastStatAt
}
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
//拆分时间区间并依次提交,控制数据库读写压力
for item, lastStatAt := range lastStatAtMap {
_lastStatAt := lastStatAt
for recentMinute.After(_lastStatAt) {
end := _lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: _lastStatAt,
Tail: end,
}
if err = statByTimeRange(item, subTimeRange); err != nil {
log.Error("user stat statuser statByTimeRange 2 faild!", log.E(err))
return
}
_lastStatAt = end
}
lastStatAtMap[item] = _lastStatAt
}
}
func statByTimeRange(item string, timeRange timerange.TimeRange) error {
method, ok := methodMap[Itemtype(item)]
if !ok {
return nil
}
if method.MethodType == IncType {
//装配INC基础统计数据
incDocList, err := method.GetIncDocListMethod(timeRange)
if err != nil {
log.Error("user stat statByTimeRange GetIncDocListMethod faild!", log.Any("item", item), log.E(err))
return err
}
//提交Inc统计数据
log.Info("user stat statByTimeRange IncCommitMethod committing...", log.Any("item", item))
if err = method.IncCommitMethod(incDocList, item, timeRange.Tail); err != nil {
log.Error("user stat statByTimeRange IncCommitMethod faild!", log.Any("item", item), log.E(err))
return err
}
log.Info("user stat statByTimeRange IncCommitMethod committed.", log.Any("item", item))
} else if method.MethodType == SetType {
//装配SET基础统计数据
setDocList, err := method.GetSetDocListMethod(timeRange)
if err != nil {
log.Error("user stat statByTimeRange GetSetDocListMethod faild!", log.Any("item", item), log.E(err))
return err
}
//提交SET统计数据
log.Info("user stat statByTimeRange SetCommitMethod committing...", log.Any("item", item))
if err = method.SetCommitMethod(setDocList, item, timeRange.Tail); err != nil {
log.Error("user stat statByTimeRange SetCommitMethod faild!", log.Any("item", item), log.E(err))
return err
}
log.Info("user stat statByTimeRange SetCommitMethod committed.", log.Any("item", item))
}
return nil
}
func commitIncDoc(incDocList []IncDoc, item string, recordAt time.Time) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(incDocList) != 0 {
if err := statusermod.ChangeIncStatTrans(trans, incDocList); err != nil {
return fmt.Errorf("user stat commitCount ChangeIncStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneItemTrans(trans, statrecordmod.UserStatJob, item, recordAt); err != nil {
return fmt.Errorf("user stat commitIncDoc UpsertOneItemTrans faild!, err:%+v\n", err)
}
return nil
}, opt)
}
func commitSetDoc(setDocList []SetDoc, item string, recordAt time.Time) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(setDocList) != 0 {
err := statusermod.ChangeSetStatTrans(trans, setDocList)
if err != nil {
return fmt.Errorf("user stat commitCount ChangeSetStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneItemTrans(trans, statrecordmod.UserStatJob, item, recordAt); err != nil {
return fmt.Errorf("user stat commitSetDoc UpsertOneItemTrans faild!, item: %s+v, err:%+v\n", item, err)
}
return nil
}, opt)
}
skd/job/stat/tagplay/method.go
+25
View File
@@ -0,0 +1,25 @@
package tagplay
import (
"91porn-server/common/timeutil/timerange"
"91porn-server/models/l/playlgmod"
"91porn-server/models/s/stattagplaymod"
)
type IncDoc = stattagplaymod.IncDoc
func GetTagPlayIncDocList(timeRange timerange.TimeRange) ([]IncDoc, error) {
playLogList, err := playlgmod.GetTagPlayByTimeRange(timeRange.Head, timeRange.Tail)
if err != nil {
return []IncDoc{}, err
}
tagPlayList := make([]IncDoc, len(playLogList))
for i, v := range playLogList {
tagPlayList[i] = IncDoc{
VideoID: v.GroupID.VideoID,
TagID: v.GroupID.TagID,
PlayCount: v.PlayCount,
}
}
return tagPlayList, nil
}
skd/job/stat/tagplay/stat.go
+71
View File
@@ -0,0 +1,71 @@
package tagplay
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/s/stattagplaymod"
"91porn-server/skd/skdg"
)
func RunStat(defaultStatDate time.Time) {
log.Info("tag play stat start...")
defer log.Info("tag play stat end...")
// 获取上次统计的记录时间
lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.TagPlayStatJob, defaultStatDate) //第一次从默认记录时间开始统计
if err != nil {
log.Error("tag play RunStat lastRecordTime faild!", log.E(err))
return
}
now := time.Now()
// 按照每个钟的5分钟刻度统计: 0,05,10,15,20,25,30,35,40,45,50,55
// 获取最近的时间刻度
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastStatAt) {
end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
// 时间区间5分钟
subTimeRange := timerange.TimeRange{
Head: lastStatAt,
Tail: end,
}
//5分钟内的统计数据
incDocList, err := GetTagPlayIncDocList(subTimeRange)
if err != nil {
log.Error("tag play RunStat GetTagPlayList err!", log.E(err))
return
}
//每次循环都应提交
log.Info("tag play stat committing...")
if err = commit(incDocList, subTimeRange.Tail); err != nil {
log.Error("tag play RunStat commit err! ", log.E(err))
return
}
log.Info("tag play stat committed.")
// 统计数据
lastStatAt = end
}
}
func commit(statData []IncDoc, recordTime time.Time) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(statData) != 0 {
if err := stattagplaymod.ChangeStatTrans(trans, statData); err != nil {
return fmt.Errorf("tag play commit ChangeStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.TagPlayStatJob, recordTime); err != nil {
return fmt.Errorf("tag play commit UpsertOneTrans faild!, err:%+v\n", err)
}
return nil
}, opt)
}
skd/job/stat/video/method.go
+130
View File
@@ -0,0 +1,130 @@
package video
import (
"time"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/l/payvidlgmod"
"91porn-server/models/l/playlgmod"
"91porn-server/models/s/statvidmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ObjectID = primitive.ObjectID
type TimeRange = timerange.TimeRange
type VideoStatIncDoc = statvidmod.VideoStatIncDoc
type VideoStatSetDoc = statvidmod.VideoStatSetDoc
type VideoStatDoc = statvidmod.VideoStatDoc
type VideoStatDocMap = statvidmod.VideoStatDocMap //ObjectID->VideoStatDoc
// fillPayCoin
func fillPayCoin(timeRange TimeRange, docMap *VideoStatDocMap) error {
incomeMap, err := payvidlgmod.PayCoinMapByTime(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for vid, v := range incomeMap {
if _, existed := docM[vid]; !existed {
docM[vid] = &VideoStatDoc{}
}
val := v
docM[vid].Income = &val
}
return nil
}
// fillPayCount
func fillPayCount(timeRange TimeRange, docMap *VideoStatDocMap) error {
payCountMap, err := payvidlgmod.PayCountMapByTime(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for vid, v := range payCountMap {
if _, existed := docM[vid]; !existed {
docM[vid] = &VideoStatDoc{}
}
val := v
docM[vid].PayCount = &val
}
return nil
}
// fillPlayCount
func fillPlayCount(timeRange TimeRange, docMap *VideoStatDocMap) error {
playCountMap, err := playlgmod.PlayCountMapByTime(timeRange.Head, timeRange.Tail, []int{0, 1, 2}) //0，免费 1付费 2.试看
if err != nil {
return err
}
docM := *docMap
for vid, v := range playCountMap {
if _, existed := docM[vid]; !existed {
docM[vid] = &VideoStatDoc{}
}
val := v
docM[vid].PlayCount = &val
}
return nil
}
// fillPayPlayCount
func fillPayPlayCount(timeRange TimeRange, docMap *VideoStatDocMap) error {
payPlayCountMap, err := playlgmod.PlayCountMapByTime(timeRange.Head, timeRange.Tail, []int{1}) //0，免费 1付费 2.试看
if err != nil {
return err
}
docM := *docMap
for vid, v := range payPlayCountMap {
if _, existed := docM[vid]; !existed {
docM[vid] = &VideoStatDoc{}
}
val := v
docM[vid].PayPlayCount = &val
}
return nil
}
func fillTax(timeRange TimeRange, docMap *VideoStatDocMap) error {
payPlayCountMap, err := payvidlgmod.TaxAmountMapByTime(timeRange.Head, timeRange.Tail) //0，免费 1付费 2.试看
if err != nil {
return err
}
docM := *docMap
for vid, v := range payPlayCountMap {
if _, existed := docM[vid]; !existed {
docM[vid] = &VideoStatDoc{}
}
val := v
docM[vid].Tax = &val
}
return nil
}
// fillStatDocMap
func fillStatDocMap(sumDate time.Time, timeRange TimeRange, docMap *VideoStatDocMap) error {
//收入
if err := fillPayCoin(timeRange, docMap); err != nil {
return err
}
//购买次数
if err := fillPayCount(timeRange, docMap); err != nil {
return err
}
//视屏播放次数
if err := fillPlayCount(timeRange, docMap); err != nil {
return err
}
//付费视屏播放次数
if err := fillPayPlayCount(timeRange, docMap); err != nil {
return err
}
//税
return fillTax(timeRange, docMap)
}
skd/job/stat/video/stat.go
+97
View File
@@ -0,0 +1,97 @@
package video
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/s/statvidmod"
"91porn-server/skd/skdg"
)
func RunStat(defaultStatDate time.Time) {
log.Info("video stat running...")
defer log.Info("video stat finished...")
now := time.Now()
//从历史提交记录中获取上次记录时间
lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.VideoStatJob, defaultStatDate) //第一次从默认记录时间开始统计
if err != nil {
log.Error("video stat RunVideoStat lastRecordTime faild!", log.E(err))
return
}
//过去几天的数据按一日为单位统计，帮助数据恢复
dayHead := timerange.LocDayRange(now).Head
for dayHead.After(lastStatAt) {
end := lastStatAt.AddDate(0, 0, 1)
//防止时间超出范围
if end.After(dayHead) {
end = dayHead
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err = statByTimeRange(subTimeRange); err != nil {
log.Error("video RunStat statByTimeRange faild!", log.E(err))
return
}
lastStatAt = end
}
//当天的数据按照5分钟统计
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
//拆分时间区间并依次提交,控制数据库读写压力
for recentMinute.After(lastStatAt) {
end := lastStatAt.Add(statrecordmod.OneMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err = statByTimeRange(subTimeRange); err != nil {
log.Error("video RunStat statByTimeRange faild!", log.E(err))
return
}
lastStatAt = end
}
}
func statByTimeRange(timeRange timerange.TimeRange) error {
sumDate := timerange.LocDayRange(timeRange.Head).Head //获取此时的零日点
//装配基础统计数据
statDocMap := make(VideoStatDocMap)
if err := fillStatDocMap(sumDate, timeRange, &statDocMap); err != nil {
log.Error("video stat RunVideoStat fillStatDocMap faild!", log.E(err))
return err
}
//提交渠道统计数据
log.Info("videoStat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
if err := commit(sumDate, timeRange.Tail, statDocMap); err != nil {
log.Error("video stat RunVideoStat commit faild!", log.E(err))
return err
}
log.Info("videoStat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
return nil
}
// commit 提交统计数据
func commit(sumDate time.Time, recordTime time.Time, docMap VideoStatDocMap) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(docMap) != 0 {
if err := statvidmod.ChangeStatTrans(trans, sumDate, recordTime, docMap); err != nil {
return fmt.Errorf("video stat commit ChangeStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.VideoStatJob, recordTime); err != nil {
return fmt.Errorf("video stat commit UpsertOneTrans faild!, err:%+v\n", err)
}
return nil
}, opt)
}
skd/job/stat/vidtotal/method.go
+46
View File
@@ -0,0 +1,46 @@
package vidtotal
import (
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statvidmod"
"91porn-server/models/s/statvidtotalmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ObjectID = primitive.ObjectID
type TimeRange = timerange.TimeRange
type VideoTotalIncDoc = statvidtotalmod.IncDoc
type VideoTotalIncDocMap = statvidtotalmod.IncDocMap
type RecordAtGTEAndLTMatch = statvidmod.RecordAtGTEAndLTMatch
func getVideoTotalIncDocMap(timeRange TimeRange) (VideoTotalIncDocMap, error) {
recordAtGTEAndLTMatch := RecordAtGTEAndLTMatch{GTE: &timeRange.Head, LT: &timeRange.Tail}
groupList, err := statvidmod.SumByRecordAt(recordAtGTEAndLTMatch)
if err != nil {
return nil, err
}
m := toVideoTotalIncDocMap(groupList)
return m, nil
}
func toVideoTotalIncDocMap(groupList []statvidmod.SumGroup) VideoTotalIncDocMap {
m := make(VideoTotalIncDocMap, len(groupList))
for _, v := range groupList {
_, ok := m[v.Vid]
if !ok {
m[v.Vid] = &VideoTotalIncDoc{}
}
doc := m[v.Vid]
doc.Income = v.Income
doc.PayCount = v.PayCount
doc.PlayCount = v.PlayCount
doc.PayPlayCount = v.PayPlayCount
doc.Tax = v.Tax
}
return m
}
skd/job/stat/vidtotal/stat.go
+96
View File
@@ -0,0 +1,96 @@
package vidtotal
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
"91porn-server/models/s/statvidtotalmod"
"91porn-server/skd/skdg"
)
func RunStat(defaultStatDate time.Time) {
log.Info("video total stat running...")
defer log.Info("video total stat finished...")
now := time.Now()
//从历史提交记录中获取上次记录时间
lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.VideoTotalStatJob, defaultStatDate) //第一次从默认记录时间开始统计
if err != nil {
log.Error("video total stat RunVideoStat lastRecordTime faild!", log.E(err))
return
}
//过去几天的数据按一日为单位统计，帮助数据恢复
dayHead := timerange.LocDayRange(now).Head
for dayHead.After(lastStatAt) {
end := lastStatAt.AddDate(0, 0, 1)
//防止时间超出范围
if end.After(dayHead) {
end = dayHead
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err = statByTimeRange(subTimeRange); err != nil {
log.Error("video total RunStat statByTimeRange faild!", log.E(err))
return
}
lastStatAt = end
}
//当天的数据按照5分钟统计
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
//拆分时间区间并依次提交,控制数据库读写压力
for recentMinute.After(lastStatAt) {
end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err = statByTimeRange(subTimeRange); err != nil {
log.Error("video total RunStat statByTimeRange faild!", log.E(err))
return
}
lastStatAt = end
}
}
func statByTimeRange(timeRange timerange.TimeRange) error {
//装配基础统计数据
statDocMap, err := getVideoTotalIncDocMap(timeRange)
if err != nil {
log.Error("video total stat RunVideoStat fillStatDocMap faild!", log.E(err))
return err
}
//提交视屏总计数据
log.Info("videoTotalStat committing...")
if err = commit(timeRange.Tail, statDocMap); err != nil {
log.Error("video total stat RunVideoStat commit faild!", log.E(err))
return err
}
log.Info("videoTotalStat committed.")
return nil
}
// commit 提交统计数据
func commit(recordTime time.Time, docMap VideoTotalIncDocMap) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(docMap) != 0 {
if err := statvidtotalmod.ChangeStatTrans(trans, recordTime, docMap); err != nil {
return fmt.Errorf("video total stat commit ChangeStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.VideoTotalStatJob, recordTime); err != nil {
return fmt.Errorf("video total stat commit UpsertOneTrans faild!, err:%+v\n", err)
}
return nil
}, opt)
}
skd/job/stat/withorder/method.go
+139
View File
@@ -0,0 +1,139 @@
package withorder
import (
"time"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statordermod"
"91porn-server/models/v/wdordmod"
)
type WithDrawOrderStateDoc = statordermod.WithDrawOrderStateDoc
type WitOrderStateMap = statordermod.WitOrderStateMap
func fillWitOrderStatDocMap(sumDate time.Time, timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
//订单数 OrderCount
if err := fillWitOrderCount(timeRange, docMap); err != nil {
return err
}
//完成订单数 CompleteOrderCount
if err := fillWitCompleteOrderCount(timeRange, docMap); err != nil {
return err
}
//订单金额 OrderAmount
if err := fillWitOrderAmount(timeRange, docMap); err != nil {
return err
}
//实际兑换金额 CompleteOrderAmount
if err := fillWitCompleteOrderPayAmount(timeRange, docMap); err != nil {
return err
}
//三方已出款订单数 fillWitPaiedOrderCount
if err := fillWitPaiedOrderCount(timeRange, docMap); err != nil {
return err
}
//三方已出款金额 fillWitPaiedMoney
return fillWitPaiedMoney(timeRange, docMap)
}
// fillWitOrderCount fillWitOrderCount
func fillWitOrderCount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
orderCountMap, err := wdordmod.ChannelOrderCountMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range orderCountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &WithDrawOrderStateDoc{}
}
val := v
docM[channel].OrderCount = &val
}
return nil
}
// fillWitCompleteOrderCount fillWitCompleteOrderCount
func fillWitCompleteOrderCount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
completeOrderCountMap, err := wdordmod.ChannelCompleteOrderCountMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range completeOrderCountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &WithDrawOrderStateDoc{}
}
val := v
docM[channel].CompleteOrderCount = &val
}
return nil
}
// fillWitOrderAmount fillWitOrderAmount
func fillWitOrderAmount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
orderAmountMap, err := wdordmod.ChannelOrderMoneyMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range orderAmountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &WithDrawOrderStateDoc{}
}
val := v
docM[channel].OrderAmount = &val
}
return nil
}
// fillWitCompleteOrderPayAmount fillWitCompleteOrderPayAmount
func fillWitCompleteOrderPayAmount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
completeOrderAmountMap, err := wdordmod.ChannelCompleteOrderPayMoneyMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range completeOrderAmountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &WithDrawOrderStateDoc{}
}
val := v
docM[channel].CompleteOrderAmount = &val
}
return nil
}
// fillWitPaiedOrderCount fillWitPaiedOrderCount
func fillWitPaiedOrderCount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
receivedOrderCountMap, err := wdordmod.ChannelReceivedOrderCountMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range receivedOrderCountMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &WithDrawOrderStateDoc{}
}
val := v
docM[channel].PaidOrderCount = &val
}
return nil
}
// fillWitPaiedMoney fillWitPaiedMoney
func fillWitPaiedMoney(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
receivedMoneyMap, err := wdordmod.ChannelReceivedMoneyMap(timeRange.Head, timeRange.Tail)
if err != nil {
return err
}
docM := *docMap
for channel, v := range receivedMoneyMap {
if _, existed := docM[channel]; !existed {
docM[channel] = &WithDrawOrderStateDoc{}
}
val := v
docM[channel].PaidAmount = &val
}
return nil
}
skd/job/stat/withorder/stat.go
+97
View File
@@ -0,0 +1,97 @@
package withorder
import (
"fmt"
"time"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statordermod"
"91porn-server/models/s/statrecordmod"
"91porn-server/skd/skdg"
)
func RunStat(defaultStatDate time.Time) {
log.Info("withorder stat running...")
defer log.Info("withorder stat finished...")
now := time.Now()
//从历史提交记录中获取上次记录时间
lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.WithdrawOrderStatJob, defaultStatDate) //第一次从默认记录时间开始统计
if err != nil {
log.Error("withorder RunStat lastRecordTime faild!", log.E(err))
return
}
//过去几天的数据按一日为单位统计，帮助数据恢复
dayHead := timerange.LocDayRange(now).Head
for dayHead.After(lastStatAt) {
end := lastStatAt.AddDate(0, 0, 1)
//防止时间超出范围
if end.After(dayHead) {
end = dayHead
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err = statByTimeRange(subTimeRange); err != nil {
log.Error("withorder RunStat statByTimeRange 1 faild!", log.E(err))
return
}
lastStatAt = end
}
//当天的数据按照5分钟统计
recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
//拆分时间区间并依次提交,控制数据库读写压力
for recentMinute.After(lastStatAt) {
end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
//防止时间超出范围
if end.After(recentMinute) {
end = recentMinute
}
subTimeRange := timerange.TimeRange{ //连续子切片
Head: lastStatAt,
Tail: end,
}
if err = statByTimeRange(subTimeRange); err != nil {
log.Error("withorder RunStat statByTimeRange 2 faild!", log.E(err))
return
}
lastStatAt = end
}
}
func statByTimeRange(timeRange timerange.TimeRange) error {
sumDate := timerange.LocDayRange(timeRange.Head).Head //获取此时的零日点
//装配渠道统计数据
witOrderStateMap := make(WitOrderStateMap)
if err := fillWitOrderStatDocMap(sumDate, timeRange, &witOrderStateMap); err != nil {
log.Error("withorder RunStat fillWitOrderStatDocMap faild!", log.E(err))
return err
}
//提交渠道统计数据
log.Info("withorderStat withdraw committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
if err := commitWit(sumDate, timeRange.Tail, witOrderStateMap); err != nil {
log.Error("withorder RunStat commitWit faild!", log.E(err))
return err
}
log.Info("withorderStat withdraw committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
return nil
}
// commit 提交订单统计数据
func commitWit(sumDate time.Time, recordTime time.Time, stateMap WitOrderStateMap) error {
opt := (&db.TransOpts{}).SetReEntry(10)
return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
if len(stateMap) != 0 {
if err := statordermod.ChangeWithDrawStatTrans(trans, sumDate, recordTime, stateMap); err != nil {
return fmt.Errorf("stat commitWit ChangeWithDrawStatTrans faild!, err:%+v\n", err)
}
}
//提交本次修改记录
if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.WithdrawOrderStatJob, recordTime); err != nil {
return fmt.Errorf("stat commitWit UpsertOneTrans faild!, err:%+v\n", err)
}
return nil
}, opt)
}
skd/job/sync.go
+74
View File
@@ -0,0 +1,74 @@
package job
import (
"91porn-server/skd/job/elastic/esmedia"
"91porn-server/skd/job/elastic/esuser"
"sync"
"time"
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/skd/job/elastic/estag"
"91porn-server/skd/job/elastic/esvideo"
"91porn-server/skd/skdg"
)
var syncIsRuining = false
// start 定时任务 开启
func runSync() {
//防止重入
if syncIsRuining {
return
}
syncIsRuining = true
defer func() {
syncIsRuining = false
}()
doSync() //做统计业务
}
// 做统计业务
func doSync() {
defaultSyncDate, err := getDefaultSyncDate()
if err != nil {
log.Error("defaultStatDate invalid", log.E(err))
return
}
wg := sync.WaitGroup{}
wg.Add(5)
// 同步数据
{
common.Go(func() {
defer wg.Done()
esuser.RunSync(defaultSyncDate)
})
common.Go(func() {
defer wg.Done()
esvideo.RunSync(defaultSyncDate)
})
common.Go(func() {
defer wg.Done()
esmedia.RunSync(defaultSyncDate)
})
common.Go(func() {
defer wg.Done()
estag.RunSync(defaultSyncDate)
})
common.Go(func() {
defer wg.Done()
vidDate, err := time.ParseInLocation("2006-01-02", "2023-09-22", time.Local)
if err == nil {
defaultSyncDate = vidDate
}
// 同步短视频数据到REDIS
//rdvideo.RunSync(defaultSyncDate)
})
}
wg.Wait()
}
func getDefaultSyncDate() (time.Time, error) {
return time.ParseInLocation("2006-01-02", skdg.Conf.StartDate, time.Local)
}
skd/job/sync_im_users.go
+176
View File
@@ -0,0 +1,176 @@
package job
import (
"encoding/json"
"fmt"
"time"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/skd/skdg"
"91porn-server/services/srv_im"
"91porn-server/web/webg"
)
const (
phaseQueued = "queued"
phaseScanningDialogs = "scanning_chat_sessions"
phaseRegisteringUsers = "registering_users"
phaseAddingFriends = "adding_friends"
phaseDone = "done"
syncIMUsersTriggerKey = "im:sync_users:trigger"
syncIMUsersLockKey = "im:sync_users:lock"
syncIMUsersStatusKey = "im:sync_users:status"
syncIMUsersTriggerTTL = 10 * time.Minute
syncIMUsersLockTTL = 1 * time.Hour
)
// SyncIMUsersTaskStatus 旧私聊会话用户同步任务状态。
// 状态完全落在 Redis 里——web 进程写 trigger、读 status；skd 进程消费 trigger、执行任务、写 status。
type SyncIMUsersTaskStatus struct {
Running bool `json:"running"`
Phase string `json:"phase,omitempty"`
StartedAt *time.Time `json:"startedAt,omitempty"`
FinishedAt *time.Time `json:"finishedAt,omitempty"`
LastStats srv_im.SyncDialogUsersStats `json:"lastStats"`
LastError string `json:"lastError,omitempty"`
}
// imSyncRedis 拿当前进程可用的 Redis 客户端：
// skd 主进程用 skdg.Redis；web 主进程用 webg.Redis。
// 两端写的是同一个 Redis 实例，靠 key 协议互通。
func imSyncRedis() *redis.Client {
if skdg.Redis != nil {
return skdg.Redis
}
if webg.Redis != nil {
return webg.Redis
}
return nil
}
// RequestSyncIMUsers 由 web admin 接口调用：写 trigger key，由 skd 接管执行。
// 返回 accepted=true 表示成功排队；false 表示已有任务在跑或在排队。
func RequestSyncIMUsers() (accepted bool, status SyncIMUsersTaskStatus) {
status = GetSyncIMUsersStatus()
if status.Running {
return false, status
}
r := imSyncRedis()
if r == nil {
status.LastError = "redis unavailable"
return false, status
}
if existing, _ := r.Get(syncIMUsersTriggerKey); existing != nil && *existing != "" {
// 已经有 trigger 在排队，等 skd 消费
return false, status
}
if err := r.Set(syncIMUsersTriggerKey, "1", syncIMUsersTriggerTTL); err != nil {
status.LastError = err.Error()
return false, status
}
now := time.Now()
status = SyncIMUsersTaskStatus{
Phase: phaseQueued,
StartedAt: &now,
LastStats: srv_im.SyncDialogUsersStats{Enabled: true},
}
saveSyncIMUsersStatus(status)
return true, status
}
// GetSyncIMUsersStatus 读 Redis 里最新状态，任何进程都能调
func GetSyncIMUsersStatus() SyncIMUsersTaskStatus {
r := imSyncRedis()
if r == nil {
return SyncIMUsersTaskStatus{}
}
raw, err := r.Get(syncIMUsersStatusKey)
if err != nil || raw == nil || *raw == "" {
return SyncIMUsersTaskStatus{}
}
var status SyncIMUsersTaskStatus
if err := json.Unmarshal([]byte(*raw), &status); err != nil {
return SyncIMUsersTaskStatus{}
}
return status
}
// RunPendingSyncIMUsers 由 skd cron 定时调用：检测 trigger 并接管执行同步
func RunPendingSyncIMUsers() {
r := imSyncRedis()
if r == nil {
return
}
// 没有 trigger 就直接退
trig, _ := r.Get(syncIMUsersTriggerKey)
if trig == nil || *trig == "" {
return
}
// 加锁防止 skd 多副本同时跑
ok, err := r.Setnx_NewOK(syncIMUsersLockKey, 1, syncIMUsersLockTTL)
if err != nil || !ok {
return
}
defer func() { _, _ = r.Del(syncIMUsersLockKey) }()
// 消费 trigger，避免下个 tick 重复拉起
_, _ = r.Del(syncIMUsersTriggerKey)
now := time.Now()
status := SyncIMUsersTaskStatus{
Running: true,
Phase: phaseScanningDialogs,
StartedAt: &now,
LastStats: srv_im.SyncDialogUsersStats{Enabled: true},
}
saveSyncIMUsersStatus(status)
stats, runErr := runSyncIMUsersOnce(&status)
finished := time.Now()
status.Running = false
status.Phase = phaseDone
status.FinishedAt = &finished
status.LastStats = stats
if runErr != nil {
status.LastError = runErr.Error()
log.Error("RunPendingSyncIMUsers failed", log.E(runErr), log.Any("stats", stats))
} else {
status.LastError = ""
log.Info("RunPendingSyncIMUsers done", log.Any("stats", stats))
}
saveSyncIMUsersStatus(status)
}
func runSyncIMUsersOnce(status *SyncIMUsersTaskStatus) (stats srv_im.SyncDialogUsersStats, err error) {
defer func() {
if rec := recover(); rec != nil {
err = fmt.Errorf("sync panic: %v", rec)
}
}()
status.Phase = phaseRegisteringUsers
saveSyncIMUsersStatus(*status)
stats, err = srv_im.SyncDialogUsers()
status.Phase = phaseAddingFriends
status.LastStats = stats
saveSyncIMUsersStatus(*status)
return stats, err
}
func saveSyncIMUsersStatus(status SyncIMUsersTaskStatus) {
r := imSyncRedis()
if r == nil {
return
}
raw, err := json.Marshal(status)
if err != nil {
return
}
if err := r.Set(syncIMUsersStatusKey, string(raw), 0); err != nil {
log.Warn("saveSyncIMUsersStatus set failed", log.E(err))
}
}
skd/job/syncspiderdata/audiobookspider/method.go
+147
View File
@@ -0,0 +1,147 @@
package audiobookspider
import (
"fmt"
"time"
"91porn-server/common/httputil"
"91porn-server/common/log"
"91porn-server/models/v/audioanchormod"
"91porn-server/models/v/audiobookmod"
"91porn-server/skd/skdg"
)
func SyncDataFromSpider(data []AudioBook) error {
dataLen := len(data)
if dataLen == 0 {
return nil
}
now := time.Now()
la := make([]audiobookmod.AudioBookUpsertEdit, dataLen)
laa := make([]audioanchormod.EditSelector, dataLen)
for i := range data {
temp := audiobookmod.AudioBookUpsertEdit{
Title: data[i].Title,
Anchor: data[i].Announcer,
Cover: data[i].Cover,
FType: data[i].Category,
Summary: data[i].Desc,
Number: data[i].Id,
UpdatedAt: now,
}
temp.TotalEpisode = len(data[i].EpisodeList)
if data[i].Status == 1 {
temp.IsActive = true
} else {
temp.IsActive = false
}
if data[i].Announcer == "" {
temp.IsActive = false
}
temp.ContentSet = make([]audiobookmod.Content, len(data[i].EpisodeList))
for di := range data[i].EpisodeList {
temp.ContentSet[di] = audiobookmod.Content{
EpisodeNumber: di + 1,
ListenPermission: 0,
Name: data[i].EpisodeList[di].Name,
URL: data[i].EpisodeList[di].FileName,
MediaSize: data[i].EpisodeList[di].Size,
}
}
temp.TrialSet = make([]audiobookmod.Content, len(data[i].TrialEpisodeList))
for di := range data[i].TrialEpisodeList {
temp.TrialSet[di] = audiobookmod.Content{
EpisodeNumber: di + 1,
ListenPermission: 3,
Name: data[i].TrialEpisodeList[di].Name,
URL: data[i].TrialEpisodeList[di].FileName,
MediaSize: data[i].TrialEpisodeList[di].Size,
}
}
tempa := audioanchormod.EditSelector{
Name: data[i].Announcer,
Avatar: data[i].AnnouncerCover,
UpdatedAt: now}
la[i] = temp
laa[i] = tempa
}
log.Info("audiobook sync audiobookSpider data committing...")
if err := audiobookmod.InsertBulket(la); err != nil {
log.Info("audiobook sync audiobookSpider data commit failed.", log.E(err))
return err
}
if err := audioanchormod.InsertBulket(laa); err != nil {
log.Info("audiobook sync audiobookSpider data commit failed.", log.E(err))
return err
}
log.Info("audiobook sync audiobookSpider data committed.")
return nil
}
type Episode struct {
Name string `json:"name" bson:"name" ` //集数名称
Size int64 `json:"size" bson:"size" ` //本集mp3文件大小
FileName string `json:"fileName" bson:"fileName"` //小说文件
}
type AudioBook struct {
Id int64 `json:"id" bson:"_id"`
SpiderId int64 `json:"spiderId" bson:"spiderId"` //爬虫库的表Id
Timestamp int64 `json:"timestamp" bson:"timestamp" ` //时间戳
Title string `json:"title" bson:"title"` //标题
Announcer string `json:"announcer" bson:"announcer"` //播音员
Category string `json:"category" bson:"category"` //分类
Desc string `json:"desc" bson:"desc"` //描述
Cover string `json:"cover" bson:"cover" ` //封面
AnnouncerCover string `json:"announcerCover"` //播音员封面
EpisodeList []Episode `json:"episodeList" bson:"episodeList" ` //集数列表
TrialEpisodeList []Episode `json:"trialEpisodeList" bson:"trialEpisodeList"` //试听集数列表
Status int `json:"status" bson:"status"` //状态 1:上架 2:下架
CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间
UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt" ` //更新时间
}
// /api/stat/novel/audio/sync POST
type SyncAudioBookReq struct {
SyncSize int64 `json:"syncSize" binding:"required"`
StartTime time.Time `json:"startTime" binding:"required"`
}
type SyncAudioBookResp struct {
Data []AudioBook `json:"data"`
}
// todo skd
// 获取ES同步数据
func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []AudioBook, err error) {
p := SyncAudioBookReq{
SyncSize: 10000,
StartTime: start,
}
var msg struct {
Code int `json:"code"`
Data SyncAudioBookResp `json:"data"`
}
url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/audio/sync"
code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
if err != nil || code != 200 {
log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "POSTJsonWithJResp", err),
log.Any("start", start),
log.Any("url", end),
log.Any("code", code),
)
return
}
if msg.Code != 200 {
log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
log.Any("start", start),
log.Any("url", end),
log.Any("code", code),
log.Any("msg", msg),
)
return
}
data = msg.Data.Data
return
}
skd/job/syncspiderdata/audiobookspider/sync.go
+56
View File
@@ -0,0 +1,56 @@
package audiobookspider
import (
"fmt"
"time"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
)
func RunSync(defaultDate time.Time) {
log.Info("Audiobook sync audiobookSpider data start...")
defer log.Info("Audiobook sync audiobookSpider data end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncAudiobookJob, defaultDate)
if err != nil {
log.Error("Audiobook sync audiobookSpider data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now().Add(-time.Minute * 10)
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("Audiobook sync audiobookSpider commit err! ", log.E(err))
return
}
if len(data) > 0 {
lastSyncAt = data[len(data)-1].UpdatedAt
} else {
lastSyncAt = end
}
}
}
func commit(data []AudioBook, syncTime time.Time) error {
if err := SyncDataFromSpider(data); err != nil {
return fmt.Errorf("Audiobook sync audiobookSpider commit audiobook data SyncDataFromSpider failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncAudiobookJob, syncTime); err != nil {
return fmt.Errorf("Audiobook sync audiobookSpider commit audiobook data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/syncspiderdata/fictionspider/method.go
+99
View File
@@ -0,0 +1,99 @@
package fictionspider
import (
"fmt"
"time"
"91porn-server/common/httputil"
"91porn-server/common/log"
"91porn-server/models/v/fictionmod"
"91porn-server/skd/skdg"
)
func SyncDataFromSpider(data []Novel) error {
now := time.Now()
if len(data) == 0 {
return nil
}
ll := make([]fictionmod.FictionUpsert, len(data))
for i := range data {
temp := fictionmod.FictionUpsert{
Title: data[i].Title,
FType: data[i].Type,
Summary: data[i].Desc,
ContentURL: data[i].FileName,
Number: data[i].Id,
UpdatedAt: now,
}
if data[i].Status == 1 {
temp.IsActive = true
} else {
temp.IsActive = false
}
ll[i] = temp
}
log.Info("fiction sync fictionSpider data committing...")
if err := fictionmod.InsertBulket(ll); err != nil {
log.Info("fiction sync fictionSpider data commit failed.")
return err
}
log.Info("fiction sync fictionSpider data committed.")
return nil
}
type Novel struct {
Id int64 `json:"id" bson:"_id"`
SpiderId int64 `json:"spiderId" bson:"spiderId"` //爬虫库的表Id
Title string `json:"title" bson:"title" ` //小说标题
Type string `json:"type" bson:"type" ` //小说类型
Desc string `json:"desc" bson:"desc" ` //描述
FileName string `json:"fileName" bson:"fileName" ` //小说文件
Timestamp int64 `json:"timestamp" bson:"timestamp" ` //时间戳
Status int `json:"status" bson:"status"` //状态 1:上架 2:下架
CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间
UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"` //更新时间
}
// /api/stat/novel/sync POST
type SyncNovelReq struct {
SyncSize int64 `json:"syncSize" binding:"required"`
StartTime time.Time `json:"startTime" binding:"required"`
}
type SyncNovelResp struct {
Data []Novel `json:"data"`
}
// todo skd
// 获取ES同步数据
func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []Novel, err error) {
p := SyncNovelReq{
SyncSize: 10000,
StartTime: start,
}
var msg struct {
Code int `json:"code"`
Data SyncNovelResp `json:"data"`
}
url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/sync"
code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
if err != nil || code != 200 {
log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
log.Any("start", start),
log.Any("url", end),
log.Any("code", code),
)
return
}
if msg.Code != 200 {
log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
log.Any("start", start),
log.Any("url", end),
log.Any("code", code),
log.Any("msg", msg),
)
return
}
data = msg.Data.Data
return
}
skd/job/syncspiderdata/fictionspider/sync.go
+56
View File
@@ -0,0 +1,56 @@
package fictionspider
import (
"fmt"
"time"
"91porn-server/common/log"
"91porn-server/common/timeutil/timerange"
"91porn-server/models/s/statrecordmod"
)
func RunSync(defaultDate time.Time) {
log.Info("fiction sync fictionSpider data start...")
defer log.Info("fiction sync fictionSpider data end.")
// 获取上次数据同步的记录时间
lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncfictionJob, defaultDate)
if err != nil {
log.Error("fiction sync fictionSpider data RunSync get last record time failed", log.E(err))
return
}
var now = time.Now().Add(-time.Minute * 10)
var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
for recentMinute.After(lastSyncAt) {
var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
if end.After(recentMinute) {
end = recentMinute
}
var subTimeRange = timerange.TimeRange{
Head: lastSyncAt,
Tail: end,
}
data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
if err != nil {
return
}
if err = commit(data, subTimeRange.Tail); err != nil {
log.Error("fiction sync fictionSpider commit err! ", log.E(err))
return
}
if len(data) > 0 {
lastSyncAt = data[len(data)-1].UpdatedAt
} else {
lastSyncAt = end
}
}
}
func commit(data []Novel, syncTime time.Time) error {
if err := SyncDataFromSpider(data); err != nil {
return fmt.Errorf("fiction sync fictionSpider commit fiction data SyncDataFromSpider failed!, err:%+v\n", err)
}
if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncfictionJob, syncTime); err != nil {
return fmt.Errorf("fiction sync fictionSpider commit fiction data UpsertOneTrans failed!, err:%+v\n", err)
}
return nil
}
skd/job/user/privilege.go
+16
View File
@@ -0,0 +1,16 @@
package user
import (
"91porn-server/common/log"
"91porn-server/models/v/advanceordermod"
)
// ResetLimit 重置预售卡预付权益
func ResetLimit() {
err := advanceordermod.ResetToday()
if err != nil {
log.Error("重置预售卡预付权益错误", log.E(err))
}
log.Info("--重置预售卡预付权益完成！")
}
skd/main.go
+161
View File
@@ -0,0 +1,161 @@
package main
import (
"91porn-server/common/cachev2"
"91porn-server/common/dataReport"
"context"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"91porn-server/common"
"91porn-server/common/conf"
"91porn-server/common/db"
"91porn-server/common/laosiji"
"91porn-server/common/log"
recommendqueue "91porn-server/common/shortrecommend"
"91porn-server/common/tg"
"91porn-server/models"
"91porn-server/models/Init"
"91porn-server/skd/job"
"91porn-server/skd/service"
"91porn-server/skd/skdg"
)
func init() {
cfg := skdg.GlobalConfig{}
log.Info("load config start...")
if err := conf.LoadJSON("config/skd.json", &cfg); err != nil {
log.Error("startUp get config error: %v", log.E(err))
panic(err)
}
skdg.Conf = &cfg
log.Info("load config end..")
log.Init(log.Options{Level: skdg.Conf.Log.Level})
log.Info("daichong conf Domain", log.Any("donmain", skdg.Conf.DaiChong.Domain))
}
var httpServer *http.Server
func main() {
time.Local, _ = time.LoadLocation("Asia/Shanghai")
common.Go(func() {
startUp()
httpServerStart()
job.Start()
})
waitQuit()
}
func startUp() {
if skdg.Conf.Base.Env == "prod" {
skdg.Bot = tg.New(tg.TianTian_Bot, fmt.Sprintf("短视频[env:%s][Skd-Server]", skdg.Conf.Base.Env))
common.SetPanicHandler(func(s string) {
_ = skdg.Bot.Send(tg.Msg{ChatId: tg.ChatIdServerTeam, Text: `\[SKD-Server\]` + s})
})
}
ds := db.InitDS(initMongodbParam(models.RegisterPool))
if len(ds) > 0 {
for k, v := range ds {
switch k {
case models.VideoDb:
skdg.VideoDB = v
case models.LogDb:
skdg.LogDB = v
case models.StatDb:
skdg.StatDB = v
}
}
}
dataReport.Init(dataReport.Config{
ApiUrl: skdg.Conf.DataReport.Url,
})
laosiji.InitConfig(laosiji.Config{
AppID: skdg.Conf.LSJ.AppID,
APIKey: skdg.Conf.LSJ.APIKey,
APIUrl: skdg.Conf.LSJ.APIUrl,
ImageYuan: skdg.Conf.LSJ.ImageYuan,
NoticeURL: skdg.Conf.LSJ.NoticeURL,
})
skdg.InitRedis()
if err := recommendqueue.SetKeyTTLHours(skdg.Conf.ShortRecommendKeyTTL()); err != nil {
log.Error("short recommend key TTL config invalid, use default",
log.E(err), log.Any("configuredHours", skdg.Conf.ShortRecommendKeyTTL()))
_ = recommendqueue.SetKeyTTLHours(72)
}
// 初始化缓存操作中间件
cachev2.Init(skdg.Redis, log.ZapLog)
skdg.InitElastic()
skdg.InitStatic()
Init.InitMongo()
Init.ESInit()
if skdg.Conf.Switch.NotifyExpire {
service.DoRegister()
}
// service.RobotPublishComment()
// service.Init()
// service.VideoInfo.FullUpdate()
}
func waitQuit() {
//graceful exit
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT)
<-ch
log.Info("Recv Signal, Shutdown Server Start...")
log.Info("Pleace Wait Job Stop...")
job.Stop()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if httpServer != nil {
if err := httpServer.Shutdown(ctx); err != nil {
log.Error("Server Shutdown error", log.E(err))
}
}
db.CloseDS()
if err := skdg.RedisClose(); err != nil {
log.Error("redis close error", log.E(err))
return
}
log.Warn("Shutdown Server OK...")
}
func initMongodbParam(registerPool []models.Register) ([]db.DBmap, []db.Register) {
pri := db.DBmap{
Key: models.VideoDb,
URL: skdg.Conf.Mongo.VideoDbUrl,
}
sta := db.DBmap{
Key: models.StatDb,
URL: skdg.Conf.Mongo.StatDbUrl,
}
log := db.DBmap{
Key: models.LogDb,
URL: skdg.Conf.Mongo.LogDbUrl,
}
dbgister := make([]db.Register, len(registerPool))
for i, v := range registerPool {
dbgister[i] = db.Register{Key: v.Key, Table: v.Table}
}
dbmaps := []db.DBmap{pri, sta, log}
return dbmaps, dbgister
}
func httpServerStart() {
common.Go(func() {
//开始http
port := skdg.Conf.Base.Port
httpServer = &http.Server{
Addr: fmt.Sprintf(":%d", port),
}
log.Info("Start Listen ...", log.Any("port", port))
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatal("server listen err", log.E(err))
}
// Block Here
log.Info("httpServer quit...")
})
}
skd/once/main.go
+77
View File
@@ -0,0 +1,77 @@
package main
import (
"sync"
"91porn-server/common/conf"
"91porn-server/common/db"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/Init"
"91porn-server/skd/skdg"
)
/*
本函数会将mongo库内的video_info数据, audiobook数据, fiction数据写入es
一般在项目部署, 数据导入完成以后运行一次便可
*/
func init() {
cfg := skdg.GlobalConfig{}
log.Info("load config start...")
if err := conf.LoadJSON("config/skd.json", &cfg); err != nil {
log.Error("startUp get config error: %v", log.E(err))
panic(err)
}
skdg.Conf = &cfg
log.Info("load config end..")
log.Init(log.Options{Level: skdg.Conf.Log.Level})
log.Info("daichong conf Domain", log.Any("donmain", skdg.Conf.DaiChong.Domain))
}
func main() {
ds := db.InitDS(initMongodbParam(models.RegisterPool))
if len(ds) > 0 {
for k, v := range ds {
switch k {
case models.VideoDb:
skdg.VideoDB = v
case models.LogDb:
skdg.LogDB = v
case models.StatDb:
skdg.StatDB = v
}
}
}
skdg.InitElastic()
Init.InitMongo()
log.Info("所有同步开始")
wg := new(sync.WaitGroup)
wg.Add(3)
go syncAudioBook(wg)
go syncVideoInfo(wg)
go syncFiction(wg)
wg.Wait()
log.Info("所有同步结束")
}
func initMongodbParam(registerPool []models.Register) ([]db.DBmap, []db.Register) {
pri := db.DBmap{
Key: models.VideoDb,
URL: skdg.Conf.Mongo.VideoDbUrl,
}
sta := db.DBmap{
Key: models.StatDb,
URL: skdg.Conf.Mongo.StatDbUrl,
}
log := db.DBmap{
Key: models.LogDb,
URL: skdg.Conf.Mongo.LogDbUrl,
}
dbgister := make([]db.Register, len(registerPool))
for i, v := range registerPool {
dbgister[i] = db.Register{Key: v.Key, Table: v.Table}
}
dbmaps := []db.DBmap{pri, sta, log}
return dbmaps, dbgister
}
skd/once/syncAudioBook.go
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"sync"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/audiobookmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
func syncAudioBook(wg *sync.WaitGroup) {
log.Info("audiobook sync start...")
for i := int64(0); ; i++ {
opts := options.Find().SetSkip(i * 1000).SetLimit(1000).SetSort(bson.D{{Key: "_id", Value: 1}})
au, err := audiobookmod.GetListByCond(bson.M{}, opts)
if err != nil {
panic(err)
}
var source = elastic.M{}
for i := range au {
tmp := au[i]
source[au[i].ID.Hex()] = tmp
}
if err := skdg.VideoES.Bulk(models.ESInfoAudioBookTable, source); err != nil {
panic(err)
}
auLen := len(au)
if len(au) < 1000 {
break
}
log.Info("audiobook sync:", log.Any("start", i*1000), log.Any("end", i*1000+int64(auLen)))
}
log.Info("audiobook sync complete...")
wg.Done()
}
skd/once/syncFiction.go
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"sync"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/fictionmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
func syncFiction(wg *sync.WaitGroup) {
log.Info("fiction sync start...")
for i := int64(0); ; i++ {
opts := options.Find().SetSkip(i * 1000).SetLimit(1000).SetSort(bson.D{{Key: "_id", Value: 1}})
fc, err := fictionmod.GetListByCond(bson.M{}, opts)
if err != nil {
panic(err)
}
var source = elastic.M{}
for i := range fc {
tmp := fc[i]
source[fc[i].ID.Hex()] = tmp
}
if err := skdg.VideoES.Bulk(models.ESInfoFictionTable, source); err != nil {
panic(err)
}
fcLen := len(fc)
if fcLen < 1000 {
break
}
log.Info("fiction sync:", log.Any("start", i*1000), log.Any("end", i*1000+int64(fcLen)))
}
log.Info("fiction sync complete...")
wg.Done()
}
skd/once/syncVideoInfo.go
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"sync"
"91porn-server/common/elastic"
"91porn-server/common/log"
"91porn-server/models"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
func syncVideoInfo(wg *sync.WaitGroup) {
log.Info("video sync start...")
for i := int64(0); ; i++ {
opts := options.Find().SetSkip(i * 1000).SetLimit(1000).SetSort(bson.D{{Key: "_id", Value: 1}})
vd, err := vidmod.GetListByCond(bson.M{}, opts)
if err != nil {
panic(err)
}
var source = elastic.M{}
for i := range vd {
tmp := vd[i]
source[vd[i].ID.Hex()] = tmp
}
if err := skdg.VideoES.Bulk(models.ESInfoVideoTable, source); err != nil {
panic(err)
}
vdLen := len(vd)
if vdLen < 1000 {
break
}
log.Info("video sync:", log.Any("start", i*1000), log.Any("end", i*1000+int64(vdLen)))
}
log.Info("video sync complete...")
wg.Done()
}
skd/service/chatrobot.go
+218
View File
@@ -0,0 +1,218 @@
package service
import (
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/common/stderr"
"91porn-server/models/commod"
"91porn-server/models/v/modulesectionmod"
"91porn-server/models/v/modulevidmod"
"91porn-server/models/v/tagmod"
"91porn-server/models/v/vidmod"
"91porn-server/models/v/vidtimeonlinemod"
"91porn-server/web/service/video_media_service"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"strconv"
"strings"
"time"
)
type vidTimeOnlineInfo struct {
VidId primitive.ObjectID `json:"vidId"`
VidType string `json:"vidType"`
SourceURL string `json:"sourceURL"`
ReviewAccount string `json:"reviewAccount"`
}
var vidTimeOnlineIsRuining = false
// 视频自动上架
func VidTimeOnline() {
common.Go(func() {
log.Info("video time online start...")
if vidTimeOnlineIsRuining {
log.Info("video time online already executed")
return
}
vidTimeOnlineIsRuining = true
defer func() {
vidTimeOnlineIsRuining = false
}()
log.Info("video time online execute")
handleVidTimeOnline()
log.Info("video time online end...")
})
}
func handleVidTimeOnline() {
//每次取十条数据，定期上线
vidInfs, err := vidtimeonlinemod.GetByPage(commod.Page{PageNumber: 1, PageSize: 100})
if err != nil {
log.Error("video time online eonlinemod.GetByPage err", log.E(err))
return
}
if len(vidInfs) == 0 {
log.Info("video time online vidInfs == 0")
return
}
//上架
resIds := passVids(vidInfs)
//从自动上架表中移除已经上架的帖子数据
if len(resIds) > 0 {
if err = vidtimeonlinemod.DeleteByVids(resIds); err != nil {
log.Warn("video time online vidtimeonlinemod.DeleteByVids err", log.Any("err", err))
return
}
}
log.Info("video time online resIds", log.Any("resIds", resIds))
}
// passVids 批量自动上架
func passVids(vidIds []vidtimeonlinemod.VidTimeOnlineModel) (resIds []primitive.ObjectID) {
var uploadRecord []vidmod.UploadMediaVideo
var records []modulevidmod.SectionVideo
for _, vid := range vidIds {
//非图集才需要同步
isCover := vid.VidType == vidmod.COVER || vid.VidType == vidmod.PIC
log.Info("vid time online", log.Any("isCover", isCover), log.Any("vid", vid))
// 查询视频信息
videoInfo, err := vidmod.GetVideoInfo(vid.VidId.Hex())
if err != nil {
log.Warn("vid time online passVids vidmod.GetVideoInfo err,", log.Any("err", err), log.Any("vid", vid.VidId))
return resIds
}
log.Info("vid time online", log.Any("status", videoInfo.Status), log.Any("MDSID", videoInfo.MDSID), log.Any("videoInfo.Filename", videoInfo.Filename))
// 媒资库的不做任何校验，因为给过来的这部分数据的sourceID有问题
if videoInfo.MDSID == "" && !isCover && !strings.Contains(videoInfo.Filename, "mzk_") {
if code := SyncFileFromFs(vid.VidId.Hex()); code != stderr.UpLoadFileComplete {
log.Warn("vid time online passVids syncFileFromFs err,", log.Any("code", code), log.Any("vid", vid.VidId))
continue
}
}
tagSort := make(bson.M, len(videoInfo.Tags))
for _, v := range videoInfo.Tags {
tagSort[v.Hex()] = 0
}
if videoInfo.Status == 1 {
resIds = append(resIds, videoInfo.ID)
continue
}
if (videoInfo.NewsType == vidmod.SP || videoInfo.NewsType == vidmod.SHORT) && !strings.Contains(videoInfo.Filename, "mzk_") {
var width, height int
resolutions := strings.Split(videoInfo.Resolution, "*")
if len(resolutions) >= 2 {
width, _ = strconv.Atoi(resolutions[0])
height, _ = strconv.Atoi(resolutions[1])
}
if len(resolutions) != 2 {
resolutions = strings.Split(videoInfo.Resolution, "x")
if len(resolutions) >= 2 {
width, _ = strconv.Atoi(resolutions[0])
height, _ = strconv.Atoi(resolutions[1])
}
}
var tags string
if len(videoInfo.Tags) > 0 {
for _, v := range videoInfo.Tags {
tagData, err := tagmod.FindOneTagByID(v)
if err == nil {
if tags != "" {
tags += "," + tagData.TagName
} else {
tags += tagData.TagName
}
}
}
}
uploadRecord = append(uploadRecord, vidmod.UploadMediaVideo{
FsResourceId: videoInfo.SourceID,
HashId: videoInfo.MD5,
Title: videoInfo.Title,
Length: int(videoInfo.PlayTime),
FileSize: videoInfo.Size,
M3u8Src: videoInfo.SourceURL,
Height: height,
Width: width,
CoverImage: videoInfo.Cover,
TagsText: tags,
})
}
video, err := vidmod.PassVidsOnlineTime(vid.VidId, 1, "", vid.ReviewAccount, vid.OnlineTime, tagSort)
if err != nil {
log.Warn("vid time online passVids vidmod.PassVids err,", log.Any("err", err), log.Any("vid", video.ID))
return resIds
}
if !vid.SectionId.IsZero() {
// 获取专题
section, err := modulesectionmod.GetBySectionByID(vid.SectionId)
if err != nil {
log.Error("vid time online modulesectionmod.GetBySectionByID fail", log.Any("vid.SectionId", vid.SectionId), log.E(err))
continue
}
if !section.ID.IsZero() {
// 判断当前专题下是否已经有该记录,没有则添加
sectionVids, err := modulevidmod.GetBySectionIDAndVids(section.ID, []primitive.ObjectID{vid.VidId})
if err != nil {
log.Error("vid time online modulevidmod.GetBySectionIDAndVids( fail", log.Any("vid.SectionId", vid.SectionId), log.Any("vid.VidId", vid.VidId), log.E(err))
continue
}
if len(sectionVids) == 0 {
records = append(records, modulevidmod.SectionVideo{
SectionID: section.ID,
VideoID: vid.VidId,
VideoReviewedAt: vid.OnlineTime,
})
}
// 修改帖子的mId
update := bson.M{}
update["updatedAt"] = time.Now()
update["mId"] = section.SubModuleID.Hex()
vidmod.UpdateOneByID(vid.VidId, update)
}
}
resIds = append(resIds, video.ID)
}
if len(records) > 0 {
err := modulevidmod.InsertMany(records)
if err != nil {
log.Error("vid time online AddVidTimeOnlineInfo moduleVidMod InsertMany error", log.Any("records", records), log.Any("err", err))
return resIds
}
}
log.Info("vid time online", log.Any("uploadRecord", uploadRecord))
// 处理上传媒资库
if len(uploadRecord) > 0 {
var err error
var sourceIDs []string
var errSourceIDs []string
for _, info := range uploadRecord {
if info.Length > 300 {
err = video_media_service.UploadSkdVideoMedia(info)
} else {
err = video_media_service.UploadSkdShortVideoMedia(info)
}
if err != nil {
errSourceIDs = append(errSourceIDs, info.FsResourceId)
log.Info(fmt.Sprintf("vid time online UploadVideoMedia err%v;errSourceIDs%v", err, errSourceIDs))
continue
}
sourceIDs = append(sourceIDs, info.FsResourceId)
}
if err != nil {
log.Info(fmt.Sprintf("vid time online UploadVideoMedia err%v;errSourceIDs%v", err, errSourceIDs))
return resIds
}
log.Info(fmt.Sprintf("vid time online UploadVideoMedia sucess sourceIDs%v", sourceIDs))
}
return resIds
}
skd/service/contentreviewser/acg.go
+68
View File
@@ -0,0 +1,68 @@
package contentreviewser
import (
"91porn-server/common/log"
"91porn-server/models/v/contentreviewmod"
"91porn-server/models/v/mediacontentmod"
"91porn-server/models/v/mediamod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// acgScanner ACG 扫描策略：每个 Media 可能产生 1 个 ACG_MEDIA + N 个 ACG_CONTENT 命中
// 仅匹配 title / summary / 章节名 / 章节内容（不检测 tag）
type acgScanner struct{}
func (s *acgScanner) CountTotal() (int64, error) {
return mediamod.CountForReview()
}
func (s *acgScanner) ScanBatch(taskID, lastID primitive.ObjectID, terms []string) (*ScanResult, error) {
batch, err := mediamod.FindForReviewBatch(lastID, scanBatchSize)
if err != nil {
return nil, err
}
if len(batch) == 0 {
return &ScanResult{Done: true}, nil
}
issues := make([]*contentreviewmod.ReviewIssue, 0)
for _, m := range batch {
issues = append(issues, collectMediaIssues(taskID, m, terms)...)
}
return &ScanResult{
Issues: issues,
Checked: int64(len(batch)),
NextID: batch[len(batch)-1].ID,
}, nil
}
// collectMediaIssues 检测一个 Media 自身及其全部 MediaContent
func collectMediaIssues(taskID primitive.ObjectID, m *mediamod.Media, terms []string) []*contentreviewmod.ReviewIssue {
out := make([]*contentreviewmod.ReviewIssue, 0)
mb := newIssueBuilder(taskID, contentreviewmod.IssueTargetACGMedia, m.ID).
MatchTitle("标题", m.Title, terms).
MatchContent("内容", m.Summary, terms)
if mb.HasHits() {
out = append(out, mb.Build())
}
contents, err := mediacontentmod.QueryAllList(bson.M{"mediaId": m.ID, "isActive": true})
if err != nil {
log.Warn("query media contents fail", log.Any("mid", m.ID), log.E(err))
return out
}
for _, c := range contents {
cb := newIssueBuilder(taskID, contentreviewmod.IssueTargetACGContent, c.ID).
WithMediaID(m.ID).
MatchTitle("章节名", c.Name, terms).
MatchContent("章节内容", c.Text, terms)
if cb.HasHits() {
out = append(out, cb.Build())
}
}
return out
}
skd/service/contentreviewser/issue_builder.go
+92
View File
@@ -0,0 +1,92 @@
package contentreviewser
import (
"fmt"
"strings"
"91porn-server/models/v/contentreviewmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// issueBuilder 用 Builder 模式构建 ReviewIssue：
// 通过链式调用 MatchXxx 收集各字段的命中，HasHits/Build 在所有匹配后调用
type issueBuilder struct {
issue *contentreviewmod.ReviewIssue
hitSet map[string]struct{}
descLines []string
}
func newIssueBuilder(taskID primitive.ObjectID, targetType contentreviewmod.IssueTargetType, targetID primitive.ObjectID) *issueBuilder {
return &issueBuilder{
issue: &contentreviewmod.ReviewIssue{
TaskID: taskID,
TargetType: targetType,
TargetID: targetID,
},
hitSet: make(map[string]struct{}),
}
}
// WithMediaID 设置关联的 MediaID（仅 ACG_CONTENT 需要）
func (b *issueBuilder) WithMediaID(id primitive.ObjectID) *issueBuilder {
b.issue.MediaID = id
return b
}
// MatchTitle 匹配标题字段；命中时存入高亮版本并累计描述
// label 是描述文案中的字段名（如"标题"、"章节名"）
func (b *issueBuilder) MatchTitle(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.Title = hl
b.addHits(label, hits)
}
return b
}
func (b *issueBuilder) MatchContent(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.Content = hl
b.addHits(label, hits)
}
return b
}
func (b *issueBuilder) MatchTags(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.Tags = hl
b.addHits(label, hits)
}
return b
}
func (b *issueBuilder) MatchRichText(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.RichText = hl
b.addHits(label, hits)
}
return b
}
// HasHits 是否至少有一个字段命中
func (b *issueBuilder) HasHits() bool {
return len(b.hitSet) > 0
}
// Build 收尾：拼装 Description 与去重后的 HitWords
func (b *issueBuilder) Build() *contentreviewmod.ReviewIssue {
hits := make([]string, 0, len(b.hitSet))
for h := range b.hitSet {
hits = append(hits, h)
}
b.issue.HitWords = hits
b.issue.Description = strings.Join(b.descLines, "\n")
return b.issue
}
func (b *issueBuilder) addHits(label string, hits []string) {
b.descLines = append(b.descLines, fmt.Sprintf("%s命中 %s", label, strings.Join(hits, "、")))
for _, h := range hits {
b.hitSet[h] = struct{}{}
}
}
skd/service/contentreviewser/matcher.go
+114
View File
@@ -0,0 +1,114 @@
package contentreviewser
import (
"sort"
"strings"
"91porn-server/models/v/contentreviewmod"
)
const (
markPrefix = "<mark>"
markSuffix = "</mark>"
)
// allTerms 把快照展开为一组待匹配的词
func allTerms(snaps []contentreviewmod.SensitiveWordSnap) []string {
seen := make(map[string]struct{})
out := make([]string, 0, len(snaps))
for _, s := range snaps {
if s.Word == "" {
continue
}
if _, ok := seen[s.Word]; ok {
continue
}
seen[s.Word] = struct{}{}
out = append(out, s.Word)
}
// 长词优先，避免短词先替换破坏长词的匹配
sort.Slice(out, func(i, j int) bool {
return len(out[i]) > len(out[j])
})
return out
}
// findHits 返回 input 中命中的词（去重，保持长→短顺序）
func findHits(input string, terms []string) []string {
if input == "" || len(terms) == 0 {
return nil
}
hits := make([]string, 0)
seen := make(map[string]struct{})
for _, t := range terms {
if t == "" {
continue
}
if _, ok := seen[t]; ok {
continue
}
if strings.Contains(input, t) {
seen[t] = struct{}{}
hits = append(hits, t)
}
}
return hits
}
// highlight 用 <mark>...</mark> 包裹命中的词
// hits 已按长度降序；为避免再次包裹已 mark 的内容，分段替换
func highlight(input string, hits []string) string {
if input == "" || len(hits) == 0 {
return input
}
result := input
for _, h := range hits {
if h == "" {
continue
}
result = replaceNotInMark(result, h)
}
return result
}
// replaceNotInMark 在 input 中把 term 全部替换为 <mark>term</mark>，
// 但跳过已经包在 <mark>...</mark> 内的片段（避免嵌套）
func replaceNotInMark(input, term string) string {
if term == "" {
return input
}
var b strings.Builder
i := 0
for i < len(input) {
// 已经在 mark 块里，整段原样输出
if strings.HasPrefix(input[i:], markPrefix) {
end := strings.Index(input[i:], markSuffix)
if end < 0 {
b.WriteString(input[i:])
return b.String()
}
b.WriteString(input[i : i+end+len(markSuffix)])
i += end + len(markSuffix)
continue
}
if strings.HasPrefix(input[i:], term) {
b.WriteString(markPrefix)
b.WriteString(term)
b.WriteString(markSuffix)
i += len(term)
continue
}
b.WriteByte(input[i])
i++
}
return b.String()
}
// matchAndHighlight 在 input 上一步到位：返回命中词列表 + 高亮后的文本
func matchAndHighlight(input string, terms []string) ([]string, string) {
hits := findHits(input, terms)
if len(hits) == 0 {
return nil, input
}
return hits, highlight(input, hits)
}
skd/service/contentreviewser/runner.go
+129
View File
@@ -0,0 +1,129 @@
package contentreviewser
import (
"fmt"
"runtime/debug"
"sync/atomic"
"91porn-server/common/log"
"91porn-server/models/v/contentreviewmod"
"91porn-server/models/v/sensitivewordmod"
"91porn-server/skd/service/export_task"
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
)
// tgChatID 内容审查任务完成通知群（与导出任务复用同一群）
const tgChatID int64 = -1003399433452
var runningFlag int32 // 进程内互斥：0=空闲 1=运行中
// TryRunPendingTask 调度入口，由 skd 的 cron 定时调用
// skd 单进程，仅需 atomic 互斥防止 cron tick 重入；每次只挑一个最早的任务处理
// 取到 pending 视为首跑；取到心跳超时的 running 视为续跑，从 lastTargetId 继续
func TryRunPendingTask() {
if !atomic.CompareAndSwapInt32(&runningFlag, 0, 1) {
return
}
defer atomic.StoreInt32(&runningFlag, 0)
task, err := contentreviewmod.PickNextTask()
if err != nil {
log.Warn("content review: pick next fail", log.E(err))
return
}
if task == nil {
return
}
if task.ID.IsZero() {
return
}
isResume := task.Status == contentreviewmod.TaskStatusRunning
if err := contentreviewmod.AcquireTask(task.ID, isResume); err != nil {
log.Warn("content review: acquire task fail", log.Any("taskId", task.ID), log.E(err))
return
}
if isResume {
log.Info("content review: resume stale task",
log.Any("taskId", task.ID), log.Any("lastTargetId", task.LastTargetID))
}
executeTask(task)
}
// executeTask 真正执行扫描
func executeTask(task *contentreviewmod.ReviewTask) {
defer func() {
if r := recover(); r != nil {
stack := string(debug.Stack())
log.Error("content review task panic",
log.Any("taskId", task.ID), log.Any("panic", r), log.Any("stack", stack))
_ = contentreviewmod.MarkFailed(task.ID, fmt.Sprintf("panic: %v", r))
}
}()
log.Info("content review task start",
log.Any("taskId", task.ID), log.Any("type", task.TaskType))
checked, issues, err := runScan(task)
if err != nil {
log.Error("content review task fail",
log.Any("taskId", task.ID), log.E(err))
_ = contentreviewmod.MarkFailed(task.ID, err.Error())
return
}
if err := contentreviewmod.MarkFinished(task.ID, checked, issues); err != nil {
log.Error("content review mark finished fail", log.E(err))
return
}
// 发送 TG
if sendTGNotify(task, checked, issues) {
_ = contentreviewmod.MarkTgSent(task.ID)
}
log.Info("content review task done",
log.Any("taskId", task.ID), log.Any("checked", checked), log.Any("issues", issues))
}
func sendTGNotify(task *contentreviewmod.ReviewTask, checked, issues int64) bool {
if export_task.Bot == nil {
log.Warn("content review: tg bot not initialized, skip notify")
return false
}
typeName := "视频帖子"
if task.TaskType == contentreviewmod.TaskTypeACG {
typeName = "ACG"
}
operator := task.Operator
if operator == "" {
operator = "-"
}
text := fmt.Sprintf(
"【%s 内容检测任务完成】\n操作人: %s\n类型: %s\n任务ID: %s\n总记录: %d\n已检测: %d\n命中: %d",
export_task.GetProName(), operator, typeName, task.ID.Hex(), task.TotalCount, checked, issues,
)
if _, err := export_task.Bot.Send(tgbotapi.NewMessage(tgChatID, text)); err != nil {
log.Warn("content review: send tg fail", log.Any("taskId", task.ID), log.E(err))
return false
}
return true
}
// BuildSnapshot 拉取当前启用的敏感词作为快照（web 创建任务时复用）
func BuildSnapshot() ([]contentreviewmod.SensitiveWordSnap, error) {
enabled := sensitivewordmod.StatusEnabled
list, err := sensitivewordmod.FindAll(&sensitivewordmod.ListReq{Status: &enabled})
if err != nil {
return nil, err
}
out := make([]contentreviewmod.SensitiveWordSnap, 0, len(list))
for _, w := range list {
out = append(out, contentreviewmod.SensitiveWordSnap{
Word: w.Word,
Category: w.Category,
})
}
return out, nil
}
skd/service/contentreviewser/scanner.go
+95
View File
@@ -0,0 +1,95 @@
package contentreviewser
import (
"fmt"
"91porn-server/common/log"
"91porn-server/models/v/contentreviewmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const scanBatchSize = 100
// ScanResult 一批扫描的产出
type ScanResult struct {
Issues []*contentreviewmod.ReviewIssue // 本批生成的命中记录
Checked int64 // 本批实际扫描的目标数量
NextID primitive.ObjectID // 下一批的游标
Done bool // true 表示已扫完，runScan 退出循环
}
// Scanner 内容审查扫描策略；具体任务类型实现各自的 ScanBatch
type Scanner interface {
CountTotal() (int64, error)
ScanBatch(taskID, lastID primitive.ObjectID, terms []string) (*ScanResult, error)
}
// newScanner Strategy 工厂：根据任务类型返回具体扫描器
func newScanner(taskType string) (Scanner, error) {
switch taskType {
case contentreviewmod.TaskTypeVideo:
return &videoScanner{}, nil
case contentreviewmod.TaskTypeACG:
return &acgScanner{}, nil
default:
return nil, fmt.Errorf("unsupported task type: %s", taskType)
}
}
// runScan Template Method：把"分页游标 + 进度回写 + 批量插入"骨架抽出，差异由 Scanner 提供
// 支持从 task.LastTargetID 断点续跑
func runScan(task *contentreviewmod.ReviewTask) (int64, int64, error) {
scanner, err := newScanner(task.TaskType)
if err != nil {
return task.CheckedCount, task.IssueCount, err
}
terms := allTerms(task.SnapshotWords)
total := task.TotalCount
if total <= 0 {
t, err := scanner.CountTotal()
if err != nil {
return task.CheckedCount, task.IssueCount, fmt.Errorf("count total: %w", err)
}
total = t
}
// 同步到内存对象，便于 executeTask 拼 TG 消息时正确取值
task.TotalCount = total
lastID := task.LastTargetID
checked := task.CheckedCount
issues := task.IssueCount
flushProgress(task.ID, lastID, checked, issues, total)
for {
result, err := scanner.ScanBatch(task.ID, lastID, terms)
if err != nil {
return checked, issues, fmt.Errorf("scan batch: %w", err)
}
if result.Done {
break
}
if len(result.Issues) > 0 {
if err := contentreviewmod.InsertIssues(result.Issues); err != nil {
log.Warn("insert issues fail", log.Any("count", len(result.Issues)), log.E(err))
} else {
issues += int64(len(result.Issues))
}
}
checked += result.Checked
lastID = result.NextID
flushProgress(task.ID, lastID, checked, issues, total)
}
return checked, issues, nil
}
func flushProgress(id, lastID primitive.ObjectID, checked, issues, total int64) {
if err := contentreviewmod.UpdateProgress(id, lastID, checked, issues, total); err != nil {
log.Warn("update progress fail", log.E(err))
}
}
skd/service/contentreviewser/video.go
+42
View File
@@ -0,0 +1,42 @@
package contentreviewser
import (
"91porn-server/models/v/contentreviewmod"
"91porn-server/models/v/vidmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// videoScanner 视频帖子扫描策略；匹配 title / content / richText（不检测 tag）
type videoScanner struct{}
func (s *videoScanner) CountTotal() (int64, error) {
return vidmod.CountForReview()
}
func (s *videoScanner) ScanBatch(taskID, lastID primitive.ObjectID, terms []string) (*ScanResult, error) {
batch, err := vidmod.FindForReviewBatch(lastID, scanBatchSize)
if err != nil {
return nil, err
}
if len(batch) == 0 {
return &ScanResult{Done: true}, nil
}
issues := make([]*contentreviewmod.ReviewIssue, 0, len(batch))
for _, v := range batch {
b := newIssueBuilder(taskID, contentreviewmod.IssueTargetVideo, v.ID).
MatchTitle("标题", v.Title, terms).
MatchContent("内容", v.Content, terms).
MatchRichText("富文本", v.RichText, terms)
if b.HasHits() {
issues = append(issues, b.Build())
}
}
return &ScanResult{
Issues: issues,
Checked: int64(len(batch)),
NextID: batch[len(batch)-1].ID,
}, nil
}
skd/service/doregister.go
+119
View File
@@ -0,0 +1,119 @@
package service
import (
"fmt"
"strconv"
"strings"
"time"
"91porn-server/common"
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/common/redis"
"91porn-server/common/usertruth"
"91porn-server/models/l/registermod"
"91porn-server/models/v/usermod"
"91porn-server/skd/skdg"
)
const (
RestartSignal string = "RESTART"
NotifyKeyPrefix string = "notify:user:new:register:"
TrimPrefix string = "notify:"
)
var MessageCh = make(chan *redis.Message, 100)
var ChStatus = make(chan string)
// DoRegister
func DoRegister() {
common.Go(func() { Sub(redisconst.Db0ExpiredChannel) })
common.Go(func() {
for {
select {
case v, ok := <-MessageCh:
if !ok {
log.Warn("MessChan is nil waiting some second")
}
if v != nil {
LoadNewRegister(v.Payload)
}
case v, ok := <-ChStatus:
if !ok {
log.Warn("Restart signal is not receive")
}
if v == RestartSignal {
Sub(redisconst.Db0ExpiredChannel)
}
}
}
})
}
// 接收过期的键 将信息存入到数据库中 只处理指定的过期键 notify:user:new:register
func LoadNewRegister(expiredKey interface{}) {
if v, ok := expiredKey.(string); ok {
if strings.HasPrefix(v, NotifyKeyPrefix) {
common.Go(func() {
key := strings.TrimPrefix(v, TrimPrefix)
uidstr := strings.TrimPrefix(v, NotifyKeyPrefix)
uid, _ := strconv.ParseUint(uidstr, 10, 64)
log.Info("Expired Key And Handle Data", log.Any("expiredKey", expiredKey), log.Any("needHandleKey", key), log.Any("uid", uid))
handleRegisterData(key, uid)
})
}
}
}
func handleRegisterData(key string, uid uint64) {
fileds := skdg.Redis.Hkeys(key)
//获取IP
ip, _ := skdg.Redis.Hget(key, "ip")
if len(fileds) > 0 {
values, err := skdg.Redis.HMget(key, fileds)
if err != nil {
log.Warn("[Method] HandleRegisterData HMget Data from redis error", log.Any("key", key), log.E(err))
}
qCountMap := make(map[string]int64)
var qsArray []registermod.QueryStat
rgister := &registermod.NewRegisterLog{}
for i := 0; i < len(fileds); i++ {
if v, ok := values[i].(string); ok {
cnt, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
qCountMap[fileds[i]] = cnt
qs := registermod.QueryStat{QueryUrl: fileds[i], Count: cnt}
qsArray = append(qsArray, qs)
}
}
_, _ = skdg.Redis.Del(key)
rgister.CreatedAt = time.Now()
rgister.UID = uid
rgister.Query = qsArray
rgister.IP = ip
_ = registermod.InsertRegisterLog(rgister)
score := usertruth.Score(qCountMap)
//为区别默认值0 将计算出来的得分为0的情况 设置为-1
if score == 0 {
score = -1
}
_, _ = usermod.Update(uid, usermod.UserSelector{TrueScore: &score})
log.Info("[Method] HandleRegisterData success", log.Any("key", key), log.Any("trueScore", score), log.Any("uid", uid))
}
}
// 订阅
func Sub(channel string) {
subpush, err := skdg.Redis.Subscribe(channel)
if err != nil {
ChStatus <- RestartSignal
log.Warn(fmt.Sprintf("skd sever sub channel %s err", channel), log.E(err))
}
ch := subpush.Channel()
for msg := range ch {
message := &redis.Message{Channel: msg.Channel, Payload: msg.Payload}
MessageCh <- message
}
}
skd/service/export_task/advance_order_export.go
+102
View File
@@ -0,0 +1,102 @@
package export_task
import (
"91porn-server/common/log"
"91porn-server/models/v/advanceordermod"
"91porn-server/models/v/export_task_mod"
"encoding/json"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
func ExecExportAdvanceOrderTask(task *export_task_mod.ExportTask) (total int64, err error) {
arg := advanceordermod.QueryAllCond{}
err = json.Unmarshal([]byte(task.Param), &arg)
if err != nil {
return
}
var skip int64
var size int64 = 100
index := 1
path := fmt.Sprintf("./temp/%v%v", "预售订单列表-", task.ID.Hex())
zipName := fmt.Sprintf("%v_%v_预售订单列表_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
var records []*advanceordermod.AdvanceOrderExport
excelList := []string{}
for {
fmt.Println("skip", skip, " now ", time.Now().Format("2006-01-02 15:04:05"))
opt := options.Find().SetSkip(skip).SetLimit(size).SetSort(bson.M{"createdAt": -1})
list, err := advanceordermod.QueryAllDocument(arg.Filter(), opt)
if err != nil {
log.Error("ExecExportAdvanceOrderTask advanceordermod.QueryAllDocument fail", log.E(err))
return total, err
}
for _, v := range list {
item := &advanceordermod.AdvanceOrderExport{
ID: v.ID,
UID: v.UID,
AdvOid: v.AdvOid,
BalOid: v.BalOid,
ProductID: v.ProductID,
TotalAmount: v.TotalAmount,
AdvanceAmount: v.AdvanceAmount,
BalanceAmount: v.BalanceAmount,
StartTime: v.StartTime,
EndTime: v.EndTime,
CreatedAt: v.CreatedAt,
}
switch v.Status {
case advanceordermod.AdvanceProcessing:
item.Status = "预付中"
case advanceordermod.AdvanceSUCCESS:
item.Status = "预付成功"
case advanceordermod.BalanceProcessing:
item.Status = "尾款预付中"
case advanceordermod.BalanceSUCCESS:
item.Status = "尾款预付成功"
}
records = append(records, item)
}
var fileName = fmt.Sprintf("预售订单列表_%v.xlsx", index)
if len(records) >= maxDataNum {
// 获取到的数据已经达到了单文件最大限制数据量
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportAdvanceOrderTask SaveExcel fail", log.E(err))
return total, err
}
// 重新归0
records = []*advanceordermod.AdvanceOrderExport{}
index = index + 1
excelList = append(excelList, filePath)
fmt.Println("index:", index)
total = total + int64(len(records))
} else if len(list) < int(size) {
if len(records) > 0 {
// 数据库已经取不到更多的数据了
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportAdvanceOrderTask SaveExcel fail", log.E(err))
return total, err
}
excelList = append(excelList, filePath)
total = total + int64(len(records))
}
// 结束循环
break
}
skip = skip + size
}
// 发送到tg
err = SendTg(task.Admin, excelList, zipName)
if err != nil {
log.Error("ExecExportVideoTask SendTg fail", log.E(err))
return
}
return total, nil
}
skd/service/export_task/base.go
+282
View File
@@ -0,0 +1,282 @@
package export_task
import (
"91porn-server/common/file"
"91porn-server/common/log"
commonWorker "91porn-server/common/worker"
"91porn-server/models/v/export_task_mod"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"time"
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
"github.com/pkg/errors"
"github.com/xuri/excelize/v2"
"go.mongodb.org/mongo-driver/bson"
)
const (
proName = "91PORN"
tgToken = "5852134617:AAEO0_GPZiUJS_LeR5dM7byEfOpnRg4dGqU"
maxDataNum = 20000 // 每个excel存储的数据量
maxWorkerNum = 3
maxSize = 1024 * 1024 * 20
)
var Bot *tgbotapi.BotAPI
var worker *commonWorker.Worker
func init() {
bot, err := tgbotapi.NewBotAPI(tgToken)
if err != nil || bot == nil {
log.Fatal("初始化tgbot失败", log.E(err))
}
Bot = bot
worker = commonWorker.NewWorker(maxWorkerNum)
}
var execExportTaskRuining bool
func ExecExportTask() {
//防止重入
if execExportTaskRuining {
return
}
execExportTaskRuining = true
defer func() {
execExportTaskRuining = false
}()
// 获取任务列表
list, err := export_task_mod.GetTaskList()
if err != nil {
log.Error("ExecExportTask export_task_mod.GetTaskList fail", log.E(err))
return
}
for _, task := range list {
switch task.Type {
case export_task_mod.ExportUserTask:
CallExportFunc(task, ExecExportUserTask)
case export_task_mod.ExportVidTask:
CallExportFunc(task, ExecExportVideoTask)
case export_task_mod.ExportAdvanceOrderTask:
CallExportFunc(task, ExecExportAdvanceOrderTask)
case export_task_mod.ExportProductHistoryTask:
CallExportFunc(task, ExecExportProductHistoryTask)
}
}
}
func GetProName() string {
//if skdg.Conf.Base.Env == "test" {
// return "测试环境_" + proName
//}
return proName
}
func CallExportFunc(task *export_task_mod.ExportTask, f func(t *export_task_mod.ExportTask) (int64, error)) {
if task == nil {
return
}
worker.Exec(func() {
// 已经执行过了
if task.Status != 0 {
return
}
//key := fmt.Sprintf("ExecExportTask:%v", task.ID.Hex())
//ok, err := skdg.Redis.SetNX(key, 1, time.Minute*30)
//if err != nil {
// log.Error("ExecExportTask fail", log.E(err))
// return
//}
//if !ok {
// log.Info("ExportTask already executed")
// return
//}
//defer skdg.Redis.Del(key)
start := time.Now().Unix()
total, err := f(task)
if err != nil {
log.Error("CallExportFunc fail", log.Any("task", task), log.E(err))
// 修改任务信息
export_task_mod.UpdateTask(task.ID, bson.M{
"status": 2,
"reason": err.Error(),
})
return
}
// 修改任务信息
export_task_mod.UpdateTask(task.ID, bson.M{
"status": 1,
"time": time.Now().Unix() - start, // 耗时
"total": total,
})
})
return
}
// SendTg 发送到tg
func SendTg(admin string, excelList []string, zipFileName string) (err error) {
exportPath, err := filepath.Abs("temp")
if err != nil {
return
}
// 尝试创建
file.MakeDir(exportPath)
zipList, err := Zip(excelList, zipFileName)
if err != nil {
log.Error("SendTg Zip fail", log.Any("excelList", excelList), log.Any("zipFileName", zipFileName), log.E(err))
return
}
// 测试环境不需要发送到tg群
//if skdg.Conf.Base.Env == "test" {
// return
//}
if len(zipList) == 0 {
return
}
var chatId int64 = -1003399433452
content := fmt.Sprintf("%v总共有%v个压缩包，请查收", admin, len(zipList))
_, err = Bot.Send(tgbotapi.NewMessage(chatId, content))
if err != nil {
log.Error("tgbot send fail", log.E(err))
return errors.Wrap(err, "发送TG消息失败")
}
for _, zipName := range zipList {
fileBytes, err := os.ReadFile("./temp/" + zipName)
if err != nil {
log.Error("ReadFile fail", log.Any("excelList", excelList), log.Any("zipFileName", zipFileName), log.E(err))
return errors.Wrap(err, "打开压缩文件失败")
}
// 创建要发送的文件
tFile := tgbotapi.FileBytes{Name: zipName, Bytes: fileBytes}
//生成csv发送到tg群
newDocument := tgbotapi.NewDocumentUpload(chatId, tFile)
_, err = Bot.Send(newDocument)
if err != nil {
log.Error("tgbot send fail", log.E(err))
return errors.Wrap(err, "发送TG消息失败")
}
}
return nil
}
func Zip(filePathList []string, zipFileName string) (zipList []string, err error) {
if len(filePathList) == 0 {
return
}
var currentSize int64
zipIndex := 1
currentZipName := fmt.Sprintf("%v_%v.zip", zipFileName, zipIndex)
args := []string{currentZipName}
for k, filePath := range filePathList {
// 获取文件大小
stat, err := os.Stat(filePath)
if err != nil {
return nil, err
}
currentSize = currentSize + stat.Size()
args = append(args, filePath)
// 判断当前所有文件加起来是否已经超过20M，超过就直接压缩,否则就接着等待压缩
if currentSize < maxSize && k != (len(filePathList)-1) {
continue
}
cmd1 := exec.Command("zip", args...)
out1, err := cmd1.CombinedOutput()
if err != nil {
return nil, err
}
fmt.Println(string(out1))
// 移动文件到目录里
cmd2 := exec.Command("mv", currentZipName, "./temp")
if err != nil {
return nil, err
}
out2, err := cmd2.CombinedOutput()
if err != nil {
return nil, err
}
fmt.Println(string(out2))
zipList = append(zipList, currentZipName)
// 重置
zipIndex = zipIndex + 1
currentZipName = fmt.Sprintf("%v_%v.zip", zipFileName, zipIndex)
args = []string{currentZipName}
currentSize = 0
}
return
}
// SaveExcel 保存数据到excel中
func SaveExcel(records interface{}, path string, fileName string) (filePath string, err error) {
sheet := "sheet1"
xlsx := excelize.NewFile() // new file
index, _ := xlsx.NewSheet(sheet) // new sheet
xlsx.SetActiveSheet(index) // set active (default) sheet
t := reflect.TypeOf(records)
if t.Kind() != reflect.Slice {
panic("records must be slice")
}
s := reflect.ValueOf(records)
for i := 0; i < s.Len(); i++ {
elem := s.Index(i).Interface()
elemType := reflect.TypeOf(elem)
elemValue := reflect.ValueOf(elem)
if elemType.Kind() == reflect.Ptr {
elemType = elemType.Elem()
elemValue = elemValue.Elem()
}
if elemType.Kind() != reflect.Struct {
panic("record in slice must be a struct")
}
k := 0
for j := 0; j < elemType.NumField(); j++ {
field := elemType.Field(j)
tag := field.Tag.Get("xlsx")
if tag == "-" {
continue
}
if tag == "" {
tag = field.Name
}
column, _ := excelize.ColumnNumberToName(k + 1)
k++
name := tag
// 设置表头
if i == 0 {
_ = xlsx.SetCellValue(sheet, fmt.Sprintf("%s%d", column, i+1), name)
}
// 判断是否是时间类型
colValue := elemValue.Field(j).Interface()
if colTime, ok := colValue.(time.Time); ok {
colValue = colTime.Format("2006-01-02 15:04:05")
}
// 设置内容
_ = xlsx.SetCellValue(sheet, fmt.Sprintf("%s%d", column, i+2), colValue)
}
}
// 保存的目录看是否存在不存在就创建
_, err = os.Stat(path)
if os.IsNotExist(err) {
err = os.MkdirAll(path, os.ModePerm)
if err != nil {
return "", err
}
}
filePath = path + "/" + fileName
err = xlsx.SaveAs(filePath)
if err != nil {
return "", err
}
return filePath, nil
}
skd/service/export_task/product_history_export.go
+115
View File
@@ -0,0 +1,115 @@
package export_task
import (
"91porn-server/common/log"
"91porn-server/models/v/advanceordermod"
"91porn-server/models/v/export_task_mod"
"91porn-server/models/v/prdcthsomod"
"91porn-server/models/v/productmod"
"encoding/json"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
func ExecExportProductHistoryTask(task *export_task_mod.ExportTask) (total int64, err error) {
arg := prdcthsomod.ProductHistoryQueryReq{}
err = json.Unmarshal([]byte(task.Param), &arg)
if err != nil {
return
}
var skip int64
var size int64 = 100
index := 1
path := fmt.Sprintf("./temp/%v%v", "会员卡购买记录-", task.ID.Hex())
zipName := fmt.Sprintf("%v_%v_会员卡购买记录_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
var records []*prdcthsomod.ProductHistoryExport
excelList := []string{}
for {
fmt.Println("skip", skip, " now ", time.Now().Format("2006-01-02 15:04:05"))
opt := options.Find().SetSkip(skip).SetLimit(size).SetSort(bson.M{"createdAt": -1})
list, err := prdcthsomod.FindList(arg.GetCond(), opt)
if err != nil {
log.Error("ExecExportProductHistoryTask prdcthsomod.FindList fail", log.E(err))
return total, err
}
for _, v := range list {
item := &prdcthsomod.ProductHistoryExport{
ID: v.ID, // 交易订单号
UID: v.UID, // 用户id
ProductID: v.ProductID, // 会员卡id
Name: v.Name, // 商品名字
IsUpgrade: v.IsUpgrade, // 是否VIP升级
Amount: v.Amount, // 花费的余额
Income: v.Income, // 花费的收益
SysType: v.SysType, // 设备系统类型 ios pc android
CreatedAt: v.CreatedAt,
}
if v.ProductSnapShot != nil {
item.Duration = v.ProductSnapShot.Duration
item.VipLevel = v.ProductSnapShot.VipLevel
item.OriginalPrice = v.ProductSnapShot.OriginalPrice
if v.ProductSnapShot.DiscountedPriceIos != nil {
item.DiscountedPriceIos = *v.ProductSnapShot.DiscountedPriceIos
}
if v.ProductSnapShot.DiscountedPriceAnd != nil {
item.DiscountedPriceAnd = *v.ProductSnapShot.DiscountedPriceAnd
}
}
switch v.ProductType {
case productmod.VIP:
item.ProductType = "会员卡"
case productmod.AdvanceCard:
item.ProductType = "预售卡"
}
switch v.AdvanceOrderStatus {
case advanceordermod.AdvanceSUCCESS:
item.AdvanceOrderStatus = "预付成功"
case advanceordermod.BalanceSUCCESS:
item.AdvanceOrderStatus = "尾款支付成功"
}
records = append(records, item)
}
var fileName = fmt.Sprintf("预售订单列表_%v.xlsx", index)
if len(records) >= maxDataNum {
// 获取到的数据已经达到了单文件最大限制数据量
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportProductHistoryTask SaveExcel fail", log.E(err))
return total, err
}
// 重新归0
records = []*prdcthsomod.ProductHistoryExport{}
index = index + 1
excelList = append(excelList, filePath)
fmt.Println("index:", index)
total = total + int64(len(records))
} else if len(list) < int(size) {
if len(records) > 0 {
// 数据库已经取不到更多的数据了
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportProductHistoryTask SaveExcel fail", log.E(err))
return total, err
}
excelList = append(excelList, filePath)
total = total + int64(len(records))
}
// 结束循环
break
}
skip = skip + size
}
// 发送到tg
err = SendTg(task.Admin, excelList, zipName)
if err != nil {
log.Error("ExecExportProductHistoryTask SendTg fail", log.E(err))
return
}
return total, nil
}
skd/service/export_task/user_export.go
+192
View File
@@ -0,0 +1,192 @@
package export_task
import (
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/models/l/playlgmod"
"91porn-server/models/v/export_task_mod"
"91porn-server/models/v/usermod"
"encoding/json"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"sync"
)
// ExecExportUserTask 执行用户导出任务
func ExecExportUserTask(task *export_task_mod.ExportTask) (total int64, err error) {
type userListReq struct {
usermod.UserListSelector
common.StandQuery
IsPretendAcc *int `form:"isPretendAcc" json:"isPretendAcc"` //是否马甲账号
}
var arg userListReq
err = json.Unmarshal([]byte(task.Param), &arg)
if err != nil {
return
}
if arg.StartTime == nil || arg.EndTime == nil {
return 0, errors.New("没有选择时间")
}
cond, _ := common.StandQueryMap(arg.StandQuery, arg.UserListSelector)
//马甲账户
if arg.IsPretendAcc != nil && *arg.IsPretendAcc == 1 {
cond["devID"] = bson.M{"$regex": usermod.SystemDevIDPrex, "$options": "i"}
}
var skip int64
var size int64 = 500
records := []usermod.ExportUser{}
zipName := fmt.Sprintf("%v_%v_用户列表_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
index := 1
path := fmt.Sprintf("./temp/%v%v", "用户列表-", task.ID.Hex())
excelList := []string{}
// 最后一条记录的时间
for {
userList, err := usermod.ExportFindMany(cond, skip, size)
if err != nil {
log.Error("ExecExportUserTask fail", log.E(err))
return total, err
}
videoCntMap := make(map[uint64]int)
unDealVideoCntMap := make(map[uint64]int)
// 针对91PORN做的优化，不然数据太多，导出数据文本太大，速度慢
//if arg.IsPretendAcc != nil && *arg.IsPretendAcc == 1 {
// videoCntMap, unDealVideoCntMap, _ = vidmod.GetVideosByUIDs(getUIDs(userList))
//}
eUsers := encodeUsers(userList, videoCntMap, unDealVideoCntMap)
records = append(records, eUsers...)
var fileName = fmt.Sprintf("用户列表_%v.xlsx", index)
// 最多容纳5000条
if len(records) >= maxDataNum {
// 直接写入excel
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
return total, err
}
// 重置
records = []usermod.ExportUser{}
index = index + 1
excelList = append(excelList, filePath)
total = total + int64(len(records))
} else if len(userList) < int(size) {
if len(records) > 0 {
// 直接写入excel
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
return total, err
}
excelList = append(excelList, filePath)
total = total + int64(len(records))
}
// 已经没有更多数据了，直接返回
break
}
skip = skip + size
}
// 发送到tg
err = SendTg(task.Admin, excelList, zipName)
if err != nil {
log.Error("ExecExportUserTask SendTg fail", log.E(err))
return
}
return total, nil
}
func encodeUsers(users []*usermod.User, videoCntMap map[uint64]int, unDealVideoCntMap map[uint64]int) []usermod.ExportUser {
usersLen := len(users)
uids := make([]uint64, 0, usersLen)
uInfos := make([]usermod.ExportUser, 0, usersLen)
for _, u := range users {
if u != nil {
user := usermod.ExportUser{
UID: u.UID,
//DevID: u.DevID,
DevType: u.DevType,
RegisterIP: u.RegisterIP,
Mobile: u.Mobile,
//Gender: u.Gender,
Channel: u.DistrictCode,
//Name: u.Name,
PromotionCode: u.PromCode,
//Summary: u.Summary,
//Region: u.Region,
//Birthday: u.Birthday,
//VipLevel: u.VipLevel,
VipExpireDate: u.VipExpireDate,
CreatedAt: u.CreatedAt,
//MobileBindAt: u.MobileBindAt,
//HasLocked: u.HasLocked,
//HasBanned: u.HasBanned,
//TotalVideoCnt: videoCntMap[u.UID],
//UndealVideoCnt: unDealVideoCntMap[u.UID],
LastVisitAt: u.LastVisitAt,
}
uInfos = append(uInfos, user)
uids = append(uids, u.UID)
}
}
wCnt := setArray2Map(calWatchedVideoCnt(uids))
for i, u := range uInfos {
uInfos[i].WatchCount = wCnt[u.UID]
}
return uInfos
}
func getUIDs(users []*usermod.User) []uint64 {
uids := make([]uint64, 0, len(users))
for _, v := range users {
if v == nil {
continue
}
uids = append(uids, v.UID)
}
return uids
}
func setArray2Map(datas []playlgmod.UIDCount) map[uint64]int {
m := make(map[uint64]int)
for _, d := range datas {
m[d.UID] = d.Count
}
return m
}
func calWatchedVideoCnt(uids []uint64) []playlgmod.UIDCount {
le := len(uids)
if le < 1000 {
datas, _ := playlgmod.HasWatchedVideoCnt(uids)
return datas
}
//创建多个协程去拉取
const goRoutineCnt = 10
var wg sync.WaitGroup
var mData [goRoutineCnt][]playlgmod.UIDCount
size := le / goRoutineCnt
wg.Add(goRoutineCnt)
for i := 0; i < goRoutineCnt-1; i++ {
common.GoParam(i, func(i int) {
defer wg.Done()
begin := i * size
end := begin + size - 1
mData[i], _ = playlgmod.HasWatchedVideoCnt(uids[begin:end])
})
}
common.Go(func() {
defer wg.Done()
begin := (goRoutineCnt - 1) * size
end := le - 1
mData[goRoutineCnt-1], _ = playlgmod.HasWatchedVideoCnt(uids[begin:end])
})
wg.Wait()
datas := []playlgmod.UIDCount{}
for i := 0; i < goRoutineCnt; i++ {
datas = append(datas, mData[i]...)
}
return datas
}
skd/service/export_task/video_export.go
+219
View File
@@ -0,0 +1,219 @@
package export_task
import (
"91porn-server/common/log"
"91porn-server/models/v/export_task_mod"
"91porn-server/models/v/modulevidmod"
"91porn-server/models/v/tagmod"
"91porn-server/models/v/vidmod"
"91porn-server/web/vidhelp"
"encoding/json"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"time"
)
func ExecExportVideoTask(task *export_task_mod.ExportTask) (total int64, err error) {
arg := vidmod.ListReq{}
err = json.Unmarshal([]byte(task.Param), &arg)
if err != nil {
return
}
cond, err := assembleConditions(arg)
if err != nil {
return
}
secID, cond, _, err := dealVidListCond(cond)
if err != nil {
return
}
var skip int64
var size int64 = 100
index := 1
path := fmt.Sprintf("./temp/%v%v", "帖子列表-", task.ID.Hex())
zipName := fmt.Sprintf("%v_%v_帖子列表_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
var records []*vidmod.WebVideo
excelList := []string{}
for {
fmt.Println("skip", skip, " now ", time.Now().Format("2006-01-02 15:04:05"))
list, err := vidmod.ExportFindMany(cond, skip, size)
if err != nil {
log.Error("ExecExportVideoTask vidmod.ExportFindMany fail", log.E(err))
return total, err
}
infos, err := vidhelp.EncodeVideoInfo(list, secID)
if err != nil {
log.Error("ExecExportVideoTask vidhelp.EncodeVideoInfo fail", log.E(err))
return total, err
}
records = append(records, infos...)
var fileName = fmt.Sprintf("帖子列表_%v.xlsx", index)
if len(records) >= maxDataNum {
// 获取到的数据已经达到了单文件最大限制数据量
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
return total, err
}
// 重新归0
records = []*vidmod.WebVideo{}
index = index + 1
excelList = append(excelList, filePath)
fmt.Println("index:", index)
total = total + int64(len(records))
} else if len(list) < int(size) {
if len(records) > 0 {
// 数据库已经取不到更多的数据了
filePath, err := SaveExcel(records, path, fileName)
if err != nil {
log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
return total, err
}
excelList = append(excelList, filePath)
total = total + int64(len(records))
}
// 结束循环
break
}
skip = skip + size
}
// 发送到tg
err = SendTg(task.Admin, excelList, zipName)
if err != nil {
log.Error("ExecExportVideoTask SendTg fail", log.E(err))
return
}
return total, nil
}
func dealVidListCond(cond bson.M) (primitive.ObjectID, bson.M, []primitive.ObjectID, error) {
var secID primitive.ObjectID
var err error
sectionID, ok := cond["sectionID"]
if ok {
secID, err = primitive.ObjectIDFromHex(sectionID.(string))
if err != nil {
log.Error("primitive.ObjectIDFromHex", log.Any("sectionID", sectionID), log.E(err))
return secID, cond, nil, err
}
var flag bool
isSorted, ok := cond["isSortedUnderModule"]
if ok {
flag = isSorted.(bool)
}
videoIDs, err := modulevidmod.GetBySectionID(secID, flag)
if err != nil {
return secID, cond, nil, err
}
cond["_id"] = bson.M{"$in": videoIDs}
delete(cond, "sectionID")
delete(cond, "isSortedUnderModule")
return secID, cond, videoIDs, nil
}
delete(cond, "sectionID")
delete(cond, "isSortedUnderModule")
return secID, cond, nil, nil
}
func assembleConditions(req vidmod.ListReq) (m map[string]interface{}, err error) {
m = make(map[string]interface{})
if req.ShowType != nil {
m["showType"] = *req.ShowType
}
if req.Status != 4 {
if req.Status == 7 {
m["status"] = map[string]int{"$gt": 0}
} else {
m["status"] = req.Status
}
}
if req.IsFree == 0 {
m["coins"] = map[string]int{"$gt": 0}
}
if req.IsFree == 1 {
m["coins"] = map[string]int{"$eq": 0}
}
if req.IsUserUp == 2 {
m["publisherID"] = map[string]int{"$lt": 115000}
}
if req.IsUserUp == 1 {
m["publisherID"] = map[string]int{"$gt": 115000}
}
if len(req.Title) != 0 {
m["title"] = map[string]string{"$regex": req.Title, "$options": "i"}
}
if req.UID > 0 {
m["publisherID"] = req.UID
}
if req.Chosen == 1 {
m["chosen"] = true
}
if req.Chosen == 2 {
m["chosen"] = false
}
if req.FreeArea == 1 {
m["freeArea"] = true
}
if req.FreeArea == 2 {
m["freeArea"] = false
}
if !req.End.IsZero() {
m["createdAt"] = map[string]time.Time{"$gte": req.Start, "$lt": req.End}
}
if len(req.Tag) != 0 {
id, err := tagmod.GetTagIDByName(req.Tag)
if err != nil {
return m, err
}
m["tags"] = id
}
if len(req.ID) != 0 {
oid, err := primitive.ObjectIDFromHex(req.ID)
if err != nil {
return m, err
}
m["_id"] = oid
}
//通过初始价格判断是否是马甲账号
if req.IsPretendAcc == 1 {
m["coins"] = vidmod.PretendAccInitCoins
}
if req.IsPretendAcc == 2 {
//i := make(map[string]int64)
//i["$ne"] = vidmod.PretendAccInitCoins
m["coins"] = map[string]int64{"$ne": vidmod.PretendAccInitCoins}
}
m["deleteAt"] = bson.M{"$exists": false}
if req.NewsType != "" {
m["newsType"] = req.NewsType
}
if req.LiaoBaTop != nil {
m["liaoBaTop"] = *req.LiaoBaTop
}
if req.SectionID != "" {
m["sectionID"] = req.SectionID
}
m["isSortedUnderModule"] = req.IsSortedUnderModule
if req.IsRecommended != nil {
if *req.IsRecommended {
m["recoWeight"] = bson.M{"$gte": 0}
} else {
m["recoWeight"] = bson.M{"$lt": 0}
}
}
if req.Key == "likeRate" || req.Key == "purchaseRate" {
m[req.Key] = req.Value
}
if req.IsHappinessPlazaTop != nil && *req.IsHappinessPlazaTop {
if *req.IsHappinessPlazaTop {
m["happinessPlazaTop"] = bson.M{"$gt": 0}
} else {
m["happinessPlazaTop"] = bson.M{"$lte": 0}
}
}
return m, nil
}
skd/service/fakeVideoInfo.go
+122
View File
@@ -0,0 +1,122 @@
package service
import (
"sync"
"time"
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/common/stderr"
"91porn-server/models/v/chatrobotmod"
"91porn-server/models/v/vidmod"
"github.com/robfig/cron/v3"
"go.mongodb.org/mongo-driver/bson"
)
const SingleQueryLimit = 5000
var VideoInfo *VideoInfoFaker
type VideoInfoFaker struct {
once sync.Once
limit int64 // 限制每次查询的数据量
mu *sync.Mutex
}
func newVideoInfoFaker() *VideoInfoFaker {
return &VideoInfoFaker{
once: sync.Once{},
limit: SingleQueryLimit,
mu: &sync.Mutex{},
}
}
// FullUpdate 单次更新，程序启动后执行一次
func (f *VideoInfoFaker) FullUpdate() {
f.once.Do(
func() {
f.mu.Lock()
log.Debug("full update start...")
common.Go(func() {
defer f.mu.Unlock()
start := time.Now()
totalUpdate, err := f.fakeVideoInfo()
log.Info("FullUpdate fihish",
log.Any("time(s)", time.Since(start).Seconds()),
log.Any("total", totalUpdate),
log.E(err))
})
},
)
}
// CronUpdate 定时更新
func (f *VideoInfoFaker) CronUpdate() {
f.mu.Lock()
defer f.mu.Unlock()
start := time.Now()
totalUpdate, err := f.fakeVideoInfo()
log.Info("CronUpdate fihish",
log.Any("time(s)", time.Since(start).Seconds()),
log.Any("total", totalUpdate),
log.E(err))
}
func (f *VideoInfoFaker) RegisterCronJob(cron *cron.Cron) {
f.mu.Lock()
defer f.mu.Unlock()
_, _ = cron.AddFunc("0 0 10 */1 * *", f.CronUpdate) // 每隔三天，早上10点，仅在周一到周六执行
}
// FakeVideoInfo 更新视频相关数据
func (f *VideoInfoFaker) fakeVideoInfo() (int64, error) {
robotConf, err := chatrobotmod.GetRobotConfByType(chatrobotmod.Comment)
if err != nil || robotConf.ID.IsZero() {
return 0, err
}
if (robotConf.MaxFakeLikes <= 0 && robotConf.MaxFakePlayCount <= 0) ||
(robotConf.MinFakeLikes >= robotConf.MaxFakeLikes &&
robotConf.MinFakePlayCount >= robotConf.MaxFakePlayCount) {
return 0, stderr.RobotInvalidConf
}
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
pageSize, page, totalUpdate := f.limit, int64(0), int64(0)
for pageSize == f.limit {
videoList, err := vidmod.GetVideoListLtMinPlayLikeCount(int64(robotConf.MinFakeLikes),
int64(robotConf.MinFakePlayCount), page*pageSize, pageSize)
if err != nil {
return totalUpdate, err
}
cnt := int64(0)
for _, video := range videoList {
set := bson.M{}
if video.FakeLikeCount < robotConf.MinFakeLikes {
destLike := common.RandInt(robotConf.MinFakeLikes, robotConf.MaxFakeLikes)
set["fakeLikeCount"] = destLike
}
if video.FakePlayCount < robotConf.MinFakePlayCount {
destPlay := common.RandInt(robotConf.MinFakePlayCount, robotConf.MaxFakePlayCount)
video.FakePlayCount = destPlay
set["fakePlayCount"] = destPlay
}
if set["fakePlayCount"] != 0 || set["fakeLikeCount"] != 0 {
// log.Info("update", log.Any("set", set), log.Any("vid", video.ID), log.Any("origin like", video.FakeLikeCount), log.Any("origin play", video.PlayCount))
res, err := vidmod.UpdateOneByID(video.ID, set)
if err != nil {
// 遇到错误继续执行
log.Error("fakeVideoInfo UpdateOneByID", log.Any("videoID", video.ID), log.E(err))
} else {
cnt += res.ModifiedCount
}
}
}
page++
pageSize = int64(len(videoList))
totalUpdate += cnt
log.Info("fake progress", log.Any("page", page), log.Any("total", totalUpdate))
<-ticker.C
}
return totalUpdate, nil
}
skd/service/init.go
+5
View File
@@ -0,0 +1,5 @@
package service
func Init() {
VideoInfo = newVideoInfoFaker()
}
skd/service/promotionCode.go
+58
View File
@@ -0,0 +1,58 @@
package service
import (
"91porn-server/common"
"91porn-server/common/constant/redisconst"
"91porn-server/common/log"
"91porn-server/models/v/usermod"
"91porn-server/skd/skdg"
)
// 填充推广码
func MakePromotionCode() {
redisKey := redisconst.PromotionCodeKey()
var low int64 = 5000
total, err := skdg.Redis.SCard(redisKey)
if err != nil {
log.Warn("service MakePromotionCode SCard error", log.E(err), log.Any("total", total))
return
}
if total < low {
savePromotionCode(total)
}
}
func savePromotionCode(total int64) {
var max int64 = 9999
size := 100
redisKey := redisconst.PromotionCodeKey()
if total < max {
promotionCodeCollection := make([]string, size)
for i := 0; i < size; i++ {
p, err := getPromotionCode()
if err != nil {
break
}
promotionCodeCollection[i] = p
}
count, err := skdg.Redis.SAdd(redisKey, promotionCodeCollection)
if err != nil {
log.Warn("service savePromotionCode SAdd error", log.E(err), log.Any("total", total))
return
}
savePromotionCode(total + count)
}
}
// 获取不重复的推广码
func getPromotionCode() (string, error) {
promotionCode := common.InvitePromotionCodeGenera()
u, err := usermod.FindUserPromotionCode(promotionCode)
if err != nil {
return "", err
}
if u == nil {
return promotionCode, nil
}
return getPromotionCode()
}
skd/service/updownload.go
+102
View File
@@ -0,0 +1,102 @@
package service
import (
"91porn-server/common"
"91porn-server/common/httputil"
"91porn-server/common/log"
"91porn-server/common/stderr"
"91porn-server/models/v/vidmod"
"91porn-server/skd/skdg"
"strconv"
"strings"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// SyncFileFromFs 从文件服务器同步文件信息
func SyncFileFromFs(vID string) (code stderr.Code) {
v, _ := vidmod.GetVideoInfo(vID)
resp, err := PullFileInfo(v.SourceID)
if err != nil {
return stderr.ErrConnectToFs
}
if resp.Code == stderr.UpLoadFileComplete {
oid, _ := primitive.ObjectIDFromHex(resp.Data.ID)
sourceID := resp.Data.ID
playTime := resp.Data.PlayTime
freeTime := vidmod.GetFreeTime(resp.Data.PlayTime)
if v.FreeTime != 0 && freeTime != v.FreeTime {
freeTime = v.FreeTime
}
seriesCover := v.SeriesCover
resolution := strconv.FormatInt(int64(resp.Data.Width), 10) + "*" + strconv.FormatInt(int64(resp.Data.Height), 10)
width := resp.Data.Width
height := resp.Data.Height
identifi := common.IdentifiVideoInfo(int64(resp.Data.Width), int64(resp.Data.Height))
direction := string(identifi.Direction)
quality := string(identifi.Quality)
md5 := resp.Data.CheckSum
actor := strings.Join(resp.Data.Actors, ",")
size := resp.Data.Size
filename := resp.Data.Filename
via := resp.Data.Via
ratio := resp.Data.Ratio
update := vidmod.WebVideoUpdateDoc{
ID: oid,
SourceID: &sourceID,
PlayTime: &playTime,
FreeTime: &freeTime,
SeriesCover: &seriesCover,
Resolution: &resolution,
Width: &width,
Height: &height,
MD5: &md5,
Actor: &actor,
Size: &size,
Filename: &filename,
Via: &via,
Ratio: &ratio,
Direction: &direction,
Quality: &quality,
}
if v.Cover == "" {
cover := resp.Data.FieldNameFs + "-1.jpg"
update.Cover = &cover
}
if v.CoverThumb == "" {
coverThumb := resp.Data.FieldNameFs + "-2.jpg"
update.CoverThumb = &coverThumb
}
_, _ = vidmod.UpdateVideoResolutionPlayTime(v.ID, update)
}
return resp.Code
}
// PullFileInfo 获取文件转码情况及详情
func PullFileInfo(id string) (respBody vidmod.PuFinfoResp, err error) {
client := httputil.Client()
code, err := client.POSTWithJResp(&respBody, common.BindUrl(skdg.Conf.Url.PullFileInfo, id), nil, nil)
log.Info("http method PullFileInfo response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code))
if err != nil {
log.Error("Pull FileInfo From Aws ", log.Any("Error", err))
respBody.Code = stderr.ErrConnectToFs
return
}
switch respBody.Data.Status {
case vidmod.Merging, vidmod.MergeCompleted:
respBody.Code = stderr.FileMerging
case vidmod.MergeError:
respBody.Code = stderr.ErrMergeFile
case vidmod.Converting:
respBody.Code = stderr.FileConverting
case vidmod.ConvertError:
respBody.Code = stderr.ErrConvertFile
case vidmod.UploadLoadingToFs:
respBody.Code = stderr.UploadLoadingToFs
case vidmod.FileUploadError:
respBody.Code = stderr.ErrUploadError
case vidmod.ConvertCompleted, vidmod.Completed:
respBody.Code = stderr.UpLoadFileComplete
}
return
}
skd/service/versionser.go
+33
View File
@@ -0,0 +1,33 @@
package service
import (
"91porn-server/common"
"91porn-server/common/crypt"
)
func GetTFUrl(tf string) string {
if tf != "" {
type tfStc struct {
Url string `json:"url"`
Name string `json:"name"`
}
var tfArr []tfStc
_ = crypt.JSON2Struct(tf, &tfArr)
if len(tfArr) == 0 {
return tf
}
weight := 100 / len(tfArr)
chioce := make([]common.Choice, len(tfArr))
for i := range tfArr {
chioce[i] = common.Choice{
Weight: weight,
Item: tfArr[i].Url,
}
}
ch, _ := common.WeightedChoice(chioce)
if v, ok := ch.Item.(string); ok {
return v
}
}
return tf
}
skd/skdg/elastic.go
+24
View File
@@ -0,0 +1,24 @@
package skdg
import (
"os"
"91porn-server/common/elastic"
"91porn-server/common/log"
)
func InitElastic() {
opts := elastic.Options{
Address: []string{Conf.Elastic.VideoUrl},
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 30,
UserName: Conf.Elastic.UserName,
PassWord: Conf.Elastic.PassWord,
}
c, err := elastic.InitElastic(opts)
if err != nil {
log.Error("startUp VideoES InitElastic error", log.E(err))
os.Exit(1)
}
VideoES = c
}
skd/skdg/redisg.go
+28
View File
@@ -0,0 +1,28 @@
package skdg
import (
"os"
"91porn-server/common/log"
"91porn-server/common/redis"
)
func InitRedis() {
var err error
Redis, err = redis.WithURL(Conf.Redis.URL)
if err != nil {
log.Error("startUp udb redis error", log.E(err))
os.Exit(1)
}
}
func RedisClose() (err error) {
if Redis != nil {
if err := Redis.Close(); err != nil {
log.Error("RUdb Redis Close Error", log.E(err))
return err
}
log.Info("RUdb Redis Close OK")
}
return nil
}
skd/skdg/shortrecommend.go
+49
View File
@@ -0,0 +1,49 @@
package skdg
const (
defaultShortRecommendCron = "CRON_TZ=Asia/Shanghai 0 5/10 * * * ?"
defaultShortRecommendKeyTTLHours = 72
defaultProdInitializationLimit = 50_000
)
// ShortRecommendConfig 控制定时队列生成。Enabled 未配置时默认启用。
type ShortRecommendConfig struct {
Enabled *bool `json:"enabled"`
Cron string `json:"cron"`
KeyTTLHours int `json:"keyTTLHours"`
MaxInitializationWrites *int64 `json:"maxInitializationWrites"`
}
func (c *GlobalConfig) ShortRecommendEnabled() bool {
return c == nil || c.ShortRecommend.Enabled == nil || *c.ShortRecommend.Enabled
}
func (c *GlobalConfig) ShortRecommendCron() string {
if c == nil || c.ShortRecommend.Cron == "" {
return defaultShortRecommendCron
}
return c.ShortRecommend.Cron
}
func (c *GlobalConfig) ShortRecommendKeyTTL() int {
if c == nil || c.ShortRecommend.KeyTTLHours <= 0 {
return defaultShortRecommendKeyTTLHours
}
return c.ShortRecommend.KeyTTLHours
}
func (c *GlobalConfig) ShortRecommendInitializationLimit() int64 {
if c == nil {
return 0
}
if c.ShortRecommend.MaxInitializationWrites != nil {
if *c.ShortRecommend.MaxInitializationWrites < 0 {
return 0
}
return *c.ShortRecommend.MaxInitializationWrites
}
if c.Base.Env == "prod" {
return defaultProdInitializationLimit
}
return 0
}
skd/skdg/shortrecommend_test.go
+51
View File
@@ -0,0 +1,51 @@
package skdg
import "testing"
func TestShortRecommendDefaultsPreserveLegacyDeployment(t *testing.T) {
var nilConfig *GlobalConfig
if !nilConfig.ShortRecommendEnabled() {
t.Fatal("nil config should default to enabled")
}
if got := nilConfig.ShortRecommendCron(); got != defaultShortRecommendCron {
t.Fatalf("cron = %q, want %q", got, defaultShortRecommendCron)
}
if got := nilConfig.ShortRecommendKeyTTL(); got != 72 {
t.Fatalf("TTL hours = %d, want 72", got)
}
}
func TestShortRecommendExplicitConfig(t *testing.T) {
disabled := false
cfg := &GlobalConfig{}
cfg.ShortRecommend.Enabled = &disabled
cfg.ShortRecommend.Cron = "0 0 5 * * ?"
cfg.ShortRecommend.KeyTTLHours = 96
initializationLimit := int64(10_000)
cfg.ShortRecommend.MaxInitializationWrites = &initializationLimit
if cfg.ShortRecommendEnabled() {
t.Fatal("configured disabled should be honored")
}
if got := cfg.ShortRecommendCron(); got != cfg.ShortRecommend.Cron {
t.Fatalf("cron = %q", got)
}
if got := cfg.ShortRecommendKeyTTL(); got != 96 {
t.Fatalf("TTL hours = %d", got)
}
if got := cfg.ShortRecommendInitializationLimit(); got != 10_000 {
t.Fatalf("initialization limit = %d", got)
}
}
func TestShortRecommendProductionDefaultsToFailClosedInitializationGuard(t *testing.T) {
cfg := &GlobalConfig{}
cfg.Base.Env = "prod"
if got := cfg.ShortRecommendInitializationLimit(); got != defaultProdInitializationLimit {
t.Fatalf("initialization limit = %d, want %d", got, defaultProdInitializationLimit)
}
unlimited := int64(0)
cfg.ShortRecommend.MaxInitializationWrites = &unlimited
if got := cfg.ShortRecommendInitializationLimit(); got != 0 {
t.Fatalf("explicit initialization limit = %d, want 0", got)
}
}
skd/skdg/skdg.go
+105
View File
@@ -0,0 +1,105 @@
package skdg
import (
"91porn-server/common/constant"
"91porn-server/common/db"
"91porn-server/common/elastic"
"91porn-server/common/redis"
"91porn-server/common/tg"
)
type GlobalConfig struct {
Base struct {
Port int64 `json:"port"`
Env string `json:"env"`
// 媒体分片(TS)鉴权签名密钥对象(含 keyVersion/key)；未配置时回退内置默认(版本 default + constant.MediaSourceAuthKey)
TsAuth constant.TsAuthKeyConfig `json:"tsAuth"`
} `json:"base"`
Mongo struct {
VideoDbUrl string `json:"videoDbUrl"` //主数据库 存放强业务相关的数据
StatDbUrl string `json:"statDbUrl"` //统计数据库 存放统计 日志 渠道相关的数据
LogDbUrl string `json:"logDbUrl"` //日志数据库
SpiderDbUrl string `json:"spiderDbUrl"` //爬虫库
} `json:"mongo"`
Log struct {
Level string `json:"level"`
} `json:"log"`
Redis struct {
URL string `json:"url"`
} `json:"redis"`
ShortRecommend ShortRecommendConfig `json:"shortRecommend"`
DataReport struct {
Url string `json:"url"`
AppUrl string `json:"appUrl"`
} `json:"dataReport"`
DaiChong struct {
AppID string `json:"appId"`
AppSecret string `json:"appSecret"`
PlatName string `json:"platName"`
Domain string `json:"domain"`
} `json:"daichong"`
LSJ struct {
AppID string `json:"appId"`
APIKey string `json:"apiKey"`
APIUrl string `json:"apiUrl"`
ImageYuan string `json:"imageYuan"`
NoticeURL string `json:"noticeUrl"`
} `json:"lsj"`
Hevc struct {
EnableTranscode *bool `json:"enableTranscode"` // nil: 仅生产环境默认开启；true: 显式开启；false: 所有环境熔断
PullSecret string `json:"pullSecret"` // 与 App 共享的 H265 拉流签名密钥，至少 32 个随机字节
} `json:"hevc"`
AdCenter struct {
ApiDomain string `json:"apiDomain"` // api 地址
MerchantCode string `json:"merchantCode"` // 商户 code
AppCode string `json:"appCode"` // 应用 code
AesKey string `json:"aesKey"` // 数据解密 KEY
} `json:"adCenter"`
ImV2 struct {
BaseURL string `json:"baseUrl"`
DynamicConfigDomain string `json:"dynamicConfigDomain"`
SocketURL string `json:"socketUrl"`
MerchantCode string `json:"merchantCode"`
TenantCode string `json:"tenantCode"`
AppKey string `json:"appKey"`
ClientID string `json:"clientId"`
ClientSecret string `json:"clientSecret"`
} `json:"imv2"`
Elastic struct {
VideoUrl string `json:"videoUrl"`
UserName string `json:"userName"`
PassWord string `json:"passWord"`
} `json:"elastic"`
Url struct {
CheckUrl string `json:"checkUrl"`
SpiderDataURL string `json:"spiderDataUrl"` //统计中心数据同步域名
PullFileInfo string `json:"pullFileInfo"`
AiServer string `json:"aiServer"` //ai订单请求地址
AppApiUrl string `json:"appApiUrl"` // App 对外 HTTPS origin（无路径），供 H265 云转码拉取源 m3u8
} `json:"url"`
Switch struct {
NotifyExpire bool `json:"notifyExpire"`
} `json:"switch"`
StartDate string `json:"startDate"`
MediaResourcesCfg struct {
UploadMediaUrl string `json:"uploadMediaUrl"` // 请求域名
ProId string `json:"proId"` // 商户号
AppSecret string `json:"appSecret"` // 商户密钥
} `json:"mediaResourceCfg"` //媒体资源库配置
}
var (
Conf *GlobalConfig
Redis *redis.Client
VideoES *elastic.Client
Static *StaticSource
StatDB *db.MongoDB
VideoDB *db.MongoDB
LogDB *db.MongoDB
SpiderDB *db.MongoDB
Bot *tg.Bot
)
type StaticSource struct {
Portrait []string `json:"portrait"`
}
skd/skdg/static.go
+20
View File
@@ -0,0 +1,20 @@
package skdg
import (
"fmt"
"91porn-server/common/conf"
)
func InitStatic() {
initStatic()
}
func initStatic() {
static := StaticSource{}
err := conf.LoadJSON("config/static.json", &static)
if err != nil {
fmt.Printf("startUp load static error: %v\n", err)
}
Static = &static
}