Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-15 13:57:10 +08:00
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
@@ -0,0 +1,147 @@
+package audiobookspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/httputil"
+	"91porn-server/common/log"
+	"91porn-server/models/v/audioanchormod"
+	"91porn-server/models/v/audiobookmod"
+	"91porn-server/skd/skdg"
+)
+
+func SyncDataFromSpider(data []AudioBook) error {
+	dataLen := len(data)
+	if dataLen == 0 {
+		return nil
+	}
+	now := time.Now()
+	la := make([]audiobookmod.AudioBookUpsertEdit, dataLen)
+	laa := make([]audioanchormod.EditSelector, dataLen)
+	for i := range data {
+		temp := audiobookmod.AudioBookUpsertEdit{
+			Title:     data[i].Title,
+			Anchor:    data[i].Announcer,
+			Cover:     data[i].Cover,
+			FType:     data[i].Category,
+			Summary:   data[i].Desc,
+			Number:    data[i].Id,
+			UpdatedAt: now,
+		}
+		temp.TotalEpisode = len(data[i].EpisodeList)
+		if data[i].Status == 1 {
+			temp.IsActive = true
+		} else {
+			temp.IsActive = false
+		}
+		if data[i].Announcer == "" {
+			temp.IsActive = false
+		}
+		temp.ContentSet = make([]audiobookmod.Content, len(data[i].EpisodeList))
+		for di := range data[i].EpisodeList {
+			temp.ContentSet[di] = audiobookmod.Content{
+				EpisodeNumber:    di + 1,
+				ListenPermission: 0,
+				Name:             data[i].EpisodeList[di].Name,
+				URL:              data[i].EpisodeList[di].FileName,
+				MediaSize:        data[i].EpisodeList[di].Size,
+			}
+		}
+		temp.TrialSet = make([]audiobookmod.Content, len(data[i].TrialEpisodeList))
+		for di := range data[i].TrialEpisodeList {
+			temp.TrialSet[di] = audiobookmod.Content{
+				EpisodeNumber:    di + 1,
+				ListenPermission: 3,
+				Name:             data[i].TrialEpisodeList[di].Name,
+				URL:              data[i].TrialEpisodeList[di].FileName,
+				MediaSize:        data[i].TrialEpisodeList[di].Size,
+			}
+		}
+		tempa := audioanchormod.EditSelector{
+			Name:      data[i].Announcer,
+			Avatar:    data[i].AnnouncerCover,
+			UpdatedAt: now}
+		la[i] = temp
+		laa[i] = tempa
+	}
+
+	log.Info("audiobook sync audiobookSpider data committing...")
+	if err := audiobookmod.InsertBulket(la); err != nil {
+		log.Info("audiobook sync audiobookSpider data commit failed.", log.E(err))
+		return err
+	}
+	if err := audioanchormod.InsertBulket(laa); err != nil {
+		log.Info("audiobook sync audiobookSpider data commit failed.", log.E(err))
+		return err
+	}
+	log.Info("audiobook sync audiobookSpider data committed.")
+	return nil
+}
+
+type Episode struct {
+	Name     string `json:"name" bson:"name" `        //集数名称
+	Size     int64  `json:"size" bson:"size" `        //本集mp3文件大小
+	FileName string `json:"fileName" bson:"fileName"` //小说文件
+}
+
+type AudioBook struct {
+	Id               int64     `json:"id" bson:"_id"`
+	SpiderId         int64     `json:"spiderId" bson:"spiderId"`                 //爬虫库的表Id
+	Timestamp        int64     `json:"timestamp" bson:"timestamp" `              //时间戳
+	Title            string    `json:"title" bson:"title"`                       //标题
+	Announcer        string    `json:"announcer" bson:"announcer"`               //播音员
+	Category         string    `json:"category" bson:"category"`                 //分类
+	Desc             string    `json:"desc" bson:"desc"`                         //描述
+	Cover            string    `json:"cover" bson:"cover" `                      //封面
+	AnnouncerCover   string    `json:"announcerCover"`                           //播音员封面
+	EpisodeList      []Episode `json:"episodeList" bson:"episodeList" `          //集数列表
+	TrialEpisodeList []Episode `json:"trialEpisodeList" bson:"trialEpisodeList"` //试听集数列表
+	Status           int       `json:"status" bson:"status"`                     //状态  1:上架 2:下架
+	CreatedAt        time.Time `json:"createdAt" bson:"createdAt" `              //创建时间
+	UpdatedAt        time.Time `json:"updatedAt" bson:"updatedAt" `              //更新时间
+}
+
+// /api/stat/novel/audio/sync POST
+type SyncAudioBookReq struct {
+	SyncSize  int64     `json:"syncSize" binding:"required"`
+	StartTime time.Time `json:"startTime" binding:"required"`
+}
+
+type SyncAudioBookResp struct {
+	Data []AudioBook `json:"data"`
+}
+
+// todo skd
+// 获取ES同步数据
+func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []AudioBook, err error) {
+	p := SyncAudioBookReq{
+		SyncSize:  10000,
+		StartTime: start,
+	}
+	var msg struct {
+		Code int               `json:"code"`
+		Data SyncAudioBookResp `json:"data"`
+	}
+	url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/audio/sync"
+	code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
+	if err != nil || code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "POSTJsonWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+		)
+		return
+	}
+	if msg.Code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+			log.Any("msg", msg),
+		)
+		return
+	}
+	data = msg.Data.Data
+	return
+}
@@ -0,0 +1,56 @@
+package audiobookspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("Audiobook sync audiobookSpider data start...")
+	defer log.Info("Audiobook sync audiobookSpider data end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncAudiobookJob, defaultDate)
+	if err != nil {
+		log.Error("Audiobook sync audiobookSpider data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 10)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("Audiobook sync audiobookSpider commit err! ", log.E(err))
+			return
+		}
+		if len(data) > 0 {
+			lastSyncAt = data[len(data)-1].UpdatedAt
+		} else {
+			lastSyncAt = end
+		}
+	}
+}
+
+func commit(data []AudioBook, syncTime time.Time) error {
+	if err := SyncDataFromSpider(data); err != nil {
+		return fmt.Errorf("Audiobook sync audiobookSpider commit audiobook data SyncDataFromSpider failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncAudiobookJob, syncTime); err != nil {
+		return fmt.Errorf("Audiobook sync audiobookSpider commit audiobook data  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}
@@ -0,0 +1,99 @@
+package fictionspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/httputil"
+	"91porn-server/common/log"
+	"91porn-server/models/v/fictionmod"
+	"91porn-server/skd/skdg"
+)
+
+func SyncDataFromSpider(data []Novel) error {
+	now := time.Now()
+	if len(data) == 0 {
+		return nil
+	}
+	ll := make([]fictionmod.FictionUpsert, len(data))
+	for i := range data {
+		temp := fictionmod.FictionUpsert{
+			Title:      data[i].Title,
+			FType:      data[i].Type,
+			Summary:    data[i].Desc,
+			ContentURL: data[i].FileName,
+			Number:     data[i].Id,
+			UpdatedAt:  now,
+		}
+		if data[i].Status == 1 {
+			temp.IsActive = true
+		} else {
+			temp.IsActive = false
+		}
+		ll[i] = temp
+	}
+	log.Info("fiction sync fictionSpider data committing...")
+	if err := fictionmod.InsertBulket(ll); err != nil {
+		log.Info("fiction sync fictionSpider data commit failed.")
+		return err
+	}
+	log.Info("fiction sync fictionSpider data committed.")
+	return nil
+}
+
+type Novel struct {
+	Id        int64     `json:"id" bson:"_id"`
+	SpiderId  int64     `json:"spiderId" bson:"spiderId"`    //爬虫库的表Id
+	Title     string    `json:"title" bson:"title"  `        //小说标题
+	Type      string    `json:"type" bson:"type"   `         //小说类型
+	Desc      string    `json:"desc" bson:"desc"  `          //描述
+	FileName  string    `json:"fileName" bson:"fileName"  `  //小说文件
+	Timestamp int64     `json:"timestamp" bson:"timestamp" ` //时间戳
+	Status    int       `json:"status" bson:"status"`        //状态  1:上架 2:下架
+	CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间
+	UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"`  //更新时间
+}
+
+// /api/stat/novel/sync POST
+type SyncNovelReq struct {
+	SyncSize  int64     `json:"syncSize" binding:"required"`
+	StartTime time.Time `json:"startTime" binding:"required"`
+}
+
+type SyncNovelResp struct {
+	Data []Novel `json:"data"`
+}
+
+// todo skd
+// 获取ES同步数据
+func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []Novel, err error) {
+	p := SyncNovelReq{
+		SyncSize:  10000,
+		StartTime: start,
+	}
+	var msg struct {
+		Code int           `json:"code"`
+		Data SyncNovelResp `json:"data"`
+	}
+	url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/sync"
+	code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
+	if err != nil || code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+		)
+		return
+	}
+	if msg.Code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+			log.Any("msg", msg),
+		)
+		return
+	}
+	data = msg.Data.Data
+	return
+}
@@ -0,0 +1,56 @@
+package fictionspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("fiction sync fictionSpider data start...")
+	defer log.Info("fiction sync fictionSpider data end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncfictionJob, defaultDate)
+	if err != nil {
+		log.Error("fiction sync fictionSpider data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 10)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("fiction sync fictionSpider commit err! ", log.E(err))
+			return
+		}
+		if len(data) > 0 {
+			lastSyncAt = data[len(data)-1].UpdatedAt
+		} else {
+			lastSyncAt = end
+		}
+	}
+}
+
+func commit(data []Novel, syncTime time.Time) error {
+	if err := SyncDataFromSpider(data); err != nil {
+		return fmt.Errorf("fiction sync fictionSpider commit fiction data SyncDataFromSpider failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncfictionJob, syncTime); err != nil {
+		return fmt.Errorf("fiction sync fictionSpider commit fiction data  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}