Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-15 13:57:10 +08:00
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
@@ -0,0 +1,99 @@
+package fictionspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/httputil"
+	"91porn-server/common/log"
+	"91porn-server/models/v/fictionmod"
+	"91porn-server/skd/skdg"
+)
+
+func SyncDataFromSpider(data []Novel) error {
+	now := time.Now()
+	if len(data) == 0 {
+		return nil
+	}
+	ll := make([]fictionmod.FictionUpsert, len(data))
+	for i := range data {
+		temp := fictionmod.FictionUpsert{
+			Title:      data[i].Title,
+			FType:      data[i].Type,
+			Summary:    data[i].Desc,
+			ContentURL: data[i].FileName,
+			Number:     data[i].Id,
+			UpdatedAt:  now,
+		}
+		if data[i].Status == 1 {
+			temp.IsActive = true
+		} else {
+			temp.IsActive = false
+		}
+		ll[i] = temp
+	}
+	log.Info("fiction sync fictionSpider data committing...")
+	if err := fictionmod.InsertBulket(ll); err != nil {
+		log.Info("fiction sync fictionSpider data commit failed.")
+		return err
+	}
+	log.Info("fiction sync fictionSpider data committed.")
+	return nil
+}
+
+type Novel struct {
+	Id        int64     `json:"id" bson:"_id"`
+	SpiderId  int64     `json:"spiderId" bson:"spiderId"`    //爬虫库的表Id
+	Title     string    `json:"title" bson:"title"  `        //小说标题
+	Type      string    `json:"type" bson:"type"   `         //小说类型
+	Desc      string    `json:"desc" bson:"desc"  `          //描述
+	FileName  string    `json:"fileName" bson:"fileName"  `  //小说文件
+	Timestamp int64     `json:"timestamp" bson:"timestamp" ` //时间戳
+	Status    int       `json:"status" bson:"status"`        //状态  1:上架 2:下架
+	CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间
+	UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"`  //更新时间
+}
+
+// /api/stat/novel/sync POST
+type SyncNovelReq struct {
+	SyncSize  int64     `json:"syncSize" binding:"required"`
+	StartTime time.Time `json:"startTime" binding:"required"`
+}
+
+type SyncNovelResp struct {
+	Data []Novel `json:"data"`
+}
+
+// todo skd
+// 获取ES同步数据
+func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []Novel, err error) {
+	p := SyncNovelReq{
+		SyncSize:  10000,
+		StartTime: start,
+	}
+	var msg struct {
+		Code int           `json:"code"`
+		Data SyncNovelResp `json:"data"`
+	}
+	url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/sync"
+	code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
+	if err != nil || code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+		)
+		return
+	}
+	if msg.Code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+			log.Any("msg", msg),
+		)
+		return
+	}
+	data = msg.Data.Data
+	return
+}
@@ -0,0 +1,56 @@
+package fictionspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("fiction sync fictionSpider data start...")
+	defer log.Info("fiction sync fictionSpider data end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncfictionJob, defaultDate)
+	if err != nil {
+		log.Error("fiction sync fictionSpider data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 10)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("fiction sync fictionSpider commit err! ", log.E(err))
+			return
+		}
+		if len(data) > 0 {
+			lastSyncAt = data[len(data)-1].UpdatedAt
+		} else {
+			lastSyncAt = end
+		}
+	}
+}
+
+func commit(data []Novel, syncTime time.Time) error {
+	if err := SyncDataFromSpider(data); err != nil {
+		return fmt.Errorf("fiction sync fictionSpider commit fiction data SyncDataFromSpider failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncfictionJob, syncTime); err != nil {
+		return fmt.Errorf("fiction sync fictionSpider commit fiction data  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}