100 lines
2.8 KiB
Go
100 lines
2.8 KiB
Go
package fictionspider
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"91porn-server/common/httputil"
|
|
"91porn-server/common/log"
|
|
"91porn-server/models/v/fictionmod"
|
|
"91porn-server/skd/skdg"
|
|
)
|
|
|
|
func SyncDataFromSpider(data []Novel) error {
|
|
now := time.Now()
|
|
if len(data) == 0 {
|
|
return nil
|
|
}
|
|
ll := make([]fictionmod.FictionUpsert, len(data))
|
|
for i := range data {
|
|
temp := fictionmod.FictionUpsert{
|
|
Title: data[i].Title,
|
|
FType: data[i].Type,
|
|
Summary: data[i].Desc,
|
|
ContentURL: data[i].FileName,
|
|
Number: data[i].Id,
|
|
UpdatedAt: now,
|
|
}
|
|
if data[i].Status == 1 {
|
|
temp.IsActive = true
|
|
} else {
|
|
temp.IsActive = false
|
|
}
|
|
ll[i] = temp
|
|
}
|
|
log.Info("fiction sync fictionSpider data committing...")
|
|
if err := fictionmod.InsertBulket(ll); err != nil {
|
|
log.Info("fiction sync fictionSpider data commit failed.")
|
|
return err
|
|
}
|
|
log.Info("fiction sync fictionSpider data committed.")
|
|
return nil
|
|
}
|
|
|
|
type Novel struct {
|
|
Id int64 `json:"id" bson:"_id"`
|
|
SpiderId int64 `json:"spiderId" bson:"spiderId"` //爬虫库的表Id
|
|
Title string `json:"title" bson:"title" ` //小说标题
|
|
Type string `json:"type" bson:"type" ` //小说类型
|
|
Desc string `json:"desc" bson:"desc" ` //描述
|
|
FileName string `json:"fileName" bson:"fileName" ` //小说文件
|
|
Timestamp int64 `json:"timestamp" bson:"timestamp" ` //时间戳
|
|
Status int `json:"status" bson:"status"` //状态 1:上架 2:下架
|
|
CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间
|
|
UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"` //更新时间
|
|
}
|
|
|
|
// /api/stat/novel/sync POST
|
|
type SyncNovelReq struct {
|
|
SyncSize int64 `json:"syncSize" binding:"required"`
|
|
StartTime time.Time `json:"startTime" binding:"required"`
|
|
}
|
|
|
|
type SyncNovelResp struct {
|
|
Data []Novel `json:"data"`
|
|
}
|
|
|
|
// todo skd
|
|
// 获取ES同步数据
|
|
func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []Novel, err error) {
|
|
p := SyncNovelReq{
|
|
SyncSize: 10000,
|
|
StartTime: start,
|
|
}
|
|
var msg struct {
|
|
Code int `json:"code"`
|
|
Data SyncNovelResp `json:"data"`
|
|
}
|
|
url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/sync"
|
|
code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
|
|
if err != nil || code != 200 {
|
|
log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
|
|
log.Any("start", start),
|
|
log.Any("url", end),
|
|
log.Any("code", code),
|
|
)
|
|
return
|
|
}
|
|
if msg.Code != 200 {
|
|
log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
|
|
log.Any("start", start),
|
|
log.Any("url", end),
|
|
log.Any("code", code),
|
|
log.Any("msg", msg),
|
|
)
|
|
return
|
|
}
|
|
data = msg.Data.Data
|
|
return
|
|
}
|