package fictionspider import ( "fmt" "time" "91porn-server/common/httputil" "91porn-server/common/log" "91porn-server/models/v/fictionmod" "91porn-server/skd/skdg" ) func SyncDataFromSpider(data []Novel) error { now := time.Now() if len(data) == 0 { return nil } ll := make([]fictionmod.FictionUpsert, len(data)) for i := range data { temp := fictionmod.FictionUpsert{ Title: data[i].Title, FType: data[i].Type, Summary: data[i].Desc, ContentURL: data[i].FileName, Number: data[i].Id, UpdatedAt: now, } if data[i].Status == 1 { temp.IsActive = true } else { temp.IsActive = false } ll[i] = temp } log.Info("fiction sync fictionSpider data committing...") if err := fictionmod.InsertBulket(ll); err != nil { log.Info("fiction sync fictionSpider data commit failed.") return err } log.Info("fiction sync fictionSpider data committed.") return nil } type Novel struct { Id int64 `json:"id" bson:"_id"` SpiderId int64 `json:"spiderId" bson:"spiderId"` //爬虫库的表Id Title string `json:"title" bson:"title" ` //小说标题 Type string `json:"type" bson:"type" ` //小说类型 Desc string `json:"desc" bson:"desc" ` //描述 FileName string `json:"fileName" bson:"fileName" ` //小说文件 Timestamp int64 `json:"timestamp" bson:"timestamp" ` //时间戳 Status int `json:"status" bson:"status"` //状态 1:上架 2:下架 CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间 UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"` //更新时间 } // /api/stat/novel/sync POST type SyncNovelReq struct { SyncSize int64 `json:"syncSize" binding:"required"` StartTime time.Time `json:"startTime" binding:"required"` } type SyncNovelResp struct { Data []Novel `json:"data"` } // todo skd // 获取ES同步数据 func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []Novel, err error) { p := SyncNovelReq{ SyncSize: 10000, StartTime: start, } var msg struct { Code int `json:"code"` Data SyncNovelResp `json:"data"` } url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/sync" code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p) if err != nil || code != 200 { log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err), log.Any("start", start), log.Any("url", end), log.Any("code", code), ) return } if msg.Code != 200 { log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err), log.Any("start", start), log.Any("url", end), log.Any("code", code), log.Any("msg", msg), ) return } data = msg.Data.Data return }