package updownloadser import ( "context" "errors" "mime" "net/http" "path/filepath" "runtime/debug" "strconv" "strings" "time" "91porn-server/common" "91porn-server/common/constant" "91porn-server/common/elastic" "91porn-server/common/httputil" "91porn-server/common/log" "91porn-server/common/stderr" "91porn-server/common/timeutil" v10 "91porn-server/common/v10" "91porn-server/models" "91porn-server/models/commod" "91porn-server/models/l/pullgmod" "91porn-server/models/v/fsuidmod" "91porn-server/models/v/idmod" "91porn-server/models/v/locmod" "91porn-server/models/v/tagmod" "91porn-server/models/v/usermod" "91porn-server/models/v/vidmod" "91porn-server/web/webg" "go.mongodb.org/mongo-driver/bson/primitive" ) var cities = []string{"上海", "北京", "深圳", "成都", "杭州", "重庆", "广州", "西安", "武汉"} // 从server-file 查找用户信息 func SearchUserFromFS(id string) (respBody vidmod.PublishResport, err error) { var params = map[string]string{ "uid": id, } code, err := httputil.DefaultClientGetWithResp(&respBody, webg.Conf.URL.SearchUserUrl, nil, params) log.Info("http method SearchUserFromFS response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("search user info wrong ", log.E(err)) } return } // SendVidCover2FS 传送图片 func SendVidCover2FS(id string, ext string, fileData string) (respBody commod.Resp, err error) { var params = map[string]string{ "fileData": fileData, "ext": ext, } code, err := httputil.DefaultClientPostJsonWithResp(&respBody, common.BindUrl(webg.Conf.URL.UploadImgUrl, id), nil, params) log.Info("http method SendVidCover2FS response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("file upload wrong ", log.E(err)) } return } // SendFile2FS 传送视频文件到FS func SendFile2FS(fileID string, fileData string, pos int64, total int64) (respBody vidmod.AwsResport, err error) { var header = map[string]string{ "Content-Type": "application/json", } var params = map[string]interface{}{ "taskId": fileID, "fileData": fileData, "pos": pos, "totalPos": total, "type": "sp", } c, cancle := context.WithTimeout(context.Background(), 5*time.Minute) defer cancle() code, err := httputil.DefaultClientPostJsonWithRespWithCtx(c, &respBody, webg.Conf.URL.UploadUrl, header, params) log.Info("http method SendFile2FS response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("file upload wrong ", log.E(err)) return } return } // SendImageToFS 传送图片 func SendImageToFS(fileName, fileData string) (respBody vidmod.FsSendSingleResp, err error) { var params = map[string]interface{}{ "fileData": fileData, "fileName": fileName, } code, err := httputil.DefaultClientPostJsonWithResp(&respBody, webg.Conf.URL.SendSingleFile, nil, params) log.Info("http method SendImageToFS response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("file upload wrong ", log.E(err)) } return } type FileInfo struct { FileName *string `json:"fileName"` // 文件名称 FileData *string `json:"fileData"` // 文件存储在那台服务器 Resize bool `json:"resize"` } type InfoBatch struct { Batch []*FileInfo `json:"batch"` //Info array } // SendImageToFSBatch 传送图片 func SendImageToFSBatch(batch InfoBatch) (respBody vidmod.FsSendBatchResp, err error) { if len(batch.Batch) == 0 { err = errors.New("no file") return } c, cancle := context.WithTimeout(context.Background(), 5*time.Minute) defer cancle() code, err := httputil.DefaultClientPostJsonWithRespWithCtx(c, &respBody, webg.Conf.URL.SendBatchFile, nil, batch) log.Info("http method SendImageToFSBatch response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("SendImageToFSBatch file upload wrong ", log.E(err)) } return } // FsIO 获取io.ready func FsIO(source, mds string) (data []byte, err error) { _, data, err = httputil.DefaultClientGetBytes(common.BindUrl(getOriginUrl(mds), source), nil) return data, err } func getOriginUrl(mds string) string { switch mds { case constant.MediaSourcePMS: return webg.Conf.URL.CdnUrl case constant.MediaSourceSP: return webg.Conf.URL.OriginUrl case constant.MediaSourceJH1B: return webg.Conf.URL.OriginUrl case constant.MediaSourceLaoSiJi: return webg.Conf.URL.LaoSiJiOriginUrl default: return webg.Conf.URL.OriginUrl } } // 获取马甲账号电话号码 func genMobileAndDevID(uid uint64) (string, string) { //马甲账号电话号码, 2的字符串拼接uid orgin := "12222222222" struid := strconv.FormatUint(uid, 10) l := 11 - len(struid) mobile := orgin[:l] + struid devID := usermod.SystemDevIDPrex + struid return mobile, devID } // 随机获取生日 func getBirthDate() string { var initBirthday = "1993-01-20" initTime, _ := time.Parse("2006-01-02", initBirthday) randomTime := common.RandInt(7*24*60*60, 7*365*24*60*60) return time.Unix(initTime.UTC().Unix()+int64(randomTime), 0).Format("2006-01-02") } // 游客注册 func register(fsid string) (uid uint64, city string, err error) { var portrait, summary, name string uid, err = idmod.GetNextUID() if err != nil { return } mobile, devID := genMobileAndDevID(uid) createdAt := time.Now() if len(webg.Static.Portrait) > 0 { portrait = webg.Static.Portrait[common.RandInt(0, len(webg.Static.Portrait))] } if len(webg.Static.Cities) > 0 { city = webg.Static.Cities[common.RandInt(0, len(webg.Static.Cities))] } else { city = cities[common.RandInt(0, len(cities))] } if len(webg.Static.Names) > 0 { name = webg.Static.Names[common.RandInt(0, len(webg.Static.Names))] } else { name = "游客" + strconv.FormatInt(time.Now().Unix(), 10) } publisher, err := SearchUserFromFS(fsid) if err != nil { return } if publisher.Code == http.StatusOK && publisher.Data.UserName != "" { createdAt = publisher.Data.CreatedAt portrait = publisher.Data.Portrait name = publisher.Data.UserName summary = publisher.Data.Summary } u := usermod.User{ UID: uid, DevID: devID, Name: name, Portrait: portrait, Gender: "female", Summary: summary, Birthday: getBirthDate(), Region: city, RegisterIP: "", WatchCount: 10, //初始化观看次数 默认为10 PromCode: "", DevToken: "", VideoDeduction: 3, //默认视频扣量 30% VipExpireDate: time.Unix(0, 0), DevType: "xxxx", SysType: constant.SysTypeIOS, Mobile: mobile, MobileBindAt: &createdAt, Ver: "3.0.1", } err = usermod.InsertUser(&u) return } // getUIDByFsid 获取文件服发布者id映射的uid func getUIDByFsid(fsid string) (uint64, string, error) { uid, err := fsuidmod.FindFsid2UID(fsid) if err != nil { return uid, "", err } user, err := usermod.FindUserByUID(uid) if user != nil && err == nil { return user.UID, user.Region, nil } var city string if uid == 0 { //生成用户 从fs 获取用户信息 uid, city, err = register(fsid) if err != nil { return 0, "", err } //插入映射关系 _ = fsuidmod.InsertFsid2UID(fsid, uid) } return uid, city, nil } // PullFile 同步文件 func PullFile(pullParam vidmod.PullReq) (code stderr.Code, data map[string]interface{}) { var pullResp []vidmod.AwsPullResp var failIds []string var count int var failCount int var lastID string var page = 1 awsPullReq := pullParam.AwsPullReq awsPullReq.Status = "Completed" rStatus, err := strconv.Atoi(pullParam.RStatus) if err != nil { return stderr.Failure, nil } coins, err := strconv.ParseInt(pullParam.Coins, 10, 64) if err != nil { return stderr.Failure, nil } defer func() { if lastID != "" { pullLog := pullgmod.PullLog{ LastID: lastID, Count: count, FailCount: failCount, FailIDs: failIds, CreatedAt: time.Now(), VidType: awsPullReq.Type, NewUpdatedAt: awsPullReq.NewUpdateAt, } _ = pullgmod.InsertPullLog(pullLog) } if r := recover(); r != nil { code = stderr.Failure log.Warn("pull file info occour error", log.Any("Detail", r), log.Any("stack", string(debug.Stack()))) } }() pl, err := pullgmod.FindPullRecord(awsPullReq.Type, awsPullReq.NewUpdateAt) if err != nil { return stderr.Failure, nil } if pl != nil && !pullParam.Retry { awsPullReq.ID = pl.LastID } for { awsPullReq.Page = page param := common.StructToMap(awsPullReq) resp, err := PullFileInfoFromFS(param) if err != nil || len(resp.Data) == 0 || resp.Code != http.StatusOK { break } pullResp = resp.Data city := pullParam.Cityes[common.RandInt(0, len(pullParam.Cityes))] for _, pr := range pullResp { if len(pr.Tags) == 0 { continue } tags := make([]primitive.ObjectID, 0, len(pr.Tags)) for _, t := range pr.Tags { tid, suc := HandleTag(t, pr.FieldNameFs+"-1.jpg") if !suc { continue } tags = append(tags, tid) } var userId uint64 randomUid := pullParam.UID[common.RandInt(0, len(pullParam.UID))] if pr.PublishID != "" { uid, regcity, err := getUIDByFsid(pr.PublishID) city = regcity userId = uid if err != nil { userId = randomUid user, err := usermod.FindUserByUID(userId) if user != nil && err == nil { city = user.Region } } } else { userId = randomUid user, err := usermod.FindUserByUID(userId) if user != nil && err == nil { city = user.Region } } resCode := HandleFileBase(userId, city, tags, awsPullReq.NewUpdateAt, pr, rStatus, coins) if resCode != stderr.Success { failIds = append(failIds, pr.ID) failCount++ } lastID = pr.ID count++ } page++ } data = make(map[string]interface{}) data["pullCount"] = count data["failCount"] = failCount data["lastID"] = lastID return stderr.Success, data } func PullSeriesFile(pullParam vidmod.PullReq) (code stderr.Code, data map[string]interface{}) { var pullResp []vidmod.AwsPullSeriesResp var failIds []string var count int var failCount int var lastID string var page = 1 awsPullReq := pullParam.AwsPullReq awsPullReq.Type = "SP" awsPullReq.Status = "Completed" rStatus, err := strconv.Atoi(pullParam.RStatus) if err != nil { return stderr.Failure, nil } coins, err := strconv.ParseInt(pullParam.Coins, 10, 64) if err != nil { return stderr.Failure, nil } defer func() { if lastID != "" { pullLog := pullgmod.PullLog{ LastID: lastID, Count: count, FailCount: failCount, FailIDs: failIds, CreatedAt: time.Now(), VidType: pullParam.SyncType, NewUpdatedAt: awsPullReq.NewUpdateAt, } _ = pullgmod.InsertPullLog(pullLog) } if r := recover(); r != nil { code = stderr.Failure log.Warn("pull file info occour error", log.Any("Detail", r), log.Any("stack", string(debug.Stack()))) } }() pl, err := pullgmod.FindPullRecord(pullParam.SyncType, awsPullReq.NewUpdateAt) if err != nil { return stderr.Failure, nil } if pl != nil && !pullParam.Retry { awsPullReq.ID = pl.LastID } for { awsPullReq.Page = page param := common.StructToMap(awsPullReq) resp, err := PullSeriesFromFS(param) if err != nil || len(resp.Data) == 0 || resp.Code != http.StatusOK { break } pullResp = resp.Data city := pullParam.Cityes[common.RandInt(0, len(pullParam.Cityes))] for _, pr := range pullResp { tags := make([]primitive.ObjectID, 0, len(pr.Tags)) for _, t := range pr.Tags { tid, suc := HandleTag(t, pr.CoverImg) if !suc { continue } tags = append(tags, tid) } var userId uint64 randomUid := pullParam.UID[common.RandInt(0, len(pullParam.UID))] userId = randomUid user, err := usermod.FindUserByUID(userId) if user != nil && err == nil { city = user.Region } resCode := HandleSeriesBase(userId, city, tags, awsPullReq.NewUpdateAt, pr, rStatus, coins) if resCode != stderr.Success { failIds = append(failIds, pr.ID) failCount++ } lastID = pr.ID count++ } page++ } data = make(map[string]interface{}) data["pullCount"] = count data["failCount"] = failCount data["lastID"] = lastID return stderr.Success, data } // HandleFileBase 处理文件基础信息 func HandleFileBase(uid uint64, city string, tags []primitive.ObjectID, newUpdatedAt string, req vidmod.AwsPullResp, status int, coins int64) (code stderr.Code) { defer func() { if err := recover(); err != nil { code = stderr.Failure return } }() if len(tags) == 0 { return stderr.Failure } //过滤调重复的视频 如果重复则跳过此视频 if vidmod.IsExist(req.CheckSum, req.Size) { return stderr.Failure } l := locmod.Location{ City: city, } if err := locmod.InsertLocationInfo(l); err != nil { return stderr.ErrDbInsertError } lid, err := locmod.GetLocationIDByCity(city) if err != nil { return stderr.ErrDbQueryError } if vidmod.IsExistBySocID(req.ID) { return stderr.ErrDbInputExist } code = insertBase(uid, tags, lid, newUpdatedAt, req, status, coins) for _, t := range tags { if err := tagmod.IncreaseTagVidCount(t, 1); err != nil { log.Warn("UpdateTag tag video count err %+v\n", log.E(err)) continue } } return code } // HandleSeriesBase 处理套图基础信息 func HandleSeriesBase(uid uint64, city string, tags []primitive.ObjectID, newUpdatedAt string, req vidmod.AwsPullSeriesResp, status int, coins int64) (code stderr.Code) { defer func() { if err := recover(); err != nil { code = stderr.Failure return } }() l := locmod.Location{ City: city, } if err := locmod.InsertLocationInfo(l); err != nil { return stderr.ErrDbInsertError } lid, err := locmod.GetLocationIDByCity(city) if err != nil { return stderr.ErrDbQueryError } if vidmod.IsExistBySocID(req.ID) { return stderr.ErrDbInputExist } code = insertSeriesBase(uid, tags, lid, newUpdatedAt, req, status, coins) for _, t := range tags { err := tagmod.IncreaseTagVidCount(t, 1) if err != nil { log.Warn("UpdateTag tag video count err %+v\n", log.E(err)) continue } } return code } func insertBase(uid uint64, tags []primitive.ObjectID, lid primitive.ObjectID, newUpdatedAt string, req vidmod.AwsPullResp, status int, coins int64) (code stderr.Code) { oid, _ := primitive.ObjectIDFromHex(req.ID) flagTime := timeutil.RandReduceTime(time.Now(), 30*24*time.Hour) v := vidmod.VideoModel{ ID: oid, PublisherID: uid, NewsType: vidmod.SP, Title: strings.TrimSpace(req.Title), Tags: tags, SourceID: req.ID, Cover: req.FieldNameFs + "-1.jpg", CoverThumb: req.FieldNameFs + "-2.jpg", SourceURL: req.FieldNameFs + ".m3u8", Filename: req.Filename, PlayTime: req.PlayTime, SeriesCover: req.CoverImg, Activity: []primitive.ObjectID{}, MimeType: mime.TypeByExtension(strings.ToLower(filepath.Ext(req.Filename))), Resolution: strconv.FormatInt(int64(req.Width), 10) + "*" + strconv.FormatInt(int64(req.Height), 10), Width: req.Width, Height: req.Height, Ratio: req.Ratio, Via: req.Via, Rating: 0, PlayCount: 0, LikeCount: 0, FakeLikeCount: common.RandInt(50, 500), CommentCount: 0, FakePlayCount: common.RandInt(200, 3000), ShareCount: common.RandInt(500, 10000), Status: status, Location: lid, FreeTime: vidmod.MinFreeTime, Coins: coins, Size: req.Size, MD5: req.CheckSum, Actor: strings.Join(req.Actors, ","), NewUpdatedAt: newUpdatedAt, CreatedAt: flagTime, UpdatedAt: flagTime, } if _, err := vidmod.InsertBase(v); err != nil { return stderr.ErrDbInsertError } return stderr.Success } func insertSeriesBase(uid uint64, tags []primitive.ObjectID, lid primitive.ObjectID, newUpdatedAt string, req vidmod.AwsPullSeriesResp, status int, coins int64) (code stderr.Code) { oid, _ := primitive.ObjectIDFromHex(req.ID) flagTime := timeutil.RandAddTime(time.Now(), 3*24*time.Hour) v := vidmod.VideoModel{ ID: oid, PublisherID: uid, NewsType: vidmod.COVER, Title: strings.TrimSpace(req.Title), Tags: tags, SourceID: req.ID, Cover: req.CoverImg, SeriesCover: req.SeriesCover, SeriesNum: int(req.Number), Activity: []primitive.ObjectID{}, MimeType: req.MimeType, Via: req.Via, Rating: 0, PlayCount: 0, LikeCount: 0, FakeLikeCount: common.RandInt(50, 500), CommentCount: 0, FakePlayCount: common.RandInt(200, 3000), ShareCount: common.RandInt(300, 10000), Status: status, Location: lid, FreeTime: vidmod.MinFreeTime, Coins: coins, NewUpdatedAt: newUpdatedAt, CreatedAt: flagTime, UpdatedAt: flagTime, } if _, err := vidmod.InsertBase(v); err != nil { return stderr.ErrDbInsertError } return stderr.Success } // HandleTag 处理标签 func HandleTag(tagname, img string) (tagID primitive.ObjectID, suc bool) { tMod := tagmod.Tag{} tagInfo, err := tagmod.FindOneTagByName(strings.TrimSpace(tagname)) if err == nil && !tagInfo.ID.IsZero() { return tagInfo.ID, true } tMod.TagName = v10.ExtractPureChar(tagname) tMod.CoverImg = img tMod.SortCode = 999 tMod.IsActive = true tMod.CreatedAt = time.Now() tMod.UpdatedAt = time.Now() id, err := tagmod.InsertOne(&tMod) if err != nil { return primitive.NilObjectID, false } return id, true } // PullFileInfoFromFS PullFileInfoFromFS func PullFileInfoFromFS(params map[string]interface{}) (respBody vidmod.Resport, err error) { code, err := httputil.DefaultClientPostJsonWithResp(&respBody, webg.Conf.URL.PullFileUrl, nil, params) log.Info("http method PullFileInfoFromFS response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("Pull FileInfo From Aws ", log.E(err)) return } return } // PullSeriesFromFS PullSeriesFromFS func PullSeriesFromFS(params map[string]interface{}) (respBody vidmod.SeriesResport, err error) { code, err := httputil.DefaultClientPostJsonWithResp(&respBody, webg.Conf.URL.PullSeriesUrl, nil, params) log.Info("http method PullSeriesFromFS response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("Pull Series From Aws ", log.E(err)) return } return } // PullFileInfo 获取文件转码情况及详情 func PullFileInfo(id string) (respBody vidmod.PuFinfoResp, err error) { code, err := httputil.DefaultClientPostJsonWithResp(&respBody, common.BindUrl(webg.Conf.URL.PullFileInfo, id), nil, nil) log.Info("http method PullFileInfo response code ==>", log.Any("statusCode", code), log.Any("respCode", respBody.Code)) if err != nil { log.Error("Pull FileInfo From Aws ", log.E(err)) respBody.Code = stderr.ErrConnectToFs return } switch respBody.Data.Status { case vidmod.Merging, vidmod.MergeCompleted: respBody.Code = stderr.FileMerging case vidmod.MergeError: respBody.Code = stderr.ErrMergeFile case vidmod.Converting: respBody.Code = stderr.FileConverting case vidmod.ConvertError: respBody.Code = stderr.ErrConvertFile case vidmod.UploadLoadingToFs: respBody.Code = stderr.UploadLoadingToFs case vidmod.FileUploadError: respBody.Code = stderr.ErrUploadError case vidmod.ConvertCompleted, vidmod.Completed: respBody.Code = stderr.UpLoadFileComplete } return } // 新导入的视频数据同步到ES func SyncNewImportVideoToElastic(newUpdatedAt string, start, end time.Time) { var page, pageSize int64 = 1, 50 for { nVideos, err := vidmod.GetNewImportVideo(newUpdatedAt, start, end, page, pageSize) if err != nil || len(nVideos) == 0 { return } // 获取标签名字 var tagsID []primitive.ObjectID for _, v := range nVideos { tagsID = append(tagsID, v.Tags...) } tags, err := tagmod.FindTagsByIDS(tagsID) if err != nil { return } var tagsNameMap = make(map[primitive.ObjectID]string) for _, v := range tags { tagsNameMap[v.ID] = v.TagName } var source = elastic.M{} for _, v := range nVideos { var tagsName []string for _, v1 := range v.Tags { tagsName = append(tagsName, tagsNameMap[v1]) } var tmp = vidmod.ESVideo{ ID: v.ID, PublisherID: v.PublisherID, Title: v.Title, Tags: v.Tags, TagsName: tagsName, Filename: v.Filename, PlayCount: v.PlayCount, PurchaseCount: v.PurchaseCount, LikeCount: v.LikeCount, CommentCount: v.CommentCount, ShareCount: v.ShareCount, FakeLikeCount: v.FakeLikeCount, FakeShareCount: v.FakeShareCount, FakePlayCount: v.FakePlayCount, Status: v.Status, Location: v.Location, CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, } source[v.ID.Hex()] = tmp } if err = webg.VideoES.Bulk(models.ESInfoVideoTable, source); err != nil { continue } log.Info("sync video info to elastic successfully running at page", log.Any("page", page)) page++ } }