diff --git a/internal/app/api/handler/bilbil_picture.go b/internal/app/api/handler/bilbil_picture.go index 7649a37..73aecd1 100644 --- a/internal/app/api/handler/bilbil_picture.go +++ b/internal/app/api/handler/bilbil_picture.go @@ -17,7 +17,6 @@ func BilibiliLatestPics(s *service.BilbilPicture) func(ctx *gin.Context) { _ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err)) return } - if resp, err := s.Latest(ctx, req); err != nil { _ = ctx.Error(err) return @@ -30,7 +29,11 @@ func BilibiliLatestPics(s *service.BilbilPicture) func(ctx *gin.Context) { func BilibiliRecommendPics(s *service.BilbilPicture) func(ctx *gin.Context) { return func(ctx *gin.Context) { var req idl.BilibiliPictureRecommendReq - if resp, err := s.Recommend(ctx,req); err != nil { + if err := ctx.ShouldBindQuery(&req); err != nil { + _ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err)) + return + } + if resp, err := s.Recommend(ctx, req); err != nil { _ = ctx.Error(err) return } else { diff --git a/internal/app/api/idl/bilibili_picture.go b/internal/app/api/idl/bilibili_picture.go index 52d09da..8394b1a 100644 --- a/internal/app/api/idl/bilibili_picture.go +++ b/internal/app/api/idl/bilibili_picture.go @@ -48,6 +48,7 @@ type BilibiliPictureLatestReq struct { } type BilibiliPictureRecommendReq struct { + Page int `form:"page,default=1" binding:"omitempty,gt=0"` TopicID int `form:"topic_id"` } type BilibiliPicturesCommonResp struct { @@ -61,6 +62,7 @@ type BilibiliPicturesLatestResp struct { type BilibiliPicturesRecommendResp struct { BilibiliPicturesCommonResp + Page int `json:"page"` Total int `json:"total"` } type BilibiliDynamicDTO struct { @@ -74,7 +76,7 @@ type BilibiliDynamicPicture struct { Height float64 `json:"img_height"` Size float64 `json:"img_size"` Width float64 `json:"img_width"` - ImgSrc string `json:"image_src"` + ImgSrc string `json:"img_src"` } func (BilibiliDynamic) TableName() string { @@ -87,6 +89,5 @@ type BilibiliPictureRepository interface { Update(updates map[string]interface{}, dynamicID uint64) error FindAllByPubDate(from, to time.Time, page, size int64) (list []*BilibiliDynamic, err error) Latest(page, size, topicID int) (list []*BilibiliDynamic, err error) - //推荐暂时先默认50个 - Recommend(from, to time.Time, size, topicID int) (list []*BilibiliDynamic, err error) + Recommend(from, to time.Time, page, size, topicID int) (list []*BilibiliDynamic, err error) } diff --git a/internal/app/api/service/bilbil_picture.go b/internal/app/api/service/bilbil_picture.go index d460ce3..e788f85 100644 --- a/internal/app/api/service/bilbil_picture.go +++ b/internal/app/api/service/bilbil_picture.go @@ -10,7 +10,7 @@ import ( ) const ( - picRecommendDefaultSize = 50 + picRecommendDefaultSize = 20 ) type BilbilPicture struct { @@ -46,12 +46,13 @@ func (service *BilbilPicture) Recommend(ctx context.Context, req idl.BilibiliPic tx := service.db.WithContext(ctx) picRepository := repository.NewBilibiliPicture(tx) now := time.Now() - list, err := picRepository.Recommend(now.Add(-(3 * 24 * time.Hour)), now, picRecommendDefaultSize, req.TopicID) + list, err := picRepository.Recommend(now.Add(-(3 * 24 * time.Hour)), now, req.Page, picRecommendDefaultSize, req.TopicID) if err != nil { return nil, err } resp := idl.BilibiliPicturesRecommendResp{ Total: len(list), + Page: req.Page, } for i := range list { resp.Result = append(resp.Result, &idl.BilibiliDynamicDTO{ diff --git a/internal/app/spider/picture.go b/internal/app/spider/picture.go index 30ca805..9967aba 100644 --- a/internal/app/spider/picture.go +++ b/internal/app/spider/picture.go @@ -119,15 +119,14 @@ func (p *Picture) spider() error { for _, v := range data.Cards { switch v.Desc.Type { case bilibili.DynamicDraw: - dynamicID, _ := strconv.ParseUint(v.Desc.DynamicID, 10, 64) - if dynamicID <= *curMaxDynamicID { + if v.Desc.DynamicID <= *curMaxDynamicID { //后面所有的都是爬过的，提前结束，后续也不再请求api exist = true break } dynamic := &idl.BilibiliDynamic{ UID: v.Desc.UID, - DynamicID: dynamicID, + DynamicID: v.Desc.DynamicID, TopicName: topicName, TopicID: topicID, View: v.Desc.View, diff --git a/internal/app/spider/picture_test.go b/internal/app/spider/picture_test.go new file mode 100644 index 0000000..29cf4a6 --- /dev/null +++ b/internal/app/spider/picture_test.go @@ -0,0 +1,75 @@ +package spider + +import ( + "testing" + + "git.vtb.link/eoefans/internal/app/api/idl" + "git.vtb.link/eoefans/internal/pkg/bilibili" + "git.vtb.link/eoefans/internal/pkg/database" + "git.vtb.link/eoefans/internal/repository" + "go.uber.org/zap" +) + +func TestFindCurMaxDynamicIDByTopicName(t *testing.T) { + db, err := database.NewDatabase(&database.Options{ + Type: "mysql", + DSN: "root:123456@tcp(127.0.0.1:3306)/eoes?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai", + Debug: true, + SetMaxIdleConns: 2, + SetMaxOpenConns: 4, + SetConnMaxLifetime: 6, + }) + if err != nil { + t.Error(err) + return + } + id, err := repository.NewBilibiliPicture(db).FindMaxDynamicID("EOE的魔法盒") + if err != nil { + t.Error(err) + return + } + if id != nil { + t.Log(*id) + } +} +func TestInsertPicture(t *testing.T) { + db, err := database.NewDatabase(&database.Options{ + Type: "mysql", + DSN: "root:123456@tcp(127.0.0.1:3306)/eoes?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai", + Debug: true, + SetMaxIdleConns: 2, + SetMaxOpenConns: 4, + SetConnMaxLifetime: 6, + }) + if err != nil { + t.Error(err) + return + } + sdk := bilibili.NewSDK(&zap.Logger{}) + var dynamicID uint64 = 752843351274815520 + res, err := sdk.Dynamic(uint64(dynamicID)) + if err != nil { + t.Error(err) + return + } + pictures, err := parsePicturesFromCard(res.Card.Card) + if err != nil { + t.Error(err) + return + } + items := []*idl.BilibiliDynamic{{ + UID: res.Card.Desc.UID, + DynamicID: 752843351274815520, + Pictures: pictures, + View: res.Card.Desc.View, + Repost: res.Card.Desc.Repost, + Comment: res.Card.Desc.Comment, + Like: res.Card.Desc.Like, + SentAt: res.Card.Desc.TimeStamp, + }} + err = repository.NewBilibiliPicture(db).Create(items) + if err != nil { + t.Error(err) + return + } +} diff --git a/internal/app/spider/update_dynamic.go b/internal/app/spider/update_dynamic.go index 76986ab..be6b572 100644 --- a/internal/app/spider/update_dynamic.go +++ b/internal/app/spider/update_dynamic.go @@ -80,16 +80,16 @@ func (u *UpdateDynamic) spider() error { return nil } for _, v := range list { - dynamicCard, err := u.sdk.Dynamic(v.DynamicID) + dynamic, err := u.sdk.Dynamic(v.DynamicID) if err != nil { u.logger.Error("Dynamic error", zap.Int("dynamic_id", int(v.DynamicID)), zap.Error(err)) continue } updates := map[string]interface{}{ - "view": dynamicCard.Desc.View, - "repost": dynamicCard.Desc.Repost, - "comment": dynamicCard.Desc.Comment, - "like": dynamicCard.Desc.Like, + "view": dynamic.Card.Desc.View, + "repost": dynamic.Card.Desc.Repost, + "comment": dynamic.Card.Desc.Comment, + "like": dynamic.Card.Desc.Like, } if err := repo.Update(updates, v.DynamicID); err != nil { u.logger.Error("Dynamic Update error", zap.Int("dynamic_id", int(v.DynamicID)), zap.Error(err)) diff --git a/internal/pkg/bilibili/api_test.go b/internal/pkg/bilibili/api_test.go new file mode 100644 index 0000000..7ba11d2 --- /dev/null +++ b/internal/pkg/bilibili/api_test.go @@ -0,0 +1,30 @@ +package bilibili + +import ( + "testing" + + "go.uber.org/zap" +) + +func TestDynamic(t *testing.T) { + sdk := NewSDK(&zap.Logger{}) + var dynamicID uint64 = 752843351274815520 + res, err := sdk.Dynamic(uint64(dynamicID)) + if err != nil { + t.Error(err) + return + } + t.Log(res) +} + +func TestDynamicList(t *testing.T) { + sdk := NewSDK(&zap.Logger{}) + name := "EOE的魔法盒" + offset := 0 + res, err := sdk.TopicDynamics(name, uint64(offset)) + if err != nil { + t.Error(err) + return + } + t.Log(len(res.Cards)) +} diff --git a/internal/pkg/bilibili/video.go b/internal/pkg/bilibili/video.go index 8d8b3b8..1abf454 100644 --- a/internal/pkg/bilibili/video.go +++ b/internal/pkg/bilibili/video.go @@ -60,12 +60,15 @@ type ResponseBasic struct { Data interface{} `json:"data"` } -type DynamicInfo struct { +type DynamicList struct { Cards []DynamicCard `json:"cards"` HasMore uint `json:"has_more"` Offset string `json:"offset"` } +type Dynamic struct { + Card DynamicCard `json:"card"` +} type DynamicCard struct { Desc struct { Type DynamicType `json:"type"` @@ -74,7 +77,7 @@ type DynamicCard struct { Comment uint64 `json:"comment"` Like uint64 `json:"like"` UID uint64 `json:"uid"` - DynamicID string `json:"dynamic_id"` + DynamicID uint64 `json:"dynamic_id"` TimeStamp uint64 `json:"timestamp"` } `json:"desc"` Card string `json:"card"` @@ -92,7 +95,7 @@ type DynamicPicture struct { Height float64 `json:"img_height"` Size float64 `json:"img_size"` Width float64 `json:"img_width"` - ImgSrc string `json:"image_src"` + ImgSrc string `json:"img_src"` } type VideoSearchInfo struct { Type string `json:"type"` @@ -350,7 +353,7 @@ func (sdk *SDK) VideoWebTagInfo(aid string) (data *VideoTagResponse, err error) return data, nil } -func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *DynamicInfo, err error) { +func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *DynamicList, err error) { params := url.Values{} params.Add("topic_name", topicName) url := fmt.Sprintf(topicHistory, offsetDynamicId) @@ -361,8 +364,9 @@ func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *D return data, nil } -func (sdk *SDK) Dynamic(dynamicId uint64) (data *DynamicCard, err error) { - url := fmt.Sprintf(topicHistory, dynamicId) +func (sdk *SDK) Dynamic(dynamicId uint64) (data *Dynamic, err error) { + url := fmt.Sprintf(dynamic, dynamicId) + fmt.Println(url) if err = sdk.fastGet(url, &data); err != nil { return nil, err } diff --git a/internal/repository/bilibili_picture.go b/internal/repository/bilibili_picture.go index 2b083e7..e526771 100644 --- a/internal/repository/bilibili_picture.go +++ b/internal/repository/bilibili_picture.go @@ -57,9 +57,13 @@ func (impl *BilibiliPictureMysqlImpl) Latest(page, size, topicID int) (list []*i if topicID != 0 { conn = conn.Where("topic_id = ?", topicID) } + offset := (page - 1) * size + if offset < 0 { + offset = -1 + } err = conn.Select("dynamic_id,pictures,sent_at"). Order("sent_at DESC"). - Offset((page - 1) * size). + Offset(offset). Limit(size).Find(&list).Error if err != nil { return nil, err @@ -67,14 +71,19 @@ func (impl *BilibiliPictureMysqlImpl) Latest(page, size, topicID int) (list []*i return list, nil } -func (impl *BilibiliPictureMysqlImpl) Recommend(from, to time.Time, size, topicID int) (list []*idl.BilibiliDynamic, err error) { +func (impl *BilibiliPictureMysqlImpl) Recommend(from, to time.Time, page, size, topicID int) (list []*idl.BilibiliDynamic, err error) { conn := impl.tx.Table(idl.BilibiliDynamic{}.TableName()) if topicID != 0 { conn = conn.Where("topic_id = ?", topicID) } + offset := (page - 1) * size + if offset < 0 { + offset = -1 + } err = conn.Select("dynamic_id,pictures,sent_at"). Where("sent_at >= ? AND sent_at <= ?", from.Unix(), to.Unix()). Order("favor DESC"). + Offset(offset). Limit(size).Find(&list).Error if err != nil { return nil, err