[feature] 调整图片推荐接口

internal/app/api/handler/bilbil_picture.go

@@ -17,7 +17,6 @@ func BilibiliLatestPics(s *service.BilbilPicture) func(ctx *gin.Context) {
 			_ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err))
 			return
 		}
-
 		if resp, err := s.Latest(ctx, req); err != nil {
 			_ = ctx.Error(err)
 			return
@@ -30,7 +29,11 @@ func BilibiliLatestPics(s *service.BilbilPicture) func(ctx *gin.Context) {
 func BilibiliRecommendPics(s *service.BilbilPicture) func(ctx *gin.Context) {
 	return func(ctx *gin.Context) {
 		var req idl.BilibiliPictureRecommendReq
-		if resp, err := s.Recommend(ctx,req); err != nil {
+		if err := ctx.ShouldBindQuery(&req); err != nil {
+			_ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err))
+			return
+		}
+		if resp, err := s.Recommend(ctx, req); err != nil {
 			_ = ctx.Error(err)
 			return
 		} else {

internal/app/api/idl/bilibili_picture.go

@@ -48,6 +48,7 @@ type BilibiliPictureLatestReq struct {
 }
 
 type BilibiliPictureRecommendReq struct {
+	Page    int `form:"page,default=1" binding:"omitempty,gt=0"`
 	TopicID int `form:"topic_id"`
 }
 type BilibiliPicturesCommonResp struct {
@@ -61,6 +62,7 @@ type BilibiliPicturesLatestResp struct {
 
 type BilibiliPicturesRecommendResp struct {
 	BilibiliPicturesCommonResp
+	Page  int `json:"page"`
 	Total int `json:"total"`
 }
 type BilibiliDynamicDTO struct {
@@ -74,7 +76,7 @@ type BilibiliDynamicPicture struct {
 	Height float64 `json:"img_height"`
 	Size   float64 `json:"img_size"`
 	Width  float64 `json:"img_width"`
-	ImgSrc string  `json:"image_src"`
+	ImgSrc string  `json:"img_src"`
 }
 
 func (BilibiliDynamic) TableName() string {
@@ -87,6 +89,5 @@ type BilibiliPictureRepository interface {
 	Update(updates map[string]interface{}, dynamicID uint64) error
 	FindAllByPubDate(from, to time.Time, page, size int64) (list []*BilibiliDynamic, err error)
 	Latest(page, size, topicID int) (list []*BilibiliDynamic, err error)
-	//推荐暂时先默认50个
-	Recommend(from, to time.Time, size, topicID int) (list []*BilibiliDynamic, err error)
+	Recommend(from, to time.Time, page, size, topicID int) (list []*BilibiliDynamic, err error)
 }

internal/app/api/service/bilbil_picture.go

@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	picRecommendDefaultSize = 50
+	picRecommendDefaultSize = 20
 )
 
 type BilbilPicture struct {
@@ -46,12 +46,13 @@ func (service *BilbilPicture) Recommend(ctx context.Context, req idl.BilibiliPic
 	tx := service.db.WithContext(ctx)
 	picRepository := repository.NewBilibiliPicture(tx)
 	now := time.Now()
-	list, err := picRepository.Recommend(now.Add(-(3 * 24 * time.Hour)), now, picRecommendDefaultSize, req.TopicID)
+	list, err := picRepository.Recommend(now.Add(-(3 * 24 * time.Hour)), now, req.Page, picRecommendDefaultSize, req.TopicID)
 	if err != nil {
 		return nil, err
 	}
 	resp := idl.BilibiliPicturesRecommendResp{
 		Total: len(list),
+		Page:  req.Page,
 	}
 	for i := range list {
 		resp.Result = append(resp.Result, &idl.BilibiliDynamicDTO{

internal/app/spider/picture.go

@@ -119,15 +119,14 @@ func (p *Picture) spider() error {
 			for _, v := range data.Cards {
 				switch v.Desc.Type {
 				case bilibili.DynamicDraw:
-					dynamicID, _ := strconv.ParseUint(v.Desc.DynamicID, 10, 64)
-					if dynamicID <= *curMaxDynamicID {
+					if v.Desc.DynamicID <= *curMaxDynamicID {
 						//后面所有的都是爬过的，提前结束，后续也不再请求api
 						exist = true
 						break
 					}
 					dynamic := &idl.BilibiliDynamic{
 						UID:       v.Desc.UID,
-						DynamicID: dynamicID,
+						DynamicID: v.Desc.DynamicID,
 						TopicName: topicName,
 						TopicID:   topicID,
 						View:      v.Desc.View,

internal/app/spider/picture_test.go

@@ -0,0 +1,75 @@
+package spider
+
+import (
+	"testing"
+
+	"git.vtb.link/eoefans/internal/app/api/idl"
+	"git.vtb.link/eoefans/internal/pkg/bilibili"
+	"git.vtb.link/eoefans/internal/pkg/database"
+	"git.vtb.link/eoefans/internal/repository"
+	"go.uber.org/zap"
+)
+
+func TestFindCurMaxDynamicIDByTopicName(t *testing.T) {
+	db, err := database.NewDatabase(&database.Options{
+		Type:               "mysql",
+		DSN:                "root:123456@tcp(127.0.0.1:3306)/eoes?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai",
+		Debug:              true,
+		SetMaxIdleConns:    2,
+		SetMaxOpenConns:    4,
+		SetConnMaxLifetime: 6,
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	id, err := repository.NewBilibiliPicture(db).FindMaxDynamicID("EOE的魔法盒")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if id != nil {
+		t.Log(*id)
+	}
+}
+func TestInsertPicture(t *testing.T) {
+	db, err := database.NewDatabase(&database.Options{
+		Type:               "mysql",
+		DSN:                "root:123456@tcp(127.0.0.1:3306)/eoes?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai",
+		Debug:              true,
+		SetMaxIdleConns:    2,
+		SetMaxOpenConns:    4,
+		SetConnMaxLifetime: 6,
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	sdk := bilibili.NewSDK(&zap.Logger{})
+	var dynamicID uint64 = 752843351274815520
+	res, err := sdk.Dynamic(uint64(dynamicID))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	pictures, err := parsePicturesFromCard(res.Card.Card)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	items := []*idl.BilibiliDynamic{{
+		UID:       res.Card.Desc.UID,
+		DynamicID: 752843351274815520,
+		Pictures:  pictures,
+		View:      res.Card.Desc.View,
+		Repost:    res.Card.Desc.Repost,
+		Comment:   res.Card.Desc.Comment,
+		Like:      res.Card.Desc.Like,
+		SentAt:    res.Card.Desc.TimeStamp,
+	}}
+	err = repository.NewBilibiliPicture(db).Create(items)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+}

internal/app/spider/update_dynamic.go

@@ -80,16 +80,16 @@ func (u *UpdateDynamic) spider() error {
 			return nil
 		}
 		for _, v := range list {
-			dynamicCard, err := u.sdk.Dynamic(v.DynamicID)
+			dynamic, err := u.sdk.Dynamic(v.DynamicID)
 			if err != nil {
 				u.logger.Error("Dynamic error", zap.Int("dynamic_id", int(v.DynamicID)), zap.Error(err))
 				continue
 			}
 			updates := map[string]interface{}{
-				"view":    dynamicCard.Desc.View,
-				"repost":  dynamicCard.Desc.Repost,
-				"comment": dynamicCard.Desc.Comment,
-				"like":    dynamicCard.Desc.Like,
+				"view":    dynamic.Card.Desc.View,
+				"repost":  dynamic.Card.Desc.Repost,
+				"comment": dynamic.Card.Desc.Comment,
+				"like":    dynamic.Card.Desc.Like,
 			}
 			if err := repo.Update(updates, v.DynamicID); err != nil {
 				u.logger.Error("Dynamic Update error", zap.Int("dynamic_id", int(v.DynamicID)), zap.Error(err))

internal/pkg/bilibili/api_test.go

@@ -0,0 +1,30 @@
+package bilibili
+
+import (
+	"testing"
+
+	"go.uber.org/zap"
+)
+
+func TestDynamic(t *testing.T) {
+	sdk := NewSDK(&zap.Logger{})
+	var dynamicID uint64 = 752843351274815520
+	res, err := sdk.Dynamic(uint64(dynamicID))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	t.Log(res)
+}
+
+func TestDynamicList(t *testing.T) {
+	sdk := NewSDK(&zap.Logger{})
+	name := "EOE的魔法盒"
+	offset := 0
+	res, err := sdk.TopicDynamics(name, uint64(offset))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	t.Log(len(res.Cards))
+}

internal/pkg/bilibili/video.go

@@ -60,12 +60,15 @@ type ResponseBasic struct {
 	Data    interface{} `json:"data"`
 }
 
-type DynamicInfo struct {
+type DynamicList struct {
 	Cards   []DynamicCard `json:"cards"`
 	HasMore uint          `json:"has_more"`
 	Offset  string        `json:"offset"`
 }
 
+type Dynamic struct {
+	Card DynamicCard `json:"card"`
+}
 type DynamicCard struct {
 	Desc struct {
 		Type      DynamicType `json:"type"`
@@ -74,7 +77,7 @@ type DynamicCard struct {
 		Comment   uint64      `json:"comment"`
 		Like      uint64      `json:"like"`
 		UID       uint64      `json:"uid"`
-		DynamicID string      `json:"dynamic_id"`
+		DynamicID uint64      `json:"dynamic_id"`
 		TimeStamp uint64      `json:"timestamp"`
 	} `json:"desc"`
 	Card string `json:"card"`
@@ -92,7 +95,7 @@ type DynamicPicture struct {
 	Height float64 `json:"img_height"`
 	Size   float64 `json:"img_size"`
 	Width  float64 `json:"img_width"`
-	ImgSrc string  `json:"image_src"`
+	ImgSrc string  `json:"img_src"`
 }
 type VideoSearchInfo struct {
 	Type         string        `json:"type"`
@@ -350,7 +353,7 @@ func (sdk *SDK) VideoWebTagInfo(aid string) (data *VideoTagResponse, err error)
 	return data, nil
 }
 
-func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *DynamicInfo, err error) {
+func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *DynamicList, err error) {
 	params := url.Values{}
 	params.Add("topic_name", topicName)
 	url := fmt.Sprintf(topicHistory, offsetDynamicId)
@@ -361,8 +364,9 @@ func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *D
 	return data, nil
 }
 
-func (sdk *SDK) Dynamic(dynamicId uint64) (data *DynamicCard, err error) {
-	url := fmt.Sprintf(topicHistory, dynamicId)
+func (sdk *SDK) Dynamic(dynamicId uint64) (data *Dynamic, err error) {
+	url := fmt.Sprintf(dynamic, dynamicId)
+	fmt.Println(url)
 	if err = sdk.fastGet(url, &data); err != nil {
 		return nil, err
 	}

internal/repository/bilibili_picture.go

@@ -57,24 +57,33 @@ func (impl *BilibiliPictureMysqlImpl) Latest(page, size, topicID int) (list []*i
 	if topicID != 0 {
 		conn = conn.Where("topic_id = ?", topicID)
 	}
+	offset := (page - 1) * size
+	if offset < 0 {
+		offset = -1
+	}
 	err = conn.Select("dynamic_id,pictures,sent_at").
 		Order("sent_at DESC").
-		Offset((page - 1) * size).
+		Offset(offset).
 		Limit(size).Find(&list).Error
 	if err != nil {
 		return nil, err
 	}
 	return list, nil
 }
 
-func (impl *BilibiliPictureMysqlImpl) Recommend(from, to time.Time, size, topicID int) (list []*idl.BilibiliDynamic, err error) {
+func (impl *BilibiliPictureMysqlImpl) Recommend(from, to time.Time, page, size, topicID int) (list []*idl.BilibiliDynamic, err error) {
 	conn := impl.tx.Table(idl.BilibiliDynamic{}.TableName())
 	if topicID != 0 {
 		conn = conn.Where("topic_id = ?", topicID)
 	}
+	offset := (page - 1) * size
+	if offset < 0 {
+		offset = -1
+	}
 	err = conn.Select("dynamic_id,pictures,sent_at").
 		Where("sent_at >= ? AND sent_at <= ?", from.Unix(), to.Unix()).
 		Order("favor DESC").
+		Offset(offset).
 		Limit(size).Find(&list).Error
 	if err != nil {
 		return nil, err