Skip to content

Commit

Permalink
[feature] 动态图片爬虫与api
Browse files Browse the repository at this point in the history
  • Loading branch information
kyrie committed Jan 21, 2023
1 parent 2cef8ca commit 4c15ebc
Show file tree
Hide file tree
Showing 11 changed files with 434 additions and 77 deletions.
2 changes: 2 additions & 0 deletions cmd/spider/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ func newSpider() fx.Option {
fx.Provide(spider.NewVideo),
fx.Provide(spider.NewUpdate),
fx.Provide(spider.NewPicture),
fx.Provide(spider.NewUpdateDynamic),
fx.Provide(bilibili.NewSDK),
fx.Provide(health.NewCheckServer),
fx.Invoke(lc),
Expand All @@ -34,6 +35,7 @@ func lc(
spiderVideo *spider.Video,
spiderUpdate *spider.Update,
spiderPicture *spider.Picture,
spiderUpdatePicture *spider.UpdateDynamic,
checkServer *health.CheckServer,
shutdown fx.Shutdowner,
) {
Expand Down
18 changes: 18 additions & 0 deletions database/init.sql
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,21 @@ INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', '柚恩', 80);
INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', '柚恩不加糖', 100);
INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', 'EOE', 80);
INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', 'EOE组合', 100);

create table bilibili_dynamics (
id bigint unsigned not null auto_increment primary key comment 'id',
uid bigint unsigned not null comment 'B站用户id',
dynamic_id bigint unsigned not null comment 'B站动态id',
pictures json not null comment '图片',
topic_name varchar(64) comment '话题名称',
topic_id bigint unsigned not null comment '话题id',
view_nums bigint unsigned not null default '0' comment '看过的数量',
repost bigint unsigned not null default '0' comment '转发数量',
comment_nums bigint unsigned not null default '0' comment '评论数量',
favor bigint unsigned not null default '0' comment '点赞数量',
sent_at bigint unsigned not null comment '动态发生时间',
created_at bigint unsigned not null comment '创建时间',
updated_at bigint unsigned not null comment '更新时间',
index idx_dynamic_id(dynamic_id) comment '动态id索引',
index idx_sent_at(sent_at) comment '发送时间索引'
)Engine=InnoDB comment '动态' charset 'utf8mb4';
40 changes: 40 additions & 0 deletions internal/app/api/handler/bilbil_picture.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package handler

import (
"net/http"

"git.vtb.link/eoefans/internal/app/api/apperrors"
"git.vtb.link/eoefans/internal/app/api/help"
"git.vtb.link/eoefans/internal/app/api/idl"
"git.vtb.link/eoefans/internal/app/api/service"
"github.com/gin-gonic/gin"
)

func BilibiliLatestPics(s *service.BilbilPicture) func(ctx *gin.Context) {
return func(ctx *gin.Context) {
var req idl.BilibiliPictureLatestReq
if err := ctx.ShouldBindQuery(&req); err != nil {
_ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err))
return
}

if resp, err := s.Latest(ctx, req); err != nil {
_ = ctx.Error(err)
return
} else {
ctx.JSON(http.StatusOK, help.SuccessJson(resp))
}
}
}

func BilibiliRecommendPics(s *service.BilbilPicture) func(ctx *gin.Context) {
return func(ctx *gin.Context) {
var req idl.BilibiliPictureRecommendReq
if resp, err := s.Recommend(ctx,req); err != nil {
_ = ctx.Error(err)
return
} else {
ctx.JSON(http.StatusOK, help.SuccessJson(resp))
}
}
}
96 changes: 81 additions & 15 deletions internal/app/api/idl/bilibili_picture.go
Original file line number Diff line number Diff line change
@@ -1,26 +1,92 @@
package idl

type BilibiliPicture struct {
ID uint64 `gorm:"primarykey"`
Url string `gorm:"column:url"`
DynamicID uint64 `gorm:"column:dynamic_id"`
TopicName string `gorm:"column:topic_name"`
SentAt uint64 `gorm:"column:sent_at"`
CreatedAt uint64 `gorm:"autoCreateTime"`
UpdatedAt uint64 `gorm:"autoUpdateTime"`
import (
"database/sql/driver"
"encoding/json"
"errors"
"time"
)

// 图片来源于动态,以动态为单位
type BilibiliDynamic struct {
ID uint64 `gorm:"primarykey"`
UID uint64 `gorm:"column:uid"`
DynamicID uint64 `gorm:"column:dynamic_id"`
Pictures BilibiliDynamicPictures `gorm:"column:pictures;"`
TopicName string `gorm:"column:topic_name"`
TopicID uint64 `gorm:"column:topic_id"`
View uint64 `gorm:"column:view_nums"`
Repost uint64 `gorm:"column:repost"`
Comment uint64 `gorm:"column:comment_nums"`
Like uint64 `gorm:"column:favor"`
SentAt uint64 `gorm:"column:sent_at"`
CreatedAt uint64 `gorm:"autoCreateTime"`
UpdatedAt uint64 `gorm:"autoUpdateTime"`
}

func (p BilibiliDynamicPictures) Value() (driver.Value, error) {
return json.Marshal(p)
}

func (c *BilibiliDynamicPictures) Scan(input interface{}) error {
data, ok := input.([]byte)
if !ok {
return errors.New("invalid input in Scan")
}
result := BilibiliDynamicPictures{}
err := json.Unmarshal(data, &result)
if err != nil {
return err
}
*c = result
return nil
}

type BilibiliPictureLatestReq struct {
Page int `form:"page,default=1" binding:"omitempty,gt=0"`
TopicID int `form:"topic_id"`
}

type BilibiliPictureRecommendReq struct {
TopicID int `form:"topic_id"`
}
type BilibiliPicturesCommonResp struct {
Result []*BilibiliDynamicDTO `json:"result"`
}
type BilibiliPicturesLatestResp struct {
BilibiliPicturesCommonResp
Page int `json:"page"`
Total int `json:"total"`
}

type BilibiliPicturesRecommendResp struct {
BilibiliPicturesCommonResp
Total int `json:"total"`
}
type BilibiliDynamicDTO struct {
DynamicID uint64 `json:"dynamic_id"`
Pictures BilibiliDynamicPictures `json:"pictures"`
SentAt uint64 `json:"sent_at"`
}

type BilibiliPictureDTO struct {
ID uint64 `json:"id"`
Url string `json:"url"`
CreatedAt uint64 `json:"created_at"`
type BilibiliDynamicPictures []BilibiliDynamicPicture
type BilibiliDynamicPicture struct {
Height float64 `json:"img_height"`
Size float64 `json:"img_size"`
Width float64 `json:"img_width"`
ImgSrc string `json:"image_src"`
}

func (BilibiliPicture) TableName() string {
return "bilibili_pictures"
func (BilibiliDynamic) TableName() string {
return "bilibili_dynamics"
}

type BilibiliPictureRepository interface {
Create(items []*BilibiliPicture) error
Create(items []*BilibiliDynamic) error
FindMaxDynamicID(topicName string) (*uint64, error)
Update(updates map[string]interface{}, dynamicID uint64) error
FindAllByPubDate(from, to time.Time, page, size int64) (list []*BilibiliDynamic, err error)
Latest(page, size, topicID int) (list []*BilibiliDynamic, err error)
//推荐暂时先默认50个
Recommend(from, to time.Time, size, topicID int) (list []*BilibiliDynamic, err error)
}
10 changes: 8 additions & 2 deletions internal/app/api/router/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ func Provide() fx.Option {

func InitRouters(
bvService *service.BilbilVideo,
picService *service.BilbilPicture,
authService *service.Auth,
userService *service.User,
errMiddlewares *middlewares.ErrorInterceptor,
Expand All @@ -24,9 +25,14 @@ func InitRouters(
// http 异常处理
r.Use(errMiddlewares.Handler)

// 视频搜素
// 视频搜索
r.GET("/v1/video-interface/advanced-search", handler.BilibiliVideoSearch(bvService))

//图片
picApi := r.Group("/v1/pic")
{
picApi.GET("/latest", handler.BilibiliLatestPics(picService))
picApi.GET("/recommend", handler.BilibiliRecommendPics(picService))
}
// Auth相关
authApi := r.Group("/v1/auth")
{
Expand Down
64 changes: 64 additions & 0 deletions internal/app/api/service/bilbil_picture.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package service

import (
"context"
"time"

"git.vtb.link/eoefans/internal/app/api/idl"
"git.vtb.link/eoefans/internal/repository"
"gorm.io/gorm"
)

const (
picRecommendDefaultSize = 50
)

type BilbilPicture struct {
db *gorm.DB
}

func NewBilibiliPicture(db *gorm.DB) *BilbilPicture {
return &BilbilPicture{db: db}
}

func (service *BilbilPicture) Latest(ctx context.Context, req idl.BilibiliPictureLatestReq) (*idl.BilibiliPicturesLatestResp, error) {
tx := service.db.WithContext(ctx)
picRepository := repository.NewBilibiliPicture(tx)
list, err := picRepository.Latest(req.Page, defaultQuerySize, req.TopicID)
if err != nil {
return nil, err
}
resp := idl.BilibiliPicturesLatestResp{
Page: req.Page,
Total: len(list),
}
for i := range list {
resp.Result = append(resp.Result, &idl.BilibiliDynamicDTO{
DynamicID: list[i].DynamicID,
Pictures: list[i].Pictures,
SentAt: list[i].SentAt,
})
}
return &resp, nil
}

func (service *BilbilPicture) Recommend(ctx context.Context, req idl.BilibiliPictureRecommendReq) (*idl.BilibiliPicturesRecommendResp, error) {
tx := service.db.WithContext(ctx)
picRepository := repository.NewBilibiliPicture(tx)
now := time.Now()
list, err := picRepository.Recommend(now.Add(-(3 * 24 * time.Hour)), now, picRecommendDefaultSize, req.TopicID)
if err != nil {
return nil, err
}
resp := idl.BilibiliPicturesRecommendResp{
Total: len(list),
}
for i := range list {
resp.Result = append(resp.Result, &idl.BilibiliDynamicDTO{
DynamicID: list[i].DynamicID,
Pictures: list[i].Pictures,
SentAt: list[i].SentAt,
})
}
return &resp, nil
}
1 change: 1 addition & 0 deletions internal/app/provide.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func MiddlewareProvider() fx.Option {
func ServiceProvider() fx.Option {
return fx.Provide(
service.NewBilbilVideo,
service.NewBilibiliPicture,
service.NewAuth,
service.NewUser,
)
Expand Down
Loading

0 comments on commit 4c15ebc

Please sign in to comment.