diff --git a/cmd/spider/main.go b/cmd/spider/main.go index 6a8c1ad..0100d9f 100644 --- a/cmd/spider/main.go +++ b/cmd/spider/main.go @@ -22,13 +22,23 @@ func newSpider() fx.Option { video_analysis.Provide(), fx.Provide(spider.NewVideo), fx.Provide(spider.NewUpdate), + fx.Provide(spider.NewPicture), + fx.Provide(spider.NewUpdateDynamic), fx.Provide(bilibili.NewSDK), fx.Provide(health.NewCheckServer), fx.Invoke(lc), ) } -func lc(lifecycle fx.Lifecycle, spiderVideo *spider.Video, spiderUpdate *spider.Update, checkServer *health.CheckServer, shutdown fx.Shutdowner) { +func lc( + lifecycle fx.Lifecycle, + spiderVideo *spider.Video, + spiderUpdate *spider.Update, + spiderPicture *spider.Picture, + spiderUpdatePicture *spider.UpdateDynamic, + checkServer *health.CheckServer, + shutdown fx.Shutdowner, +) { lifecycle.Append(fx.Hook{ OnStart: func(ctx context.Context) error { return spiderVideo.Run(ctx) @@ -53,6 +63,18 @@ func lc(lifecycle fx.Lifecycle, spiderVideo *spider.Video, spiderUpdate *spider. }, }) + lifecycle.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return spiderPicture.Run(ctx) + }, + OnStop: func(ctx context.Context) error { + if err := spiderPicture.Stop(ctx); err != nil { + return err + } + return shutdown.Shutdown() + }, + }) + lifecycle.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { diff --git a/database/init.sql b/database/init.sql index 0b17e63..b79bc8c 100644 --- a/database/init.sql +++ b/database/init.sql @@ -109,3 +109,21 @@ INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', '柚恩', 80); INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', '柚恩不加糖', 100); INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', 'EOE', 80); INSERT INTO video_analysis (type, `key`, score) VALUES ('tag', 'EOE组合', 100); + +create table bilibili_dynamics ( + id bigint unsigned not null auto_increment primary key comment 'id', + uid bigint unsigned not null comment 'B站用户id', + dynamic_id bigint unsigned not null comment 'B站动态id', + pictures json not null comment '图片', + topic_name varchar(64) comment '话题名称', + topic_id bigint unsigned not null comment '话题id', + view_nums bigint unsigned not null default '0' comment '看过的数量', + repost bigint unsigned not null default '0' comment '转发数量', + comment_nums bigint unsigned not null default '0' comment '评论数量', + favor bigint unsigned not null default '0' comment '点赞数量', + sent_at bigint unsigned not null comment '动态发生时间', + created_at bigint unsigned not null comment '创建时间', + updated_at bigint unsigned not null comment '更新时间', + index idx_dynamic_id(dynamic_id) comment '动态id索引', + index idx_sent_at(sent_at) comment '发送时间索引' +)Engine=InnoDB comment '动态' charset 'utf8mb4'; \ No newline at end of file diff --git a/internal/app/api/handler/bilbil_picture.go b/internal/app/api/handler/bilbil_picture.go new file mode 100644 index 0000000..73aecd1 --- /dev/null +++ b/internal/app/api/handler/bilbil_picture.go @@ -0,0 +1,43 @@ +package handler + +import ( + "net/http" + + "git.vtb.link/eoefans/internal/app/api/apperrors" + "git.vtb.link/eoefans/internal/app/api/help" + "git.vtb.link/eoefans/internal/app/api/idl" + "git.vtb.link/eoefans/internal/app/api/service" + "github.com/gin-gonic/gin" +) + +func BilibiliLatestPics(s *service.BilbilPicture) func(ctx *gin.Context) { + return func(ctx *gin.Context) { + var req idl.BilibiliPictureLatestReq + if err := ctx.ShouldBindQuery(&req); err != nil { + _ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err)) + return + } + if resp, err := s.Latest(ctx, req); err != nil { + _ = ctx.Error(err) + return + } else { + ctx.JSON(http.StatusOK, help.SuccessJson(resp)) + } + } +} + +func BilibiliRecommendPics(s *service.BilbilPicture) func(ctx *gin.Context) { + return func(ctx *gin.Context) { + var req idl.BilibiliPictureRecommendReq + if err := ctx.ShouldBindQuery(&req); err != nil { + _ = ctx.Error(apperrors.NewValidationError(400, err.Error()).Wrap(err)) + return + } + if resp, err := s.Recommend(ctx, req); err != nil { + _ = ctx.Error(err) + return + } else { + ctx.JSON(http.StatusOK, help.SuccessJson(resp)) + } + } +} diff --git a/internal/app/api/idl/bilibili_picture.go b/internal/app/api/idl/bilibili_picture.go new file mode 100644 index 0000000..8394b1a --- /dev/null +++ b/internal/app/api/idl/bilibili_picture.go @@ -0,0 +1,93 @@ +package idl + +import ( + "database/sql/driver" + "encoding/json" + "errors" + "time" +) + +// 图片来源于动态，以动态为单位 +type BilibiliDynamic struct { + ID uint64 `gorm:"primarykey"` + UID uint64 `gorm:"column:uid"` + DynamicID uint64 `gorm:"column:dynamic_id"` + Pictures BilibiliDynamicPictures `gorm:"column:pictures;"` + TopicName string `gorm:"column:topic_name"` + TopicID uint64 `gorm:"column:topic_id"` + View uint64 `gorm:"column:view_nums"` + Repost uint64 `gorm:"column:repost"` + Comment uint64 `gorm:"column:comment_nums"` + Like uint64 `gorm:"column:favor"` + SentAt uint64 `gorm:"column:sent_at"` + CreatedAt uint64 `gorm:"autoCreateTime"` + UpdatedAt uint64 `gorm:"autoUpdateTime"` +} + +func (p BilibiliDynamicPictures) Value() (driver.Value, error) { + return json.Marshal(p) +} + +func (c *BilibiliDynamicPictures) Scan(input interface{}) error { + data, ok := input.([]byte) + if !ok { + return errors.New("invalid input in Scan") + } + result := BilibiliDynamicPictures{} + err := json.Unmarshal(data, &result) + if err != nil { + return err + } + *c = result + return nil +} + +type BilibiliPictureLatestReq struct { + Page int `form:"page,default=1" binding:"omitempty,gt=0"` + TopicID int `form:"topic_id"` +} + +type BilibiliPictureRecommendReq struct { + Page int `form:"page,default=1" binding:"omitempty,gt=0"` + TopicID int `form:"topic_id"` +} +type BilibiliPicturesCommonResp struct { + Result []*BilibiliDynamicDTO `json:"result"` +} +type BilibiliPicturesLatestResp struct { + BilibiliPicturesCommonResp + Page int `json:"page"` + Total int `json:"total"` +} + +type BilibiliPicturesRecommendResp struct { + BilibiliPicturesCommonResp + Page int `json:"page"` + Total int `json:"total"` +} +type BilibiliDynamicDTO struct { + DynamicID uint64 `json:"dynamic_id"` + Pictures BilibiliDynamicPictures `json:"pictures"` + SentAt uint64 `json:"sent_at"` +} + +type BilibiliDynamicPictures []BilibiliDynamicPicture +type BilibiliDynamicPicture struct { + Height float64 `json:"img_height"` + Size float64 `json:"img_size"` + Width float64 `json:"img_width"` + ImgSrc string `json:"img_src"` +} + +func (BilibiliDynamic) TableName() string { + return "bilibili_dynamics" +} + +type BilibiliPictureRepository interface { + Create(items []*BilibiliDynamic) error + FindMaxDynamicID(topicName string) (*uint64, error) + Update(updates map[string]interface{}, dynamicID uint64) error + FindAllByPubDate(from, to time.Time, page, size int64) (list []*BilibiliDynamic, err error) + Latest(page, size, topicID int) (list []*BilibiliDynamic, err error) + Recommend(from, to time.Time, page, size, topicID int) (list []*BilibiliDynamic, err error) +} diff --git a/internal/app/api/router/router.go b/internal/app/api/router/router.go index b097203..7b70798 100644 --- a/internal/app/api/router/router.go +++ b/internal/app/api/router/router.go @@ -15,6 +15,7 @@ func Provide() fx.Option { func InitRouters( bvService *service.BilbilVideo, + picService *service.BilbilPicture, authService *service.Auth, userService *service.User, toolService *service.Tool, @@ -25,9 +26,14 @@ func InitRouters( // http 异常处理 r.Use(errMiddlewares.Handler) - // 视频搜素 + // 视频搜索 r.GET("/v1/video-interface/advanced-search", handler.BilibiliVideoSearch(bvService)) - + //图片 + picApi := r.Group("/v1/pic") + { + picApi.GET("/latest", handler.BilibiliLatestPics(picService)) + picApi.GET("/recommend", handler.BilibiliRecommendPics(picService)) + } // Auth相关 authApi := r.Group("/v1/auth") { diff --git a/internal/app/api/service/bilbil_picture.go b/internal/app/api/service/bilbil_picture.go new file mode 100644 index 0000000..e788f85 --- /dev/null +++ b/internal/app/api/service/bilbil_picture.go @@ -0,0 +1,65 @@ +package service + +import ( + "context" + "time" + + "git.vtb.link/eoefans/internal/app/api/idl" + "git.vtb.link/eoefans/internal/repository" + "gorm.io/gorm" +) + +const ( + picRecommendDefaultSize = 20 +) + +type BilbilPicture struct { + db *gorm.DB +} + +func NewBilibiliPicture(db *gorm.DB) *BilbilPicture { + return &BilbilPicture{db: db} +} + +func (service *BilbilPicture) Latest(ctx context.Context, req idl.BilibiliPictureLatestReq) (*idl.BilibiliPicturesLatestResp, error) { + tx := service.db.WithContext(ctx) + picRepository := repository.NewBilibiliPicture(tx) + list, err := picRepository.Latest(req.Page, defaultQuerySize, req.TopicID) + if err != nil { + return nil, err + } + resp := idl.BilibiliPicturesLatestResp{ + Page: req.Page, + Total: len(list), + } + for i := range list { + resp.Result = append(resp.Result, &idl.BilibiliDynamicDTO{ + DynamicID: list[i].DynamicID, + Pictures: list[i].Pictures, + SentAt: list[i].SentAt, + }) + } + return &resp, nil +} + +func (service *BilbilPicture) Recommend(ctx context.Context, req idl.BilibiliPictureRecommendReq) (*idl.BilibiliPicturesRecommendResp, error) { + tx := service.db.WithContext(ctx) + picRepository := repository.NewBilibiliPicture(tx) + now := time.Now() + list, err := picRepository.Recommend(now.Add(-(3 * 24 * time.Hour)), now, req.Page, picRecommendDefaultSize, req.TopicID) + if err != nil { + return nil, err + } + resp := idl.BilibiliPicturesRecommendResp{ + Total: len(list), + Page: req.Page, + } + for i := range list { + resp.Result = append(resp.Result, &idl.BilibiliDynamicDTO{ + DynamicID: list[i].DynamicID, + Pictures: list[i].Pictures, + SentAt: list[i].SentAt, + }) + } + return &resp, nil +} diff --git a/internal/app/provide.go b/internal/app/provide.go index 087d300..0c4be2b 100644 --- a/internal/app/provide.go +++ b/internal/app/provide.go @@ -33,6 +33,7 @@ func MiddlewareProvider() fx.Option { func ServiceProvider() fx.Option { return fx.Provide( service.NewBilbilVideo, + service.NewBilibiliPicture, service.NewAuth, service.NewUser, service.NewTool, diff --git a/internal/app/spider/picture.go b/internal/app/spider/picture.go new file mode 100644 index 0000000..9967aba --- /dev/null +++ b/internal/app/spider/picture.go @@ -0,0 +1,179 @@ +package spider + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "git.vtb.link/eoefans/internal/app/api/idl" + "git.vtb.link/eoefans/internal/pkg/bilibili" + "git.vtb.link/eoefans/internal/repository" + "github.com/pkg/errors" + "go.uber.org/zap" + "gorm.io/gorm" +) + +type Picture struct { + stopChan chan bool + db *gorm.DB + logger *zap.Logger + sdk *bilibili.SDK + isRunning bool +} + +func NewPicture(db *gorm.DB, logger *zap.Logger, sdk *bilibili.SDK) *Picture { + return &Picture{ + stopChan: make(chan bool), + db: db, + logger: logger, + sdk: sdk, + } +} + +func (p *Picture) Stop(ctx context.Context) error { + p.logger.Info("stopping spider server") + + for { + select { + case <-ctx.Done(): + return errors.New("shutdown spider server timeout") + default: + if err := p.stop(); err != nil { + return errors.Wrap(err, "shutdown spider server error") + } + return nil + } + } +} + +func (p *Picture) stop() error { + p.stopChan <- true + p.isRunning = false + return nil +} + +func (p *Picture) Run(ctx context.Context) error { + tk := time.NewTicker(60 * time.Minute) + p.isRunning = true + + go func() { + if err := p.spider(); err != nil { + p.logger.Error("start spider server error", zap.Error(err)) + } + }() + + go func(_tk *time.Ticker) { + for { + select { + case <-_tk.C: + p.logger.Info("[tick] picture spider", zap.Time("time", time.Now())) + if err := p.spider(); err != nil { + p.logger.Error("start picture server error", zap.Error(err)) + } + case <-p.stopChan: + return + } + } + }(tk) + + return nil +} + +func (p *Picture) spider() error { + //把当前数据库最大的动态ID查出来 + //调用接口，将大于当前动态ID的都入DB，如果存在小于的，则可提前结束，尽量保证没有重复数据 + topicsMap := map[string]uint64{ + bilibili.TopicNameGoGo: bilibili.TopicIDGoGo, + bilibili.TopicNameMino: bilibili.TopicIDMino, + bilibili.TopicNameUn: bilibili.TopicIDUn, + bilibili.TopicNameMoMo: bilibili.TopicIDMoMo, + bilibili.TopicNameWan: bilibili.TopicIDWan, + bilibili.TopicNameEOE: bilibili.TopicIDEOE, + } + for topicName, topicID := range topicsMap { + curMaxDynamicID, err := repository.NewBilibiliPicture(p.db).FindMaxDynamicID(topicName) + if err != nil { + p.logger.Error("FindMaxDynamicID error", zap.String("topic_name", topicName), zap.Error(err)) + continue + } + var hasMore uint = 1 + var offset uint64 = 0 + exist := false //判断有没有已经爬过 + for hasMore == 1 && !exist { + data, err := p.sdk.TopicDynamics(topicName, offset) + if err != nil { + p.logger.Error("TopicDynamics error", zap.String("topic_name", topicName), zap.String("offset", fmt.Sprintf("%d", offset)), zap.Error(err)) + time.Sleep(500 * time.Millisecond) + break + } + hasMore = data.HasMore + dynamicID, err := strconv.ParseUint(data.Offset, 10, 64) + if err == nil { + offset = dynamicID + } else { + hasMore = 0 + } + items := make([]*idl.BilibiliDynamic, 0) + for _, v := range data.Cards { + switch v.Desc.Type { + case bilibili.DynamicDraw: + if v.Desc.DynamicID <= *curMaxDynamicID { + //后面所有的都是爬过的，提前结束，后续也不再请求api + exist = true + break + } + dynamic := &idl.BilibiliDynamic{ + UID: v.Desc.UID, + DynamicID: v.Desc.DynamicID, + TopicName: topicName, + TopicID: topicID, + View: v.Desc.View, + Repost: v.Desc.Repost, + Comment: v.Desc.Comment, + Like: v.Desc.Like, + SentAt: v.Desc.TimeStamp, + } + pictures, err := parsePicturesFromCard(v.Card) + if err != nil { + p.logger.Error("ParsePicturesFromCard error", zap.String("topic_name", topicName), zap.String("offset", fmt.Sprintf("%d", offset)), zap.Error(err)) + continue + } + if len(pictures) == 0 { + continue + } + dynamic.Pictures = pictures + items = append(items, dynamic) + default: + continue + } + } + //插入数据 + if len(items) != 0 { + err := repository.NewBilibiliPicture(p.db).Create(items) + if err != nil { + p.logger.Error("Create bilibli_pictures error", zap.String("topic_name", topicName), zap.String("offset", fmt.Sprintf("%d", offset)), zap.Error(err)) + } + } + } + } + return nil +} + +func parsePicturesFromCard(data string) ([]idl.BilibiliDynamicPicture, error) { + var content bilibili.DynamicCardContent + if err := json.Unmarshal([]byte(data), &content); err != nil { + return nil, err + } + pics := make([]idl.BilibiliDynamicPicture, 0, len(content.Item.Pictures)) + for _, v := range content.Item.Pictures { + pics = append(pics, idl.BilibiliDynamicPicture{ + Height: v.Height, + Size: v.Size, + Width: v.Width, + ImgSrc: v.ImgSrc, + }) + } + return pics, nil +} diff --git a/internal/app/spider/picture_test.go b/internal/app/spider/picture_test.go new file mode 100644 index 0000000..29cf4a6 --- /dev/null +++ b/internal/app/spider/picture_test.go @@ -0,0 +1,75 @@ +package spider + +import ( + "testing" + + "git.vtb.link/eoefans/internal/app/api/idl" + "git.vtb.link/eoefans/internal/pkg/bilibili" + "git.vtb.link/eoefans/internal/pkg/database" + "git.vtb.link/eoefans/internal/repository" + "go.uber.org/zap" +) + +func TestFindCurMaxDynamicIDByTopicName(t *testing.T) { + db, err := database.NewDatabase(&database.Options{ + Type: "mysql", + DSN: "root:123456@tcp(127.0.0.1:3306)/eoes?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai", + Debug: true, + SetMaxIdleConns: 2, + SetMaxOpenConns: 4, + SetConnMaxLifetime: 6, + }) + if err != nil { + t.Error(err) + return + } + id, err := repository.NewBilibiliPicture(db).FindMaxDynamicID("EOE的魔法盒") + if err != nil { + t.Error(err) + return + } + if id != nil { + t.Log(*id) + } +} +func TestInsertPicture(t *testing.T) { + db, err := database.NewDatabase(&database.Options{ + Type: "mysql", + DSN: "root:123456@tcp(127.0.0.1:3306)/eoes?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai", + Debug: true, + SetMaxIdleConns: 2, + SetMaxOpenConns: 4, + SetConnMaxLifetime: 6, + }) + if err != nil { + t.Error(err) + return + } + sdk := bilibili.NewSDK(&zap.Logger{}) + var dynamicID uint64 = 752843351274815520 + res, err := sdk.Dynamic(uint64(dynamicID)) + if err != nil { + t.Error(err) + return + } + pictures, err := parsePicturesFromCard(res.Card.Card) + if err != nil { + t.Error(err) + return + } + items := []*idl.BilibiliDynamic{{ + UID: res.Card.Desc.UID, + DynamicID: 752843351274815520, + Pictures: pictures, + View: res.Card.Desc.View, + Repost: res.Card.Desc.Repost, + Comment: res.Card.Desc.Comment, + Like: res.Card.Desc.Like, + SentAt: res.Card.Desc.TimeStamp, + }} + err = repository.NewBilibiliPicture(db).Create(items) + if err != nil { + t.Error(err) + return + } +} diff --git a/internal/app/spider/update_dynamic.go b/internal/app/spider/update_dynamic.go new file mode 100644 index 0000000..d01369b --- /dev/null +++ b/internal/app/spider/update_dynamic.go @@ -0,0 +1,104 @@ +package spider + +import ( + "context" + + "git.vtb.link/eoefans/internal/pkg/bilibili" + "git.vtb.link/eoefans/internal/repository" + + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" + "gorm.io/gorm" +) + +type UpdateDynamic struct { + stopChan chan bool + db *gorm.DB + logger *zap.Logger + sdk *bilibili.SDK +} + +func NewUpdateDynamic(db *gorm.DB, logger *zap.Logger, sdk *bilibili.SDK) *Update { + return &Update{ + stopChan: make(chan bool), + db: db, + logger: logger, + sdk: sdk, + } +} + +func (u *UpdateDynamic) Stop(ctx context.Context) error { + u.logger.Info("stopping spider server") + + for { + select { + case <-ctx.Done(): + return errors.New("shutdown dynamic update spider server timeout") + default: + close(u.stopChan) + return nil + } + } +} + +func (u *UpdateDynamic) Run(ctx context.Context) error { + go func() { + if err := u.spider(); err != nil { + u.logger.Error("start dynamic update error", zap.Error(err)) + } + }() + + tk := time.NewTicker(180 * time.Minute) + go func(_tk *time.Ticker) { + for { + select { + case <-_tk.C: + u.logger.Info("[tick] dynamic update spider", zap.Time("time", time.Now())) + if err := u.spider(); err != nil { + u.logger.Error("start dynamic update error", zap.Error(err)) + } + case <-u.stopChan: + return + } + } + }(tk) + + return nil +} + +func (u *UpdateDynamic) spider() error { + tx := u.db.WithContext(context.TODO()) + repo := repository.NewBilibiliPicture(tx) + + size := 100 + for p := 1; true; p++ { + list, err := repo.FindAllByPubDate(time.Now().Add(-(3 * 24 * time.Hour)), time.Now(), int64(p), int64(size)) + if err != nil { + u.logger.Error("[UpdateDynamic spider()]FindAllByPubDate error", zap.Int("page", p), zap.Error(err)) + return nil + } + for _, v := range list { + dynamic, err := u.sdk.Dynamic(v.DynamicID) + if err != nil { + u.logger.Error("Dynamic error", zap.Int("dynamic_id", int(v.DynamicID)), zap.Error(err)) + continue + } + updates := map[string]interface{}{ + "view_nums": dynamic.Card.Desc.View, + "repost": dynamic.Card.Desc.Repost, + "comment_nums": dynamic.Card.Desc.Comment, + "favor": dynamic.Card.Desc.Like, + } + if err := repo.Update(updates, v.DynamicID); err != nil { + u.logger.Error("Dynamic Update error", zap.Int("dynamic_id", int(v.DynamicID)), zap.Error(err)) + } + } + if len(list) < size { + break + } + } + + return nil +} diff --git a/internal/pkg/bilibili/api_test.go b/internal/pkg/bilibili/api_test.go new file mode 100644 index 0000000..7ba11d2 --- /dev/null +++ b/internal/pkg/bilibili/api_test.go @@ -0,0 +1,30 @@ +package bilibili + +import ( + "testing" + + "go.uber.org/zap" +) + +func TestDynamic(t *testing.T) { + sdk := NewSDK(&zap.Logger{}) + var dynamicID uint64 = 752843351274815520 + res, err := sdk.Dynamic(uint64(dynamicID)) + if err != nil { + t.Error(err) + return + } + t.Log(res) +} + +func TestDynamicList(t *testing.T) { + sdk := NewSDK(&zap.Logger{}) + name := "EOE的魔法盒" + offset := 0 + res, err := sdk.TopicDynamics(name, uint64(offset)) + if err != nil { + t.Error(err) + return + } + t.Log(len(res.Cards)) +} diff --git a/internal/pkg/bilibili/video.go b/internal/pkg/bilibili/video.go index 576a762..1abf454 100644 --- a/internal/pkg/bilibili/video.go +++ b/internal/pkg/bilibili/video.go @@ -3,6 +3,7 @@ package bilibili import ( "fmt" "net/http" + "net/url" "git.vtb.link/eoefans/internal/pkg/httpclient" "github.com/go-resty/resty/v2" @@ -14,12 +15,36 @@ const ( webVideoSearchURL = "https://api.bilibili.com/x/web-interface/search/type?context=&search_type=video&page=%d&order=pubdate&keyword=%s&duration=0&category_id=&tids_2=&__refresh__=true&_extra=&tids=0&highlight=1&single_column=0" webVideoInfoURL = "https://api.bilibili.com/x/web-interface/view?bvid=%s" webVideoTagInfoURL = "https://api.bilibili.com/x/web-interface/view/detail/tag?aid=%s" + topicHistory = "https://api.vc.bilibili.com/topic_svr/v1/topic_svr/topic_history?offset_dynamic_id=%d&" + dynamic = "https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/get_dynamic_detail?dynamic_id=%d" ) const ( cookie = `buvid3=84D177FC-9F90-100D-8BE2-09EE5D9912C823748infoc; b_nut=1672450523; i-wanna-go-back=-1; b_lsid=10D3F3189_18565D1C03E; _uuid=1F95104D7-B7410-FFAD-4263-1EF4BFB14B61023203infoc; buvid_fp=beb12c3870e59666cfee50c6431e8b1a; buvid4=2F7A4DEB-FE17-1579-67FF-E0004E5BFE6D24471-022123109-ytYURKXzrIAOEIOhWNJfdQ%3D%3D; SESSDATA=239cfdaf%2C1688002658%2Ca176f%2Ac2; bili_jct=f4be98b5fb468b646b7260f10ba547e1; DedeUserID=554518161; DedeUserID__ckMd5=d1653a8f679dcf53; CURRENT_FNVAL=4048; sid=870jixri; is-2022-channel=1; hit-new-style-dyn=0; hit-dyn-v2=1; b_ut=5; innersign=1; rpdid=|(u~|||~Y)m)0J\'uY~kRl|~uu'` ) +type DynamicType uint + +const ( + DynamicDraw DynamicType = 2 //图片动态 +) +const ( + //topicHistory用topic_id查出来的数据有问题，故暂时用topic_name + TopicNameWan = "小莞熊在这里" + TopicNameUn = "柚恩的蜜罐子" + TopicNameGoGo = "GOGO队立大功！" //中文感叹号 + TopicNameMoMo = "虞你在一起" + TopicNameMino = "和米诺的对抗路日常" + TopicNameEOE = "EOE的魔法盒" + + TopicIDWan uint64 = 28953983 + TopicIDUn uint64 = 28950030 + TopicIDGoGo uint64 = 29067608 + TopicIDMoMo uint64 = 28948378 + TopicIDMino uint64 = 29069147 + TopicIDEOE uint64 = 29156150 +) + type SDK struct { logger *zap.Logger } @@ -35,6 +60,43 @@ type ResponseBasic struct { Data interface{} `json:"data"` } +type DynamicList struct { + Cards []DynamicCard `json:"cards"` + HasMore uint `json:"has_more"` + Offset string `json:"offset"` +} + +type Dynamic struct { + Card DynamicCard `json:"card"` +} +type DynamicCard struct { + Desc struct { + Type DynamicType `json:"type"` + View uint64 `json:"view"` + Repost uint64 `json:"repost"` + Comment uint64 `json:"comment"` + Like uint64 `json:"like"` + UID uint64 `json:"uid"` + DynamicID uint64 `json:"dynamic_id"` + TimeStamp uint64 `json:"timestamp"` + } `json:"desc"` + Card string `json:"card"` +} + +// Card是json字符串，需要进一步解析 +type DynamicCardContent struct { + Item struct { + Pictures DynamicPictures `json:"pictures"` + } `json:"item"` +} + +type DynamicPictures []DynamicPicture +type DynamicPicture struct { + Height float64 `json:"img_height"` + Size float64 `json:"img_size"` + Width float64 `json:"img_width"` + ImgSrc string `json:"img_src"` +} type VideoSearchInfo struct { Type string `json:"type"` Id int `json:"id"` @@ -290,3 +352,23 @@ func (sdk *SDK) VideoWebTagInfo(aid string) (data *VideoTagResponse, err error) } return data, nil } + +func (sdk *SDK) TopicDynamics(topicName string, offsetDynamicId uint64) (data *DynamicList, err error) { + params := url.Values{} + params.Add("topic_name", topicName) + url := fmt.Sprintf(topicHistory, offsetDynamicId) + url = url + params.Encode() + if err = sdk.fastGet(url, &data); err != nil { + return nil, err + } + return data, nil +} + +func (sdk *SDK) Dynamic(dynamicId uint64) (data *Dynamic, err error) { + url := fmt.Sprintf(dynamic, dynamicId) + fmt.Println(url) + if err = sdk.fastGet(url, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/internal/repository/bilibili_picture.go b/internal/repository/bilibili_picture.go new file mode 100644 index 0000000..e526771 --- /dev/null +++ b/internal/repository/bilibili_picture.go @@ -0,0 +1,92 @@ +package repository + +import ( + "time" + + "git.vtb.link/eoefans/internal/app/api/idl" + "gorm.io/gorm" +) + +func NewBilibiliPicture(tx *gorm.DB) idl.BilibiliPictureRepository { + return &BilibiliPictureMysqlImpl{tx: tx} +} + +type BilibiliPictureMysqlImpl struct { + tx *gorm.DB +} + +func (impl *BilibiliPictureMysqlImpl) Create(items []*idl.BilibiliDynamic) error { + if len(items) == 0 { + return nil + } + return impl.tx.Table(idl.BilibiliDynamic{}.TableName()).Create(&items).Error +} + +func (impl *BilibiliPictureMysqlImpl) Update(updates map[string]interface{}, dynamicID uint64) error { + if len(updates) == 0 { + return nil + } + return impl.tx.Table(idl.BilibiliDynamic{}.TableName()).Where("dynamic_id=?", dynamicID).Updates(updates).Error +} + +func (impl *BilibiliPictureMysqlImpl) FindMaxDynamicID(topicName string) (*uint64, error) { + var id uint64 + conn := impl.tx.Table(idl.BilibiliDynamic{}.TableName()) + err := conn.Select("max(dynamic_id) as id").Where("topic_name = ?", topicName).Group("dynamic_id").Scan(&id).Error + if err != nil { + return nil, err + } + return &id, nil +} + +func (impl *BilibiliPictureMysqlImpl) FindAllByPubDate(from, to time.Time, page, size int64) (list []*idl.BilibiliDynamic, err error) { + err = impl.tx.Table(idl.BilibiliDynamic{}.TableName()). + Where("sent_at >= ? AND sent_at <= ?", from.Unix(), to.Unix()). + Select("dynamic_id"). + Offset(int((page - 1) * size)).Limit(int(size)). + Order("sent_at DESC"). + Find(&list).Error + if err != nil { + return nil, err + } + return list, nil +} + +func (impl *BilibiliPictureMysqlImpl) Latest(page, size, topicID int) (list []*idl.BilibiliDynamic, err error) { + conn := impl.tx.Table(idl.BilibiliDynamic{}.TableName()) + if topicID != 0 { + conn = conn.Where("topic_id = ?", topicID) + } + offset := (page - 1) * size + if offset < 0 { + offset = -1 + } + err = conn.Select("dynamic_id,pictures,sent_at"). + Order("sent_at DESC"). + Offset(offset). + Limit(size).Find(&list).Error + if err != nil { + return nil, err + } + return list, nil +} + +func (impl *BilibiliPictureMysqlImpl) Recommend(from, to time.Time, page, size, topicID int) (list []*idl.BilibiliDynamic, err error) { + conn := impl.tx.Table(idl.BilibiliDynamic{}.TableName()) + if topicID != 0 { + conn = conn.Where("topic_id = ?", topicID) + } + offset := (page - 1) * size + if offset < 0 { + offset = -1 + } + err = conn.Select("dynamic_id,pictures,sent_at"). + Where("sent_at >= ? AND sent_at <= ?", from.Unix(), to.Unix()). + Order("favor DESC"). + Offset(offset). + Limit(size).Find(&list).Error + if err != nil { + return nil, err + } + return list, nil +}