diff --git a/cmd/migration/main.go b/cmd/migration/main.go new file mode 100644 index 0000000..fff4583 --- /dev/null +++ b/cmd/migration/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "git.vtb.link/eoefans/internal/app/api/idl" + "github.com/google/uuid" + "github.com/lib/pq" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var oldDB *gorm.DB +var newDB *gorm.DB + +const ( + oldDSN = "vlink02mysql:8715-3314fa3525fc@tcp(vtb-link-02-mysql.mysql.database.azure.com:3306)/eoefans-01?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai" + newDSN = "host=c.vlink01cosmosdb.postgres.database.azure.com port=5432 dbname=citus user=citus password=83cc-cddd5e342f63 sslmode=require TimeZone=Asia/Shanghai" +) + +func init() { + var err error + if oldDB, err = gorm.Open(mysql.Open(oldDSN)); err != nil { + panic(err) + } + + if newDB, err = gorm.Open(postgres.Open(newDSN)); err != nil { + panic(err) + } +} + +func main() { + mPc() +} + +type NewBilibiliVideo struct { + idl.BilibiliVideo + UUID uuid.UUID `gorm:"column:id;type:uuid;primary_key"` + TagList pq.StringArray `gorm:"column:tag_list;type:varchar(100)[]"` +} + +func mVideo() { + var list []*idl.BilibiliVideo + result := oldDB.Table("bilbil_video").Order("id DESC").Find(&list) + + if result.Error != nil { + panic(result.Error) + } + + //for _, video := range list { + // new := &NewBilibiliVideo{ + // BilibiliVideo: *video, + // UUID: uuid.New(), + // TagList: strings.Split(video.Tag, ","), + // } + // + // result = newDB.Table("bilbil_video").Clauses(clause.OnConflict{DoNothing: true}).Create(new) + // if result.Error != nil { + // panic(result.Error) + // } + //} + // + //fmt.Println(list) +} + +func mPD() { + var list []*idl.BilibiliDynamic + result := oldDB.Table("bilibili_dynamics").Order("id").Find(&list) + if result.Error != nil { + panic(result.Error) + } + + for _, dynamic := range list { + result = newDB.Table("bilibili_dynamics").Create(dynamic) + if result.Error != nil { + panic(result.Error) + } + } +} + +func mPc() { + var list []*idl.BilibiliPicture + result := oldDB.Table("bilibili_pictures").Order("id").Find(&list) + + if result.Error != nil { + panic(result.Error) + } + + for _, picture := range list { + result = newDB.Table("bilibili_pictures").Create(picture) + if result.Error != nil { + panic(result.Error) + } + } +} diff --git a/config/config.template.yml b/config/config.template.yml index 9025b5b..2fb3e03 100644 --- a/config/config.template.yml +++ b/config/config.template.yml @@ -16,8 +16,9 @@ smsClient: ssl: true insecureSkipVerify: false db: - type: mysql # mysql - dsn: USER:PASSWORD@tcp(HOST:PORT)/DATABASE?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai + type: mysql # mysql postgres +# dsn: host=HOST user=USER password=PASSWORD dbname=DATABASE port=PORT sslmode=disable TimeZone=Asia/Shanghai # POSTGRES DSN + dsn: USER:PASSWORD@tcp(HOST:PORT)/DATABASE?charset=utf8mb4&parseTime=true&loc=Asia%2FShanghai # MYSQL DSN debug: true setMaxIdleConns: 2 # 设置空闲连接池中连接的最大数量 setMaxOpenConns: 4 # 设置打开数据库连接的最大数量 diff --git a/go.mod b/go.mod index 74d85e4..61bd9ac 100644 --- a/go.mod +++ b/go.mod @@ -31,11 +31,15 @@ require ( github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-json v0.10.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.1 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -62,4 +66,5 @@ require ( google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/postgres v1.4.8 // indirect ) diff --git a/go.sum b/go.sum index 93e56c5..727ec4d 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,13 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= +github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= +github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -196,6 +203,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -262,6 +271,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -299,6 +309,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -336,6 +347,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -369,6 +381,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -391,6 +405,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -430,12 +446,16 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -443,6 +463,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -496,6 +517,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -609,7 +631,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= +gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= +gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/app/api/idl/bilbil_video.go b/internal/app/api/idl/bilbil_video.go index 121cf81..e9c64d1 100644 --- a/internal/app/api/idl/bilbil_video.go +++ b/internal/app/api/idl/bilbil_video.go @@ -1,6 +1,8 @@ package idl import ( + "github.com/google/uuid" + "github.com/lib/pq" "time" "git.vtb.link/eoefans/internal/app/api/util/query_parser" @@ -27,7 +29,7 @@ const ( ) type BilibiliVideo struct { - Id uint64 `json:"-"` + Id uint64 `json:"-" gorm:"-"` Bvid string `json:"bvid"` Aid uint64 `json:"aid"` Name string `json:"name"` @@ -55,6 +57,12 @@ type BilibiliVideo struct { UpdatedAt uint64 `json:"updated_at" gorm:"autoUpdateTime"` } +type NewBilibiliVideo struct { + BilibiliVideo + UUID uuid.UUID `gorm:"column:id;type:uuid;primary_key"` + TagList pq.StringArray `gorm:"column:tag_list;type:varchar(64)[]"` +} + type BilibiliVideoOrder string const ( diff --git a/internal/app/api/idl/bilibili_picture.go b/internal/app/api/idl/bilibili_picture.go index 3dd5d08..b826c24 100644 --- a/internal/app/api/idl/bilibili_picture.go +++ b/internal/app/api/idl/bilibili_picture.go @@ -35,7 +35,7 @@ type BilibiliPicture struct { ImgSrc string `gorm:"column:img_src"` ImgAttr BilibiliPictureAttr `gorm:"column:img_attr"` Feedback DynamicFeedback `gorm:"column:feedback"` - Verify bool `gorm:"column:verify"` + Verify int `gorm:"column:verify"` CreatedAt uint64 `gorm:"autoCreateTime"` UpdatedAt uint64 `gorm:"autoUpdateTime"` DeletedAt gorm.DeletedAt `gorm:"index"` @@ -48,7 +48,7 @@ type BilibiliDynamic struct { UID uint64 `gorm:"column:uid"` DynamicID uint64 `gorm:"column:dynamic_id"` Feedback DynamicFeedback `gorm:"column:feedback"` - Verify bool `gorm:"column:verify"` + Verify int `gorm:"column:verify"` Pictures BilibiliDynamicPictures `gorm:"column:pictures"` PicturesNum int `gorm:"column:pictures_num"` TopicDetails *BilibiliDynamicTopicDetails `gorm:"topic_details"` @@ -173,7 +173,7 @@ type BilibiliRandomPicture struct { ImgID uint64 `json:"img_id"` DynamicID uint64 `json:"dynamic_id"` Tags []BilibiliRandomPictureTag `json:"tags"` - Verify bool `json:"verify"` + Verify int `json:"verify"` } type BilibiliRandomPictureTag struct { TagID uint64 `json:"tag_id"` diff --git a/internal/app/api/service/bilbil_picture.go b/internal/app/api/service/bilbil_picture.go index f9284a0..f76fa29 100644 --- a/internal/app/api/service/bilbil_picture.go +++ b/internal/app/api/service/bilbil_picture.go @@ -131,7 +131,7 @@ func (service *BilbilPicture) Latest(ctx context.Context, req idl.BilibiliPictur } verifyCount := 0 for j := range list[i].Pics { - if list[i].Pics[j].Verify { + if list[i].Pics[j].Verify == 1 { verifyCount += 1 } pic := idl.BilibiliDynamicPicture{ @@ -169,7 +169,7 @@ func (service *BilbilPicture) Recommend(ctx context.Context, req idl.BilibiliPic } verifyCount := 0 for j := range list[i].Pics { - if list[i].Pics[j].Verify { + if list[i].Pics[j].Verify == 1 { verifyCount += 1 } pic := idl.BilibiliDynamicPicture{ diff --git a/internal/pkg/database/gorm.go b/internal/pkg/database/gorm.go index 567de73..802d80a 100644 --- a/internal/pkg/database/gorm.go +++ b/internal/pkg/database/gorm.go @@ -1,6 +1,7 @@ package database import ( + "gorm.io/driver/postgres" "strings" "time" @@ -71,6 +72,8 @@ func newDialector(t, dsn string) (gorm.Dialector, error) { switch strings.ToLower(t) { case "mysql": return mysql.Open(dsn), nil + case "postgres": + return postgres.Open(dsn), nil default: return nil, errors.New("unsupported database type") } diff --git a/internal/repository/bilbil_video.go b/internal/repository/bilbil_video.go index 2e53deb..dffe1a7 100644 --- a/internal/repository/bilbil_video.go +++ b/internal/repository/bilbil_video.go @@ -22,7 +22,7 @@ const ( ) func NewBilibiliVideo(tx *gorm.DB) idl.BilibiliVideoRepository { - return &BilibiliVideoMysqlImpl{tx: tx} + return &BilibiliVideoPostgresImpl{tx: tx} } type BilibiliVideoMysqlImpl struct { diff --git a/internal/repository/bilbil_video_pg.go b/internal/repository/bilbil_video_pg.go new file mode 100644 index 0000000..f958773 --- /dev/null +++ b/internal/repository/bilbil_video_pg.go @@ -0,0 +1,162 @@ +package repository + +import ( + "fmt" + "git.vtb.link/eoefans/internal/app/api/idl" + "git.vtb.link/eoefans/internal/app/api/util/query_parser" + "github.com/google/uuid" + "github.com/lib/pq" + "github.com/pkg/errors" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "strings" + "time" +) + +type BilibiliVideoPostgresImpl struct { + tx *gorm.DB +} + +func toNewBilibiliVideo(video *idl.BilibiliVideo) *idl.NewBilibiliVideo { + return &idl.NewBilibiliVideo{ + BilibiliVideo: *video, + UUID: uuid.New(), + TagList: strings.Split(video.Tag, ","), + } +} + +func toBilibiliVideo(video *idl.NewBilibiliVideo) *idl.BilibiliVideo { + return &video.BilibiliVideo +} + +func toBilibiliVideoList(videos []*idl.NewBilibiliVideo) []*idl.BilibiliVideo { + var list []*idl.BilibiliVideo + for _, video := range videos { + list = append(list, &video.BilibiliVideo) + } + + return list +} + +func (impl *BilibiliVideoPostgresImpl) FindAllByPubDate(from, to time.Time, page, size int64) (list []*idl.BilibiliVideo, total int64, err error) { + var newList []*idl.NewBilibiliVideo + result := impl.tx.Table(bilibiliVideoTableName). + Where("pubdate >= ? AND pubdate <= ?", from.Unix(), to.Unix()). + Offset(int((page - 1) * size)).Limit(int(size)). + Order("pubdate DESC"). + Find(&newList) + + if result == nil { + return nil, 0, errors.Wrap(result.Error, fmt.Sprintf("select %s error", bilibiliVideoTableName)) + } + + result = impl.tx.Table(bilibiliVideoTableName). + Select("id"). + Where("pubdate >= ? AND pubdate <= ?", from.Second(), to). + Count(&total) + + if result == nil { + return nil, 0, errors.Wrap(result.Error, fmt.Sprintf("count from %s error", bilibiliVideoTableName)) + } + + return toBilibiliVideoList(newList), total, nil +} + +func (impl *BilibiliVideoPostgresImpl) Search(queryItems []query_parser.QueryItem, order idl.BilibiliVideoOrder, page, size int64) (list []*idl.BilibiliVideo, total int64, err error) { + var newList []*idl.NewBilibiliVideo + resp := impl.builderQueryItems(impl.tx, queryItems).Table(bilibiliVideoTableName). + Where(fmt.Sprintf("%s.status = ?", bilibiliVideoTableName), idl.BilibiliVideoEnabledStatus). + Order(fmt.Sprintf("%s DESC", order)). + Offset(int((page - 1) * size)).Limit(int(size)). + Find(&newList) + + if resp.Error != nil { + return nil, 0, errors.Wrap(resp.Error, fmt.Sprintf("select %s error", bilibiliVideoTableName)) + } + + resp = impl.builderQueryItems(impl.tx, queryItems).Table(bilibiliVideoTableName). + Where(fmt.Sprintf("%s.status = ?", bilibiliVideoTableName), idl.BilibiliVideoEnabledStatus). + Count(&total) + + if resp.Error != nil { + return nil, 0, errors.Wrap(resp.Error, fmt.Sprintf("count from %s error", bilibiliVideoTableName)) + } + + return toBilibiliVideoList(newList), total, nil +} + +func (impl *BilibiliVideoPostgresImpl) builderQueryItems(tx *gorm.DB, queryItems []query_parser.QueryItem) *gorm.DB { + for _, item := range queryItems { + if strings.ToLower(item.Key) == "tag" { + switch item.Type { + case query_parser.TypeAND: + tx = tx.Where("tag_list @> ARRAY[?]::varchar(64)[]", pq.StringArray(item.Values)) + case query_parser.TypeOR: + tx = tx.Where("tag_list && ARRAY[?]::varchar(64)[]", pq.StringArray(item.Values)) + } + continue + } + + switch item.Type { + case query_parser.TypeAND: + for _, value := range item.Values { + tx = tx.Where(fmt.Sprintf("%s = ?", item.Key), value) + } + case query_parser.TypeOR: + tx = tx.Where(fmt.Sprintf("%s IN (?)", item.Key), item.Values) + case query_parser.TypeBetween: + v := item.GetBetweenValues() + tx = tx.Where(fmt.Sprintf("%s BETWEEN ? AND ?", item.Key), v[0], v[1]) + } + } + + return tx +} + +func (impl *BilibiliVideoPostgresImpl) Shield(bvid string) error { + result := impl.tx.Table(bilibiliVideoTableName). + Where("bvid = ? AND status = ?", bvid, idl.BilibiliVideoEnabledStatus). + UpdateColumn("status", idl.BilibiliVideoDisabledStatus) + + if result.Error != nil { + return errors.Wrap(result.Error, fmt.Sprintf("update %s fail", bilibiliVideoTableName)) + } + + return nil +} + +func (impl *BilibiliVideoPostgresImpl) Failure(bvid string) error { + result := impl.tx.Table(bilibiliVideoTableName). + Where("bvid = ? AND status = ?", bvid, idl.BilibiliVideoEnabledStatus). + UpdateColumn("status", idl.BilibiliVideoFailureStatus) + + if result.Error != nil { + return errors.Wrap(result.Error, fmt.Sprintf("update %s fail", bilibiliVideoTableName)) + } + + return nil +} + +func (impl *BilibiliVideoPostgresImpl) Save(e *idl.BilibiliVideo) error { + newVideo := toNewBilibiliVideo(e) + result := impl.tx.Table(bilibiliVideoTableName).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "bvid"}}, + DoUpdates: clause.AssignmentColumns(append(strings.Split(onUpdateFields, ","), "tag_list")), + }).Create(&newVideo) + + if result.Error != nil { + return errors.Wrap(result.Error, fmt.Sprintf("install %s fail", bilibiliVideoTableName)) + } + + return nil +} + +func (impl *BilibiliVideoPostgresImpl) FindAllByBvidList(bvidList []string) (list []*idl.BilibiliVideo, err error) { + var newList []*idl.NewBilibiliVideo + result := impl.tx.Table(bilibiliVideoTableName).Where("bvid IN (?)", bvidList).Find(&newList) + if result.Error != nil { + return nil, errors.Wrap(result.Error, fmt.Sprintf("select %s error", bilibiliVideoTableName)) + } + + return toBilibiliVideoList(newList), nil +}