mirror of
https://github.com/element-hq/dendrite.git
synced 2025-03-14 14:15:35 +00:00
Merge 78281d5a78 into f4506a0d82
This commit is contained in:
Alex
GitHub
commit
e5b0559cce
12 changed files with 1415 additions and 4 deletions
4
go.mod
4
go.mod
|
|
@ -14,6 +14,7 @@ require (
|
|||
github.com/docker/go-connections v0.5.0
|
||||
github.com/eyedeekay/goSam v0.32.54
|
||||
github.com/eyedeekay/onramp v0.33.8
|
||||
github.com/foxcpp/go-mockdns v1.1.0
|
||||
github.com/getsentry/sentry-go v0.14.0
|
||||
github.com/gologme/log v1.3.0
|
||||
github.com/google/go-cmp v0.6.0
|
||||
|
|
@ -49,6 +50,7 @@ require (
|
|||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
|
||||
golang.org/x/image v0.18.0
|
||||
golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b
|
||||
golang.org/x/net v0.33.0
|
||||
golang.org/x/sync v0.10.0
|
||||
golang.org/x/term v0.27.0
|
||||
gopkg.in/h2non/bimg.v1 v1.1.9
|
||||
|
|
@ -107,6 +109,7 @@ require (
|
|||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/miekg/dns v1.1.57 // indirect
|
||||
github.com/minio/highwayhash v1.0.3 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/term v0.5.0 // indirect
|
||||
|
|
@ -142,7 +145,6 @@ require (
|
|||
go.opentelemetry.io/otel/trace v1.32.0 // indirect
|
||||
go.uber.org/mock v0.4.0 // indirect
|
||||
golang.org/x/mod v0.19.0 // indirect
|
||||
golang.org/x/net v0.33.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
|
|
|
|||
48
go.sum
48
go.sum
|
|
@ -106,6 +106,8 @@ github.com/eyedeekay/sam3 v0.33.8/go.mod h1:ytbwLYLJlW6UA92Ffyc6oioWTKnGeeUMr9CL
|
|||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=
|
||||
github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
|
||||
github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
|
||||
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
|
||||
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
|
||||
|
|
@ -248,8 +250,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
|||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
|
||||
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
|
||||
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
|
||||
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
|
||||
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
|
||||
github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
|
||||
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
|
|
@ -364,6 +366,7 @@ github.com/yggdrasil-network/yggquic v0.0.0-20241212194307-0d495106021f h1:nqinj
|
|||
github.com/yggdrasil-network/yggquic v0.0.0-20241212194307-0d495106021f/go.mod h1:TVCKOUWiXR9cAqr3eDpKvXkVkTph38xwk0wjcvfrtKI=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
|
||||
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
|
||||
|
|
@ -392,6 +395,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
|
|
@ -412,6 +419,10 @@ golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b/go.mod h1:EiXZlVfUTaAyySF
|
|||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
|
||||
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
|
|
@ -420,11 +431,22 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
|
||||
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -436,22 +458,40 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -464,6 +504,10 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
|
||||
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
|
||||
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
|
|||
76
internal/netcontext.go
Normal file
76
internal/netcontext.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrDeniedAddress = fmt.Errorf("address is denied")
|
||||
)
|
||||
|
||||
func GetDialer(allowNetworks []string, denyNetworks []string, dialTimeout time.Duration) *net.Dialer {
|
||||
if len(allowNetworks) == 0 && len(denyNetworks) == 0 {
|
||||
return &net.Dialer{
|
||||
Timeout: dialTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
return &net.Dialer{
|
||||
Timeout: time.Second * 5,
|
||||
ControlContext: allowDenyNetworksControl(allowNetworks, denyNetworks),
|
||||
}
|
||||
}
|
||||
|
||||
// allowDenyNetworksControl is used to allow/deny access to certain networks
|
||||
func allowDenyNetworksControl(allowNetworks, denyNetworks []string) func(_ context.Context, network string, address string, conn syscall.RawConn) error {
|
||||
return func(_ context.Context, network string, address string, conn syscall.RawConn) error {
|
||||
if network != "tcp4" && network != "tcp6" {
|
||||
return fmt.Errorf("%s is not a safe network type", network)
|
||||
}
|
||||
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s is not a valid host/port pair: %s", address, err)
|
||||
}
|
||||
|
||||
ipaddress := net.ParseIP(host)
|
||||
if ipaddress == nil {
|
||||
return fmt.Errorf("%s is not a valid IP address", host)
|
||||
}
|
||||
|
||||
if !isAllowed(ipaddress, allowNetworks, denyNetworks) {
|
||||
return ErrDeniedAddress
|
||||
}
|
||||
|
||||
return nil // allow connection
|
||||
}
|
||||
}
|
||||
|
||||
func isAllowed(ip net.IP, allowCIDRs []string, denyCIDRs []string) bool {
|
||||
if inRange(ip, denyCIDRs) {
|
||||
return false
|
||||
}
|
||||
if inRange(ip, allowCIDRs) {
|
||||
return true
|
||||
}
|
||||
return false // "should never happen"
|
||||
}
|
||||
|
||||
func inRange(ip net.IP, CIDRs []string) bool {
|
||||
for i := 0; i < len(CIDRs); i++ {
|
||||
cidr := CIDRs[i]
|
||||
_, network, err := net.ParseCIDR(cidr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if network.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
@ -153,6 +153,10 @@ func moveFile(src types.Path, dst types.Path) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func MoveFile(src types.Path, dst types.Path) error {
|
||||
return moveFile(src, dst)
|
||||
}
|
||||
|
||||
func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) {
|
||||
tmpDir, err := createTempDir(absBasePath)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -308,10 +308,11 @@ func (r *downloadRequest) respondFromLocalFile(
|
|||
return nil, fmt.Errorf("fileutils.GetPathFromBase64Hash: %w", err)
|
||||
}
|
||||
file, err := os.Open(filePath)
|
||||
defer file.Close() // nolint: errcheck, staticcheck, megacheck
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("os.Open: %w", err)
|
||||
}
|
||||
defer file.Close() // nolint: errcheck, staticcheck, megacheck
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("file.Stat: %w", err)
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ package routing
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/element-hq/dendrite/federationapi/routing"
|
||||
"github.com/element-hq/dendrite/internal"
|
||||
"github.com/element-hq/dendrite/internal/httputil"
|
||||
"github.com/element-hq/dendrite/mediaapi/storage"
|
||||
"github.com/element-hq/dendrite/mediaapi/types"
|
||||
|
|
@ -88,6 +91,7 @@ func Setup(
|
|||
MXCToResult: map[string]*types.RemoteRequestResult{},
|
||||
}
|
||||
|
||||
// v1 url_preview endpoint requiring auth
|
||||
downloadHandler := makeDownloadAPI("download_unauthed", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false)
|
||||
v3mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
|
||||
v3mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
|
||||
|
|
@ -102,6 +106,15 @@ func Setup(
|
|||
v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions)
|
||||
v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions)
|
||||
|
||||
var dialer *net.Dialer
|
||||
if cfg.FederationAPI.AllowNetworkCIDRs != nil || cfg.FederationAPI.DenyNetworkCIDRs != nil {
|
||||
dialer = internal.GetDialer(cfg.FederationAPI.AllowNetworkCIDRs, cfg.FederationAPI.DenyNetworkCIDRs, time.Duration(cfg.MediaAPI.UrlPreviewTimeout))
|
||||
}
|
||||
urlPreviewHandler := httputil.MakeAuthAPI("preview_url", userAPI, makeUrlPreviewHandler(&cfg.MediaAPI, dialer, rateLimits, db, activeThumbnailGeneration))
|
||||
v1mux.Handle("/preview_url", urlPreviewHandler).Methods(http.MethodGet, http.MethodOptions)
|
||||
// That method is deprecated according to spec but still in use
|
||||
v3mux.Handle("/preview_url", urlPreviewHandler).Methods(http.MethodGet, http.MethodOptions)
|
||||
|
||||
v1mux.Handle("/thumbnail/{serverName}/{mediaId}",
|
||||
httputil.MakeHTTPAPI("thumbnail", userAPI, cfg.Global.Metrics.Enabled, makeDownloadAPI("thumbnail_authed_client", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false), httputil.WithAuth()),
|
||||
).Methods(http.MethodGet, http.MethodOptions)
|
||||
|
|
|
|||
696
mediaapi/routing/url_preview.go
Normal file
696
mediaapi/routing/url_preview.go
Normal file
|
|
@ -0,0 +1,696 @@
|
|||
// Copyright 2024 New Vector Ltd.
|
||||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
// Please see LICENSE files in the repository root for full details.
|
||||
|
||||
package routing
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/element-hq/dendrite/internal/httputil"
|
||||
"github.com/element-hq/dendrite/mediaapi/storage"
|
||||
"github.com/element-hq/dendrite/mediaapi/types"
|
||||
"github.com/element-hq/dendrite/setup/config"
|
||||
userapi "github.com/element-hq/dendrite/userapi/api"
|
||||
|
||||
"github.com/element-hq/dendrite/mediaapi/fileutils"
|
||||
"github.com/element-hq/dendrite/mediaapi/thumbnailer"
|
||||
|
||||
"github.com/matrix-org/gomatrixserverlib/spec"
|
||||
"github.com/matrix-org/util"
|
||||
"github.com/pkg/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrorMissingUrl = errors.New("missing url")
|
||||
ErrorUnsupportedContentType = errors.New("unsupported content type")
|
||||
ErrorFileTooLarge = errors.New("file too large")
|
||||
ErrorTimeoutThumbnailGenerator = errors.New("timeout waiting for thumbnail generator")
|
||||
ErrNoMetadataFound = errors.New("no metadata found")
|
||||
ErrorUrlDenied = errors.New("url is in the urls deny list")
|
||||
)
|
||||
|
||||
func makeUrlPreviewHandler(
|
||||
cfg *config.MediaAPI,
|
||||
dialer *net.Dialer,
|
||||
rateLimits *httputil.RateLimits,
|
||||
db storage.Database,
|
||||
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
|
||||
) func(req *http.Request, device *userapi.Device) util.JSONResponse {
|
||||
|
||||
activeUrlPreviewRequests := &types.ActiveUrlPreviewRequests{Url: map[string]*types.UrlPreviewResult{}}
|
||||
urlPreviewCache := &types.UrlPreviewCache{Records: map[string]*types.UrlPreviewCacheRecord{}}
|
||||
urlDenyList := createUrlDenyList(cfg)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
t := time.Now().Unix()
|
||||
urlPreviewCache.Lock()
|
||||
for k, record := range urlPreviewCache.Records {
|
||||
if record.Created < (t - int64(cfg.UrlPreviewCacheTime)) {
|
||||
delete(urlPreviewCache.Records, k)
|
||||
}
|
||||
}
|
||||
urlPreviewCache.Unlock()
|
||||
time.Sleep(time.Duration(60) * time.Second)
|
||||
}
|
||||
}()
|
||||
|
||||
httpHandler := func(req *http.Request, device *userapi.Device) util.JSONResponse {
|
||||
req = util.RequestWithLogging(req)
|
||||
|
||||
// log := util.GetLogger(req.Context())
|
||||
// Here be call to the url preview handler
|
||||
pUrl := req.URL.Query().Get("url")
|
||||
ts := req.URL.Query().Get("ts")
|
||||
if pUrl == "" {
|
||||
return util.ErrorResponse(ErrorMissingUrl)
|
||||
}
|
||||
_ = ts
|
||||
|
||||
logger := util.GetLogger(req.Context()).WithFields(log.Fields{
|
||||
"url": pUrl,
|
||||
})
|
||||
// Check rate limits
|
||||
if r := rateLimits.Limit(req, device); r != nil {
|
||||
return *r
|
||||
}
|
||||
|
||||
// Check if the url is in the deny list
|
||||
if checkIsURLDenied(urlDenyList, pUrl) {
|
||||
return util.ErrorResponse(ErrorUrlDenied)
|
||||
}
|
||||
|
||||
urlParsed, perr := url.Parse(pUrl)
|
||||
if perr != nil {
|
||||
return util.ErrorResponse(ErrorMissingUrl)
|
||||
}
|
||||
|
||||
hash := getHashFromString(pUrl)
|
||||
|
||||
// Get for url preview from in-memory cache
|
||||
if response, ok := checkInternalCacheResponse(urlPreviewCache, pUrl); ok {
|
||||
return response
|
||||
}
|
||||
|
||||
if urlPreviewCached, err := loadUrlPreviewResponse(req.Context(), cfg, db, hash); err == nil {
|
||||
logger.Debug("Loaded url preview from the cache")
|
||||
// Put in into the cache for further usage
|
||||
defer func() {
|
||||
if _, ok := urlPreviewCache.Records[pUrl]; !ok {
|
||||
|
||||
urlPreviewCacheItem := &types.UrlPreviewCacheRecord{
|
||||
Created: time.Now().Unix(),
|
||||
Preview: urlPreviewCached,
|
||||
}
|
||||
urlPreviewCache.Lock()
|
||||
urlPreviewCache.Records[pUrl] = urlPreviewCacheItem
|
||||
defer urlPreviewCache.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: urlPreviewCached,
|
||||
}
|
||||
}
|
||||
|
||||
// Check if there is an active request
|
||||
if response, ok := checkActivePreviewResponse(activeUrlPreviewRequests, pUrl); ok {
|
||||
return response
|
||||
}
|
||||
|
||||
// Start new url preview request
|
||||
activeUrlPreviewRequest := &types.UrlPreviewResult{Cond: sync.NewCond(&sync.Mutex{})}
|
||||
activeUrlPreviewRequests.Url[pUrl] = activeUrlPreviewRequest
|
||||
activeUrlPreviewRequests.Unlock()
|
||||
|
||||
// we defer caching the url preview response as well as signalling the waiting goroutines
|
||||
// about the completion of the request
|
||||
defer func() {
|
||||
|
||||
urlPreviewCacheItem := &types.UrlPreviewCacheRecord{
|
||||
Created: time.Now().Unix(),
|
||||
}
|
||||
if activeUrlPreviewRequest.Error != nil {
|
||||
urlPreviewCacheItem.Error = activeUrlPreviewRequest.Error
|
||||
} else {
|
||||
urlPreviewCacheItem.Preview = activeUrlPreviewRequest.Preview
|
||||
// Store the response file for further usage
|
||||
err := storeUrlPreviewResponse(req.Context(), cfg, db, *device, hash, activeUrlPreviewRequest.Preview, logger)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to store url preview response")
|
||||
}
|
||||
}
|
||||
|
||||
urlPreviewCache.Lock()
|
||||
urlPreviewCache.Records[pUrl] = urlPreviewCacheItem
|
||||
defer urlPreviewCache.Unlock()
|
||||
|
||||
activeUrlPreviewRequests.Lock()
|
||||
activeUrlPreviewRequests.Url[pUrl].Cond.Broadcast()
|
||||
delete(activeUrlPreviewRequests.Url, pUrl)
|
||||
defer activeUrlPreviewRequests.Unlock()
|
||||
}()
|
||||
|
||||
resp, err := downloadUrl(pUrl, dialer, time.Duration(cfg.UrlPreviewTimeout)*time.Second)
|
||||
if err != nil {
|
||||
activeUrlPreviewRequest.Error = err
|
||||
} else {
|
||||
defer resp.Body.Close() // nolint: errcheck
|
||||
|
||||
var result *types.UrlPreview
|
||||
var err error
|
||||
var mediaData *types.MediaMetadata
|
||||
var width, height int
|
||||
|
||||
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
|
||||
// The url is a webpage - get data from the meta tags
|
||||
result = getPreviewFromHTML(resp, urlParsed)
|
||||
if result.ImageUrl != "" {
|
||||
// In case of an image in the preview we download it
|
||||
if imgReader, derr := downloadUrl(result.ImageUrl, dialer, time.Duration(cfg.UrlPreviewTimeout)*time.Second); derr == nil {
|
||||
mediaData, width, height, _ = downloadAndStoreImage("url_preview", req.Context(), imgReader, cfg, device, db, activeThumbnailGeneration, logger)
|
||||
}
|
||||
// We don't show the original image in the preview
|
||||
// as it is insecure for room members
|
||||
result.ImageUrl = ""
|
||||
}
|
||||
} else if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
|
||||
// The url is an image link
|
||||
mediaData, width, height, err = downloadAndStoreImage("somefile", req.Context(), resp, cfg, device, db, activeThumbnailGeneration, logger)
|
||||
if err == nil {
|
||||
result = &types.UrlPreview{}
|
||||
}
|
||||
} else {
|
||||
return util.ErrorResponse(errors.New("Unsupported content type"))
|
||||
}
|
||||
|
||||
// In case of any error happened during the page/image download
|
||||
// we store the error instead of the preview
|
||||
if err != nil {
|
||||
activeUrlPreviewRequest.Error = err
|
||||
} else {
|
||||
// We have a mediadata so we have an image in the preview
|
||||
if mediaData != nil {
|
||||
result.ImageUrl = fmt.Sprintf("mxc://%s/%s", mediaData.Origin, mediaData.MediaID)
|
||||
result.ImageWidth = width
|
||||
result.ImageHeight = height
|
||||
result.ImageType = mediaData.ContentType
|
||||
result.ImageSize = mediaData.FileSizeBytes
|
||||
}
|
||||
|
||||
activeUrlPreviewRequest.Preview = result
|
||||
}
|
||||
}
|
||||
|
||||
// Return eather the error or the preview
|
||||
if activeUrlPreviewRequest.Error != nil {
|
||||
return util.ErrorResponse(activeUrlPreviewRequest.Error)
|
||||
} else {
|
||||
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: activeUrlPreviewRequest.Preview,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return httpHandler
|
||||
|
||||
}
|
||||
|
||||
func checkInternalCacheResponse(urlPreviewCache *types.UrlPreviewCache, url string) (util.JSONResponse, bool) {
|
||||
if cacheRecord, ok := urlPreviewCache.Records[url]; ok {
|
||||
if cacheRecord.Error != nil {
|
||||
return util.ErrorResponse(cacheRecord.Error), true
|
||||
}
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: cacheRecord.Preview,
|
||||
}, true
|
||||
}
|
||||
return util.JSONResponse{}, false
|
||||
}
|
||||
|
||||
func checkActivePreviewResponse(activeUrlPreviewRequests *types.ActiveUrlPreviewRequests, url string) (util.JSONResponse, bool) {
|
||||
activeUrlPreviewRequests.Lock()
|
||||
if activeUrlPreviewRequest, ok := activeUrlPreviewRequests.Url[url]; ok {
|
||||
activeUrlPreviewRequests.Unlock()
|
||||
// Wait for it to complete
|
||||
activeUrlPreviewRequest.Cond.L.Lock()
|
||||
defer activeUrlPreviewRequest.Cond.L.Unlock()
|
||||
activeUrlPreviewRequest.Cond.Wait()
|
||||
|
||||
if activeUrlPreviewRequest.Error != nil {
|
||||
return util.ErrorResponse(activeUrlPreviewRequest.Error), true
|
||||
}
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: activeUrlPreviewRequest.Preview,
|
||||
}, true
|
||||
}
|
||||
return util.JSONResponse{}, false
|
||||
}
|
||||
|
||||
func downloadUrl(url string, dialer *net.Dialer, t time.Duration) (*http.Response, error) {
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
}
|
||||
if dialer != nil {
|
||||
tr.DialContext = dialer.DialContext
|
||||
}
|
||||
|
||||
client := http.Client{Timeout: t, Transport: tr}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, errors.New("HTTP status code: " + strconv.Itoa(resp.StatusCode))
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func getPreviewFromHTML(resp *http.Response, urlParsed *url.URL) *types.UrlPreview {
|
||||
|
||||
fields := getMetaFieldsFromHTML(resp)
|
||||
preview := &types.UrlPreview{
|
||||
Title: fields["og:title"],
|
||||
Description: fields["og:description"],
|
||||
Type: fields["og:type"],
|
||||
Url: fields["og:url"],
|
||||
}
|
||||
|
||||
if fields["og:title"] == "" {
|
||||
preview.Title = urlParsed.String()
|
||||
}
|
||||
if fields["og:image"] != "" {
|
||||
preview.ImageUrl = fields["og:image"]
|
||||
} else if fields["og:image:url"] != "" {
|
||||
preview.ImageUrl = fields["og:image:url"]
|
||||
} else if fields["og:image:secure_url"] != "" {
|
||||
preview.ImageUrl = fields["og:image:secure_url"]
|
||||
}
|
||||
|
||||
if preview.ImageUrl != "" {
|
||||
if imgUrl, err := url.Parse(preview.ImageUrl); err == nil {
|
||||
// Use the same scheme and host as the original URL if empty
|
||||
if imgUrl.Scheme == "" {
|
||||
imgUrl.Scheme = urlParsed.Scheme
|
||||
}
|
||||
// Use the same host as the original URL if empty
|
||||
if imgUrl.Host == "" {
|
||||
imgUrl.Host = urlParsed.Host
|
||||
}
|
||||
preview.ImageUrl = imgUrl.String()
|
||||
} else {
|
||||
preview.ImageUrl = ""
|
||||
}
|
||||
}
|
||||
|
||||
return preview
|
||||
}
|
||||
|
||||
func downloadAndStoreImage(
|
||||
filename string,
|
||||
ctx context.Context,
|
||||
req *http.Response,
|
||||
cfg *config.MediaAPI,
|
||||
dev *userapi.Device,
|
||||
db storage.Database,
|
||||
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
|
||||
logger *log.Entry,
|
||||
|
||||
) (*types.MediaMetadata, int, int, error) {
|
||||
|
||||
var width, height int
|
||||
|
||||
userid := types.MatrixUserID(dev.UserID)
|
||||
|
||||
reqReader := req.Body.(io.Reader)
|
||||
if cfg.MaxFileSizeBytes > 0 {
|
||||
reqReader = io.LimitReader(reqReader, int64(cfg.MaxFileSizeBytes)+1)
|
||||
}
|
||||
hash, bytesWritten, tmpDir, fileErr := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath)
|
||||
if fileErr != nil {
|
||||
logger.WithError(fileErr).WithFields(log.Fields{
|
||||
"MaxFileSizeBytes": cfg.MaxFileSizeBytes,
|
||||
}).Warn("Error while transferring file")
|
||||
return nil, width, height, fileErr
|
||||
}
|
||||
defer fileutils.RemoveDir(tmpDir, logger)
|
||||
|
||||
// Check if temp file size exceeds max file size configuration
|
||||
if cfg.MaxFileSizeBytes > 0 && bytesWritten > types.FileSizeBytes(cfg.MaxFileSizeBytes) {
|
||||
return nil, 0, 0, ErrorFileTooLarge
|
||||
}
|
||||
|
||||
// Check if we already have this file
|
||||
existingMetadata, err := db.GetMediaMetadataByHash(
|
||||
ctx, hash, cfg.Matrix.ServerName,
|
||||
)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to get media metadata by hash")
|
||||
return nil, width, height, err
|
||||
}
|
||||
|
||||
if existingMetadata != nil {
|
||||
|
||||
logger.WithField("mediaID", existingMetadata.MediaID).Debug("media already exists")
|
||||
// Here we have to read the image to get it's size
|
||||
filePath, pathErr := fileutils.GetPathFromBase64Hash(existingMetadata.Base64Hash, cfg.AbsBasePath)
|
||||
if pathErr != nil {
|
||||
return nil, width, height, pathErr
|
||||
}
|
||||
width, height, err = thumbnailer.GetImageSize(string(filePath))
|
||||
if err != nil {
|
||||
return nil, width, height, err
|
||||
}
|
||||
return existingMetadata, width, height, nil
|
||||
}
|
||||
|
||||
tmpFileName := filepath.Join(string(tmpDir), "content")
|
||||
fileType, typeErr := detectFileType(tmpFileName, logger)
|
||||
if typeErr != nil {
|
||||
logger.WithError(err).Error("unable to detect file type")
|
||||
return nil, width, height, typeErr
|
||||
}
|
||||
logger.WithField("contentType", fileType).Debug("uploaded file is an image")
|
||||
|
||||
var thumbnailPath string
|
||||
|
||||
if cfg.UrlPreviewThumbnailSize.Width != 0 {
|
||||
// Create a thumbnail from the image
|
||||
thumbnailPath = tmpFileName + ".thumbnail"
|
||||
|
||||
width, height, err = createThumbnail(types.Path(tmpFileName), types.Path(thumbnailPath), types.ThumbnailSize(cfg.UrlPreviewThumbnailSize),
|
||||
hash, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, logger)
|
||||
if err != nil {
|
||||
if errors.Is(err, thumbnailer.ErrThumbnailTooLarge) {
|
||||
// In case the image is smaller than the thumbnail size
|
||||
// we don't create a thumbnail
|
||||
thumbnailPath = tmpFileName
|
||||
width, height, err = thumbnailer.GetImageSize(thumbnailPath)
|
||||
if err != nil {
|
||||
return nil, width, height, err
|
||||
}
|
||||
} else {
|
||||
return nil, width, height, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No thumbnail size specified, use the original image
|
||||
thumbnailPath = tmpFileName
|
||||
width, height, err = thumbnailer.GetImageSize(thumbnailPath)
|
||||
if err != nil {
|
||||
return nil, width, height, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
thumbnailFileInfo, statErr := os.Stat(thumbnailPath)
|
||||
if statErr != nil {
|
||||
logger.WithError(statErr).Error("unable to get thumbnail file info")
|
||||
return nil, width, height, statErr
|
||||
}
|
||||
|
||||
r := &uploadRequest{
|
||||
MediaMetadata: &types.MediaMetadata{
|
||||
Origin: cfg.Matrix.ServerName,
|
||||
},
|
||||
Logger: logger,
|
||||
}
|
||||
|
||||
// Move the thumbnail to the media store
|
||||
mediaID, mediaErr := r.generateMediaID(ctx, db)
|
||||
if mediaErr != nil {
|
||||
logger.WithError(mediaErr).Error("unable to generate media ID")
|
||||
return nil, width, height, mediaErr
|
||||
}
|
||||
mediaMetaData := &types.MediaMetadata{
|
||||
MediaID: mediaID,
|
||||
Origin: cfg.Matrix.ServerName,
|
||||
ContentType: types.ContentType(fileType),
|
||||
FileSizeBytes: types.FileSizeBytes(thumbnailFileInfo.Size()),
|
||||
UploadName: types.Filename(filename),
|
||||
CreationTimestamp: spec.Timestamp(time.Now().Unix()),
|
||||
Base64Hash: hash,
|
||||
UserID: userid,
|
||||
}
|
||||
|
||||
finalPath, pathErr := fileutils.GetPathFromBase64Hash(mediaMetaData.Base64Hash, cfg.AbsBasePath)
|
||||
if pathErr != nil {
|
||||
logger.WithError(pathErr).Error("unable to get path from base64 hash")
|
||||
return nil, width, height, pathErr
|
||||
}
|
||||
err = fileutils.MoveFile(types.Path(thumbnailPath), types.Path(finalPath))
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to move thumbnail file")
|
||||
return nil, width, height, err
|
||||
}
|
||||
// Store the metadata in the database
|
||||
err = db.StoreMediaMetadata(ctx, mediaMetaData)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to store media metadata")
|
||||
return nil, width, height, err
|
||||
}
|
||||
|
||||
return mediaMetaData, width, height, nil
|
||||
}
|
||||
|
||||
func createThumbnail(src types.Path, dst types.Path, size types.ThumbnailSize, hash types.Base64Hash, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, logger *log.Entry) (int, int, error) {
|
||||
timeout := time.After(30 * time.Second)
|
||||
for {
|
||||
// Check if we have too many thumbnail generators running
|
||||
// If so, wait up to 30 seconds for one to finish
|
||||
if len(activeThumbnailGeneration.PathToResult) < maxThumbnailGenerators {
|
||||
|
||||
activeThumbnailGeneration.Lock()
|
||||
activeThumbnailGeneration.PathToResult[string(hash)] = nil
|
||||
activeThumbnailGeneration.Unlock()
|
||||
|
||||
defer func() {
|
||||
activeThumbnailGeneration.Lock()
|
||||
delete(activeThumbnailGeneration.PathToResult, string(hash))
|
||||
activeThumbnailGeneration.Unlock()
|
||||
}()
|
||||
|
||||
width, height, err := thumbnailer.CreateThumbnailFromFile(src, dst, size, logger)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to create thumbnail")
|
||||
return 0, 0, err
|
||||
}
|
||||
return width, height, nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-timeout:
|
||||
logger.Error("timed out waiting for thumbnail generator")
|
||||
return 0, 0, ErrorTimeoutThumbnailGenerator
|
||||
default:
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func storeUrlPreviewResponse(ctx context.Context, cfg *config.MediaAPI, db storage.Database, user userapi.Device, hash types.Base64Hash, preview *types.UrlPreview, logger *log.Entry) error {
|
||||
|
||||
jsonPreview, err := json.Marshal(preview)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, bytes.NewReader(jsonPreview), cfg.AbsBasePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fileutils.RemoveDir(tmpDir, logger)
|
||||
|
||||
r := &uploadRequest{
|
||||
MediaMetadata: &types.MediaMetadata{
|
||||
Origin: cfg.Matrix.ServerName,
|
||||
},
|
||||
Logger: logger,
|
||||
}
|
||||
|
||||
mediaID, err := r.generateMediaID(ctx, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mediaMetaData := &types.MediaMetadata{
|
||||
MediaID: mediaID,
|
||||
Origin: cfg.Matrix.ServerName,
|
||||
ContentType: "application/json",
|
||||
FileSizeBytes: types.FileSizeBytes(bytesWritten),
|
||||
UploadName: types.Filename("url_preview.json"),
|
||||
CreationTimestamp: spec.Timestamp(time.Now().Unix()),
|
||||
Base64Hash: hash,
|
||||
UserID: types.MatrixUserID(user.UserID),
|
||||
}
|
||||
|
||||
_, _, err = fileutils.MoveFileWithHashCheck(tmpDir, mediaMetaData, cfg.AbsBasePath, logger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = db.StoreMediaMetadata(ctx, mediaMetaData)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to store media metadata")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadUrlPreviewResponse(ctx context.Context, cfg *config.MediaAPI, db storage.Database, hash types.Base64Hash) (*types.UrlPreview, error) {
|
||||
if mediaMetadata, err := db.GetMediaMetadataByHash(ctx, hash, cfg.Matrix.ServerName); err == nil && mediaMetadata != nil {
|
||||
// Get the response file
|
||||
filePath, err := fileutils.GetPathFromBase64Hash(mediaMetadata.Base64Hash, cfg.AbsBasePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := os.ReadFile(string(filePath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var preview types.UrlPreview
|
||||
err = json.Unmarshal(data, &preview)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &preview, nil
|
||||
}
|
||||
return nil, ErrNoMetadataFound
|
||||
}
|
||||
|
||||
func detectFileType(filePath string, logger *log.Entry) (string, error) {
|
||||
// Check if the file is an image.
|
||||
// Otherwise return an error
|
||||
file, err := os.Open(string(filePath))
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to open image file")
|
||||
return "", err
|
||||
}
|
||||
defer file.Close() // nolint: errcheck
|
||||
|
||||
buf := make([]byte, 512)
|
||||
|
||||
_, err = file.Read(buf)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to read file")
|
||||
return "", err
|
||||
}
|
||||
|
||||
fileType := http.DetectContentType(buf)
|
||||
if !strings.HasPrefix(fileType, "image") {
|
||||
logger.WithField("contentType", fileType).Debugf("uploaded file is not an image")
|
||||
return "", ErrorUnsupportedContentType
|
||||
}
|
||||
return fileType, nil
|
||||
}
|
||||
|
||||
func getHashFromString(s string) types.Base64Hash {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte(s))
|
||||
return types.Base64Hash(base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)))
|
||||
}
|
||||
|
||||
func getMetaFieldsFromHTML(resp *http.Response) map[string]string {
|
||||
htmlTokens := html.NewTokenizer(resp.Body)
|
||||
ogValues := map[string]string{}
|
||||
fieldsToGet := []string{
|
||||
"og:title",
|
||||
"og:description",
|
||||
"og:image",
|
||||
"og:image:url",
|
||||
"og:image:secure_url",
|
||||
"og:type",
|
||||
"og:url",
|
||||
}
|
||||
fieldsMap := make(map[string]bool, len(fieldsToGet))
|
||||
for _, field := range fieldsToGet {
|
||||
fieldsMap[field] = true
|
||||
ogValues[field] = ""
|
||||
}
|
||||
|
||||
headTagOpened := false
|
||||
for {
|
||||
tokenType := htmlTokens.Next()
|
||||
if tokenType == html.ErrorToken {
|
||||
break
|
||||
}
|
||||
token := htmlTokens.Token()
|
||||
|
||||
// Check if there was opened a head tag
|
||||
if tokenType == html.StartTagToken && token.Data == "head" {
|
||||
headTagOpened = true
|
||||
}
|
||||
// We search for meta tags only inside the head tag if it exists
|
||||
if headTagOpened && tokenType == html.EndTagToken && token.Data == "head" {
|
||||
break
|
||||
}
|
||||
if (tokenType == html.SelfClosingTagToken || tokenType == html.StartTagToken) && token.Data == "meta" {
|
||||
var propertyName string
|
||||
var propertyContent string
|
||||
for _, attr := range token.Attr {
|
||||
if attr.Key == "property" {
|
||||
propertyName = attr.Val
|
||||
}
|
||||
if attr.Key == "content" {
|
||||
propertyContent = attr.Val
|
||||
}
|
||||
if propertyName != "" && propertyContent != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Push the values to the map if they are in the required fields list
|
||||
if propertyName != "" && propertyContent != "" {
|
||||
if _, ok := fieldsMap[propertyName]; ok {
|
||||
ogValues[propertyName] = propertyContent
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ogValues
|
||||
}
|
||||
|
||||
func createUrlDenyList(cfg *config.MediaAPI) []*regexp.Regexp {
|
||||
denyList := make([]*regexp.Regexp, len(cfg.UrlPreviewDenylist))
|
||||
for i, pattern := range cfg.UrlPreviewDenylist {
|
||||
denyList[i] = regexp.MustCompile(pattern)
|
||||
}
|
||||
return denyList
|
||||
}
|
||||
|
||||
func checkIsURLDenied(urldenylist []*regexp.Regexp, url string) bool {
|
||||
// Check if the url is in the deny list
|
||||
for _, pattern := range urldenylist {
|
||||
if pattern.MatchString(url) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
461
mediaapi/routing/url_preview_test.go
Normal file
461
mediaapi/routing/url_preview_test.go
Normal file
|
|
@ -0,0 +1,461 @@
|
|||
// Copyright 2024 New Vector Ltd.
|
||||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
// Please see LICENSE files in the repository root for full details.
|
||||
|
||||
package routing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/element-hq/dendrite/internal"
|
||||
"github.com/element-hq/dendrite/internal/httputil"
|
||||
"github.com/element-hq/dendrite/internal/sqlutil"
|
||||
"github.com/element-hq/dendrite/mediaapi/fileutils"
|
||||
"github.com/element-hq/dendrite/mediaapi/storage"
|
||||
"github.com/element-hq/dendrite/mediaapi/types"
|
||||
"github.com/element-hq/dendrite/setup/config"
|
||||
userapi "github.com/element-hq/dendrite/userapi/api"
|
||||
"github.com/foxcpp/go-mockdns"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var tests = []map[string]interface{}{
|
||||
{
|
||||
"test": `<html>
|
||||
<head>
|
||||
<title>Title</title>
|
||||
<meta property="og:title" content="test_title"/>
|
||||
<meta property="og:description" content="test_description" ></meta>
|
||||
<meta property="og:image" content="test.png">
|
||||
<meta property="og:image:url" content="test2.png"/><meta>
|
||||
<meta property="og:image:secure_url" content="test3.png">
|
||||
<meta property="og:type" content="image/jpeg" />
|
||||
<meta property="og:url" content="/image.jpg" />
|
||||
</head>
|
||||
</html>
|
||||
`,
|
||||
"expected": map[string]string{
|
||||
"og:title": "test_title",
|
||||
"og:description": "test_description",
|
||||
"og:image": "test.png",
|
||||
"og:image:url": "test2.png",
|
||||
"og:image:secure_url": "test3.png",
|
||||
"og:type": "image/jpeg",
|
||||
"og:url": "/image.jpg",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func Test_getMetaFieldsFromHTML(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
r := &http.Response{Body: io.NopCloser(strings.NewReader(test["test"].(string)))}
|
||||
result := getMetaFieldsFromHTML(r)
|
||||
fmt.Println(result)
|
||||
for k, v := range test["expected"].(map[string]string) {
|
||||
if val, ok := result[k]; ok {
|
||||
if val != v {
|
||||
t.Errorf("Values don't match: expected %s, got %s", v, val)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("Not found %s in the test HTML", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test_LoadStorePreview(t *testing.T) {
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Errorf("failed to get current working directory: %v", err)
|
||||
}
|
||||
|
||||
maxSize := config.FileSizeBytes(8)
|
||||
logger := log.New().WithField("mediaapi", "test")
|
||||
testdataPath := filepath.Join(wd, "./testdata")
|
||||
|
||||
g := &config.Global{}
|
||||
g.Defaults(config.DefaultOpts{Generate: true})
|
||||
cfg := &config.MediaAPI{
|
||||
Matrix: g,
|
||||
MaxFileSizeBytes: maxSize,
|
||||
BasePath: config.Path(testdataPath),
|
||||
AbsBasePath: config.Path(testdataPath),
|
||||
DynamicThumbnails: false,
|
||||
}
|
||||
|
||||
// create testdata folder and remove when done
|
||||
_ = os.Mkdir(testdataPath, os.ModePerm)
|
||||
defer fileutils.RemoveDir(types.Path(testdataPath), nil)
|
||||
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
|
||||
db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
|
||||
ConnectionString: "file::memory:?cache=shared",
|
||||
MaxOpenConnections: 100,
|
||||
MaxIdleConnections: 2,
|
||||
ConnMaxLifetimeSeconds: -1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("error opening mediaapi database: %v", err)
|
||||
}
|
||||
|
||||
testPreview := &types.UrlPreview{
|
||||
Title: "test_title",
|
||||
Description: "test_description",
|
||||
ImageUrl: "test_url.png",
|
||||
ImageType: "image/png",
|
||||
ImageSize: types.FileSizeBytes(100),
|
||||
ImageHeight: 100,
|
||||
ImageWidth: 100,
|
||||
Type: "video",
|
||||
Url: "video.avi",
|
||||
}
|
||||
|
||||
hash := getHashFromString("testhash")
|
||||
device := userapi.Device{
|
||||
ID: "1",
|
||||
UserID: "user",
|
||||
}
|
||||
err = storeUrlPreviewResponse(context.Background(), cfg, db, device, hash, testPreview, logger)
|
||||
if err != nil {
|
||||
t.Errorf("Can't store urel preview response: %v", err)
|
||||
}
|
||||
|
||||
filePath, err := fileutils.GetPathFromBase64Hash(hash, cfg.AbsBasePath)
|
||||
if err != nil {
|
||||
t.Errorf("Can't get stored file path: %v", err)
|
||||
}
|
||||
_, err = os.Stat(filePath)
|
||||
if err != nil {
|
||||
t.Errorf("Can't get stored file info: %v", err)
|
||||
|
||||
}
|
||||
|
||||
loadedPreview, err := loadUrlPreviewResponse(context.Background(), cfg, db, hash)
|
||||
if err != nil {
|
||||
t.Errorf("Can't load the preview: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(loadedPreview, testPreview) {
|
||||
t.Errorf("Stored and loaded previews not equal: stored=%v, loaded=%v", testPreview, loadedPreview)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Blacklist(t *testing.T) {
|
||||
|
||||
tests := map[string]interface{}{
|
||||
"entrys": []string{
|
||||
"drive.google.com",
|
||||
"https?://altavista.com/someurl",
|
||||
"https?://(www.)?google.com",
|
||||
"http://stackoverflow.com",
|
||||
},
|
||||
"tests": map[string]bool{
|
||||
"https://drive.google.com/path": true,
|
||||
"http://altavista.com": false,
|
||||
"http://altavista.com/someurl": true,
|
||||
"https://altavista.com/someurl": true,
|
||||
"https://stackoverflow.com": false,
|
||||
},
|
||||
}
|
||||
|
||||
cfg := &config.MediaAPI{
|
||||
UrlPreviewDenylist: tests["entrys"].([]string),
|
||||
}
|
||||
denylist := createUrlDenyList(cfg)
|
||||
|
||||
for url, expected := range tests["tests"].(map[string]bool) {
|
||||
value := checkIsURLDenied(denylist, url)
|
||||
if value != expected {
|
||||
t.Errorf("Blacklist %v: expected=%v, got=%v", url, expected, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test_ActiveRequestWaiting(t *testing.T) {
|
||||
activeRequests := &types.ActiveUrlPreviewRequests{
|
||||
Url: map[string]*types.UrlPreviewResult{
|
||||
"someurl": &types.UrlPreviewResult{
|
||||
Cond: sync.NewCond(&sync.Mutex{}),
|
||||
Preview: &types.UrlPreview{},
|
||||
Error: nil,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
successResults := 0
|
||||
successResultsLock := &sync.Mutex{}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
go func() {
|
||||
if res, ok := checkActivePreviewResponse(activeRequests, "someurl"); ok {
|
||||
if res.Code != 200 {
|
||||
t.Errorf("Unsuccess result: %v", res)
|
||||
}
|
||||
successResultsLock.Lock()
|
||||
defer successResultsLock.Unlock()
|
||||
successResults++
|
||||
return
|
||||
}
|
||||
t.Errorf("url %v not found in active requests", "someurl")
|
||||
}()
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(1) * time.Second)
|
||||
successResultsLock.Lock()
|
||||
if successResults != 0 {
|
||||
t.Error("Subroutines haven't waited for the result")
|
||||
}
|
||||
successResultsLock.Unlock()
|
||||
activeRequests.Url["someurl"].Cond.Broadcast()
|
||||
to := time.After(1 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case <-to:
|
||||
t.Errorf("Test timed out, results=%v", successResults)
|
||||
return
|
||||
default:
|
||||
}
|
||||
successResultsLock.Lock()
|
||||
if successResults == 3 {
|
||||
break
|
||||
}
|
||||
successResultsLock.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func Test_UrlPreviewHandler(t *testing.T) {
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Errorf("failed to get current working directory: %v", err)
|
||||
}
|
||||
|
||||
maxSize := config.FileSizeBytes(1024 * 1024)
|
||||
testdataPath := filepath.Join(wd, "./testdata")
|
||||
|
||||
g := &config.Global{}
|
||||
g.Defaults(config.DefaultOpts{Generate: true})
|
||||
cfg := &config.MediaAPI{
|
||||
Matrix: g,
|
||||
MaxFileSizeBytes: maxSize,
|
||||
BasePath: config.Path(testdataPath),
|
||||
AbsBasePath: config.Path(testdataPath),
|
||||
DynamicThumbnails: false,
|
||||
}
|
||||
cfg2 := &config.MediaAPI{
|
||||
Matrix: g,
|
||||
MaxFileSizeBytes: maxSize,
|
||||
BasePath: config.Path(testdataPath),
|
||||
AbsBasePath: config.Path(testdataPath),
|
||||
UrlPreviewThumbnailSize: config.ThumbnailSize{
|
||||
Width: 10,
|
||||
Height: 10,
|
||||
},
|
||||
MaxThumbnailGenerators: 10,
|
||||
DynamicThumbnails: false,
|
||||
}
|
||||
|
||||
// create testdata folder and remove when done
|
||||
_ = os.Mkdir(testdataPath, os.ModePerm)
|
||||
defer fileutils.RemoveDir(types.Path(testdataPath), nil)
|
||||
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
|
||||
db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
|
||||
ConnectionString: "file::memory:?cache=shared",
|
||||
MaxOpenConnections: 100,
|
||||
MaxIdleConnections: 2,
|
||||
ConnMaxLifetimeSeconds: -1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("error opening mediaapi database: %v", err)
|
||||
}
|
||||
db2, err2 := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
|
||||
ConnectionString: "file::memory:",
|
||||
MaxOpenConnections: 100,
|
||||
MaxIdleConnections: 2,
|
||||
ConnMaxLifetimeSeconds: -1,
|
||||
})
|
||||
if err2 != nil {
|
||||
t.Errorf("error opening mediaapi database: %v", err)
|
||||
}
|
||||
|
||||
db3, err3 := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{
|
||||
ConnectionString: "file::memory:?",
|
||||
MaxOpenConnections: 100,
|
||||
MaxIdleConnections: 2,
|
||||
ConnMaxLifetimeSeconds: -1,
|
||||
})
|
||||
if err3 != nil {
|
||||
t.Errorf("error opening mediaapi database: %v", err)
|
||||
}
|
||||
|
||||
activeThumbnailGeneration := &types.ActiveThumbnailGeneration{
|
||||
PathToResult: map[string]*types.ThumbnailGenerationResult{},
|
||||
}
|
||||
rateLimits := &httputil.RateLimits{}
|
||||
device := userapi.Device{
|
||||
ID: "1",
|
||||
UserID: "user",
|
||||
}
|
||||
|
||||
handler := makeUrlPreviewHandler(cfg, nil, rateLimits, db, activeThumbnailGeneration)
|
||||
// this handler is to test filecache
|
||||
handler2 := makeUrlPreviewHandler(cfg, nil, rateLimits, db, activeThumbnailGeneration)
|
||||
// this handler is to test image resize
|
||||
handler3 := makeUrlPreviewHandler(cfg2, nil, rateLimits, db2, activeThumbnailGeneration)
|
||||
|
||||
responseBody := `<html>
|
||||
<head>
|
||||
<title>Title</title>
|
||||
<meta property="og:title" content="test_title"/>
|
||||
<meta property="og:description" content="test_description" ></meta>
|
||||
<meta property="og:image:url" content="/test.png">
|
||||
<meta property="og:type" content="image/jpeg" />
|
||||
<meta property="og:url" content="/image.jpg" />
|
||||
</head>
|
||||
</html>`
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.RequestURI == "/test.png" || r.RequestURI == "/test2.png" || r.RequestURI == "/test3.png" {
|
||||
w.Header().Add("Content-Type", "image/jpeg")
|
||||
http.ServeFile(w, r, "../bimg-96x96-crop.jpg")
|
||||
return
|
||||
}
|
||||
w.Write([]byte(responseBody))
|
||||
}))
|
||||
|
||||
ur, _ := url.Parse("/?url=" + srv.URL)
|
||||
req := &http.Request{
|
||||
Method: "GET",
|
||||
URL: ur,
|
||||
}
|
||||
result := handler(req, &device)
|
||||
assert.Equal(t, result.Code, 200, "Response code mismatch")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
|
||||
assert.Greater(t, result.JSON.(*types.UrlPreview).ImageSize, types.FileSizeBytes(0), "Image size missmatch")
|
||||
|
||||
// Test only image response
|
||||
ur2, _ := url.Parse("/?url=" + srv.URL + "/test.png")
|
||||
result = handler(&http.Request{
|
||||
Method: "GET",
|
||||
URL: ur2,
|
||||
}, &device)
|
||||
assert.Equal(t, result.Code, 200, "Response code mismatch")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
|
||||
assert.Greater(t, result.JSON.(*types.UrlPreview).ImageHeight, int(0), "height missmatch")
|
||||
assert.Greater(t, result.JSON.(*types.UrlPreview).ImageWidth, int(0), "width missmatch")
|
||||
|
||||
srcSize := result.JSON.(*types.UrlPreview).ImageSize
|
||||
srcHeight := result.JSON.(*types.UrlPreview).ImageHeight
|
||||
srcWidth := result.JSON.(*types.UrlPreview).ImageWidth
|
||||
|
||||
// Test image resize
|
||||
ur3, _ := url.Parse("/?url=" + srv.URL + "/test2.png")
|
||||
result = handler3(&http.Request{
|
||||
Method: "GET",
|
||||
URL: ur3,
|
||||
}, &device)
|
||||
assert.Equal(t, result.Code, 200, "Response code mismatch")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
|
||||
assert.Less(t, result.JSON.(*types.UrlPreview).ImageSize, srcSize, "thumbnail file size missmatch")
|
||||
assert.Less(t, result.JSON.(*types.UrlPreview).ImageHeight, srcHeight, "thumbnail height missmatch")
|
||||
assert.Less(t, result.JSON.(*types.UrlPreview).ImageWidth, srcWidth, "thumbnail width missmatch")
|
||||
|
||||
// Test to not image resize if the requested size is large than image itself
|
||||
cfg2.UrlPreviewThumbnailSize = config.ThumbnailSize{
|
||||
Width: 1000,
|
||||
Height: 1000,
|
||||
}
|
||||
handler3 = makeUrlPreviewHandler(cfg2, nil, rateLimits, db3, activeThumbnailGeneration)
|
||||
ur3, _ = url.Parse("/?url=" + srv.URL + "/test3.png")
|
||||
result = handler3(&http.Request{
|
||||
Method: "GET",
|
||||
URL: ur3,
|
||||
}, &device)
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageHeight, srcHeight, "thumbnail file size missmatch")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageWidth, srcWidth, "thumbnail file size missmatch")
|
||||
|
||||
// Test denied addresses
|
||||
|
||||
dns := SetupFakeResolver()
|
||||
defer func(t *testing.T) {
|
||||
t.Helper()
|
||||
err = dns.Close()
|
||||
assert.NoError(t, err)
|
||||
}(t)
|
||||
defer mockdns.UnpatchNet(net.DefaultResolver)
|
||||
|
||||
// this handler is to test allow/deny nets
|
||||
denyNets := []string{"192.168.1.1/24", "172.15.1.0/24"}
|
||||
allowNets := []string{"127.0.0.1/24"}
|
||||
dialer := internal.GetDialer(allowNets, denyNets, time.Duration(5*time.Second))
|
||||
handler4 := makeUrlPreviewHandler(cfg, dialer, rateLimits, db, activeThumbnailGeneration)
|
||||
|
||||
serverUrlParsed, err := url.Parse(srv.URL)
|
||||
assert.NoError(t, err)
|
||||
tests := map[string]int{
|
||||
"http://deny1.example.com/test.png": 500,
|
||||
"http://deny2.example.com/test.png": 500,
|
||||
fmt.Sprintf("http://allow.example.com:%s/test.png", serverUrlParsed.Port()): 200,
|
||||
}
|
||||
for serverUrl, code := range tests {
|
||||
ur4, _ := url.Parse("/?url=" + serverUrl)
|
||||
result = handler4(&http.Request{
|
||||
Method: "GET",
|
||||
URL: ur4,
|
||||
}, &device)
|
||||
assert.Equal(t, result.Code, code, "Deny: Response code mismatch: %s", result.JSON)
|
||||
}
|
||||
|
||||
srv.Close()
|
||||
|
||||
// Test in-memory cache
|
||||
result = handler(req, &device)
|
||||
assert.Equal(t, result.Code, 200, "Response code mismatch")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
|
||||
|
||||
// Test response file cache
|
||||
result = handler2(req, &device)
|
||||
assert.Equal(t, result.Code, 200, "Response code mismatch")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).Title, "test_title")
|
||||
assert.Equal(t, result.JSON.(*types.UrlPreview).ImageUrl[:6], "mxc://", "Image response not found")
|
||||
|
||||
}
|
||||
|
||||
// SetupFakeResolver sets up Fake DNS server to resolve SRV records.
|
||||
func SetupFakeResolver() *mockdns.Server {
|
||||
|
||||
testZone := map[string]mockdns.Zone{
|
||||
"allow.example.com.": {
|
||||
A: []string{"127.0.0.1"},
|
||||
},
|
||||
"deny1.example.com.": {
|
||||
A: []string{"192.168.1.10"},
|
||||
},
|
||||
"deny2.example.com.": {
|
||||
A: []string{"172.15.1.10"},
|
||||
},
|
||||
}
|
||||
|
||||
srv, _ := mockdns.NewServer(testZone, true)
|
||||
srv.PatchNet(net.DefaultResolver)
|
||||
|
||||
return srv
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ package thumbnailer
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"image/draw"
|
||||
|
||||
|
|
@ -34,6 +35,8 @@ import (
|
|||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var ErrThumbnailTooLarge = errors.New("thumbnail is larger than original")
|
||||
|
||||
// GenerateThumbnails generates the configured thumbnail sizes for the source file
|
||||
func GenerateThumbnails(
|
||||
ctx context.Context,
|
||||
|
|
@ -266,3 +269,44 @@ func adjustSize(dst types.Path, img image.Image, w, h int, crop bool, logger *lo
|
|||
|
||||
return out.Bounds().Max.X, out.Bounds().Max.Y, nil
|
||||
}
|
||||
|
||||
func CreateThumbnailFromFile(
|
||||
src types.Path,
|
||||
dst types.Path,
|
||||
config types.ThumbnailSize,
|
||||
logger *log.Entry,
|
||||
) (width int, height int, err error) {
|
||||
img, err := readFile(string(src))
|
||||
if err != nil {
|
||||
logger.WithError(err).WithFields(log.Fields{
|
||||
"src": src,
|
||||
}).Error("Failed to read image")
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Check if request is larger than original
|
||||
if config.Width >= img.Bounds().Dx() && config.Height >= img.Bounds().Dy() {
|
||||
return img.Bounds().Dx(), img.Bounds().Dy(), ErrThumbnailTooLarge
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
width, height, err = adjustSize(dst, img, config.Width, config.Height, config.ResizeMethod == types.Crop, logger)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
logger.WithFields(log.Fields{
|
||||
"ActualWidth": width,
|
||||
"ActualHeight": height,
|
||||
"processTime": time.Since(start),
|
||||
}).Info("Generated thumbnail")
|
||||
|
||||
return width, height, nil
|
||||
}
|
||||
|
||||
func GetImageSize(src string) (width int, height int, err error) {
|
||||
img, err := readFile(src)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return img.Bounds().Dx(), img.Bounds().Dy(), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,6 +92,40 @@ type ActiveThumbnailGeneration struct {
|
|||
PathToResult map[string]*ThumbnailGenerationResult
|
||||
}
|
||||
|
||||
type UrlPreviewCache struct {
|
||||
sync.Mutex
|
||||
Records map[string]*UrlPreviewCacheRecord
|
||||
}
|
||||
|
||||
type UrlPreviewCacheRecord struct {
|
||||
Created int64
|
||||
Preview *UrlPreview
|
||||
Error error
|
||||
}
|
||||
|
||||
type UrlPreview struct {
|
||||
ImageSize FileSizeBytes `json:"matrix:image:size"`
|
||||
Description string `json:"og:description"`
|
||||
ImageUrl string `json:"og:image"`
|
||||
ImageType ContentType `json:"og:image:type"`
|
||||
ImageHeight int `json:"og:image:height"`
|
||||
ImageWidth int `json:"og:image:width"`
|
||||
Title string `json:"og:title"`
|
||||
Type string `json:"og:type"`
|
||||
Url string `json:"og:url"`
|
||||
}
|
||||
|
||||
type UrlPreviewResult struct {
|
||||
Cond *sync.Cond
|
||||
Preview *UrlPreview
|
||||
Error error
|
||||
}
|
||||
|
||||
type ActiveUrlPreviewRequests struct {
|
||||
sync.Mutex
|
||||
Url map[string]*UrlPreviewResult
|
||||
}
|
||||
|
||||
// Crop indicates we should crop the thumbnail on resize
|
||||
const Crop = "crop"
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,17 @@ type MediaAPI struct {
|
|||
|
||||
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
|
||||
ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
|
||||
|
||||
// Deny list of urls
|
||||
UrlPreviewDenylist []string `yaml:"url_preview_denylist"`
|
||||
|
||||
// The time in seconds to cache URL previews for
|
||||
UrlPreviewCacheTime int `yaml:"url_preview_cache_time"`
|
||||
|
||||
// The timeout in milliseconds for fetching URL previews
|
||||
UrlPreviewTimeout int `yaml:"url_preview_timeout"`
|
||||
|
||||
UrlPreviewThumbnailSize ThumbnailSize `yaml:"url_preview_thumbnail_size"`
|
||||
}
|
||||
|
||||
// DefaultMaxFileSizeBytes defines the default file size allowed in transfers
|
||||
|
|
@ -38,6 +49,9 @@ var DefaultMaxFileSizeBytes = FileSizeBytes(10485760)
|
|||
func (c *MediaAPI) Defaults(opts DefaultOpts) {
|
||||
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
|
||||
c.MaxThumbnailGenerators = 10
|
||||
c.UrlPreviewCacheTime = 10
|
||||
c.UrlPreviewTimeout = 10000
|
||||
|
||||
if opts.Generate {
|
||||
c.ThumbnailSizes = []ThumbnailSize{
|
||||
{
|
||||
|
|
@ -76,4 +90,11 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors) {
|
|||
if c.Matrix.DatabaseOptions.ConnectionString == "" {
|
||||
checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
|
||||
}
|
||||
|
||||
// If MaxFileSizeBytes overflows int64, default to DefaultMaxFileSizeBytes
|
||||
if c.MaxFileSizeBytes+1 <= 0 {
|
||||
c.MaxFileSizeBytes = DefaultMaxFileSizeBytes
|
||||
fmt.Printf("Configured MediaApi.MaxFileSizeBytes overflows int64, defaulting to %d bytes", DefaultMaxFileSizeBytes)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -314,3 +314,18 @@ func Test_SigningIdentityFor(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_MediaAPIConfigVerify(t *testing.T) {
|
||||
config := &MediaAPI{
|
||||
Matrix: &Global{DatabaseOptions: DatabaseOptions{}},
|
||||
Database: DatabaseOptions{},
|
||||
MaxFileSizeBytes: FileSizeBytes(^int64(0)),
|
||||
}
|
||||
|
||||
configErrs := &ConfigErrors{}
|
||||
|
||||
config.Verify(configErrs)
|
||||
if config.MaxFileSizeBytes != DefaultMaxFileSizeBytes {
|
||||
t.Errorf("config.MediaAPI.MaxFileSizeBytes got = %v, want %v", config.MaxFileSizeBytes, DefaultMaxFileSizeBytes)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue