lovely-projects/dendrite
lovely-projects/dendrite
1
0
Fork 0
mirror of https://github.com/element-hq/dendrite.git synced 2025-03-14 14:15:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into msc3861

Browse source
This commit is contained in:
Roman Isaev 2025-02-03 13:28:37 +00:00 • committed by GitHub
commit 9ab9a8da8e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 8 additions and 280 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -51,7 +51,6 @@ require (
golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b
golang.org/x/sync v0.10.0
golang.org/x/term v0.28.0
gopkg.in/h2non/bimg.v1 v1.1.9
gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.5.1
maunium.net/go/mautrix v0.15.1

2
go.sum
View file

@ -488,8 +488,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM=
gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI=

27
mediaapi/README.md
View file

@ -1,27 +0,0 @@
# Media API
This server is responsible for serving `/media` requests as per:
http://matrix.org/docs/spec/client_server/r0.2.0.html#id43
## Scaling libraries
### nfnt/resize (default)
Thumbnailing uses https://github.com/nfnt/resize by default which is a pure golang image scaling library relying on image codecs from the standard library. It is ISC-licensed.
It is multi-threaded and uses Lanczos3 so produces sharp images. Using Lanczos3 all the way makes it slower than some other approaches like bimg. (~845ms in total for pre-generating 32x32-crop, 96x96-crop, 320x240-scale, 640x480-scale and 800x600-scale from a given JPEG image on a given machine.)
See the sample below for image quality with nfnt/resize:
![](nfnt-96x96-crop.jpg)
### bimg (uses libvips C library)
Alternatively one can use `go build -tags bimg` to use bimg from https://github.com/h2non/bimg (MIT-licensed) which uses libvips from https://github.com/jcupitt/libvips (LGPL v2.1+ -licensed). libvips is a C library and must be installed/built separately. See the github page for details. Also note that libvips in turn has dependencies with a selection of FOSS licenses.
bimg and libvips have significantly better performance than nfnt/resize but produce slightly less-sharp images. bimg uses a box filter for downscaling to within about 200% of the target scale and then uses Lanczos3 for the last bit. This is a much faster approach but comes at the expense of sharpness. (~295ms in total for pre-generating 32x32-crop, 96x96-crop, 320x240-scale, 640x480-scale and 800x600-scale from a given JPEG image on a given machine.)
See the sample below for image quality with bimg:
![](bimg-96x96-crop.jpg)

3
mediaapi/mediaapi.go
View file

@ -7,6 +7,8 @@
package mediaapi
import (
"github.com/sirupsen/logrus"
"github.com/element-hq/dendrite/internal/httputil"
"github.com/element-hq/dendrite/internal/sqlutil"
"github.com/element-hq/dendrite/mediaapi/routing"
@ -14,7 +16,6 @@ import (
"github.com/element-hq/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/fclient"
"github.com/sirupsen/logrus"
)
// AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.

3
mediaapi/thumbnailer/thumbnailer.go
View file

@ -14,10 +14,11 @@ import (
"path/filepath"
"sync"
log "github.com/sirupsen/logrus"
"github.com/element-hq/dendrite/mediaapi/storage"
"github.com/element-hq/dendrite/mediaapi/types"
"github.com/element-hq/dendrite/setup/config"
log "github.com/sirupsen/logrus"
)
type thumbnailFitness struct {

241
mediaapi/thumbnailer/thumbnailer_bimg.go
View file

@ -1,241 +0,0 @@
// Copyright 2024 New Vector Ltd.
// Copyright 2017 Vector Creations Ltd
//
// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
// Please see LICENSE files in the repository root for full details.
//go:build bimg
// +build bimg
package thumbnailer
import (
"context"
"os"
"time"
"github.com/element-hq/dendrite/mediaapi/storage"
"github.com/element-hq/dendrite/mediaapi/types"
"github.com/element-hq/dendrite/setup/config"
log "github.com/sirupsen/logrus"
"gopkg.in/h2non/bimg.v1"
)
// GenerateThumbnails generates the configured thumbnail sizes for the source file
func GenerateThumbnails(
ctx context.Context,
src types.Path,
configs []config.ThumbnailSize,
mediaMetadata *types.MediaMetadata,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
maxThumbnailGenerators int,
db storage.Database,
logger *log.Entry,
) (busy bool, errorReturn error) {
buffer, err := bimg.Read(string(src))
if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to read src file")
return false, err
}
img := bimg.NewImage(buffer)
for _, config := range configs {
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy, err = createThumbnail(
ctx, src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration,
maxThumbnailGenerators, db, logger,
)
if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
return false, err
}
if busy {
return true, nil
}
}
return false, nil
}
// GenerateThumbnail generates the configured thumbnail size for the source file
func GenerateThumbnail(
ctx context.Context,
src types.Path,
config types.ThumbnailSize,
mediaMetadata *types.MediaMetadata,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
maxThumbnailGenerators int,
db storage.Database,
logger *log.Entry,
) (busy bool, errorReturn error) {
buffer, err := bimg.Read(string(src))
if err != nil {
logger.WithError(err).WithFields(log.Fields{
"src": src,
}).Error("Failed to read src file")
return false, err
}
img := bimg.NewImage(buffer)
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy, err = createThumbnail(
ctx, src, img, config, mediaMetadata, activeThumbnailGeneration,
maxThumbnailGenerators, db, logger,
)
if err != nil {
logger.WithError(err).WithFields(log.Fields{
"src": src,
}).Error("Failed to generate thumbnails")
return false, err
}
if busy {
return true, nil
}
return false, nil
}
// createThumbnail checks if the thumbnail exists, and if not, generates it
// Thumbnail generation is only done once for each non-existing thumbnail.
func createThumbnail(
ctx context.Context,
src types.Path,
img *bimg.Image,
config types.ThumbnailSize,
mediaMetadata *types.MediaMetadata,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
maxThumbnailGenerators int,
db storage.Database,
logger *log.Entry,
) (busy bool, errorReturn error) {
logger = logger.WithFields(log.Fields{
"Width": config.Width,
"Height": config.Height,
"ResizeMethod": config.ResizeMethod,
})
// Check if request is larger than original
if isLargerThanOriginal(config, img) {
return false, nil
}
dst := GetThumbnailPath(src, config)
// Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration
isActive, busy, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, maxThumbnailGenerators, logger)
if err != nil {
return false, err
}
if busy {
return true, nil
}
if isActive {
// Note: This is an active request that MUST broadcastGeneration to wake up waiting goroutines!
// Note: broadcastGeneration uses mutexes and conditions from activeThumbnailGeneration
defer func() {
// Note: errorReturn is the named return variable so we wrap this in a closure to re-evaluate the arguments at defer-time
if err := recover(); err != nil {
broadcastGeneration(dst, activeThumbnailGeneration, config, err.(error), logger)
panic(err)
}
broadcastGeneration(dst, activeThumbnailGeneration, config, errorReturn, logger)
}()
}
exists, err := isThumbnailExists(ctx, dst, config, mediaMetadata, db, logger)
if err != nil || exists {
return false, err
}
start := time.Now()
width, height, err := resize(dst, img, config.Width, config.Height, config.ResizeMethod == "crop", logger)
if err != nil {
return false, err
}
logger.WithFields(log.Fields{
"ActualWidth": width,
"ActualHeight": height,
"processTime": time.Now().Sub(start),
}).Debugf("Generated thumbnail %q", dst)
stat, err := os.Stat(string(dst))
if err != nil {
return false, err
}
thumbnailMetadata := &types.ThumbnailMetadata{
MediaMetadata: &types.MediaMetadata{
MediaID: mediaMetadata.MediaID,
Origin: mediaMetadata.Origin,
// Note: the code currently always creates a JPEG thumbnail
ContentType: types.ContentType("image/jpeg"),
FileSizeBytes: types.FileSizeBytes(stat.Size()),
},
ThumbnailSize: types.ThumbnailSize{
Width: config.Width,
Height: config.Height,
ResizeMethod: config.ResizeMethod,
},
}
err = db.StoreThumbnail(ctx, thumbnailMetadata)
if err != nil {
logger.WithError(err).WithFields(log.Fields{
"ActualWidth": width,
"ActualHeight": height,
}).Error("Failed to store thumbnail metadata in database.")
return false, err
}
return false, nil
}
func isLargerThanOriginal(config types.ThumbnailSize, img *bimg.Image) bool {
imgSize, err := img.Size()
if err == nil && config.Width >= imgSize.Width && config.Height >= imgSize.Height {
return true
}
return false
}
// resize scales an image to fit within the provided width and height
// If the source aspect ratio is different to the target dimensions, one edge will be smaller than requested
// If crop is set to true, the image will be scaled to fill the width and height with any excess being cropped off
func resize(dst types.Path, inImage *bimg.Image, w, h int, crop bool, logger *log.Entry) (int, int, error) {
inSize, err := inImage.Size()
if err != nil {
return -1, -1, err
}
options := bimg.Options{
Type: bimg.JPEG,
Quality: 85,
}
if crop {
options.Width = w
options.Height = h
options.Crop = true
} else {
inAR := float64(inSize.Width) / float64(inSize.Height)
outAR := float64(w) / float64(h)
if inAR > outAR {
// input has wider AR than requested output so use requested width and calculate height to match input AR
options.Width = w
options.Height = int(float64(w) / inAR)
} else {
// input has narrower AR than requested output so use requested height and calculate width to match input AR
options.Width = int(float64(h) * inAR)
options.Height = h
}
}
newImage, err := inImage.Process(options)
if err != nil {
return -1, -1, err
}
if err = bimg.Write(string(dst), newImage); err != nil {
logger.WithError(err).Error("Failed to resize image")
return -1, -1, err
}
return options.Width, options.Height, nil
}

11
mediaapi/thumbnailer/thumbnailer_nfnt.go
View file

@ -4,9 +4,6 @@
// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
// Please see LICENSE files in the repository root for full details.
//go:build !bimg
// +build !bimg
package thumbnailer
import (
@ -20,18 +17,18 @@ import (
// Imported for png codec
_ "image/png"
"os"
"time"
// Imported for webp codec
_ "golang.org/x/image/webp"
"os"
"time"
"github.com/nfnt/resize"
log "github.com/sirupsen/logrus"
"github.com/element-hq/dendrite/mediaapi/storage"
"github.com/element-hq/dendrite/mediaapi/types"
"github.com/element-hq/dendrite/setup/config"
"github.com/nfnt/resize"
log "github.com/sirupsen/logrus"
)
// GenerateThumbnails generates the configured thumbnail sizes for the source file