mattermost/server/channels/app/opengraph.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.

package app

import (
	"html"
	"io"
	"net/url"
	"time"

	"github.com/dyatlov/go-opengraph/opengraph"
	ogImage "github.com/dyatlov/go-opengraph/opengraph/types/image"
	"github.com/pkg/errors"
	"golang.org/x/net/html/charset"

	"github.com/mattermost/mattermost/server/public/model"
	"github.com/mattermost/mattermost/server/public/shared/mlog"
	"github.com/mattermost/mattermost/server/v8/channels/app/oembed"
)

const (
	MaxOpenGraphResponseSize   = 1024 * 1024 * 50
	openGraphMetadataCacheSize = 10000
)

func (a *App) GetOpenGraphMetadata(requestURL string) ([]byte, error) {
	var ogJSONGeneric []byte
	err := a.Srv().openGraphDataCache.Get(requestURL, &ogJSONGeneric)
	if err == nil {
		return ogJSONGeneric, nil
	}

	res, err := a.HTTPService().MakeClient(false).Get(requestURL)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	graph := a.parseOpenGraphMetadata(requestURL, res.Body, res.Header.Get("Content-Type"))

	ogJSON, err := graph.ToJSON()
	if err != nil {
		return nil, err
	}
	err = a.Srv().openGraphDataCache.SetWithExpiry(requestURL, ogJSON, 1*time.Hour)
	if err != nil {
		return nil, err
	}

	return ogJSON, nil
}

func (a *App) parseOpenGraphMetadata(requestURL string, body io.Reader, contentType string) *opengraph.OpenGraph {
	og := opengraph.NewOpenGraph()
	body = forceHTMLEncodingToUTF8(io.LimitReader(body, MaxOpenGraphResponseSize), contentType)

	if err := og.ProcessHTML(body); err != nil {
		mlog.Warn("parseOpenGraphMetadata processing failed", mlog.String("requestURL", requestURL), mlog.Err(err))
	}

	makeOpenGraphURLsAbsolute(og, requestURL)

	openGraphDecodeHTMLEntities(og)

	og = filterSVGImagesFromOpenGraph(og)

	// If image proxy enabled modify open graph data to feed though proxy
	if toProxyURL := a.ImageProxyAdder(); toProxyURL != nil {
		og = openGraphDataWithProxyAddedToImageURLs(og, toProxyURL)
	}

	// The URL should be the link the user provided in their message, not a redirected one.
	if og.URL != "" {
		og.URL = requestURL
	}

	return og
}

func forceHTMLEncodingToUTF8(body io.Reader, contentType string) io.Reader {
	r, err := charset.NewReader(body, contentType)
	if err != nil {
		mlog.Warn("forceHTMLEncodingToUTF8 failed to convert", mlog.String("contentType", contentType), mlog.Err(err))
		return body
	}
	return r
}

func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) {
	parsedRequestURL, err := url.Parse(requestURL)
	if err != nil {
		mlog.Warn("makeOpenGraphURLsAbsolute failed to parse url", mlog.String("requestURL", requestURL), mlog.Err(err))
		return
	}

	makeURLAbsolute := func(resultURL string) string {
		if resultURL == "" {
			return resultURL
		}

		parsedResultURL, err := url.Parse(resultURL)
		if err != nil {
			mlog.Warn("makeOpenGraphURLsAbsolute failed to parse result", mlog.String("requestURL", requestURL), mlog.Err(err))
			return resultURL
		}

		if parsedResultURL.IsAbs() {
			return resultURL
		}

		return parsedRequestURL.ResolveReference(parsedResultURL).String()
	}

	og.URL = makeURLAbsolute(og.URL)

	for _, image := range og.Images {
		image.URL = makeURLAbsolute(image.URL)
		image.SecureURL = makeURLAbsolute(image.SecureURL)
	}

	for _, audio := range og.Audios {
		audio.URL = makeURLAbsolute(audio.URL)
		audio.SecureURL = makeURLAbsolute(audio.SecureURL)
	}

	for _, video := range og.Videos {
		video.URL = makeURLAbsolute(video.URL)
		video.SecureURL = makeURLAbsolute(video.SecureURL)
	}
}

func openGraphDataWithProxyAddedToImageURLs(ogdata *opengraph.OpenGraph, toProxyURL func(string) string) *opengraph.OpenGraph {
	for _, image := range ogdata.Images {
		var url string
		if image.SecureURL != "" {
			url = image.SecureURL
		} else {
			url = image.URL
		}

		image.URL = ""
		image.SecureURL = toProxyURL(url)
	}

	return ogdata
}

// filterSVGImagesFromOpenGraph removes SVG images from OpenGraph metadata.
func filterSVGImagesFromOpenGraph(og *opengraph.OpenGraph) *opengraph.OpenGraph {
	if og == nil || len(og.Images) == 0 {
		return og
	}

	og.Images = model.FilterSVGImages(og.Images)
	return og
}

func openGraphDecodeHTMLEntities(og *opengraph.OpenGraph) {
	og.Title = html.UnescapeString(og.Title)
	og.Description = html.UnescapeString(og.Description)
}

func (a *App) parseOpenGraphFromOEmbed(requestURL string, body io.Reader) (*opengraph.OpenGraph, error) {
	oEmbedResponse, err := oembed.ResponseFromJSON(io.LimitReader(body, MaxOpenGraphResponseSize))
	if err != nil {
		return nil, errors.Wrap(err, "parseOpenGraphFromOEmbed: Unable to parse oEmbed response")
	}

	og := &opengraph.OpenGraph{
		Type:  "opengraph",
		Title: oEmbedResponse.Title,
		URL:   requestURL,
	}

	if oEmbedResponse.ThumbnailURL != "" {
		og.Images = append(og.Images, &ogImage.Image{
			Type:   "image",
			URL:    oEmbedResponse.ThumbnailURL,
			Width:  uint64(oEmbedResponse.ThumbnailWidth),
			Height: uint64(oEmbedResponse.ThumbnailHeight),
		})
	}

	og = filterSVGImagesFromOpenGraph(og)

	if toProxyURL := a.ImageProxyAdder(); toProxyURL != nil {
		og = openGraphDataWithProxyAddedToImageURLs(og, toProxyURL)
	}

	return og, nil
}