Unfloyded/Legamunt
0
0
Fork 0
Code Activity

Improve encoding detection for link cards (#30780)

Browse source
This commit is contained in:
David Roetzel 2024-06-21 14:51:10 +02:00 • committed by GitHub
parent 4e6db5e284
commit 2cab1c7b09
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 36 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

15
app/lib/link_details_extractor.rb
View file

@ -269,16 +269,21 @@ class LinkDetailsExtractor
end
def document
@document ||= Nokogiri::HTML(@html, nil, encoding)
@document ||= detect_encoding_and_parse_document
end
def encoding
@encoding ||= begin
guess = detector.detect(@html, @html_charset)
guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
def detect_encoding_and_parse_document
[detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
document = Nokogiri::HTML(@html, nil, encoding)
return document if document.to_s.valid_encoding?
end
end
def detect_encoding
guess = detector.detect(@html, @html_charset)
guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
end
def detector
@detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
detector.strip_tags = true