CloudToys/REI-Fixed
1
0
Fork 0
Code

Track trending tags (#7638)

Browse Source 
* Track trending tags

- Half-life of 1 day
- Historical usage in daily buckets (last 7 days stored)
- GET /api/v1/trends

Fix #271

* Add trends to web UI

* Don't render compose form on search route, adjust search results header

* Disqualify tag from trends if it's in disallowed hashtags setting

* Count distinct accounts using tag, ignore silenced accounts
This commit is contained in:
Eugen Rochko 2018-05-27 21:45:30 +02:00 committed by GitHub
parent 63c7b91572
commit 9bd23dc4e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 310 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
app/controllers/api/v1/trends_controller.rb Normal file
View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
class Api::V1::TrendsController < Api::BaseController
before_action :set_tags
respond_to :json
def index
render json: @tags, each_serializer: REST::TagSerializer
end
private
def set_tags
@tags = TrendingTags.get(limit_param(10))
end
end

32
app/javascript/mastodon/actions/trends.js Normal file
View File

@ -0,0 +1,32 @@
import api from '../api';
export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST';
export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS';
export const TRENDS_FETCH_FAIL = 'TRENDS_FETCH_FAIL';
export const fetchTrends = () => (dispatch, getState) => {
dispatch(fetchTrendsRequest());
api(getState)
.get('/api/v1/trends')
.then(({ data }) => dispatch(fetchTrendsSuccess(data)))
.catch(err => dispatch(fetchTrendsFail(err)));
};
export const fetchTrendsRequest = () => ({
type: TRENDS_FETCH_REQUEST,
skipLoading: true,
});
export const fetchTrendsSuccess = trends => ({
type: TRENDS_FETCH_SUCCESS,
trends,
skipLoading: true,
});
export const fetchTrendsFail = error => ({
type: TRENDS_FETCH_FAIL,
error,
skipLoading: true,
skipAlert: true,
});

59
app/javascript/mastodon/features/compose/components/search_results.js
View File

@ -1,23 +1,75 @@
import React from 'react';
import PropTypes from 'prop-types';
import ImmutablePropTypes from 'react-immutable-proptypes';
import { FormattedMessage } from 'react-intl';
import { FormattedMessage, FormattedNumber } from 'react-intl';
import AccountContainer from '../../../containers/account_container';
import StatusContainer from '../../../containers/status_container';
import { Link } from 'react-router-dom';
import ImmutablePureComponent from 'react-immutable-pure-component';
import { Sparklines, SparklinesCurve } from 'react-sparklines';
const shortNumberFormat = number => {
if (number < 1000) {
return <FormattedNumber value={number} />;
} else {
return <React.Fragment><FormattedNumber value={number / 1000} maximumFractionDigits={1} />K</React.Fragment>;
}
};
export default class SearchResults extends ImmutablePureComponent {
static propTypes = {
results: ImmutablePropTypes.map.isRequired,
trends: ImmutablePropTypes.list,
fetchTrends: PropTypes.func.isRequired,
};
componentDidMount () {
const { fetchTrends } = this.props;
fetchTrends();
}
render () {
const { results } = this.props;
const { results, trends } = this.props;
let accounts, statuses, hashtags;
let count = 0;
if (results.isEmpty()) {
return (
<div className='search-results'>
<div className='trends'>
<div className='trends__header'>
<i className='fa fa-fire fa-fw' />
<FormattedMessage id='trends.header' defaultMessage='Trending now' />
</div>
{trends && trends.map(hashtag => (
<div className='trends__item' key={hashtag.get('name')}>
<div className='trends__item__name'>
<Link to={`/timelines/tag/${hashtag.get('name')}`}>
#<span>{hashtag.get('name')}</span>
</Link>
<FormattedMessage id='trends.count_by_accounts' defaultMessage='{count} {rawCount, plural, one {person} other {people}} talking' values={{ rawCount: hashtag.getIn(['history', 0, 'accounts']), count: <strong>{shortNumberFormat(hashtag.getIn(['history', 0, 'accounts']))}</strong> }} />
</div>
<div className='trends__item__current'>
{shortNumberFormat(hashtag.getIn(['history', 0, 'uses']))}
</div>
<div className='trends__item__sparkline'>
<Sparklines width={50} height={28} data={hashtag.get('history').reverse().map(day => day.get('uses')).toArray()}>
<SparklinesCurve style={{ fill: 'none' }} />
</Sparklines>
</div>
</div>
))}
</div>
</div>
);
}
if (results.get('accounts') && results.get('accounts').size > 0) {
count += results.get('accounts').size;
accounts = (
@ -48,7 +100,7 @@ export default class SearchResults extends ImmutablePureComponent {
{results.get('hashtags').map(hashtag => (
<Link key={hashtag} className='search-results__hashtag' to={`/timelines/tag/${hashtag}`}>
#{hashtag}
{hashtag}
</Link>
))}
</div>
@ -58,6 +110,7 @@ export default class SearchResults extends ImmutablePureComponent {
return (
<div className='search-results'>
<div className='search-results__header'>
<i className='fa fa-search fa-fw' />
<FormattedMessage id='search_results.total' defaultMessage='{count, number} {count, plural, one {result} other {results}}' values={{ count }} />
</div>

8
app/javascript/mastodon/features/compose/containers/search_results_container.js
View File

@ -1,8 +1,14 @@
import { connect } from 'react-redux';
import SearchResults from '../components/search_results';
import { fetchTrends } from '../../../actions/trends';
const mapStateToProps = state => ({
results: state.getIn(['search', 'results']),
trends: state.get('trends'),
});
export default connect(mapStateToProps)(SearchResults);
const mapDispatchToProps = dispatch => ({
fetchTrends: () => dispatch(fetchTrends()),
});
export default connect(mapStateToProps, mapDispatchToProps)(SearchResults);

4
app/javascript/mastodon/features/compose/index.js
View File

@ -101,7 +101,7 @@ export default class Compose extends React.PureComponent {
{(multiColumn || isSearchPage) && <SearchContainer /> }
<div className='drawer__pager'>
<div className='drawer__inner' onFocus={this.onFocus}>
{!isSearchPage && <div className='drawer__inner' onFocus={this.onFocus}>
<NavigationContainer onClose={this.onBlur} />
<ComposeFormContainer />
{multiColumn && (
@ -109,7 +109,7 @@ export default class Compose extends React.PureComponent {
<img alt='' draggable='false' src={elephantUIPlane} />
</div>
)}
</div>
</div>}
<Motion defaultStyle={{ x: isSearchPage ? 0 : -100 }} style={{ x: spring(showSearch || isSearchPage ? 0 : -100, { stiffness: 210, damping: 20 }) }}>
{({ x }) => (

2
app/javascript/mastodon/reducers/index.js
View File

@ -26,6 +26,7 @@ import height_cache from './height_cache';
import custom_emojis from './custom_emojis';
import lists from './lists';
import listEditor from './list_editor';
import trends from './trends';
const reducers = {
dropdown_menu,
@ -55,6 +56,7 @@ const reducers = {
custom_emojis,
lists,
listEditor,
trends,
};
export default combineReducers(reducers);

13
app/javascript/mastodon/reducers/trends.js Normal file
View File

@ -0,0 +1,13 @@
import { TRENDS_FETCH_SUCCESS } from '../actions/trends';
import { fromJS } from 'immutable';
const initialState = null;
export default function trendsReducer(state = initialState, action) {
switch(action.type) {
case TRENDS_FETCH_SUCCESS:
return fromJS(action.trends);
default:
return state;
}
};

83
app/javascript/styles/mastodon/components.scss
View File

@ -3334,9 +3334,15 @@ a.status-card {
color: $dark-text-color;
background: lighten($ui-base-color, 2%);
border-bottom: 1px solid darken($ui-base-color, 4%);
padding: 15px 10px;
font-size: 14px;
padding: 15px;
font-weight: 500;
font-size: 16px;
cursor: default;
.fa {
display: inline-block;
margin-right: 5px;
}
}
.search-results__section {
@ -5209,3 +5215,76 @@ noscript {
background: $ui-base-color;
}
}
.trends {
&__header {
color: $dark-text-color;
background: lighten($ui-base-color, 2%);
border-bottom: 1px solid darken($ui-base-color, 4%);
font-weight: 500;
padding: 15px;
font-size: 16px;
cursor: default;
.fa {
display: inline-block;
margin-right: 5px;
}
}
&__item {
display: flex;
align-items: center;
padding: 15px;
border-bottom: 1px solid lighten($ui-base-color, 8%);
&:last-child {
border-bottom: 0;
}
&__name {
flex: 1 1 auto;
color: $dark-text-color;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
strong {
font-weight: 500;
}
a {
color: $darker-text-color;
text-decoration: none;
font-size: 14px;
font-weight: 500;
display: block;
&:hover,
&:focus,
&:active {
span {
text-decoration: underline;
}
}
}
}
&__current {
width: 100px;
font-size: 24px;
line-height: 36px;
font-weight: 500;
text-align: center;
color: $secondary-text-color;
}
&__sparkline {
width: 50px;
path {
stroke: lighten($highlight-text-color, 6%) !important;
}
}
}
}

16
app/models/tag.rb
View File

@ -21,6 +21,22 @@ class Tag < ApplicationRecord
name
end
def history
days = []
7.times do |i|
day = i.days.ago.beginning_of_day.to_i
days << {
day: day.to_s,
uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
}
end
days
end
class << self
def search_for(term, limit = 5)
pattern = sanitize_sql_like(term.strip) + '%'

61
app/models/trending_tags.rb Normal file
View File

@ -0,0 +1,61 @@
# frozen_string_literal: true
class TrendingTags
KEY = 'trending_tags'
HALF_LIFE = 1.day.to_i
MAX_ITEMS = 500
EXPIRE_HISTORY_AFTER = 7.days.seconds
class << self
def record_use!(tag, account, at_time = Time.now.utc)
return if disallowed_hashtags.include?(tag.name) || account.silenced?
increment_vote!(tag.id, at_time)
increment_historical_use!(tag.id, at_time)
increment_unique_use!(tag.id, account.id, at_time)
end
def get(limit)
tag_ids = redis.zrevrange(KEY, 0, limit).map(&:to_i)
tags = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h
tag_ids.map { |tag_id| tags[tag_id] }.compact
end
private
def increment_vote!(tag_id, at_time)
redis.zincrby(KEY, (2**((at_time.to_i - epoch) / HALF_LIFE)).to_f, tag_id.to_s)
redis.zremrangebyrank(KEY, 0, -MAX_ITEMS) if rand < (2.to_f / MAX_ITEMS)
end
def increment_historical_use!(tag_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
redis.incrby(key, 1)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_unique_use!(tag_id, account_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
redis.pfadd(key, account_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
# The epoch needs to be 2.5 years in the future if the half-life is one day
# While dynamic, it will always be the same within one year
def epoch
@epoch ||= Date.new(Date.current.year + 2.5, 10, 1).to_datetime.to_i
end
def disallowed_hashtags
return @disallowed_hashtags if defined?(@disallowed_hashtags)
@disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
@disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
@disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
end
def redis
Redis.current
end
end
end

11
app/serializers/rest/tag_serializer.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
class REST::TagSerializer < ActiveModel::Serializer
include RoutingHelper
attributes :name, :url, :history
def url
tag_url(object)
end
end

6
app/services/process_hashtags_service.rb
View File

@ -4,8 +4,10 @@ class ProcessHashtagsService < BaseService
def call(status, tags = [])
tags = Extractor.extract_hashtags(status.text) if status.local?
tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag|
status.tags << Tag.where(name: tag).first_or_initialize(name: tag)
tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
tag = Tag.where(name: name).first_or_create(name: name)
status.tags << tag
TrendingTags.record_use!(tag, status.account, status.created_at)
end
end
end

1
config/routes.rb
View File

@ -254,6 +254,7 @@ Rails.application.routes.draw do
resources :mutes, only: [:index]
resources :favourites, only: [:index]
resources :reports, only: [:index, :create]
resources :trends, only: [:index]
namespace :apps do
get :verify_credentials, to: 'credentials#show'

1
package.json
View File

@ -97,6 +97,7 @@
"react-redux-loading-bar": "^2.9.3",
"react-router-dom": "^4.1.1",
"react-router-scroll-4": "^1.0.0-beta.1",
"react-sparklines": "^1.7.0",
"react-swipeable-views": "^0.12.3",
"react-textarea-autosize": "^5.2.1",
"react-toggle": "^4.0.1",

6
yarn.lock
View File

@ -6124,6 +6124,12 @@ react-router@^4.2.0:
prop-types "^15.5.4"
warning "^3.0.0"
react-sparklines@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/react-sparklines/-/react-sparklines-1.7.0.tgz#9b1d97e8c8610095eeb2ad658d2e1fcf91f91a60"
dependencies:
prop-types "^15.5.10"
react-swipeable-views-core@^0.12.11:
version "0.12.11"
resolved "https://registry.yarnpkg.com/react-swipeable-views-core/-/react-swipeable-views-core-0.12.11.tgz#3cf2b4daffbb36f9d69bd19bf5b2d5370b6b2c1b"