Commit 6b833015 authored by Eugen Rochko's avatar Eugen Rochko

Move friend-finding to Sidekiq, display progress using React

Fix #1
parent e37ee660
......@@ -49,3 +49,4 @@ end
gem 'mini_racer', platforms: :ruby
gem 'webpacker', '~> 3.5'
gem 'active_model_serializers', '~> 0.10'
......@@ -50,6 +50,11 @@ GEM
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
active_model_serializers (0.10.7)
actionpack (>= 4.1, < 6)
activemodel (>= 4.1, < 6)
case_transform (>= 0.2)
jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
activejob (5.2.0)
activesupport (= 5.2.0)
globalid (>= 0.3.6)
......@@ -82,6 +87,8 @@ GEM
msgpack (~> 1.0)
buftok (0.2.0)
builder (3.2.3)
case_transform (0.2)
activesupport
chronic_duration (0.10.6)
numerizer (~> 0.1.1)
coderay (1.1.2)
......@@ -142,6 +149,7 @@ GEM
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
jsonapi-renderer (0.2.0)
jwt (1.5.6)
libv8 (6.3.292.48.1)
listen (3.0.8)
......@@ -336,6 +344,7 @@ PLATFORMS
ruby
DEPENDENCIES
active_model_serializers (~> 0.10)
better_errors (~> 2.4)
binding_of_caller (~> 0.7)
bootsnap
......
......@@ -609,3 +609,31 @@ h4 {
border-radius: 10px;
overflow: hidden;
}
.progress-bar {
width: 100%;
border-radius: 4px;
background: $darkest;
position: relative;
text-align: center;
& > div {
border-radius: 4px;
&:first-child {
position: absolute;
top: 0;
left: 0;
height: 100%;
background: $success;
}
&:last-child {
position: relative;
z-index: 2;
width: 100%;
padding: 10px 0;
color: #fff;
}
}
}
......@@ -5,16 +5,23 @@ require 'twitter'
class FriendsController < ApplicationController
before_action :authenticate_user!
MAX_INSTANCES = 20
MIN_INSTANCES = 4
def index
session[:job_id] = FindFriendsWorker.perform_async(current_user.id) unless job_id.present?
session[:job_id] = FindFriendsWorker.perform_async(current_user.id) unless job_exists?
end
def results
render json: Oj.dump(friends)
render json: friends, each_serializer: UserSerializer
end
def domains
render json: friends_domains
end
def status
render json: Oj.dump(Sidekiq::Status::get_all(job_id))
render json: Sidekiq::Status::get_all(job_id)
end
private
......@@ -23,15 +30,58 @@ class FriendsController < ApplicationController
session[:job_id]
end
def job_exists?
job_id.present? && Sidekiq::Status::get_all(job_id).key?('status')
end
def friends
return unless Sidekiq::Status::complete?(job_id)
User.where(id: Authorization.where(provider: :twitter, uid: twitter_friend_ids).map(&:user_id))
.includes(:twitter, :mastodon)
.reject { |user| user.mastodon.nil? }
data_map = Rails.cache.fetch("#{current_user.id}/friends") { [] }.map { |d| [d.first, d] }.to_h
return [] if data_map.empty?
User.where(id: data_map.keys).includes(:mastodon, :twitter).map do |user|
user.relative_account_id = data_map[user.id][1]
user.following = data_map[user.id][2]
user
end
end
def friends_domains
data = Rails.cache.fetch("#{current_user.id}/friends")
return default_domains.sample(MIN_INSTANCES) if data.empty?
Authorization.where(provider: 'mastodon', user_id: data.map(&:first))
.map(&:uid)
.map { |uid| uid.split('@').last }
.inject(Hash.new(0)) { |h, k| h[k] += 1; h }
.sort_by { |k, v| -v }
.take(MAX_INSTANCES)
.map { |k, _| fetch_instance_info(k) }
.compact
end
def default_domains
%w(
octodon.social
mastodon.art
niu.moe
todon.nl
soc.ialis.me
scifi.fyi
hostux.social
mstdn.maud.io
mastodon.sdf.org
x0r.be
toot.cafe
)
end
def twitter_friend_ids
Rails.cache.read("#{current_user.id}/twitter-friends")
def fetch_instance_info(host)
Rails.cache.fetch("instance:#{host}", expires_in: 1.week) { Oj.load(HTTP.get("https://#{host}/api/v1/instance").to_s, mode: :strict) }
rescue HTTP::Error, OpenSSL::SSL::SSLError, Oj::ParseError
nil
end
end
# frozen_string_literal: true
class HelloWorldController < ApplicationController
layout "hello_world"
def index
@hello_world_props = { name: "Stranger" }
end
end
import { FINDER_PROGRESS, FINDER_RESULTS, FINDER_DOMAINS } from '../constants/finderConstants';
import axios from 'axios';
export const fetchProgress = () => dispatch => axios.get('/friends/status.json').then(({ data }) => {
dispatch({
type: FINDER_PROGRESS,
data,
});
if (data.status === 'complete') {
dispatch(fetchResults());
dispatch(fetchDomains());
} else if (data.status === 'failed') {
console.error('failed');
} else {
setTimeout(() => dispatch(fetchProgress()), 1000);
}
});
export const fetchResults = () => dispatch => axios.get('/friends/results.json').then(({ data }) => {
dispatch({
type: FINDER_RESULTS,
data,
});
});
export const fetchDomains = () => dispatch => axios.get('/friends/domains.json').then(({ data }) => {
dispatch({
type: FINDER_DOMAINS,
data,
});
});
import PropTypes from 'prop-types';
import React from 'react';
import { Motion, StaggeredMotion, spring, presets } from 'react-motion';
import { FormattedMessage, FormattedNumber } from 'react-intl';
export default class HelloWorld extends React.Component {
export default class HelloWorld extends React.PureComponent {
static propTypes = {
name: PropTypes.string.isRequired, // this is passed from the Rails view
status: PropTypes.string.isRequired,
total: PropTypes.number.isRequired,
at: PropTypes.number.isRequired,
results: PropTypes.array.isRequired,
domains: PropTypes.array.isRequired,
inProgress: PropTypes.bool.isRequired,
fetchProgress: PropTypes.func.isRequired,
fetchResults: PropTypes.func.isRequired,
mastodonIsConnected: PropTypes.bool.isRequired,
};
/**
* @param props - Comes from your rails view.
*/
constructor(props) {
super(props);
// How to set initial state in ES6 class syntax
// https://reactjs.org/docs/state-and-lifecycle.html#adding-local-state-to-a-class
this.state = { name: this.props.name };
componentDidMount () {
const { fetchProgress } = this.props;
fetchProgress();
}
updateName = (name) => {
this.setState({ name });
};
render () {
const { status, inProgress, at, total, results, domains, mastodonIsConnected } = this.props;
if (inProgress) {
const pct = total > 0 ? (at / total).toFixed(2) * 100 : 10;
const label = total > 0 ? <span>{at} / {total}</span> : 'Preparing';
return (
<div>
<div className='page-heading'>
<h3>
Searching for your friends...
<small>Please wait while your Twitter friends are being fetched</small>
</h3>
</div>
<div className='progress-bar'>
<Motion defaultStyle={{ x: 0 }} style={{ x: spring(pct, presets.gentle) }}>
{value => <div style={{ width: `${value.x}%` }} />}
</Motion>
<div>{label}</div>
</div>
</div>
);
}
render() {
return (
<div>
<h3>
Hello, {this.state.name}!
</h3>
<hr />
<form >
<label htmlFor="name">
Say hello to:
</label>
<input
id="name"
type="text"
value={this.state.name}
onChange={(e) => this.updateName(e.target.value)}
/>
</form>
<div className={`page-heading ${!mastodonIsConnected ? 'bottomless' : ''}`}>
<h3>
Your friends
<small>Here are your Twitter friends who are on Mastodon:</small>
</h3>
</div>
{!mastodonIsConnected && <div className='connect-prompt'>
For your friends to find you as well, you still need to <a target='_blank' href='/users/auth/mastodon'>login via Mastodon</a>
</div>}
<StaggeredMotion defaultStyles={results.map(_ => ({ height: 0 }))} styles={prevInterpolatedStyles => prevInterpolatedStyles.map((_, i) => {
return i == 0
? { height: spring(80, presets.gentle) }
: { height: spring(prevInterpolatedStyles[i - 1].height, presets.gentle) };
})}>
{interpolatingStyles => (
<div className='grid'>
{interpolatingStyles.map((style, i) => (
<a target='_blank' href={results[i].mastodon_url} style={{ height: style.height }} key={results[i].mastodon_username} className='user-card' title={`@${results[i].twitter_username} on Twitter`}>
<div className='avatar'><img src={results[i].avatar_url} /></div>
{results[i].following && <div className='following-indicator'>
<i className='fa fa-check' />
</div>}
<div className='name'>
<span className='display-name'>{results[i].display_name}</span>
<span className='username'>@{results[i].mastodon_username}</span>
</div>
</a>
))}
</div>
)}
</StaggeredMotion>
<div className='page-heading'>
<h3>
Your friends' instances
<small>Here are the servers your friends are using:</small>
</h3>
</div>
<div className='grid'>
{domains.map(domain => (
<a target='_blank' className='instance-card' href={`https://${domain.uri}/about`} key={domain.uri} style={{ backgroundImage: `url(${domain.thumbnail})` }}>
<div className='info'>
<span className='title'>{domain.title}</span>
<span className='uri'>{domain.uri}</span>
{domain.stats && <span className='users'> (<FormattedMessage id='num_users' defaultMessage='{formatted_count} {count, plural, one {person} other {people}}' values={{ count: domain.stats.user_count, formatted_count: <FormattedNumber value={domain.stats.user_count} /> }} />)</span>}
</div>
</a>
))}
</div>
</div>
);
}
......
/* eslint-disable import/prefer-default-export */
export const FINDER_PROGRESS = 'FINDER_PROGRESS';
export const FINDER_RESULTS = 'FINDER_RESULTS';
export const FINDER_DOMAINS = 'FINDER_DOMAINS';
// Simple example of a React "smart" component
import { connect } from 'react-redux';
import HelloWorld from '../components/HelloWorld';
import * as actions from '../actions/finderActionCreators';
// Which part of the Redux global state does our component want to receive as props?
const mapStateToProps = (state) => ({ ...state.finder, mastodonIsConnected: state.user.mastodonIsConnected });
// Don't forget to actually use connect!
// Note that we don't export HelloWorld, but the redux "connected" version of it.
// See https://github.com/reactjs/react-redux/blob/master/docs/api.md#examples
export default connect(mapStateToProps, actions)(HelloWorld);
import { combineReducers } from 'redux';
import { FINDER_PROGRESS, FINDER_RESULTS, FINDER_DOMAINS } from '../constants/finderConstants';
const initialState = {
total: 0,
at: 0,
status: null,
inProgress: true,
results: [],
domains: [],
};
const finder = (state = initialState, action) => {
switch (action.type) {
case FINDER_PROGRESS:
return { ...state, status: action.data.status, total: action.data.total, at: action.data.at };
case FINDER_RESULTS:
return { ...state, inProgress: false, results: action.data };
case FINDER_DOMAINS:
return { ...state, domains: action.data };
default:
return state;
}
};
const user = (state = '', action) => {
return state;
};
const finderReducer = combineReducers({ finder, user });
export default finderReducer;
import React from 'react';
import { Provider } from 'react-redux';
import { IntlProvider, addLocaleData } from 'react-intl';
import en from 'react-intl/locale-data/en';
import configureStore from '../store/helloWorldStore';
import HelloWorldContainer from '../containers/HelloWorldContainer';
addLocaleData([...en]);
// See documentation for https://github.com/reactjs/react-redux.
// This is how you get props from the Rails view into the redux store.
// This code here binds your smart component to the redux store.
const HelloWorldApp = (props) => (
<IntlProvider locale='en'>
<Provider store={configureStore(props)}>
<HelloWorldContainer />
</Provider>
</IntlProvider>
);
export default HelloWorldApp;
import { createStore, applyMiddleware, compose } from 'redux';
import finderReducer from '../reducers/finderReducer';
import thunk from 'redux-thunk';
const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ || compose;
const configureStore = (railsProps) => (
createStore(finderReducer, railsProps, composeEnhancers(applyMiddleware(thunk)))
);
export default configureStore;
import ReactOnRails from 'react-on-rails';
import HelloWorld from '../bundles/HelloWorld/components/HelloWorld';
import HelloWorldApp from '../bundles/HelloWorld/startup/HelloWorldApp';
// This is how react_on_rails can see the HelloWorld in the browser.
ReactOnRails.register({
HelloWorld,
HelloWorldApp,
});
// Run this example by adding <%= javascript_pack_tag 'hello_react' %> to the head of your layout file,
// like app/views/layouts/application.html.erb. All it does is render <div>Hello React</div> at the bottom
// of the page.
import React from 'react'
import ReactDOM from 'react-dom'
import PropTypes from 'prop-types'
const Hello = props => (
<div>Hello {props.name}!</div>
)
Hello.defaultProps = {
name: 'David'
}
Hello.propTypes = {
name: PropTypes.string
}
document.addEventListener('DOMContentLoaded', () => {
ReactDOM.render(
<Hello name="React" />,
document.body.appendChild(document.createElement('div')),
)
})
# frozen_string_literal: true
class UserSerializer < ActiveModel::Serializer
attributes :mastodon_url, :mastodon_username, :twitter_username,
:display_name, :avatar_url, :following
def mastodon_url
object.mastodon.info['url'] || object.mastodon.profile_url
end
def mastodon_username
object.mastodon.uid
end
def twitter_username
object.twitter.display_name
end
def display_name
object.mastodon.info['display_name'].presence ||
object.mastodon.info['username'] ||
object.mastodon.display_name.presence ||
object.mastodon.uid.split('@').first
end
def avatar_url
object.mastodon.info['avatar'] unless object.mastodon.info['avatar'].blank?
end
end
.grid
- friends.each do |user|
= link_to user.mastodon.info['url'] || user.mastodon.profile_url, class: 'user-card', title: "@#{user.twitter.display_name} on Twitter" do
.avatar
= image_tag user.mastodon.info['avatar'] unless user.mastodon.info['avatar'].blank?
- if user.following
.following-indicator= fa_icon('check')
.name
%span.display-name= user.mastodon.info['display_name'].presence || user.mastodon.info['username'] || user.mastodon.display_name
%span.username= "@#{user.mastodon.uid}"
.grid
- instances.each do |instance_info|
= link_to "https://#{instance_info['uri']}/about", class: 'instance-card', style: "background-image: url(#{instance_info['thumbnail']})" do
.info
%span.title= instance_info['title']
%span.uri= instance_info['uri']
- if instance_info['stats'].is_a?(Hash)
%span.users
= surround '(', ')' do
= number_with_delimiter instance_info['stats']['user_count']
= 'person'.pluralize(instance_info['stats']['user_count'])
.page-heading{ class: current_user.mastodon.nil? ? 'bottomless' : '' }
%h3
Your friends
%small Here are your Twitter friends who are on Mastodon:
- if current_user.mastodon.nil?
.connect-prompt
For your friends to find you as well, you still need to
= link_to 'login via Mastodon', user_mastodon_omniauth_authorize_path
%pre= Sidekiq::Status::get_all(session[:job_id]).inspect
= react_component("HelloWorld", props: { name: "Stranger" }, prerender: false)
= react_component("HelloWorldApp", props: { user: { mastodonIsConnected: !current_user.mastodon.nil? } }, prerender: false)
......@@ -5,7 +5,8 @@ class FindFriendsWorker
include Sidekiq::Status::Worker
def perform(user_id)
client = User.find(user_id).twitter_client
current_user = User.find(user_id)
client = current_user.twitter_client
all_friend_ids = []
twitter_user = client.user
......@@ -15,12 +16,47 @@ class FindFriendsWorker
client.friend_ids.each do |friend_id|
all_friend_ids << friend_id
at all_friend_ids.size
sleep 1
end
rescue Twitter::Error::TooManyRequests => error
sleep error.rate_limit.reset_in + 1
retry
end
Rails.cache.write("#{user_id}/twitter-friends", all_friend_ids, expires_in: 45.minutes)
users = User.where(id: Authorization.where(provider: :twitter, uid: all_friend_ids).map(&:user_id))
.includes(:twitter, :mastodon)
.reject { |user| user.mastodon.nil? }
unless current_user.mastodon.nil?
users.each do |user|
begin
user.relative_account_id = Rails.cache.fetch("#{current_user.id}/#{current_user.mastodon.domain}/#{user.mastodon.uid}", expires_in: 1.week) do
account, _ = current_user.mastodon_client.perform_request(:get, '/api/v1/accounts/search', q: user.mastodon.uid, resolve: 'true', limit: 1)
next if account.nil?
account['id']
end
rescue Mastodon::Error, HTTP::Error, OpenSSL::SSL::SSLError
next
end
end
set_relationships!(current_user, users)
end
Rails.cache.write("#{current_user.id}/friends", users.map { |u| [u.id, u.relative_account_id, u.following] })
end
private
def set_relationships!(current_user, users)
account_map = users.map { |user| [user.relative_account_id, user] }.to_h
account_ids = users.collect { |user| user.relative_account_id }.compact
param_str = account_ids.map { |id| "id[]=#{id}" }.join('&')
current_user.mastodon_client.perform_request(:get, "/api/v1/accounts/relationships?#{param_str}").each do |relationship|
account_map[relationship['id']].following = relationship['following']
end
rescue Mastodon::Error, HTTP::Error, OpenSSL::SSL::SSLError
nil
end
end
Sidekiq.configure_client do |config|
Sidekiq::Status.configure_client_middleware config
Sidekiq::Status.configure_client_middleware config, expiration: 1.day
end
Sidekiq.configure_server do |config|
Sidekiq::Status.configure_server_middleware config
Sidekiq::Status.configure_client_middleware config
Sidekiq::Status.configure_server_middleware config, expiration: 1.day
Sidekiq::Status.configure_client_middleware config, expiration: 1.day
end
# frozen_string_literal: true
Rails.application.routes.draw do
get 'hello_world', to: 'hello_world#index'
devise_for :users, controllers: { omniauth_callbacks: 'users/omniauth_callbacks' }
devise_scope :user do
......@@ -8,6 +9,12 @@ Rails.application.routes.draw do
end
resources :friends, only: :index do
collection do
get :status
get :results
get :domains
end
member do
get :follow
end
......
......@@ -38,7 +38,7 @@ development:
host: localhost
port: 3035
public: localhost:3035
hmr: false
hmr: rue
# Inline should be set to true if using HMR
inline: true
overlay: true
......
{
"dependencies": {
"@rails/webpacker": "3.5",
"axios": "^0.18.0",
"babel-preset-react": "^6.24.1",
"prop-types": "^15.6.1",
"react": "^16.4.1",
"react-dom": "^16.4.1",
"react-on-rails": "11.0.7"
"react-intl": "^2.4.0",
"react-motion": "^0.5.2",
"react-on-rails": "11.0.7",
"react-redux": "^5.0.7",
"redux": "^4.0.0",
"redux-thunk": "^2.3.0"
},
"devDependencies": {
"webpack-dev-server": "2.11.2"
......
......@@ -292,6 +292,13 @@ aws4@^1.2.1:
version "1.7.0"
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.7.0.tgz#d4d0e9b9dbfca77bf08eeb0a8a471550fe39e289"
axios@^0.18.0:
version "0.18.0"
resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102"
dependencies:
follow-redirects "^1.3.0"
is-buffer "^1.1.5"
babel-code-frame@^6.26.0:
version "6.26.0"
resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b"
......@@ -2284,7 +2291,7 @@ flush-write-stream@^1.0.0:
inherits "^2.0.1"
readable-stream "^2.0.4"
follow-redirects@^1.0.0:
follow-redirects@^1.0.0, follow-redirects@^1.3.0:
version "1.5.0"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.0.tgz#234f49cf770b7f35b40e790f636ceba0c3a0ab77"
dependencies:
......@@ -2628,6 +2635,10 @@ hoek@2.x.x:
version "2.16.3"
resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"
hoist-non-react-statics@^2.5.0:
version "2.5.4"
resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-2.5.4.tgz#fc3b1ac05d2ae3abedec84eba846511b0d4fcc4f"