Commit d0254b1d authored by Eugen Rochko's avatar Eugen Rochko

Add sidekiq

parent 6e0da2a3
......@@ -25,8 +25,10 @@ gem 'httplog', '~> 0.99'
gem 'hiredis', '~> 0.6'
gem 'redis', '~> 3.3', require: ['redis', 'redis/connection/hiredis']
gem 'sidekiq', '~> 5.0'
gem 'sidekiq-status'
gem 'redis-rails', '~> 5.0'
gem 'oj'
gem 'react_on_rails', '11.0.0'
group :development, :test do
gem 'pry-rails', '~> 0.3'
......
......@@ -78,6 +78,8 @@ GEM
msgpack (~> 1.0)
buftok (0.2.0)
builder (3.2.3)
chronic_duration (0.10.6)
numerizer (~> 0.1.1)
coderay (1.1.2)
colorize (0.8.1)
concurrent-ruby (1.0.5)
......@@ -107,7 +109,7 @@ GEM
railties (>= 3.2, < 5.2)
globalid (0.4.0)
activesupport (>= 4.2.0)
hamlit (2.8.4)
hamlit (2.8.8)
temple (>= 0.8.0)
thor
tilt
......@@ -116,7 +118,7 @@ GEM
activesupport (>= 4.0.1)
hamlit (>= 1.2.0)
railties (>= 4.0.1)
hashie (3.5.6)
hashie (3.5.7)
hiredis (0.6.1)
http (3.0.0)
addressable (~> 2.3)
......@@ -165,7 +167,8 @@ GEM
nio4r (2.1.0)
nokogiri (1.8.1)
mini_portile2 (~> 2.3.0)
oauth (0.5.3)
numerizer (0.1.1)
oauth (0.5.4)
oauth2 (1.4.0)
faraday (>= 0.8, < 0.13)
jwt (~> 1.0)
......@@ -173,7 +176,7 @@ GEM
multi_xml (~> 0.5)
rack (>= 1.2, < 3)
oj (3.3.10)
omniauth (1.6.1)
omniauth (1.8.1)
hashie (>= 3.4.6, < 3.6.0)
rack (>= 1.6.2, < 3)
omniauth-mastodon (0.9.3)
......@@ -197,7 +200,7 @@ GEM
pry (>= 0.10.4)
public_suffix (3.0.1)
puma (3.11.0)
rack (2.0.3)
rack (2.0.5)
rack-protection (2.0.0)
rack
rack-test (0.6.3)
......@@ -230,10 +233,18 @@ GEM
method_source
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
rainbow (2.2.2)
rake
rake (12.3.0)
rb-fsevent (0.10.2)
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
react_on_rails (11.0.0)
addressable
connection_pool
execjs (~> 2.5)
rails (>= 3.2)
rainbow (~> 2.2)
redis (3.3.5)
redis-actionpack (5.0.2)
actionpack (>= 4.0, < 6)
......@@ -270,6 +281,9 @@ GEM
connection_pool (~> 2.2, >= 2.2.0)
rack-protection (>= 1.5.0)
redis (>= 3.3.4, < 5)
sidekiq-status (1.0.1)
chronic_duration
sidekiq (>= 3.0)
simple_oauth (0.3.1)
spring (2.0.2)
activesupport (>= 4.2)
......@@ -327,10 +341,12 @@ DEPENDENCIES
puma (~> 3.10)
rails (~> 5.1.0)
rails_12factor
react_on_rails (= 11.0.0)
redis (~> 3.3)
redis-rails (~> 5.0)
sass-rails (~> 5.0)
sidekiq (~> 5.0)
sidekiq-status
spring
spring-watcher-listen (~> 2.0.0)
twitter!
......
......@@ -4,110 +4,34 @@ require 'twitter'
class FriendsController < ApplicationController
before_action :authenticate_user!
before_action :set_page
before_action :set_friends
before_action :set_top_instances
before_action :set_next_page
before_action :paginate_friends
before_action :set_relationships, if: -> { current_user&.mastodon }
rescue_from Twitter::Error do |e|
redirect_to root_path, alert: "Twitter error: #{e}"
def index
session[:job_id] = FindFriendsWorker.perform_async(current_user.id) unless job_id.present?
end
PER_PAGE_FRIENDS = 20
MAX_INSTANCES = 20
MIN_INSTANCES = 4
def index; end
private
def set_page
@page = (params['page'] || 1).to_i
def results
render json: Oj.dump(friends)
end
def set_next_page
@next_page = @friends.size > (@page * PER_PAGE_FRIENDS) ? @page + 1 : nil
def status
render json: Oj.dump(Sidekiq::Status::get_all(job_id))
end
def set_friends
@friends = User.where(id: Authorization.where(provider: :twitter, uid: twitter_friend_ids).map(&:user_id))
.includes(:twitter, :mastodon)
.reject { |user| user.mastodon.nil? }
end
def set_top_instances
@top_instances = friends_domains.map { |k, _| fetch_instance_info(k) }.compact
end
private
def paginate_friends
@friends = @friends.slice([(@page - 1) * PER_PAGE_FRIENDS, @friends.size].min, PER_PAGE_FRIENDS)
.map { |user| fetch_account_id(user) }
def job_id
session[:job_id]
end
def friends_domains
return default_domains.sample(MIN_INSTANCES) if @friends.empty?
@friends.collect { |user| user&.mastodon&.uid }
.compact
.map { |uid| uid.split('@').last }
.inject(Hash.new(0)) { |h, k| h[k] += 1; h }
.sort_by { |k, v| -v }
.take(MAX_INSTANCES)
end
def friends
return unless Sidekiq::Status::complete?(job_id)
def default_domains
%w(
octodon.social
mastodon.art
niu.moe
todon.nl
soc.ialis.me
scifi.fyi
hostux.social
mstdn.maud.io
mastodon.sdf.org
x0r.be
toot.cafe
)
User.where(id: Authorization.where(provider: :twitter, uid: twitter_friend_ids).map(&:user_id))
.includes(:twitter, :mastodon)
.reject { |user| user.mastodon.nil? }
end
def twitter_friend_ids
Rails.cache.fetch("#{current_user.id}/twitter-friends", expires_in: 15.minutes) { current_user.twitter_client.friend_ids.to_a }
end
def fetch_instance_info(host)
Rails.cache.fetch("instance:#{host}", expires_in: 1.week) { Oj.load(HTTP.get("https://#{host}/api/v1/instance").to_s, mode: :strict) }
rescue HTTP::Error, OpenSSL::SSL::SSLError, Oj::ParseError
nil
end
def fetch_account_id(user)
user.tap do |user|
next if current_user.mastodon.nil?
begin
user.relative_account_id = Rails.cache.fetch("#{current_user.id}/#{current_user.mastodon.domain}/#{user.mastodon.uid}", expires_in: 1.week) do
account, _ = current_user.mastodon_client.perform_request(:get, '/api/v1/accounts/search', q: user.mastodon.uid, resolve: 'true', limit: 1)
next if account.nil?
account['id']
end
rescue Mastodon::Error, HTTP::Error, OpenSSL::SSL::SSLError
next
end
end
end
def set_relationships
account_map = @friends.map { |user| [user.relative_account_id, user] }.to_h
account_ids = @friends.collect { |user| user.relative_account_id }.compact
param_str = account_ids.map { |id| "id[]=#{id}" }.join('&')
current_user.mastodon_client.perform_request(:get, "/api/v1/accounts/relationships?#{param_str}").each do |relationship|
account_map[relationship['id']].following = relationship['following']
end
rescue Mastodon::Error, HTTP::Error, OpenSSL::SSL::SSLError
nil
Rails.cache.read("#{current_user.id}/twitter-friends")
end
end
......@@ -8,30 +8,4 @@
For your friends to find you as well, you still need to
= link_to 'login via Mastodon', user_mastodon_omniauth_authorize_path
- if @friends.empty?
%p.empty-message Well, this is unfortunate. Looks like none of your Twitter friends are on Mastodon yet. Or maybe they are, but haven't signed in to this bridge service.
- if current_user.mastodon.nil?
%p.empty-message Here are some instances you could choose to make your account on:
- else
%p.empty-message Here are some instances you could recommend to your friends:
- else
= render 'friends_grid', friends: @friends
- if @next_page || @page > 1
.pagination
- if @page > 1
= link_to friends_path(page: @page - 1) do
= fa_icon('chevron-left fw')
Previous
- if @next_page
= link_to friends_path(page: @next_page) do
Next
= fa_icon('chevron-right fw')
.page-heading
%h3
Your friends' instances
%small Here are the instances your friends are using:
= render 'instances_grid', instances: @top_instances
%pre= Sidekiq::Status::get_all(session[:job_id]).inspect
# frozen_string_literal: true
class FindFriendsWorker
include Sidekiq::Worker
include Sidekiq::Status::Worker
def perform(user_id)
client = User.find(user_id).twitter_client
all_friend_ids = []
twitter_user = client.user
total twitter_user.friends_count
begin
client.friend_ids.each do |friend_id|
all_friend_ids << friend_id
at all_friend_ids.size
end
rescue Twitter::Error::TooManyRequests => error
sleep error.rate_limit.reset_in + 1
retry
end
Rails.cache.write("#{user_id}/twitter-friends", all_friend_ids, expires_in: 45.minutes)
end
end
Sidekiq.configure_client do |config|
Sidekiq::Status.configure_client_middleware config
end
Sidekiq.configure_server do |config|
Sidekiq::Status.configure_server_middleware config
Sidekiq::Status.configure_client_middleware config
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment