Add sidekiq

Gemfile

@@ -25,8 +25,10 @@ gem 'httplog', '~> 0.99'
 gem 'hiredis', '~> 0.6'
 gem 'redis', '~> 3.3', require: ['redis', 'redis/connection/hiredis']
 gem 'sidekiq', '~> 5.0'
+gem 'sidekiq-status'
 gem 'redis-rails', '~> 5.0'
 gem 'oj'
+gem 'react_on_rails', '11.0.0'
 
 group :development, :test do
   gem 'pry-rails', '~> 0.3'

Gemfile.lock

@@ -78,6 +78,8 @@ GEM
       msgpack (~> 1.0)
     buftok (0.2.0)
     builder (3.2.3)
+    chronic_duration (0.10.6)
+      numerizer (~> 0.1.1)
     coderay (1.1.2)
     colorize (0.8.1)
     concurrent-ruby (1.0.5)
@@ -107,7 +109,7 @@ GEM
       railties (>= 3.2, < 5.2)
     globalid (0.4.0)
       activesupport (>= 4.2.0)
-    hamlit (2.8.4)
+    hamlit (2.8.8)
       temple (>= 0.8.0)
       thor
       tilt
@@ -116,7 +118,7 @@ GEM
       activesupport (>= 4.0.1)
       hamlit (>= 1.2.0)
       railties (>= 4.0.1)
-    hashie (3.5.6)
+    hashie (3.5.7)
     hiredis (0.6.1)
     http (3.0.0)
       addressable (~> 2.3)
@@ -165,7 +167,8 @@ GEM
     nio4r (2.1.0)
     nokogiri (1.8.1)
       mini_portile2 (~> 2.3.0)
-    oauth (0.5.3)
+    numerizer (0.1.1)
+    oauth (0.5.4)
     oauth2 (1.4.0)
       faraday (>= 0.8, < 0.13)
       jwt (~> 1.0)
@@ -173,7 +176,7 @@ GEM
       multi_xml (~> 0.5)
       rack (>= 1.2, < 3)
     oj (3.3.10)
-    omniauth (1.6.1)
+    omniauth (1.8.1)
       hashie (>= 3.4.6, < 3.6.0)
       rack (>= 1.6.2, < 3)
     omniauth-mastodon (0.9.3)
@@ -197,7 +200,7 @@ GEM
       pry (>= 0.10.4)
     public_suffix (3.0.1)
     puma (3.11.0)
-    rack (2.0.3)
+    rack (2.0.5)
     rack-protection (2.0.0)
       rack
     rack-test (0.6.3)
@@ -230,10 +233,18 @@ GEM
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
+    rainbow (2.2.2)
+      rake
     rake (12.3.0)
     rb-fsevent (0.10.2)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
+    react_on_rails (11.0.0)
+      addressable
+      connection_pool
+      execjs (~> 2.5)
+      rails (>= 3.2)
+      rainbow (~> 2.2)
     redis (3.3.5)
     redis-actionpack (5.0.2)
       actionpack (>= 4.0, < 6)
@@ -270,6 +281,9 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (>= 3.3.4, < 5)
+    sidekiq-status (1.0.1)
+      chronic_duration
+      sidekiq (>= 3.0)
     simple_oauth (0.3.1)
     spring (2.0.2)
       activesupport (>= 4.2)
@@ -327,10 +341,12 @@ DEPENDENCIES
   puma (~> 3.10)
   rails (~> 5.1.0)
   rails_12factor
+  react_on_rails (= 11.0.0)
   redis (~> 3.3)
   redis-rails (~> 5.0)
   sass-rails (~> 5.0)
   sidekiq (~> 5.0)
+  sidekiq-status
   spring
   spring-watcher-listen (~> 2.0.0)
   twitter!

app/controllers/friends_controller.rb

@@ -4,110 +4,34 @@ require 'twitter'
 
 class FriendsController < ApplicationController
   before_action :authenticate_user!
-  before_action :set_page
-  before_action :set_friends
-  before_action :set_top_instances
-  before_action :set_next_page
-  before_action :paginate_friends
-  before_action :set_relationships, if: -> { current_user&.mastodon }
 
-  rescue_from Twitter::Error do |e|
-    redirect_to root_path, alert: "Twitter error: #{e}"
+  def index
+    session[:job_id] = FindFriendsWorker.perform_async(current_user.id) unless job_id.present?
   end
 
-  PER_PAGE_FRIENDS = 20
-  MAX_INSTANCES    = 20
-  MIN_INSTANCES    = 4
-
-  def index; end
-
-  private
-
-  def set_page
-    @page = (params['page'] || 1).to_i
+  def results
+    render json: Oj.dump(friends)
   end
 
-  def set_next_page
-    @next_page = @friends.size > (@page * PER_PAGE_FRIENDS) ? @page + 1 : nil
+  def status
+    render json: Oj.dump(Sidekiq::Status::get_all(job_id))
   end
 
-  def set_friends
-    @friends = User.where(id: Authorization.where(provider: :twitter, uid: twitter_friend_ids).map(&:user_id))
-                   .includes(:twitter, :mastodon)
-                   .reject { |user| user.mastodon.nil? }
-  end
-
-  def set_top_instances
-    @top_instances = friends_domains.map { |k, _| fetch_instance_info(k) }.compact
-  end
+  private
 
-  def paginate_friends
-    @friends = @friends.slice([(@page - 1) * PER_PAGE_FRIENDS, @friends.size].min, PER_PAGE_FRIENDS)
-                       .map { |user| fetch_account_id(user) }
+  def job_id
+    session[:job_id]
   end
 
-  def friends_domains
-    return default_domains.sample(MIN_INSTANCES) if @friends.empty?
-
-    @friends.collect { |user| user&.mastodon&.uid }
-            .compact
-            .map { |uid| uid.split('@').last }
-            .inject(Hash.new(0)) { |h, k| h[k] += 1; h }
-            .sort_by { |k, v| -v }
-            .take(MAX_INSTANCES)
-  end
+  def friends
+    return unless Sidekiq::Status::complete?(job_id)
 
-  def default_domains
-    %w(
-      octodon.social
-      mastodon.art
-      niu.moe
-      todon.nl
-      soc.ialis.me
-      scifi.fyi
-      hostux.social
-      mstdn.maud.io
-      mastodon.sdf.org
-      x0r.be
-      toot.cafe
-    )
+    User.where(id: Authorization.where(provider: :twitter, uid: twitter_friend_ids).map(&:user_id))
+        .includes(:twitter, :mastodon)
+        .reject { |user| user.mastodon.nil? }
   end
 
   def twitter_friend_ids
-    Rails.cache.fetch("#{current_user.id}/twitter-friends", expires_in: 15.minutes) { current_user.twitter_client.friend_ids.to_a }
-  end
-
-  def fetch_instance_info(host)
-    Rails.cache.fetch("instance:#{host}", expires_in: 1.week) { Oj.load(HTTP.get("https://#{host}/api/v1/instance").to_s, mode: :strict) }
-  rescue HTTP::Error, OpenSSL::SSL::SSLError, Oj::ParseError
-    nil
-  end
-
-  def fetch_account_id(user)
-    user.tap do |user|
-      next if current_user.mastodon.nil?
-
-      begin
-        user.relative_account_id = Rails.cache.fetch("#{current_user.id}/#{current_user.mastodon.domain}/#{user.mastodon.uid}", expires_in: 1.week) do
-          account, _ = current_user.mastodon_client.perform_request(:get, '/api/v1/accounts/search', q: user.mastodon.uid, resolve: 'true', limit: 1)
-          next if account.nil?
-          account['id']
-        end
-      rescue Mastodon::Error, HTTP::Error, OpenSSL::SSL::SSLError
-        next
-      end
-    end
-  end
-
-  def set_relationships
-    account_map = @friends.map { |user| [user.relative_account_id, user] }.to_h
-    account_ids = @friends.collect { |user| user.relative_account_id }.compact
-    param_str   = account_ids.map { |id| "id[]=#{id}" }.join('&')
-
-    current_user.mastodon_client.perform_request(:get, "/api/v1/accounts/relationships?#{param_str}").each do |relationship|
-      account_map[relationship['id']].following = relationship['following']
-    end
-  rescue Mastodon::Error, HTTP::Error, OpenSSL::SSL::SSLError
-    nil
+    Rails.cache.read("#{current_user.id}/twitter-friends")
   end
 end

app/views/friends/index.html.haml

@@ -8,30 +8,4 @@
     For your friends to find you as well, you still need to
     = link_to 'login via Mastodon', user_mastodon_omniauth_authorize_path
 
-- if @friends.empty?
-  %p.empty-message Well, this is unfortunate. Looks like none of your Twitter friends are on Mastodon yet. Or maybe they are, but haven't signed in to this bridge service.
-
-  - if current_user.mastodon.nil?
-    %p.empty-message Here are some instances you could choose to make your account on:
-  - else
-    %p.empty-message Here are some instances you could recommend to your friends:
-- else
-  = render 'friends_grid', friends: @friends
-
-  - if @next_page || @page > 1
-    .pagination
-      - if @page > 1
-        = link_to friends_path(page: @page - 1) do
-          = fa_icon('chevron-left fw')
-          Previous
-      - if @next_page
-        = link_to friends_path(page: @next_page) do
-          Next
-          = fa_icon('chevron-right fw')
-
-  .page-heading
-    %h3
-      Your friends' instances
-      %small Here are the instances your friends are using:
-
-= render 'instances_grid', instances: @top_instances
+%pre= Sidekiq::Status::get_all(session[:job_id]).inspect

app/workers/find_friends_worker.rb

+# frozen_string_literal: true
+
+class FindFriendsWorker
+  include Sidekiq::Worker
+  include Sidekiq::Status::Worker
+
+  def perform(user_id)
+    client         = User.find(user_id).twitter_client
+    all_friend_ids = []
+    twitter_user   = client.user
+
+    total twitter_user.friends_count
+
+    begin
+      client.friend_ids.each do |friend_id|
+        all_friend_ids << friend_id
+        at all_friend_ids.size
+      end
+    rescue Twitter::Error::TooManyRequests => error
+      sleep error.rate_limit.reset_in + 1
+      retry
+    end
+
+    Rails.cache.write("#{user_id}/twitter-friends", all_friend_ids, expires_in: 45.minutes)
+  end
+end

config/initializers/sidekiq.rb

+Sidekiq.configure_client do |config|
+  Sidekiq::Status.configure_client_middleware config
+end
+
+Sidekiq.configure_server do |config|
+  Sidekiq::Status.configure_server_middleware config
+  Sidekiq::Status.configure_client_middleware config
+end