Improve deployments to not require including all the docs' index.json in git

Gemfile

@@ -6,6 +6,9 @@ gem 'thor'
 gem 'pry', '~> 0.11.0'
 gem 'activesupport', '~> 5.2', require: false
 gem 'yajl-ruby', require: false
+gem 'html-pipeline'
+gem 'typhoeus'
+gem 'nokogiri'
 
 group :app do
   gem 'rack'
@@ -31,9 +34,6 @@ group :development do
 end
 
 group :docs do
-  gem 'typhoeus'
-  gem 'nokogiri'
-  gem 'html-pipeline'
   gem 'image_optim'
   gem 'image_optim_pack', platforms: :ruby
   gem 'progress_bar', require: false

Rakefile

@@ -13,6 +13,11 @@ end
 namespace :assets do
   desc 'Compile all assets'
   task :precompile do
+    Bundler.require
+
+    load 'tasks/docs.thor'
+    DocsCLI.new.prepare_deploy
+
     load 'tasks/assets.thor'
     AssetsCLI.new.compile
   end

lib/tasks/docs.thor

@@ -172,6 +172,43 @@ class DocsCLI < Thor
     handle_doc_not_found_error(error)
   end
 
+  desc 'prepare_deploy', 'Internal task executed before deployment'
+  def prepare_deploy
+    puts 'Docs -- BEGIN'
+
+    require 'open-uri'
+    require 'thread'
+
+    docs = Docs.all_versions
+    time = Time.now.to_i
+    mutex = Mutex.new
+
+    (1..6).map do
+      Thread.new do
+        while doc = docs.shift
+          dir = File.join(Docs.store_path, doc.path)
+          FileUtils.mkpath(dir)
+
+          ['index.json', 'meta.json'].each do |filename|
+            open("https://docs.devdocs.io/#{doc.path}/#{filename}?#{time}") do |file|
+              mutex.synchronize do
+                path = File.join(dir, filename)
+                File.write(path, file.read)
+              end
+            end
+          end
+
+          puts "Docs -- Downloaded #{doc.slug}"
+        end
+      end
+    end.map(&:join)
+
+    puts 'Docs -- Generating manifest...'
+    generate_manifest
+
+    puts 'Docs -- DONE'
+  end
+
   private
 
   def find_docs(names)