Implement get_latest_version for the remaining scrapers

lib/docs/scrapers/nginx.rb

@@ -25,5 +25,12 @@ module Docs
       &copy; 2011-2018 Nginx, Inc.<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://nginx.org/en/download.html', options) do |doc|
+        table = doc.at_css('#content > table').inner_html
+        block.call table.scan(/nginx-([0-9.]+)</)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/nginx_lua_module.rb

@@ -4,6 +4,9 @@ module Docs
     self.slug = 'nginx_lua_module'
     self.release = '0.10.13'
     self.base_url = "https://github.com/openresty/lua-nginx-module/tree/v#{self.release}/"
+    self.links = {
+      code: 'https://github.com/openresty/lua-nginx-module'
+    }
 
     html_filters.push 'nginx_lua_module/clean_html', 'nginx_lua_module/entries', 'title'
 
@@ -15,5 +18,12 @@ module Docs
       &copy; 2009&ndash;2018 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_tags('openresty', 'lua-nginx-module', options) do |tags|
+        tag = tags.find { |tag| !tag['name'].include?('rc') }
+        block.call tag['name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/nim.rb

@@ -17,5 +17,11 @@ module Docs
       &copy; 2006&ndash;2018 Andreas Rumpf<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://nim-lang.org/docs/overview.html', options) do |doc|
+        block.call doc.at_css('.container > .docinfo > tbody > tr:last-child > td').content.strip
+      end
+    end
   end
 end

lib/docs/scrapers/node.rb

@@ -46,5 +46,11 @@ module Docs
       self.release = '4.9.1'
       self.base_url = 'https://nodejs.org/dist/latest-v4.x/docs/api/'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://nodejs.org/en/', options) do |doc|
+        block.call doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/nokogiri2.rb

@@ -19,5 +19,11 @@ module Docs
       Patrick Mahoney, Yoko Harada, Akinori Musha, John Shahid, Lars Kanis<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('sparklemotion', 'nokogiri', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/npm.rb

@@ -29,5 +29,11 @@ module Docs
       Licensed under the npm License.<br>
       npm is a trademark of npm, Inc.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('npm', 'cli', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/numpy.rb

@@ -49,5 +49,11 @@ module Docs
       self.release = '1.10.4'
       self.base_url = "https://docs.scipy.org/doc/numpy-#{self.release}/reference/"
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('numpy', 'numpy', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/openjdk.rb

@@ -86,5 +86,26 @@ module Docs
     def read_file(path)
       File.read(path).force_encoding('iso-8859-1').encode('utf-8') rescue nil
     end
+
+    def get_latest_version(options, &block)
+      latest_version = 8
+      current_attempt = latest_version
+      attempts = 0
+
+      while attempts < 3
+        current_attempt += 1
+
+        fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", options) do |doc|
+          if doc.at_css('.perror').nil?
+            latest_version = current_attempt
+            attempts = 0
+          else
+            attempts += 1
+          end
+        end
+      end
+
+      block.call latest_version.to_s
+    end
   end
 end

lib/docs/scrapers/opentsdb.rb

@@ -18,5 +18,11 @@ module Docs
       &copy; 2010&ndash;2016 The OpenTSDB Authors<br>
       Licensed under the GNU LGPLv2.1+ and GPLv3+ licenses.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('OpenTSDB', 'opentsdb', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/padrino.rb

@@ -23,5 +23,11 @@ module Docs
     stub 'index2' do
       request_one(url_for('index')).body
     end
+
+    def get_latest_version(options, &block)
+      get_github_tags('padrino', 'padrino-framework', options) do |tags|
+        block.call tags[0]['name']
+      end
+    end
   end
 end

lib/docs/scrapers/pandas.rb

@@ -49,5 +49,12 @@ module Docs
       self.release = '0.18.1'
       self.base_url = "http://pandas.pydata.org/pandas-docs/version/#{self.release}/"
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', options) do |doc|
+        label = doc.at_css('.body > .section > p').content
+        block.call label.scan(/Version: ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/perl.rb

@@ -43,5 +43,11 @@ module Docs
       self.release = '5.20.2'
       self.base_url = "https://perldoc.perl.org/#{self.release}/"
     end
+
+    def get_latest_version(options, &block)
+      fetch('https://perldoc.perl.org/static/perlversion.js', options) do |body|
+        block.call body.scan(/>Perl ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/phalcon.rb

@@ -29,5 +29,11 @@ module Docs
       self.release = '2.0.13'
       self.base_url = 'https://docs.phalconphp.com/en/2.0.0/'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://docs.phalconphp.com/', options) do |doc|
+        block.call doc.at_css('.custom-select__version').content.strip.sub(/Version /, '')
+      end
+    end
   end
 end

lib/docs/scrapers/phaser.rb

@@ -25,5 +25,11 @@ module Docs
       &copy; 2016 Richard Davey, Photon Storm Ltd.<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('photonstorm', 'phaser', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/phoenix.rb

@@ -46,5 +46,11 @@ module Docs
         HTML
       end
     }
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', options) do |doc|
+        block.call doc.at_css('.sidebar-projectVersion').content.strip[1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/php.rb

@@ -66,5 +66,12 @@ module Docs
       &copy; 1997&ndash;2018 The PHP Documentation Group<br>
       Licensed under the Creative Commons Attribution License v3.0 or later.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', options) do |doc|
+        label = doc.at_css('tbody.gen-changelog > tr > td').content
+        block.call label.split(',').last.strip
+      end
+    end
   end
 end

lib/docs/scrapers/phpunit.rb

@@ -37,5 +37,12 @@ module Docs
       self.release = '4.8'
       self.base_url = "https://phpunit.de/manual/#{release}/en/"
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://phpunit.readthedocs.io/', options) do |doc|
+        label = doc.at_css('.rst-current-version').content.strip
+        block.call label.scan(/v: ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/postgresql.rb

@@ -80,5 +80,12 @@ module Docs
 
       html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://www.postgresql.org/docs/current/index.html', options) do |doc|
+        label = doc.at_css('#pgContentWrap h1.title').content
+        block.call label.scan(/([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/pug.rb

@@ -18,6 +18,10 @@ module Docs
       Licensed under the MIT license.
     HTML
 
+    def get_latest_version(options, &block)
+      get_npm_version('pug', options, &block)
+    end
+
     private
 
     def parse(response) # Hook here because Nokogori removes whitespace from textareas

lib/docs/scrapers/puppeteer.rb

@@ -14,5 +14,11 @@ module Docs
       &copy; 2017 Google Inc<br>
       Licensed under the Apache License 2.0.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', options) do |contents|
+        block.call contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/pygame.rb

@@ -17,5 +17,11 @@ module Docs
       &copy; Pygame Developpers.<br>
       Licensed under the GNU LGPL License version 2.1.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('pygame', 'pygame', options) do |release|
+        block.call release['tag_name']
+      end
+    end
   end
 end

lib/docs/scrapers/python.rb

@@ -50,5 +50,11 @@ module Docs
 
       html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://docs.python.org/', options) do |doc|
+        block.call doc.at_css('.version_switcher_placeholder').content
+      end
+    end
   end
 end

lib/docs/scrapers/q.rb

@@ -19,5 +19,9 @@ module Docs
       &copy; 2009&ndash;2017 Kristopher Michael Kowal<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_npm_version('q', options, &block)
+    end
   end
 end

lib/docs/scrapers/qt.rb

@@ -117,5 +117,12 @@ module Docs
       self.release = '5.6'
       self.base_url = 'https://doc.qt.io/qt-5.6/'
     end
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://doc.qt.io/qt-5/index.html', options) do |doc|
+        label = doc.at_css('.mainContent h1.title').content
+        block.call label.sub(/Qt /, '')
+      end
+    end
   end
 end

lib/docs/scrapers/ramda.rb

@@ -15,6 +15,12 @@ module Docs
       &copy; 2013&ndash;2016 Scott Sauyet and Michael Hurley<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://ramdajs.com/docs/', options) do |doc|
+        block.call doc.at_css('.navbar-brand > .version').content[1..-1]
+      end
+    end
   end
 end
 

lib/docs/scrapers/react.rb

@@ -30,5 +30,11 @@ module Docs
       &copy; 2013&ndash;present Facebook Inc.<br>
       Licensed under the Creative Commons Attribution 4.0 International Public License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://reactjs.org/docs/getting-started.html', options) do |doc|
+        block.call doc.at_css('a[href="/versions"]').content.strip[1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/react_native.rb

@@ -30,5 +30,11 @@ module Docs
       &copy; 2015&ndash;2018 Facebook Inc.<br>
       Licensed under the Creative Commons Attribution 4.0 International Public License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', options) do |doc|
+        block.call doc.at_css('header > a > h3').content
+      end
+    end
   end
 end

lib/docs/scrapers/redis.rb

@@ -19,5 +19,12 @@ module Docs
       &copy; 2009&ndash;2018 Salvatore Sanfilippo<br>
       Licensed under the Creative Commons Attribution-ShareAlike License 4.0.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', options) do |body|
+        body = body.lines[1..-1].join
+        block.call body.scan(/Redis ([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/redux.rb

@@ -20,5 +20,9 @@ module Docs
     stub '' do
       request_one('http://redux.js.org/index.html').body
     end
+
+    def get_latest_version(options, &block)
+      get_npm_version('redux', options, &block)
+    end
   end
 end

lib/docs/scrapers/relay.rb

@@ -18,5 +18,11 @@ module Docs
       &copy; 2013&ndash;present Facebook Inc.<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('http://facebook.github.io/relay/en/', options) do |doc|
+        block.call doc.at_css('header > a > h3').content[1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/requirejs.rb

@@ -30,5 +30,9 @@ module Docs
       &copy; jQuery Foundation and other contributors<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_npm_version('requirejs', options, &block)
+    end
   end
 end

lib/docs/scrapers/rethinkdb.rb

@@ -58,6 +58,12 @@ module Docs
       CODE
     end
 
+    def get_latest_version(options, &block)
+      get_latest_github_release('rethinkdb', 'rethinkdb', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
+
     private
 
     def process_response?(response)

lib/docs/scrapers/rust.rb

@@ -39,6 +39,13 @@ module Docs
       Licensed under the Apache License, Version 2.0 or the MIT license, at your option.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://www.rust-lang.org/', options) do |doc|
+        label = doc.at_css('.button-download + p > a').content
+        block.call label.sub(/Version /, '')
+      end
+    end
+
     private
 
     REDIRECT_RGX = /http-equiv="refresh"/i

lib/docs/scrapers/sass.rb

@@ -23,5 +23,11 @@ module Docs
       &copy; 2006&ndash;2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('sass', 'sass', 'VERSION', options) do |contents|
+        block.call contents.strip
+      end
+    end
   end
 end

lib/docs/scrapers/scikit_image.rb

@@ -20,5 +20,11 @@ module Docs
       &copy; 2011 the scikit-image team<br>
       Licensed under the BSD 3-clause License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_tags('scikit-image', 'scikit-image', options) do |tags|
+        block.call tags[0]['name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/scikit_learn.rb

@@ -25,5 +25,11 @@ module Docs
       Licensed under the 3-clause BSD License.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://scikit-learn.org/stable/documentation.html', options) do |doc|
+        label = doc.at_css('.body h1').content
+        block.call label.scan(/([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/sinon.rb

@@ -52,5 +52,11 @@ module Docs
       self.release = '1.17.7'
       self.base_url = "https://sinonjs.org/releases/v#{release}/"
     end
+
+    def get_latest_version(options, &block)
+      fetch('https://sinonjs.org/', options) do |body|
+        block.call body.scan(/\/releases\/v([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/socketio.rb

@@ -20,5 +20,9 @@ module Docs
       &copy; 2014&ndash;2015 Automattic<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_npm_version('socket.io', options, &block)
+    end
   end
 end

lib/docs/scrapers/sqlite.rb

@@ -41,6 +41,12 @@ module Docs
 
     options[:attribution] = 'SQLite is in the Public Domain.'
 
+    def get_latest_version(options, &block)
+      fetch_doc('https://sqlite.org/chronology.html', options) do |doc|
+        block.call doc.at_css('#chrontab > tbody > tr > td:last-child > a').content
+      end
+    end
+
     private
 
     def parse(response)

lib/docs/scrapers/statsmodels.rb

@@ -21,5 +21,10 @@ module Docs
       Licensed under the 3-clause BSD License.
     HTML
 
+    def get_latest_version(options, &block)
+      fetch_doc('http://www.statsmodels.org/stable/', options) do |doc|
+        block.call doc.at_css('.sphinxsidebarwrapper h3 + p > b').content
+      end
+    end
   end
 end

lib/docs/scrapers/symfony.rb

@@ -70,5 +70,11 @@ module Docs
       self.release = '2.7.35'
       self.base_url = "https://api.symfony.com/#{version}/"
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('symfony', 'symfony', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/tcl_tk.rb

@@ -25,5 +25,12 @@ module Docs
     options[:attribution] = <<-HTML
       Licensed under <a href="http://tcl.tk/software/tcltk/license.html">Tcl/Tk terms</a>
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', options) do |doc|
+        label = doc.at_css('h2').content
+        block.call label.scan(/Tk([0-9.]+)/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/tensorflow.rb

@@ -56,6 +56,12 @@ module Docs
         /\Aextend/]
     end
 
+    def get_latest_version(options, &block)
+      get_latest_github_release('tensorflow', 'tensorflow', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
+
     private
 
     def parse(response)

lib/docs/scrapers/terraform.rb

@@ -18,5 +18,11 @@ module Docs
       &copy; 2018 HashiCorp</br>
       Licensed under the MPL 2.0 License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', options) do |contents|
+        block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/twig.rb

@@ -28,5 +28,11 @@ module Docs
       self.release = '1.34.3'
       self.base_url = 'https://twig.symfony.com/doc/1.x/'
     end
+
+    def get_latest_version(options, &block)
+      get_github_tags('twigphp', 'Twig', options) do |tags|
+        block.call tags[0]['name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/typescript.rb

@@ -24,6 +24,12 @@ module Docs
       &copy; Microsoft and other contributors<br>
       Licensed under the Apache License, Version 2.0.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('Microsoft', 'TypeScript', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end
 

lib/docs/scrapers/underscore.rb

@@ -20,5 +20,11 @@ module Docs
       &copy; 2009&ndash;2018 Jeremy Ashkenas, DocumentCloud and Investigative Reporters &amp; Editors<br>
       Licensed under the MIT License.
     HTML
+
+    def get_latest_version(options, &block)
+      fetch_doc('https://underscorejs.org/', options) do |doc|
+        block.call doc.at_css('.version').content[1...-1]
+      end
+    end
   end
 end

lib/docs/scrapers/vagrant.rb

@@ -18,5 +18,11 @@ module Docs
       &copy; 2010&ndash;2018 Mitchell Hashimoto<br>
       Licensed under the MPL 2.0 License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', options) do |contents|
+        block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
+      end
+    end
   end
 end

lib/docs/scrapers/vue.rb

@@ -32,5 +32,11 @@ module Docs
       self.root_path = '/guide/index.html'
       self.initial_paths = %w(/api/index.html)
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('vuejs', 'vue', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/vulkan.rb

@@ -20,5 +20,11 @@ module Docs
       Licensed under the Creative Commons Attribution 4.0 International License.<br>
       Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.
     HTML
+
+    def get_latest_version(options, &block)
+      get_github_tags('KhronosGroup', 'Vulkan-Docs', options) do |tags|
+        block.call tags[0]['name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/webpack.rb

@@ -68,5 +68,9 @@ module Docs
         Licensed under the MIT License.
       HTML
     end
+
+    def get_latest_version(options, &block)
+      get_npm_version('webpack', options, &block)
+    end
   end
 end

lib/docs/scrapers/yarn.rb

@@ -20,5 +20,11 @@ module Docs
       &copy; 2016&ndash;present Yarn Contributors<br>
       Licensed under the BSD License.
     HTML
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('yarnpkg', 'yarn', options) do |release|
+        block.call release['tag_name'][1..-1]
+      end
+    end
   end
 end

lib/docs/scrapers/yii.rb

@@ -34,5 +34,11 @@ module Docs
 
       options[:container] = '.grid_9'
     end
+
+    def get_latest_version(options, &block)
+      get_latest_github_release('yiisoft', 'yii2', options) do |release|
+        block.call release['tag_name']
+      end
+    end
   end
 end

lib/tasks/updates.thor

@@ -58,7 +58,7 @@ class UpdatesCLI < Thor
 
   private
 
-  def check_doc(doc, options)
+  def check_doc(doc, opts)
     # Newer scraper versions always come before older scraper versions
     # Therefore, the first item's release value is the latest scraper version
     #
@@ -71,7 +71,7 @@ class UpdatesCLI < Thor
 
     logger.debug("Checking #{doc.name}")
 
-    instance.get_latest_version(options) do |latest_version|
+    instance.get_latest_version(opts) do |latest_version|
       return {
         name: doc.name,
         scraper_version: scraper_version,