Update and version Meteor documentation

assets/stylesheets/pages/_meteor.scss

@@ -1,15 +1,18 @@
 ._meteor {
   @extend %simple;
 
-  .note, .warning { @extend %note; }
+  .note, .warning, .subtitle-page { @extend %note; }
+  .subtitle-page { @extend %note-blue; }
   .warning { @extend %note-red; }
 
   dl.args { margin-left: 1rem; }
+  dt > code { @extend %label; }
 
+  .api-heading { overflow: hidden; }
+  .api-heading > code { font-weight: bold; }
   .locus, .src-code { float: right; }
-  .locus, .type, .src-code {
-    margin-left: .5em;
-    font-size: .9em;
-  }
+  .locus, .type, .src-code { margin-left: .5em; }
+  h2 .subtext-api { margin-top: .25rem; }
+  .locus, .subtext-api, .subtext-api > code { font-size: .75rem; }
   .locus, .type { color: $textColorLight; }
 }

lib/docs/core/scraper.rb

@@ -3,7 +3,7 @@ require 'set'
 module Docs
   class Scraper < Doc
     class << self
-      attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters, :stubs
+      attr_accessor :base_url, :root_path, :initial_paths, :initial_urls, :options, :html_filters, :text_filters, :stubs
 
       def inherited(subclass)
         super
@@ -16,6 +16,7 @@ module Docs
         subclass.base_url = base_url
         subclass.root_path = root_path
         subclass.initial_paths = initial_paths.dup
+        subclass.initial_urls = initial_urls.dup
         subclass.options = options.deep_dup
         subclass.html_filters = html_filters.inheritable_copy
         subclass.text_filters = text_filters.inheritable_copy
@@ -35,6 +36,7 @@ module Docs
     include Instrumentable
 
     self.initial_paths = []
+    self.initial_urls = []
     self.options = {}
     self.stubs = {}
 
@@ -103,7 +105,7 @@ module Docs
     end
 
     def initial_urls
-      @initial_urls ||= [root_url.to_s].concat(initial_paths.map(&method(:url_for))).freeze
+      @initial_urls ||= [root_url.to_s].concat(self.class.initial_urls).concat(initial_paths.map(&method(:url_for))).freeze
     end
 
     def pipeline

lib/docs/core/scrapers/url_scraper.rb

@@ -33,7 +33,11 @@ module Docs
         raise "Error status code (#{response.code}): #{response.url}"
       end
 
-      response.success? && response.html? && base_url.contains?(response.effective_url)
+      response.success? && response.html? && process_url?(response.effective_url)
+    end
+
+    def process_url?(url)
+      base_url.contains?(url)
     end
 
     def load_capybara_selenium

lib/docs/filters/meteor/clean_html.rb

@@ -2,44 +2,45 @@ module Docs
   class Meteor
     class CleanHtmlFilter < Filter
       def call
-        root_page? ? root : other
+        @doc = at_css('.content-wrapper')
 
-        css('pre span').each do |node|
-          node.before(node.children).remove
-        end
-
-        doc
-      end
+        css('.page-actions', '.anchor').remove
 
-      def root
-        @doc = at_css('#introduction').parent
-
-        css('.github-ribbon', '#introduction').remove
-
-        css('.selflink', 'b > em').each do |node|
+        css('.header-content', '.document-formatting', 'h2 > a', '.api', '.api-body', 'div.desc').each do |node|
           node.before(node.children).remove
         end
 
-        css('pre').each do |node|
-          node['data-language'] = node.at_css('code')['class'].include?('html') ? 'html' : 'js'
-          node.content = node.content
+        css('.anchor-offset').each do |node|
+          node.parent['id'] = node['id']
+          node.remove
         end
 
-        css('a.src-code').each do |node|
-          node.content = 'Source'
+        css('.api-heading').each do |node|
+          heading = node.at_css('h2, h3')
+          name = heading.name
+          node['id'] = heading['id']
+          heading.replace "<code>#{heading.content.strip}</code>"
+          node.name = name
         end
-      end
 
-      def other
-        @doc = at_css('#content')
-
-        css('.edit-discuss-links', '.bottom-nav', '.edit-link').remove
+        css('div.code', 'span.code', '.args .name').each do |node|
+          node.name = 'code'
+          node.remove_attribute('class')
+        end
 
         css('figure.highlight').each do |node|
-          node.inner_html = node.at_css('.code pre').inner_html.gsub('<br>', "\n")
+          node.inner_html = node.at_css('.code pre').inner_html.gsub('</div><div', "</div>\n<div").gsub('<br>', "\n")
+          node.content = node.content
           node['data-language'] = node['class'].split.last
           node.name = 'pre'
         end
+
+        css('pre.prettyprint').each do |node|
+          node['data-language'] = node['class'].include?('html') ? 'html' : 'js'
+          node.content = node.content
+        end
+
+        doc
       end
     end
   end

lib/docs/filters/meteor/entries.rb

@@ -2,32 +2,31 @@ module Docs
   class Meteor
     class EntriesFilter < Docs::EntriesFilter
       def get_name
-        at_css('#content > h1').content
+        at_css('.item-toc.current').content
       end
 
       def get_type
-        if (node = at_css('#sidebar .current')) && (node = node.ancestors('.menu-root').first.previous_element)
-          "Guide: #{node.content}"
+        if subpath.start_with?('api')
+          name
         else
-          'Guide'
+          type = at_css('.item-toc.current').ancestors('li').first.at_css('.heading-toc').try(:content) || 'Guide'
+          type.prepend 'Guide: ' if base_url.host == 'guide.meteor.com' && type != 'Guide'
+          type
         end
       end
 
       def additional_entries
-        return [] unless root_page?
-        type = nil
-
-        at_css('.full-api-toc').element_children.each_with_object [] do |node, entries|
-          link = node.at_css('a')
-          next unless link
-
-          target = link['href'].remove('#/full/')
-
-          case node.name
-          when 'h1', 'h2'
-            type = node.content.strip
-          when 'h3', 'h4'
-            entries << [node.content, target, type]
+        if slug == 'commandline'
+          css('h2[id]').map do |node|
+            [node.content, node['id']]
+          end
+        else
+          css('.title-api[id]').map do |node|
+            name = node.content.strip
+            name.sub! %r{\(.+\)}, '()'
+            name.remove! 'new '
+            name = '{{> Template.dynamic }}' if name.include?('Template.dynamic')
+            [name, node['id']]
           end
         end
       end

lib/docs/scrapers/meteor.rb

@@ -1,9 +1,11 @@
 module Docs
   class Meteor < UrlScraper
+    class << self
+      attr_accessor :guide_url
+    end
+
     self.type = 'meteor'
-    self.release = '1.3.2'
-    self.base_url = 'https://guide.meteor.com/v1.3/'
-    self.initial_paths = %w(guide)
+    self.root_path = 'index.html'
     self.links = {
       home: 'https://www.meteor.com/',
       code: 'https://github.com/meteor/meteor/'
@@ -11,26 +13,52 @@ module Docs
 
     html_filters.push 'meteor/entries', 'meteor/clean_html'
 
-    options[:skip_links] = ->(filter) { filter.root_page? }
+    options[:skip_patterns] = [/\Av\d/]
+    options[:skip] = %w(
+      CONTRIBUTING.html
+      CHANGELOG.html
+      using-packages.html
+      writing-packages.html
+    )
 
     options[:attribution] = <<-HTML
       &copy; 2011&ndash;2016 Meteor Development Group<br>
       Licensed under the MIT License.
     HTML
 
-    stub '' do
-      require 'capybara/dsl'
-      Capybara.current_driver = :selenium
-      Capybara.run_server = false
-      Capybara.app_host = 'https://docs.meteor.com'
-      Capybara.visit('/#/full/')
-      Capybara.find('.body')['innerHTML']
+    version '1.4' do
+      self.release = '1.4.0'
+      self.base_url = 'https://docs.meteor.com/'
+      self.guide_url = 'https://guide.meteor.com/'
+      self.initial_urls = [guide_url]
+    end
+
+    version '1.3' do
+      self.release = '1.3.5'
+      self.base_url = "https://docs.meteor.com/v#{self.release}/"
+      self.guide_url = 'https://guide.meteor.com/v1.3/'
+      self.initial_urls = [guide_url]
     end
 
-    stub 'guide' do
-      request_one(url_for('index.html')).body
+    def guide_url
+      @guide_url ||= URL.parse(self.class.guide_url)
     end
 
-    options[:replace_paths] = { 'index.html' => 'guide' }
+    private
+
+    def process_url?(url)
+      base_url.contains?(url) || guide_url.contains?(url)
+    end
+
+    def process_response(response)
+      original_host = @base_url.host
+      original_path = @base_url.path
+      @base_url.host = response.effective_url.host
+      @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] || '/'
+      super
+    ensure
+      @base_url.host = original_host
+      @base_url.path = original_path
+    end
   end
 end