Update Angular documentation (4.3.2)

assets/javascripts/app/app.coffee

@@ -119,6 +119,8 @@
     for slug in @settings.getDocs() when not @docs.findBy('slug', slug)
       needsSaving = true
       doc = @disabledDocs.findBy('slug', 'webpack') if slug == 'webpack~2'
+      doc = @disabledDocs.findBy('slug', 'angular') if slug == 'angular~4_typescript'
+      doc = @disabledDocs.findBy('slug', 'angular~2') if slug == 'angular~2_typescript'
       doc ||= @disabledDocs.findBy('slug_without_version', slug)
       if doc
         @disabledDocs.remove(doc)

assets/stylesheets/pages/_angular.scss

@@ -1,55 +1,22 @@
 ._angular {
-  padding-left: 1rem;
+  @extend %simple;
 
-  h1, h2, > h3, .banner, .badges, .breadcrumbs { margin-left: -1rem; }
+  .pre-title { @extend %pre-heading; }
 
-  ._mobile & {
-    padding-left: 0;
-
-    h1, h2, > h3, .banner, .badges, .breadcrumbs { margin-left: 0; }
-  }
-
-  h2 { @extend %block-heading; }
-  > h3 { @extend %block-label, %label-blue; }
-  .code-example > h4, .pre-title { @extend %pre-heading; }
-
-  p > code, dd > code, .status-badge { @extend %label; }
-
-  .l-sub-section, .alert, .banner, .breadcrumbs { @extend %note; }
+  .breadcrumbs { @extend %note; }
   .banner { @extend %note-green; }
+  code.stable { @extend %label-green; }
+  code.experimental { @extend %label-orange; }
+  code.deprecated { @extend %label-red; }
   .alert.is-important { @extend %note-red; }
   .alert.is-helpful, .breadcrumbs { @extend %note-blue; }
-  .breadcrumbs { padding-left: 2em; }
-
-  td > h3, .l-sub-section > h3, .l-sub-section > h4, .alert > h3, .alert > h4, .row-margin > h3 {
-    margin-top: .25rem;
-    font-size: 1em;
-  }
 
-  img {
-    display: block;
-    margin: 1em auto;
-
-    &[align="left"] {
-      float: left;
-      margin: 0 1em 0 0;
-    }
+  .breadcrumbs { padding-left: 2em; }
 
-    &[align="right"] {
-      float: right;
-      margin: 0 0 0 1em;
-    }
-  }
+  img { margin: 1em 0; }
 
   .location-badge {
-    text-align: right;
     font-style: italic;
-  }
-
-  .filetree {
-    white-space: normal;
-    @extend %pre;
-
-    .children { padding-left: 1em; }
+    text-align: right;
   }
 }

assets/stylesheets/pages/_simple.scss

@@ -12,7 +12,7 @@
     h1, h2, h3 { margin-left: 0; }
   }
 
-  p > code, li > code, td > code, blockquote > code { @extend %label; }
+  p > code, li > code, td > code, blockquote > code, dd > code { @extend %label; }
   blockquote { @extend %note; }
   blockquote > h4, blockquote > h5 { margin-top: .25rem; }
 }

lib/app.rb

@@ -352,7 +352,9 @@ class App < Sinatra::Application
     'yii1' => 'yii~1.1',
     'python2' => 'python~2.7',
     'xpath' => 'xslt_xpath',
-    'angular~2.0_typescript' => 'angular~2_typescript',
+    'angular~4_typescript' => 'angular',
+    'angular~2_typescript' => 'angular~2',
+    'angular~2.0_typescript' => 'angular~2',
     'angular~1.5' => 'angularjs~1.5',
     'angular~1.4' => 'angularjs~1.4',
     'angular~1.3' => 'angularjs~1.3',

lib/docs/core/response.rb

@@ -13,7 +13,7 @@ module Docs
     end
 
     def mime_type
-      @mime_type ||= headers['Content-Type'] || 'text/plain'
+      headers['Content-Type'] || 'text/plain'
     end
 
     def html?

lib/docs/filters/angular/clean_html.rb

@@ -2,103 +2,86 @@ module Docs
   class Angular
     class CleanHtmlFilter < Filter
       def call
-        container = at_css('article.docs-content')
-        badges = css('header.hero .badge, header.hero .hero-subtitle').map do |node|
-          node.name = 'span'
-          node['class'] = 'status-badge'
-          node.to_html
-        end.join(' ')
-        badges = %(<div class="badges">#{badges}</div>)
-        container.child.before(at_css('header.hero h1')).before(badges).before(css('header.hero + .banner, header.hero .breadcrumbs'))
-        @doc = container
-
-        title = at_css('h1').content.strip
         if root_page?
+          css('.card-container').remove
           at_css('h1').content = 'Angular Documentation'
-        elsif title == 'Index'
-          at_css('h1').content = result[:entries].first.name
-        elsif title == 'Angular'
-          at_css('h1').content = slug.split('/').last.gsub('-', ' ')
-        elsif at_css('.breadcrumbs') && title != result[:entries].first.name
-          at_css('h1').content = result[:entries].first.name
         end
 
-        css('pre.no-bg-with-indent').each do |node|
-          node.content = '  ' + node.content.gsub("\n", "\n  ")
-        end
-
-        css('.openParens').each do |node|
-          node.parent.name = 'pre'
-          node.parent.content = node.parent.css('code, pre').map(&:content).join("\n")
-        end
-
-        css('button.verbose', 'button.verbose + .l-verbose-section', 'a[id=top]', 'a[href="#top"]', '.sidebar').remove
+        css('br', 'hr', '.material-icons', '.header-link').remove
 
-        css('.c10', '.showcase', '.showcase-content', '.l-main-section', 'div.div', 'div[flex]', 'code-tabs', 'md-card', 'md-card-content', 'div:not([class])', 'footer', '.card-row', '.card-row-container', 'figure', 'blockquote', 'exported', 'defined', 'div.ng-scope', '.code-example header', 'section.desc', '.row', '.dart-api-entry-main', '.main-content', 'section.summary', 'span.signature').each do |node|
+        css('.content', 'article', '.api-header', 'section', '.instance-member').each do |node|
           node.before(node.children).remove
         end
 
-        css('span.badges').each do |node|
-          node.name = 'div'
+        css('label', 'h2 > em', 'h3 > em').each do |node|
+          node.name = 'code'
         end
 
-        css('pre[language]').each do |node|
-          node['data-language'] = node['language'].sub(/\Ats/, 'typescript').strip
-          node['data-language'] = 'html' if node.content.start_with?('<')
+        css('h1 + code').each do |node|
+          node.before('<p></p>')
+          while node.next_element.name == 'code'
+            node.previous_element << ' '
+            node.previous_element << node.next_element
+          end
+          node.previous_element.prepend_child(node)
         end
 
-        css('pre.prettyprint').each do |node|
-          node.content = node.content.strip
-          node['data-language'] = 'dart' if node['class'].include?('dart')
-          node['data-language'] = 'html' if node.content.start_with?('<')
+        css('td h3', '.l-sub-section > h3', '.alert h3', '.row-margin > h3', '.api-heading ~ h3', '.api-heading + h2', '.metadata-member h3').each do |node|
+          node.name = 'h4'
         end
 
-        css('.multi-line-signature').each do |node|
-          node.name = 'pre'
-          node.content = node.content.strip
+        css('.l-sub-section', '.alert', '.banner').each do |node|
+          node.name = 'blockquote'
         end
 
-        css('a[id]:empty').each do |node|
-          node.next_element['id'] = node['id'] if node.next_element
+        css('.file').each do |node|
+          node.content = node.content.strip
         end
 
-        css('a[name]:empty').each do |node|
-          node.next_element['id'] = node['name'] if node.next_element
+        css('.filetree .children').each do |node|
+          node.css('.file').each do |n|
+            n.content = "  #{n.content}"
+          end
         end
 
-        css('tr[style]').each do |node|
-          node.remove_attribute 'style'
+        css('.filetree').each do |node|
+          node.content = node.css('.file').map(&:inner_html).join("\n")
+          node.name = 'pre'
+          node.remove_attribute('class')
         end
 
-        css('h1:not(:first-child)').each do |node|
-          node.name = 'h2'
-        end unless at_css('h2')
+        css('pre').each do |node|
+          node.content = node.content.strip
 
-        css('img[style]').each do |node|
-          node['align'] ||= node['style'][/float:\s*(left|right)/, 1]
-          node['style'] = node['style'].split(';').map(&:strip).select { |s| s =~ /\Awidth|height/ }.join(';')
-        end
+          node['data-language'] = 'typescript' if node['path'].try(:ends_with?, '.ts')
+          node['data-language'] = 'html' if node['path'].try(:ends_with?, '.html')
+          node['data-language'] = 'css' if node['path'].try(:ends_with?, '.css')
+          node['data-language'] = node['language'].sub(/\Ats/, 'typescript').strip if node['language']
+          node['data-language'] ||= 'typescript' if node.content.start_with?('@')
 
-        css('.example-title + pre').each do |node|
-          node['name'] = node.previous_element.content.strip
-          node.previous_element.remove
-        end
+          node.before(%(<div class="pre-title">#{node['title']}</div>)) if node['title']
 
-        css('pre[name]').each do |node|
-          node.before(%(<div class="pre-title">#{node['name']}</div>))
-        end
+          if node['class'] && node['class'].include?('api-heading')
+            node.name = 'h3'
+            node.inner_html = "<code>#{node.inner_html}</code>"
+          end
 
-        css('a.is-button > h3').each do |node|
-          node.parent.content = node.content
+          node.remove_attribute('path')
+          node.remove_attribute('region')
+          node.remove_attribute('linenums')
+          node.remove_attribute('title')
+          node.remove_attribute('language')
+          node.remove_attribute('hidecopy')
+          node.remove_attribute('class')
         end
 
-        css('#angular-2-glossary ~ .l-sub-section').each do |node|
-          node.before(node.children).remove
-        end
+        css('h1[class]').remove_attr('class')
+        css('table[class]').remove_attr('class')
+        css('table[width]').remove_attr('width')
+        css('tr[style]').remove_attr('style')
 
-        location_badge = at_css('.location-badge')
-        if location_badge && doc.last_element_child != location_badge
-          doc.last_element_child.after(location_badge)
+        if at_css('.api-type-label.module')
+          at_css('h1').content = subpath.remove('api/')
         end
 
         doc

lib/docs/filters/angular/clean_html_v2.rb

@@ -0,0 +1,157 @@
+module Docs
+  class Angular
+    class CleanHtmlV2Filter < Filter
+      def call
+        container = at_css('article.docs-content')
+        badges = css('header.hero .badge, header.hero .hero-subtitle').map do |node|
+          node.name = 'span'
+          node['class'] = 'status-badge'
+          node.to_html
+        end.join(' ')
+        badges = %(<div class="badges">#{badges}</div>)
+        container.child.before(at_css('header.hero h1')).before(badges).before(css('header.hero + .banner, header.hero .breadcrumbs'))
+        @doc = container
+
+        title = at_css('h1').content.strip
+        if root_page?
+          at_css('h1').content = 'Angular 2 Documentation'
+        elsif title == 'Index'
+          at_css('h1').content = result[:entries].first.name
+        elsif title == 'Angular'
+          at_css('h1').content = slug.split('/').last.gsub('-', ' ')
+        elsif at_css('.breadcrumbs') && title != result[:entries].first.name
+          at_css('h1').content = result[:entries].first.name
+        end
+
+        css('pre.no-bg-with-indent').each do |node|
+          node.content = '  ' + node.content.gsub("\n", "\n  ")
+        end
+
+        css('.openParens').each do |node|
+          node.parent.name = 'pre'
+          node.parent.content = node.parent.css('code, pre').map(&:content).join("\n")
+        end
+
+        css('button.verbose', 'button.verbose + .l-verbose-section', 'a[id=top]', 'a[href="#top"]', '.sidebar', 'br').remove
+
+        css('.c10', '.showcase', '.showcase-content', '.l-main-section', 'div.div', 'div[flex]', 'code-tabs', 'md-card', 'md-card-content', 'div:not([class])', 'footer', '.card-row', '.card-row-container', 'figure', 'blockquote', 'exported', 'defined', 'div.ng-scope', '.code-example header', 'section.desc', '.row', '.dart-api-entry-main', '.main-content', 'section.summary', 'span.signature').each do |node|
+          node.before(node.children).remove
+        end
+
+        css('span.badges').each do |node|
+          node.name = 'div'
+        end
+
+        css('pre[language]').each do |node|
+          node['data-language'] = node['language'].sub(/\Ats/, 'typescript').strip
+          node['data-language'] = 'html' if node.content.start_with?('<')
+          node.remove_attribute('language')
+          node.remove_attribute('format')
+        end
+
+        css('pre.prettyprint').each do |node|
+          node.content = node.content.strip
+          node['data-language'] = 'dart' if node['class'].include?('dart')
+          node['data-language'] = 'html' if node.content.start_with?('<')
+          node.remove_attribute('class')
+        end
+
+        css('.multi-line-signature').each do |node|
+          node.name = 'pre'
+          node.content = node.content.strip
+        end
+
+        css('a[id]:empty').each do |node|
+          node.next_element['id'] = node['id'] if node.next_element
+        end
+
+        css('a[name]:empty').each do |node|
+          node.next_element['id'] = node['name'] if node.next_element
+        end
+
+        css('tr[style]').each do |node|
+          node.remove_attribute 'style'
+        end
+
+        css('h1:not(:first-child)').each do |node|
+          node.name = 'h2'
+        end unless at_css('h2')
+
+        css('img[style]').each do |node|
+          node['align'] ||= node['style'][/float:\s*(left|right)/, 1]
+          node['style'] = node['style'].split(';').map(&:strip).select { |s| s =~ /\Awidth|height/ }.join(';')
+        end
+
+        css('.example-title + pre').each do |node|
+          node['name'] = node.previous_element.content.strip
+          node.previous_element.remove
+        end
+
+        css('pre[name]').each do |node|
+          node.before(%(<div class="pre-title">#{node['name']}</div>))
+        end
+
+        css('a.is-button > h3').each do |node|
+          node.parent.content = node.content
+        end
+
+        css('#angular-2-glossary ~ .l-sub-section').each do |node|
+          node.before(node.children).remove
+        end
+
+        location_badge = at_css('.location-badge')
+        if location_badge && doc.last_element_child != location_badge
+          doc.last_element_child.after(location_badge)
+        end
+
+        css('.filetree .children').each do |node|
+          node.css('.file').each do |n|
+            n.content = "  #{n.content}"
+          end
+        end
+
+        css('.filetree').each do |node|
+          node.content = node.css('.file').map(&:inner_html).join("\n")
+          node.name = 'pre'
+          node.remove_attribute('class')
+        end
+
+        css('.status-badge').each do |node|
+          node.name = 'code'
+          node.content = node.content.strip
+          node.remove_attribute('class')
+        end
+
+        css('div.badges').each do |node|
+          node.name = 'p'
+        end
+
+        css('td h3', '.l-sub-section > h3', '.alert h3', '.row-margin > h3').each do |node|
+          node.name = 'h4'
+        end
+
+        css('.l-sub-section', '.alert', '.banner').each do |node|
+          node.name = 'blockquote'
+        end
+
+        css('.code-example > h4').each do |node|
+          node['class'] = 'pre-title'
+        end
+
+        css('.row-margin', '.ng-cloak').each do |node|
+          node.before(node.children).remove
+        end
+
+        css('*[layout]').remove_attr('layout')
+        css('*[layout-xs]').remove_attr('layout-xs')
+        css('*[flex]').remove_attr('flex')
+        css('*[flex-xs]').remove_attr('flex-xs')
+        css('*[ng-class]').remove_attr('ng-class')
+        css('*[align]').remove_attr('align')
+        css('h1, h2, h3').remove_attr('class')
+
+        doc
+      end
+    end
+  end
+end

lib/docs/filters/angular/entries.rb

@@ -2,59 +2,24 @@ module Docs
   class Angular
     class EntriesFilter < Docs::EntriesFilter
       def get_name
-        if slug.start_with?('tutorial') || slug.start_with?('guide')
-          name = at_css('.nav-list-item.is-selected, header.hero h1').content.strip
-        else
-          name = at_css('header.hero h1').content.strip
-        end
-
-        name = name.split(':').first
-
-        if mod
-          if name == 'Index'
-            return slug.split('/')[1..-2].join('/')
-          elsif name == 'Angular'
-            return slug.split('/').last.split('-').first
-          end
-        end
-
-        subtitle = at_css('.hero-subtitle').try(:content)
-        breadcrumbs = css('.breadcrumbs li').map(&:content)[2..-2]
-
-        name.prepend "#{breadcrumbs.join('.')}#" if breadcrumbs.present? && breadcrumbs[0] != name
-        name << '()' if %w(Function Method Constructor).include?(subtitle)
+        name = at_css('h1').content
+        name.prepend "#{$1}. " if subpath =~ /\-pt(\d+)/
         name
       end
 
       def get_type
-        if slug.start_with?('guide/')
-          'Guide'
-        elsif slug.start_with?('cookbook/')
-          'Cookbook'
-        elsif slug == 'glossary'
+        if slug.start_with?('guide')
           'Guide'
+        elsif slug.start_with?('tutorial')
+          'Tutorial'
+        elsif node = at_css('th:contains("npm Package")')
+          node.next_element.content.remove('@angular/')
+        elsif at_css('.api-type-label.module')
+          name.split('/').first
         else
-          type = at_css('.nav-title.is-selected').content.strip
-          type.remove! ' Reference'
-          type << ": #{mod}" if mod
-          type
+          'Miscellaneous'
         end
       end
-
-      INDEX = Set.new
-
-      def include_default_entry?
-        INDEX.add?([name, type].join(';')) ? true : false # ¯\_(ツ)_/¯
-      end
-
-      private
-
-      def mod
-        return @mod if defined?(@mod)
-        @mod = slug[/api\/([\w\-\.]+)\//, 1]
-        @mod.remove! 'angular2.' if @mod
-        @mod
-      end
     end
   end
 end

lib/docs/filters/angular/entries_v2.rb

@@ -0,0 +1,60 @@
+module Docs
+  class Angular
+    class EntriesV2Filter < Docs::EntriesFilter
+      def get_name
+        if slug.start_with?('tutorial') || slug.start_with?('guide')
+          name = at_css('.nav-list-item.is-selected, header.hero h1').content.strip
+        else
+          name = at_css('header.hero h1').content.strip
+        end
+
+        name = name.split(':').first
+
+        if mod
+          if name == 'Index'
+            return slug.split('/')[1..-2].join('/')
+          elsif name == 'Angular'
+            return slug.split('/').last.split('-').first
+          end
+        end
+
+        subtitle = at_css('.hero-subtitle').try(:content)
+        breadcrumbs = css('.breadcrumbs li').map(&:content)[2..-2]
+
+        name.prepend "#{breadcrumbs.join('.')}#" if breadcrumbs.present? && breadcrumbs[0] != name
+        name << '()' if %w(Function Method Constructor).include?(subtitle)
+        name
+      end
+
+      def get_type
+        if slug.start_with?('guide/')
+          'Guide'
+        elsif slug.start_with?('cookbook/')
+          'Cookbook'
+        elsif slug == 'glossary'
+          'Guide'
+        else
+          type = at_css('.nav-title.is-selected').content.strip
+          type.remove! ' Reference'
+          type << ": #{mod}" if mod
+          type
+        end
+      end
+
+      INDEX = Set.new
+
+      def include_default_entry?
+        INDEX.add?([name, type].join(';')) ? true : false # ¯\_(ツ)_/¯
+      end
+
+      private
+
+      def mod
+        return @mod if defined?(@mod)
+        @mod = slug[/api\/([\w\-\.]+)\//, 1]
+        @mod.remove! 'angular2.' if @mod
+        @mod
+      end
+    end
+  end
+end

lib/docs/scrapers/angular.rb

@@ -1,56 +1,110 @@
+require 'yajl/json_gem'
+
 module Docs
   class Angular < UrlScraper
     self.type = 'angular'
-    self.root_path = 'api/'
     self.links = {
       home: 'https://angular.io/',
       code: 'https://github.com/angular/angular'
     }
 
-    html_filters.push 'angular/entries', 'angular/clean_html'
-
-    options[:skip_patterns] = [/deprecated/, /VERSION-let/]
-    options[:skip] = %w(
-      index.html
-      styleguide.html
-      quickstart.html
-      cheatsheet.html
-      guide/cheatsheet.html
-      guide/style-guide.html)
-
-    options[:replace_paths] = {
-      'testing/index.html'  => 'guide/testing.html',
-      'guide/glossary.html' => 'glossary.html',
-      'tutorial'            => 'tutorial/',
-      'api'                 => 'api/'
-    }
-
-    options[:fix_urls] = -> (url) do
-      url.sub! %r{\A(https://(?:v2\.)?angular\.io/docs/.+/)index\.html\z}, '\1'
-      url
-    end
+    options[:max_image_size] = 256_000
 
     options[:attribution] = <<-HTML
       &copy; 2010&ndash;2017 Google, Inc.<br>
       Licensed under the Creative Commons Attribution License 4.0.
     HTML
 
-    stub 'api/' do
-      base_url = URL.parse(self.base_url)
-      capybara = load_capybara_selenium
-      capybara.app_host = base_url.origin
-      capybara.visit(base_url.path + 'api/')
-      capybara.execute_script('return document.body.innerHTML')
-    end
+    version do
+      self.release = '4.3.2'
+      self.base_url = 'https://angular.io/'
+      self.root_path = 'docs'
+
+      html_filters.push 'angular/clean_html', 'angular/entries'
+
+      options[:follow_links] = false
+      options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
+      options[:fix_urls_before_parse] = ->(url) do
+        url.sub! %r{\Aguide/}, '/guide/'
+        url.sub! %r{\Atutorial/}, '/tutorial/'
+        url.sub! %r{\Aapi/}, '/api/'
+        url.sub! %r{\Agenerated/}, '/generated/'
+        url
+      end
+
+      private
+
+      def initial_urls
+        initial_urls = []
 
-    version '4 TypeScript' do
-      self.release = '4.0.3'
-      self.base_url = 'https://angular.io/docs/ts/latest/'
+        Request.run 'https://angular.io/generated/navigation.json' do |response|
+          data = JSON.parse(response.body)
+          dig = ->(entry) do
+            initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
+            entry['children'].each(&dig) if entry['children']
+          end
+          data['SideNav'].each(&dig)
+        end
+
+        Request.run 'https://angular.io/generated/docs/api/api-list.json' do |response|
+          data = JSON.parse(response.body)
+          dig = ->(entry) do
+            initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']
+            initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']
+            entry['items'].each(&dig) if entry['items']
+          end
+          data.each(&dig)
+        end
+
+        initial_urls
+      end
+
+      def handle_response(response)
+        if response.mime_type.include?('json')
+          response.options[:response_body] = JSON.parse(response.body)['contents']
+          response.headers['Content-Type'] = 'text/html'
+          response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
+          response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
+        end
+        super
+      end
     end
 
-    version '2 TypeScript' do
+    version '2' do
       self.release = '2.4.10'
       self.base_url = 'https://v2.angular.io/docs/ts/latest/'
+      self.root_path = 'api/'
+
+      html_filters.push 'angular/entries_v2', 'angular/clean_html_v2'
+
+      stub 'api/' do
+        base_url = URL.parse(self.base_url)
+        capybara = load_capybara_selenium
+        capybara.app_host = base_url.origin
+        capybara.visit(base_url.path + 'api/')
+        capybara.execute_script('return document.body.innerHTML')
+      end
+
+      options[:skip_patterns] = [/deprecated/, /VERSION-let/]
+      options[:skip] = %w(
+        index.html
+        styleguide.html
+        quickstart.html
+        cheatsheet.html
+        guide/cheatsheet.html
+        guide/style-guide.html)
+
+      options[:replace_paths] = {
+        'testing/index.html'  => 'guide/testing.html',
+        'guide/glossary.html' => 'glossary.html',
+        'tutorial'            => 'tutorial/',
+        'api'                 => 'api/'
+      }
+
+      options[:fix_urls] = -> (url) do
+        url.sub! %r{\A(https://(?:v2\.)?angular\.io/docs/.+/)index\.html\z}, '\1'
+        url
+      end
     end
 
     private