Update & improve MDN scrapers

Closes 260.

lib/docs/filters/css/entries.rb

@@ -9,9 +9,6 @@ module Docs
         linear-gradient radial-gradient repeating-linear-gradient
         repeating-radial-gradient var)
 
-      PSEUDO_ELEMENT_SLUGS = %w(::after ::before ::first-letter ::first-line
-        ::selection)
-
       VALUE_SLUGS = %w(auto inherit initial none normal unset)
 
       ADDITIONAL_ENTRIES = {
@@ -39,6 +36,7 @@ module Docs
         'transform-function' => [
           %w(matrix() matrix() Functions),
           %w(matrix3d() matrix3d() Functions),
+          %w(perspective() perspective() Functions),
           %w(rotate() rotate() Functions),
           %w(rotate3d() rotate3d() Functions),
           %w(rotateX() rotateX() Functions),
@@ -60,18 +58,21 @@ module Docs
 
       def get_name
         case type
-        when 'Data Types' then "<#{super.remove ' value'}>"
-        when 'Functions'  then "#{super}()"
+        when 'Data Types'    then "<#{super.remove ' value'}>"
+        when 'Functions'     then "#{super}()"
+        when 'Miscellaneous' then slug.to_s.gsub('_', ' ').gsub('/', ': ')
         else super
         end
       end
 
       def get_type
-        if slug.end_with? 'selectors'
+        if slug.end_with?('selectors')
           'Selectors'
-        elsif slug.start_with? ':'
-          PSEUDO_ELEMENT_SLUGS.include?(slug) ? 'Pseudo-elements' : 'Pseudo-classes'
-        elsif slug.start_with? '@'
+        elsif slug.start_with?('::')
+          'Pseudo-elements'
+        elsif slug.start_with?(':')
+          'Pseudo-classes'
+        elsif slug.start_with?('@')
           'At-rules'
         elsif DATA_TYPE_SLUGS.include?(slug)
           'Data Types'
@@ -79,6 +80,8 @@ module Docs
           'Functions'
         elsif VALUE_SLUGS.include?(slug)
           'Values'
+        elsif slug =~ /\A[a-z]+_/i
+          'Miscellaneous'
         else
           'Properties'
         end

lib/docs/filters/dom/entries.rb

@@ -4,6 +4,7 @@ module Docs
       TYPE_BY_SPEC = {
         'Battery Status'      => 'Battery Status',
         'Canvas '             => 'Canvas',
+        'CSS Font Loading'    => 'CSS',
         'CSS Object Model'    => 'CSS',
         'Cryptography'        => 'Web Cryptography',
         'Device Orientation'  => 'Device Orientation',
@@ -19,6 +20,8 @@ module Docs
         'Navigation Timing'   => 'Navigation Timing',
         'Network Information' => 'Network Information',
         'Push API'            => 'Push',
+        'Shadow DOM'          => 'Shadow DOM',
+        'Server-Sent Events'  => 'Server-Sent Events',
         'Service Workers'     => 'Service Workers',
         'Web Animations'      => 'Animation',
         'Web Audio'           => 'Web Audio',
@@ -64,6 +67,8 @@ module Docs
         'StyleSheet'          => 'CSS',
         'Stylesheet'          => 'CSS',
         'SVG'                 => 'SVG',
+        'timing'              => 'Navigation Timing',
+        'Timing'              => 'Navigation Timing',
         'Touch'               => 'Touch',
         'TreeWalker'          => 'TreeWalker',
         'URL'                 => 'URL',
@@ -74,15 +79,22 @@ module Docs
       TYPE_BY_NAME_INCLUDES = {
         'ChildNode'     => 'Node',
         'Crypto'        => 'Web Cryptography',
+        'FormData'      => 'XMLHTTPRequest',
+        'ImageBitmap'   => 'Canvas',
         'ImageData'     => 'Canvas',
         'IndexedDB'     => 'IndexedDB',
+        'Media Source'  => 'Media',
         'MediaStream'   => 'Media',
         'NodeList'      => 'Node',
         'Path2D'        => 'Canvas',
+        'Server-sent'   => 'Server-Sent Events',
         'ServiceWorker' => 'Service Workers',
         'TextMetrics'   => 'Canvas',
         'udio'          => 'Web Audio',
+        'WebSocket'     => 'Web Sockets',
         'WebGL'         => 'WebGL',
+        'WEBGL'         => 'WebGL',
+        'WebRTC'        => 'WebRTC',
         'WebVR'         => 'WebVR',
         'Worker'        => 'Web Workers' }
 
@@ -111,6 +123,7 @@ module Docs
         name.sub! 'API.', 'API: '
         # Comment.Comment => Comment.constructor
         name.sub! %r{\A(\w+)\.\1\z}, '\1.constructor' unless name == 'window.window'
+        name.prepend 'XMLHttpRequest.' if slug.start_with?('XMLHttpRequest/') && !name.start_with?('XMLHttpRequest')
         name
       end
 
@@ -160,6 +173,28 @@ module Docs
         content = node.content
         SKIP_CONTENT.none? { |str| content.include?(str) }
       end
+
+      def additional_entries
+        entries = []
+
+        if slug == 'history' || slug == 'XMLHttpRequest'
+          css('dt a[title*="not yet been written"]').each do |node|
+            next if node.parent.at_css('.obsolete')
+            name = node.content.sub('History', 'history')
+            id = node.parent['id'] = name.parameterize
+            entries << [name, id]
+          end
+        end
+
+        if slug == 'XMLHttpRequest'
+          css('h2[id="Methods_2"] ~ h3').each do |node|
+            break if node.content == 'Non-standard methods'
+            entries << ["#{name}.#{node.content}", node['id']]
+          end
+        end
+
+        entries
+      end
     end
   end
 end

lib/docs/filters/dom_events/entries.rb

@@ -3,12 +3,15 @@ module Docs
     class EntriesFilter < Docs::EntriesFilter
       TYPE_BY_INFO = {
         'applicationCache' => 'Application Cache',
+        'Battery'          => 'Battery',
         'Clipboard'        => 'Clipboard',
         'CSS'              => 'CSS',
         'Drag'             => 'Drag & Drop',
         'Focus'            => 'Focus',
         'Fullscreen'       => 'Fullscreen',
+        'Gamepad'          => 'Gamepad',
         'HashChange'       => 'History',
+        'IDB'              => 'IndexedDB',
         'IndexedDB'        => 'IndexedDB',
         'Keyboard'         => 'Keyboard',
         'edia'             => 'Media',
@@ -16,6 +19,7 @@ module Docs
         'Offline'          => 'Offline',
         'Orientation'      => 'Device',
         'Sensor'           => 'Device',
+        'SVG'              => 'SVG',
         'Page Visibility'  => 'Page Visibility',
         'Pointer'          => 'Mouse',
         'PopState'         => 'History',
@@ -29,6 +33,7 @@ module Docs
         'WebSocket'        => 'WebSocket',
         'Web Audio'        => 'Web Audio',
         'Web Messaging'    => 'Web Messaging',
+        'WebRTC'           => 'WebRTC',
         'Wheel'            => 'Mouse',
         'Worker'           => 'Web Workers' }
 
@@ -61,6 +66,10 @@ module Docs
           'Miscellaneous'
         end
       end
+
+      def include_default_entry?
+        !doc.content.include?('Firefox OS specific')
+      end
     end
   end
 end

lib/docs/scrapers/mdn/dom.rb

@@ -54,7 +54,7 @@ module Docs
       /\A\/InstallTrigger/,
       /\A\/Entity/,
       /\A\/Settings/,
-      /\A\/Telephony/,
+      /telephony/i,
       /\A\/NFC_API/,
       /\A\/Window\/\w+bar/i,
       /\A\/Apps/,

lib/docs/scrapers/mdn/dom_events.rb

@@ -10,6 +10,7 @@ module Docs
     options[:root_title] = 'DOM Events'
 
     options[:skip] = %w(/MozOrientation)
+    options[:skip_patterns] = [/\A\/moz/i]
 
     options[:fix_urls] = ->(url) do
       url.sub! 'https://developer.mozilla.org/en-US/Mozilla_event_reference',      DomEvents.base_url