Update Docker documentation (20.10.6)

lib/docs/filters/docker/clean_html_old.rb

@@ -1,37 +0,0 @@
-module Docs
-  class Docker
-    class CleanHtmlOldFilter < Filter
-      def call
-        if root_page?
-          doc.inner_html = "<h1>Docker Documentation</h1>"
-          return doc
-        end
-
-        @doc = at_css('#DocumentationText')
-
-        at_css('h2').name = 'h1' unless at_css('h1')
-
-        css('.anchorLink', '.reading-time', 'hr', '> div[style*="margin-top"]:last-child').remove
-
-        css('h1 + h1').each do |node|
-          node.name = 'h2'
-        end
-
-        css('pre').each do |node|
-          node.content = node.content
-          node['data-language'] = node.parent['class'][/language-(\w+)/, 1] if node.parent['class']
-        end
-
-        css('div.highlighter-rouge').each do |node|
-          node.before(node.children).remove
-        end
-
-        css('code.highlighter-rouge').each do |node|
-          node.content = node.content.gsub(/\s+/, ' ').strip
-        end
-
-        doc
-      end
-    end
-  end
-end

lib/docs/filters/docker/clean_html_very_old.rb

@@ -1,22 +0,0 @@
-module Docs
-  class Docker
-    class CleanHtmlVeryOldFilter < Filter
-      def call
-        if root_page?
-          doc.inner_html = "<h1>Docker Documentation</h1>"
-          return doc
-        end
-
-        @doc = at_css('#content')
-
-        at_css('h2').name = 'h1' unless at_css('h1')
-
-        css('pre').each do |node|
-          node.content = node.content
-        end
-
-        doc
-      end
-    end
-  end
-end

lib/docs/filters/docker/entries.rb

@@ -10,20 +10,7 @@ module Docs
 
       def get_name
         return NAME_BY_SUBPATH[subpath] if NAME_BY_SUBPATH[subpath]
-        return at_css('h1').content unless nav_link
-
-        name = nav_link.content.strip
-        name.capitalize! if name == 'exoscale'
-        name.remove! ' (base command)'
-
-        if name =~ /\A[a-z\-\s]+\z/
-          name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./
-          name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./
-        else
-          name << " (#{product})" if name !~ /#{product}/i && !subpath.start_with?('get-started')
-        end
-
-        name
+        at_css('h1').content
       end
 
       def get_type
@@ -37,11 +24,6 @@ module Docs
         product
       end
 
-      def nav_link
-        return @nav_link if defined?(@nav_link)
-        @nav_link = at_css('.currentPage')
-      end
-
       def product
         @product ||= subpath.split('/').first.capitalize
       end

lib/docs/filters/docker/entries_old.rb

@@ -1,75 +0,0 @@
-module Docs
-  class Docker
-    class EntriesOldFilter < Docs::EntriesFilter
-      NAME_BY_SUBPATH = {
-        'engine/' => 'Engine',
-        'compose/' => 'Compose',
-        'machine/' => 'Machine'
-      }
-
-      def get_name
-        return NAME_BY_SUBPATH[subpath] if NAME_BY_SUBPATH[subpath]
-        return at_css('h1').content unless nav_link
-
-        name = nav_link.content.strip
-        name.capitalize! if name == 'exoscale'
-        name.remove! ' (base command)'
-
-        if name =~ /\A[a-z\-\s]+\z/
-          name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./
-          name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./
-        else
-          name << " (#{product})" if name !~ /#{product}/i
-        end
-
-        name
-      end
-
-      TYPE_BY_SUBPATH = {
-        'engine/' => 'Engine',
-        'compose/' => 'Compose',
-        'machine/' => 'Machine'
-      }
-
-      def get_type
-        return TYPE_BY_SUBPATH[subpath] if TYPE_BY_SUBPATH[subpath]
-        return 'Engine: CLI'         if subpath.start_with?('engine/reference/commandline/')
-        return 'Engine: Admin Guide' if subpath.start_with?('engine/admin/')
-        return 'Engine: Security'    if subpath.start_with?('engine/security/')
-        return 'Engine: Extend'      if subpath.start_with?('engine/extend/')
-        return 'Engine: Get Started' if subpath.start_with?('engine/getstarted')
-        return 'Engine: Tutorials'   if subpath.start_with?('engine/tutorials/')
-        return product if !nav_link && subpath =~ /\A\w+\/[\w\-]+\/\z/
-
-        leaves = nav_link.ancestors('li.leaf').reverse
-        return product if leaves.length <= 2
-
-        type = leaves[0..1].map { |node| node.at_css('> a').content.strip }.join(': ')
-        type.remove! %r{\ADocker }
-        type.remove! ' Engine'
-        type.sub! %r{Command[\-\s]line reference}i, 'CLI'
-        type.sub! 'CLI reference', 'CLI'
-        type
-      end
-
-      def nav_link
-        return @nav_link if defined?(@nav_link)
-        @nav_link = at_css('.currentPage')
-
-        unless @nav_link
-          link = at_css('#DocumentationText li a')
-          return unless link
-          link = at_css(".docsidebarnav_section a[href='#{link['href']}']")
-          return unless link
-          @nav_link = link.ancestors('.menu-closed').first.at_css('a')
-        end
-
-        @nav_link
-      end
-
-      def product
-        @product ||= subpath.split('/').first.capitalize
-      end
-    end
-  end
-end

lib/docs/filters/docker/entries_very_old.rb

@@ -1,56 +0,0 @@
-module Docs
-  class Docker
-    class EntriesVeryOldFilter < Docs::EntriesFilter
-      def get_name
-        name = nav_link ? nav_link.content.strip : at_css('#content h1').content.strip
-        name.capitalize! if name == 'exoscale'
-
-        if name =~ /\A[a-z\-\s]+\z/
-          name.prepend 'docker ' if subpath =~ /engine\/reference\/commandline\/./
-          name.prepend 'docker-compose  ' if subpath =~ /compose\/reference\/./
-          name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./
-          name.prepend 'swarm ' if subpath =~ /swarm\/reference\/./ && name != 'swarm'
-        else
-          name << " (#{product})" if name !~ /#{product}/i
-        end
-
-        name
-      end
-
-      def get_type
-        unless nav_link
-          return 'Engine: User guide' if subpath.start_with?('engine/userguide')
-        end
-
-        type = nav_link.ancestors('article').to_a.reverse.to_a[0..1].map do |node|
-          node.at_css('> button').content.strip
-        end.join(': ')
-
-        type.remove! %r{\ADocker }
-        type.remove! %r{ Engine}
-        type.sub! %r{Command[\-\s]line reference}i, 'CLI'
-        type = 'Engine: Reference' if type == 'Engine: reference'
-        type
-      end
-
-      def nav_link
-        return @nav_link if defined?(@nav_link)
-        @nav_link = at_css('#multiple .active')
-
-        unless @nav_link
-          link = at_css('#content li a')
-          return unless link
-          link = at_css("#multiple a[href='#{link['href']}']")
-          return unless link
-          @nav_link = link.ancestors('article').first.at_css('button')
-        end
-
-        @nav_link
-      end
-
-      def product
-        @product ||= subpath.split('/').first.capitalize
-      end
-    end
-  end
-end

lib/docs/scrapers/docker.rb

@@ -1,11 +1,31 @@
 module Docs
   class Docker < UrlScraper
+    include MultipleBaseUrls
+
+    self.name = 'Docker'
     self.type = 'simple'
+    self.release = '20.10.6'
+
+    self.base_urls = [
+      'https://docs.docker.com/',
+      'https://docs.docker.com/machine/'
+    ]
+
     self.links = {
       home: 'https://docker.com/',
       code: 'https://github.com/docker/docker'
     }
 
+    html_filters.push 'docker/entries', 'docker/clean_html'
+
+    options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//]
+
+    options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/]
+
+    options[:skip] = [
+      'engine/userguide/networking/get-started-overlay/'
+    ]
+
     options[:trailing_slash] = true
 
     options[:fix_urls] = ->(url) do
@@ -21,242 +41,54 @@ module Docs
       Docker, Inc. and other parties may also have trademark rights in other terms used herein.
     HTML
 
+    options[:replace_paths] = {
+      'engine/userguide/' => 'config/daemon',
+      'engine/userguide/labels-custom-metadata/' => 'config',
+      'engine/swarm/networking/' => 'network/overlay',
+      'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images/dockerfile_best-practices/',
+      'engine/userguide/networking/get-started-overlay/' => 'network/network-tutorial-overlay/',
+      'engine/userguide/networking/' => 'network/',
+      'engine/reference/api/' => 'develop/sdk',
+      'engine/api/latest/' => 'develop/sdk',
+      'get-started/part3/' => 'get-started/04_sharing_app/',
+      'engine/security/https/' => 'engine/security/protect-access/',
+      'compose/aspnet-mssql-compose/' => 'samples/aspnet-mssql-compose/',
+      'engine/examples/dotnetcore/' => 'samples/dotnetcore/'
+    }
+
     version '19' do
       self.release = '19.03'
-      self.base_url = "https://docs.docker.com/"
-
-      html_filters.push 'docker/entries', 'docker/clean_html'
-
-      options[:container] = '.wrapper .container-fluid .row'
-
-      options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//]
-      options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/]
-
-      options[:replace_paths] = {
-        'install/linux/ubuntu/' => 'install/linux/docker-ce',
-        'get-started/part1' => 'get-started',
-        'engine/installation/' => 'install',
-        'engine/installation/linux/linux-postinstall/' => 'install/linux',
-        'compose/overview/' => 'compose',
-        'docker-cloud/' => 'docker-hub',
-        'datacenter/install/linux/' => 'ee',
-        'engine/userguide/' => 'config/daemon',
-        'engine/admin/' => 'config/daemon',
-        'opensource/get-help/' => 'opensource',
-        'engine/tutorials/dockerimages/' => 'get-started',
-        'engine/admin/volumes/bind-mounts/' => 'storage',
-        'engine/tutorials/dockervolumes/' => 'storage',
-        'engine/admin/volumes/volumes/' => 'storage',
-        'engine/userguide/labels-custom-metadata/' => 'config',
-        'engine/userguide/eng-image/multistage-build/' => 'develop/develop-images',
-        'engine/swarm/networking/' => 'network',
-        'engine/admin/resource_constraints/' => 'config/containers',
-        'engine/admin/logging/overview/' => 'config/containers/logging',
-        'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images',
-        'engine/tutorials/dockerrepos/' => 'get-started',
-        'engine/userguide/networking/' => 'network',
-        'engine/userguide/networking/get-started-overlay/' => 'network',
-        'engine/reference/commandline/swarm_join_token/' => 'engine/reference/commandline',
-        'engine/understanding-docker/' => 'engine',
-        'engine/userguide/dockervolumes/' => 'storage',
-        'engine/installation/binaries/' => 'install/linux/docker-ce',
-        'engine/userguide/networking/default_network/dockerlinks/' => 'network',
-        'engine/reference/api/' => 'develop/sdk',
-        'engine/admin/systemd/' => 'config/daemon',
-        'engine/userguide/storagedriver/imagesandcontainers/' => 'storage/storagedriver',
-        'engine/api/' => 'develop/sdk',
-        'engine/userguide/networking/get-started-overlay' => 'network',
-        'engine/userguide/networking/overlay-security-model/' => 'network',
-        'engine/installation/linux/docker-ce/binaries/' => 'install/linux/docker-ce',
-        'engine/admin/volumes/' => 'storage/volumes/',
-        'engine/userguide/networking//' => 'network',
-        'engine/reference/commandline' => 'engine/reference/commandline/docker',
-        'engine/reference/commandline/' => 'engine/reference/commandline/docker/',
-      }
+      self.base_url = "https://docs.docker.com"
     end
 
     version '18' do
       self.release = '18.09'
-      self.base_url = "https://docs.docker.com/v#{release}/"
-
-      html_filters.push 'docker/entries', 'docker/clean_html'
-
-      options[:container] = '.wrapper .container-fluid .row'
-
-      options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//]
-      options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/]
-
-      options[:replace_paths] = {
-        'install/linux/ubuntu/' => 'install/linux/docker-ce',
-        'get-started/part1' => 'get-started',
-        'engine/installation/' => 'install',
-        'engine/installation/linux/linux-postinstall/' => 'install/linux',
-        'compose/overview/' => 'compose',
-        'datacenter/install/linux/' => 'ee',
-        'engine/userguide/' => 'config/daemon',
-        'engine/admin/' => 'config/daemon',
-        'opensource/get-help/' => 'opensource',
-        'engine/tutorials/dockerimages/' => 'get-started',
-        'engine/admin/volumes/bind-mounts/' => 'storage',
-        'engine/tutorials/dockervolumes/' => 'storage',
-        'engine/admin/volumes/volumes/' => 'storage',
-        'engine/userguide/labels-custom-metadata/' => 'config',
-        'engine/reference/api/' => 'develop/sdk',
-        'engine/userguide/eng-image/multistage-build/' => 'develop/develop-images',
-        'engine/swarm/networking/' => 'network',
-        'engine/admin/resource_constraints/' => 'config/containers',
-        'engine/admin/logging/overview/' => 'config/containers/logging',
-        'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images',
-        'engine/tutorials/dockerrepos/' => 'get-started',
-        'engine/userguide/networking/' => 'network',
-        'engine/userguide/networking/get-started-overlay/' => 'network',
-        'engine/understanding-docker/' => 'engine',
-        'engine/reference/commandline/swarm_join_token/' => 'engine/reference/commandline',
-        'engine/userguide/dockervolumes/' => 'storage',
-        'engine/admin/systemd/' => 'config/daemon',
-        'engine/userguide/storagedriver/imagesandcontainers/' => 'storage/storagedriver',
-        'engine/installation/binaries/' => 'install/linux/docker-ce',
-        'engine/userguide/networking/default_network/dockerlinks/' => 'network',
-        'engine/userguide/networking/overlay-security-model/' => 'network',
-        'engine/userguide/networking/get-started-overlay' => 'network',
-        'engine/api/' => 'develop/sdk',
-        'engine/installation/linux/docker-ce/binaries/' => 'install/linux/docker-ce',
-        'engine/admin/volumes/' => 'storage/volumes/',
-        'engine/userguide/networking//' => 'network',
-        'engine/reference/commandline' => 'engine/reference/commandline/docker',
-        'engine/reference/commandline/' => 'engine/reference/commandline/docker/',
-      }
+      self.base_url = "https://docs.docker.com"
     end
 
     version '17' do
       self.release = '17.12'
-      self.base_url = "https://docs.docker.com/v#{release}/"
-
-      html_filters.push 'docker/entries', 'docker/clean_html'
-
-      options[:container] = '.wrapper .container-fluid .row'
-
-      options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//]
-      options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/]
-
-      options[:replace_paths] = {
-        'get-started/part1' => 'get-started',
-        'engine/installation/' => 'install',
-        'engine/installation/linux/linux-postinstall/' => 'install/linux',
-        'opensource/get-help/' => 'opensource',
-        'engine/admin/volumes/volumes/' => 'storage',
-        'engine/tutorials/dockerimages/' => 'get-started',
-        'engine/admin/volumes/bind-mounts/' => 'storage',
-        'engine/tutorials/dockervolumes/' => 'storage',
-        'datacenter/install/aws/' => 'docker-for-aws',
-        'engine/userguide/' => 'config/daemon',
-        'engine/admin/' => 'config/daemon',
-        'engine/userguide/labels-custom-metadata/' => 'config',
-        'engine/userguide/eng-image/multistage-build/' => 'develop/develop-images',
-        'engine/swarm/networking/' => 'network',
-        'engine/admin/resource_constraints/' => 'config/containers',
-        'engine/admin/logging/overview/' => 'config/containers/logging',
-        'engine/understanding-docker/' => 'engine',
-        'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images',
-        'engine/tutorials/dockerrepos/' => 'get-started',
-        'engine/userguide/networking/' => 'network',
-        'engine/reference/commandline/swarm_join_token/' => 'edge/engine/reference/commandline',
-        'engine/userguide/networking/get-started-overlay/' => 'network',
-        'engine/userguide/dockervolumes/' => 'storage',
-        'engine/installation/binaries/' => 'install/linux/docker-ce',
-        'engine/userguide/networking/default_network/dockerlinks/' => 'network',
-        'engine/reference/api/' => 'develop/sdk',
-        'engine/admin/live-restore/' => 'config/containers',
-        'engine/api/' => 'develop/sdk',
-        'engine/userguide/networking/get-started-overlay' => 'network',
-        'security/security/' => 'engine/security',
-        'engine/installation/linux/docker-ce/binaries/' => 'install/linux/docker-ce',
-        'engine/reference/commandline/' => 'edge/engine/reference/commandline',
-        'engine/admin/systemd/' => 'config/daemon',
-        'engine/userguide/storagedriver/imagesandcontainers/' => 'storage/storagedriver',
-        'engine/userguide/networking/overlay-security-model/' => 'network',
-        'engine/admin/volumes/' => 'storage/volumes/',
-        'engine/userguide/networking//' => 'network',
-      }
-    end
-
-    module OldOptions
-      def self.included(klass)
-        klass.options[:only_patterns] = [/\Aengine\//, /\Acompose\//, /\Amachine\//]
-        klass.options[:skip_patterns] = [/\Aengine\/api\/v/, /\Aengine\/installation/]
-        klass.options[:skip] = %w(
-          /
-          engine/userguide/
-          engine/examples/
-          engine/reference/
-          engine/reference/api/
-          engine/reference/api/docker_remote_api_v1.24/
-          engine/getstarted/linux_install_help/
-          machine/reference/
-          machine/drivers/
-          machine/examples/
-          compose/reference/
-        )
-        klass.options[:replace_paths] = {
-          'engine/userguide/networking/dockernetworks/' => 'engine/userguide/networking/',
-          'engine/userguide/dockervolumes/'             => 'engine/tutorials/dockervolumes/',
-          'engine/reference/logging/overview/'          => 'engine/admin/logging/overview/',
-          'engine/reference/commandline/daemon/'        => 'engine/reference/commandline/dockerd/',
-          'engine/reference/commandline/'               => 'engine/reference/commandline/docker/',
-          'engine/reference/api/docker_remote_api/'     => 'engine/api/',
-          'engine/swarm/how-swarm-mode-works/'          => 'engine/swarm/how-swarm-mode-works/nodes/',
-          'engine/tutorials/dockerizing/'               => 'engine/getstarted/step_one/',
-          'engine/tutorials/usingdocker/'               => 'engine/getstarted/step_three/',
-          'engine/tutorials/dockerimages/'              => 'engine/getstarted/step_four/',
-          'engine/tutorials/dockerrepos/'               => 'engine/getstarted/step_six/'
-        }
-      end
+      self.base_url = "https://docs.docker.com"
     end
 
     version '1.13' do
-      include OldOptions
-
       self.release = '1.13'
-      self.base_url = "https://docs.docker.com/v#{self.version}/"
-
-      html_filters.push 'docker/entries_old', 'docker/clean_html_old'
-
-      options[:container] = '.container-fluid .row'
+      self.base_url = "https://docs.docker.com"
     end
 
     version '1.12' do
-      include OldOptions
-
       self.release = '1.12'
-      self.base_url = "https://docs.docker.com/v#{self.version}/"
-
-      html_filters.push 'docker/entries_old', 'docker/clean_html_old'
-
-      options[:container] = '.container-fluid .row'
+      self.base_url = "https://docs.docker.com"
     end
 
     version '1.11' do
-      include OldOptions
-
       self.release = '1.11'
-      self.base_url = "https://docs.docker.com/v#{self.version}/"
-
-      html_filters.push 'docker/entries_very_old', 'docker/clean_html_very_old'
-
-      options[:container] = '#docs'
-      options[:only_patterns] << /\Aswarm\//
+      self.base_url = "https://docs.docker.com"
     end
 
     version '1.10' do
-      include OldOptions
-
       self.release = '1.10'
-      self.base_url = "https://docs.docker.com/v#{self.version}/"
-
-      html_filters.push 'docker/entries_very_old', 'docker/clean_html_very_old'
-
-      options[:container] = '#docs'
-      options[:only_patterns] << /\Aswarm\//
+      self.base_url = "https://docs.docker.com"
     end
 
     def get_latest_version(opts)