Codebase list ruby-cms-scanner / 3649ef4
Update upstream source from tag 'upstream/0.0.44.1' Update to upstream version '0.0.44.1' with Debian dir 72674ec701df28b981453186297e37b36a2217ca Sophie Brun 5 years ago
28 changed file(s) with 173 addition(s) and 121 deletion(s). Raw diff Collapse all Expand all
00 AllCops:
1 TargetRubyVersion: 2.3
1 TargetRubyVersion: 2.4
22 Exclude:
33 - '*.gemspec'
44 - 'vendor/**/*'
99 Max: 120
1010 MethodLength:
1111 Max: 18
12 Exclude:
13 - app/controllers/core/cli_options.rb
1214 Lint/UriEscapeUnescape:
1315 Enabled: false
1416 Metrics/AbcSize:
11 sudo: false
22 cache: bundler
33 rvm:
4 - 2.3.0
5 - 2.3.1
6 - 2.3.2
7 - 2.3.3
8 - 2.3.4
9 - 2.3.5
10 - 2.3.6
11 - 2.3.7
12 - 2.3.8
134 - 2.4.1
145 - 2.4.2
156 - 2.4.3
167 - 2.4.4
178 - 2.4.5
9 - 2.4.6
1810 - 2.5.0
1911 - 2.5.1
2012 - 2.5.2
4545 OptBoolean.new(['--random-user-agent', '--rua',
4646 'Use a random user-agent for each scan']),
4747 OptFilePath.new(['--user-agents-list FILE-PATH',
48 'List of agents to use with --random-user-agent'], exists: true, advanced: true),
48 'List of agents to use with --random-user-agent'],
49 exists: true,
50 advanced: true,
51 default: APP_DIR.join('user_agents.txt')),
4952 OptCredentials.new(['--http-auth login:password']),
5053 OptPositiveInteger.new(['-t', '--max-threads VALUE', 'The max threads to use'],
5154 default: 5),
66 # Core Controller
77 class Core < Base
88 def setup_cache
9 return unless parsed_options[:cache_dir]
9 return unless NS::ParsedCli.cache_dir
1010
11 storage_path = File.join(parsed_options[:cache_dir], Digest::MD5.hexdigest(target.url))
11 storage_path = File.join(NS::ParsedCli.cache_dir, Digest::MD5.hexdigest(target.url))
1212
1313 Typhoeus::Config.cache = Cache::Typhoeus.new(storage_path)
14 Typhoeus::Config.cache.clean if parsed_options[:clear_cache]
14 Typhoeus::Config.cache.clean if NS::ParsedCli.clear_cache
1515 end
1616
1717 def before_scan
2222 end
2323
2424 def maybe_output_banner_help_and_version
25 output('banner') if parsed_options[:banner]
26 output('help', help: option_parser.simple_help, simple: true) if parsed_options[:help]
27 output('help', help: option_parser.full_help, simple: false) if parsed_options[:hh]
28 output('version') if parsed_options[:version]
25 output('banner') if NS::ParsedCli.banner
26 output('help', help: option_parser.simple_help, simple: true) if NS::ParsedCli.help
27 output('help', help: option_parser.full_help, simple: false) if NS::ParsedCli.hh
28 output('version') if NS::ParsedCli.version
2929
30 exit(NS::ExitCode::OK) if parsed_options[:help] || parsed_options[:hh] || parsed_options[:version]
30 exit(NS::ExitCode::OK) if NS::ParsedCli.help || NS::ParsedCli.hh || NS::ParsedCli.version
3131 end
3232
3333 # Checks that the target is accessible, raises related errors otherwise
4242 when 401
4343 raise Error::HTTPAuthRequired
4444 when 403
45 raise Error::AccessForbidden, parsed_options[:random_user_agent]
45 raise Error::AccessForbidden, NS::ParsedCli.random_user_agent
4646 when 407
4747 raise Error::ProxyAuthRequired
4848 end
5353
5454 return if target.in_scope?(effective_url)
5555
56 raise Error::HTTPRedirect, effective_url unless parsed_options[:ignore_main_redirect]
56 raise Error::HTTPRedirect, effective_url unless NS::ParsedCli.ignore_main_redirect
5757
5858 target.homepage_res = res
5959 end
1414 end
1515
1616 def run
17 mode = parsed_options[:interesting_findings_detection] || parsed_options[:detection_mode]
17 mode = NS::ParsedCli.interesting_findings_detection || NS::ParsedCli.detection_mode
1818 findings = target.interesting_findings(mode: mode)
1919
2020 output('findings', findings: findings) unless findings.empty?
66 s.name = 'cms_scanner'
77 s.version = CMSScanner::VERSION
88 s.platform = Gem::Platform::RUBY
9 s.required_ruby_version = '>= 2.3'
9 s.required_ruby_version = '>= 2.4'
1010 s.authors = ['WPScanTeam']
1111 s.email = ['[email protected]']
1212 s.summary = 'CMS Scanner Framework (experimental)'
3131 s.add_development_dependency 'rake', '~> 12.3'
3232 s.add_development_dependency 'rspec', '~> 3.8.0'
3333 s.add_development_dependency 'rspec-its', '~> 1.2.0'
34 s.add_development_dependency 'rubocop', '~> 0.66.0'
34 s.add_development_dependency 'rubocop', '~> 0.67.1'
3535 s.add_development_dependency 'simplecov', '~> 0.16.1'
3636 s.add_development_dependency 'webmock', '~> 3.5.1'
3737 end
2020 s.executables = ['cmsscan']
2121 s.require_paths = ['lib']
2222
23 s.add_dependency 'cms_scanner', '~> 0.0.42.0'
23 s.add_dependency 'cms_scanner', '~> 0.0.44.1'
2424
2525 s.add_development_dependency 'bundler', '>= 1.6'
2626 s.add_development_dependency 'coveralls', '~> 0.8.0'
6161
6262 @user_agents = []
6363
64 # The user_agents_list is managed by the CLI options, with the default being
65 # APP_DIR/user_agents.txt
6466 File.open(user_agents_list).each do |line|
6567 next if line == "\n" || line[0, 1] == '#'
6668
6870 end
6971
7072 @user_agents
71 end
72
73 # @return [ String ] The path to the user agents list
74 def user_agents_list
75 @user_agents_list ||= File.join(APP_DIR, 'user_agents.txt')
7673 end
7774
7875 # @param [ value ] The throttle time in milliseconds
1313 def initialize(parsed_options = {})
1414 self.throttle = 0
1515
16 load_options(parsed_options)
16 load_options(parsed_options.dup)
1717 end
1818
1919 private_class_method :new
2121 # Reset all the class attibutes
2222 # Currently only used in specs
2323 def self.reset
24 @@target = nil
25 @@parsed_options = nil
26 @@datastore = nil
27 @@formatter = nil
24 @@target = nil
25 @@datastore = nil
26 @@formatter = nil
2827 end
2928
3029 # @return [ Target ]
3130 def target
32 @@target ||= NS::Target.new(parsed_options[:url], parsed_options)
31 @@target ||= NS::Target.new(NS::ParsedCli.url, NS::ParsedCli.options)
3332 end
3433
3534 # @param [ OptParsevalidator::OptParser ] parser
4241 @@option_parser
4342 end
4443
45 # Set the parsed options and initialize the browser
46 # with them
47 #
48 # @param [ Hash ] options
49 def self.parsed_options=(options)
50 @@parsed_options = options
51
52 NS::Browser.instance(options)
53 end
54
55 # @return [ Hash ]
56 def parsed_options
57 @@parsed_options ||= {}
58 end
59
6044 # @return [ Hash ]
6145 def datastore
6246 @@datastore ||= {}
6448
6549 # @return [ Formatter::Base ]
6650 def formatter
67 @@formatter ||= NS::Formatter.load(parsed_options[:format], datastore[:views])
51 @@formatter ||= NS::Formatter.load(NS::ParsedCli.format, datastore[:views])
6852 end
6953
7054 # @see Formatter#output
8367
8468 # @return [ Boolean ]
8569 def user_interaction?
86 formatter.user_interaction? && !parsed_options[:output]
70 formatter.user_interaction? && !NS::ParsedCli.output
8771 end
8872
8973 # @return [ String ]
10791
10892 # @return [ Hash ] All the instance variable keys (and their values) and the verbose value
10993 def instance_variable_values
110 h = { verbose: parsed_options[:verbose] }
94 h = { verbose: NS::ParsedCli.verbose }
11195 instance_variables.each do |a|
11296 s = a.to_s
11397 n = s[1..s.size]
3434 end
3535
3636 def run
37 parsed_options = option_parser.results
38 first.class.option_parser = option_parser
39 first.class.parsed_options = parsed_options
37 NS::ParsedCli.options = option_parser.results
38 first.class.option_parser = option_parser # To be able to output the help when -h/--hh
4039
41 redirect_output_to_file(parsed_options[:output]) if parsed_options[:output]
40 redirect_output_to_file(NS::ParsedCli.output) if NS::ParsedCli.output
4241
43 Timeout.timeout(parsed_options[:max_scan_duration], NS::Error::MaxScanDurationReached) do
42 Timeout.timeout(NS::ParsedCli.max_scan_duration, NS::Error::MaxScanDurationReached) do
4443 each(&:before_scan)
4544
4645 @running = true
4847 each(&:run)
4948 end
5049 ensure
51 Browser.instance.hydra.abort
50 NS::Browser.instance.hydra.abort
5251
5352 # Reverse is used here as the app/controllers/core#after_scan finishes the output
5453 # and must be the last one to be executed. It also guarantee that stats will be output
102102 'Request timed out.'
103103 elsif response.code.zero?
104104 "No response from remote server. WAF/IPS? (#{response.return_message})"
105 elsif response.code.to_s =~ /^50/
105 elsif /^50/.match?(response.code.to_s)
106106 'Server error, try reducing the number of threads.'
107107 else
108108 "Unknown response received Code: #{response.code}\nBody: #{response.body}"
0 # frozen_string_literal: true
1
2 module CMSScanner
3 # Class to hold the parsed CLI options and have them available via
4 # methods, such as #verbose?, rather than from the hash.
5 # This is similar to an OpenStruct, but class wise (rather than instance), and with
6 # the logic to update the Browser options accordinly
7 class ParsedCli
8 # @return [ Hash ]
9 def self.options
10 @options ||= {}
11 end
12
13 # Sets the CLI options, and put them into the Browser as well
14 # @param [ Hash ] options
15 def self.options=(options)
16 @options = options.dup || {}
17
18 NS::Browser.reset
19 NS::Browser.instance(@options)
20 end
21
22 # @return [ Boolean ]
23 def self.verbose?
24 options[:verbose] ? true : false
25 end
26
27 # Unknown methods will return nil, this is the expected behaviour
28 # rubocop:disable Style/MissingRespondToMissing
29 def self.method_missing(method_name, *_args, &_block)
30 super if method_name == :new
31
32 options[method_name.to_sym]
33 end
34 # rubocop:enable Style/MissingRespondToMissing
35 end
36 end
3131 formatter.output('@scan_aborted',
3232 reason: e.is_a?(Interrupt) ? 'Canceled by User' : e.message,
3333 trace: e.backtrace,
34 verbose: controllers.first.parsed_options[:verbose] ||
34 verbose: NS::ParsedCli.verbose ||
3535 run_error_exit_code == NS::ExitCode::EXCEPTION)
3636 ensure
3737 formatter.beautify
6060 at_exit do
6161 exit(run_error_exit_code) if run_error
6262
63 controller = controllers.first
64
6563 # The parsed_option[:url] must be checked to avoid raising erros when only -h/-v are given
66 exit(NS::ExitCode::VULNERABLE) if controller.parsed_options[:url] && controller.target.vulnerable?
64 exit(NS::ExitCode::VULNERABLE) if NS::ParsedCli.url && controllers.first.target.vulnerable?
6765 exit(NS::ExitCode::OK)
6866 end
6967 end
1919 # which can be huge (~ 2Go)
2020 res = head_and_get(path, [200], get: params.merge(headers: { 'Range' => 'bytes=0-700' }))
2121
22 res.body =~ pattern ? true : false
22 res.body&.match?(pattern) ? true : false
2323 end
2424
2525 # @param [ String ] path
11
22 # Version
33 module CMSScanner
4 VERSION = '0.0.43.2'
4 VERSION = '0.0.44.1'
55 end
2323 require 'cms_scanner/numeric' # Adds a Numeric#bytes_to_human
2424 # Custom Libs
2525 require 'cms_scanner/scan'
26 require 'cms_scanner/parsed_cli'
2627 require 'cms_scanner/helper'
2728 require 'cms_scanner/exit_code'
2829 require 'cms_scanner/errors'
00 # frozen_string_literal: true
11
22 describe CMSScanner::Controller::Core do
3 subject(:core) { described_class.new }
4 let(:target_url) { 'http://example.com/' }
5 let(:cli_args) { "--url #{target_url}" }
6 let(:parsed_options) { rspec_parsed_options(cli_args) }
3 subject(:core) { described_class.new }
4 let(:target_url) { 'http://example.com/' }
5 let(:cli_args) { "--url #{target_url}" }
76
87 before do
9 CMSScanner::Browser.reset
10 described_class.parsed_options = parsed_options
8 CMSScanner::ParsedCli.options = rspec_parsed_options(cli_args)
119 end
1210
1311 describe '#cli_options' do
1412 its(:cli_options) { should_not be_empty }
1513 its(:cli_options) { should be_a Array }
1614
17 it 'contaisn the expected options' do
15 it 'contains the expected options' do
1816 expect(core.cli_options.map(&:to_sym)).to match_array(
1917 %i[
2018 banner cache_dir cache_ttl clear_cache connect_timeout cookie_jar cookie_string
3634 context 'when cache_dir' do
3735 let(:cli_args) { "#{super()} --cache-dir #{CACHE}" }
3836 let(:cache) { Typhoeus::Config.cache }
39 let(:storage) { File.join(parsed_options[:cache_dir], Digest::MD5.hexdigest(target_url)) }
37 let(:storage) { File.join(CMSScanner::ParsedCli.cache_dir, Digest::MD5.hexdigest(target_url)) }
4038
4139 before { core.setup_cache }
4240 after { Typhoeus::Config.cache = nil }
260258 let(:cli_args) { "#{super()} --proxy-auth user:p@ss" }
261259
262260 it 'raises an error' do
263 expect(CMSScanner::Browser.instance.proxy_auth).to eq(parsed_options[:proxy_auth])
261 expect(CMSScanner::Browser.instance.proxy_auth).to eq(CMSScanner::ParsedCli.proxy_auth)
264262
265263 expect { core.before_scan }.to raise_error(CMSScanner::Error::ProxyAuthRequired)
266264 end
272270 let(:cli_args) { "#{super()} --proxy-auth user:pass" }
273271
274272 it 'raises an error' do
275 expect(CMSScanner::Browser.instance.proxy_auth).to eq(parsed_options[:proxy_auth])
273 expect(CMSScanner::Browser.instance.proxy_auth).to eq(CMSScanner::ParsedCli.proxy_auth)
276274
277275 expect { core.before_scan }.to_not raise_error
278276 end
33 subject(:controller) { described_class.new }
44 let(:target_url) { 'http://example.com/' }
55 let(:cli_args) { "--url #{target_url}" }
6 let(:parsed_options) { rspec_parsed_options(cli_args) }
76
87 before do
9 CMSScanner::Browser.reset
10 described_class.parsed_options = parsed_options
8 CMSScanner::ParsedCli.options = rspec_parsed_options(cli_args)
119 end
1210
1311 its(:before_scan) { should be_nil }
2624 before do
2725 expect(controller.target).to receive(:interesting_findings)
2826 .with(
29 mode: parsed_options[:interesting_findings_detection] || parsed_options[:detection_mode]
27 mode: CMSScanner::ParsedCli.interesting_findings_detection ||
28 CMSScanner::ParsedCli.detection_mode
3029 ).and_return(stubbed)
3130 end
3231
1313 let(:parsed_options) { { url: target_url, format: formatter.to_s.underscore.dasherize } }
1414
1515 before do
16 controller.class.parsed_options = parsed_options
16 CMSScanner::ParsedCli.options = parsed_options
1717 # Resets the formatter to ensure the correct one is loaded
1818 controller.class.class_variable_set(:@@formatter, nil)
1919 end
8080 expected = case sym
8181 when :user_agent
8282 browser.default_user_agent
83 when :user_agents_list
84 File.join(CMSScanner::APP_DIR, 'user_agents.txt')
8583 when :throttle
8684 0.0
8785 end
218216 context 'when --random-user-agent' do
219217 let(:options) { super().merge(random_user_agent: true) }
220218
221 it 'select a random UA in the user_agents' do
219 it 'selects a random UA in the user_agents' do
220 expect(browser).to receive(:user_agents_list).and_return(FIXTURES.join('user_agents.txt'))
221
222222 expect(browser.user_agent).to_not eql browser.default_user_agent
223
223224 # Should not pick up a random one each time
224225 expect(browser.user_agent).to eql browser.user_agent
225226 end
33 let(:target_url) { 'http://wp.lab/' }
44
55 before do
6 scanner = CMSScanner::Scan.new
7 scanner.controllers.first.class.parsed_options = rspec_parsed_options("--url #{target_url}")
6 CMSScanner::Scan.new # To initialize the start memory
7 CMSScanner::ParsedCli.options = rspec_parsed_options("--url #{target_url}")
88 end
99
1010 describe 'typhoeus memoize' do
22 describe CMSScanner::Controller do
33 subject(:controller) { described_class::Base.new }
44
5 context 'when parsed_options' do
6 before do
7 described_class::Base.option_parser = nil
8 described_class::Base.parsed_options = parsed_options
9 end
5 before do
6 described_class::Base.option_parser = nil
107
11 let(:parsed_options) { { url: 'http://example.com/' } }
8 CMSScanner::ParsedCli.options = parsed_options
9 end
1210
13 its(:option_parser) { should be nil }
14 its(:parsed_options) { should eq(parsed_options) }
15 its(:formatter) { should be_a CMSScanner::Formatter::Cli }
16 its(:user_interaction?) { should be true }
17 its(:tmp_directory) { should eql '/tmp/cms_scanner' }
18 its(:target) { should be_a CMSScanner::Target }
19 its('target.scope.domains') { should eq [PublicSuffix.parse('example.com')] }
11 let(:parsed_options) { { url: 'http://example.com/' } }
2012
21 context 'when output option' do
22 let(:parsed_options) { super().merge(output: '/tmp/spec.txt') }
13 its(:option_parser) { should be nil }
14 its(:formatter) { should be_a CMSScanner::Formatter::Cli }
15 its(:user_interaction?) { should be true }
16 its(:tmp_directory) { should eql '/tmp/cms_scanner' }
17 its(:target) { should be_a CMSScanner::Target }
18 its('target.scope.domains') { should eq [PublicSuffix.parse('example.com')] }
2319
24 its(:user_interaction?) { should be false }
25 end
20 context 'when output option' do
21 let(:parsed_options) { super().merge(output: '/tmp/spec.txt') }
2622
27 describe '#render' do
28 it 'calls the formatter#render' do
29 expect(controller.formatter).to receive(:render).with('test', { verbose: nil }, 'base')
30 controller.render('test')
31 end
23 its(:user_interaction?) { should be false }
24 end
25
26 describe '#render' do
27 it 'calls the formatter#render' do
28 expect(controller.formatter).to receive(:render).with('test', { verbose: nil }, 'base')
29 controller.render('test')
3230 end
3331 end
3432 end
33 module Controller
44 class Spec < Base
55 def before_scan
6 output('help', help: option_parser.simple_help, simple: true) if parsed_options[:help]
6 output('help', help: option_parser.simple_help, simple: true) if NS::ParsedCli.help
77
8 exit(NS::ExitCode::OK) if parsed_options[:help]
8 exit(NS::ExitCode::OK) if NS::ParsedCli.help
99 end
1010 end
1111
5757 [base, spec].each { |c| expect(c).to receive(:before_scan).ordered }
5858 [base, spec].each { |c| expect(c).to receive(:run).ordered }
5959
60 expect(hydra).to receive(:abort).ordered
60 expect_any_instance_of(Typhoeus::Hydra).to receive(:abort)
6161
6262 [spec, base].each { |c| expect(c).to receive(:after_scan).ordered }
6363
7979 .ordered
8080 .with('help', hash_including(:help, :simple), 'spec')
8181
82 expect(hydra).to receive(:abort).ordered
82 expect_any_instance_of(Typhoeus::Hydra).to receive(:abort)
8383
8484 expect { controllers.run }.to raise_error(SystemExit)
8585 end
9797 let(:max_scan_duration) { 1 }
9898
9999 it 'raises an exception' do
100 expect(hydra).to receive(:abort).ordered
100 expect_any_instance_of(Typhoeus::Hydra).to receive(:abort)
101101
102102 controllers.reverse_each { |c| expect(c).to receive(:after_scan).ordered }
103103
0 # frozen_string_literal: true
1
2 describe CMSScanner::ParsedCli do
3 subject(:parsed_cli) { described_class }
4 let(:options) { { key: 'value', cache_ttl: 10 } }
5
6 describe '#options=' do
7 it 'sets them, reset the Browser and pass them to it' do
8 expect(CMSScanner::Browser.instance.cache_ttl).to eql nil # Not yet set
9
10 parsed_cli.options = options
11 expect(CMSScanner::Browser.instance.cache_ttl).to eql 10
12 end
13
14 context 'when the options are modified from the top after being passed' do
15 it 'does not modify them' do
16 parsed_cli.options = options
17
18 options[:key3] = 'value3'
19
20 expect(parsed_cli.options).to eql(key: 'value', cache_ttl: 10)
21 end
22 end
23
24 context 'when passing nil' do
25 it 'sets an empty hash' do
26 parsed_cli.options = nil
27
28 expect(parsed_cli.options).to eql({})
29 end
30 end
31 end
32
33 describe '.options, .verbose etc' do
34 it 'has the correct values' do
35 parsed_cli.options = options
36
37 expect(parsed_cli.options).to eql options
38
39 expect(parsed_cli.verbose?).to be false
40
41 expect(parsed_cli.key).to eql 'value'
42 expect(parsed_cli.cache_ttl).to eql 10
43
44 expect(parsed_cli.nothing).to eql nil
45 end
46 end
47 end
5656 let(:target_url) { 'http://ex.lo/' }
5757
5858 before do
59 scanner.controllers.first.class.parsed_options = { url: target_url }
59 SubScanner::ParsedCli.options = { url: target_url }
6060 end
6161
6262 describe '#app_name' do
11
22 shared_examples CMSScanner::Target::Platform::PHP do
33 before do
4 if path =~ /\.log\z/i
4 if /\.log\z/i.match?(path)
55 expect(target).to receive(:head_or_get_params).and_return(method: :head)
66
77 stub_request(:head, target.url(path)).and_return(status: head_status)
1515 RSpec.configure do |config|
1616 config.expect_with :rspec do |c|
1717 c.syntax = :expect
18 end
19
20 config.before(:each) do
21 # Needed for rspec to run w/o error due to the at_exit hook calling the controller#target
22 CMSScanner::Controller::Core.parsed_options = { url: 'http://ex.lo' }
2318 end
2419 end
2520