Commit 84fd21cbc36a0c739238050885da345ffc92f8c9 - ruby-cms-scanner

+8

-8

.rubocop.yml less more

4	4	- '*.gemspec'
5	5	- 'vendor/*/'
6	6	- 'example/*/'
7		ClassVars:
8		Enabled: false
9		LineLength:
10		Max: 120
11		MethodLength:
12		Max: 18
13		Exclude:
14		- app/controllers/core/cli_options.rb
15	7	Lint/UriEscapeUnescape:
16	8	Enabled: false
17	9	Metrics/AbcSize:

21	13	- 'spec/*/'
22	14	Metrics/CyclomaticComplexity:
23	15	Max: 10
	16	Metrics/LineLength:
	17	Max: 120
	18	Metrics/MethodLength:
	19	Max: 18
	20	Exclude:
	21	- app/controllers/core/cli_options.rb
24	22	Metrics/PerceivedComplexity:
25	23	Max: 9
	24	Style/ClassVars:
	25	Enabled: false
26	26	Style/Documentation:
27	27	Enabled: false
28	28	Style/FormatStringToken:

+7

-4

.travis.yml less more

7	7	- 2.4.4
8	8	- 2.4.5
9	9	- 2.4.6
	10	- 2.4.7
	11	- 2.4.8
	12	- 2.4.9
10	13	- 2.5.0
11	14	- 2.5.1
12	15	- 2.5.2
13	16	- 2.5.3
14	17	- 2.5.4
15	18	- 2.5.5
	19	- 2.5.6
	20	- 2.5.7
16	21	- 2.6.0
17	22	- 2.6.1
18	23	- 2.6.2
19	24	- 2.6.3
20		- ruby-head
	25	- 2.6.4
	26	- 2.6.5
21	27	before_install:
22	28	- "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
23	29	- gem update --system
24		matrix:
25		allow_failures:
26		- rvm: ruby-head
27	30	script:
28	31	- bundle exec rubocop
29	32	- bundle exec rspec

+1

-1

app/controllers/core.rb less more

68	68	def after_scan
69	69	@stop_time = Time.now
70	70	@elapsed = @stop_time - @start_time
71		@used_memory = memory_usage - @start_memory
	71	@used_memory = GetProcessMem.new.bytes - @start_memory
72	72
73	73	output('finished',
74	74	cached_requests: NS.cached_requests,

+1

-1

app/finders/interesting_findings/fantastico_fileslist.rb less more

10	10	res = target.head_and_get(path)
11	11
12	12	return if res.body.strip.empty?
13		return unless res.headers && res.headers['Content-Type'] =~ %r{\Atext/plain}
	13	return unless res.headers && res.headers['Content-Type']&.start_with?('text/plain')
14	14
15	15	NS::Model::FantasticoFileslist.new(target.url(path), confidence: 70, found_by: found_by)
16	16	end

+8

-5

cms_scanner.gemspec less more

18	18	s.test_files = []
19	19	s.require_paths = ['lib']
20	20
	21	s.add_dependency 'get_process_mem', '~> 0.2.5'
21	22	s.add_dependency 'nokogiri', '~> 1.10.4'
22		s.add_dependency 'opt_parse_validator', '~> 1.8.0'
	23	s.add_dependency 'opt_parse_validator', '~> 1.8.1'
23	24	s.add_dependency 'public_suffix', '>= 3.0', '< 4.1'
24	25	s.add_dependency 'ruby-progressbar', '~> 1.10.0'
25	26	s.add_dependency 'typhoeus', '~> 1.3.0'
26	27	s.add_dependency 'xmlrpc', '~> 0.3'
27	28	s.add_dependency 'yajl-ruby', '~> 1.4.1' # Better JSON parser regarding memory usage
28	29
	30	s.add_dependency 'sys-proctable', '~> 1.2.2' # Required by get_process_mem for Windows OS.
	31
29	32	s.add_development_dependency 'bundler', '>= 1.6'
30	33	s.add_development_dependency 'coveralls', '~> 0.8.0'
31		s.add_development_dependency 'rake', '~> 12.3'
32		s.add_development_dependency 'rspec', '~> 3.8.0'
	34	s.add_development_dependency 'rake', '~> 13.0'
	35	s.add_development_dependency 'rspec', '~> 3.9.0'
33	36	s.add_development_dependency 'rspec-its', '~> 1.3.0'
34		s.add_development_dependency 'rubocop', '~> 0.74.0'
35		s.add_development_dependency 'rubocop-performance', '~> 1.4.0'
	37	s.add_development_dependency 'rubocop', '~> 0.76.0'
	38	s.add_development_dependency 'rubocop-performance', '~> 1.5.0'
36	39	s.add_development_dependency 'simplecov', '~> 0.16.1'
37	40	s.add_development_dependency 'webmock', '~> 3.7.0'
38	41	end

+14

-13

example/.rubocop.yml less more

	0	require: rubocop-performance
0	1	AllCops:
1		TargetRubyVersion: 2.3
	2	TargetRubyVersion: 2.4
2	3	Exclude:
3		- '*.gemspec'
4		- 'vendor/*/'
5		LineLength:
	4	- '*.gemspec'
	5	- 'vendor/*/'
	6	Metrics/AbcSize:
	7	Max: 25
	8	Metrics/CyclomaticComplexity:
	9	Max: 10
	10	Metrics/LineLength:
6	11	Max: 120
7		ClassVars:
	12	Metrics/MethodLength:
	13	Max: 17
	14	Metrics/PerceivedComplexity:
	15	Max: 9
	16	Style/ClassVars:
8	17	Enabled: false
9	18	Style/RescueModifier:
10	19	Enabled: false
11	20	Style/SignalException:
12	21	EnforcedStyle: semantic
13		MethodLength:
14		Max: 17
15		Metrics/AbcSize:
16		Max: 25
17		Metrics/CyclomaticComplexity:
18		Max: 10
19		Metrics/PerceivedComplexity:
20		Max: 9

+3

-19

example/.travis.yml less more

1	1	sudo: false
2	2	cache: bundler
3	3	rvm:
4		- 2.3.0
5		- 2.3.1
6		- 2.3.2
7		- 2.3.3
8		- 2.3.4
9		- 2.3.5
10		- 2.3.6
11		- 2.3.7
12		- 2.3.8
13		- 2.4.1
14		- 2.4.2
15		- 2.4.3
16		- 2.4.4
17		- 2.4.5
18		- 2.5.0
19		- 2.5.1
20		- 2.5.2
21		- 2.5.3
22		- 2.6.0
	4	- 2.4.9
	5	- 2.5.7
	6	- 2.6.5
23	7	- ruby-head
24	8	before_install:
25	9	- "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"

+12

-9

example/cmsscan.gemspec less more

20	20	s.executables = ['cmsscan']
21	21	s.require_paths = ['lib']
22	22
23		s.add_dependency 'cms_scanner', '~> 0.0.44.1'
	23	s.add_dependency 'cms_scanner', '~> 0.6.2'
24	24
25		s.add_development_dependency 'bundler', '>= 1.6'
26		s.add_development_dependency 'coveralls', '~> 0.8.0'
27		s.add_development_dependency 'rake', '~> 12.3'
28		s.add_development_dependency 'rspec', '~> 3.8.0'
29		s.add_development_dependency 'rspec-its', '~> 1.2.0'
30		s.add_development_dependency 'rubocop', '~> 0.66.0'
31		s.add_development_dependency 'simplecov', '~> 0.16.1'
32		s.add_development_dependency 'webmock', '~> 3.5.1'
	25	s.add_development_dependency 'bundler', '>= 1.6'
	26	s.add_development_dependency 'coveralls', '~> 0.8.0'
	27	s.add_development_dependency 'memory_profiler', '~> 0.9.13'
	28	s.add_development_dependency 'rake', '~> 13.0'
	29	s.add_development_dependency 'rspec', '~> 3.9.0'
	30	s.add_development_dependency 'rspec-its', '~> 1.3.0'
	31	s.add_development_dependency 'rubocop', '~> 0.76.0'
	32	s.add_development_dependency 'rubocop-performance', '~> 1.5.0'
	33	s.add_development_dependency 'simplecov', '~> 0.16.1'
	34	s.add_development_dependency 'stackprof', '~> 0.2.12'
	35	s.add_development_dependency 'webmock', '~> 3.7.0'
33	36	end

+10

-4

lib/cms_scanner/finders/finder.rb less more

19	19
20	20	# @return [ String ] The titleized name of the finder
21	21	def titleize
22		self.class.to_s.demodulize.underscore.titleize
	22	# Put a _ char before any digits except those at the end, which will be replaced by a space
	23	# Otherwise, class such as Error404Page are returned as Error404 Page instead of Error 404 page
	24	# The keep_id_suffix is to concevert classes such as CssId to Css Id instead of Css
	25
	26	@titleize \|\|= self.class.to_s.demodulize.gsub(/(\d+)[a-z]+/i, '_\0').titleize(keep_id_suffix: true)
23	27	end
24	28
25	29	# @param [ Hash ] _opts

49	53	@hydra \|\|= browser.hydra
50	54	end
51	55
52		# @param [ String, Symbol ] klass
	56	# @param [String, Class ] klass
53	57	# @return [ String ]
54		def found_by(klass = self)
	58	def found_by(klass = self.class)
55	59	caller_locations.each do \|call\|
56	60	label = call.label
57	61
58	62	next unless %w[aggressive passive].include? label
59	63
60		return "#{klass.titleize} (#{label.capitalize} Detection)"
	64	title = klass.to_s.demodulize.gsub(/(\d+)[a-z]+/i, '_\0').titleize(keep_id_suffix: true)
	65
	66	return "#{title} (#{label.capitalize} Detection)"
61	67	end
62	68	nil
63	69	end

+0

-5

lib/cms_scanner/helper.rb less more

4	4	$stdout.reopen(file, 'w')
5	5	$stdout.sync = true
6	6	end
7
8		# @return [ Integer ] The memory of the current process in Bytes
9		def memory_usage
10		`ps -o rss= -p #{Process.pid}`.to_i * 1024 # ps returns the value in KB
11		end

+1

-1

lib/cms_scanner/numeric.rb less more

5	5	def bytes_to_human
6	6	units = %w[B KB MB GB TB]
7	7	e = abs.zero? ? abs : (Math.log(abs) / Math.log(1024)).floor
8		s = format('%.3f', (abs.to_f / 1024**e))
	8	s = format('%<s>.3f', s: (abs.to_f / 1024**e))
9	9
10	10	s.sub(/\.?0*$/, ' ' + units[e])
11	11	end

+1

-1

lib/cms_scanner/scan.rb less more

5	5	attr_reader :run_error
6	6
7	7	def initialize
8		NS.start_memory = memory_usage
	8	NS.start_memory = GetProcessMem.new.bytes
9	9
10	10	controllers << NS::Controller::Core.new
11	11

+1

-6

lib/cms_scanner/target/hashes.rb less more

26	26	# @note This is used to detect potential custom 404 responding with a 200
27	27	# @return [ String ] The hash of a 404
28	28	def error_404_hash
29		@error_404_hash \|\|= self.class.page_hash(non_existant_page_url)
30		end
31
32		# @return [ String ] The URL of an unlikely existant page
33		def non_existant_page_url
34		uri.join(Digest::MD5.hexdigest(rand(999_999_999).to_s) + '.html').to_s
	29	@error_404_hash \|\|= self.class.page_hash(error_404_res)
35	30	end
36	31
37	32	# @param [ Typhoeus::Response, String ] page

+1

-1

lib/cms_scanner/version.rb less more

1	1
2	2	# Version
3	3	module CMSScanner
4		VERSION = '0.6.0'
	4	VERSION = '0.7.1'
5	5	end

+16

-0

lib/cms_scanner/web_site.rb less more

52	52	# @return [ String ]
53	53	def homepage_url
54	54	@homepage_url \|\|= homepage_res.effective_url
	55	end
	56
	57	# @return [ Typhoeus::Response ]
	58	def error_404_res
	59	@error_404_res \|\|= NS::Browser.get(error_404_url)
	60	end
	61
	62	# @return [ String ] The URL of an unlikely existant page
	63	def error_404_url
	64	@error_404_url \|\|= non_existant_page_url
	65	end
	66
	67	# @return [ String ] The URL of an unlikely existant page
	68	# TODO: This will be removed in the next major version (0.7)
	69	def non_existant_page_url
	70	uri.join(Digest::MD5.hexdigest(rand(999_999).to_s)[0..6] + '.html').to_s
55	71	end
56	72
57	73	# Checks if the remote website is up.

+1

-0

lib/cms_scanner.rb less more

5	5	require 'yajl/json_gem'
6	6	require 'public_suffix'
7	7	require 'addressable/uri'
	8	require 'get_process_mem'
8	9	require 'ruby-progressbar'
9	10	require 'opt_parse_validator'
10	11	require 'active_support/concern'

+10

-1

spec/app/finders/interesting_findings/fantastico_fileslist_spec.rb less more

30	30	end
31	31	end
32	32
	33	context 'when no Content-Type header' do
	34	it 'return nil' do
	35	stub_request(:get, file_url)
	36	.to_return(status: 200, body: 'not empty', headers: {})
	37
	38	expect(finder.aggressive).to eql nil
	39	end
	40	end
	41
33	42	context 'when not a text/plain Content-Type' do
34	43	it 'return nil' do
35	44	stub_request(:get, file_url)
36		.to_return(status: 200, body: 'not empty', headers: { 'Content-Type' => 'text/html ' })
	45	.to_return(status: 200, body: 'not empty', headers: { 'Content-Type' => 'text/html' })
37	46
38	47	expect(finder.aggressive).to eql nil
39	48	end

+43

-19

spec/lib/finders/finder_spec.rb less more

1	1
2	2	describe CMSScanner::Finders::Finder do
3	3	subject(:finder) { described_class.new('target') }
	4
	5	its(:titleize) { should eql 'Finder' }
	6	its(:browser) { should be_a CMSScanner::Browser }
	7	its(:hydra) { should be_a Typhoeus::Hydra }
4	8
5	9	describe '#create_progress_bar' do
6	10	before { finder.create_progress_bar(opts) }

39	43	end
40	44	end
41	45
42		its(:browser) { should be_a CMSScanner::Browser }
	46	class SpecCallerLocation
	47	attr_reader :call
43	48
44		its(:hydra) { should be_a Typhoeus::Hydra }
	49	def initialize(call)
	50	@call = call
	51	end
	52
	53	def label
	54	@label \|\|= call[/`([^']+)'$/, 1]
	55	end
	56
	57	def to_s
	58	call
	59	end
	60	end
45	61
46	62	describe '#found_by' do
47	63	context 'when no klass supplied' do

53	69	end
54	70	end
55	71
56		# TODO: make the below work
57		# context 'when aggressive match' do
58		# it 'returns the expected string' do
59		# expect(finder).to receive(:caller_locations)
60		# .and_return([Thread::Backtrace::Location.new("/aaaaa/file.rb:xx:in `aggressive'")])
61		#
62		# expect(finder.found_by).to eql 'Finder (Aggressive Detection)'
63		# end
64		# end
	72	context 'when aggressive match' do
	73	it 'returns the expected string' do
	74	expect(finder).to receive(:caller_locations)
	75	.and_return([SpecCallerLocation.new("/aaaaa/file.rb:xx:in `aggressive'")])
	76
	77	expect(finder.found_by).to eql 'Finder (Aggressive Detection)'
	78	end
	79	end
65	80	end
66	81
67		# context 'when class supplied' do
68		# it 'returns the expected string' do
69		# expect(finder).to receive(:caller_locations)
70		# .and_return(["/aaaaa/file.rb:xx:in `passive'"])
71		#
72		# expect(finder.found_by('Rspec')).to eql 'Rspec (Passive Detection)'
73		# end
74		# end
	82	{
	83	Rspec: 'Rspec', Error404Page: 'Error 404 Page',
	84	CssId: 'Css Id', Something12Db2: 'Something 12 Db2'
	85	}.each do \|klass, expected_title\|
	86	context "when class #{klass} supplied" do
	87	it 'returns the expected string' do
	88	allow(finder).to receive(:caller_locations)
	89	.and_return([SpecCallerLocation.new("/aaaaa/file.rb:xx:in `passive'")])
	90
	91	expected = "#{expected_title} (Passive Detection)"
	92
	93	# klass = Object.const_set(klass_name, Class.new(described_class))
	94
	95	expect(finder.found_by(klass)).to eql expected
	96	end
	97	end
	98	end
75	99	end
76	100	end

+1

-1

spec/lib/target/hashes_spec.rb less more

66	66
67	67	describe '#error_404_hash' do
68	68	it 'returns the md5sum of the 404 page' do
69		stub_request(:any, /.*/).to_return(status: 404, body: '404 page !')
	69	stub_request(:any, ERROR_404_URL_PATTERN).to_return(status: 404, body: '404 page !')
70	70
71	71	expect(target.error_404_hash).to eql md5sum('404 page !')
72	72	end

+11

-0

spec/lib/web_site_spec.rb less more

60	60	expect(web_site.url('/sub/file.txt')).to eql 'http://e.org/sub/file.txt'
61	61	end
62	62	end
	63	end
	64	end
	65
	66	describe '#error_404_url' do
	67	its(:error_404_url) { should match ERROR_404_URL_PATTERN }
	68
	69	it 'returns the same url when called more than once' do
	70	url1 = web_site.error_404_url
	71	url2 = web_site.error_404_url
	72
	73	expect(url1).to eql url2
63	74	end
64	75	end
65	76

+9

-8

spec/spec_helper.rb less more

77	77	end
78	78	# rubocop:enabled all
79	79
80		SPECS = Pathname.new(__FILE__).dirname
81		CACHE = SPECS.join('cache')
82		FIXTURES = SPECS.join('fixtures')
83		FIXTURES_VIEWS = FIXTURES.join('views')
84		FIXTURES_FINDERS = FIXTURES.join('finders')
85		FIXTURES_MODELS = FIXTURES.join('models')
86		FIXTURES_CONTROLLERS = FIXTURES.join('controllers')
87		APP_VIEWS = File.join(CMSScanner::APP_DIR, 'views')
	80	SPECS = Pathname.new(__FILE__).dirname
	81	CACHE = SPECS.join('cache')
	82	FIXTURES = SPECS.join('fixtures')
	83	FIXTURES_VIEWS = FIXTURES.join('views')
	84	FIXTURES_FINDERS = FIXTURES.join('finders')
	85	FIXTURES_MODELS = FIXTURES.join('models')
	86	FIXTURES_CONTROLLERS = FIXTURES.join('controllers')
	87	APP_VIEWS = File.join(CMSScanner::APP_DIR, 'views')
	88	ERROR_404_URL_PATTERN = %r{/[a-z\d]{7}\.html$}