38 lines
888 B
Ruby
38 lines
888 B
Ruby
# frozen_string_literal: true
|
|
|
|
module Gitlab
|
|
module RobotsTxt
|
|
class Parser
|
|
attr_reader :disallow_rules
|
|
|
|
def initialize(content)
|
|
@raw_content = content
|
|
|
|
@disallow_rules = parse_raw_content!
|
|
end
|
|
|
|
def disallowed?(path)
|
|
disallow_rules.any? { |rule| path =~ rule }
|
|
end
|
|
|
|
private
|
|
|
|
# This parser is very basic as it only knows about `Disallow:` lines,
|
|
# and simply ignores all other lines.
|
|
#
|
|
# Order of predecence, 'Allow:`, etc are ignored for now.
|
|
def parse_raw_content!
|
|
@raw_content.each_line.map do |line|
|
|
if line.start_with?('Disallow:')
|
|
value = line.sub('Disallow:', '').strip
|
|
value = Regexp.escape(value).gsub('\*', '.*')
|
|
Regexp.new("^#{value}")
|
|
else
|
|
nil
|
|
end
|
|
end.compact
|
|
end
|
|
end
|
|
end
|
|
end
|