1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
|
module Aws
class ResourceNotFoundError < StandardError
end
# Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
# web services raise this type of error.
# Attribute inherited by RuntimeError:
# message - the text of the error, generally as returned by AWS in its XML response.
class AwsError < RuntimeError
# either an array of errors where each item is itself an array of [code, message]),
# or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
attr_reader :errors
# Request id (if exists)
attr_reader :request_id
# Response HTTP error code
attr_reader :http_code
# Raw request text data to AWS
attr_reader :request_data
attr_reader :response
def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil, response=nil)
@errors = errors
@request_id = request_id
@http_code = http_code
@request_data = request_data
@response = response
msg = @errors.is_a?(Array) ? @errors.map { |code, m| "#{code}: #{m}" }.join("; ") : @errors.to_s
msg += "\nREQUEST=#{@request_data} " unless @request_data.nil?
msg += "\nREQUEST ID=#{@request_id} " unless @request_id.nil?
super(msg)
end
# Does any of the error messages include the regexp +pattern+?
# Used to determine whether to retry request.
def include?(pattern_or_string)
if pattern_or_string.is_a?(String)
puts 'convert to pattern'
pattern_or_string = /#{pattern_or_string}/
end
if @errors.is_a?(Array)
@errors.each { |code, msg| return true if code =~ pattern_or_string }
else
return true if @errors_str =~ pattern_or_string
end
false
end
# Generic handler for AwsErrors. +aws+ is the Aws::S3, Aws::EC2, or Aws::SQS
# object that caused the exception (it must provide last_request and last_response). Supported
# boolean options are:
# * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
# * <tt>:puts</tt> do a "puts" of the error
# * <tt>:raise</tt> re-raise the error after logging
def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
# Only log & notify if not user error
if !options[:raise] || system_error?($!)
error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
puts error_text if options[:puts]
# Log the error
if options[:log]
request = aws.last_request ? aws.last_request.path : '-none-'
response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
@response = response
aws.logger.error error_text
aws.logger.error "Request was: #{request}"
aws.logger.error "Response was: #{response}"
end
end
raise if options[:raise] # re-raise an exception
return nil
end
# True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
# Used to force logging.
def self.system_error?(e)
!e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
end
end
# Simplified version
class AwsError2 < RuntimeError
# Request id (if exists)
attr_reader :request_id
# Response HTTP error code
attr_reader :http_code
# Raw request text data to AWS
attr_reader :request_data
attr_reader :response
attr_reader :errors
def initialize(http_code=nil, request_id=nil, request_data=nil, response=nil)
@request_id = request_id
@http_code = http_code
@request_data = request_data
@response = response
# puts '@response=' + @response.inspect
if @response
ref = XmlSimple.xml_in(@response, {"ForceArray"=>false})
# puts "refxml=" + ref.inspect
msg = "#{ref['Error']['Code']}: #{ref['Error']['Message']}"
else
msg = "#{@http_code}: REQUEST(#{@request_data})"
end
msg += "\nREQUEST ID=#{@request_id} " unless @request_id.nil?
super(msg)
end
end
class AWSErrorHandler
# 0-100 (%)
DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
@@reiteration_start_delay = 0.2
def self.reiteration_start_delay
@@reiteration_start_delay
end
def self.reiteration_start_delay=(reiteration_start_delay)
@@reiteration_start_delay = reiteration_start_delay
end
@@reiteration_time = 5
def self.reiteration_time
@@reiteration_time
end
def self.reiteration_time=(reiteration_time)
@@reiteration_time = reiteration_time
end
@@close_on_error = true
def self.close_on_error
@@close_on_error
end
def self.close_on_error=(close_on_error)
@@close_on_error = close_on_error
end
@@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
def self.close_on_4xx_probability
@@close_on_4xx_probability
end
def self.close_on_4xx_probability=(close_on_4xx_probability)
@@close_on_4xx_probability = close_on_4xx_probability
end
# params:
# :reiteration_time
# :errors_list
# :close_on_error = true | false
# :close_on_4xx_probability = 1-100
def initialize(aws, parser, params={}) #:nodoc:
@aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
@parser = parser # parser to parse Amazon response
@started_at = Time.now
@stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
@errors_list = params[:errors_list] || []
@reiteration_delay = @@reiteration_start_delay
@retries = 0
# close current HTTP(S) connection on 5xx, errors from list and 4xx errors
@close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
@close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
end
# Returns false if
def check(request, options={}) #:nodoc:
result = false
error_found = false
redirect_detected = false
error_match = nil
last_errors_text = ''
response = @aws.last_response
# log error
request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
# is this a redirect?
# yes!
if response.is_a?(Net::HTTPRedirection)
redirect_detected = true
else
# no, it's an error ...
@aws.logger.debug("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
@aws.logger.debug("##### #{@aws.class.name} request: #{request_text_data} ####")
end
# Check response body: if it is an Amazon XML document or not:
if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
@aws.class.bench_xml.add! do
error_parser = RightErrorResponseParser.new
error_parser.parse(response)
@aws.last_errors = error_parser.errors
@aws.last_request_id = error_parser.requestID
last_errors_text = @aws.last_errors.flatten.join("\n")
# on redirect :
if redirect_detected
location = response['location'] || "#{request[:protocol]}://#{error_parser.instance_variable_get(:'@endpoint')}"
# ... log information and ...
@aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
@aws.logger.info("##### New location: #{location} #####")
# ... fix the connection data
request[:server] = URI.parse(location).host
request[:protocol] = URI.parse(location).scheme
request[:port] = URI.parse(location).port
end
end
else # ... it is not a xml document(probably just a html page?)
@aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
@aws.last_request_id = '-undefined-'
last_errors_text = response.message
end
# now - check the error
unless redirect_detected
@errors_list.each do |error_to_find|
if last_errors_text[/#{error_to_find}/i]
error_found = true
error_match = error_to_find
@aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
break
end
end
end
# check the time has gone from the first error come
if redirect_detected || error_found
# Close the connection to the server and recreate a new one.
# It may have a chance that one server is a semi-down and reconnection
# will help us to connect to the other server
if !redirect_detected && @close_on_error
@aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
end
# puts 'OPTIONS3=' + options.inspect
if options[:retries].nil? || @retries < options[:retries]
if (Time.now < @stop_at)
@retries += 1
unless redirect_detected
@aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
sleep @reiteration_delay
@reiteration_delay *= 2
# Always make sure that the fp is set to point to the beginning(?)
# of the File/IO. TODO: it assumes that offset is 0, which is bad.
if (request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
begin
request[:request].body_stream.pos = 0
rescue Exception => e
@logger.warn("Retry may fail due to unable to reset the file pointer" +
" -- #{self.class.name} : #{e.inspect}")
end
end
else
@aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
end
result = @aws.request_info(request, @parser, options)
else
@aws.logger.warn("##### Ooops, time is over... ####")
end
else
@aws.logger.info("##### Stopped retrying because retries=#{@retries} and max=#{options[:retries]} ####")
end
# aha, this is unhandled error:
elsif @close_on_error
# Is this a 5xx error ?
if @aws.last_response.code.to_s[/^5\d\d$/]
@aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
# Is this a 4xx error ?
elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
@aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
"probability: #{@close_on_4xx_probability}%"
end
end
result
end
end
end
|