File: recursive_hot_file_loader.rb

package info (click to toggle)
ruby-fog-openstack 1.1.5-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 11,784 kB
  • sloc: ruby: 47,937; makefile: 5; sh: 4
file content (220 lines) | stat: -rw-r--r-- 7,903 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
require 'set'
require 'yaml'
require 'open-uri'
require 'objspace'
require 'fog/core'

module Fog
  module OpenStack
    module OrchestrationUtil
      #
      # Resolve get_file resources found in a HOT template populating
      #  a files Hash conforming to Heat Specs
      #  https://developer.openstack.org/api-ref/orchestration/v1/index.html?expanded=create-stack-detail#stacks
      #
      # Files present in :files are not processed further. The others
      #   are added to the Hash. This behavior is the same implemented in openstack-infra/shade
      #   see https://github.com/openstack-infra/shade/blob/1d16f64fbf376a956cafed1b3edd8e51ccc16f2c/shade/openstackcloud.py#L1200
      #
      # This implementation just process nested templates but not resource
      #  registries.
      class RecursiveHotFileLoader
        attr_reader :files
        attr_reader :template

        def initialize(template, files = nil)
          # According to https://github.com/fog/fog-openstack/blame/master/docs/orchestration.md#L122
          #  templates can be either String or Hash.
          #  If it's an Hash, we deep_copy it so the passed argument
          #  is not modified by get_file_contents.
          template = deep_copy(template)
          @visited = Set.new
          @files = files || {}
          @template = get_template_contents(template)
        end

        # Return string
        def url_join(prefix, suffix)
          if prefix
            # URI.join replaces prefix parts before a
            #  trailing slash. See https://docs.ruby-lang.org/en/2.3.0/URI.html.
            prefix += '/' unless prefix.to_s.end_with?("/")
            suffix = URI.join(prefix, suffix)
            # Force URI to use traditional file scheme representation.
            suffix.host = "" if suffix.scheme == "file"
          end
          suffix.to_s
        end

        # Retrieve a template content.
        #
        # @param template_file can be either:
        #          - a raw_template string
        #          - an URI string
        #          - an Hash containing the parsed template.
        #
        # XXX: we could use named parameters
        # and better mimic heatclient implementation.
        def get_template_contents(template_file)
          Fog::Logger.debug("get_template_contents [#{template_file}]")

          raise "template_file should be Hash or String, not #{template_file.class.name}" unless
            template_file.kind_of?(String) || template_file.kind_of?(Hash)

          local_base_url = url_join("file:/", File.absolute_path(Dir.pwd))

          if template_file.kind_of?(Hash)
            template_base_url = local_base_url
            template = template_file
          elsif template_is_raw?(template_file)
            template_base_url = local_base_url
            template = YAML.safe_load(template_file, :permitted_classes => [Date])
          elsif template_is_url?(template_file)
            template_file = normalise_file_path_to_url(template_file)
            template_base_url = base_url_for_url(template_file)
            raw_template = read_uri(template_file)
            template = YAML.safe_load(raw_template, :permitted_classes => [Date])

            Fog::Logger.debug("Template visited: #{@visited}")
            @visited.add(template_file)
          else
            raise "template_file is not a string of the expected form"
          end

          get_file_contents(template, template_base_url)

          template
        end

        # Traverse the template tree looking for get_file and type
        #   and populating the @files attribute with their content.
        #   Resource referenced by get_file and type are eventually
        #   replaced with their absolute URI as done in heatclient
        #   and shade.
        #
        def get_file_contents(from_data, base_url)
          Fog::Logger.debug("Processing #{from_data} with base_url #{base_url}")

          # Recursively traverse the tree
          #   if recurse_data is Array or Hash
          recurse_data = from_data.kind_of?(Hash) ? from_data.values : from_data
          recurse_data.each { |value| get_file_contents(value, base_url) } if recurse_data.kind_of?(Array)

          # I'm on a Hash, process it.
          return unless from_data.kind_of?(Hash)
          from_data.each do |key, value|
            next if ignore_if(key, value)

            # Resolve relative paths.
            str_url = url_join(base_url, value)

            next if @files.key?(str_url)

            file_content = read_uri(str_url)

            # get_file should not recurse hot templates.
            if key == "type" && template_is_raw?(file_content) && !@visited.include?(str_url)
              template = get_template_contents(str_url)
              file_content = YAML.dump(template)
            end

            @files[str_url] = file_content
            # replace the data value with the normalised absolute URL as required
            #  by https://docs.openstack.org/heat/pike/template_guide/hot_spec.html#get-file
            from_data[key] = str_url
          end
        end

        private

        # Retrive the content of a local or remote file.
        #
        # @param A local or remote uri.
        #
        # @raise ArgumentError if it's not a valid uri
        #
        # Protect open-uri from malign arguments like
        #  - "|ls"
        #  - multiline strings
        def read_uri(uri_or_filename)
          remote_schemes = %w[http https ftp]
          Fog::Logger.debug("Opening #{uri_or_filename}")

          begin
            # Validate URI to protect from open-uri attacks.
            url = URI(uri_or_filename)

            if remote_schemes.include?(url.scheme)
              # Remote schemes must contain an host.
              raise ArgumentError if url.host.nil?

              # Encode URI with spaces.
              uri_or_filename = uri_or_filename.gsub(/ /, "%20")
            end
          rescue URI::InvalidURIError
            raise ArgumentError, "Not a valid URI: #{uri_or_filename}"
          end

          # TODO: A future revision may implement a retry.
          content = ''
          # open-uri doesn't open "file:///" uris.
          uri_or_filename = uri_or_filename.sub(/^file:/, "")

          open(uri_or_filename) { |f| content = f.read }
          content
        end

        # Return true if the file is an heat template, false otherwise.
        def template_is_raw?(content)
          htv = content.strip.index("heat_template_version")
          # Tolerate some leading character in case of a json template.
          htv && htv < 5
        end

        # Return true if it's an URI, false otherwise.
        def template_is_url?(path)
          normalise_file_path_to_url(path)
          true
        rescue ArgumentError, URI::InvalidURIError
          false
        end

        # Return true if I should I process this this file.
        #
        # @param [String] An heat template key
        #
        def ignore_if(key, value)
          return true if key != 'get_file' && key != 'type'

          return true unless value.kind_of?(String)

          return true if key == 'type' &&
                         !value.end_with?('.yaml', '.template')

          false
        end

        # Returns the string baseurl of the given url.
        def base_url_for_url(url)
          parsed = URI(url)
          parsed_dir = File.dirname(parsed.path)
          url_join(parsed, parsed_dir)
        end

        def normalise_file_path_to_url(path)
          # Nothing to do on URIs
          return path if URI(path).scheme

          path = File.absolute_path(path)
          url_join('file:/', path)
        end

        def deep_copy(item)
          return item if item.kind_of?(String)

          YAML.safe_load(YAML.dump(item), :permitted_classes => [Date])
        end
      end
    end
  end
end