File: loaders.rb

package info (click to toggle)
puppet-agent 7.23.0-1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm
  • size: 19,092 kB
  • sloc: ruby: 245,074; sh: 456; makefile: 38; xml: 33
file content (542 lines) | stat: -rw-r--r-- 21,569 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
module Puppet::Pops
# This is the container for all Loader instances. Each Loader instance has a `loader_name` by which it can be uniquely
# identified within this container.
# A Loader can be private or public. In general, code will have access to the private loader associated with the
# location of the code. It will be parented by a loader that in turn have access to other public loaders that
# can load only such entries that have been publicly available. The split between public and private is not
# yet enforced in Puppet.
#
# The name of a private loader should always end with ' private'
#
class Loaders
  class LoaderError < Puppet::Error; end

  attr_reader :static_loader
  attr_reader :puppet_system_loader
  attr_reader :puppet_cache_loader
  attr_reader :public_environment_loader
  attr_reader :private_environment_loader
  attr_reader :environment

  def self.new(environment, for_agent = false, load_from_pcore = true)
    environment.lock.synchronize do
      obj = environment.loaders
      if obj.nil?
        obj = self.allocate
        obj.send(:initialize, environment, for_agent, load_from_pcore)
      end
      obj
    end
  end

  def initialize(environment, for_agent, load_from_pcore = true)
    # Protect against environment havoc
    raise ArgumentError.new(_("Attempt to redefine already initialized loaders for environment")) unless environment.loaders.nil?
    environment.loaders = self
    @environment = environment
    @loaders_by_name = {}

    add_loader_by_name(self.class.static_loader)

    # Create the set of loaders
    # 1. Puppet, loads from the "running" puppet - i.e. bundled functions, types, extension points and extensions
    #    These cannot be cached since a  loaded instance will be bound to its closure scope which holds on to
    #    a compiler and all loaded types. Subsequent request would find remains of the environment that loaded
    #    the content. PUP-4461.
    #
    @puppet_system_loader = create_puppet_system_loader()

    # 2. Cache loader(optional) - i.e. what puppet stores on disk via pluginsync; gate behind the for_agent flag.
    # 3. Environment loader - i.e. what is bound across the environment, may change for each setup
    #    TODO: loaders need to work when also running in an agent doing catalog application. There is no
    #    concept of environment the same way as when running as a master (except when doing apply).
    #    The creation mechanisms should probably differ between the two.
    @private_environment_loader =
      if for_agent
        @puppet_cache_loader = create_puppet_cache_loader
        create_environment_loader(environment, @puppet_cache_loader, load_from_pcore)
      else
        create_environment_loader(environment, @puppet_system_loader, load_from_pcore)
      end

    Pcore.init_env(@private_environment_loader)

    # 4. module loaders are set up from the create_environment_loader, they register themselves
  end

  # Called after loader has been added to Puppet Context as :loaders so that dynamic types can
  # be pre-loaded with a fully configured loader system
  def pre_load
    @puppet_system_loader.load(:type, 'error')
  end

  # Clears the cached static and puppet_system loaders (to enable testing)
  #
  def self.clear
    @@static_loader = nil
    Puppet::Pops::Types::TypeFactory.clear
    Model.class_variable_set(:@@pcore_ast_initialized, false)
    Model.register_pcore_types
  end

  # Calls {#loaders} to obtain the {{Loaders}} instance and then uses it to find the appropriate loader
  # for the given `module_name`, or for the environment in case `module_name` is `nil` or empty.
  #
  # @param module_name [String,nil] the name of the module
  # @return [Loader::Loader] the found loader
  # @raise [Puppet::ParseError] if no loader can be found
  # @api private
  def self.find_loader(module_name)
    loaders.find_loader(module_name)
  end

  def self.static_implementation_registry
    if !class_variable_defined?(:@@static_implementation_registry) || @@static_implementation_registry.nil?
      ir = Types::ImplementationRegistry.new
      Types::TypeParser.type_map.values.each { |t| ir.register_implementation(t.simple_name, t.class.name) }
      @@static_implementation_registry = ir
    end
    @@static_implementation_registry
  end

  def self.static_loader
    # The static loader can only be changed after a reboot
    if !class_variable_defined?(:@@static_loader) || @@static_loader.nil?
      @@static_loader = Loader::StaticLoader.new()
      @@static_loader.register_aliases
      Pcore.init(@@static_loader, static_implementation_registry)
    end
    @@static_loader
  end

  def self.implementation_registry
    loaders = Puppet.lookup(:loaders) { nil }
    loaders.nil? ? nil : loaders.implementation_registry
  end

  def register_implementations(obj_classes, name_authority)
    self.class.register_implementations_with_loader(obj_classes, name_authority, @private_environment_loader)
  end

  # Register implementations using the global static loader
  def self.register_static_implementations(obj_classes)
    register_implementations_with_loader(obj_classes, Pcore::RUNTIME_NAME_AUTHORITY, static_loader)
  end

  def self.register_implementations_with_loader(obj_classes, name_authority, loader)
    types = obj_classes.map do |obj_class|
      type = obj_class._pcore_type
      typed_name = Loader::TypedName.new(:type, type.name, name_authority)
      entry = loader.loaded_entry(typed_name)
      loader.set_entry(typed_name, type) if entry.nil? || entry.value.nil?
      type
    end
    # Resolve lazy so that all types can cross reference each other
    types.each { |type| type.resolve(loader) }
  end

  # Register the given type with the Runtime3TypeLoader. The registration will not happen unless
  # the type system has been initialized.
  #
  # @param name [String,Symbol] the name of the entity being set
  # @param origin [URI] the origin or the source where the type is defined
  # @api private
  def self.register_runtime3_type(name, origin)
    loaders = Puppet.lookup(:loaders) { nil }
    return nil if loaders.nil?

    rt3_loader = loaders.runtime3_type_loader
    return nil if rt3_loader.nil?

    name = name.to_s
    caps_name = Types::TypeFormatter.singleton.capitalize_segments(name)
    typed_name = Loader::TypedName.new(:type, name)
    rt3_loader.set_entry(typed_name, Types::PResourceType.new(caps_name), origin)
    nil
  end

  # Finds a loader to use when deserializing a catalog and then subsequenlty use user
  # defined types found in that catalog.
  #
  def self.catalog_loader
    loaders = Puppet.lookup(:loaders) { nil }
    if loaders.nil?
      loaders = Loaders.new(Puppet.lookup(:current_environment), true)
      Puppet.push_context(:loaders => loaders)
    end
    loaders.find_loader(nil)
  end

  # Finds the `Loaders` instance by looking up the :loaders in the global Puppet context
  #
  # @return [Loaders] the loaders instance
  # @raise [Puppet::ParseError] if loader has been bound to the global context
  # @api private
  def self.loaders
    loaders = Puppet.lookup(:loaders) { nil }
    raise Puppet::ParseError, _("Internal Error: Puppet Context ':loaders' missing") if loaders.nil?
    loaders
  end

  # Lookup a loader by its unique name.
  #
  # @param [String] loader_name the name of the loader to lookup
  # @return [Loader] the found loader
  # @raise [Puppet::ParserError] if no loader is found
  def [](loader_name)
    loader = @loaders_by_name[loader_name]
    if loader.nil?
      # Unable to find the module private loader. Try resolving the module
      loader = private_loader_for_module(loader_name[0..-9]) if loader_name.end_with?(' private')
      raise Puppet::ParseError, _("Unable to find loader named '%{loader_name}'") % { loader_name: loader_name } if loader.nil?
    end
    loader
  end

  # Finds the appropriate loader for the given `module_name`, or for the environment in case `module_name`
  # is `nil` or empty.
  #
  # @param module_name [String,nil] the name of the module
  # @return [Loader::Loader] the found loader
  # @raise [Puppet::ParseError] if no loader can be found
  # @api private
  def find_loader(module_name)
    if module_name.nil? || EMPTY_STRING == module_name
      # Use the public environment loader
      public_environment_loader
    else
      # TODO : Later check if definition is private, and then add it to private_loader_for_module
      #
      loader = public_loader_for_module(module_name)
      if loader.nil?
        raise Puppet::ParseError, _("Internal Error: did not find public loader for module: '%{module_name}'") % { module_name: module_name }
      end
      loader
    end
  end

  def implementation_registry
    # Environment specific implementation registry
    @implementation_registry ||= Types::ImplementationRegistry.new(self.class.static_implementation_registry)
  end

  def static_loader
    self.class.static_loader
  end

  def puppet_system_loader
    @puppet_system_loader
  end

  def runtime3_type_loader
    @runtime3_type_loader
  end

  def public_loader_for_module(module_name)
    md = @module_resolver[module_name] || (return nil)
    # Note, this loader is not resolved until there is interest in the visibility of entities from the
    # perspective of something contained in the module. (Many request may pass through a module loader
    # without it loading anything.
    # See {#private_loader_for_module}, and not in {#configure_loaders_for_modules}
    md.public_loader
  end

  def private_loader_for_module(module_name)
    md = @module_resolver[module_name] || (return nil)
    # Since there is interest in the visibility from the perspective of entities contained in the
    # module, it must be resolved (to provide this visibility).
    # See {#configure_loaders_for_modules}
    unless md.resolved?
      @module_resolver.resolve(md)
    end
    md.private_loader
  end

  def add_loader_by_name(loader)
    name = loader.loader_name
    if @loaders_by_name.include?(name)
      raise Puppet::ParseError, _("Internal Error: Attempt to redefine loader named '%{name}'") % { name: name }
    end
    @loaders_by_name[name] = loader
  end

  # Load the main manifest for the given environment
  #
  # There are two sources that can be used for the initial parse:
  #
  #   1. The value of `Puppet[:code]`: Puppet can take a string from
  #     its settings and parse that as a manifest. This is used by various
  #     Puppet applications to read in a manifest and pass it to the
  #     environment as a side effect. This is attempted first.
  #   2. The contents of the environment's +manifest+ attribute: Puppet will
  #     try to load the environment manifest. The manifest must be a file.
  #
  # @return [Model::Program] The manifest parsed into a model object
  def load_main_manifest
    parser = Parser::EvaluatingParser.singleton
    parsed_code = Puppet[:code]
    program = if parsed_code != ""
      parser.parse_string(parsed_code, 'unknown-source-location')
    else
      file = @environment.manifest

      # if the manifest file is a reference to a directory, parse and combine
      # all .pp files in that directory
      if file == Puppet::Node::Environment::NO_MANIFEST
        nil
      elsif File.directory?(file)
        raise Puppet::Error, "manifest of environment '#{@environment.name}' appoints directory '#{file}'. It must be a file"
      elsif File.exist?(file)
        parser.parse_file(file)
      else
        raise Puppet::Error, "manifest of environment '#{@environment.name}' appoints '#{file}'. It does not exist"
      end
    end
    instantiate_definitions(program, public_environment_loader) unless program.nil?
    program
  rescue Puppet::ParseErrorWithIssue => detail
    detail.environment = @environment.name
    raise
  rescue => detail
    msg = _('Could not parse for environment %{env}: %{detail}') % { env: @environment, detail: detail }
    error = Puppet::Error.new(msg)
    error.set_backtrace(detail.backtrace)
    raise error
  end

  # Add 4.x definitions found in the given program to the given loader.
  def instantiate_definitions(program, loader)
    program.definitions.each { |d| instantiate_definition(d, loader) }
    nil
  end

  # Add given 4.x definition to the given loader.
  def instantiate_definition(definition, loader)
    case definition
    when Model::PlanDefinition
      instantiate_PlanDefinition(definition, loader)
    when Model::FunctionDefinition
      instantiate_FunctionDefinition(definition, loader)
    when Model::TypeAlias
      instantiate_TypeAlias(definition, loader)
    when Model::TypeMapping
      instantiate_TypeMapping(definition, loader)
    else
      raise Puppet::ParseError, "Internal Error: Unknown type of definition - got '#{definition.class}'"
    end
  end

  private

  def instantiate_PlanDefinition(plan_definition, loader)
    typed_name, f = Loader::PuppetPlanInstantiator.create_from_model(plan_definition, loader)
    loader.set_entry(typed_name, f, plan_definition.locator.to_uri(plan_definition))
    nil
  end

  def instantiate_FunctionDefinition(function_definition, loader)
    # Instantiate Function, and store it in the loader
    typed_name, f = Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader)
    loader.set_entry(typed_name, f, function_definition.locator.to_uri(function_definition))
    nil
  end

  def instantiate_TypeAlias(type_alias, loader)
    # Bind the type alias to the loader using the alias
    Puppet::Pops::Loader::TypeDefinitionInstantiator.create_from_model(type_alias, loader)
    nil
  end

  def instantiate_TypeMapping(type_mapping, loader)
    tf = Types::TypeParser.singleton
    lhs = tf.interpret(type_mapping.type_expr, loader)
    rhs = tf.interpret_any(type_mapping.mapping_expr, loader)
    implementation_registry.register_type_mapping(lhs, rhs)
    nil
  end

  def create_puppet_system_loader()
    Loader::ModuleLoaders.system_loader_from(static_loader, self)
  end

  def create_puppet_cache_loader()
    Loader::ModuleLoaders.cached_loader_from(puppet_system_loader, self)
  end

  def create_environment_loader(environment, parent_loader, load_from_pcore = true)
    # This defines where to start parsing/evaluating - the "initial import" (to use 3x terminology)
    # Is either a reference to a single .pp file, or a directory of manifests. If the environment becomes
    # a module and can hold functions, types etc. then these are available across all other modules without
    # them declaring this dependency - it is however valuable to be able to treat it the same way
    # bindings and other such system related configuration.

    # This is further complicated by the many options available:
    # - The environment may not have a directory, the code comes from one appointed 'manifest' (site.pp)
    # - The environment may have a directory and also point to a 'manifest'
    # - The code to run may be set in settings (code)

    # Further complication is that there is nothing specifying what the visibility is into
    # available modules. (3x is everyone sees everything).
    # Puppet binder currently reads confdir/bindings - that is bad, it should be using the new environment support.

    # env_conf is setup from the environment_dir value passed into Puppet::Environments::Directories.new
    env_conf = Puppet.lookup(:environments).get_conf(environment.name)
    env_path = env_conf.nil? || !env_conf.is_a?(Puppet::Settings::EnvironmentConf) ? nil : env_conf.path_to_env

    if Puppet[:tasks]
      loader = Loader::ModuleLoaders.environment_loader_from(parent_loader, self, env_path)
    else
      # Create the 3.x resource type loader
      static_loader.runtime_3_init
      # Create pcore resource type loader, if applicable
      pcore_resource_type_loader = if load_from_pcore && env_path
                                     Loader::ModuleLoaders.pcore_resource_type_loader_from(parent_loader, self, env_path)
                                   else
                                     nil
                                   end
      @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(parent_loader, self, environment, pcore_resource_type_loader))

      if env_path.nil?
        # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?
        loader = add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@runtime3_type_loader, Loader::ENVIRONMENT, environment))
      else
        # View the environment as a module to allow loading from it - this module is always called 'environment'
        loader = Loader::ModuleLoaders.environment_loader_from(@runtime3_type_loader, self, env_path)
      end
    end

    # An environment has a module path even if it has a null loader
    configure_loaders_for_modules(loader, environment)
    # modules should see this loader
    @public_environment_loader = loader

    # Code in the environment gets to see all modules (since there is no metadata for the environment)
    # but since this is not given to the module loaders, they can not load global code (since they can not
    # have prior knowledge about this
    loader = add_loader_by_name(Loader::DependencyLoader.new(loader, Loader::ENVIRONMENT_PRIVATE, @module_resolver.all_module_loaders(), environment))

    # The module loader gets the private loader via a lazy operation to look up the module's private loader.
    # This does not work for an environment since it is not resolved the same way.
    # TODO: The EnvironmentLoader could be a specialized loader instead of using a ModuleLoader to do the work.
    #       This is subject to future design - an Environment may move more in the direction of a Module.
    @public_environment_loader.private_loader = loader
    loader
  end

  def configure_loaders_for_modules(parent_loader, environment)
    @module_resolver = mr = ModuleResolver.new(self)
    environment.modules.each do |puppet_module|
      # Create data about this module
      md = LoaderModuleData.new(puppet_module)
      mr[puppet_module.name] = md
      md.public_loader = Loader::ModuleLoaders.module_loader_from(parent_loader, self, md.name, md.path)
    end
    # NOTE: Do not resolve all modules here - this is wasteful if only a subset of modules / functions are used
    #       The resolution is triggered by asking for a module's private loader, since this means there is interest
    #       in the visibility from that perspective.
    #       If later, it is wanted that all resolutions should be made up-front (to capture errors eagerly, this
    #       can be introduced (better for production), but may be irritating in development mode.
  end

  # =LoaderModuleData
  # Information about a Module and its loaders.
  # TODO: should have reference to real model element containing all module data; this is faking it
  # TODO: Should use Puppet::Module to get the metadata (as a hash) - a somewhat blunt instrument, but that is
  #       what is available with a reasonable API.
  #
  class LoaderModuleData

    attr_accessor :public_loader
    attr_accessor :private_loader
    attr_accessor :resolutions

    # The Puppet::Module this LoaderModuleData represents in the loader configuration
    attr_reader :puppet_module

    # @param puppet_module [Puppet::Module] the module instance for the module being represented
    #
    def initialize(puppet_module)
      @puppet_module = puppet_module
      @resolutions = []
      @public_loader = nil
      @private_loader = nil
    end

    def name
      @puppet_module.name
    end

    def version
      @puppet_module.version
    end

    def path
      @puppet_module.path
    end

    def resolved?
      !@private_loader.nil?
    end

    def restrict_to_dependencies?
      @puppet_module.has_metadata?
    end

    def unmet_dependencies?
      @puppet_module.unmet_dependencies.any?
    end

    def dependency_names
      @puppet_module.dependencies_as_modules.collect(&:name)
    end
  end

  # Resolves module loaders - resolution of model dependencies is done by Puppet::Module
  #
  class ModuleResolver

    def initialize(loaders)
      @loaders = loaders
      @index = {}
      @all_module_loaders = nil
    end

    def [](name)
      @index[name]
    end

    def []=(name, module_data)
      @index[name] = module_data
    end

    def all_module_loaders
      @all_module_loaders ||= @index.values.map {|md| md.public_loader }
    end

    def resolve(module_data)
      if module_data.resolved?
        nil
      else
        module_data.private_loader =
          if module_data.restrict_to_dependencies?
            create_loader_with_dependencies_first(module_data)
          else
            create_loader_with_all_modules_visible(module_data)
          end
      end
    end

    private

    def create_loader_with_all_modules_visible(from_module_data)
      @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", all_module_loaders(), @loaders.environment))
    end

    def create_loader_with_dependencies_first(from_module_data)
      dependency_loaders = from_module_data.dependency_names.collect { |name| @index[name].public_loader }
      visible_loaders = dependency_loaders + (all_module_loaders() - dependency_loaders)
      @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", visible_loaders, @loaders.environment))
    end
  end
end
end