#!/usr/bin/env python3 # Copyright 2022 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse import glob import json import copy import os import sys # This script runs in Chromium and ChromiumOS. try: # Chromium. _SRC_PATH = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..', '..')) sys.path.append(os.path.join(_SRC_PATH, 'third_party')) import pyyaml except ImportError: # ChromiumOS. # Some consumers (ChromiumOS ebuilds) of this script have pyyaml already in # their environment and may not have access to the full Chromium repository # nor the exact file structure. Import based on the Chromium third_party may # not always be possible. # Those consumers may install pyyaml in their environment and import it with # the name of the module instead as per the library documentation. # For compatibility reasons, please refer to # https://source.chromium.org/chromium/chromium/src/+/main:third_party/pyyaml/README.chromium # to know which version of pyyaml to use. import yaml as pyyaml def _SafeListDir(directory): '''Wrapper around os.listdir() that ignores files created by Finder.app.''' # On macOS, Finder.app creates .DS_Store files when a user visit a # directory causing failure of the script laters on because there # are no such group as .DS_Store. Skip the file to prevent the error. return filter(lambda name:(name != '.DS_Store'),sorted(os.listdir(directory))) TEMPLATES_PATH = os.path.join( os.path.dirname(__file__), 'templates') DEFAULT_TEMPLATES_GEN_PATH = os.path.join( os.path.dirname(__file__), 'policy_templates.json') POLICY_DEFINITIONS_KEY = 'policy_definitions' def _SubstituteSchemaRefNames(node, child_key, common_schema, parent_refs, refs_seen): '''Converts recursively objects with the key '$ref' into their actual schema. ''' if (not isinstance(node, dict) or not child_key in node or not isinstance(node[child_key], dict)): return if '$ref' in node[child_key]: ref_name = node[child_key]['$ref'] # If the parent has the same id as child, leave the child's ref to avoid # infinite loop. if ref_name not in parent_refs: node[child_key] = copy.deepcopy(common_schema[ref_name]) parent_refs.add(ref_name) # If the schema has been seen already, only keep a reference to the first to # avoid having the same ref defined at multiple places. if ref_name not in refs_seen: node[child_key]['id'] = ref_name refs_seen.add(ref_name) for ck in sorted(node[child_key].keys()): # Copy parents ref so that parents are unique for each child branch and do # not mix with sibling nodes. _SubstituteSchemaRefNames(node[child_key], ck, common_schema, parent_refs.copy(), refs_seen) def _SubstituteSchemaRefs(policies, common_schema): '''Converts objects with the key '$ref' into their actual schema. Args: policies: List of policies. common_schema: Dictionary of schemas by their ref names.''' policy_list = [policy for policy in policies if 'schema' in policy] refs_seen = set() for policy in sorted(policy_list, key=lambda policy: policy['id']): parent_refs = set() _SubstituteSchemaRefNames(policy, 'schema', common_schema, parent_refs, refs_seen) parent_refs = set() _SubstituteSchemaRefNames(policy, 'validation_schema', common_schema, parent_refs, refs_seen) def _BuildPolicyTemplate(data): ''' Converts data into the format of the old policy_templates.json so that it can be used in policy_templates.grd Schema : { "policy_definitions": { "type": "list", "items": { "id": { "type": "number" }, "name": "string", "policies": { "type": "list", "items": "string" } // for policy groups. // See components/policy/resources/new_policy_templates/policy.yaml // for the other variables/ } }, "messages": { "type": "Object", "patternProperties": { "[A-Za-z]": { "desc": "string", "text": "String" } } }, //components/policy/resources/templates/manual_device_policy_proto_map.yaml // Includes policies where generate_device_proto is true. "device_policy_proto_map": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": "String" } } }, // Only lists of 2 items "legacy_device_policy_proto_map": { "type": "list", "items": "String" }, //components/policy/resources/templates/messages.yaml "messages": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": { "text": string, "description": "string" } } }, "risk_tag_definitions": { "type": "list", "items": { "name": "string", "description": "string", "user-description": "string" } }, "policy_atomic_group_definitions": { "type": "list", "items": { "id": { "type": "number" }, "name": "string", "caption": "string", "policies": { "type": "list", "items": "string" } // for policy groups. // See components/policy/resources/new_policy_templates/policy.yaml // for the other variables/ } }, "placeholders": { "type": "list" }, "deleted_policy_ids": { "type": "list", "items": { "type": "number" } }, "deleted_atomic_policy_group_ids": { "type": "list", "items": { "type": "number" } }, "highest_id_currently_used": { "type": "number" }, "highest_atomic_group_id_currently_used": { "type": "number" }, } ''' policy_name_id = { name: id for id, name in data['policies']['policies'].items() if name } atomic_group_name_id = { name: id for id, name in data['policies']['atomic_groups'].items() if name } policy_groups = [{ 'name': group_name, 'type': 'group', 'caption': group['caption'], 'desc': group['desc'], 'policies': list(group['policies']) } for group_name, group in data[POLICY_DEFINITIONS_KEY].items()] policies = [] atomic_groups = [] for group in data[POLICY_DEFINITIONS_KEY].values(): for policy_name, policy in group['policies'].items(): policies.append({ 'id': policy_name_id[policy_name], 'name': policy_name, **policy }) for name, atomic_group in group['policy_atomic_groups'].items(): atomic_groups.append({ 'id': atomic_group_name_id[name], 'name': name, **atomic_group }) device_policy_proto_map = data['manual_device_policy_proto_map'].copy() for policy in policies: if not policy.get('device_only', False): continue if not policy.get('generate_device_proto', True): continue device_policy_proto_map[policy['name']] = policy['name'] + '.value' result = { POLICY_DEFINITIONS_KEY: policies + policy_groups, 'deleted_policy_ids': [id for id, name in data['policies']['policies'].items() if not name], 'highest_id_currently_used': len(data['policies']['policies']), 'policy_atomic_group_definitions': atomic_groups, 'deleted_atomic_policy_group_ids': [ id for id, name in data['policies']['atomic_groups'].items() if not name ], 'highest_atomic_group_id_currently_used': len(data['policies']['atomic_groups']), 'placeholders': [], 'legacy_device_policy_proto_map': [], 'device_policy_proto_map': device_policy_proto_map, 'messages': data['messages'], 'risk_tag_definitions': [{'name': name, **value} for name, value in data['risk_tag_definitions'].items()] } for key, values in data['legacy_device_policy_proto_map'].items(): for item in values: result['legacy_device_policy_proto_map'].append([key, item]) _SubstituteSchemaRefs(result[POLICY_DEFINITIONS_KEY], data['common_schemas']) return result def _GetMetadata(): '''Returns an object containing the policy metadata in order to build the policy definition template.''' result = {} for file in _SafeListDir(TEMPLATES_PATH): filename = os.fsdecode(file) file_basename, file_extension = os.path.splitext(filename) if not file_extension == ".yaml": continue with open(os.path.join(TEMPLATES_PATH, filename), encoding='utf-8') as f: result[file_basename] = pyyaml.safe_load(f) return result def _GetPoliciesAndGroups(): '''Returns an object containing the policy groups with their details, policies and atomic policy groups in order to build the policy definition template. ''' result = {} policy_definitions_path = os.path.join(TEMPLATES_PATH, POLICY_DEFINITIONS_KEY) for group_name in _SafeListDir(policy_definitions_path): result[group_name] = {'policies': {}, 'policy_atomic_groups': {}} group_path = os.path.join(policy_definitions_path, group_name) if not os.path.isdir(group_path): continue for file in _SafeListDir(group_path): filename = os.fsdecode(file) file_basename, file_extension = os.path.splitext(filename) file_path = os.path.join(group_path, filename) if file_extension != '.yaml': continue with open(file_path, encoding='utf-8') as f: data = pyyaml.safe_load(f) if file_basename == '.group.details': result[group_name].update(data) elif file_basename == 'policy_atomic_groups': result[group_name]['policy_atomic_groups'].update(data) else: result[group_name]['policies'][file_basename] = data return result def _LoadPolicies(): ''' Loads all the yaml files used to define policies and their metadata into a single object. This is a direct representation of the data structure of the directories and their files. Schema : { //components/policy/resources/templates/policies.yaml "policies":{ // Map of policy atomic group ID to policy atomic group name. "atomic_groups": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": "String" } } } // Map of policy ID to policy name. "policies": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": "String" } } } } "policy_definitions": { "type": "Object", "patternProperties": { // Dictionary of policy groups "[A-Za-z]": { "type": "Object", // c/policy/resources/new_policy_templates/.groups.details.yaml "properties": { "caption": {"type": "string"}, "description": {"type": "string"} }, "patternProperties": { // Dictionary of policies "[A-Za-z]": { "type": "Object", "properties": { // c/policy/resources/new_policy_templates/policy.yaml } } } } } } }, // components/policy/resources/templates/manual_device_policy_proto_map.yaml "manual_device_policy_proto_map": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": "String" } } }, // components/policy/resources/templates/legacy_device_policy_proto_map.yaml "legacy_device_policy_proto_map": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "list", "items": "String" } } }, // components/policy/resources/templates/messages.yaml "messages": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": { "text": string, "description": "string" } } } }, // components/policy/resources/templates/risk_tag_definitions.yaml "risk_tag_definitions": { "type": "Object", "patternProperties": { "[A-Za-z]": { "type": "Object", "properties": { "description": string, "user-description": "string" } } } } } ''' return { POLICY_DEFINITIONS_KEY: _GetPoliciesAndGroups(), **_GetMetadata() } def GetPolicyTemplates(): '''Returns an object containing the policy templates. ''' template = _LoadPolicies() return _BuildPolicyTemplate(template) def _WriteDepFile(dep_file, target, source_files): '''Writes a dep file for `target` at `dep_file` with `source_files` as the dependencies. Args: dep_file: A path to the dependencies file for this script. target: The build target. source_files: A list of the dependencies for the build target. ''' with open(dep_file, "w") as f: f.write(target) f.write(": ") f.write(' '.join(source_files)) def main(): '''Generates the a JSON file at `dest` with all the policy definitions. If `dest` is not specified, a file name 'policy_templates.json' will be generated in the same directory as the script. Args: dest: A path to the policy templates generated definitions. depfile: A path to the dependencies file for this script. ''' parser = argparse.ArgumentParser() parser.add_argument('--dest', dest='dest') parser.add_argument('--depfile', dest='deps_file') args = parser.parse_args() if args.dest: path = os.path.join(args.dest) else: path = DEFAULT_TEMPLATES_GEN_PATH policy_templates = GetPolicyTemplates() with open(path, 'w+', encoding='utf-8') as dest: json.dump(policy_templates, dest, indent=2, sort_keys=True) files = sorted([f.replace('\\', '/') for f in glob.glob(TEMPLATES_PATH + '/**/*.yaml', recursive=True)]) if args.deps_file: _WriteDepFile(args.deps_file, args.dest, files) if '__main__' == __name__: sys.exit(main())