--- "/c/Program Files/JetBrains/PyCharm Community Edition 2020.3.2/plugins/python-ce/helpers/generator3/module_redeclarator.py" 2020-12-30 17:08:24.000000000 +0000 +++ pycharm_helpers/plugins/python-ce/helpers/generator3/module_redeclarator.py 2021-01-27 14:13:18.876000000 +0000 @@ -2,6 +2,15 @@ from generator3.util_methods import * from generator3.util_methods import get_relative_path_by_qname from generator3.docstring_parsing import * +import re +import enum + +# Add patterns for identifying declarations we care about +RTYPE_PATTERN = re.compile(r"[:@]rtype:\s*(.*)") +TYPE_PATTERN = re.compile(r"[:@]type\s*:\s*(.*)") +PARAM_PATTERN = re.compile(r"[:@]param\s+([^:]*)\s+([^: ]*):") +NAMESPACE_PATTERN = re.compile(r"([^[\]]+\.)*([a-zA-Z][a-zA-Z0-9_]+)") +DEFAULTS = re.compile(r"(=.*)") class emptylistdict(dict): @@ -79,6 +88,8 @@ # we write things into buffers out-of-order self.header_buf = Buf(self) self.imports_buf = Buf(self) + # each class gets its own list of dependency imports, and so do functions + self.func_imports_buf = Buf(self) self.functions_buf = Buf(self) self.classes_buf = Buf(self) self.classes_buffs = list() @@ -94,6 +105,8 @@ self.doing_builtins = doing_builtins self.ret_type_cache = {} self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names + # if we use Typing hints, import some things for it + self.used_typing = False def _initializeQApp4(self): try: # QtGui should be imported _before_ QtCore package. @@ -129,19 +142,23 @@ if self.split_modules: last_pkg_dir = build_pkg_structure(self.cache_dir, self.qname) with fopen(os.path.join(last_pkg_dir, "__init__.py"), "w") as init: - for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf): + for buf in (self.header_buf, self.imports_buf, self.classes_buf): buf.flush(init) data = "" - for buf in self.classes_buffs: + for (buf,imports) in self.classes_buffs: with fopen(os.path.join(last_pkg_dir, buf.name) + '.py', "w") as dummy: self.header_buf.flush(dummy) self.imports_buf.flush(dummy) + imports.flush(dummy) buf.flush(dummy) data += self.create_local_import(buf.name) init.write(data) - self.footer_buf.flush(init) + + # Write out functions last so they can reference the classes imported above + for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): + buf.flush(init) else: last_pkg_dir = build_pkg_structure(self.cache_dir, '.'.join(qname_parts[:-1])) # In some rare cases submodules of a binary might have been generated earlier than the module @@ -155,13 +172,16 @@ else: skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py') with fopen(skeleton_path, "w") as mod: - for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf): + for buf in (self.header_buf, self.imports_buf, self.classes_buf): buf.flush(mod) - for buf in self.classes_buffs: + for (buf,imports) in self.classes_buffs: + imports.flush(mod) buf.flush(mod) - self.footer_buf.flush(mod) + # Write out functions last so they can reference the classes imported above + for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): + buf.flush(mod) # Some builtin classes effectively change __init__ signature without overriding it. # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS @@ -177,6 +197,50 @@ data += "." data += name + " import " + name + "\n" return data + + # Parse a typing declaration and add the actual types references + def add_import_types(self, import_types, type_decl): + if '[' not in type_decl: + import_types.add(type_decl) + return + + match = re.match('^[tT]uple\[(.*)\]$', type_decl) + if match: + for i in match.group(1).split(','): + self.add_import_types(import_types, i.strip()) + return + + match = re.match('^[lL]ist\[(.*)\]$', type_decl) + if match: + self.add_import_types(import_types, match.group(1).strip()) + return + + match = re.match('^[cC]allable\[\[(.*)\]\s*,\s*(.*)\]$', type_decl) + if match: + for i in match.group(1).split(','): + self.add_import_types(import_types, i.strip()) + self.add_import_types(import_types, match.group(2).strip()) + return + + # For a given type that's referenced, figure out where to import it from and add + # to the list + def process_import_type(self, used_imports, p_modname, classname, import_type): + parent = None + + if import_type == '...': + return + + if import_type in dir(sys.modules[p_modname]): + if import_type != classname: + parent = '.' + child = import_type + elif '.' in import_type: + imp_split = import_type.split('.') + parent = '.'.join(imp_split) + child = imp_split[-1] + + if parent is not None and child not in used_imports[parent]: + used_imports[parent].append(child) def find_imported_name(self, item): """ @@ -433,7 +497,43 @@ if first_param: seq.insert(0, first_param) seq = make_names_unique(seq) - return (seq, ret_type, doc_node) + + import_types = set() + ret_hint = None + + # Try to use :rtype: to add explicit type annotations to return type, since PyCharm + # doesn't parse rtype properly (at least with split modules) + if ret_type is None and ':rtype:' in signature_string: + result = RTYPE_PATTERN.search(signature_string) + if result is not None: + type_decl = result.group(1).strip() + if type_decl != class_name: + self.add_import_types(import_types, type_decl) + ret_hint = NAMESPACE_PATTERN.sub(r'\2', type_decl) + else: + ret_hint = "'" + type_decl + "'" + self.used_typing = True + + # Also use :param: to add necessary imports and fix up the parameters. + # PyCharm supports parsing :param: but the skeletons need the right imports + # and it fails for namespaced parameters (since the 'raw' parameter type + # foo.bar refers to a generated submodule called bar with the skeleton bar + # inside. + for p in PARAM_PATTERN.findall(signature_string): + type_decl = p[0] + if type_decl != class_name: + self.add_import_types(import_types, type_decl) + try: + defaults_match = [DEFAULTS.search(s) for s in seq] + idx = [DEFAULTS.sub('', s) for s in seq].index(p[1]) + seq[idx] = '{}: {}'.format(p[1], NAMESPACE_PATTERN.sub(r'\2', p[0])) + if defaults_match[idx]: + seq[idx] += defaults_match[idx].group(1) + except ValueError: + note("Warning: Unrecognised parameter {} in parameter list {}".format(p, seq)) + pass + + return (seq, ret_type, doc_node, list(import_types), ret_hint) def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False): """ @@ -453,18 +553,23 @@ overloads.append(part[i + len(signature):]) if len(overloads) > 1: docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads] + import_types = [] ret_types = [] for result in docstring_results: rt = result[1] if rt and rt not in ret_types: ret_types.append(rt) + imps = result[3] + for imp in imps: + if imp and imp not in import_types: + import_types.append(imp) if ret_types: ret_literal = " or ".join(ret_types) else: ret_literal = None param_lists = [result[0] for result in docstring_results] spec = build_signature(func_name, restore_parameters_for_overloads(param_lists)) - return (spec, ret_literal, "restored from __doc__ with multiple overloads") + return (spec, ret_literal, "restored from __doc__ with multiple overloads", import_types) # find the first thing to look like a definition prefix_re = re.compile(r"\s*(?:(\w+)[ \t]+)?" + func_id + r"\s*\(") # "foo(..." or "int foo(..." @@ -472,18 +577,21 @@ # parse the part that looks right if match: ret_hint = match.group(1) - params, ret_literal, doc_note = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) + params, ret, doc_note, import_types, ret_hint = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) spec = func_name + flatten(params) - return (spec, ret_literal, doc_note) + # if we got a type hint, put it on the function declaration + if ret_hint: + spec = spec + ' -> ' + ret_hint + return (spec, ret, doc_note, import_types) else: - return (None, None, None) + return (None, None, None, []) def is_predefined_builtin(self, module_name, class_name, func_name): return self.doing_builtins and module_name == BUILTIN_MOD_NAME and ( class_name, func_name) in PREDEFINED_BUILTIN_SIGS - def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None): + def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None, used_imports=None): """ Restore function argument list as best we can. @param out output function of a Buf @@ -531,6 +639,8 @@ deco = "classmethod" elif type(p_func).__name__.startswith('staticmethod'): deco = "staticmethod" + elif str(descriptor).startswith('= 30 for item_name in [cls_item[0] for cls_item in cls_list]: buf = ClassBuf(item_name, self) - self.classes_buffs.append(buf) + imports = ClassBuf(item_name + '_imports', self) + self.classes_buffs.append((buf,imports)) out = buf.out if item_name in omitted_names: out(0, "# definition of ", item_name, " omitted") continue item = classes[item_name] - self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir) + used_imports = emptylistdict() + self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir, used_imports=used_imports) + # if we don't have split modules we can't import dependencies, but we also + # have an ordering constraint - classes need to be declared after any classes + # they reference in type hints. This is broken either way though even with the + # return literals + if not self.split_modules: + func_used_imports['.'] = [] + self.output_import_froms(imports.out, used_imports) self._defined[item_name] = True out(0, "") # empty line after each item @@ -1083,19 +1255,38 @@ for value in values_to_add: self.footer_buf.out(0, value) # imports: last, because previous parts could alter used_imports or hidden_imports - self.output_import_froms() + + out = self.imports_buf.out + self.output_import_froms(out, self.used_imports) + + if self.hidden_imports: + self.add_import_header_if_needed() + for mod_name in sorted_no_case(self.hidden_imports.keys()): + out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) + out(0, "") # empty line after group + + if self.used_typing: + self.add_import_header_if_needed() + out(0, 'from typing import List, Tuple, Callable, Any') + out(0, "") # empty line after group + if self.imports_buf.isEmpty(): - self.imports_buf.out(0, "# no imports") - self.imports_buf.out(0, "") # empty line after imports + out(0, "# no imports") + out(0, "") # empty line after imports - def output_import_froms(self): + def output_import_froms(self, out, imports_list): """Mention all imported names known within the module, wrapping as per PEP.""" - out = self.imports_buf.out - if self.used_imports: + if imports_list: self.add_import_header_if_needed() - for mod_name in sorted_no_case(self.used_imports.keys()): - import_names = self.used_imports[mod_name] - if import_names: + for mod_name in sorted_no_case(imports_list.keys()): + import_names = imports_list[mod_name] + if mod_name == '.': + # if this is a local import, we need to treat it specially to import + # the class inside the referenced module + for n in import_names: + out(0, "from .%s import %s" % (n, n)) # empty line after group + out(0, "") # empty line after group + elif import_names: self._defined[mod_name] = True right_pos = 0 # tracks width of list to fold it at right margin import_heading = "from % s import (" % mod_name @@ -1127,10 +1318,4 @@ out(0, "") # empty line after group - if self.hidden_imports: - self.add_import_header_if_needed() - for mod_name in sorted_no_case(self.hidden_imports.keys()): - out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) - out(0, "") # empty line after group -