# Copyright 2020 The Emscripten Authors.  All rights reserved.
# Emscripten is available under two separate licenses, the MIT license and the
# University of Illinois/NCSA Open Source License.  Both these licenses can be
# found in the LICENSE file.

import atexit
import json
import logging
import multiprocessing
import os
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
from subprocess import STDOUT, PIPE

from . import diagnostics
from . import response_file
from . import shared
from . import webassembly
from . import config
from .toolchain_profiler import ToolchainProfiler
from .shared import Settings, CLANG_CC, CLANG_CXX, PYTHON
from .shared import LLVM_NM, EMCC, EMAR, EMXX, EMRANLIB, WASM_LD, LLVM_AR
from .shared import LLVM_LINK, LLVM_OBJCOPY
from .shared import try_delete, run_process, check_call, exit_with_error
from .shared import configuration, path_from_root, EXPECTED_BINARYEN_VERSION
from .shared import asmjs_mangle, DEBUG
from .shared import EM_BUILD_VERBOSE, TEMP_DIR
from .shared import CANONICAL_TEMP_DIR, LLVM_DWARFDUMP, demangle_c_symbol_name, asbytes
from .shared import get_emscripten_temp_dir, exe_suffix, is_c_symbol
from .utils import which, WINDOWS

logger = logging.getLogger('building')

#  Building
multiprocessing_pool = None
binaryen_checked = False

# internal caches
internal_nm_cache = {}
# cache results of nm - it can be slow to run
uninternal_nm_cache = {}
# Stores the object files contained in different archive files passed as input
ar_contents = {}
_is_ar_cache = {}
# the exports the user requested
user_requested_exports = []


class ObjectFileInfo(object):
  def __init__(self, returncode, output, defs=set(), undefs=set(), commons=set()):
    self.returncode = returncode
    self.output = output
    self.defs = defs
    self.undefs = undefs
    self.commons = commons

  def is_valid_for_nm(self):
    return self.returncode == 0


# llvm-ar appears to just use basenames inside archives. as a result, files
# with the same basename will trample each other when we extract them. to help
# warn of such situations, we warn if there are duplicate entries in the
# archive
def warn_if_duplicate_entries(archive_contents, archive_filename):
  if len(archive_contents) != len(set(archive_contents)):
    msg = '%s: archive file contains duplicate entries. This is not supported by emscripten. Only the last member with a given name will be linked in which can result in undefined symbols. You should either rename your source files, or use `emar` to create you archives which works around this issue.' % archive_filename
    warned = set()
    for i in range(len(archive_contents)):
      curr = archive_contents[i]
      if curr not in warned and curr in archive_contents[i + 1:]:
        msg += '\n   duplicate: %s' % curr
        warned.add(curr)
    diagnostics.warning('emcc', msg)


# This function creates a temporary directory specified by the 'dir' field in
# the returned dictionary. Caller is responsible for cleaning up those files
# after done.
def extract_archive_contents(archive_file):
  lines = run_process([LLVM_AR, 't', archive_file], stdout=PIPE).stdout.splitlines()
  # ignore empty lines
  contents = [l for l in lines if len(l)]
  if len(contents) == 0:
    logger.debug('Archive %s appears to be empty (recommendation: link an .so instead of .a)' % archive_file)
    return {
      'returncode': 0,
      'dir': None,
      'files': []
    }

  # `ar` files can only contains filenames. Just to be sure,  verify that each
  # file has only as filename component and is not absolute
  for f in contents:
    assert not os.path.dirname(f)
    assert not os.path.isabs(f)

  warn_if_duplicate_entries(contents, archive_file)

  # create temp dir
  temp_dir = tempfile.mkdtemp('_archive_contents', 'emscripten_temp_')

  # extract file in temp dir
  proc = run_process([LLVM_AR, 'xo', archive_file], stdout=PIPE, stderr=STDOUT, cwd=temp_dir)
  abs_contents = [os.path.join(temp_dir, c) for c in contents]

  # check that all files were created
  missing_contents = [x for x in abs_contents if not os.path.exists(x)]
  if missing_contents:
    exit_with_error('llvm-ar failed to extract file(s) ' + str(missing_contents) + ' from archive file ' + f + '! Error:' + str(proc.stdout))

  return {
    'returncode': proc.returncode,
    'dir': temp_dir,
    'files': abs_contents
  }


# Due to a python pickling issue, the following two functions must be at top
# level, or multiprocessing pool spawn won't find them.
def g_llvm_nm_uncached(filename):
  return llvm_nm_uncached(filename)


def g_multiprocessing_initializer(*args):
  for item in args:
    (key, value) = item.split('=', 1)
    if key == 'EMCC_POOL_CWD':
      os.chdir(value)
    else:
      os.environ[key] = value


def unique_ordered(values):
  """return a list of unique values in an input list, without changing order
  (list(set(.)) would change order randomly).
  """
  seen = set()

  def check(value):
    if value in seen:
      return False
    seen.add(value)
    return True

  return list(filter(check, values))


# clear internal caches. this is not normally needed, except if the clang/LLVM
# used changes inside this invocation of Building, which can happen in the benchmarker
# when it compares different builds.
def clear():
  internal_nm_cache.clear()
  uninternal_nm_cache.clear()
  ar_contents.clear()
  _is_ar_cache.clear()


def get_num_cores():
  return int(os.environ.get('EMCC_CORES', multiprocessing.cpu_count()))


# Multiprocessing pools are very slow to build up and tear down, and having
# several pools throughout the application has a problem of overallocating
# child processes. Therefore maintain a single centralized pool that is shared
# between all pooled task invocations.
def get_multiprocessing_pool():
  global multiprocessing_pool
  if not multiprocessing_pool:
    cores = get_num_cores()
    if DEBUG:
      # When in EMCC_DEBUG mode, only use a single core in the pool, so that
      # logging is not all jumbled up.
      cores = 1

    # If running with one core only, create a mock instance of a pool that does not
    # actually spawn any new subprocesses. Very useful for internal debugging.
    if cores == 1:
      class FakeMultiprocessor(object):
        def map(self, func, tasks, *args, **kwargs):
          results = []
          for t in tasks:
            results += [func(t)]
          return results

        def map_async(self, func, tasks, *args, **kwargs):
          class Result:
            def __init__(self, func, tasks):
              self.func = func
              self.tasks = tasks

            def get(self, timeout):
              results = []
              for t in tasks:
                results += [func(t)]
              return results

          return Result(func, tasks)

      multiprocessing_pool = FakeMultiprocessor()
    else:
      child_env = [
        # Multiprocessing pool children must have their current working
        # directory set to a safe path that is guaranteed not to die in
        # between of executing commands, or otherwise the pool children will
        # have trouble spawning subprocesses of their own.
        'EMCC_POOL_CWD=' + path_from_root(),
        # Multiprocessing pool children can't spawn their own linear number of
        # children, that could cause a quadratic amount of spawned processes.
        'EMCC_CORES=1'
      ]
      multiprocessing_pool = multiprocessing.Pool(processes=cores, initializer=g_multiprocessing_initializer, initargs=child_env)

      def close_multiprocessing_pool():
        global multiprocessing_pool
        try:
          # Shut down the pool explicitly, because leaving that for Python to do at process shutdown is buggy and can generate
          # noisy "WindowsError: [Error 5] Access is denied" spam which is not fatal.
          multiprocessing_pool.terminate()
          multiprocessing_pool.join()
          multiprocessing_pool = None
        except OSError as e:
          # Mute the "WindowsError: [Error 5] Access is denied" errors, raise all others through
          if not (sys.platform.startswith('win') and isinstance(e, WindowsError) and e.winerror == 5):
            raise
      atexit.register(close_multiprocessing_pool)

  return multiprocessing_pool


# .. but for Popen, we cannot have doublequotes, so provide functionality to
# remove them when needed.
def remove_quotes(arg):
  if isinstance(arg, list):
    return [remove_quotes(a) for a in arg]

  if arg.startswith('"') and arg.endswith('"'):
    return arg[1:-1].replace('\\"', '"')
  elif arg.startswith("'") and arg.endswith("'"):
    return arg[1:-1].replace("\\'", "'")
  else:
    return arg


def get_building_env(cflags=[]):
  env = os.environ.copy()
  # point CC etc. to the em* tools.
  env['CC'] = EMCC
  env['CXX'] = EMXX
  env['AR'] = EMAR
  env['LD'] = EMCC
  env['NM'] = LLVM_NM
  env['LDSHARED'] = EMCC
  env['RANLIB'] = EMRANLIB
  env['EMSCRIPTEN_TOOLS'] = path_from_root('tools')
  if cflags:
    env['CFLAGS'] = env['EMMAKEN_CFLAGS'] = ' '.join(cflags)
  env['HOST_CC'] = CLANG_CC
  env['HOST_CXX'] = CLANG_CXX
  env['HOST_CFLAGS'] = "-W" # if set to nothing, CFLAGS is used, which we don't want
  env['HOST_CXXFLAGS'] = "-W" # if set to nothing, CXXFLAGS is used, which we don't want
  env['PKG_CONFIG_LIBDIR'] = path_from_root('system', 'local', 'lib', 'pkgconfig') + os.path.pathsep + path_from_root('system', 'lib', 'pkgconfig')
  env['PKG_CONFIG_PATH'] = os.environ.get('EM_PKG_CONFIG_PATH', '')
  env['EMSCRIPTEN'] = path_from_root()
  env['PATH'] = path_from_root('system', 'bin') + os.pathsep + env['PATH']
  env['CROSS_COMPILE'] = path_from_root('em') # produces /path/to/emscripten/em , which then can have 'cc', 'ar', etc appended to it
  return env


# Returns a clone of the given environment with all directories that contain
# sh.exe removed from the PATH.  Used to work around CMake limitation with
# MinGW Makefiles, where sh.exe is not allowed to be present.
def remove_sh_exe_from_path(env):
  env = env.copy()
  if not WINDOWS:
    return env
  path = env['PATH'].split(';')
  path = [p for p in path if not os.path.exists(os.path.join(p, 'sh.exe'))]
  env['PATH'] = ';'.join(path)
  return env


def handle_cmake_toolchain(args, env):
  def has_substr(args, substr):
    return any(substr in s for s in args)

  # Append the Emscripten toolchain file if the user didn't specify one.
  if not has_substr(args, '-DCMAKE_TOOLCHAIN_FILE'):
    args.append('-DCMAKE_TOOLCHAIN_FILE=' + path_from_root('cmake', 'Modules', 'Platform', 'Emscripten.cmake'))
  node_js = config.NODE_JS

  if not has_substr(args, '-DCMAKE_CROSSCOMPILING_EMULATOR'):
    node_js = config.NODE_JS[0].replace('"', '\"')
    args.append('-DCMAKE_CROSSCOMPILING_EMULATOR="%s"' % node_js)

  # On Windows specify MinGW Makefiles or ninja if we have them and no other
  # toolchain was specified, to keep CMake from pulling in a native Visual
  # Studio, or Unix Makefiles.
  if WINDOWS and '-G' not in args:
    if which('mingw32-make'):
      args += ['-G', 'MinGW Makefiles']
    elif which('ninja'):
      args += ['-G', 'Ninja']

  # CMake has a requirement that it wants sh.exe off PATH if MinGW Makefiles
  # is being used. This happens quite often, so do this automatically on
  # behalf of the user. See
  # http://www.cmake.org/Wiki/CMake_MinGW_Compiler_Issues
  if WINDOWS and 'MinGW Makefiles' in args:
    env = remove_sh_exe_from_path(env)

  return (args, env)


def configure(args, stdout=None, stderr=None, env=None, cflags=[], **kwargs):
  if env:
    env = env.copy()
  else:
    env = get_building_env(cflags=cflags)
  if 'cmake' in args[0]:
    # Note: EMMAKEN_JUST_CONFIGURE shall not be enabled when configuring with
    #       CMake. This is because CMake does expect to be able to do
    #       config-time builds with emcc.
    args, env = handle_cmake_toolchain(args, env)
  else:
    # When we configure via a ./configure script, don't do config-time
    # compilation with emcc, but instead do builds natively with Clang. This
    # is a heuristic emulation that may or may not work.
    env['EMMAKEN_JUST_CONFIGURE'] = '1'
  if EM_BUILD_VERBOSE >= 2:
    stdout = None
  if EM_BUILD_VERBOSE >= 1:
    stderr = None
  print('configure: ' + shared.shlex_join(args), file=sys.stderr)
  run_process(args, stdout=stdout, stderr=stderr, env=env, **kwargs)


def make(args, stdout=None, stderr=None, env=None, cflags=[], **kwargs):
  if env is None:
    env = get_building_env(cflags=cflags)

  # On Windows prefer building with mingw32-make instead of make, if it exists.
  if WINDOWS:
    if args[0] == 'make':
      mingw32_make = which('mingw32-make')
      if mingw32_make:
        args[0] = mingw32_make

    if 'mingw32-make' in args[0]:
      env = remove_sh_exe_from_path(env)

  # On Windows, run the execution through shell to get PATH expansion and
  # executable extension lookup, e.g. 'sdl2-config' will match with
  # 'sdl2-config.bat' in PATH.
  if EM_BUILD_VERBOSE >= 2:
    stdout = None
  if EM_BUILD_VERBOSE >= 1:
    stderr = None
  print('make: ' + ' '.join(args), file=sys.stderr)
  run_process(args, stdout=stdout, stderr=stderr, env=env, shell=WINDOWS, **kwargs)


def make_paths_absolute(f):
  if f.startswith('-'):  # skip flags
    return f
  else:
    return os.path.abspath(f)


# Runs llvm-nm in parallel for the given list of files.
# The results are populated in uninternal_nm_cache
# multiprocessing_pool: An existing multiprocessing pool to reuse for the operation, or None
# to have the function allocate its own.
def parallel_llvm_nm(files):
  with ToolchainProfiler.profile_block('parallel_llvm_nm'):
    pool = get_multiprocessing_pool()
    object_contents = pool.map(g_llvm_nm_uncached, files)

    for i, file in enumerate(files):
      if object_contents[i].returncode != 0:
        logger.debug('llvm-nm failed on file ' + file + ': return code ' + str(object_contents[i].returncode) + ', error: ' + object_contents[i].output)
      uninternal_nm_cache[file] = object_contents[i]
    return object_contents


def read_link_inputs(files):
  with ToolchainProfiler.profile_block('read_link_inputs'):
    # Before performing the link, we need to look at each input file to determine which symbols
    # each of them provides. Do this in multiple parallel processes.
    archive_names = [] # .a files passed in to the command line to the link
    object_names = [] # .o/.bc files passed in to the command line to the link
    for f in files:
      absolute_path_f = make_paths_absolute(f)

      if absolute_path_f not in ar_contents and is_ar(absolute_path_f):
        archive_names.append(absolute_path_f)
      elif absolute_path_f not in uninternal_nm_cache and is_bitcode(absolute_path_f):
        object_names.append(absolute_path_f)

    # Archives contain objects, so process all archives first in parallel to obtain the object files in them.
    pool = get_multiprocessing_pool()
    object_names_in_archives = pool.map(extract_archive_contents, archive_names)

    def clean_temporary_archive_contents_directory(directory):
      def clean_at_exit():
        try_delete(directory)
      if directory:
        atexit.register(clean_at_exit)

    for n in range(len(archive_names)):
      if object_names_in_archives[n]['returncode'] != 0:
        raise Exception('llvm-ar failed on archive ' + archive_names[n] + '!')
      ar_contents[archive_names[n]] = object_names_in_archives[n]['files']
      clean_temporary_archive_contents_directory(object_names_in_archives[n]['dir'])

    for o in object_names_in_archives:
      for f in o['files']:
        if f not in uninternal_nm_cache:
          object_names.append(f)

    # Next, extract symbols from all object files (either standalone or inside archives we just extracted)
    # The results are not used here directly, but populated to llvm-nm cache structure.
    parallel_llvm_nm(object_names)


def llvm_backend_args():
  # disable slow and relatively unimportant optimization passes
  args = ['-combiner-global-alias-analysis=false']

  # asm.js-style exception handling
  if Settings.DISABLE_EXCEPTION_CATCHING != 1:
    args += ['-enable-emscripten-cxx-exceptions']
  if Settings.DISABLE_EXCEPTION_CATCHING == 2:
    allowed = ','.join(Settings.EXCEPTION_CATCHING_ALLOWED or ['__fake'])
    args += ['-emscripten-cxx-exceptions-allowed=' + allowed]

  if Settings.SUPPORT_LONGJMP:
    # asm.js-style setjmp/longjmp handling
    args += ['-enable-emscripten-sjlj']

  # better (smaller, sometimes faster) codegen, see binaryen#1054
  # and https://bugs.llvm.org/show_bug.cgi?id=39488
  args += ['-disable-lsr']

  return args


def link_to_object(linker_inputs, target):
  # link using lld for the wasm backend with wasm object files,
  # other otherwise for linking of bitcode we must use our python
  # code (necessary for asm.js, for wasm bitcode see
  # https://bugs.llvm.org/show_bug.cgi?id=40654)
  if not Settings.LTO:
    link_lld(linker_inputs + ['--relocatable'], target)
  else:
    link_bitcode(linker_inputs, target)


def link_llvm(linker_inputs, target):
  # runs llvm-link to link things.
  cmd = [LLVM_LINK] + linker_inputs + ['-o', target]
  cmd = get_command_with_possible_response_file(cmd)
  check_call(cmd)


def lld_flags_for_executable(external_symbol_list):
  cmd = []
  if external_symbol_list:
    undefs = configuration.get_temp_files().get('.undefined').name
    with open(undefs, 'w') as f:
      f.write('\n'.join(external_symbol_list))
    cmd.append('--allow-undefined-file=%s' % undefs)
  else:
    cmd.append('--allow-undefined')

  if Settings.IMPORTED_MEMORY:
    cmd.append('--import-memory')

  if Settings.USE_PTHREADS:
    cmd.append('--shared-memory')

  if Settings.MEMORY64:
    cmd.append('-mwasm64')

  # wasm-ld can strip debug info for us. this strips both the Names
  # section and DWARF, so we can only use it when we don't need any of
  # those things.
  if Settings.DEBUG_LEVEL < 2 and (not Settings.EMIT_SYMBOL_MAP and
                                   not Settings.PROFILING_FUNCS and
                                   not Settings.ASYNCIFY):
    cmd.append('--strip-debug')

  if Settings.RELOCATABLE:
    if Settings.MAIN_MODULE == 2 or Settings.SIDE_MODULE == 2:
      cmd.append('--no-export-dynamic')
    else:
      cmd.append('--no-gc-sections')
      cmd.append('--export-dynamic')
  else:
    cmd.append('--export-table')
    if Settings.ALLOW_TABLE_GROWTH:
      cmd.append('--growable-table')

  if Settings.LINKABLE:
    cmd.append('--export-all')
  else:
    c_exports = [e for e in Settings.EXPORTED_FUNCTIONS if is_c_symbol(e)]
    # Strip the leading underscores
    c_exports = [demangle_c_symbol_name(e) for e in c_exports]
    if external_symbol_list:
      # Filter out symbols external/JS symbols
      c_exports = [e for e in c_exports if e not in external_symbol_list]
    for export in c_exports:
      cmd += ['--export', export]

  if Settings.RELOCATABLE:
    cmd.append('--experimental-pic')
    if Settings.SIDE_MODULE:
      cmd.append('-shared')
    else:
      cmd.append('-pie')

  if not Settings.SIDE_MODULE:
    cmd += [
      '-z', 'stack-size=%s' % Settings.TOTAL_STACK,
      '--initial-memory=%d' % Settings.INITIAL_MEMORY,
    ]

    if Settings.STANDALONE_WASM:
      # when Settings.EXPECT_MAIN is set we fall back to wasm-ld default of _start
      if not Settings.EXPECT_MAIN:
        cmd += ['--entry=_initialize']
    else:
      if Settings.EXPECT_MAIN and not Settings.IGNORE_MISSING_MAIN:
        cmd += ['--entry=main']
      else:
        cmd += ['--no-entry']
    if not Settings.ALLOW_MEMORY_GROWTH:
      cmd.append('--max-memory=%d' % Settings.INITIAL_MEMORY)
    elif Settings.MAXIMUM_MEMORY != -1:
      cmd.append('--max-memory=%d' % Settings.MAXIMUM_MEMORY)
    if not Settings.RELOCATABLE:
      cmd.append('--global-base=%s' % Settings.GLOBAL_BASE)

  return cmd


def link_lld(args, target, external_symbol_list=None):
  if not os.path.exists(WASM_LD):
    exit_with_error('linker binary not found in LLVM directory: %s', WASM_LD)
  # runs lld to link things.
  # lld doesn't currently support --start-group/--end-group since the
  # semantics are more like the windows linker where there is no need for
  # grouping.
  args = [a for a in args if a not in ('--start-group', '--end-group')]

  # Emscripten currently expects linkable output (SIDE_MODULE/MAIN_MODULE) to
  # include all archive contents.
  if Settings.LINKABLE:
    args.insert(0, '--whole-archive')
    args.append('--no-whole-archive')

  if Settings.STRICT:
    args.append('--fatal-warnings')

  cmd = [WASM_LD, '-o', target] + args
  for a in llvm_backend_args():
    cmd += ['-mllvm', a]

  # For relocatable output (generating an object file) we don't pass any of the
  # normal linker flags that are used when building and exectuable
  if '--relocatable' not in args and '-r' not in args:
    cmd += lld_flags_for_executable(external_symbol_list)

  cmd = get_command_with_possible_response_file(cmd)
  check_call(cmd)


def link_bitcode(files, target, force_archive_contents=False):
  # "Full-featured" linking: looks into archives (duplicates lld functionality)
  actual_files = []
  # Tracking unresolveds is necessary for .a linking, see below.
  # Specify all possible entry points to seed the linking process.
  # For a simple application, this would just be "main".
  unresolved_symbols = set([func[1:] for func in Settings.EXPORTED_FUNCTIONS])
  resolved_symbols = set()
  # Paths of already included object files from archives.
  added_contents = set()
  has_ar = False
  for f in files:
    if not f.startswith('-'):
      has_ar = has_ar or is_ar(make_paths_absolute(f))

  # If we have only one archive or the force_archive_contents flag is set,
  # then we will add every object file we see, regardless of whether it
  # resolves any undefined symbols.
  force_add_all = len(files) == 1 or force_archive_contents

  # Considers an object file for inclusion in the link. The object is included
  # if force_add=True or if the object provides a currently undefined symbol.
  # If the object is included, the symbol tables are updated and the function
  # returns True.
  def consider_object(f, force_add=False):
    new_symbols = llvm_nm(f)
    # Check if the object was valid according to llvm-nm. It also accepts
    # native object files.
    if not new_symbols.is_valid_for_nm():
      diagnostics.warning('emcc', 'object %s is not valid according to llvm-nm, cannot link', f)
      return False
    # Check the object is valid for us, and not a native object file.
    if not is_bitcode(f):
      exit_with_error('unknown file type: %s', f)
    provided = new_symbols.defs.union(new_symbols.commons)
    do_add = force_add or not unresolved_symbols.isdisjoint(provided)
    if do_add:
      logger.debug('adding object %s to link (forced: %d)' % (f, force_add))
      # Update resolved_symbols table with newly resolved symbols
      resolved_symbols.update(provided)
      # Update unresolved_symbols table by adding newly unresolved symbols and
      # removing newly resolved symbols.
      unresolved_symbols.update(new_symbols.undefs.difference(resolved_symbols))
      unresolved_symbols.difference_update(provided)
      actual_files.append(f)
    return do_add

  # Traverse a single archive. The object files are repeatedly scanned for
  # newly satisfied symbols until no new symbols are found. Returns true if
  # any object files were added to the link.
  def consider_archive(f, force_add):
    added_any_objects = False
    loop_again = True
    logger.debug('considering archive %s' % (f))
    contents = ar_contents[f]
    while loop_again: # repeatedly traverse until we have everything we need
      loop_again = False
      for content in contents:
        if content in added_contents:
          continue
        # Link in the .o if it provides symbols, *or* this is a singleton archive (which is
        # apparently an exception in gcc ld)
        if consider_object(content, force_add=force_add):
          added_contents.add(content)
          loop_again = True
          added_any_objects = True
    logger.debug('done running loop of archive %s' % (f))
    return added_any_objects

  read_link_inputs([x for x in files if not x.startswith('-')])

  # Rescan a group of archives until we don't find any more objects to link.
  def scan_archive_group(group):
    loop_again = True
    logger.debug('starting archive group loop')
    while loop_again:
      loop_again = False
      for archive in group:
        if consider_archive(archive, force_add=False):
          loop_again = True
    logger.debug('done with archive group loop')

  current_archive_group = None
  in_whole_archive = False
  for f in files:
    absolute_path_f = make_paths_absolute(f)
    if f.startswith('-'):
      if f in ['--start-group', '-(']:
        assert current_archive_group is None, 'Nested --start-group, missing --end-group?'
        current_archive_group = []
      elif f in ['--end-group', '-)']:
        assert current_archive_group is not None, '--end-group without --start-group'
        scan_archive_group(current_archive_group)
        current_archive_group = None
      elif f in ['--whole-archive', '-whole-archive']:
        in_whole_archive = True
      elif f in ['--no-whole-archive', '-no-whole-archive']:
        in_whole_archive = False
      else:
        # Command line flags should already be vetted by the time this method
        # is called, so this is an internal error
        assert False, 'unsupported link flag: ' + f
    elif is_ar(absolute_path_f):
      # Extract object files from ar archives, and link according to gnu ld semantics
      # (link in an entire .o from the archive if it supplies symbols still unresolved)
      consider_archive(absolute_path_f, in_whole_archive or force_add_all)
      # If we're inside a --start-group/--end-group section, add to the list
      # so we can loop back around later.
      if current_archive_group is not None:
        current_archive_group.append(absolute_path_f)
    elif is_bitcode(absolute_path_f):
      if has_ar:
        consider_object(f, force_add=True)
      else:
        # If there are no archives then we can simply link all valid object
        # files and skip the symbol table stuff.
        actual_files.append(f)
    else:
      exit_with_error('unknown file type: %s', f)

  # We have to consider the possibility that --start-group was used without a matching
  # --end-group; GNU ld permits this behavior and implicitly treats the end of the
  # command line as having an --end-group.
  if current_archive_group:
    logger.debug('--start-group without matching --end-group, rescanning')
    scan_archive_group(current_archive_group)
    current_archive_group = None

  try_delete(target)

  # Finish link
  # tolerate people trying to link a.so a.so etc.
  actual_files = unique_ordered(actual_files)

  logger.debug('emcc: linking: %s to %s', actual_files, target)
  link_llvm(actual_files, target)


def get_command_with_possible_response_file(cmd):
  # 8k is a bit of an arbitrary limit, but a reasonable one
  # for max command line size before we use a response file
  if len(' '.join(cmd)) <= 8192:
    return cmd

  logger.debug('using response file for %s' % cmd[0])
  filename = response_file.create_response_file(cmd[1:], TEMP_DIR)
  new_cmd = [cmd[0], "@" + filename]
  return new_cmd


def parse_symbols(output, include_internal=False):
  defs = []
  undefs = []
  commons = []
  for line in output.split('\n'):
    if not line or line[0] == '#':
      continue
    # e.g.  filename.o:  , saying which file it's from
    if ':' in line:
      continue
    parts = [seg for seg in line.split(' ') if len(seg)]
    # pnacl-nm will print zero offsets for bitcode, and newer llvm-nm will print present symbols
    # as  -------- T name
    if len(parts) == 3 and parts[0] == "--------" or re.match(r'^[\da-f]{8}$', parts[0]):
      parts.pop(0)
    if len(parts) == 2:
      # ignore lines with absolute offsets, these are not bitcode anyhow
      # e.g. |00000630 t d_source_name|
      status, symbol = parts
      if status == 'U':
        undefs.append(symbol)
      elif status == 'C':
        commons.append(symbol)
      elif (not include_internal and status == status.upper()) or \
           (include_internal and status in ['W', 't', 'T', 'd', 'D']):
        # FIXME: using WTD in the previous line fails due to llvm-nm behavior on macOS,
        #        so for now we assume all uppercase are normally defined external symbols
        defs.append(symbol)
  return ObjectFileInfo(0, None, set(defs), set(undefs), set(commons))


def llvm_nm_uncached(filename, stdout=PIPE, stderr=PIPE, include_internal=False):
  # LLVM binary ==> list of symbols
  proc = run_process([LLVM_NM, filename], stdout=stdout, stderr=stderr, check=False)
  if proc.returncode == 0:
    return parse_symbols(proc.stdout, include_internal)
  else:
    return ObjectFileInfo(proc.returncode, str(proc.stdout) + str(proc.stderr))


def llvm_nm(filename, stdout=PIPE, stderr=PIPE, include_internal=False):
  # Always use absolute paths to maximize cache usage
  filename = os.path.abspath(filename)

  if include_internal and filename in internal_nm_cache:
    return internal_nm_cache[filename]
  elif not include_internal and filename in uninternal_nm_cache:
    return uninternal_nm_cache[filename]

  ret = llvm_nm_uncached(filename, stdout, stderr, include_internal)

  if ret.returncode != 0:
    logger.debug('llvm-nm failed on file ' + filename + ': return code ' + str(ret.returncode) + ', error: ' + ret.output)

  # Even if we fail, write the results to the NM cache so that we don't keep trying to llvm-nm the failing file again later.
  if include_internal:
    internal_nm_cache[filename] = ret
  else:
    uninternal_nm_cache[filename] = ret

  return ret


def emcc(filename, args=[], output_filename=None, stdout=None, stderr=None, env=None):
  if output_filename is None:
    output_filename = filename + '.o'
  try_delete(output_filename)
  run_process([EMCC, filename] + args + ['-o', output_filename], stdout=stdout, stderr=stderr, env=env)


def emar(action, output_filename, filenames, stdout=None, stderr=None, env=None):
  try_delete(output_filename)
  response_filename = response_file.create_response_file(filenames, TEMP_DIR)
  cmd = [EMAR, action, output_filename] + ['@' + response_filename]
  try:
    run_process(cmd, stdout=stdout, stderr=stderr, env=env)
  finally:
    try_delete(response_filename)

  if 'c' in action:
    assert os.path.exists(output_filename), 'emar could not create output file: ' + output_filename


def get_safe_internalize():
  if Settings.LINKABLE:
    return [] # do not internalize anything

  exps = Settings.EXPORTED_FUNCTIONS
  internalize_public_api = '-internalize-public-api-'
  internalize_list = ','.join([demangle_c_symbol_name(exp) for exp in exps])

  # EXPORTED_FUNCTIONS can potentially be very large.
  # 8k is a bit of an arbitrary limit, but a reasonable one
  # for max command line size before we use a response file
  if len(internalize_list) > 8192:
    logger.debug('using response file for EXPORTED_FUNCTIONS in internalize')
    finalized_exports = '\n'.join([exp[1:] for exp in exps])
    internalize_list_file = configuration.get_temp_files().get('.response').name
    with open(internalize_list_file, 'w') as f:
      f.write(finalized_exports)
    internalize_public_api += 'file=' + internalize_list_file
  else:
    internalize_public_api += 'list=' + internalize_list

  # internalize carefully, llvm 3.2 will remove even main if not told not to
  return ['-internalize', internalize_public_api]


def opt_level_to_str(opt_level, shrink_level=0):
  # convert opt_level/shrink_level pair to a string argument like -O1
  if opt_level == 0:
    return '-O0'
  if shrink_level == 1:
    return '-Os'
  elif shrink_level >= 2:
    return '-Oz'
  else:
    return '-O' + str(min(opt_level, 3))


def js_optimizer(filename, passes):
  from . import js_optimizer
  try:
    return js_optimizer.run(filename, passes)
  except subprocess.CalledProcessError as e:
    exit_with_error("'%s' failed (%d)", ' '.join(e.cmd), e.returncode)


# run JS optimizer on some JS, ignoring asm.js contents if any - just run on it all
def acorn_optimizer(filename, passes, extra_info=None, return_output=False):
  optimizer = path_from_root('tools', 'acorn-optimizer.js')
  original_filename = filename
  if extra_info is not None:
    temp_files = configuration.get_temp_files()
    temp = temp_files.get('.js').name
    shutil.copyfile(filename, temp)
    with open(temp, 'a') as f:
      f.write('// EXTRA_INFO: ' + extra_info)
    filename = temp
  cmd = config.NODE_JS + [optimizer, filename] + passes
  # Keep JS code comments intact through the acorn optimization pass so that JSDoc comments
  # will be carried over to a later Closure run.
  if Settings.USE_CLOSURE_COMPILER:
    cmd += ['--closureFriendly']
  if not return_output:
    next = original_filename + '.jso.js'
    configuration.get_temp_files().note(next)
    check_call(cmd, stdout=open(next, 'w'))
    save_intermediate(next, '%s.js' % passes[0])
    return next
  output = check_call(cmd, stdout=PIPE).stdout
  return output


# evals ctors. if binaryen_bin is provided, it is the dir of the binaryen tool
# for this, and we are in wasm mode
def eval_ctors(js_file, binary_file, debug_info=False): # noqa
  logger.debug('Ctor evalling in the wasm backend is disabled due to https://github.com/emscripten-core/emscripten/issues/9527')
  return
  # TODO re-enable
  # cmd = [PYTHON, path_from_root('tools', 'ctor_evaller.py'), js_file, binary_file, str(Settings.INITIAL_MEMORY), str(Settings.TOTAL_STACK), str(Settings.GLOBAL_BASE), binaryen_bin, str(int(debug_info))]
  # if binaryen_bin:
  #   cmd += get_binaryen_feature_flags()
  # check_call(cmd)


def get_closure_compiler():
  # First check if the user configured a specific CLOSURE_COMPILER in thier settings
  if config.CLOSURE_COMPILER:
    return shared.CLOSURE_COMPILER

  # Otherwise use the system-shared one
  cmd = ['closure-compiler']
  if not WINDOWS:
    # Work around an issue that Closure compiler can take up a lot of memory and crash in an error
    # "FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap
    # out of memory"
    cmd.insert(-1, '--max_old_space_size=8192')
  return cmd


def check_closure_compiler(cmd, args, env, allowed_to_fail):
  try:
    output = run_process(cmd + args + ['--version'], stdout=PIPE, env=env).stdout
  except Exception as e:
    if allowed_to_fail:
      return False
    logger.warn(str(e))
    exit_with_error('closure compiler ("%s --version") did not execute properly!' % str(cmd))

  if 'Version:' not in output:
    if allowed_to_fail:
      return False
    exit_with_error('unrecognized closure compiler --version output (%s):\n%s' % (str(cmd), output))

  return True


def closure_compiler(filename, pretty=True, advanced=True, extra_closure_args=None):
  with ToolchainProfiler.profile_block('closure_compiler'):
    env = shared.env_with_node_in_path()
    user_args = []
    env_args = os.environ.get('EMCC_CLOSURE_ARGS')
    if env_args:
      user_args += shlex.split(env_args)
    if extra_closure_args:
      user_args += extra_closure_args

    # Closure compiler expects JAVA_HOME to be set *and* java.exe to be in the PATH in order
    # to enable use the java backend.  Without this it will only try the native and JavaScript
    # versions of the compiler.
    java_bin = os.path.dirname(config.JAVA)
    if java_bin:
      def add_to_path(dirname):
        env['PATH'] = env['PATH'] + os.pathsep + dirname
      add_to_path(java_bin)
      java_home = os.path.dirname(java_bin)
      env.setdefault('JAVA_HOME', java_home)

    closure_cmd = get_closure_compiler()

    # skip checking option --version which hangs with old Debian release.
    #native_closure_compiler_works = check_closure_compiler(closure_cmd, user_args, env, allowed_to_fail=True)
    #if not native_closure_compiler_works and not any(a.startswith('--platform') for a in user_args):
    #  # Run with Java Closure compiler as a fallback if the native version does not work
    #  user_args.append('--platform=java')
    #  check_closure_compiler(closure_cmd, user_args, env, allowed_to_fail=False)

    # Closure externs file contains known symbols to be extern to the minification, Closure
    # should not minify these symbol names.
    CLOSURE_EXTERNS = [path_from_root('src', 'closure-externs', 'closure-externs.js')]

    # Closure compiler needs to know about all exports that come from the asm.js/wasm module, because to optimize for small code size,
    # the exported symbols are added to global scope via a foreach loop in a way that evades Closure's static analysis. With an explicit
    # externs file for the exports, Closure is able to reason about the exports.
    if Settings.MODULE_EXPORTS and not Settings.DECLARE_ASM_MODULE_EXPORTS:
      # Generate an exports file that records all the exported symbols from asm.js/wasm module.
      module_exports_suppressions = '\n'.join(['/**\n * @suppress {duplicate, undefinedVars}\n */\nvar %s;\n' % i for i, j in Settings.MODULE_EXPORTS])
      exports_file = configuration.get_temp_files().get('_module_exports.js')
      exports_file.write(module_exports_suppressions.encode())
      exports_file.close()

      CLOSURE_EXTERNS += [exports_file.name]

    # Node.js specific externs
    if Settings.target_environment_may_be('node'):
      NODE_EXTERNS_BASE = path_from_root('third_party', 'closure-compiler', 'node-externs')
      NODE_EXTERNS = os.listdir(NODE_EXTERNS_BASE)
      NODE_EXTERNS = [os.path.join(NODE_EXTERNS_BASE, name) for name in NODE_EXTERNS
                      if name.endswith('.js')]
      CLOSURE_EXTERNS += [path_from_root('src', 'closure-externs', 'node-externs.js')] + NODE_EXTERNS

    # V8/SpiderMonkey shell specific externs
    if Settings.target_environment_may_be('shell'):
      V8_EXTERNS = [path_from_root('src', 'closure-externs', 'v8-externs.js')]
      SPIDERMONKEY_EXTERNS = [path_from_root('src', 'closure-externs', 'spidermonkey-externs.js')]
      CLOSURE_EXTERNS += V8_EXTERNS + SPIDERMONKEY_EXTERNS

    # Web environment specific externs
    if Settings.target_environment_may_be('web') or Settings.target_environment_may_be('worker'):
      BROWSER_EXTERNS_BASE = path_from_root('src', 'closure-externs', 'browser-externs')
      if os.path.isdir(BROWSER_EXTERNS_BASE):
        BROWSER_EXTERNS = os.listdir(BROWSER_EXTERNS_BASE)
        BROWSER_EXTERNS = [os.path.join(BROWSER_EXTERNS_BASE, name) for name in BROWSER_EXTERNS
                           if name.endswith('.js')]
        CLOSURE_EXTERNS += BROWSER_EXTERNS

    if Settings.MINIMAL_RUNTIME and Settings.USE_PTHREADS and not Settings.MODULARIZE:
      CLOSURE_EXTERNS += [path_from_root('src', 'minimal_runtime_worker_externs.js')]
    outfile = filename + '.cc.js'
    configuration.get_temp_files().note(outfile)

    args = ['--compilation_level', 'ADVANCED_OPTIMIZATIONS' if advanced else 'SIMPLE_OPTIMIZATIONS']
    # Keep in sync with ecmaVersion in tools/acorn-optimizer.js
    args += ['--language_in', 'ECMASCRIPT_NEXT_IN']
    # Tell closure not to do any transpiling or inject any polyfills.
    # At some point we may want to look into using this as way to convert to ES5 but
    # babel is perhaps a better tool for that.
    args += ['--language_out', 'NO_TRANSPILE']
    # Tell closure never to inject the 'use strict' directive.
    args += ['--emit_use_strict=false']

    for e in CLOSURE_EXTERNS:
      args += ['--externs', e]
    args += ['--js_output_file', outfile]

    if Settings.IGNORE_CLOSURE_COMPILER_ERRORS:
      args.append('--jscomp_off=*')
    if pretty:
      args += ['--formatting', 'PRETTY_PRINT']
    args += ['--js', filename]
    cmd = closure_cmd + args + user_args
    logger.debug('closure compiler: ' + ' '.join(cmd))

    proc = run_process(cmd, stderr=PIPE, check=False, env=env)

    # XXX Closure bug: if Closure is invoked with --create_source_map, Closure should create a
    # outfile.map source map file (https://github.com/google/closure-compiler/wiki/Source-Maps)
    # But it looks like it creates such files on Linux(?) even without setting that command line
    # flag (and currently we don't), so delete the produced source map file to not leak files in
    # temp directory.
    try_delete(outfile + '.map')

    # Print Closure diagnostics result up front.
    if proc.returncode != 0:
      logger.error('Closure compiler run failed:\n')
    elif len(proc.stderr.strip()) > 0:
      if Settings.CLOSURE_WARNINGS == 'error':
        logger.error('Closure compiler completed with warnings and -s CLOSURE_WARNINGS=error enabled, aborting!\n')
      elif Settings.CLOSURE_WARNINGS == 'warn':
        logger.warn('Closure compiler completed with warnings:\n')

    # Print input file (long wall of text!)
    if DEBUG == 2 and (proc.returncode != 0 or (len(proc.stderr.strip()) > 0 and Settings.CLOSURE_WARNINGS != 'quiet')):
      input_file = open(filename, 'r').read().splitlines()
      for i in range(len(input_file)):
        sys.stderr.write(str(i + 1) + ': ' + input_file[i] + '\n')

    if proc.returncode != 0:
      logger.error(proc.stderr) # print list of errors (possibly long wall of text if input was minified)

      # Exit and print final hint to get clearer output
      msg = 'closure compiler failed (rc: %d): %s' % (proc.returncode, shared.shlex_join(cmd))
      if not pretty:
        msg += ' the error message may be clearer with -g1 and EMCC_DEBUG=2 set'
      exit_with_error(msg)

    if len(proc.stderr.strip()) > 0 and Settings.CLOSURE_WARNINGS != 'quiet':
      # print list of warnings (possibly long wall of text if input was minified)
      if Settings.CLOSURE_WARNINGS == 'error':
        logger.error(proc.stderr)
      else:
        logger.warn(proc.stderr)

      # Exit and/or print final hint to get clearer output
      if not pretty:
        logger.warn('(rerun with -g1 linker flag for an unminified output)')
      elif DEBUG != 2:
        logger.warn('(rerun with EMCC_DEBUG=2 enabled to dump Closure input file)')

      if Settings.CLOSURE_WARNINGS == 'error':
        exit_with_error('closure compiler produced warnings and -s CLOSURE_WARNINGS=error enabled')

    return outfile


# minify the final wasm+JS combination. this is done after all the JS
# and wasm optimizations; here we do the very final optimizations on them
def minify_wasm_js(js_file, wasm_file, expensive_optimizations, minify_whitespace, debug_info):
  # start with JSDCE, to clean up obvious JS garbage. When optimizing for size,
  # use AJSDCE (aggressive JS DCE, performs multiple iterations). Clean up
  # whitespace if necessary too.
  passes = []
  if not Settings.LINKABLE:
    passes.append('JSDCE' if not expensive_optimizations else 'AJSDCE')
  if minify_whitespace:
    passes.append('minifyWhitespace')
  if passes:
    logger.debug('running cleanup on shell code: ' + ' '.join(passes))
    js_file = acorn_optimizer(js_file, passes)
  # if we can optimize this js+wasm combination under the assumption no one else
  # will see the internals, do so
  if not Settings.LINKABLE:
    # if we are optimizing for size, shrink the combined wasm+JS
    # TODO: support this when a symbol map is used
    if expensive_optimizations:
      js_file = metadce(js_file, wasm_file, minify_whitespace=minify_whitespace, debug_info=debug_info)
      # now that we removed unneeded communication between js and wasm, we can clean up
      # the js some more.
      passes = ['AJSDCE']
      if minify_whitespace:
        passes.append('minifyWhitespace')
      logger.debug('running post-meta-DCE cleanup on shell code: ' + ' '.join(passes))
      js_file = acorn_optimizer(js_file, passes)
      if Settings.MINIFY_WASM_IMPORTS_AND_EXPORTS:
        js_file = minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace=minify_whitespace, minify_exports=Settings.MINIFY_ASMJS_EXPORT_NAMES, debug_info=debug_info)
  return js_file


# run binaryen's wasm-metadce to dce both js and wasm
def metadce(js_file, wasm_file, minify_whitespace, debug_info):
  logger.debug('running meta-DCE')
  temp_files = configuration.get_temp_files()
  # first, get the JS part of the graph
  extra_info = '{ "exports": [' + ','.join(map(lambda x: '["' + x[0] + '","' + x[1] + '"]', Settings.MODULE_EXPORTS)) + ']}'
  txt = acorn_optimizer(js_file, ['emitDCEGraph', 'noPrint'], return_output=True, extra_info=extra_info)
  graph = json.loads(txt)
  # add exports based on the backend output, that are not present in the JS
  if not Settings.DECLARE_ASM_MODULE_EXPORTS:
    exports = set()
    for item in graph:
      if 'export' in item:
        exports.add(item['export'])
    for export, unminified in Settings.MODULE_EXPORTS:
      if export not in exports:
        graph.append({
          'export': export,
          'name': 'emcc$export$' + export,
          'reaches': []
        })
  # ensure that functions expected to be exported to the outside are roots
  for item in graph:
    if 'export' in item:
      export = item['export']
      # wasm backend's exports are prefixed differently inside the wasm
      export = asmjs_mangle(export)
      if export in user_requested_exports or Settings.EXPORT_ALL:
        item['root'] = True
  # in standalone wasm, always export the memory
  if not Settings.IMPORTED_MEMORY:
    graph.append({
      'export': 'memory',
      'name': 'emcc$export$memory',
      'reaches': [],
      'root': True
    })
  if not Settings.RELOCATABLE:
    graph.append({
      'export': '__indirect_function_table',
      'name': 'emcc$export$__indirect_function_table',
      'reaches': [],
      'root': True
    })
  # fix wasi imports TODO: support wasm stable with an option?
  WASI_IMPORTS = set([
    'environ_get',
    'environ_sizes_get',
    'args_get',
    'args_sizes_get',
    'fd_write',
    'fd_close',
    'fd_read',
    'fd_seek',
    'fd_fdstat_get',
    'fd_sync',
    'fd_pread',
    'fd_pwrite',
    'proc_exit',
    'clock_res_get',
    'clock_time_get',
  ])
  for item in graph:
    if 'import' in item and item['import'][1][1:] in WASI_IMPORTS:
      item['import'][0] = Settings.WASI_MODULE_NAME
  # fixup wasm backend prefixing
  for item in graph:
    if 'import' in item:
      if item['import'][1][0] == '_':
        item['import'][1] = item['import'][1][1:]
  # map import names from wasm to JS, using the actual name the wasm uses for the import
  import_name_map = {}
  for item in graph:
    if 'import' in item:
      import_name_map[item['name']] = 'emcc$import$' + item['import'][1]
  temp = temp_files.get('.txt').name
  txt = json.dumps(graph)
  with open(temp, 'w') as f:
    f.write(txt)
  # run wasm-metadce
  out = run_binaryen_command('wasm-metadce',
                             wasm_file,
                             wasm_file,
                             ['--graph-file=' + temp],
                             debug=debug_info,
                             stdout=PIPE)
  # find the unused things in js
  unused = []
  PREFIX = 'unused: '
  for line in out.splitlines():
    if line.startswith(PREFIX):
      name = line.replace(PREFIX, '').strip()
      if name in import_name_map:
        name = import_name_map[name]
      unused.append(name)
  # remove them
  passes = ['applyDCEGraphRemovals']
  if minify_whitespace:
    passes.append('minifyWhitespace')
  extra_info = {'unused': unused}
  return acorn_optimizer(js_file, passes, extra_info=json.dumps(extra_info))


def asyncify_lazy_load_code(wasm_target, debug):
  # create the lazy-loaded wasm. remove the memory segments from it, as memory
  # segments have already been applied by the initial wasm, and apply the knowledge
  # that it will only rewind, after which optimizations can remove some code
  args = ['--remove-memory', '--mod-asyncify-never-unwind']
  if Settings.OPT_LEVEL > 0:
    args.append(opt_level_to_str(Settings.OPT_LEVEL, Settings.SHRINK_LEVEL))
  run_wasm_opt(wasm_target,
               wasm_target + '.lazy.wasm',
               args=args,
               debug=debug)
  # re-optimize the original, by applying the knowledge that imports will
  # definitely unwind, and we never rewind, after which optimizations can remove
  # a lot of code
  # TODO: support other asyncify stuff, imports that don't always unwind?
  # TODO: source maps etc.
  args = ['--mod-asyncify-always-and-only-unwind']
  if Settings.OPT_LEVEL > 0:
    args.append(opt_level_to_str(Settings.OPT_LEVEL, Settings.SHRINK_LEVEL))
  run_wasm_opt(infile=wasm_target,
               outfile=wasm_target,
               args=args,
               debug=debug)


def minify_wasm_imports_and_exports(js_file, wasm_file, minify_whitespace, minify_exports, debug_info):
  logger.debug('minifying wasm imports and exports')
  # run the pass
  if minify_exports:
    # standalone wasm mode means we need to emit a wasi import module.
    # otherwise, minify even the imported module names.
    if Settings.MINIFY_WASM_IMPORTED_MODULES:
      pass_name = '--minify-imports-and-exports-and-modules'
    else:
      pass_name = '--minify-imports-and-exports'
  else:
    pass_name = '--minify-imports'
  out = run_wasm_opt(wasm_file, wasm_file,
                     [pass_name],
                     debug=debug_info,
                     stdout=PIPE)
  # TODO this is the last tool we run, after normal opts and metadce. it
  # might make sense to run Stack IR optimizations here or even -O (as
  # metadce which runs before us might open up new general optimization
  # opportunities). however, the benefit is less than 0.5%.

  # get the mapping
  SEP = ' => '
  mapping = {}
  for line in out.split('\n'):
    if SEP in line:
      old, new = line.strip().split(SEP)
      assert old not in mapping, 'imports must be unique'
      mapping[old] = new
  # apply them
  passes = ['applyImportAndExportNameChanges']
  if minify_whitespace:
    passes.append('minifyWhitespace')
  extra_info = {'mapping': mapping}
  return acorn_optimizer(js_file, passes, extra_info=json.dumps(extra_info))


def wasm2js(js_file, wasm_file, opt_level, minify_whitespace, use_closure_compiler, debug_info, symbols_file=None):
  logger.debug('wasm2js')
  args = ['--emscripten']
  if opt_level > 0:
    args += ['-O']
  if symbols_file:
    args += ['--symbols-file=%s' % symbols_file]
  wasm2js_js = run_binaryen_command('wasm2js', wasm_file,
                                    args=args,
                                    debug=debug_info,
                                    stdout=PIPE)
  if DEBUG:
    with open(os.path.join(get_emscripten_temp_dir(), 'wasm2js-output.js'), 'w') as f:
      f.write(wasm2js_js)
  # JS optimizations
  if opt_level >= 2:
    passes = []
    if not debug_info and not Settings.USE_PTHREADS:
      passes += ['minifyNames']
    if minify_whitespace:
      passes += ['minifyWhitespace']
    passes += ['last']
    if passes:
      # hackish fixups to work around wasm2js style and the js optimizer FIXME
      wasm2js_js = '// EMSCRIPTEN_START_ASM\n' + wasm2js_js + '// EMSCRIPTEN_END_ASM\n'
      wasm2js_js = wasm2js_js.replace('// EMSCRIPTEN_START_FUNCS;\n', '// EMSCRIPTEN_START_FUNCS\n')
      wasm2js_js = wasm2js_js.replace('// EMSCRIPTEN_END_FUNCS;\n', '// EMSCRIPTEN_END_FUNCS\n')
      wasm2js_js = wasm2js_js.replace('\n function $', '\nfunction $')
      wasm2js_js = wasm2js_js.replace('\n }', '\n}')
      wasm2js_js += '\n// EMSCRIPTEN_GENERATED_FUNCTIONS\n'
      temp = configuration.get_temp_files().get('.js').name
      with open(temp, 'w') as f:
        f.write(wasm2js_js)
      temp = js_optimizer(temp, passes)
      with open(temp) as f:
        wasm2js_js = f.read()
  # Closure compiler: in mode 1, we just minify the shell. In mode 2, we
  # minify the wasm2js output as well, which is ok since it isn't
  # validating asm.js.
  # TODO: in the non-closure case, we could run a lightweight general-
  #       purpose JS minifier here.
  if use_closure_compiler == 2:
    temp = configuration.get_temp_files().get('.js').name
    with open(temp, 'a') as f:
      f.write(wasm2js_js)
    temp = closure_compiler(temp, pretty=not minify_whitespace, advanced=False)
    with open(temp) as f:
      wasm2js_js = f.read()
    # closure may leave a trailing `;`, which would be invalid given where we place
    # this code (inside parens)
    wasm2js_js = wasm2js_js.strip()
    if wasm2js_js[-1] == ';':
      wasm2js_js = wasm2js_js[:-1]
  with open(js_file) as f:
    all_js = f.read()
  # quoted notation, something like Module['__wasm2jsInstantiate__']
  finds = re.findall(r'''[\w\d_$]+\[['"]__wasm2jsInstantiate__['"]\]''', all_js)
  if not finds:
    # post-closure notation, something like a.__wasm2jsInstantiate__
    finds = re.findall(r'''[\w\d_$]+\.__wasm2jsInstantiate__''', all_js)
  assert len(finds) == 1
  marker = finds[0]
  all_js = all_js.replace(marker, '(\n' + wasm2js_js + '\n)')
  # replace the placeholder with the actual code
  js_file = js_file + '.wasm2js.js'
  with open(js_file, 'w') as f:
    f.write(all_js)
  return js_file


def strip(infile, outfile, debug=False, producers=False):
  cmd = [LLVM_OBJCOPY, infile, outfile]
  if debug:
    cmd += ['--remove-section=.debug*']
  if producers:
    cmd += ['--remove-section=producers']
  check_call(cmd)


# extract the DWARF info from the main file, and leave the wasm with
# debug into as a file on the side
# TODO: emit only debug sections in the side file, and not the entire
#       wasm as well
def emit_debug_on_side(wasm_file, wasm_file_with_dwarf):
  # if the dwarf filename wasn't provided, use the default target + a suffix
  wasm_file_with_dwarf = shared.Settings.SEPARATE_DWARF
  if wasm_file_with_dwarf is True:
    wasm_file_with_dwarf = wasm_file + '.debug.wasm'
  embedded_path = shared.Settings.SEPARATE_DWARF_URL
  if not embedded_path:
    # a path was provided - make it relative to the wasm.
    embedded_path = os.path.relpath(wasm_file_with_dwarf,
                                    os.path.dirname(wasm_file))
    # normalize the path to use URL-style separators, per the spec
    embedded_path = embedded_path.replace('\\', '/').replace('//', '/')

  shutil.move(wasm_file, wasm_file_with_dwarf)
  strip(wasm_file_with_dwarf, wasm_file, debug=True)

  # embed a section in the main wasm to point to the file with external DWARF,
  # see https://yurydelendik.github.io/webassembly-dwarf/#external-DWARF
  section_name = b'\x13external_debug_info' # section name, including prefixed size
  filename_bytes = asbytes(embedded_path)
  contents = webassembly.toLEB(len(filename_bytes)) + filename_bytes
  section_size = len(section_name) + len(contents)
  with open(wasm_file, 'ab') as f:
    f.write(b'\0') # user section is code 0
    f.write(webassembly.toLEB(section_size))
    f.write(section_name)
    f.write(contents)


def apply_wasm_memory_growth(js_file):
  logger.debug('supporting wasm memory growth with pthreads')
  fixed = acorn_optimizer(js_file, ['growableHeap'])
  ret = js_file + '.pgrow.js'
  with open(fixed, 'r') as fixed_f:
    with open(ret, 'w') as ret_f:
      with open(path_from_root('src', 'growableHeap.js')) as support_code_f:
        ret_f.write(support_code_f.read() + '\n' + fixed_f.read())
  return ret


def use_unsigned_pointers_in_js(js_file):
  logger.debug('using unsigned pointers in JS')
  return acorn_optimizer(js_file, ['unsignPointers'])


def instrument_js_for_asan(js_file):
  logger.debug('instrumenting JS memory accesses for ASan')
  return acorn_optimizer(js_file, ['asanify'])


def instrument_js_for_safe_heap(js_file):
  logger.debug('instrumenting JS memory accesses for SAFE_HEAP')
  return acorn_optimizer(js_file, ['safeHeap'])


def handle_final_wasm_symbols(wasm_file, symbols_file, debug_info):
  logger.debug('handle_final_wasm_symbols')
  args = []
  if symbols_file:
    args += ['--print-function-map']
  if not debug_info:
    # to remove debug info, we just write to that same file, and without -g
    args += ['-o', wasm_file]
  else:
    # suppress the wasm-opt warning regarding "no output file specified"
    args += ['--quiet']
  # ignore stderr because if wasm-opt is run without a -o it will warn
  output = run_wasm_opt(wasm_file, args=args, stdout=PIPE)
  if symbols_file:
    with open(symbols_file, 'w') as f:
      f.write(output)


def is_ar(filename):
  try:
    if _is_ar_cache.get(filename):
      return _is_ar_cache[filename]
    header = open(filename, 'rb').read(8)
    sigcheck = header == b'!<arch>\n'
    _is_ar_cache[filename] = sigcheck
    return sigcheck
  except Exception as e:
    logger.debug('is_ar failed to test whether file \'%s\' is a llvm archive file! Failed on exception: %s' % (filename, e))
    return False


def is_bitcode(filename):
  try:
    # look for magic signature
    b = open(filename, 'rb').read(4)
    if b[:2] == b'BC':
      return True
    # on macOS, there is a 20-byte prefix which starts with little endian
    # encoding of 0x0B17C0DE
    elif b == b'\xDE\xC0\x17\x0B':
      b = bytearray(open(filename, 'rb').read(22))
      return b[20:] == b'BC'
  except IndexError:
    # not enough characters in the input
    # note that logging will be done on the caller function
    pass
  return False


def is_wasm(filename):
  magic = open(filename, 'rb').read(4)
  return magic == b'\0asm'


# Given the name of a special Emscripten-implemented system library, returns an
# array of absolute paths to JS library files inside emscripten/src/ that
# corresponds to the library name.
def map_to_js_libs(library_name):
  # Some native libraries are implemented in Emscripten as system side JS libraries
  library_map = {
    'c': [],
    'dl': [],
    'EGL': ['library_egl.js'],
    'GL': ['library_webgl.js', 'library_html5_webgl.js'],
    'webgl.js': ['library_webgl.js', 'library_html5_webgl.js'],
    'GLESv2': ['library_webgl.js'],
    # N.b. there is no GLESv3 to link to (note [f] in https://www.khronos.org/registry/implementers_guide.html)
    'GLEW': ['library_glew.js'],
    'glfw': ['library_glfw.js'],
    'glfw3': ['library_glfw.js'],
    'GLU': [],
    'glut': ['library_glut.js'],
    'm': [],
    'openal': ['library_openal.js'],
    'rt': [],
    'pthread': [],
    'X11': ['library_xlib.js'],
    'SDL': ['library_sdl.js'],
    'stdc++': [],
    'uuid': ['library_uuid.js'],
    'websocket': ['library_websocket.js']
  }

  if library_name in library_map:
    libs = library_map[library_name]
    logger.debug('Mapping library `%s` to JS libraries: %s' % (library_name, libs))
    return libs

  if library_name.endswith('.js') and os.path.isfile(path_from_root('src', 'library_' + library_name)):
    return ['library_' + library_name]

  return None


# map a linker flag to a Settings option, and apply it. this lets a user write
# -lSDL2 and it will have the same effect as -s USE_SDL=2.
def map_and_apply_to_settings(library_name):
  # most libraries just work, because the -l name matches the name of the
  # library we build. however, if a library has variations, which cause us to
  # build multiple versions with multiple names, then we need this mechanism.
  library_map = {
    # SDL2_mixer's built library name contains the specific codecs built in.
    'SDL2_mixer': [('USE_SDL_MIXER', 2)],
  }

  if library_name in library_map:
    for key, value in library_map[library_name]:
      logger.debug('Mapping library `%s` to settings changes: %s = %s' % (library_name, key, value))
      setattr(shared.Settings, key, value)
    return True

  return False


def emit_wasm_source_map(wasm_file, map_file, final_wasm):
  # source file paths must be relative to the location of the map (which is
  # emitted alongside the wasm)
  base_path = os.path.dirname(os.path.abspath(final_wasm))
  sourcemap_cmd = [PYTHON, path_from_root('tools', 'wasm-sourcemap.py'),
                   wasm_file,
                   '--dwarfdump=' + LLVM_DWARFDUMP,
                   '-o',  map_file,
                   '--basepath=' + base_path]
  check_call(sourcemap_cmd)


def get_binaryen_feature_flags():
  # start with the MVP features, add the rest as needed
  ret = ['--mvp-features']
  if Settings.USE_PTHREADS:
    ret += ['--enable-threads']
  if Settings.MEMORY64:
    ret += ['--enable-memory64']
  ret += Settings.BINARYEN_FEATURES
  return ret


def check_binaryen(bindir):
  opt = os.path.join(bindir, exe_suffix('wasm-opt'))
  if not os.path.exists(opt):
    exit_with_error('binaryen executable not found (%s). Please check your binaryen installation' % opt)
  try:
    output = run_process([opt, '--version'], stdout=PIPE).stdout
  except subprocess.CalledProcessError:
    exit_with_error('error running binaryen executable (%s). Please check your binaryen installation' % opt)
  if output:
    output = output.splitlines()[0]
  try:
    version = output.split()[2]
    version = int(version)
  except (IndexError, ValueError):
    exit_with_error('error parsing binaryen version (%s). Please check your binaryen installation (%s)' % (output, opt))

  # Allow the expected version or the following one in order avoid needing to update both
  # emscripten and binaryen in lock step in emscripten-releases.
  if version not in (EXPECTED_BINARYEN_VERSION, EXPECTED_BINARYEN_VERSION + 1):
    diagnostics.warning('version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION)


def get_binaryen_bin():
  assert Settings.WASM, 'non wasm builds should not ask for binaryen'
  global binaryen_checked
  rtn = os.path.join(config.BINARYEN_ROOT, 'bin')
  if not binaryen_checked:
    check_binaryen(rtn)
    binaryen_checked = True
  return rtn


def run_binaryen_command(tool, infile, outfile=None, args=[], debug=False, stdout=None):
  cmd = [os.path.join(get_binaryen_bin(), tool)]
  if outfile and tool == 'wasm-opt' and Settings.DEBUG_LEVEL != 3:
    # remove any dwarf debug info sections, if the debug level is <3, as
    # we don't need them; also remove them if we the level is 4, as then we
    # want a source map, which is implemented separately from dwarf.
    # note that we add this pass first, so that it doesn't interfere with
    # the final set of passes (which may generate stack IR, and nothing
    # should be run after that)
    # TODO: if lld can strip dwarf then we don't need this. atm though it can
    #       only strip all debug info or none, which includes the name section
    #       which we may need
    # TODO: once fastcomp is gone, either remove source maps entirely, or
    #       support them by emitting a source map at the end from the dwarf,
    #       and use llvm-objcopy to remove that final dwarf
    cmd += ['--strip-dwarf']
  cmd += args
  if infile:
    cmd += [infile]
  if outfile:
    cmd += ['-o', outfile]
    if Settings.ERROR_ON_WASM_CHANGES_AFTER_LINK:
      # emit some extra helpful text for common issues
      extra = ''
      # a plain -O0 build *almost* doesn't need post-link changes, except for
      # legalization. show a clear error for those (as the flags the user passed
      # in are not enough to see what went wrong)
      if shared.Settings.LEGALIZE_JS_FFI:
        extra += '\nnote: to disable int64 legalization (which requires changes after link) use -s WASM_BIGINT'
      if shared.Settings.OPT_LEVEL > 0:
        extra += '\nnote: -O2+ optimizations always require changes, build with -O0 or -O1 instead'
      exit_with_error('changes to the wasm are required after link, but disallowed by ERROR_ON_WASM_CHANGES_AFTER_LINK: ' + str(cmd) + extra)
  if debug:
    cmd += ['-g'] # preserve the debug info
  # if the features are not already handled, handle them
  if '--detect-features' not in cmd:
    cmd += get_binaryen_feature_flags()
  # if we are emitting a source map, every time we load and save the wasm
  # we must tell binaryen to update it
  if Settings.GENERATE_SOURCE_MAP and outfile:
    cmd += ['--input-source-map=' + infile + '.map']
    cmd += ['--output-source-map=' + outfile + '.map']
  ret = check_call(cmd, stdout=stdout).stdout
  if outfile:
    save_intermediate(outfile, '%s.wasm' % tool)
  return ret


def run_wasm_opt(*args, **kwargs):
  return run_binaryen_command('wasm-opt', *args, **kwargs)


save_intermediate_counter = 0


def save_intermediate(src, dst):
  if DEBUG:
    global save_intermediate_counter
    dst = 'emcc-%d-%s' % (save_intermediate_counter, dst)
    save_intermediate_counter += 1
    dst = os.path.join(CANONICAL_TEMP_DIR, dst)
    logger.debug('saving debug copy %s' % dst)
    shutil.copyfile(src, dst)