/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See https://swift.org/LICENSE.txt for license information
 See https://swift.org/CONTRIBUTORS.txt for Swift project authors
*/

import Foundation

/// A converter from a documentation bundle to an output that can be consumed by a renderer.
///
/// This protocol is primarily used for injecting mock documentation converters during testing.
///
/// ## See Also
///
/// - ``DocumentationConverter``
public protocol DocumentationConverterProtocol {
    /// Converts documentation, outputting products using the given output consumer.
    /// - Parameter outputConsumer: The output consumer for content produced during conversion.
    /// - Returns: The problems emitted during analysis of the documentation bundle and during conversion.
    /// - Throws: Throws an error if the conversion process was not able to start at all, for example if the bundle could not be read.
    /// Partial failures, such as failing to consume a single render node, are returned in the `conversionProblems` component
    /// of the returned tuple.
    mutating func convert(
        outputConsumer: some ConvertOutputConsumer
    ) throws -> (analysisProblems: [Problem], conversionProblems: [Problem])
}

/// A converter from a documentation bundle to an output that can be consumed by a renderer.
///
/// A documentation converter analyzes a documentation bundle and converts it to products that can be used by a documentation
/// renderer to render documentation. The output format of the conversion is controlled by a ``ConvertOutputConsumer``, which
/// determines what to do with the conversion products, for example, write them to disk.
///
/// You can also configure the documentation converter to emit extra metadata such as linkable entities and indexing records
/// information.
public struct DocumentationConverter: DocumentationConverterProtocol {
    let rootURL: URL?
    let emitDigest: Bool
    let documentationCoverageOptions: DocumentationCoverageOptions
    let bundleDiscoveryOptions: BundleDiscoveryOptions
    let diagnosticEngine: DiagnosticEngine
    
    private(set) var context: DocumentationContext
    private let workspace: DocumentationWorkspace
    private var currentDataProvider: DocumentationWorkspaceDataProvider?
    private var dataProvider: DocumentationWorkspaceDataProvider
    
    /// An optional closure that sets up a context before the conversion begins.
    public var setupContext: ((inout DocumentationContext) -> Void)?
    
    /// Conversion batches should be big enough to keep all cores busy but small enough not to keep
    /// around too many async blocks that update the conversion results. After running some tests it
    /// seems that more than couple hundred of a batch size doesn't bring more performance CPU-wise
    /// and it's a fair amount of async tasks to keep in memory before draining the results queue
    /// after the batch is converted.
    var batchNodeCount = 1
    
    /// The external IDs of the symbols to convert.
    ///
    /// Use this property to indicate what symbol documentation nodes should be converted. When ``externalIDsToConvert``
    /// and ``documentationPathsToConvert`` are both set, the documentation nodes that are in either arrays will be
    /// converted.
    ///
    /// If you want all the symbol render nodes to be returned as part of the conversion's response, set this property to `nil`.
    /// For Swift, the external ID of the symbol is its USR.
    var externalIDsToConvert: [String]?
    
    /// The paths of the documentation nodes to convert.
    ///
    /// Use this property to indicate what documentation nodes should be converted. When ``externalIDsToConvert``
    /// and ``documentationPathsToConvert`` are both set, the documentation nodes that are in either arrays will be
    /// converted.
    ///
    /// If you want all the render nodes to be returned as part of the conversion's response, set this property to `nil`.
    var documentPathsToConvert: [String]?
    
    /// Whether the documentation converter should include source file
    /// location metadata in any render nodes representing symbols it creates.
    ///
    /// Before setting this value to `true` please confirm that your use case doesn't include
    /// public distribution of any created render nodes as there are filesystem privacy and security
    /// concerns with distributing this data.
    var shouldEmitSymbolSourceFileURIs: Bool
    
    /// Whether the documentation converter should include access level information for symbols.
    var shouldEmitSymbolAccessLevels: Bool
    
    /// The source repository where the documentation's sources are hosted.
    var sourceRepository: SourceRepository?
    
    /// Whether the documentation converter should write documentation extension files containing markdown representations of DocC's automatic curation into the source documentation catalog.
    var experimentalModifyCatalogWithGeneratedCuration: Bool
    
    /// The identifiers and access level requirements for symbols that have an expanded version of their documentation page if the requirements are met
    var symbolIdentifiersWithExpandedDocumentation: [String: ConvertRequest.ExpandedDocumentationRequirements]? = nil
    
    /// `true` if the conversion is cancelled.
    private var isCancelled: Synchronized<Bool>? = nil

    private var processingDurationMetric: Benchmark.Duration?

    /// Creates a documentation converter given a documentation bundle's URL.
    /// 
    /// - Parameters:
    ///  - documentationBundleURL: The root URL of the documentation bundle to convert.
    ///  - emitDigest: Whether the conversion should create metadata files, such as linkable entities information.
    ///  - documentationCoverageOptions: What level of documentation coverage output should be emitted.
    ///  - currentPlatforms: The current version and beta information for platforms that may be encountered while processing symbol graph files.
    ///  - workspace: A provided documentation workspace. Creates a new empty workspace if value is `nil`.
    ///  - context: A provided documentation context.
    ///  - dataProvider: A data provider to use when registering bundles.
    ///  - externalIDsToConvert: The external IDs of the documentation nodes to convert.
    ///  - documentPathsToConvert: The paths of the documentation nodes to convert.
    ///  - bundleDiscoveryOptions: Options to configure how the converter discovers documentation bundles.
    ///  - emitSymbolSourceFileURIs: Whether the documentation converter should include
    ///    source file location metadata in any render nodes representing symbols it creates.
    /// 
    ///    Before passing `true` please confirm that your use case doesn't include public
    ///    distribution of any created render nodes as there are filesystem privacy and security
    ///    concerns with distributing this data.
    ///  - emitSymbolAccessLevels: Whether the documentation converter should include access level information for symbols.
    ///  - sourceRepository: The source repository where the documentation's sources are hosted.
    ///  - isCancelled: A wrapped boolean value used for the caller to cancel converting the documentation.
    ///   that have an expanded version of their documentation page if the access level requirement is met.
    ///  - diagnosticEngine: The diagnostic engine that collects any problems encountered from converting the documentation.
    ///  - symbolIdentifiersWithExpandedDocumentation: Identifiers and access level requirements for symbols
    ///  - experimentalModifyCatalogWithGeneratedCuration: Whether the documentation converter should write documentation extension files containing markdown representations of DocC's automatic curation into the source documentation catalog.
    public init(
        documentationBundleURL: URL?,
        emitDigest: Bool,
        documentationCoverageOptions: DocumentationCoverageOptions,
        currentPlatforms: [String : PlatformVersion]?,
        workspace: DocumentationWorkspace,
        context: DocumentationContext,
        dataProvider: DocumentationWorkspaceDataProvider,
        externalIDsToConvert: [String]? = nil,
        documentPathsToConvert: [String]? = nil,
        bundleDiscoveryOptions: BundleDiscoveryOptions,
        emitSymbolSourceFileURIs: Bool = false,
        emitSymbolAccessLevels: Bool = false,
        sourceRepository: SourceRepository? = nil,
        isCancelled: Synchronized<Bool>? = nil,
        diagnosticEngine: DiagnosticEngine = .init(),
        symbolIdentifiersWithExpandedDocumentation: [String: ConvertRequest.ExpandedDocumentationRequirements]? = nil,
        experimentalModifyCatalogWithGeneratedCuration: Bool = false
    ) {
        self.rootURL = documentationBundleURL
        self.emitDigest = emitDigest
        self.documentationCoverageOptions = documentationCoverageOptions
        self.workspace = workspace
        self.context = context
        self.dataProvider = dataProvider
        self.externalIDsToConvert = externalIDsToConvert
        self.documentPathsToConvert = documentPathsToConvert
        self.bundleDiscoveryOptions = bundleDiscoveryOptions
        self.shouldEmitSymbolSourceFileURIs = emitSymbolSourceFileURIs
        self.shouldEmitSymbolAccessLevels = emitSymbolAccessLevels
        self.sourceRepository = sourceRepository
        self.isCancelled = isCancelled
        self.diagnosticEngine = diagnosticEngine
        self.symbolIdentifiersWithExpandedDocumentation = symbolIdentifiersWithExpandedDocumentation
        self.experimentalModifyCatalogWithGeneratedCuration = experimentalModifyCatalogWithGeneratedCuration
        
        // Inject current platform versions if provided
        if var currentPlatforms {
            // Add missing platforms if their fallback platform is present.
            for (platform, fallbackPlatform) in DefaultAvailability.fallbackPlatforms where currentPlatforms[platform.displayName] == nil {
                currentPlatforms[platform.displayName] = currentPlatforms[fallbackPlatform.displayName]
            }
            self.context.externalMetadata.currentPlatforms = currentPlatforms
        }
    }
    
    /// Returns the first bundle in the source directory, if any.
    /// > Note: The result of this function is not cached, it reads the source directory and finds all bundles.
    public func firstAvailableBundle() -> DocumentationBundle? {
        return (try? dataProvider.bundles(options: bundleDiscoveryOptions)).map(sorted(bundles:))?.first
    }
    
    /// Sorts a list of bundles by the bundle identifier.
    private func sorted(bundles: [DocumentationBundle]) -> [DocumentationBundle] {
        return bundles.sorted(by: \.identifier)
    }
    
    mutating public func convert(
        outputConsumer: some ConvertOutputConsumer
    ) throws -> (analysisProblems: [Problem], conversionProblems: [Problem]) {
        defer {
            diagnosticEngine.flush()
        }
        
        // Unregister the current file data provider and all its bundles
        // when running repeated conversions.
        if let dataProvider = self.currentDataProvider {
            try workspace.unregisterProvider(dataProvider)
        }
        
        // Do additional context setup.
        setupContext?(&context)

        /*
           Asynchronously cancel registration if necessary.
           We spawn a timer that periodically checks `isCancelled` and if necessary
           disables registration in `DocumentationContext` as registration being
           the largest part of a documentation conversion.
        */
        let context = self.context
        let isCancelled = self.isCancelled
        
        // `true` if the `isCancelled` flag is set.
        func isConversionCancelled() -> Bool {
            return isCancelled?.sync({ $0 }) == true
        }

        // Run a timer that synchronizes the cancelled state between the converter and the context directly.
        // We need a timer on a separate dispatch queue because `workspace.registerProvider()` blocks
        // the current thread until it loads all symbol graphs, markdown files, and builds the topic graph
        // so in order to be able to update the context cancellation flag we need to run on a different thread.
        var cancelTimerQueue: DispatchQueue? = DispatchQueue(label: "org.swift.docc.ConvertActionCancelTimer", qos: .unspecified, attributes: .concurrent)
        let cancelTimer = DispatchSource.makeTimerSource(queue: cancelTimerQueue)
        cancelTimer.schedule(deadline: .now(), repeating: .milliseconds(500), leeway: .milliseconds(50))
        cancelTimer.setEventHandler {
            if isConversionCancelled() {
                cancelTimer.cancel()
                context.setRegistrationEnabled(false)
            }
        }
        cancelTimer.resume()
        
        // Start bundle registration
        try workspace.registerProvider(dataProvider, options: bundleDiscoveryOptions)
        self.currentDataProvider = dataProvider

        // Bundle registration is finished - stop the timer and reset the context cancellation state.
        cancelTimer.cancel()
        cancelTimerQueue = nil
        context.setRegistrationEnabled(true)
        
        // If cancelled, return early before we emit diagnostics.
        guard !isConversionCancelled() else { return ([], []) }
        
        processingDurationMetric = benchmark(begin: Benchmark.Duration(id: "documentation-processing"))
        
        let bundles = try sorted(bundles: dataProvider.bundles(options: bundleDiscoveryOptions))
        guard !bundles.isEmpty else {
            if let rootURL {
                throw Error.doesNotContainBundle(url: rootURL)
            } else {
                try outputConsumer.consume(problems: context.problems)
                throw GeneratedDataProvider.Error.notEnoughDataToGenerateBundle(options: bundleDiscoveryOptions, underlyingError: nil)
            }
        }
        
        // For now, we only support one bundle.
        let bundle = bundles.first!
        
        if experimentalModifyCatalogWithGeneratedCuration, let catalogURL = rootURL {
            let writer = GeneratedCurationWriter(context: context, catalogURL: catalogURL, outputURL: catalogURL)
            let curation = try writer.generateDefaultCurationContents()
            for (url, updatedContent) in curation {
                guard let data = updatedContent.data(using: .utf8) else { continue }
                try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true, attributes: nil)
                try? data.write(to: url, options: .atomic)
            }
        }
        
        guard !context.problems.containsErrors else {
            if emitDigest {
                try outputConsumer.consume(problems: context.problems)
            }
            return (analysisProblems: context.problems, conversionProblems: [])
        }
        
        // Precompute the render context
        let renderContext = RenderContext(documentationContext: context, bundle: bundle)
        
        try outputConsumer.consume(renderReferenceStore: renderContext.store)

        // Copy images, sample files, and other static assets.
        try outputConsumer.consume(assetsInBundle: bundle)
        
        let symbolIdentifiersMeetingRequirementsForExpandedDocumentation: [String]? = symbolIdentifiersWithExpandedDocumentation?.compactMap { (identifier, expandedDocsRequirement) -> String? in
            guard let documentationNode = context.documentationCache[identifier] else {
                return nil
            }
            
            return documentationNode.meetsExpandedDocumentationRequirements(expandedDocsRequirement) ? identifier : nil
        }
        
        let converter = DocumentationContextConverter(
            bundle: bundle,
            context: context,
            renderContext: renderContext,
            emitSymbolSourceFileURIs: shouldEmitSymbolSourceFileURIs,
            emitSymbolAccessLevels: shouldEmitSymbolAccessLevels,
            sourceRepository: sourceRepository,
            symbolIdentifiersWithExpandedDocumentation: symbolIdentifiersMeetingRequirementsForExpandedDocumentation
        )
        
        var indexingRecords = [IndexingRecord]()
        var linkSummaries = [LinkDestinationSummary]()
        var assets = [RenderReferenceType : [RenderReference]]()
        
        let references = context.knownPages
        let resultsSyncQueue = DispatchQueue(label: "Convert Serial Queue", qos: .unspecified, attributes: [])
        let resultsGroup = DispatchGroup()

        var coverageInfo = [CoverageDataEntry]()
        // No need to generate this closure more than once.
        let coverageFilterClosure = documentationCoverageOptions.generateFilterClosure()
        
        // Process render nodes in batches allowing us to release memory and sync after each batch
        // Keep track of any problems in case emitDigest == true
        var conversionProblems: [Problem] = references.concurrentPerform { identifier, results in
            // If cancelled skip all concurrent conversion work in this block.
            guard !isConversionCancelled() else { return }

            let source = context.documentURL(for: identifier)
            
            // Wrap JSON encoding in an autorelease pool to avoid retaining the autoreleased ObjC objects returned by `JSONSerialization`
            autoreleasepool {
                do {
                    let entity = try context.entity(with: identifier)

                    guard shouldConvertEntity(entity: entity, identifier: identifier) else {
                        return
                    }

                    guard let renderNode = try converter.renderNode(for: entity, at: source) else {
                        // No render node was produced for this entity, so just skip it.
                        return
                    }
                    
                    try outputConsumer.consume(renderNode: renderNode)

                    switch documentationCoverageOptions.level {
                    case .detailed, .brief:
                        let coverageEntry = try CoverageDataEntry(
                            documentationNode: entity,
                            renderNode: renderNode,
                            context: context
                        )
                        if coverageFilterClosure(coverageEntry) {
                            resultsGroup.async(queue: resultsSyncQueue) {
                                coverageInfo.append(coverageEntry)
                            }
                        }
                    case .none:
                        break
                    }
                    
                    if emitDigest {
                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: true)
                        let nodeIndexingRecords = try renderNode.indexingRecords(onPage: identifier)
                        
                        resultsGroup.async(queue: resultsSyncQueue) {
                            assets.merge(renderNode.assetReferences, uniquingKeysWith: +)
                            linkSummaries.append(contentsOf: nodeLinkSummaries)
                            indexingRecords.append(contentsOf: nodeIndexingRecords)
                        }
                    } else if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled {
                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: false)
                        
                        resultsGroup.async(queue: resultsSyncQueue) {
                            linkSummaries.append(contentsOf: nodeLinkSummaries)
                        }
                    }
                } catch {
                    recordProblem(from: error, in: &results, withIdentifier: "render-node")
                }
            }
        }
        
        // Wait for any concurrent updates to complete.
        resultsGroup.wait()
        
        // If cancelled, return before producing outputs.
        guard !isConversionCancelled() else { return ([], []) }
        
        // Write various metadata
        if emitDigest {
            do {
                try outputConsumer.consume(linkableElementSummaries: linkSummaries)
                try outputConsumer.consume(indexingRecords: indexingRecords)
                try outputConsumer.consume(assets: assets)
            } catch {
                recordProblem(from: error, in: &conversionProblems, withIdentifier: "metadata")
            }
        }
        
        if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled {
            do {
                let serializableLinkInformation = try context.linkResolver.localResolver.prepareForSerialization(bundleID: bundle.identifier)
                try outputConsumer.consume(linkResolutionInformation: serializableLinkInformation)
                
                if !emitDigest {
                    try outputConsumer.consume(linkableElementSummaries: linkSummaries)
                }
            } catch {
                recordProblem(from: error, in: &conversionProblems, withIdentifier: "link-resolver")
            }
        }
        
        if emitDigest {
            do {
                try outputConsumer.consume(problems: context.problems + conversionProblems)
            } catch {
                recordProblem(from: error, in: &conversionProblems, withIdentifier: "problems")
            }
        }

        switch documentationCoverageOptions.level {
        case .detailed, .brief:
            do {
                try outputConsumer.consume(documentationCoverageInfo: coverageInfo)
            } catch {
                recordProblem(from: error, in: &conversionProblems, withIdentifier: "coverage")
            }
        case .none:
            break
        }
        
        try outputConsumer.consume(
            buildMetadata: BuildMetadata(
                bundleDisplayName: bundle.displayName,
                bundleIdentifier: bundle.identifier
            )
        )
        
        // Log the duration of the processing (after the bundle content finished registering).
        benchmark(end: processingDurationMetric)
        // Log the finalized topic graph checksum.
        benchmark(add: Benchmark.TopicGraphHash(context: context))
        // Log the finalized list of topic anchor sections.
        benchmark(add: Benchmark.TopicAnchorHash(context: context))
        // Log the finalized external topics checksum.
        benchmark(add: Benchmark.ExternalTopicsHash(context: context))
        // Log the peak memory.
        benchmark(add: Benchmark.PeakMemory())
        
        return (analysisProblems: context.problems, conversionProblems: conversionProblems)
    }
    
    /// Whether the given entity should be converted to a render node.
    private func shouldConvertEntity(
        entity: DocumentationNode,
        identifier: ResolvedTopicReference
    ) -> Bool {
        let isDocumentPathToConvert: Bool
        if let documentPathsToConvert {
            isDocumentPathToConvert = documentPathsToConvert.contains(identifier.path)
        } else {
            isDocumentPathToConvert = true
        }
        
        let isExternalIDToConvert: Bool
        if let externalIDsToConvert {
            isExternalIDToConvert = entity.symbol.map {
                externalIDsToConvert.contains($0.identifier.precise)
            } == true
        } else {
            isExternalIDToConvert = true
        }

        // If the identifier of the entity is neither in `documentPathsToConvert`
        // nor `externalIDsToConvert`, we don't convert it to a render node.
        return isDocumentPathToConvert || isExternalIDToConvert
    }
    
    /// Record a problem from the given error in the given problem array.
    ///
    /// Creates a ``Problem`` from the given `Error` and identifier, emits it to the
    /// ``DocumentationConverter``'s ``DiagnosticEngine``, and appends it to the given
    /// problem array.
    ///
    /// - Parameters:
    ///   - error: The error that describes the problem.
    ///   - problems: The array that the created problem should be appended to.
    ///   - identifier: A unique identifier the problem.
    private func recordProblem(
        from error: Swift.Error,
        in problems: inout [Problem],
        withIdentifier identifier: String
    ) {
        let singleDiagnostic = Diagnostic(
            source: nil,
            severity: .error,
            range: nil,
            identifier: "org.swift.docc.documentation-converter.\(identifier)",
            summary: error.localizedDescription
        )
        let problem = Problem(diagnostic: singleDiagnostic, possibleSolutions: [])
        
        diagnosticEngine.emit(problem)
        problems.append(problem)
    }
    
    enum Error: DescribedError, Equatable {
        case doesNotContainBundle(url: URL)
        
        var errorDescription: String {
            switch self {
            case .doesNotContainBundle(let url):
                return """
                    The directory at '\(url)' and its subdirectories do not contain at least one \
                    valid documentation bundle. A documentation bundle is a directory ending in \
                    `.docc`.
                    Pass `--allow-arbitrary-catalog-directories` flag to convert a directory \
                    without a `.docc` extension.
                    """
            }
        }
    }
}

extension DocumentationNode {
    func meetsExpandedDocumentationRequirements(_ requirements: ConvertRequest.ExpandedDocumentationRequirements) -> Bool {
        guard let symbol else { return false }
        
        return requirements.accessControlLevels.contains(symbol.accessLevel.rawValue) && (!symbol.names.title.starts(with: "_") || requirements.canBeUnderscored)
    }
}