#-- # Copyright (c) 2005 Robert Aman # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #++ require 'feed_tools' require 'feed_tools/helpers/xml_helper' require 'rexml/document' module FeedTools # Methods for pulling remote data module HtmlHelper TIDY_OPTIONS = [ :add_xml_decl, :add_xml_space, :alt_text, :assume_xml_procins, :bare, :clean, :css_prefix, :decorate_inferred_ul, :doctype, :drop_empty_paras, :drop_font_tags, :drop_proprietary_attributes, :enclose_block_text, :enclose_text, :escape_cdata, :fix_backslash, :fix_bad_comments, :fix_uri, :hide_comments, :hide_endtags, :indent_cdata, :input_xml, :join_classes, :join_styles, :literal_attributes, :logical_emphasis, :lower_literals, :merge_divs, :ncr, :new_blocklevel_tags, :new_empty_tags, :new_inline_tags, :new_pre_tags, :numeric_entities, :output_html, :output_xhtml, :output_xml, :preserve_entities, :quote_ampersand, :quote_marks, :quote_nbsp, :repeated_attributes, :replace_color, :show_body_only, :uppercase_attributes, :uppercase_tags, :word_2000, :accessibility_check, :show_errors, :show_warnings, :break_before_br, :indent, :indent_attributes, :indent_spaces, :markup, :punctuation_wrap, :split, :tab_size, :vertical_space, :wrap, :wrap_asp, :wrap_attributes, :wrap_jste, :wrap_php, :wrap_script_literals, :wrap_sections, :ascii_chars, :char_encoding, :input_encoding, :language, :newline, :output_bom, :output_encoding, :error_file, :force_output, :gnu_emacs, :gnu_emacs_file, :keep_time, :output_file, :quiet, :slide_style, :tidy_mark, :write_back ] # Escapes all html entities def self.escape_entities(html) return nil if html.nil? escaped_html = CGI.escapeHTML(html) escaped_html.gsub!(/'/, "'") escaped_html.gsub!(/"/, """) return escaped_html end # Unescapes all html entities def self.unescape_entities(html) return nil if html.nil? unescaped_html = html unescaped_html.gsub!(/&/, "&") unescaped_html.gsub!(/&/, "&") substitute_numerical_entities = Proc.new do |s| m = $1 m = "0#{m}" if m[0] == ?x [Integer(m)].pack('U*') end unescaped_html.gsub!(/�*((?:\d+)|(?:x[a-f0-9]+));/, &substitute_numerical_entities) unescaped_html = CGI.unescapeHTML(unescaped_html) unescaped_html.gsub!(/'/, "'") unescaped_html.gsub!(/"/, "\"") return unescaped_html end # Removes all html tags from the html formatted text, but leaves # escaped entities alone. def self.strip_html_tags(html) return nil if html.nil? stripped_html = html stripped_html.gsub!(/<\/?[^>]+>/, "") return stripped_html end # Removes all html tags from the html formatted text and removes # escaped entities. def self.convert_html_to_plain_text(html) return nil if html.nil? stripped_html = html stripped_html = FeedTools::HtmlHelper.strip_html_tags(stripped_html) stripped_html = FeedTools::HtmlHelper.unescape_entities(stripped_html) stripped_html.gsub!(/‘/, "'") stripped_html.gsub!(/’/, "'") stripped_html.gsub!(/“/, "\"") stripped_html.gsub!(/”/, "\"") return stripped_html end # Returns true if the html tidy module can be used. # # Obviously, you need the tidy gem installed in order to run with html # tidy features turned on. # # This method does a fairly complicated, and probably unnecessarily # desperate search for the libtidy library. If you want this thing to # execute fast, the best thing to do is to set Tidy.path ahead of time. # If Tidy.path is set, this method doesn't do much. If it's not set, # it will do it's darnedest to find the libtidy library. If you set # the LIBTIDYPATH environment variable to the libtidy library, it should # be able to find it. # # Once the library is located, this method will run much faster. def self.tidy_enabled? # This is an override variable to keep tidy from being used even if it # is available. if FeedTools.configurations[:tidy_enabled] == false return false end if @tidy_enabled.nil? || @tidy_enabled == false @tidy_enabled = false begin require 'tidy' if Tidy.path.nil? # *Shrug*, just brute force it, I guess. There's a lot of places # this thing might be hiding in, depending on platform and general # sanity of the person who installed the thing. Most of these are # probably unlikely, but it's not like checking unlikely locations # hurts. Much. Especially if you actually find it. libtidy_locations = [ '/usr/local/lib/libtidy.dylib', '/opt/local/lib/libtidy.dylib', '/usr/lib/libtidy.dylib', '/usr/local/lib/tidylib.dylib', '/opt/local/lib/tidylib.dylib', '/usr/lib/tidylib.dylib', '/usr/local/lib/tidy.dylib', '/opt/local/lib/tidy.dylib', '/usr/lib/tidy.dylib', '/usr/local/lib/libtidy.so', '/opt/local/lib/libtidy.so', '/usr/lib/libtidy.so', '/usr/local/lib/tidylib.so', '/opt/local/lib/tidylib.so', '/usr/lib/tidylib.so', '/usr/local/lib/tidy.so', '/opt/local/lib/tidy.so', '/usr/lib/tidy.so', 'C:\Program Files\Tidy\tidy.dll', 'C:\Tidy\tidy.dll', 'C:\Ruby\bin\tidy.dll', 'C:\Ruby\tidy.dll', '/usr/local/lib', '/opt/local/lib', '/usr/lib' ] # We just made this thing up, but if someone sets it, we'll # go ahead and check it unless ENV['LIBTIDYPATH'].nil? libtidy_locations = libtidy_locations.reverse.push(ENV['LIBTIDYPATH']) end for path in libtidy_locations if File.exists? path if File.ftype(path) == "file" || File.ftype(path) == "link" Tidy.path = path @tidy_enabled = true break elsif File.ftype(path) == "directory" # Ok, now perhaps we're getting a bit more desperate lib_paths = `find #{path} -name '*tidy*' | grep '\\.\\(so\\|dylib\\)$'` # If there's more than one, grab the first one and # hope for the best, and if it doesn't work, then blame the # user for not specifying more accurately. tidy_path = lib_paths.split("\n").first unless tidy_path.nil? Tidy.path = tidy_path @tidy_enabled = true break end end end end # Still couldn't find it. unless @tidy_enabled @tidy_enabled = false end else @tidy_enabled = true end rescue LoadError # Tidy not installed, disable features that rely on tidy. @tidy_enabled = false end end return @tidy_enabled end # Tidys up the html def self.tidy_html(html, options = {}) return nil if html.nil? FeedTools::GenericHelper.validate_options(TIDY_OPTIONS, options.keys) options = { :add_xml_decl => false, :char_encoding => "utf8", :doctype => "omit", :indent => false, :logical_emphasis => true, :markup => true, :show_warnings => false, :wrap => 0 }.merge(options) if FeedTools::HtmlHelper.tidy_enabled? is_fragment = true html.gsub!(/<!'/, "&lt;!'") if (html.strip =~ /(.|\n)*/) != nil || (html.strip =~ /<\/body>(.|\n)*<\/html>$/) != nil is_fragment = false end if (html.strip =~ /<\?xml(.|\n)*\?>/) != nil is_fragment = false end options[:show_body_only] = true if is_fragment # Tidy sucks? # TODO: find the correct set of tidy options to set so # that *ugly* hacks like this aren't necessary. html = html.gsub(/\302\240/, "\240") tidy_html = Tidy.open(options) do |tidy| xml = tidy.clean(html) xml end tidy_html.strip! else tidy_html = html end if tidy_html.blank? && !html.blank? tidy_html = html.strip end return tidy_html end # Indents a text selection by a specified number of spaces. def self.indent(text, spaces) lines = text.split("\n") buffer = "" for line in lines line = " " * spaces + line buffer << line << "\n" end return buffer end # Unindents a text selection by a specified number of spaces. def self.unindent(text, spaces) lines = text.split("\n") buffer = "" for line in lines for index in 0...spaces if line[0...1] == " " line = line[1..-1] else break end end buffer << line << "\n" end return buffer end # Returns true if the type string provided indicates that something is # xml or xhtml content. def self.xml_type?(type) if [ "xml", "xhtml", "application/xhtml+xml" ].include?(type) return true elsif type != nil && type[-3..-1] == "xml" return true else return false end end # Returns true if the type string provided indicates that something is # html or xhtml content. def self.text_type?(type) return [ "text", "text/plain" ].include?(type) end # Returns true if the type string provided indicates that something is # html or xhtml content. def self.html_type?(type) return [ "html", "xhtml", "text/html", "application/xhtml+xml" ].include?(type) end # Returns true if the type string provided indicates that something is # only html (not xhtml) content. def self.only_html_type?(type) return [ "html", "text/html" ].include?(type) end # Resolves all relative uris in a block of html. def self.resolve_relative_uris(html, base_uri_sources=[]) relative_uri_attributes = [ ["a", "href"], ["applet", "codebase"], ["area", "href"], ["blockquote", "cite"], ["body", "background"], ["del", "cite"], ["form", "action"], ["frame", "longdesc"], ["frame", "src"], ["iframe", "longdesc"], ["iframe", "src"], ["head", "profile"], ["img", "longdesc"], ["img", "src"], ["img", "usemap"], ["input", "src"], ["input", "usemap"], ["ins", "cite"], ["link", "href"], ["object", "classid"], ["object", "codebase"], ["object", "data"], ["object", "usemap"], ["q", "cite"], ["script", "src"] ] # HACK: Prevent the parser from freaking out if it sees this: html.gsub!(/ HTML5::HTMLSanitizer, :encoding => 'UTF-8') else fragments = HTML5::HTMLParser.parse_fragment(html) end resolve_node = lambda do |html_node| if html_node.kind_of? REXML::Element for element_name, attribute_name in relative_uri_attributes if html_node.name.downcase == element_name attribute = html_node.attribute(attribute_name) if attribute != nil href = attribute.value href = FeedTools::UriHelper.resolve_relative_uri( href, [html_node.base_uri] | base_uri_sources) href = FeedTools::UriHelper.normalize_url(href) html_node.attribute(attribute_name).instance_variable_set( "@value", href) html_node.attribute(attribute_name).instance_variable_set( "@unnormalized", href) html_node.attribute(attribute_name).instance_variable_set( "@normalized", href) if html_node.attribute(attribute_name).value != href warn("Failed to update href to resolved value.") end end end end end if html_node.respond_to? :children for child in html_node.children resolve_node.call(child) end end html_node end fragments.each do |fragment| resolve_node.call(fragment) end html = (fragments.map do |stuff| stuff.to_s end).join("") return html end # Returns a string containing normalized xhtml from within a REXML node. def self.extract_xhtml(rexml_node) rexml_node_dup = rexml_node.deep_clone namespace_hash = FEED_TOOLS_NAMESPACES.dup normalize_namespaced_xhtml = lambda do |node, node_dup| if node.kind_of? REXML::Element node_namespace = node.namespace if node_namespace != namespace_hash['atom10'] && node_namespace != namespace_hash['atom03'] # Massive hack, relies on REXML not changing for index in 0...node.attributes.values.size attribute = node.attributes.values[index] attribute_dup = node_dup.attributes.values[index] if attribute.namespace == namespace_hash['xhtml'] attribute_dup.instance_variable_set( "@expanded_name", attribute.name) end if node_namespace == namespace_hash['xhtml'] if attribute.name == 'xmlns' node_dup.attributes.delete('xmlns') end end end if node_namespace == namespace_hash['xhtml'] node_dup.instance_variable_set("@expanded_name", node.name) end if !node_namespace.blank? && node.prefix.blank? if node_namespace != namespace_hash['xhtml'] prefix = nil for known_prefix in namespace_hash.keys if namespace_hash[known_prefix] == node_namespace prefix = known_prefix end end if prefix.nil? prefix = "unknown" + Digest::SHA1.new(node_namespace).to_s[0..4] namespace_hash[prefix] = node_namespace end node_dup.instance_variable_set("@expanded_name", "#{prefix}:#{node.name}") node_dup.instance_variable_set("@prefix", prefix) node_dup.add_namespace(prefix, node_namespace) end end end end for index in 0...node.children.size child = node.children[index] if child.kind_of? REXML::Element child_dup = node_dup.children[index] normalize_namespaced_xhtml.call(child, child_dup) end end end normalize_namespaced_xhtml.call(rexml_node, rexml_node_dup) buffer = "" rexml_node_dup.each_child do |child| if child.kind_of? REXML::Comment buffer << "" else buffer << child.to_s end end return buffer.strip end # Given a REXML node, returns its content, normalized as HTML. def self.process_text_construct(content_node, feed_type, feed_version, base_uri_sources=[]) if content_node.nil? return nil end content = nil root_node_name = nil type = FeedTools::XmlHelper.try_xpaths(content_node, "@type", :select_result_value => true) mode = FeedTools::XmlHelper.try_xpaths(content_node, "@mode", :select_result_value => true) encoding = FeedTools::XmlHelper.try_xpaths(content_node, "@encoding", :select_result_value => true) if type.nil? atom_namespaces = [ FEED_TOOLS_NAMESPACES['atom10'], FEED_TOOLS_NAMESPACES['atom03'] ] if ((atom_namespaces.include?(content_node.namespace) || atom_namespaces.include?(content_node.root.namespace)) || feed_type == "atom") type = "text" end end # Note that we're checking for misuse of type, mode and encoding here if content_node.cdatas.size > 0 content = content_node.cdatas.first.to_s.strip elsif type == "base64" || mode == "base64" || encoding == "base64" content = Base64.decode64(content_node.inner_xml.strip) elsif type == "xhtml" || mode == "xhtml" || type == "xml" || mode == "xml" || type == "application/xhtml+xml" || content_node.namespace == FEED_TOOLS_NAMESPACES['xhtml'] content = FeedTools::HtmlHelper.extract_xhtml(content_node) elsif type == "escaped" || mode == "escaped" || type == "html" || mode == "html" || type == "text/html" || mode == "text/html" content = FeedTools::HtmlHelper.unescape_entities( content_node.inner_xml.strip) elsif type == "text" || mode == "text" || type == "text/plain" || mode == "text/plain" content = FeedTools::HtmlHelper.unescape_entities( content_node.inner_xml.strip) else content = FeedTools::HtmlHelper.unescape_entities( content_node.inner_xml.strip) end if type == "text" || mode == "text" || type == "text/plain" || mode == "text/plain" content = FeedTools::HtmlHelper.escape_entities(content) end unless content.nil? content = FeedTools::HtmlHelper.resolve_relative_uris(content, [content_node.base_uri] | base_uri_sources) content = FeedTools::HtmlHelper.tidy_html(content) end if FeedTools.configurations[:tab_spaces] != nil spaces = FeedTools.configurations[:tab_spaces].to_i content.gsub!("\t", " " * spaces) unless content.blank? end content.strip unless content.blank? content = nil if content.blank? return content end # Strips semantically empty div wrapper elements def self.strip_wrapper_element(xhtml) return nil if xhtml.nil? return xhtml if xhtml.blank? begin doc = REXML::Document.new(xhtml.to_s.strip) if doc.children.size == 1 child = doc.children[0] if child.kind_of?(REXML::Element) && child.name.downcase == "div" return child.inner_xml.strip end end return xhtml.to_s.strip rescue Exception return xhtml.to_s.strip end end # Given a block of html, locates feed links with a given mime type. def self.extract_link_by_mime_type(html, mime_type) require 'feed_tools/helpers/xml_helper' # HACK: Prevent the parser from freaking out if it sees this: html = html.gsub(/(.|\n)*?<\/body>/, "") html.gsub!(/(.|\n)*?<\/script>/, "") html.gsub!(/(.|\n)*?<\/noscript>/, "") html.gsub!(//, "") html = FeedTools::HtmlHelper.tidy_html(html) document = HTML5::HTMLParser.parse(html) link_nodes = [] get_link_nodes = lambda do |root_node| html_node = nil head_node = nil return nil if !root_node.respond_to?(:children) if root_node.name.downcase == "html" && root_node.children.size > 0 html_node = root_node else for node in fragment_node.children next unless node.kind_of?(REXML::Element) if node.name.downcase == "html" && node.children.size > 0 html_node = node break end end end if html_node != nil for node in html_node.children next unless node.kind_of?(REXML::Element) if node.name.downcase == "head" head_node = node break end if node.name.downcase == "link" link_nodes << node end end if html_node != nil || !link_nodes.empty? if head_node != nil link_nodes = [] for node in head_node.children next unless node.kind_of?(REXML::Element) if node.name.downcase == "link" link_nodes << node end end end end end end get_link_nodes.call(document.root) process_link_nodes = lambda do |links| for link in links next unless link.kind_of?(REXML::Element) if link.attributes['type'].to_s.strip.downcase == mime_type.downcase && link.attributes['rel'].to_s.strip.downcase == "alternate" href = link.attributes['href'] return href unless href.blank? end end for link in links next unless link.kind_of?(REXML::Element) process_link_nodes.call(link.children) end end process_link_nodes.call(link_nodes) return nil end end end