#===========================================================================

package Sitescooper::LinksURLProcessor;

use Sitescooper::URLProcessor;
use Sitescooper::Util;
use Carp;

@ISA = qw(Sitescooper::URLProcessor);

use strict;

sub new {
  my $class = shift; $class = ref($class) || $class;
  my ($scoop, $robot, $scf, $ref, $url, $level, $upindex) = @_;
  my $self = $class->SUPER::new($scoop, $robot, $scf, $ref, $url);

  $self->{level} = $level;
  $self->{upindex} = $upindex;
  $self->{page} = undef;

  # bless ($self, $class);
  $self;
}

# ---------------------------------------------------------------------------
# Note on levels: a 2-level site has a contents page and stories off that;
# 3-level has issue links page, per-issue contents page and stories.
# 1-level has only the story page, no links.

sub start_get {
  my $self = shift;
  my $url = $self->{url};
  my $level = $self->{level};
  my ($cachefile, $page);

  my $human_level = $level + 2;

  if ($self->get_state() != $Sitescooper::URLProcessor::STATE_PRE_GET) {
    croak ("state != Sitescooper::URLProcessor::STATE_PRE_GET");
  }

  if ($Sitescooper::Main::got_intr_flag) { return; }
  if ($self->{robot}->{hit_file_size_limit}) { return; }

  my $pat = $self->{scf}->get_links_param ('links_limit_to', $url, $level);
  if (defined $pat) {
    if (!$self->match_url ($url, $pat)) {
      $self->{scoop}->dbg ("front page URL $url does not match $pat, ignoring.");
      return;
    }
  }

  $pat = $self->{scf}->get_links_param ('links_skip', $url, $level);
  if (defined $pat) {
    if ($url =~ m#^${pat}$#) {
      $self->{scoop}->verbose ("Skipping: $url"); return;
    }
  }

  my $origurl = $url;
  $url = $self->apply_url_preproc ($url);
  if (!defined $url) {
    $self->{scoop}->dbg ("URLProcess says URL should be ignored: $origurl"); return;
  }

  my $fullurl = $url; $url = Sitescooper::Util::URLWithoutAnchor ($url);
  return if (defined $self->{robot}->{already_tried_download}{$url});
  $self->{robot}->{already_tried_download}{$url} = 1;
  study $url;

  $self->{scoop}->verbose ("Reading level-".($human_level)." front page: $fullurl");
  Sitescooper::Main::set_got_intr_behaviour ('setflag');

  my $is_dynamic_html;
  if (defined $self->{scf}->{cacheable}->{$level}) {
    $is_dynamic_html = ($self->{scf}->{cacheable}->{$level} == 0);
  } elsif (defined $self->{scf}->{links_diff}->{$level}
	      && $self->{scf}->{links_diff}->{$level} != 0)
  {
    $is_dynamic_html = 1;	# pages that need diff'ing are dynamic
  } else {
    $is_dynamic_html = 1;	# index pages are usually dynamic
  }

  return unless ($self->get_page ($url, $is_dynamic_html));

  $self->{fullurl} = $fullurl;
  $self->{url} = $url;
  $self->{is_dynamic_html} = $is_dynamic_html;
  1;
}

# ---------------------------------------------------------------------------

sub finish_get {
  my ($self) = @_;

  $self->set_state ($Sitescooper::URLProcessor::STATE_POST_GET);
  if ($Sitescooper::Main::got_intr_flag) { goto interrupted; }

  my $is_dynamic_html = $self->{is_dynamic_html};
  my $fullurl = $self->{fullurl};
  my $url = $self->{url};
  my $level = $self->{level};
  my $upindex = $self->{upindex};
  my $human_level = $level + 2;

  $self->{scoop}->dbg ("finishing GET: $url");
  my $page = $self->get_url_reply();

  if ($self->does_need_http_retry()) {
    my $handler = new Sitescooper::LinksURLProcessor ($self->{scoop},
    	$self->{robot}, $self->{scf}, $self->{url},
	$self->get_http_retry_url(), $level, $upindex);
    $self->{robot}->re_request_page ($handler);
    return;
  }

  if ($Sitescooper::Main::got_intr_flag) { goto interrupted; }
  if (!defined $page) {
    $self->{scoop}->verbose ("Skipping (get_page returned nothing): $fullurl");
    return;
  }

  if ($self->{cf}->{linkslimit} > 0 && $self->{robot}->{stories_found} >= $self->{cf}->{linkslimit}) {
    $self->{scoop}->verbose ("over links limit, stopping this site.");
    $self->{robot}->{hit_file_size_limit} = 1;
    return;
  }

  my ($redir_from, $newbase);
  ($redir_from, $url, $newbase) =
	 $self->handle_redirects_and_base_url ($url, $page);

  my $life = $self->{scf}->get_story_param ('story_lifetime', $url);
  my $mod = $self->{cache}->get_last_modtime ($url);
  if (defined $mod && $mod < $life * 24 * 60 * 60) {
    $self->{scoop}->verbose ("Skipping (contents are older than ".$life." days): $url");
    return;
  }

  my $origpage = $page;
  $self->{scoop}->journal ("url", $url);
  $self->{scoop}->journal ("base_url", $newbase);
  $self->{scoop}->journal ("pre_strip_level".($human_level), $page);
  $page = $self->strip_front_page ($url, $level, $page);
  $self->{scoop}->journal ("post_strip_level".($human_level), $page);

  if ($self->{robot}->{need_title} && $origpage =~ /<title>\s*(\S.*?)\s*<\/title>/is) {
    $self->{robot}->change_title ($1);
  }

  my $cachedpage;
  if ((defined $self->{scf}->{links_diff}->{$level} &&
    		$self->{scf}->{links_diff}->{$level} != 0) ||
     $self->writing_html)
  {
    my $cobj =
    	$self->{cache}->get_cached_page_for_diff ($url);
    if (defined ($cobj)) {
      $cachedpage = $self->strip_front_page ($url, $level, $cobj->get_page());
    }
  }

  if (defined $self->{scf}->{links_diff}->{$level}
  	&& $self->{scf}->{links_diff}->{$level} != 0)
  {
    $page = $self->get_new_bits ($cachedpage, $page);
  }

  $self->{cache}->cache_page_at_commit ($url, $redir_from, $origpage);
  $self->{robot}->add_snarfed_link ($url);

  my $proc = $self->{scf}->get_links_param ('links_preproc', $url, $level);
  if (defined $proc) {
    $_ = $page;
    my $site_level = $human_level;
    if (!eval $proc."; 1;") {
      $self->sitewarn ("level-".($human_level)." HTMLPreProc failed: $@");
      # and keep the original $page
    } else {
      $page = $_;
    }
  }

  if (defined fileno Sitescooper::Main::JOURNAL) {
    # always write a text-mode version for the journal
    $self->{scoop}->journal ("to_text_level".($human_level),
    	$self->html_to_text ($newbase, $page, $Sitescooper::Main::OUT_TEXT));
  }

  my $lprint = $self->{scf}->get_links_param ('links_print', $url, $level);
  my $printed_this_front_page = 0;
  my $has_not_changed = 0;

  if ((defined $lprint && $lprint != 0) || $self->writing_html) {
    my $txtpage = $self->html_to_text_warn_about_ext_links
		      ($newbase, $page, $self->{cf}->{output_style});

    if (defined $cachedpage && !$self->{cf}->{refresh}) {
      # ensure that the cleaned-up HTML doesn't match the cleaned-up cached
      # HTML. Sometimes the ad banners will be the only things that have
      # changed between retrieves, and html_to_text will have stripped those
      # out.
      my $cachedtxt = $self->html_to_text
      		($newbase, $cachedpage, $self->{cf}->{output_style});
      if ($self->text_equals ($txtpage, $cachedtxt)) {
	$has_not_changed = 1;
      }
    }

    # avoid writing empty pages, or empty results of diffed pages
    if (!$self->text_equals ($txtpage, '')) {
      $self->{scoop}->verbose ("Printing: $fullurl");
      $self->{robot}->write_as_story (1, $url, $txtpage, undef, $upindex);

      if ($has_not_changed && !$self->{cf}->{refresh}) {
	# don't count a front page as a story if:
	#
	# 1. we're just outputting it because we're writing HTML
	# 3. and the page had not changed since previous GET
	# 4. and we're not refreshing
	#
	# What does this mean?
	# This means the text *will* be present in the output file, but not as
	# a "story", instead just as a means of following links to a "story".
	# There needs to be at least one "story" for the scoop to be valid.

	$self->{scoop}->dbg ("text has not changed, not counting this page as a story");
	$self->{robot}->{stories_found}--;
      }
    }
  }

  # this is a front page. Pages followed from this page should use this as
  # the "up a level" link.
  $upindex = $self->{robot}->{current_story_index} - 1;

  # see if there's any links to extra contents pages
  my @turnoverlinks = $self->get_contents_turnover_links ($newbase, $level, $page);

  my @links = ();
  my $wrote_sep = 0;

  # This was all getting a bit tricky, so I've redone it a bit.
  # It now does not try to strip closing tags, as it doesn't have to.
  while (1) {
    if ($Sitescooper::Main::got_intr_flag) { goto interrupted; }
    if ($self->{robot}->{hit_file_size_limit}) { last; }

    if (
      $page =~ s/<a\s+[^>]*href\s*=\s*\"([^\">]+)\"//is
      ||
      $page =~ s/<a\s+[^>]*href\s*=\s*\'([^\'>]+)\'//is
      ||
      $page =~ s/<a\s+[^>]*href\s*=\s*([^\s>]+)//is
      )
    {
      my $link = $1;
      push (@links, $link);
      next;
    }

    # support for frames
    if (
      $page =~ s/<frame\s+[^>]*src=\"([^\">]+)\"//is
       ||
      $page =~ s/<frame\s+[^>]*src=\'([^\'>]+)\'//is
       ||
      $page =~ s/<frame\s+[^>]*src=([^\s+>]+)//is
      )
    {
      my $link = $1;
      if ($self->writing_html) {
	if ($wrote_sep == 0) {
	  ${$self->{robot}->{output_file}}{'MAIN'} .= "<p><hr>\n"; $wrote_sep = 1;
	}
	${$self->{robot}->{output_file}}{'MAIN'} .=
		$self->translate_link ($fullurl, $link, $link). "<br>\n";
      }
      push (@links, $link);
      next;
    }

    # rudimentary support for My-Netscape-style RDF files
    if ($page =~ s/<item>(.*?)<link\s*[^>]*>(.+?)<\/link>(.*?)<\/item>//is)
    {
      my ($title, $link, $title2) = ($1, $2, $3);

      # <link> tags in RSS can contain other crap. Ditch it; we want the link!
      $link =~ s/^.*<url>(.*?)<\/url>.*$/$1/gis;

      $link = Sitescooper::Util::AbsoluteURL ($newbase, $link);
      if ($title =~ /<title>(.*?)<\/title>/is
	   || $title =~ /<text>(.*?)<\/text>/is
	   || $title2 =~ /<title>(.*?)<\/title>/is
	   || $title2 =~ /<text>(.*?)<\/text>/is)
      {
	$self->{robot}->set_url_title ($link, $1);
      }

      push (@links, $link);
      next;
    }

    last;		# no more links available
  }

  if ($#links >= 0) {
    $self->{scoop}->verbose ("Found ".($#links+1)." links, examining them.");
  }

  # now traverse the links and get the stories
  $self->{scoop}->journal ("links_level".($human_level), join ("\n", @links));
  my $followed_a_link = 0;

  @links = $self->{robot}->absolutify_and_grep_unseen_urls ($newbase, @links);
  foreach $_ (@links) {
    if ($self->{robot}->{hit_file_size_limit}) {
      my $msg =
      	"File size limit exceeded, skipped some stories from this site.";
      $self->sitewarn ($msg);
      if ($self->writing_html) {
	${$self->{robot}->{output_file}}{'MAIN'} .= "<hr><i>$msg</i><br>\n";
      } else {
	${$self->{robot}->{output_file}}{'MAIN'} .= "\n($msg)\n";
      }
      last;
    }

    $self->follow_front_link ($newbase, $level, $_, $upindex)
				and ($followed_a_link = 1);

    if ($Sitescooper::Main::got_intr_flag) { goto interrupted; }
  }

  # if there's more contents pages, process them as well.
  $self->{scoop}->journal ("turnover_links_level".($human_level),
						 join ("\n", @turnoverlinks));
  @turnoverlinks = $self->{robot}->absolutify_and_grep_unseen_urls ($newbase, @turnoverlinks);
  if ($#turnoverlinks >= 0) {
    my $link;
    for $link (@turnoverlinks) {
      if ($Sitescooper::Main::got_intr_flag) { goto interrupted; }
      $link = Sitescooper::Util::AbsoluteURL ($newbase, $link);
      $self->{robot}->download_front_page ($self->{url}, $link, $level)
				and ($followed_a_link = 1);
    }
  }

  $self->{scoop}->dbg ("stories found so far: ".$self->{robot}->{stories_found});

interrupted:
  Sitescooper::Main::set_got_intr_behaviour ('exit');

  ($printed_this_front_page || $followed_a_link || !$has_not_changed);
}

# ---------------------------------------------------------------------------

sub follow_front_link {
  my $self = shift;
  my ($url, $level, $nextpage, $upindex) = @_;

  $nextpage = Sitescooper::Util::AbsoluteURL ($url, $nextpage);
  return if ($nextpage !~ /^(http|file):/i);	# only supported links

  $self->{scoop}->dbg ("Link found on $url: $nextpage");

  # should we download the next front page?
  if ($level > 0) {
    return $self->{robot}->download_front_page ($url, $nextpage, $level-1, $upindex);
  }
  if ($Sitescooper::Main::got_intr_flag) { return; }

  # nope, we're onto the stories already
  $nextpage = $self->make_printable ($nextpage, 1);

  $self->{robot}->download_story_page ($url, $nextpage, 0, $upindex);
}

# ---------------------------------------------------------------------------

sub strip_front_page {
  my ($self, $url, $level, $page) = @_;
  $self->strip_html ($url, "Links", $level, '', $page);
}

# ---------------------------------------------------------------------------

sub get_contents_turnover_links {
  my $self = shift;
  my $url = shift;
  my $level = shift;
  my $page = shift;

  my $followlinks = $self->{scf}->get_links_param
  				('links_follow_links', $url, $level);
  if (!$followlinks) {
    return ();
  }

  my @turnoverlinks = ();

  while ($page =~ s,<a\s+[^>]*href\s*=\s*(?:\"|\'|%22)?([^>]+)(?:\"|\'|%22)?>(.+?)</a>,,is)
  {
    my $link = $1;
    my $txt = $2;

    push (@turnoverlinks, $link);
    # we don't do the automatic "more/next/page x of y" stuff
    # that we do with the story pages
  }

  @turnoverlinks;
}

1;