package Catmandu::Importer::MediaWiki;
use Catmandu::Sane;
use MediaWiki::API;
use Catmandu::Util qw(:is :check array_includes);
use URI::Escape;
use Moo;

#only generators that generate pages, and that have generators
my $generators = [qw(
    alllinks
    allpages
    allredirects
    alltransclusions
    backlinks
    categorymembers
    embeddedin
    exturlusage
    imageusage
    iwbacklinks
    langbacklinks
    pageswithprop
    prefixsearch
    random
    recentchanges
    watchlist
    watchlistraw
)];
my $default_args = {
    prop => "revisions",
    rvprop => "ids|flags|timestamp|user|comment|size|content|sha1|tags|userid|parsedcomment",
    rvlimit => 'max',
    gaplimit => 100,
    gapfilterredir => "nonredirects"
};
my $ns_map = {
    allpages => "gapnamespace",
    alllinks => "galnamespace",
    allredirects => "garnamespace",
    alltransclusions => "gatnamespace",
    backlinks => "gblnamespace",
    categorymembers => "gcmnamespace",
    embeddedin => "geinamespace",
    exturlusage => "geunamespace",
    imageusage => "giunamespace",
    iwbacklinks => undef,
    langbacklinks => undef,
    pageswithprop => undef,
    prefixsearch => "gpsnamespace",
    random => "grnnamespace",
    recentchanges => "grcnamespace",
    watchlist => "gwlnamespace",
    watchlistraw => "gwrnamespace"
};

has url => (
    is => 'ro',
    isa => sub { check_string($_[0]); }
);
#cf. https://www.mediawiki.org/wiki/API:Login
has lgname => ( is => 'ro' );
has lgpassword => ( is => 'ro' );
#cf. http://www.mediawiki.org/wiki/API:Lists
#cf. http://www.mediawiki.org/wiki/API:Query#Generators
has generate => (
    is => 'ro',
    isa => sub {
        array_includes($generators,$_[0]) or die("invalid generator");
    },
    lazy => 1,
    default => sub { "allpages"; }
);
has args => (
    is => 'ro',
    isa => sub { check_hash_ref($_[0]); },
    lazy => 1,
    default => sub { $default_args },
    coerce => sub {
        my $l = $_[0];
        my $h = is_hash_ref($l) ? +{ %$default_args,%$l } : $default_args;
        for(keys %$h){
            delete $h->{$_} unless defined $h->{$_};
        }
        $h;
    }
);
has mediawiki => (
    is => 'ro',
    lazy => 1,
    builder => '_build_mw'
);

with 'Catmandu::Importer';

sub _build_mw {
    my $self = $_[0];
    my $mw = MediaWiki::API->new( { api_url => $self->url() }  );

    if(is_string($ENV{LWP_TRACE})){
        $mw->{ua}->add_handler("request_send",  sub { shift->dump; return });
        $mw->{ua}->add_handler("response_done", sub { shift->dump; return });
    }

    $mw;
}
sub _fail {
    my $err = $_[0];
    die( $err->{code}.': '.$err->{details} );
}

sub generator {
    my $self = $_[0];

    my $generator = $self->generate();
    my $args = $self->args();

    sub {
        state $mw = $self->mediawiki();
        state $pages = [];
        state $cont_args = {};
        state $logged_in = 0;
        state $namespaces = [];
        state $namespace_key = $ns_map->{ $generator };
        state $siteinfo = {};

        unless($logged_in){
            #only try to login when both arguments are set
            if(is_string($self->lgname) && is_string($self->lgpassword)){
                $mw->login({ lgname => $self->lgname, lgpassword => $self->lgpassword }) or _fail($mw->{error});
            }
            $logged_in = 1;
        }

        #store continue arguments per namespace
        if(scalar(@$namespaces) == 0){
            #namespaces supported: true
            if(defined $namespace_key){

                if(is_string($args->{ $namespace_key })){
                    push @$namespaces,split(/\|/o,$args->{ $namespace_key });
                }
                else{
                    my $r = $mw->api({ action => "query", meta => "siteinfo", "siprop" => "namespaces|general" }) or _fail($mw->{error});
                    return unless defined $r;
                    $siteinfo =  $r->{query};
                    push @$namespaces,sort keys(%{ $siteinfo->{namespaces} });
                }
                $cont_args->{$_} = { continue => '' } for @$namespaces;
            }
            #namespaces supported: false
            else{

                push @$namespaces,"_none_";
                $cont_args->{_none_} = { continue => '' };

            }
        }

        return unless scalar(@$namespaces);

        if(scalar(@$pages)==0){
            #results for namespace are depleted: goto next
            unless(defined ($cont_args->{ $namespaces->[0] })){
                shift @$namespaces;
            }

            return unless scalar(@$namespaces);

            #look until you found a decent namespace
            my $res;
            while(scalar(@$namespaces)){
                my $a = {
                    %$args,
                    %{$cont_args->{ $namespaces->[0] } },
                    action => "query",
                    indexpageids => 1,
                    generator => $generator,
                    format => "json"
                };
                if(defined $namespace_key){
                    $a->{ $namespace_key } = $namespaces->[0];
                }
                #will work with generator in the future
                delete $a->{rvlimit};
                $res = $mw->api($a);
                if(!$res){
                    if ( $mw->{error}->{details} =~  /gapunknown_gapnamespace/o || $mw->{error}->{details} =~ /API has returned an empty array reference/o ){
                        shift @$namespaces;
                        return unless scalar(@$namespaces);
                        next;
                    }
                    _fail($mw->{error});
                }else{
                    last;
                }
            }


            return unless defined $res;

            $cont_args->{ $namespaces->[0] } = $res->{'continue'};

            if(exists($res->{'query'}->{'pageids'})){

                for my $pageid(@{ $res->{'query'}->{'pageids'} }){
                    #'titles, pageids or a generator was used to supply multiple pages, but the limit, startid, endid, dirNewer, user, excludeuser, start and end parameters may only be used on a single page.'
                    #which means: cannot repeat pageids when asking for full history
                    my $page = $res->{'query'}->{'pages'}->{$pageid};

                    #add source url
                    my $title = $page->{title};
                    $title =~ s/\s/_/go;

                    {
                        my $articlepath = $siteinfo->{general}->{articlepath};
                        $articlepath =~ s/\$1/${title}/;

                        #server:  http://localhost:8000
                        my $server = $siteinfo->{general}->{server};
                        #servername: localhost:8000
                        my $servername = $siteinfo->{general}->{servername};

                        my $url;
                        if( is_string($server) ){

                            $url = $server.$articlepath;

                        }
                        else{

                            my $base = $siteinfo->{general}->{base};
                            my $protocol = $base =~ /^https/o ? "https" : "http";
                            $url = "${protocol}://".$siteinfo->{general}->{servername}.$articlepath;

                        }

                        $page->{_url} = $url;

                    }

                    if(is_string($args->{rvlimit})){

                        my $a = {
                            action => "query",
                            format => "json",
                            pageids => $pageid,
                            prop => "revisions",
                            rvprop => $args->{rvprop},
                            rvlimit => $args->{rvlimit}
                        };
                        my $res2 = $mw->api($a) or _fail($mw->{error});
                        $page->{revisions} = $res2->{'query'}->{'pages'}->{$pageid}->{revisions} if $res2->{'query'}->{'pages'}->{$pageid}->{revisions};

                        #add source url
                        for my $revision(@{ $page->{revisions} } ){

                            $revision->{_url} = $page->{_url}."?oldid=".$revision->{revid};

                        }

                    }

                    push @$pages,$page;
                }
            }
        }

        shift @$pages;
    };
}

=head1 NAME

Catmandu::Importer::MediaWiki - Catmandu importer that imports pages from mediawiki

=head1 DESCRIPTION

This importer uses the query api from mediawiki to get a list of pages
that match certain requirements.

It retrieves a list of pages and their content by using the generators
from mediawiki:

L<http://www.mediawiki.org/wiki/API:Query#Generators>

The default generator is 'allpages'.

The list could also be retrieved with the module 'list':

L<http://www.mediawiki.org/wiki/API:Lists>

But this module 'list' is very limited. It retrieves a list of pages
with a limited set of attributes (pageid, ns and title).

The module 'properties' on the other hand lets you add properties:

L<http://www.mediawiki.org/wiki/API:Properties>

But the selecting parameters (titles, pageids and revids) are too specific
to execute a query in one call. One should execute a list query, and then
use the pageids to feed them to the submodule 'properties'.

To execute a query, and add properties to the pages in one call can be
accomplished by use of generators.

L<http://www.mediawiki.org/wiki/API:Query#Generators>

These parameters are set automatically, and cannot be overwritten:

action = "query"
indexpageids = 1
generator = <generate>
format = "json"

Additional parameters can be set in the constructor argument 'args'.
Arguments for a generator origin from the list module with the same name,
but must be prepended with 'g'.

=head1 ARGUMENTS

=over 4

=item generate

type: string

explanation:    type of generator to use. For a complete list, see L<http://www.mediawiki.org/wiki/API:Lists>.
                because Catmandu::Iterable already defines 'generator', this parameter has been renamed
                to 'generate'.

default: 'allpages'.

=item args

type: hash

explanation: extra arguments. These arguments are merged with the defaults.

default:

    {
        prop => "revisions",
        rvprop => "ids|flags|timestamp|user|comment|size|content",
        gaplimit => 100,
        gapfilterredir => "nonredirects"
    }

which means:

    prop             add revisions to the list of page attributes
    rvprop           specific properties for the list of revisions
    gaplimit         limit for generator 'allpages' (every 'generator' has its own limit).
    gapfilterredir   filter out redirect pages

=item lgname

type: string

explanation:    login name. Only used when both lgname and lgpassword are set.

L<https://www.mediawiki.org/wiki/API:Login>

=item lgpassword

type: string

explanation:    login password. Only used when both lgname and lgpassword are set.

=back

=head1 SYNOPSIS

    use Catmandu::Sane;
    use Catmandu::Importer::MediaWiki;

    binmode STDOUT,":utf8";

    my $importer = Catmandu::Importer::MediaWiki->new(
        url => "http://en.wikipedia.org/w/api.php",
        generate => "allpages",
        args => {
            prop => "revisions",
            rvprop => "ids|flags|timestamp|user|comment|size|content",
            gaplimit => 100,
            gapprefix => "plato",
            gapfilterredir => "nonredirects"
        }
    );
    $importer->each(sub{
        my $r = shift;
        my $content = $r->{revisions}->[0]->{"*"};
        say $r->{title};
    });

=head1 AUTHORS

Nicolas Franck C<< <nicolas.franck at ugent.be> >>

=head1 SEE ALSO

L<Catmandu>, L<MediaWiki::API>

=cut

1;