Big Sky :: WWW::MechanizeとXPathでtwitterのfriendsを全部取ってみる

2008/01/31

WWW::MechanizeとXPathでtwitterのfriendsを全部取ってみる

twitterのAPIでfriendsが100件しか取れなくなって久しいですが...
WWW::MechanizeとXPathでtwitterの全friendsを取得するサンプル作ってみました。あまりやり過ぎると、オフィシャル側に怒られそうな気もしますが...
後の使い方は、適当で...


#!/usr/local/bin/perl



use warnings;

use strict;

use LWP::Simple;

use XML::Simple;

use WWW::Mechanize;

use HTML::TreeBuilder::XPath;

use HTML::Selector::XPath qw(selector_to_xpath);

use Data::Dumper;



my $username = 'your_username';

my $password = 'your_password';



my $m = WWW::Mechanize->new(timeout => 10);

$m->get('http://twitter.com/login');

$m->submit_form(

    form_number => 1,

    fields    => {

        username_or_email  => $username,

        password           => $password,

    },

    button    => 'commit',

);



my $xpath = selector_to_xpath('tr.vcard');

my @friends;



my $num_page = 1;

while (1) {

    my $res = $m->get("http://twitter.com/friends/?page=$num_page");

    my $encoding = $res->header('Content-Encoding');

    my $content = $res->content;

    $content = Compress::Zlib::memGunzip($content) if $encoding =~ /gzip/i;

    $content = Compress::Zlib::uncompress($content) if $encoding =~ /deflate/i;



    my $tree = HTML::TreeBuilder::XPath->new;

    $tree->parse($content);

    $tree->eof;

    my @nodes = $tree->findnodes($xpath);

    for my $tr (@nodes) {

        push(@friends, {

                nick => $tr->findnodes('td/strong/a')->[0]->as_text,

                image => $tr->findvalue('td[@class="thumb"]//img/@src')->as_string,

                name => $tr->findvalue('td[@class="thumb"]//img/@alt')->as_string,

                description => $tr->findvalue('td/strong/a/@title')->as_string,

                url => $tr->findvalue('td[@class="thumb"]/a/@href')->as_string,

            });

    }

    $tree->delete;

    @nodes or last;

    $num_page++;

}



print Dumper @friends;

最近遊んでる物、ほとんどmiyagawa氏のものばっかだな...

Posted at 02:23 | WriteBacks () | Edit

Edit this entry...

wikieditish message: Ready to edit this entry.

New Name:

Title:

Body:

meta-tags: perl, twitter, xpath
meta-creation_date: 2008-01-31T02:23:27+09:00

twitterのAPIでfriendsが100件しか取れなくなって久しいですが... 
WWW::MechanizeとXPathでtwitterの全friendsを取得するサンプル作ってみました。
あまりやり過ぎると、オフィシャル側に怒られそうな気もしますが... 
後の使い方は、適当で...
<code class="code">
#!/usr/local/bin/perl 
 
use warnings; 
use strict; 
use LWP::Simple; 
use XML::Simple; 
use WWW::Mechanize; 
use HTML::TreeBuilder::XPath; 
use HTML::Selector::XPath qw(selector_to_xpath); 
use Data::Dumper; 
 
my&nbsp;$username&nbsp;= 'your_username'; 
my&nbsp;$password&nbsp;= 'your_password'; 
 
my&nbsp;$m&nbsp;= WWW::Mechanize-&gt;new(timeout =&gt; 10); 
$m-&gt;get('<a href="http://twitter.com/login">http://twitter.com/login</a>'); 
$m-&gt;submit_form( 
&nbsp;&nbsp;&nbsp;&nbsp;form_number =&gt; 1, 
&nbsp;&nbsp;&nbsp;&nbsp;fields&nbsp;&nbsp;&nbsp;&nbsp;=&gt; { 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;username_or_email&nbsp;&nbsp;=&gt; $username, 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;password&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; =&gt; $password, 
&nbsp;&nbsp;&nbsp;&nbsp;}, 
&nbsp;&nbsp;&nbsp;&nbsp;button&nbsp;&nbsp;&nbsp;&nbsp;=&gt; 'commit', 
); 
 
my&nbsp;$xpath&nbsp;= selector_to_xpath('tr.vcard'); 
my&nbsp;@friends; 
 
my&nbsp;$num_page&nbsp;= 1; 
while&nbsp;(1) { 
&nbsp;&nbsp;&nbsp;&nbsp;my&nbsp;$res&nbsp;= $m-&gt;get(&quot;<a href="http://twitter.com/friends/?page=">http://twitter.com/friends/?page=</a>$num_page&quot;); 
&nbsp;&nbsp;&nbsp;&nbsp;my&nbsp;$encoding&nbsp;= $res-&gt;header('Content-Encoding'); 
&nbsp;&nbsp;&nbsp;&nbsp;my&nbsp;$content&nbsp;= $res-&gt;content; 
&nbsp;&nbsp;&nbsp;&nbsp;$content&nbsp;= Compress::Zlib::memGunzip($content) if&nbsp;$encoding&nbsp;=~&nbsp;/gzip/i; 
&nbsp;&nbsp;&nbsp;&nbsp;$content&nbsp;= Compress::Zlib::uncompress($content) if&nbsp;$encoding&nbsp;=~&nbsp;/deflate/i; 
 
&nbsp;&nbsp;&nbsp;&nbsp;my&nbsp;$tree&nbsp;= HTML::TreeBuilder::XPath-&gt;new; 
&nbsp;&nbsp;&nbsp;&nbsp;$tree-&gt;parse($content); 
&nbsp;&nbsp;&nbsp;&nbsp;$tree-&gt;eof; 
&nbsp;&nbsp;&nbsp;&nbsp;my&nbsp;@nodes&nbsp;= $tree-&gt;findnodes($xpath); 
&nbsp;&nbsp;&nbsp;&nbsp;for&nbsp;my&nbsp;$tr&nbsp;(@nodes) { 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;push(@friends, { 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;nick =&gt; $tr-&gt;findnodes('td/strong/a')-&gt;[0]-&gt;as_text, 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;image =&gt; $tr-&gt;findvalue('td[@class=&quot;thumb&quot;]//img/@src')-&gt;as_string, 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;name =&gt; $tr-&gt;findvalue('td[@class=&quot;thumb&quot;]//img/@alt')-&gt;as_string, 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;description =&gt; $tr-&gt;findvalue('td/strong/a/@title')-&gt;as_string, 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;url =&gt; $tr-&gt;findvalue('td[@class=&quot;thumb&quot;]/a/@href')-&gt;as_string, 
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;}); 
&nbsp;&nbsp;&nbsp;&nbsp;} 
&nbsp;&nbsp;&nbsp;&nbsp;$tree-&gt;delete; 
&nbsp;&nbsp;&nbsp;&nbsp;@nodes&nbsp;or&nbsp;last; 
&nbsp;&nbsp;&nbsp;&nbsp;$num_page++; 
} 
 
print&nbsp;Dumper&nbsp;@friends; 
</code>
 
最近遊んでる物、ほとんどmiyagawa氏のものばっかだな...

Excerpt:

twitterのAPIでfriendsが100件しか取れなくなって久しいですが...WWW::MechanizeとXPathでtwitterの全friendsを取得するサンプル作ってみました。あまりやり過ぎると、オフィシャル側に怒られそうな気もしますが...後の使い方は、適当で...#!/usr/local/bin/perluse warnings;use strict;use LWP::Simple;use XML::Simple;use WWW::Mechanize;use HTML::TreeBuilder::XPath;use HTML::Selector::XPath qw(selector_to_xpath);use Data::Dumper;my $username = 'your_username';my $password = 'your_password';my $m = WWW::Mechanize->new(timeout => 10);$m->get('http://twitter.com/login');$m->submit_form(    form_number => 1,    fields    => {        username_or_email  => $username,        password           => $password,    },    button    => 'commit',);my $xpath = selector_to_xpath('tr.vcard');my @friends;my $num_page = 1;while (1) {    my $res = $m->get("http://twitter.com/friends/?page=$num_page");    my $encoding = $res->header('Content-Encoding');    my $content = $res->content;    $content = Compress::Zlib::memGunzip($content) if $encoding =~ /gzip/i;    $content = Compress::Zlib::uncompress($content) if $encoding =~ /deflate/i;    my $tree = HTML::TreeBuilder::XPath->new;    $tree->parse($content);    $tree->eof;    my @nodes = $tree->findnodes($xpath);    for my $tr (@nodes) {        push(@friends, {                nick => $tr->findnodes('td/strong/a')->[0]->as_text,                image => $tr->findvalue('td[@class="thumb"]//img/@src')->as_string,                name => $tr->findvalue('td[@class="thumb"]//img/@alt')->as_string,                description => $tr->findvalue('td/strong/a/@title')->as_string,                url => $tr->findvalue('td[@class="thumb"]/a/@href')->as_string,            });    }    $tree->delete;    @nodes or last;    $num_page++;}print Dumper @friends;最近遊んでる物、ほとんどmiyagawa氏のものばっかだな...

Ping URLs:

File:

Password:

A quick preview will be rendered here when you click "Preview" button.