#get names from tosearch.txt files in each date dir and search for those.
use LWP::Simple;
use LWP::UserAgent;
use HTTP::GetImages;
use Data::Dumper;

my $ua = LWP::UserAgent->new;
$ua->timeout(17);
$ua->env_proxy;
$ua->agent('Mozilla/5.0');
$|++;

$dir="C:\\projects\\chartsweep90s\\images";
opendir(DIR,$dir);
@files=readdir(DIR);
closedir(DIR);
print "files is @files\n";

foreach $date (sort @files)
{
	print "looking at $date\n";
	@imagelocs=();
	if ($date =~ /^[0-9]{8}$/)
	{
		print "it was good\n";
		$spot=$dir."\\".$date."\\tosearch\.txt";
		print "spot is $spot\n";
		$searchterms="";
		if (-f $spot)
		{open SRCH,"< $spot";
		$searchterms=<SRCH>;
		close SRCH;}
		print "searchterms are $searchterms\n";
		$searchterms=~s/\"//g;
		$searchurl='http://anonymouse.org/cgi-bin/anon-www.cgi/http://images.google.com/images?q=';
		#$searchurl='http://images.google.com/images?q=';
		@splitterms=split(", ",$searchterms);
		
		foreach $term (@splitterms)
		{
			if (!($term  =~/[a-z]/)) {next;}
			print "now working on term $term\n";
			$term =~ s/^\s*//;
			$term =~ s/\s*$//;
			$term =~ s/ /\+/g;
			
			$url=$searchurl.$term;
			$pagedurl=$url;
			for ($i=0;$i<1;$i++){$mult=21*$i;
				$pagedurl=$url;
				if ($mult)
				{
					$pagedurl=$url.'&start='.$mult;
				}
				
			print "looking for url $pagedurl\n";
			my $response = $ua->get($pagedurl);
			#print $response->content;
			if ($response->is_success) 
			{
				print "Just got $url for term $term\n";
				$stuff=$response->content;
				while ($stuff=~ /(http[^\=\&]*\.jpg)/g) 
				{
					$imageloc=$1;
					if (!($imageloc =~ /images\.google/))
						{
						push(@imagelocs,$imageloc); }
				}	
				while ($stuff=~ /(http[^\=\&]*\.gif)/g)
				{
					$imageloc=$1;
					if (!($imageloc =~ /images\.google/))
						{
						push(@imagelocs,$imageloc);} 
				}	
			}
			else {print "response was NOT success!\n"; print $response->content}}
		}
		print "about to do get\n if".@imagelocs;
		gett($date) if (@imagelocs);
	}
}

sub gett
{
$savedir=shift(@_);
print "imagelocs is \n";
print join("\n",@imagelocs);
print "dir is $savedir\n";
	$_ = new HTTP::GetImages (
                dir  => "images\/$savedir",
                todo => [@imagelocs],
                dont => ['ALL'],
                chat => 1,
		max_attempts => 1,
	);

        $_->print_imgs;
        #$_->print_done;
        #$_->print_failed;
        #$_->print_ignored;

        my $hash = $_->imgs_as_hash;
        foreach (keys %{$hash}){
                warn "$_ = ",$hash->{$_},"\n";
        }
	@imagelocs=();
}