getit.pl

#!/usr/bin/perl
use strict;
#use warnings;

use threads;
use Thread::Queue;
use MysqlDB;

my $dbh = new MysqlDB;

my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime time;
$year += 1900;
my %months = (
    0 => 'Jan',
    1 => 'Feb',
    2 => 'Mar',
    3 => 'Apr',
    4 => 'May',
    5 => 'Jun',
    6 => 'Jul',
    7 => 'Aug',
    8 => 'Sep',
    9 => 'Oct',
   10 => 'Nov',
   11 => 'Dec',
);

# Save the results to a file
my $ts = $year."_".$months{$mon}."_".$mday."_".$hour."-".$min;
chomp($ts);
print $ts . "\n";
my $file = "/tmp/clpanel_grab_$ts";
if (-e $file) {
  `:> $file`;
} else {
  `touch $file`;
  print "File now exists \n";
}

# This script uses the 'links' console application to do its scraping
# check for this application or else die saying please install.
my $links = `which links`;
if (!$links) {
  print "Please install links for this to work <!-- ";
  die;
}

my $pday             = $mday - 1;
my $date             = $ARGV[2] || $pday;
if((!$ARGV[0]) || (!$ARGV[1]) || (!$date)) {
	die "$0 function search date \n";
}

my @array;
$#array = 1000;
my @getit;
$#getit = 1000;

open FILE, "cl-list.txt" or die "ERROR: Investigate cl-list.txt";
@array = <FILE>;
close(FILE);

## Page head
#print "Craigslist results for $args{function} search $args{search} on $args{date} <br><br>";
#print "<table border=1><tr><th>Listing</th><th>Source</th></tr>";

## THREADS
my $a = 6; # Number of workers
my $b = 10; # How much to collect before ending the round
my $c = @array; # Max number.

my $queue = new Thread::Queue;
&loopit();

## Print based on function
sub loopit() {
    my $suburl;
    $suburl = "search/cpg?query=$ARGV[1]&catAbbreviation=cpg&addThree=" if ($ARGV[0] eq "gigs");

    $suburl = "search/jjj?query=$ARGV[1]&addOne=telecommuting" if ($ARGV[0] eq "jobs");
  
    while($c) {
        while($b && $a) {
            my $url = $array[$c];
            chomp($url);
            my $url2 = "$url/$suburl";
            my $child = threads->create(\&getit2,$url,$url2,$date),
            $c--;
            $b--;
            $a--;
        }
	my $tid = $queue->dequeue();
	my $child = threads->object($tid);
	$b++;
	$a++;
    }

    open FILE, $file;
    while (<FILE>) {
        $dbh->do($_);
    }
}

sub getit2($$$) {
    open FILE, ">>", $file;
    my $tid = threads->tid();
    my %sa;
    ($sa{url},$sa{url2},$sa{date}) = @_;
    @getit = qx(links -dump "$sa{url2}");
    for my $results (@getit) {
        if($results =~ m/$sa{date}/ && $results !~ /\[ help \]/) {
            chomp($results);
            $results =~ s/^\s+//g;
            $results =~ s/'/\\'/;
            print FILE qq{insert into results values ('','$ARGV[0]','$ARGV[1]','$results','<a href="$sa{url2}>">$sa{url}</a>');} . "\n";
	}
    }
    $queue->enqueue($tid);
    close(FILE);
}