strategy de conversion, html vers md

syncWeb.pl

@@ -1,19 +1,29 @@
 #!/usr/bin/perl
 use strict;
 use Digest::SHA qw(sha256_hex);
-use LWP::UserAgent;
+use HTTP::Tiny;
 use DBI qw(:sql_types);
 use Getopt::Long;
-use pQuery;
+use Mojo::DOM;
+use HTML::WikiConverter;
+use Switch;
+#use Data::Dumper;
 
-sub init {
-    my $driver   = "SQLite";
-    my $database = "history.db";
+sub trim {
+    (my $s = $_[0]) =~ s/^\s+|\s+$//g;
+    return $s;
+}
+
+sub connect_db($$$){
+    my ($driver, $database, $password) = @_;
     my $dsn = "DBI:$driver:dbname=$database";
     my $dbh = DBI->connect($dsn, { RaiseError => 1 })
         or die $DBI::errstr;
     print "Opened database successfully\n";
-    return $dbh;
+}
+
+sub init {
+    return connect_db("SQLite", "history.db", "");
 }
 
 sub add_History($$$){
@@ -23,14 +33,20 @@ sub add_History($$$){
     my $rv = $dbh->do($stmt) or die $DBI::errstr;
 }
 
-sub get_html($) {
-    my $ua = new LWP::UserAgent;
-    $ua->timeout(120);
-    my $url = @_ ;
-    my $request = new HTTP::Request('GET', $url);
-    my $response = $ua->request($request);
-    my $content = $response->content();
-    return $content;
+sub get_html($){
+    my ($url) = @_;
+    my $response = HTTP::Tiny->new->get($url);
+    if ($response->{success}) {
+        return $response->{content};
+    }
+    return 0;
+}
+
+sub get_content($$){
+    my($url, $selector) = @_;
+    my $content = get_html( $url );
+    my $dom = Mojo::DOM->new( $content );
+    return $dom->at( "$selector" );
 }
 
 sub visit($$){
@@ -41,15 +57,19 @@ sub visit($$){
 
 sub isNew($$$){
     my($url, $signature, $dbh) = @_;
-    my $sth = $dbh->prepare("SELECT COUNT(signature) FROM visit WHERE signature=?1 AND url=?2");
-    $sth->execute($signature, $url);
-    my $refs = $sth->fetchall_arrayref()->[0][0];
-    return ($refs)? 0 : 1 ;
+    # my $sth = $dbh->prepare("SELECT count(signature) AS COUNT FROM visit WHERE signature=?1 AND url=?2;");
+    my $sth = $dbh->prepare("SELECT count(signature) AS COUNT FROM visit WHERE signature='$signature' AND url='$url';");
+    #$sth->execute("$signature", "$url");
+    $sth->execute();
+    my $refs = $sth->fetchrow_arrayref()->[0];
+    print "SELECT count(signature) AS COUNT FROM visit WHERE signature=$signature AND url=$url\n";
+    print "$url, count : $refs\n";
+    return !($refs);
 }
 
-sub registerIfNew($$){
-    my($url, $dbh) = @_;
-    my $signature = visit($url, $dbh);
+sub registerIfNew($$$){
+    my($url, $selector,$dbh) = @_;
+    my $signature = sha256_hex( get_content($url, $selector) );
 
     if( isNew($url, $signature ,$dbh)){
 	add_History($url, $signature ,$dbh);
@@ -58,32 +78,55 @@ sub registerIfNew($$){
     return 0;
 }
 
-sub importFromUrl($){
-    my ($url) = @_;
-    print "importing $url";
+sub convertStrategy($$$){
+    my ($content, $fromFormat, $toFormat) = @_;
+    
+    switch("$fromFormat $toFormat") {
+	case "html md"  { return htmlToMd( $content ) }
+	case "a"        { print "string a" }
+	else            { return $content }
+    }
+}
+
+sub importContent($$$$){
+    my ($url, $selector, $fromFormat, $toFormat) = @_;
+    my $content = get_content( $url, $selector );
+
+    if( $content ){
+	return convertStrategy( $content, $fromFormat, $toFormat );
+    }
+    return 0;
 }
 
-sub checkUrl($){
-    my($url) = @_;
+sub checkUrl($$){
+    my($url, $selector) = @_;
     my $dbh = init;
-    if( registerIfNew($url, $dbh) ){
-        importFromUrl($url);
+    if( registerIfNew($url, $selector,$dbh) ){
+        get_content( $url, $selector )
     }
     $dbh->disconnect();
 }
 
-sub get_content($$){
-    ;
+sub htmlToMd($){
+    my($html) = @_;
+    my $wc = new HTML::WikiConverter( dialect => 'Markdown' );
+    return $wc->html2wiki( html => $html );
 }
 
 sub main($){
+    my $dbh = init;
     my($file) = @_;
-    print "Using file $file\n";
-    #checkUrl( $url );
     open(FH, '<', $file) or die $!;
     while(<FH>){
-	print "Checking $_";
-	checkUrl( $_ );
+	my($url, $selector, $fromFormat, $toFormat) = split("\t");
+	$fromFormat = trim($fromFormat);
+	$toFormat = trim($toFormat);
+	#print get_content( $url, $selector );
+	#checkUrl( $_ );
+	#print registerIfNew( $url, $selector, $dbh );
+	print " $url, $selector, $fromFormat, $toFormat ";
+	print importContent( $url, $selector, $fromFormat, $toFormat );
+	print "\n"
     }
     close(FH);
 }