prend une liste, et archive leurs signatures

install.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+echo "\nInstalling"
+echo "=========="
+echo "\nAuthor : nas"
+echo "Last Modification . 25/03/2021"
+
+echo "\nInstalling dpendencies"
+echo "------------------------"
+cpan install DBD::SQLite
+
+echo "\nCreating database"
+echo "-------------------"
+
+touch history.db
+sqlite3 -line history.db '
+CREATE TABLE visit(
+  url VARCHAR(128) NOT NULL,
+  date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  signature VARCHAR(128)
+);
+CREATE TABLE strategy(
+)
+'

syncWeb.pl

@@ -0,0 +1,94 @@
+#!/usr/bin/perl
+use strict;
+use Digest::SHA qw(sha256_hex);
+use LWP::UserAgent;
+use DBI qw(:sql_types);
+use Getopt::Long;
+use pQuery;
+
+sub init {
+    my $driver   = "SQLite";
+    my $database = "history.db";
+    my $dsn = "DBI:$driver:dbname=$database";
+    my $dbh = DBI->connect($dsn, { RaiseError => 1 })
+        or die $DBI::errstr;
+    print "Opened database successfully\n";
+    return $dbh;
+}
+
+sub add_History($$$){
+    my($url, $signature, $dbh) = @_;
+    my $stmt = qq(INSERT INTO visit (url,signature)
+        VALUES ("$url", "$signature"));
+    my $rv = $dbh->do($stmt) or die $DBI::errstr;
+}
+
+sub get_html($) {
+    my $ua = new LWP::UserAgent;
+    $ua->timeout(120);
+    my $url = @_ ;
+    my $request = new HTTP::Request('GET', $url);
+    my $response = $ua->request($request);
+    my $content = $response->content();
+    return $content;
+}
+
+sub visit($$){
+    my ($url, $dbh)= @_;
+    my $content = get_html "$url" ;
+    return sha256_hex( $content );
+}
+
+sub isNew($$$){
+    my($url, $signature, $dbh) = @_;
+    my $sth = $dbh->prepare("SELECT COUNT(signature) FROM visit WHERE signature=?1 AND url=?2");
+    $sth->execute($signature, $url);
+    my $refs = $sth->fetchall_arrayref()->[0][0];
+    return ($refs)? 0 : 1 ;
+}
+
+sub registerIfNew($$){
+    my($url, $dbh) = @_;
+    my $signature = visit($url, $dbh);
+
+    if( isNew($url, $signature ,$dbh)){
+	add_History($url, $signature ,$dbh);
+	return 1;
+    }
+    return 0;
+}
+
+sub importFromUrl($){
+    my ($url) = @_;
+    print "importing $url";
+}
+
+sub checkUrl($){
+    my($url) = @_;
+    my $dbh = init;
+    if( registerIfNew($url, $dbh) ){
+        importFromUrl($url);
+    }
+    $dbh->disconnect();
+}
+
+sub get_content($$){
+    ;
+}
+
+sub main($){
+    my($file) = @_;
+    print "Using file $file\n";
+    #checkUrl( $url );
+    open(FH, '<', $file) or die $!;
+    while(<FH>){
+	print "Checking $_";
+	checkUrl( $_ );
+    }
+    close(FH);
+}
+
+my $file;
+GetOptions ('file=s' => \$file);
+
+main( $file );