|
@@ -7,23 +7,77 @@ use Getopt::Long;
|
7
|
7
|
use Mojo::DOM;
|
8
|
8
|
use HTML::WikiConverter;
|
9
|
9
|
use Switch;
|
10
|
|
-#use Data::Dumper;
|
|
10
|
+use String::Escape;
|
|
11
|
+
|
|
12
|
+use Data::Dumper::Perltidy;
|
|
13
|
+
|
|
14
|
+my $import_list;
|
|
15
|
+my $export;
|
|
16
|
+my $config;
|
11
|
17
|
|
12
|
18
|
sub trim {
|
13
|
19
|
(my $s = $_[0]) =~ s/^\s+|\s+$//g;
|
14
|
20
|
return $s;
|
15
|
21
|
}
|
16
|
22
|
|
17
|
|
-sub connect_db($$$){
|
18
|
|
- my ($driver, $database, $password) = @_;
|
|
23
|
+sub connect_db($$$$$){
|
|
24
|
+ my ($driver, $database, $user, $password, $host) = @_;
|
|
25
|
+ my $dbh = 0;
|
19
|
26
|
my $dsn = "DBI:$driver:dbname=$database";
|
20
|
|
- my $dbh = DBI->connect($dsn, { RaiseError => 1 })
|
21
|
|
- or die $DBI::errstr;
|
22
|
|
- print "Opened database successfully\n";
|
|
27
|
+
|
|
28
|
+ if( $user && $password){
|
|
29
|
+ $dbh = DBI->connect($dsn, $user, $password, { RaiseError => 1 })
|
|
30
|
+ or die $DBI::errstr;
|
|
31
|
+ }elsif( $user ){
|
|
32
|
+ $dbh = DBI->connect($dsn, $user, { RaiseError => 1 })
|
|
33
|
+ or die $DBI::errstr;
|
|
34
|
+ }else{
|
|
35
|
+ $dbh = DBI->connect($dsn, { RaiseError => 1 })
|
|
36
|
+ or die $DBI::errstr;
|
|
37
|
+ }
|
|
38
|
+
|
|
39
|
+ return $dbh;
|
23
|
40
|
}
|
24
|
41
|
|
25
|
|
-sub init {
|
26
|
|
- return connect_db("SQLite", "history.db", "");
|
|
42
|
+sub to_mariadb($$){
|
|
43
|
+ my ($export, $content) = @_;
|
|
44
|
+
|
|
45
|
+ connect_db(
|
|
46
|
+ "mysql",
|
|
47
|
+ "$config->{'export'}->{$export}->{'connexion'}->{'base'}",
|
|
48
|
+ "$config->{'export'}->{$export}->{'connexion'}->{'user'}",
|
|
49
|
+ "$config->{'export'}->{$export}->{'connexion'}->{'password'}",
|
|
50
|
+ "$config->{'export'}->{$export}->{'connexion'}->{'host'}"
|
|
51
|
+ );
|
|
52
|
+
|
|
53
|
+ my $columns_default = "";
|
|
54
|
+ my $columns_default_value = "";
|
|
55
|
+
|
|
56
|
+ if( !"$config->{'export'}->{$export}->{'column_default'}" ){
|
|
57
|
+ $columns_default = "";
|
|
58
|
+ #for each...
|
|
59
|
+ }
|
|
60
|
+
|
|
61
|
+ print "INSERT INTO $config->{'export'}->{$export}->{'content'}->{'table'} (
|
|
62
|
+ $config->{'export'}->{$export}->{'content'}->{'column'}
|
|
63
|
+ $columns_default
|
|
64
|
+ )
|
|
65
|
+ VALUES ('$content' $columns_default_value);";
|
|
66
|
+
|
|
67
|
+ # close db here
|
|
68
|
+}
|
|
69
|
+
|
|
70
|
+sub to_file($$){
|
|
71
|
+ my($path, $content) = @_;
|
|
72
|
+ my $filename = "$path";
|
|
73
|
+ open(my $fh, '>', $filename) or die "Could not open file '$filename' $!";
|
|
74
|
+ print $fh "$content";
|
|
75
|
+ close $fh;
|
|
76
|
+ print "done\n";
|
|
77
|
+}
|
|
78
|
+
|
|
79
|
+sub init (){
|
|
80
|
+ return connect_db("SQLite", "history.db", "", "", "");
|
27
|
81
|
}
|
28
|
82
|
|
29
|
83
|
sub add_History($$$){
|
|
@@ -62,8 +116,7 @@ sub isNew($$$){
|
62
|
116
|
#$sth->execute("$signature", "$url");
|
63
|
117
|
$sth->execute();
|
64
|
118
|
my $refs = $sth->fetchrow_arrayref()->[0];
|
65
|
|
- print "SELECT count(signature) AS COUNT FROM visit WHERE signature=$signature AND url=$url\n";
|
66
|
|
- print "$url, count : $refs\n";
|
|
119
|
+
|
67
|
120
|
return !($refs);
|
68
|
121
|
}
|
69
|
122
|
|
|
@@ -78,7 +131,7 @@ sub registerIfNew($$$){
|
78
|
131
|
return 0;
|
79
|
132
|
}
|
80
|
133
|
|
81
|
|
-sub convertStrategy($$$){
|
|
134
|
+sub convert_strategy($$$){
|
82
|
135
|
my ($content, $fromFormat, $toFormat) = @_;
|
83
|
136
|
|
84
|
137
|
switch("$fromFormat $toFormat") {
|
|
@@ -113,25 +166,56 @@ sub htmlToMd($){
|
113
|
166
|
return $wc->html2wiki( html => $html );
|
114
|
167
|
}
|
115
|
168
|
|
116
|
|
-sub main($){
|
|
169
|
+sub export_strategy($$$){
|
|
170
|
+ my ($export, $format, $content) = @_;
|
|
171
|
+ print "export straty de $export $config->{'export'}->{$export}->{'type'}";
|
|
172
|
+
|
|
173
|
+ $content = convert_strategy(
|
|
174
|
+ $content,
|
|
175
|
+ $format,
|
|
176
|
+ "$config->{'export'}->{$export}->{'format'}"
|
|
177
|
+ );
|
|
178
|
+ $content = String::Escape::backslash( $content );
|
|
179
|
+
|
|
180
|
+ if( $config->{'export'}->{$export}->{'type'} eq "mariadb" ){
|
|
181
|
+ print "\nis $config->{'export'}->{$export}->{'type'} mariadb?\n";
|
|
182
|
+ to_mariadb( $export, $content );
|
|
183
|
+ }else{
|
|
184
|
+ to_file( "$config->{'export'}->{$export}->{'path'}", $content );
|
|
185
|
+ }
|
|
186
|
+}
|
|
187
|
+
|
|
188
|
+sub main($$){
|
117
|
189
|
my $dbh = init;
|
118
|
|
- my($file) = @_;
|
119
|
|
- open(FH, '<', $file) or die $!;
|
|
190
|
+ my($import_list, $export) = @_;
|
|
191
|
+
|
|
192
|
+ $config=do("./$export");
|
|
193
|
+
|
|
194
|
+ #$content = print encode 'unicode-escape',;
|
|
195
|
+ my $test = 'la""la';
|
|
196
|
+ print "+";
|
|
197
|
+ print String::Escape::backslash( $test );
|
|
198
|
+ print "\n";
|
|
199
|
+ open(FH, '<', $import_list) or die $!;
|
120
|
200
|
while(<FH>){
|
121
|
|
- my($url, $selector, $fromFormat, $toFormat) = split("\t");
|
122
|
|
- $fromFormat = trim($fromFormat);
|
123
|
|
- $toFormat = trim($toFormat);
|
124
|
|
- #print get_content( $url, $selector );
|
125
|
|
- #checkUrl( $_ );
|
126
|
|
- #print registerIfNew( $url, $selector, $dbh );
|
127
|
|
- print " $url, $selector, $fromFormat, $toFormat ";
|
128
|
|
- print importContent( $url, $selector, $fromFormat, $toFormat );
|
129
|
|
- print "\n"
|
|
201
|
+ my($url, $selector, $fromFormat, $toFormat, $export) = split("\t");
|
|
202
|
+ $fromFormat = trim( $fromFormat );
|
|
203
|
+ $toFormat = trim( $toFormat );
|
|
204
|
+ $export = trim( $export );
|
|
205
|
+
|
|
206
|
+ my $content = get_content( $url, $selector );
|
|
207
|
+
|
|
208
|
+ export_strategy( $export, $toFormat, $content );
|
|
209
|
+ if( registerIfNew( $url, $selector, $dbh ) ){
|
|
210
|
+ export_strategy( $export, $toFormat, $content );
|
|
211
|
+ }
|
130
|
212
|
}
|
131
|
213
|
close(FH);
|
132
|
214
|
}
|
133
|
215
|
|
134
|
|
-my $file;
|
135
|
|
-GetOptions ('file=s' => \$file);
|
|
216
|
+GetOptions (
|
|
217
|
+ 'import-list=s' => \$import_list,
|
|
218
|
+ 'export=s' => \$export
|
|
219
|
+);
|
136
|
220
|
|
137
|
|
-main( $file );
|
|
221
|
+main( $import_list, $export );
|