From: Raphaël Gertz Date: Wed, 24 Jan 2024 04:16:29 +0000 (+0100) Subject: Import binaries X-Git-Url: https://git.rapsys.eu/binary/commitdiff_plain/0c92ed6ef774fe87f0b45016eae86b50d2fc0d5f Import binaries --- 0c92ed6ef774fe87f0b45016eae86b50d2fc0d5f diff --git a/blacklist b/blacklist new file mode 100755 index 0000000..6faa52a --- /dev/null +++ b/blacklist @@ -0,0 +1,382 @@ +#! /usr/bin/perl + +use strict; +use warnings; + +#Load IPC::System::Simple +use IPC::System::Simple qw(capturex); + +#Load Data::Validate::IP +use Data::Validate::IP qw(is_ipv4 is_ipv6); + +#Load NetAddr::IP::Util +use NetAddr::IP::Util qw(shiftleft inet_4map6 ipv4to6); + +#Load NetAddr::IP +use NetAddr::IP qw(:nofqdn Ones); + +#Load POSIX +use POSIX qw(EXIT_SUCCESS EXIT_FAILURE); + +#Load Data::Dumper +use Data::Dumper; + +#use Net::CIDR qw(range2cidr); +#use Socket; +#use Data::Dump qw(dump); + +#IP v4 hash +my %ip4s = (); +#IP v6 hash +my %ip6s = (); +#Blacklist v4 array +my @blrule4s = (); +#Blacklist v6 array +my @blrule6s = (); + +#Init block list +my $blocklist = 0; + +#Init spam list +my $spamlist = 1; + +#Filter options +@ARGV = map { + if ($_ eq '-b' || $_ eq '--blocklist') { + $blocklist = 1; (); + } elsif ($_ eq '-nb' || $_ eq '--noblocklist') { + $blocklist = 0; (); + } elsif ($_ eq '-s' || $_ eq '--spamlist') { + $spamlist = 1; (); + } elsif ($_ eq '-ns' || $_ eq '--nospamlist') { + $spamlist = 0; (); + } else { + $_; + } +} @ARGV; + +#Show usage with invalid argument +if (scalar(@ARGV)) { + print "Usage: $0 [-b|--blocklist|-nb|--noblocklist|-s|--spamlist|-ns|--nospamlist]\n"; + exit EXIT_FAILURE; +} + +#IP whitelist +#my $iplist = qr/^(?:127\.|::1|2a01:4f8:190:22a6:|5\.9\.143\.173|85\.68\.|81\.67\.|89\.157\.|82\.241\.255\.46)/; +my %iplist = ( + ipv4 => [ + #Localhost + '127.0.0.0/8', + #Aurae + '144.76.27.210/32', + #Toulouse + '82.241.255.46/32', + #Akasha + #'89.157.132.244/32' + #'89.3.145.115/32' + '89.3.147.209/32', + #Hotel recamier (because of port scan) + '92.154.96.153/32', + #Ygg tracker (tracker.yggtracker.cc) + '31.220.0.116/32', + #Coppersurfer tracker (tracker.coppersurfer.tk) + '31.14.40.30/32', + #Pussytorrent tracker (tracker.pussytorrents.org) + '217.23.12.105/32' + ], + ipv6 => [ + #Localhost + '::1/32', + #Aurae + '2a01:4f8:191:1405::/64' + ] +); + +#IP blacklist +my %ipblist = ( + ipv4 => [ + #Trident media guard + '82.138.74.0/25', + '82.138.70.128/26', + '91.189.104.0/21', + '154.45.216.128/25', + '193.107.240.0/22' + ], + ipv6 => [ + ] +); + +#Create a new NetAddr::IP object without calling slow gethostbyname (load /etc/resolv.conf) +sub new_ipv4($) { + #Extract ip and mask + my ($ip, $mask) = split('/', shift); + #Build base struct + my $self = { + #Set mask + mask => !defined($mask)||$mask==32?Ones:shiftleft(Ones, 32 - $mask), + #Mark as ipv4 + isv6 => 0, + #Generate fake address + #XXX: NetAddr::IP expect a faked Socket6 gethostbyname struct + #XXX: see /usr/lib64/perl5/vendor_perl/NetAddr/IP/Util.pm +235 + addr => inet_4map6(ipv4to6(pack('C4', split('\.', $ip)))) + }; + #Return fake NetAddr::IP object + return bless $self, 'NetAddr::IP'; +} + +#User whitelist +my @userlist = ('rapsys', 'airlibre'); + +#Port whitelist +my %portlist = ( + tcp => [ 80, 443, 8000, 8080, 8443 ], + udp => [ 443, 8443 ] +); + +#Extract sshd.service scan +#map { +# #Extract user and ip +# if (/Failed password for (?:invalid user )?(.+) from (.+) port [0-9]+ ssh2/ && grep($_ ne $1, @userlist)) { +# #Save ip +# my $ip = $2; +# #Check if v4 ip and not in whitelist +# if (is_ipv4($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { +# #Add ip in v4 blacklist +# $ip4s{$ip}=1; +# #Check if v6 ip +# } elsif (is_ipv6($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv6}}) { +# $ip6s{$ip}=1; +# } +# } +#} capturex('journalctl', '-u', 'sshd.service'); + +#Extract kernel port scan +map { + #XXX to ignore: net-fw DROP IN=enp3s0 OUT= MAC=50:46:5d:a1:a1:85:0c:86:10:f5:c6:4b:08:00 SRC=1.39.26.11 DST=144.76.27.210 LEN=348 TOS=0x00 PREC=0x00 TTL=51 ID=52475 PROTO=ICMP TYPE=3 CODE=3 [SRC=144.76.27.210 DST=1.39.26.11 LEN=320 TOS=0x00 PREC=0x00 TTL=47 ID=52473 PROTO=UDP SPT=6700 DPT=40992 LEN=300 ] + #net-fw DROP IN=enp3s0 OUT= MAC=50:46:5d:a1:a1:85:0c:86:10:f5:c6:4b:08:00 SRC=61.227.52.153 DST=144.76.27.210 LEN=52 TOS=0x00 PREC=0x00 TTL=116 ID=29123 DF PROTO=TCP SPT=64349 DPT=445 WINDOW=8192 RES=0x00 SYN URGP=0 + #net-fw DROP IN=enp3s0 OUT= MAC=50:46:5d:a1:a1:85:0c:86:10:f5:c6:4b:08:00 SRC=110.34.70.110 DST=144.76.27.210 LEN=40 TOS=0x00 PREC=0x00 TTL=50 ID=17488 PROTO=TCP SPT=58225 DPT=34567 WINDOW=53283 RES=0x00 SYN URGP=0 + if (/net-fw DROP .* SRC=([^\s]+) [^\[]* PROTO=(TCP|UDP) .* DPT=([^\s]+)/ && (! defined ${portlist}{lc($2)} || ! scalar map { unless ($_ eq $3) { (); } } @{$portlist{lc($2)}})) { + #Save ip + my $ip = $1; + #Save proto + my $proto = lc($2); + #Save dpt + my $dpt = $3; + #Check if v4 ip and not in whitelist + #if (is_ipv4($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (is_ipv4($ip) && not scalar map { my $network = new_ipv4($_); my $netip = new_ipv4($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (!defined $ip4s{$ip}) { + %{$ip4s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip in v4 blacklist + $ip4s{$ip}{$proto}{$dpt}=1; + } elsif (is_ipv6($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv6}}) { + if (!defined $ip6s{$ip}) { + %{$ip6s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip in v6 blacklist + $ip6s{$ip}{$proto}{$dpt}=1; + } + #XXX to ignore: imap-login: Disconnected: Inactivity (no auth attempts in 180 secs): user=<>, rip=92.154.96.153, lip=144.76.27.210, TLS handshaking, session=<86h+BZAE1VdcmmCZ> + #From journalctl -f -m -t dovecot -o cat --no-hostname | grep -E '^imap-login: Disconnected: .+ user=<(.+)>, .*rip=([^,]+)' + #imap-login: Disconnected: Aborted login by logging out (auth failed, 1 attempts in 6 secs): user=, method=PLAIN, rip=103.199.18.128, lip=144.76.27.210, TLS, session= + } elsif (/^imap-login: Disconnected: (?:Too many invalid commands|Aborted login by logging out|Connection closed \((?:auth failed, [0-9]+ attempts in [0-9]+ secs|tried to use unsupported auth mechanism)\)|Connection closed: (?:read\(size=[0-9]+\)|SSL_accept\(\)|SSL_read) failed).*? user=<(.*?)>, .*?rip=(.+?), / && grep($_ ne $1, @userlist)) { + #Save ip + my $ip = $2; + #Set blacklist for 143, 993, 4190 dest ports on tcp + my %blacklist = ('tcp' => [ 143, 993, 4190 ]); + #Check if v4 ip and not in whitelist + if (is_ipv4($ip) && not scalar map { my $network = new_ipv4($_); my $netip = new_ipv4($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (!defined $ip4s{$ip}) { + %{$ip4s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v4 blacklist + map { my $proto = $_; map { $ip4s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + #Check if v6 ip + } elsif (is_ipv6($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv6}}) { + if (!defined $ip6s{$ip}) { + %{$ip6s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v6 blacklist + map { my $proto = $_; map { $ip6s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + } + #XXX to ignore: warning: non-SMTP command from lputeaux-656-1-79-153.w92-154.abo.wanadoo.fr[92.154.96.153]: GET / HTTP/1.1 + #sudo journalctl -f -m -t postfix/smtpd -t postfix/submission/smtpd -t postfix/submissions/smtpd -t postfix/smtps/smtpd -o cat --no-hostname | grep -E '^(warning: Illegal address syntax from [^ ]+\[|warning: numeric hostname: |warning: [^ ]+\[|SSL_accept error from [^ ]+\[)([0-9a-f:\.]+)(|\]:? .*)$' + #sudo journalctl -f -m -t postfix/smtpd -t postfix/submission/smtpd -t postfix/submissions/smtpd -t postfix/smtps/smtpd -o cat --no-hostname | perl -pne 'if (/^(?:warning: Illegal address syntax from [^ ]+\[|warning: numeric hostname: |warning: [^ ]+\[|SSL_accept error from [^ ]+\[)([0-9a-f:\.]+)(?:|\]:? .*)$/) { print $1.":".$_; } else { }; undef $_;' + #warning: Illegal address syntax from unknown[113.80.102.166] in MAIL command: + #warning: numeric hostname: 187.145.20.160 + #warning: non-SMTP command from unknown[2001:41d0:8:ed7b::1]: * + #warning: unknown[80.94.95.184]: SASL LOGIN authentication failed: UGFzc3dvcmQ6 + #SSL_accept error from 162.194.196.104.bc.googleusercontent.com[104.196.194.162]: lost connection + #} elsif (/^warning: (?:.*\[(.+)\]: .+ authentication failed:.*|numeric hostname: (.+)|non-SMTP command from .*\[(.+)\]: .*|Illegal address syntax from .*\[(.+)\] in .+ command: .*)$/) { + } elsif (/^(?:warning: Illegal address syntax from [^ ]+\[|warning: numeric hostname: |warning: [^ ]+\[|SSL_accept error from [^ ]+\[)([0-9a-f:\.]+)(?:|\]:? .*)$/) { + #Save ip + my $ip = $1; + #Set blacklist for 25, 465, 587 dest ports on tcp + my %blacklist = ('tcp' => [ 25, 465, 587 ]); + #Check if v4 ip and not in whitelist + if (is_ipv4($ip) && not scalar map { my $network = new_ipv4($_); my $netip = new_ipv4($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (!defined $ip4s{$ip}) { + %{$ip4s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v4 blacklist + map { my $proto = $_; map { $ip4s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + #Check if v6 ip + } elsif (is_ipv6($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv6}}) { + if (!defined $ip6s{$ip}) { + %{$ip6s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v6 blacklist + map { my $proto = $_; map { $ip6s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + } + #From journalctl -m -t sshd -o cat --no-hostname | grep -E '^Failed .+ for( invalid user)? (.+) from (.+) port [0-9]+ (ssh)2$' + #Failed keyboard-interactive/pam for invalid user supervisor from 168.0.232.246 port 26468 ssh2 + #Failed password for invalid user mc from 103.38.4.238 port 37738 ssh2 + } elsif (/^Failed .+ for(?: invalid user)? (.+) from (.+) port [0-9]+ ssh2$/ && grep($_ ne $1, @userlist)) { + #Save ip + my $ip = $2; + #Set blacklist for 22 dest port on tcp + my %blacklist = ('tcp' => [ 22 ]); + #Check if v4 ip and not in whitelist + if (is_ipv4($ip) && not scalar map { my $network = new_ipv4($_); my $netip = new_ipv4($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (!defined $ip4s{$ip}) { + %{$ip4s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v4 blacklist + map { my $proto = $_; map { $ip4s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + #Check if v6 ip + } elsif (is_ipv6($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv6}}) { + if (!defined $ip6s{$ip}) { + %{$ip6s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v6 blacklist + map { my $proto = $_; map { $ip6s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + } + } +} capturex('journalctl', '-m', '-t', 'kernel', '-t', 'dovecot', '-t', 'postfix/smtpd', '-t', 'postfix/submission/smtpd', '-t', 'postfix/submissions/smtpd', '-t', 'postfix/smtps/smtpd', '-t', 'sshd', '-o', 'cat', '--no-hostname'); + +#With spamlist +if ($spamlist) { + #Extract originating ip in spam + map { + #Open spam file for reading + open (my $fh, '<', $_) or die "Can't open < $_: $!"; + + #Lookup for X-Originating-IP header + while (<$fh>) { + if (/X-Originating-IP: (.+)$/) { + #Set ip + my $ip = ${1}; + + #Set blacklist for 80 and 443 dest ports on tcp + #my %blacklist = ('tcp' => [ 80, 443, 8000, 8080, 8443 ]); + my %blacklist = %portlist; + + #Check if v4 ip and not in whitelist + #if (is_ipv4($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (is_ipv4($ip) && not scalar map { my $network = new_ipv4($_); my $netip = new_ipv4($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv4}}) { + if (!defined $ip4s{$ip}) { + %{$ip4s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v4 blacklist + map { my $proto = $_; map { $ip4s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + } elsif (is_ipv6($ip) && not scalar map { my $network = NetAddr::IP->new($_); my $netip = NetAddr::IP->new($ip); unless ($network->contains($netip)) { (); } } @{$iplist{ipv6}}) { + if (!defined $ip6s{$ip}) { + %{$ip6s{$ip}} = ('tcp' => {}, 'udp' => {}); + } + #Add ip tuples in v6 blacklist + map { my $proto = $_; map { $ip6s{$ip}{$proto}{$_}=1; } @{$blacklist{$proto}}; } keys %blacklist; + } + } + } + + #Close spam file + close $fh or die "Can't close fh: $!"; + } glob('/var/spool/mail/*/.Junk/{new,cur,tmp}/*'); +} + +#Process each ipv4s keys +map { + #Set proto as either tcp or udp + for my $proto (('tcp', 'udp')) { + #Check if branch is empty + if (!scalar keys %{$ip4s{$_}{$proto}}) { + #Prune it + delete $ip4s{$_}{$proto}; + } + } +} keys %ip4s; + +#Process each ipv6s keys +map { + #Set proto as either tcp or udp + for my $proto (('tcp', 'udp')) { + #Check if branch is empty + if (!scalar keys %{$ip6s{$_}{$proto}}) { + #Prune it + delete $ip6s{$_}{$proto}; + } + } +} keys %ip6s; + +#Open blrule4s file for reading +open (my $fh, '<', '/etc/shorewall/blrules') or die "Can't open < /etc/shorewall/blrules: $!"; + +#Populate with comments +@blrule4s = map { chomp($_); if (/^#/) { $_; } else { (); } } <$fh>; + +#Prepend each specific ip from whitelist +map { push @blrule4s, "WHITELIST\tnet:$1\tall" if (/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\/32$/); } @{$iplist{ipv4}}; + +#Prepend each specific ip from blacklist +#TODO: check that blacklist range is not in a whitelisted one !!! +map { push @blrule4s, "DROP\tnet:$1\tall" if (/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2})$/); } @{$ipblist{ipv4}}; + +#Close blrule4s file +close $fh or die "Can't close fh: $!"; + +#With blocklist +if ($blocklist) { + #Open bt_blocklists.ipv4 file for reading + open ($fh, '<', '/usr/local/share/blacklist/shorewall.ipv4') or die "Can't open < /usr/local/share/blacklist/shorewall.ipv4: $!"; + + #Prepend bt_blocklists.ipv4 drop + map { chomp $_; push @blrule4s, "DROP\t\tnet:$_\tfw"; } <$fh>; + + #Close bt_blocklists.ipv4 file + close $fh or die "Can't close fh: $!"; +} + +#Build blacklist +map { + #Set proto from hash + for my $proto (sort keys %{$ip4s{$_}}) { + #Push rule + push @blrule4s, "DROP\t\tnet:".$_.(length($_)<12?"\t":'')."\tfw\t$proto\t".(scalar keys %{$ip4s{$_}{$proto}}>5||defined $ip4s{$_}{$proto}{0}?'#':'').join(",", sort { $a <=> $b } keys %{$ip4s{$_}{$proto}}); + } +} sort keys %ip4s; + +#Open blrule4s file for writing +open ($fh, '>', '/etc/shorewall/blrules') or die "Can't open > /etc/shorewall/blrules: $!"; + +#Inject content of blacklist +map { print $fh $_."\n"; } @blrule4s; + +#Close blrule4s file +close $fh or die "Can't close fh: $!"; + +#Print ipv6 to update hash +#XXX; right now it don't seems scanned at all... +for (sort keys %ip6s) { + #Set proto from hash + for my $proto (keys %{$ip6s{$_}}) { + #Print the ipv6 scanner + print $_."\t$proto\t".join(",", keys %{$ip6s{$_}{$proto}})."\n"; + } +} + +# TODO: add ipv6 ? + +#Restart shorewall service +capturex('systemctl', 'restart', 'shorewall.service'); diff --git a/cpack b/cpack new file mode 100755 index 0000000..d8c5523 --- /dev/null +++ b/cpack @@ -0,0 +1,79 @@ +#! /usr/bin/perl + +# Best practice +use strict; +use warnings; + +# Load POSIX +use POSIX qw(EXIT_SUCCESS EXIT_FAILURE); + +# Load CSS::Packer +use CSS::Packer; + +# Init compress +my $compress = 'pretty'; + +# Filter options +@ARGV = map { if ($_ eq '-p' || $_ eq '--pretty') { $compress = 'pretty'; (); } elsif ($_ eq '-m' || $_ eq '--minify') { $compress = 'minify'; (); } else { $_; } } @ARGV; + +# Show usage with invalid argument or stdin opened to a tty +if (scalar(@ARGV) || -t STDIN) { + print "Usage: $0 [-p|--pretty|-m|--minify] < input.css > output.css\n"; + exit EXIT_FAILURE; +} + +# Instantiate packer object +my $packer = CSS::Packer->init(); + +# Load input in variable +my $input = do { local $/; }; + +# Minify input with required compression +$packer->minify(\$input, compress => $compress); + +# Show result +print $input; + +# Exit with success +exit EXIT_SUCCESS; + +__END__ + +=head1 NAME + +Cpack - A simple perl CSS minifier + +=head1 VERSION + +Version 0.1 + +=head1 DESCRIPTION + +A fast pure Perl CSS minifier script. + +=head1 AUTHOR + +Raphaël Gertz (Rapsys) << >>. + +=head1 COPYRIGHT & LICENSE + +Copyright 2016 - 2017 Raphaël Gertz, all rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +=head1 SEE ALSO + +L + +=cut diff --git a/git-new b/git-new new file mode 100755 index 0000000..3e4ee26 --- /dev/null +++ b/git-new @@ -0,0 +1,35 @@ +#! /bin/sh + +# Set git root +GITROOT='/var/www/git' + +# make sure we have repository to create +if [ $# -le 0 ]; then + echo "Usage: $0 repository" + exit 1 +fi + +# Switch to directory +cd $GITROOT + +# Handle args +for repo in $@; do + if [ -d "$GITROOT/$repo" ]; then + echo "Warning: $GITROOT/$repo already exists" + else + # Create directory + mkdir "$GITROOT/$repo" + # Switch to directory + pushd "$GITROOT/$repo" > /dev/null + # Init bare repository + git --bare init --shared --initial-branch=master . > /dev/null + # Make it work + git update-server-info + # Allow push + git config http.receivepack true + # Fix ownership + chown -R apache:apache . + # Return in old dir + popd > /dev/null + fi +done diff --git a/jpack b/jpack new file mode 100755 index 0000000..79d7b58 --- /dev/null +++ b/jpack @@ -0,0 +1,90 @@ +#! /usr/bin/perl + +# Best practice +use strict; +use warnings; + +# Load POSIX +use POSIX qw(EXIT_SUCCESS EXIT_FAILURE); + +# Load CSS::Packer +use JavaScript::Packer; + +# Init compress +my $compress = 'best'; #clean, shrink, obfuscate or best + +#TODO: see how to handle theses options in a hash + +# Init copyright +my $copyright = ''; + +# Init remove_copyright +my $remove_copyright = 1; + +# Init no_compress_comment +my $no_compress_comment = 1; + +# Filter options +@ARGV = map { if ($_ eq '-c' || $_ eq '--clean') { $compress = 'clean'; (); } elsif ($_ eq '-s' || $_ eq '--shrink') { $compress = 'shrink'; (); } elsif ($_ eq '-o' || $_ eq '--obfuscate') { $compress = 'obfuscate'; (); } elsif ($_ eq '-b' || $_ eq '--best') { $compress = 'best'; (); } else { $_; } } @ARGV; + +# Show usage with invalid argument or stdin opened to a tty +if (scalar(@ARGV) || -t STDIN) { + print "Usage: $0 [-c|--clean|-s|--shrink|-o|--obfuscate|-b|--best] < input.js > output.js\n"; + exit EXIT_FAILURE; +} + +# Instantiate packer object +my $packer = JavaScript::Packer->init(); + +# Load input in variable +my $input = do { local $/; }; + +# Minify input with required compression +$packer->minify(\$input, { copyright => $copyright, remove_copyright => $remove_copyright, no_compress_comment => $no_compress_comment, compress => $compress }); + +# Show result +print $input; + +# Exit with success +exit EXIT_SUCCESS; + +__END__ + +=head1 NAME + +Jpack - A simple perl JavaScript minifier + +=head1 VERSION + +Version 0.1 + +=head1 DESCRIPTION + +A fast pure Perl JavaScript minifier script. + +=head1 AUTHOR + +Raphaël Gertz (Rapsys) << >>. + +=head1 COPYRIGHT & LICENSE + +Copyright 2016 - 2017 Raphaël Gertz, all rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +=head1 SEE ALSO + +L + +=cut diff --git a/vidpack b/vidpack new file mode 100755 index 0000000..ed0abf6 --- /dev/null +++ b/vidpack @@ -0,0 +1,200 @@ +#! /bin/sh + +# Brighten +#XXX: ffplay -vf "curves=all='0/0 0.25/0.375 0.5/0.75 0.75/0.875 1/1'" + +#Set document directory +docdir=/var/www/doc + +#Set video directory +viddir=/var/www/video + +#Set temporary directory +tmpdir=/tmp/vidpack + +#Without temporary directory +if [ ! -d "$tmpdir" ]; then + #Create temporary directory or die + mkdir -p "$tmpdir" || exit 1; +fi + +#Set time as utc +export TZ=utc + +#Change to document directory +pushd "$docdir" > /dev/null || exit 1 + +#Iterate on each mov or mp4 in doc dir older than 1h +for i in `find $docdir -maxdepth 1 -type f,l -name '*.mov' -o -name '*.mp4' -mmin +60`; do + #Set file date + fdate=$(date -u --rfc-3339=ns -r $i | perl -pne 's/\..*$/.000000Z/; s/ /T/'); + + #Without file date + if [ -z "$fdate" ]; then + #File date must exists + exit 1; + fi + + #Set name date + #XXX: may miss if filename don't match /^(.*[^0-9])?YYYYMMDD[^0-9]?HHMMSS([^0-9].*)?$/ + ndate=$(echo $i | perl -ne 'print if s/^(?:.*[^0-9])?([0-9]{4})([0-9]{2})([0-9]{2})(?:[^0-9])?([0-9]{2})([0-9]{2})([0-9]{2})(?:[^0-9].*)?$/\1-\2-\3T\4:\5:\6.000000Z/'); + + #Without name date + if [ -z "$ndate" ]; then + #Set name date + #XXX: catch when filename match only /^(.*[^0-9])?20YYMMDD([^0-9].*)?$/ + ndate=$(echo $i | perl -ne 'print if s/^(?:.*[^0-9])?(20[0-9]{2})([0-9]{2})([0-9]{2})(?:[^0-9].*)?$/\1-\2-\3T00:00:00.000000Z/'); + fi + + #Set create date + cdate=$(ffprobe -hide_banner -v quiet -show_entries format_tags=date,creation_time,com.apple.quicktime.creationdate -of default=nokey=1:noprint_wrappers=1 -i $i | perl -ne '$l = $_ unless /^$/; END { chomp $l; print $l }'); + + #Set best date + date=$(echo -e "$fdate\n$ndate\n$cdate" | perl -ne '$l = $_ unless /^(0000:00:00 00:00:00|)$/; END { chomp $l; print $l }'); + + #Set dest filename + dest="$viddir/$(echo $date | perl -pne 's%^([0-9]{4})-([0-9]{2})-([0-9]{2}).*$%\1/\2/\3%')/$(echo ${i#$docdir/} | perl -pne 's/\.(?:mov|mp4)$//')"; + + #Without dest directory + if [ ! -d "$(dirname $dest)" ]; then + #Create dest directory or die + mkdir -p "$(dirname $dest)" || exit 1; + fi + + #Set passlog filename + passlog="$tmpdir/$(echo $i | perl -pne 's%(?:.*/)([^/]+)\.(?:mov|mp4)$%\1%')"; + + #With passlog lock file + if [ -f "$passlog.lock" ]; then + #Display error + echo "Operation in progress: $passlog-0.log exists" 1>&2 + + #Exit with failure + exit 0; + fi + + #Set location + #XXX: see https://stackoverflow.com/questions/65231616/what-is-the-difference-between-location-and-location-eng-metadata-of-a-mp4-f + #XXX: drop location name after / + location=$(ffprobe -hide_banner -v quiet -show_entries format_tags=location,location-eng,com.apple.quicktime.location.ISO6709 -of default=nokey=1:noprint_wrappers=1 -i $i 2> /dev/null | perl -ne 's/\/.*$/\//g; $l = $_ unless /^$/; END { chomp $l; print $l }') + + #With location + if [ ! -z "$location" ]; then + #Replace with location arguments + location=" -metadata location=\"$location\""; + #Without location + else + #Prevent empty location + location=""; + fi + + #Set source rate + #TODO: use instead which works on webm: ffprobe -hide_banner -v quiet -select_streams v:0 -show_entries format=bit_rate -of default=nokey=1:noprint_wrappers=1 -i "$i" | perl -ne 'chomp; print' + sourcerate=$(ffprobe -hide_banner -v quiet -select_streams v:0 -show_entries stream=bit_rate -of default=nokey=1:noprint_wrappers=1 "$i" | perl -ne 'chomp; print'); + + #Set bitrate + bitrate=256k + + #With sourcerate >= 5120k (1080p) + if [ $sourcerate -ge 5120000 ]; then + #Set bitrate + bitrate=5120k + #With sourcerate >= 2560k (720p) + elif [ $sourcerate -ge 2560000 ]; then + #Set bitrate + bitrate=2560k + #With sourcerate >= 1024k + elif [ $sourcerate -ge 1024000 ]; then + #Set bitrate + bitrate=1024k + #With sourcerate >= 512k + elif [ $sourcerate -ge 512000 ]; then + #Set bitrate + bitrate=512k + fi + + #With dest.webm existing + if [ ! -f "$dest.webm" ]; then + #Create passlog lock file + touch "$passlog.lock" + + #With 1080p sourcerate + if [ $sourcerate -ge 5120000 ]; then + #Set bitrate + bitrate=5120k + + #First pass + ffmpeg -i $i -v error -passlogfile $passlog.1080p -c:v libvpx-vp9 -b:v $bitrate -pass 1 -an -f null /dev/null < /dev/null; + + #With first pass + if [ $? = 0 ]; then + #Second pass + ffmpeg -y -i $i -v error -passlogfile $passlog.1080p -c:v libvpx-vp9 -b:v $bitrate -pass 2 -pix_fmt yuv420p -c:a libopus -movflags +faststart -metadata creation_time="$date"$location -fflags +bitexact $dest.1080p.webm < /dev/null; + fi + + #With passlog file + if [ -f "$passlog.1080p-0.log" ]; then + #Remove passlog file + unlink "$passlog.1080p-0.log"; + fi + + #Set bitrate + bitrate=1024k + fi + + #With sourcerate >= 2560k (720p) + if [ $sourcerate -ge 2560000 ]; then + #Set bitrate + bitrate=2560k + + #First pass + ffmpeg -i $i -v error -passlogfile $passlog.720p -c:v libvpx-vp9 -b:v $bitrate -pass 1 -an -f null /dev/null < /dev/null; + + #With first pass + if [ $? = 0 ]; then + #Second pass + ffmpeg -y -i $i -v error -passlogfile $passlog.720p -c:v libvpx-vp9 -b:v $bitrate -pass 2 -pix_fmt yuv420p -c:a libopus -movflags +faststart -metadata creation_time="$date"$location -fflags +bitexact $dest.720p.webm < /dev/null; + fi + + #With passlog file + if [ -f "$passlog.720p-0.log" ]; then + #Remove passlog file + unlink "$passlog.720p-0.log"; + fi + + #Set bitrate + bitrate=1024k + fi + + #First pass + ffmpeg -i $i -v error -passlogfile $passlog -c:v libvpx-vp9 -b:v $bitrate -pass 1 -an -f null /dev/null < /dev/null; + + #With first pass + if [ $? = 0 ]; then + #Second pass + ffmpeg -y -i $i -v error -passlogfile $passlog -c:v libvpx-vp9 -b:v $bitrate -pass 2 -pix_fmt yuv420p -c:a libopus -movflags +faststart -metadata creation_time="$date"$location -fflags +bitexact $dest.webm < /dev/null; + fi + + #With passlog file + if [ -f "$passlog-0.log" ]; then + #Remove passlog file + unlink "$passlog-0.log"; + fi + + #With passlog lock file + if [ -f "$passlog.lock" ]; then + #Remove passlog lock file + unlink "$passlog.lock"; + fi + + fi +done + +#Revert to old dir +popd > /dev/null || exit 1; + +#Remove temporary directory +rmdir "$tmpdir" || true; + +#Exit with success +exit 0;