Ajout de apt-mirror et des patchs appliqués
This commit is contained in:
parent
1a7578ef26
commit
6bf601f8f0
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,237 @@
|
|||
--- /usr/bin/apt-mirror.old 2017-05-29 14:02:33.000000000 +0200
|
||||
+++ /usr/bin/apt-mirror 2020-04-11 02:09:10.913855091 +0200
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
-apt-mirror [configfile]
|
||||
+apt-mirror [--[no-]progress] [--verbose] [configfile]
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
@@ -16,14 +16,34 @@
|
||||
the whole Debian GNU/Linux distribution or any other apt sources.
|
||||
|
||||
Main features:
|
||||
- * It uses a config similar to APT's F<sources.list>
|
||||
- * It's fully pool compliant
|
||||
- * It supports multithreaded downloading
|
||||
- * It supports multiple architectures at the same time
|
||||
- * It can automatically remove unneeded files
|
||||
- * It works well on an overloaded Internet connection
|
||||
- * It never produces an inconsistent mirror including while mirroring
|
||||
- * It works on all POSIX compliant systems with Perl and wget
|
||||
+
|
||||
+=over
|
||||
+
|
||||
+=item *
|
||||
+It uses a config similar to APT's F<sources.list>
|
||||
+
|
||||
+=item *
|
||||
+It's fully pool compliant
|
||||
+
|
||||
+=item *
|
||||
+It supports multithreaded downloading
|
||||
+
|
||||
+=item *
|
||||
+It supports multiple architectures at the same time
|
||||
+
|
||||
+=item *
|
||||
+It can automatically remove unneeded files
|
||||
+
|
||||
+=item *
|
||||
+It works well on an overloaded Internet connection
|
||||
+
|
||||
+=item *
|
||||
+It never produces an inconsistent mirror including while mirroring
|
||||
+
|
||||
+=item *
|
||||
+It works on all POSIX compliant systems with Perl and wget
|
||||
+
|
||||
+=back
|
||||
|
||||
=head1 COMMENTS
|
||||
|
||||
@@ -94,6 +114,7 @@
|
||||
use File::Path qw(make_path);
|
||||
use File::Basename;
|
||||
use Fcntl qw(:flock);
|
||||
+use Getopt::Long;
|
||||
|
||||
my $config_file;
|
||||
|
||||
@@ -128,11 +149,17 @@
|
||||
my @childrens = ();
|
||||
my %skipclean = ();
|
||||
my %clean_directory = ();
|
||||
+my $verbose = 0;
|
||||
+my $progress = 1;
|
||||
|
||||
######################################################################################
|
||||
## Setting up $config_file variable
|
||||
|
||||
$config_file = "/etc/apt/mirror.list"; # Default value
|
||||
+GetOptions('verbose|v+', \$verbose,
|
||||
+ 'progress|p!', \$progress,
|
||||
+ ) or die "Usage: apt-mirror [--verbose] [--[no-]progress] [configfile]\n";
|
||||
+
|
||||
if ( $_ = shift )
|
||||
{
|
||||
die("apt-mirror: invalid config file specified") unless -e $_;
|
||||
@@ -256,7 +283,9 @@
|
||||
open URLS, ">" . get_variable("var_path") . "/$stage-urls.$i" or die("apt-mirror: can't write to intermediate file ($stage-urls.$i)");
|
||||
foreach (@part) { print URLS "$_\n"; }
|
||||
close URLS or die("apt-mirror: can't close intermediate file ($stage-urls.$i)");
|
||||
-
|
||||
+ if ($verbose >= 2) {
|
||||
+ print join("\n ", "Downloading batch $i:", @part), "\n";
|
||||
+ }
|
||||
$pid = fork();
|
||||
|
||||
die("apt-mirror: can't do fork in download_urls") if !defined($pid);
|
||||
@@ -274,14 +303,17 @@
|
||||
$nthreads--;
|
||||
}
|
||||
|
||||
- print "Begin time: " . localtime() . "\n[" . scalar(@childrens) . "]... ";
|
||||
+ print "Begin time: " . localtime() . "\n[" . scalar(@childrens) . "]... "
|
||||
+ if $progress;
|
||||
while ( scalar @childrens )
|
||||
{
|
||||
my $dead = wait();
|
||||
@childrens = grep { $_ != $dead } @childrens;
|
||||
- print "[" . scalar(@childrens) . "]... ";
|
||||
+ print "[" . scalar(@childrens) . "]... "
|
||||
+ if $progress;
|
||||
}
|
||||
- print "\nEnd time: " . localtime() . "\n\n";
|
||||
+ print "\nEnd time: " . localtime() . "\n\n"
|
||||
+ if $progress;
|
||||
}
|
||||
|
||||
## Parse config
|
||||
@@ -600,12 +632,13 @@
|
||||
close STREAM;
|
||||
}
|
||||
|
||||
-print "Processing translation indexes: [";
|
||||
+print "Processing translation indexes: ["
|
||||
+ if $progress;
|
||||
|
||||
foreach (@config_binaries)
|
||||
{
|
||||
my ( $arch, $uri, $distribution, @components ) = @{$_};
|
||||
- print "T";
|
||||
+ print "T" if $progress;
|
||||
if (@components)
|
||||
{
|
||||
$url = $uri . "/dists/" . $distribution . "/";
|
||||
@@ -618,7 +651,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
-print "]\n\n";
|
||||
+print "]\n\n" if $progress;
|
||||
|
||||
push( @index_urls, sort keys %urls_to_download );
|
||||
download_urls( "translation", sort keys %urls_to_download );
|
||||
@@ -691,12 +724,13 @@
|
||||
}
|
||||
}
|
||||
|
||||
-print "Processing DEP-11 indexes: [";
|
||||
+print "Processing DEP-11 indexes: ["
|
||||
+ if $progress;
|
||||
|
||||
foreach (@config_binaries)
|
||||
{
|
||||
my ( $arch, $uri, $distribution, @components ) = @{$_};
|
||||
- print "D";
|
||||
+ print "D" if $progress;
|
||||
if (@components)
|
||||
{
|
||||
$url = $uri . "/dists/" . $distribution . "/";
|
||||
@@ -709,7 +743,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
-print "]\n\n";
|
||||
+print "]\n\n" if $progress;
|
||||
|
||||
push( @index_urls, sort keys %urls_to_download );
|
||||
download_urls( "dep11", sort keys %urls_to_download );
|
||||
@@ -815,12 +849,16 @@
|
||||
{ # Packages index
|
||||
$skipclean{ remove_double_slashes( $path . "/" . $lines{"Filename:"} ) } = 1;
|
||||
print FILES_ALL remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n";
|
||||
+ print "ALL: " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n"
|
||||
+ if $verbose >= 3;
|
||||
print FILES_MD5 $lines{"MD5sum:"} . " " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n" if defined $lines{"MD5sum:"};
|
||||
print FILES_SHA1 $lines{"SHA1:"} . " " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n" if defined $lines{"SHA1:"};
|
||||
print FILES_SHA256 $lines{"SHA256:"} . " " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n" if defined $lines{"SHA256:"};
|
||||
if ( need_update( $mirror . "/" . $lines{"Filename:"}, $lines{"Size:"} ) )
|
||||
{
|
||||
print FILES_NEW remove_double_slashes( $uri . "/" . $lines{"Filename:"} ) . "\n";
|
||||
+ print "NEW: " . remove_double_slashes( $uri . "/" . $lines{"Filename:"} ) . "\n"
|
||||
+ if $verbose >= 1;
|
||||
add_url_to_download( $uri . "/" . $lines{"Filename:"}, $lines{"Size:"} );
|
||||
}
|
||||
}
|
||||
@@ -833,10 +871,14 @@
|
||||
die("apt-mirror: invalid Sources format") if @file != 3;
|
||||
$skipclean{ remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) } = 1;
|
||||
print FILES_ALL remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n";
|
||||
+ print "ALL: " . remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n"
|
||||
+ if $verbose >= 3;
|
||||
print FILES_MD5 $file[0] . " " . remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n";
|
||||
if ( need_update( $mirror . "/" . $lines{"Directory:"} . "/" . $file[2], $file[1] ) )
|
||||
{
|
||||
print FILES_NEW remove_double_slashes( $uri . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n";
|
||||
+ print "NEW: " . remove_double_slashes( $uri . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n"
|
||||
+ if $verbose >= 1;
|
||||
add_url_to_download( $uri . "/" . $lines{"Directory:"} . "/" . $file[2], $file[1] );
|
||||
}
|
||||
}
|
||||
@@ -846,12 +888,13 @@
|
||||
close STREAM;
|
||||
}
|
||||
|
||||
-print "Processing indexes: [";
|
||||
+print "Processing indexes: ["
|
||||
+ if $progress;
|
||||
|
||||
foreach (@config_sources)
|
||||
{
|
||||
my ( $uri, $distribution, @components ) = @{$_};
|
||||
- print "S";
|
||||
+ print "S" if $progress;
|
||||
if (@components)
|
||||
{
|
||||
my $component;
|
||||
@@ -869,7 +912,7 @@
|
||||
foreach (@config_binaries)
|
||||
{
|
||||
my ( $arch, $uri, $distribution, @components ) = @{$_};
|
||||
- print "P";
|
||||
+ print "P" if $progress;
|
||||
if (@components)
|
||||
{
|
||||
my $component;
|
||||
@@ -886,7 +929,8 @@
|
||||
|
||||
clear_stat_cache();
|
||||
|
||||
-print "]\n\n";
|
||||
+print "]\n\n"
|
||||
+ if $progress;
|
||||
|
||||
close FILES_ALL;
|
||||
close FILES_NEW;
|
||||
@@ -1019,6 +1063,7 @@
|
||||
foreach (@rm_files)
|
||||
{
|
||||
print CLEAN "rm -f '$_'\n";
|
||||
+ print " $_\n" if $verbose >= 1;
|
||||
print CLEAN "echo -n '[" . int( 100 * $i / $total ) . "\%]'\n" unless $i % 500;
|
||||
print CLEAN "echo -n .\n" unless $i % 10;
|
||||
$i++;
|
|
@ -0,0 +1,21 @@
|
|||
--- /usr/bin/apt-mirror.old 2017-05-29 14:02:33.000000000 +0200
|
||||
+++ /usr/bin/apt-mirror 2020-04-11 01:42:20.959628172 +0200
|
||||
@@ -520,7 +520,7 @@
|
||||
if ( @parts == 3 )
|
||||
{
|
||||
my ( $sha1, $size, $filename ) = @parts;
|
||||
- if ( $filename =~ m{^$component/i18n/Translation-[^./]*\.bz2$} )
|
||||
+ if ( $filename =~ m{^$component/i18n/Translation-[^./]*\.(gz|bz2|xz)$} )
|
||||
{
|
||||
add_url_to_download( $dist_uri . $filename, $size );
|
||||
}
|
||||
@@ -826,7 +826,8 @@
|
||||
}
|
||||
else
|
||||
{ # Sources index
|
||||
- foreach ( split( /\n/, $lines{"Files:"} ) )
|
||||
+ # foreach ( split( /\n/, $lines{"Files:"} ) )
|
||||
+ foreach ( split( /\n/, $lines{"Checksums-Sha256:"} ) )
|
||||
{
|
||||
next if $_ eq '';
|
||||
my @file = split;
|
|
@ -0,0 +1,102 @@
|
|||
--- /usr/bin/apt-mirror.old 2017-05-29 14:02:33.000000000 +0200
|
||||
+++ /usr/bin/apt-mirror 2020-04-11 02:08:01.820901199 +0200
|
||||
@@ -722,6 +722,99 @@
|
||||
}
|
||||
|
||||
######################################################################################
|
||||
+## by-hash SHA256 files download
|
||||
+
|
||||
+%urls_to_download = ();
|
||||
+
|
||||
+sub find_by_hash_sha256_files_in_release
|
||||
+{
|
||||
+ # Look in the dists/$DIST/Release file for the by-hash SHA256 files that belong
|
||||
+ # to the given component and architecture.
|
||||
+
|
||||
+ my $dist_uri = shift;
|
||||
+ my $component = shift;
|
||||
+ my $arch = shift;
|
||||
+ my ( $release_uri, $release_path, $line ) = '';
|
||||
+
|
||||
+ $release_uri = $dist_uri . "Release";
|
||||
+ $release_path = get_variable("skel_path") . "/" . sanitise_uri($release_uri);
|
||||
+
|
||||
+ unless ( open STREAM, "<$release_path" )
|
||||
+ {
|
||||
+ warn( "Failed to open Release file from " . $release_uri );
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ my $checksums = 0;
|
||||
+ while ( $line = <STREAM> )
|
||||
+ {
|
||||
+ chomp $line;
|
||||
+ if ($checksums)
|
||||
+ {
|
||||
+ if ( $line =~ /^ +(.*)$/ )
|
||||
+ {
|
||||
+ my @parts = split( / +/, $1 );
|
||||
+ if ( @parts == 3 )
|
||||
+ {
|
||||
+ my ( $sha256, $size, $filename ) = @parts;
|
||||
+ my $dirname = dirname($filename);
|
||||
+ my $sha256_filename = '/'.$dirname.'/by-hash/SHA256/'.$sha256;
|
||||
+ {
|
||||
+ add_url_to_download( $dist_uri . $sha256_filename );
|
||||
+ }
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ warn("Malformed checksum line \"$1\" in $release_uri");
|
||||
+ }
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ $checksums = 0;
|
||||
+ }
|
||||
+ }
|
||||
+ if ( not $checksums )
|
||||
+ {
|
||||
+ if ( $line eq "SHA256:" )
|
||||
+ {
|
||||
+ $checksums = 1;
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+print "Processing SHA256 by-hash files ["
|
||||
+ if $progress;
|
||||
+
|
||||
+foreach (@config_binaries)
|
||||
+{
|
||||
+ my ( $arch, $uri, $distribution, @components ) = @{$_};
|
||||
+ print "D" if $progress;
|
||||
+ if (@components)
|
||||
+ {
|
||||
+ $url = $uri . "/dists/" . $distribution . "/";
|
||||
+
|
||||
+ my $component;
|
||||
+ foreach $component (@components)
|
||||
+ {
|
||||
+ find_by_hash_sha256_files_in_release( $url, $component, $arch );
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+print "]\n\n" if $progress;
|
||||
+
|
||||
+push( @index_urls, sort keys %urls_to_download );
|
||||
+download_urls( "by-hash-SHA256", sort keys %urls_to_download );
|
||||
+
|
||||
+foreach ( keys %urls_to_download )
|
||||
+{
|
||||
+ s[^(\w+)://][];
|
||||
+ s[~][%7E]g if get_variable("_tilde");
|
||||
+ $skipclean{$_} = 1;
|
||||
+}
|
||||
+
|
||||
+######################################################################################
|
||||
## Main download preparations
|
||||
|
||||
%urls_to_download = ();
|
Loading…
Reference in New Issue