You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
484 lines
19 KiB
484 lines
19 KiB
#!/usr/bin/perl
|
|
|
|
# Copyright 2011 Chris Nighswonger
|
|
#
|
|
# This is free software; you can redistribute it and/or modify it under the
|
|
# terms of the GNU General Public License as published by the Free Software
|
|
# Foundation; either version 2 of the License, or (at your option) any later
|
|
# version.
|
|
#
|
|
# This is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
|
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along with
|
|
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
|
|
# Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
use Modern::Perl;
|
|
|
|
use Pod::Usage;
|
|
use POSIX qw(strftime);
|
|
use LWP::Simple;
|
|
use Text::CSV;
|
|
use Encode qw/encode/;
|
|
use HTML::TableExtract;
|
|
use Getopt::Long;
|
|
use Template;
|
|
use File::Basename;
|
|
use File::Spec;
|
|
use lib('.');
|
|
use encoding('utf8');
|
|
use Text::CSV; # used to store bugzilla full descriptions
|
|
|
|
# TODO:
|
|
# 1. Paramatize!
|
|
# 2. Add optional verbose output
|
|
# 3. Add exit status code
|
|
|
|
# git log --pretty=format:'%s' v3.04.05..HEAD | grep -Eo '([B|b]ug|BZ)?(\s|:|-|_|,)?\s?[0-9]{4}\s' | grep -Eo '[0-9]{4}' - | sort -u
|
|
|
|
#release_notes_3_6_0.txt
|
|
|
|
|
|
sub trim {
|
|
my ($s) = @_;
|
|
$s =~ s/^\s*(.*?)\s*$/$1/s;
|
|
return $s;
|
|
}
|
|
|
|
# try to retrieve the current version number from kohaversion.pl
|
|
my $version = undef;
|
|
eval {
|
|
require 'kohaversion.pl';
|
|
$version = kohaversion();
|
|
};
|
|
|
|
my $tag = undef;
|
|
my $HEAD = "HEAD";
|
|
my $template = undef;
|
|
my $html_template = undef;
|
|
my $rnotes_txt = undef;
|
|
my $rnotes_html = undef;
|
|
my $help = 0;
|
|
my $verbose = 0;
|
|
my $html = 0; # if set, will create a HTML version of the release notes
|
|
my $login = '';
|
|
my $password = '';
|
|
my $commit_changes = 0;
|
|
|
|
GetOptions(
|
|
't|tag:s' => \$tag,
|
|
'head:s' => \$HEAD,
|
|
'template:s' => \$template,
|
|
'r|rnotes:s' => \$rnotes_txt,
|
|
'v|version:s' => \$version,
|
|
'c|commit' => \$commit_changes,
|
|
'html' => \$html,
|
|
'help|h' => \$help,
|
|
'verbose' => \$verbose,
|
|
'u' => \$login,
|
|
'p' => \$password,
|
|
);
|
|
|
|
if ($help) {
|
|
pod2usage( -verbose => 2 );
|
|
exit;
|
|
}
|
|
|
|
print "Creating release notes for version $version\n\n";
|
|
# variations on a theme of version numbers...
|
|
|
|
my $reltools = File::Spec->rel2abs( dirname(__FILE__) );
|
|
my $tt;
|
|
$tt = Template->new(
|
|
{
|
|
INCLUDE_PATH => File::Spec->rel2abs( dirname(__FILE__) ),
|
|
ENCODING => 'utf8',
|
|
}
|
|
) || die $tt->error(), "\n";
|
|
my %arguments;
|
|
|
|
die "No usable version" unless $version =~ m/(\d)\.(\d\d)\.(\d\d)(\.\d+)?(-\w*)?/g;
|
|
my $major = $1;
|
|
my $minor = $2;
|
|
my $release = $3;
|
|
my $expanded_minor = $2;
|
|
my $expanded_release = $3;
|
|
my $additional = $5;
|
|
$minor =~ s/^0*(\d+)$/$1/;
|
|
$release =~ s/^0*(\d+)$/$1/;
|
|
my $shortversion = "$major.$minor.$release";
|
|
my $minorversion = "$major.$minor";
|
|
$arguments{minorversion} = $minorversion;
|
|
$arguments{shortversion} = $shortversion;
|
|
$arguments{shortversion} .= "$additional" if ($additional);
|
|
$arguments{expandedversion} = "$major.$expanded_minor.$expanded_release";
|
|
$arguments{expandedversion} .= "$additional" if ($additional);
|
|
$arguments{line} = "$major." . ($minor % 2 ? $minor + 1 : $minor);
|
|
$arguments{MAJOR} = $release?0:1; # major release if the last number is 0
|
|
|
|
# description is a hash used to store bugzilla descriptions
|
|
# bugzilla descriptions are slow to retrieve from bugzilla, and can't be updated
|
|
# so, retrieve them once, and store them for re-use if needed
|
|
# we store them in a CSV file so, they can be modified manually if needed, for more clarity
|
|
# and we often need more clarity or clean some technical informations
|
|
my %descriptions;
|
|
if (-e "$reltools/descriptions/descriptions-$shortversion.csv") {
|
|
my $csv = Text::CSV->new ( { sep_char => '|', binary => 1 } ) # should set binary attribute.
|
|
or die "Cannot use CSV: ".Text::CSV->error_diag ();
|
|
|
|
open my $fh, "<:encoding(utf8)", "$reltools/descriptions/descriptions-$shortversion.csv" or die "$reltools/descriptions-$shortversion.csv: $!";
|
|
while ( my $row = $csv->getline( $fh ) ) {
|
|
# uncomment the next line if you've problem of CSV reading (like "" in descriptions)
|
|
# you'll see the last valid line read
|
|
# print "read : $row->[0]\n";
|
|
$descriptions{$row->[0]} = $row->[2];
|
|
}
|
|
$csv->eof or $csv->error_diag();
|
|
close $fh;
|
|
}
|
|
|
|
$template = "templates/release_notes_tmpl.tt" unless $template;
|
|
$html_template = "templates/release_notes_tmpl_html.tt";
|
|
$rnotes_txt = "misc/release_notes/release_notes_${major}_${minor}_${release}.txt" unless $rnotes_txt;
|
|
$rnotes_html = "misc/release_notes/release_notes_${major}_${minor}_${release}.html" unless $rnotes_html;
|
|
|
|
my $pootle = "http://translate.koha-community.org/projects/$major.$minor/";
|
|
$pootle = "http://translate.koha-community.org/" unless defined(get($pootle));
|
|
|
|
my $translationpage = get($pootle);
|
|
my @translations = ( {language => 'English (USA)'} );
|
|
|
|
my $translations_parser = HTML::TableExtract->new(
|
|
headers => [ "Name", "Progress", "Total", "Need Translation", "Last Activity" ]
|
|
);
|
|
$translations_parser->parse(encode('UTF-8',$translationpage));
|
|
|
|
foreach my $language ($translations_parser->rows) {
|
|
next if !defined $language;
|
|
my $name = trim( @$language[0] );
|
|
my $progress = trim( @$language[1] );
|
|
push @translations, { language => "$name ($progress%)"} if ($progress > 50);
|
|
}
|
|
|
|
$arguments{translations} = \@translations;
|
|
|
|
$arguments{releaseteam} = "templates/release_team_$arguments{line}".($html?'_html':'').".tt";
|
|
$arguments{new_devs} = "templates/new_devs_$arguments{line}".($html?'_html':'').".tt";
|
|
|
|
print "Using template: $template and release notes file: $rnotes_txt\n";
|
|
print "Using template: $html_template and release notes file: $rnotes_html\n" if defined $html;
|
|
print "\n";
|
|
|
|
my $git_add = 1 unless -e "misc/release_notes/$rnotes_txt";
|
|
|
|
$tag = `git describe --abbrev=0` unless $tag;
|
|
chomp $tag;
|
|
|
|
my @bug_list = ();
|
|
my @git_log = qx|git log --pretty=format:'%s' $tag..$HEAD|;
|
|
|
|
$arguments{branch} = `git branch | grep '*' | sed -e 's/^* //' -e 's#/#-#'`;
|
|
chomp $arguments{branch};
|
|
$arguments{downloadlink} = "http://download.koha-community.org/koha-$arguments{expandedversion}.tar.gz";
|
|
|
|
foreach (@git_log) {
|
|
if ($_ =~ m/([B|b]ug|BZ)?\s?(?<![a-z]|\.)(\d{3,5})[\s|:|,]/g) {
|
|
# print "$&\n"; # Uncomment this line and the die below to view exact matches
|
|
push @bug_list, $2;
|
|
}
|
|
}
|
|
#die "Done for now...\n"; #XXX
|
|
#@bug_list = sort {$a <=> $b} @bug_list;
|
|
my %seen = ();
|
|
@bug_list = grep{!$seen{$_}++} (sort {$a <=> $b} @bug_list);
|
|
|
|
print "Found " . scalar @bug_list . " bugs in this search\n\n" if $verbose;
|
|
|
|
# http://bugs.koha-community.org/bugzilla3/buglist.cgi?bug_id=2629%2C2847%2C3958%2C4161%2C5150%2C5885%2C5945%2C5974%2C6390%2C6471%2C6475%2C6628%2C6629%2C6679%2C6799%2C6895%2C6955%2C6963%2C6977%2C6989%2C6994%2C7061%2C7069%2C7076%2C7084%2C7085%2C7095%2C7117%2C7128%2C7134%2C7138%2C7146%2C7184%2C7185%2C7188%2C7207%2C7221&bug_id_type=anyexact&query_format=advanced&ctype=csv
|
|
|
|
if (scalar @bug_list) {
|
|
my $url = "http://bugs.koha-community.org/bugzilla3/buglist.cgi?order=component%2Cbug_severity%2Cbug_id&bug_id=";
|
|
$url .= join '%2C', @bug_list;
|
|
$url .= "&bug_id_type=anyexact&query_format=advanced&ctype=csv&columnlist=bug_severity%2Cshort_desc%2Ccomponent";
|
|
|
|
print "URL: $url\n" if $verbose;
|
|
|
|
my @csv_file = split /\n/, get($url);
|
|
my $csv = Text::CSV->new();
|
|
|
|
# Extract the column names
|
|
$csv->parse(shift @csv_file);
|
|
my @columns = $csv->fields;
|
|
|
|
# the current component for the 3 cases (highlights, bugfixes, enhancements)
|
|
my ($current_highlight,$current_bugfix,$current_enhancement,$current_newfeature) = ('','','','');
|
|
#The lists of highlights, bugfixes and enhancement
|
|
# the component_xxx contains an array of hash reset for each component
|
|
# the xxx contains an array of hash with component & component_xxx array of hash
|
|
my (@component_highlights,@highlights);
|
|
my (@component_bugfixes,@bugfixes);
|
|
my (@component_enhancements,@enhancements);
|
|
my (@component_newfeatures,@newfeatures);
|
|
my $nb_enhancements = 0;
|
|
my $nb_newfeatures = 0;
|
|
my $nb_bugfixes = 0;
|
|
while (scalar @csv_file) {
|
|
$csv->parse(shift @csv_file);
|
|
my @fields = $csv->fields;
|
|
$fields[2] = ucfirst( $fields[2] )
|
|
unless ( $fields[2] =~ m/^koha-/ # it's a koha-* script
|
|
or $fields[2] =~ m/^t\// ); # it's about tests in t/
|
|
if ($fields[1] =~ m/(blocker|critical|major)/) {
|
|
if ($current_highlight && $fields[3] ne $current_highlight) {
|
|
my @t=@component_highlights;
|
|
push @highlights, { component => $current_highlight, list => \@t };
|
|
@component_highlights=();
|
|
}
|
|
$current_highlight=$fields[3];
|
|
push @component_highlights, { number=> $fields[0],severity=> $fields[1], short_desc=> $fields[2] };
|
|
$nb_bugfixes++;
|
|
}
|
|
elsif ($fields[1] =~ m/(normal|minor|trivial)/) {
|
|
if ($current_bugfix && $fields[3] ne $current_bugfix) {
|
|
my @t=@component_bugfixes;
|
|
push @bugfixes, { component => $current_bugfix, list => \@t };
|
|
@component_bugfixes=();
|
|
}
|
|
$current_bugfix=$fields[3];
|
|
push @component_bugfixes, { number=> $fields[0],severity=> $fields[1], short_desc=> $fields[2] };
|
|
$nb_bugfixes++;
|
|
} else { # enhancements & new feature
|
|
#
|
|
# if bugzilla login and password have been provided, retrieve the description of the bug
|
|
#
|
|
my $description;
|
|
if ($login && $password) {
|
|
if ( $descriptions{$fields[0]} ) {
|
|
# print "stored $fields[0]\n";
|
|
$description = $descriptions{$fields[0]};
|
|
} else {
|
|
# print "retrieving $fields[0]\n";
|
|
my $bugdetail = `bugz -u $login -p $password -b http://bugs.koha-community.org/bugzilla3/ get $fields[0]`;
|
|
$bugdetail =~ /\[Comment \#0\].*?\n-------------------------------------------------------------------------------\n(.*)\[Comment \#1\]/s;
|
|
$description = $1;
|
|
#
|
|
# append this to the storable description
|
|
# no one can change bug description, so once we've got it, remember it !
|
|
$descriptions{$fields[0]} = $description;
|
|
# OK, save the file with the new bug found
|
|
# FIXME not very efficient to save on each bug added
|
|
my $csv = Text::CSV->new ();
|
|
my $fh;
|
|
open $fh, ">:encoding(utf8)", "$reltools/descriptions/descriptions-$shortversion.csv" or die "$reltools/descriptions/descriptions-$shortversion.csv: $!";
|
|
print $fh "number|shortdesc|fulldesc\n";
|
|
|
|
foreach my $desc (keys %descriptions) {
|
|
$descriptions{$desc} =~ s/\|/ /g;
|
|
$descriptions{$desc} =~ s/"/ /g;
|
|
print $fh $desc."||\"".$descriptions{$desc}."\"\n";
|
|
}
|
|
close $fh or die "$reltools/descriptions/descriptions-$shortversion.csv: $!";
|
|
|
|
}
|
|
|
|
if ($html) {
|
|
# do some basic formatting if we are in html mode, if the description is multilined
|
|
$description =~ s/^ / /mg;
|
|
$description =~ s/^ / /mg;
|
|
$description =~ s/([a-zA-Z0-9 ,])\n/$1 /mg;
|
|
$description =~ s/</</g;
|
|
$description =~ s/>/>/g;
|
|
$description =~ s/\n/<br\/>/g;
|
|
}
|
|
}
|
|
if ($fields[1] =~ m/enhancement/) {
|
|
if ($current_enhancement && $fields[3] ne $current_enhancement) {
|
|
my @t=@component_enhancements;
|
|
push @enhancements, { component => $current_enhancement, list => \@t };
|
|
@component_enhancements=();
|
|
}
|
|
$current_enhancement=$fields[3];
|
|
push @component_enhancements, { number=> $fields[0],severity=> $fields[1], short_desc=> $fields[2], description => $description };
|
|
$nb_enhancements++;
|
|
} else { # new feature
|
|
if ($current_newfeature && $fields[3] ne $current_newfeature) {
|
|
my @t=@component_newfeatures;
|
|
push @newfeatures, { component => $current_newfeature, list => \@t };
|
|
@component_newfeatures=();
|
|
}
|
|
$current_newfeature=$fields[3];
|
|
push @component_newfeatures, { number=> $fields[0],severity=> $fields[1], short_desc=> $fields[2], description => $description };
|
|
$nb_newfeatures++;
|
|
|
|
}
|
|
}
|
|
}
|
|
# push the last components
|
|
push @highlights, { component => $current_highlight, list => \@component_highlights };
|
|
push @bugfixes, { component => $current_bugfix, list => \@component_bugfixes };
|
|
push @enhancements, { component => $current_enhancement, list => \@component_enhancements };
|
|
push @newfeatures, { component => $current_newfeature, list => \@component_newfeatures };
|
|
|
|
$arguments{highlights} = \@highlights;
|
|
$arguments{bugfixes} = \@bugfixes;
|
|
$arguments{enhancements} = \@enhancements;
|
|
$arguments{newfeatures} = \@newfeatures;
|
|
$arguments{nb_bugfixes} = $nb_bugfixes;
|
|
$arguments{nb_enhancements} = $nb_enhancements;
|
|
$arguments{nb_newfeatures} = $nb_newfeatures;
|
|
}
|
|
|
|
my $sysprefs_path = 'installer/data/mysql/mandatory/sysprefs.sql';
|
|
my $old_sysprefs_path = 'installer/data/mysql/sysprefs.sql'; # Before 23895
|
|
my @prev_pref_script = qx{git show $tag:$sysprefs_path 2> /dev/null};
|
|
@prev_pref_script = qx{git show $tag:$old_sysprefs_path} unless @prev_pref_script;
|
|
|
|
my @current_pref_script = `git show HEAD:$sysprefs_path 2> /dev/null`;
|
|
@current_pref_script = `git show HEAD:$old_sysprefs_path` unless @current_pref_script;
|
|
|
|
my %prev_sysprefs =
|
|
map { lc($_) => $_ }
|
|
map { /\(\s*'([^']+)'/; $1 }
|
|
grep { /\(\s*'/ }
|
|
@prev_pref_script;
|
|
|
|
my %current_sysprefs =
|
|
map { lc($_) => $_ }
|
|
map { /\(\s*'([^']+)'/; $1 }
|
|
grep { /\(\s*'/ }
|
|
@current_pref_script;
|
|
|
|
my @sysprefs;
|
|
foreach my $key (sort keys %current_sysprefs) {
|
|
push @sysprefs, { name => $current_sysprefs{$key} }
|
|
unless exists $prev_sysprefs{$key} or
|
|
$key eq 'independentbranches' # special case for renamed pref
|
|
}
|
|
$arguments{sysprefs} = \@sysprefs;
|
|
|
|
# Now we'll alphabetize the contributors based on surname (or at least the last word on their line)
|
|
my @contribs;
|
|
my @contributor_list;
|
|
open my $contribs_lines, '-|', "git log --pretty=short $tag..$HEAD | git shortlog -s | sort -k3 -";
|
|
while ( my $line = <$contribs_lines> ) {
|
|
my ( $commits, $name ) = $line =~ /\s*(\d*)\s*(.*)\s*$/;
|
|
push @contributor_list, { name => $name, commits => $commits } ;
|
|
}
|
|
|
|
my @signers;
|
|
my @signer_list;
|
|
open my $sign_lines, '-|', "git log $tag..$HEAD | grep 'Signed-off-by' | sed -e 's/^.*Signed-off-by: //' | sed -e 's/ <.*\$//' | sort -k3 - | uniq -c";
|
|
while ( my $line = <$sign_lines> ) {
|
|
my ( $signoffs, $name ) = $line =~ /\s*(\d*)\s*(.*)\s*$/;
|
|
push @signer_list, { name => $name, signoffs => $signoffs } ;
|
|
}
|
|
|
|
my @sponsor_list = map { {name => $_} }
|
|
qx(git log $tag..$HEAD | grep 'Sponsored-by' | sed -e 's/^.*Sponsored-by: //' | sort | uniq);
|
|
|
|
# contributing companies, with their number of commits, by alphabetical order
|
|
# companies are retrieved from the email address.
|
|
# generic emails like hotmail.com, gmail.com are cumulated in a "unitentified" contributor
|
|
my %domain_map;
|
|
|
|
open (my $domainmapfh, File::Spec->rel2abs( dirname(__FILE__) . '/gitdm/domain-map' ));
|
|
while (<$domainmapfh>) {
|
|
chomp $_;
|
|
$_ =~ m/^([^# ]*)\s*(.*)$/;
|
|
$domain_map{$1} = $2;
|
|
}
|
|
close ($domainmapfh);
|
|
|
|
|
|
my %companies_list;
|
|
foreach (map { {name => $_->[1]} }
|
|
sort { $a->[0] cmp $b->[0] }
|
|
map { [(split /\s+/, $_)[scalar(@contribs = split /\s+/, $_)-1], $_] }
|
|
qx(git log --pretty=short $tag..$HEAD | git shortlog -s -e | sort -k3 -) ) {
|
|
$_->{name} =~ /(\d+).*@(.*)>/;
|
|
my ($nbpatch,$company) = ($1,$2);
|
|
if ($company =~ /o2\.pl|gmail\.com|hotmail\.com|\(none\)/) {
|
|
$companies_list{unidentified} += $nbpatch;
|
|
} else {
|
|
if ($domain_map{$company}) {
|
|
$company = $domain_map{$company};
|
|
}
|
|
$companies_list{$company} += $nbpatch;
|
|
}
|
|
}
|
|
my @companies_list;
|
|
foreach (sort {$a cmp $b} keys %companies_list) {
|
|
push @companies_list, { name => $_, commits => $companies_list{$_} };
|
|
}
|
|
|
|
$arguments{contributors} = \@contributor_list;
|
|
$arguments{signers} = \@signer_list;
|
|
$arguments{sponsors} = \@sponsor_list;
|
|
$arguments{companies} = \@companies_list;
|
|
$arguments{date} = strftime "%d %b %Y", gmtime;
|
|
|
|
# Add autogenerated blurb to the bottom
|
|
my $time_stamp = strftime("%d %b %Y %T", gmtime);
|
|
$arguments{timestamp} = "##### Autogenerated release notes updated last on $time_stamp Z #####";
|
|
|
|
eval {
|
|
$tt->process($template, \%arguments, $rnotes_txt, {binmode => ":utf8"});
|
|
};
|
|
|
|
if ($@) {
|
|
die $tt->error(), "\n";
|
|
}
|
|
|
|
eval {
|
|
$tt->process($html_template, \%arguments, $rnotes_html, {binmode => ":utf8"});
|
|
} if defined $html;
|
|
|
|
if ($@) {
|
|
warn $tt->error();
|
|
}
|
|
|
|
if ($commit_changes) {
|
|
if ($git_add) {
|
|
print "Adding file to repo...\n";
|
|
my @add_results = qx|git add misc/release_notes/$rnotes_txt|;
|
|
}
|
|
print "Commiting changes...\n";
|
|
my @commit_results = qx|git commit -m "Release Notes for $version $time_stamp Z" misc/release_notes/$rnotes_txt|;
|
|
}
|
|
|
|
exit 0;
|
|
=head1 NAME
|
|
|
|
get_bugs.pl - Generate release notes
|
|
|
|
=head1 USAGE
|
|
|
|
=over
|
|
|
|
=item get_bugs.pl [-t] [-h] [--template:template] [-r:notes] [-v:version] [-c] [--html][--help][--verbose][-u bugzilla_login][-p bugzilla password]
|
|
|
|
This script will generate releases notes from a template file (that can be specified), by retrieving all bugs in git since a given tag
|
|
The script retrieve the patch description from bugzilla, and, if login/password provided, the comment 0 (detailled description)
|
|
It also generate contributors & signers & sponsor list (using git informations)
|
|
|
|
|
|
=back
|
|
|
|
=head1 PARAMETERS
|
|
|
|
t|tag specify where the release start from. If not specified, it's the last stable .0 release
|
|
head NEED DOC,
|
|
template The template file to use to generate the release notes,
|
|
r|rnotes NEED DOC,
|
|
v|version the version to generate. Calculated from kohaversion if not provided
|
|
c|commit NEED DOC
|
|
html if set, the notes will be generated also in HTML format (useful for koha-community.org)
|
|
help|h display this help
|
|
verbose verbose
|
|
u a bugzilla login. If provided, will append the description/comment 0 to each enhancement
|
|
p a bugzilla password. If provided, will append the description/comment 0 to each enhancement
|
|
|
|
|
|
=cut
|
|
|