scripts/release-notes: Update for machine-xml changes, diff package list

A bit of a decent rework to the release-notes tool, coping with the
machine-xml rework (to be one package with different repos/versions per
platform, see cd986e4cf84a526ee9653f1282e7889a9312f658)

We also do a legal-info build for each platform, as the resulting
manifest.csv allows us to work out what packages have been
updated/added/removed for each platform.

Also add the '--platform' option, to produce release notes for only
one platform.

Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

1 files changed, 220 insertions, 92 deletions

diff --git a/openpower/scripts/release-notes b/openpower/scripts/release-notes
index 5df61547..4231aa04 100755
--- a/openpower/scripts/release-notes
+++ b/openpower/scripts/release-notes
@@ -1,133 +1,264 @@
 #!/bin/env perl
+
 use strict;
 
+use Getopt::Long;
+use DBI;
+
+# We use an in-memory SQLite database as SQL is good for doing queries
+# of differences between data sets, and doing that in perl is annoying.
+my $db = "dbi:SQLite:dbname=release-notes.sqlite";
+#my $dbh = DBI->connect("dbi:SQLite:dbname=:memory:","","");
+my $dbh = DBI->connect($db,undef,undef, {AutoCommit => 1, RaiseError=>1});
+
 my $repos =
 {
     'op-build' => { REPO => "http://github.com/open-power/op-build" },
     'hostboot' => { REPO => "http://github.com/open-power/hostboot" ,
-                    DIR => "openpower/package/hostboot",
-                    PACKAGE => "HOSTBOOT" },
+                    DIR => "openpower/package/hostboot" },
     'sbe'      => { REPO => "http://github.com/open-power/sbe" ,
-                    DIR => "openpower/package/sbe",
-                    PACKAGE => "SBE" },
+                    DIR => "openpower/package/sbe"},
     'skiboot'  => { REPO => "http://github.com/open-power/skiboot" ,
-                    DIR => "openpower/package/skiboot",
-                    PACKAGE => "SKIBOOT" },
+                    DIR => "openpower/package/skiboot"},
     'occ'      => { REPO => "http://github.com/open-power/occ" ,
-                    DIR => "openpower/package/occ",
-                    PACKAGE => "OCC" },
+                    DIR => "openpower/package/occ"},
     'pnor'     => { REPO => "http://github.com/open-power/pnor" ,
-                    DIR => "openpower/package/openpower-pnor",
-                    PACKAGE => "OPENPOWER_PNOR" },
-    'palmetto-xml' => { REPO => "http://github.com/open-power/palmetto-xml" ,
-                    DIR => "openpower/package/palmetto-xml",
-                    PACKAGE => "PALMETTO_XML" },
-    'habanero-xml' => { REPO => "http://github.com/open-power/habanero-xml" ,
-                    DIR => "openpower/package/habanero-xml",
-                    PACKAGE => "HABANERO_XML" },
-    'firestone-xml' => { REPO => "http://github.com/open-power/firestone-xml" ,
-                    DIR => "openpower/package/firestone-xml",
-                    PACKAGE => "FIRESTONE_XML" },
-    'garrison-xml'  => { REPO => "http://github.com/open-power/garrison-xml" ,
-                    DIR => "openpower/package/garrison-xml",
-                    PACKAGE => "GARRISON_XML" },
-    'barreleye-xml'  => { REPO => "http://github.com/open-power/barreleye-xml" ,
-                    DIR => "openpower/package/barreleye-xml",
-                    PACKAGE => "BARRELEYE_XML" },
-    'romulus-xml'  => { REPO => "http://github.com/open-power/romulus-xml" ,
-                    DIR => "openpower/package/romulus-xml",
-                    PACKAGE => "ROMULUS_XML" },
-    'witherspoon-xml'  => { REPO => "http://github.com/open-power/witherspoon-xml" ,
-                    DIR => "openpower/package/witherspoon-xml",
-                    PACKAGE => "WITHERSPOON_XML" },
-    'zaius-xml'  => { REPO => "http://github.com/open-power/zaius-xml" ,
-                    DIR => "openpower/package/zaius-xml",
-                    PACKAGE => "ZAIUS_XML" },
+                    DIR => "openpower/package/openpower-pnor"},
     'petitboot'     => { REPO => "http://github.com/open-power/petitboot" ,
-                    DIR => "openpower/package/petitboot",
-                    PACKAGE => "PETITBOOT" },
+			 DIR => "openpower/package/petitboot"},
 };
 
-my $begin_release = shift;
-my $end_release = shift;
+my $begin_release;
+my $end_release;
+my $begin_worktree;
+my $end_worktree;
+my $platform;
 
-foreach my $repo (keys %{$repos})
+GetOptions("begin-release=s" => \$begin_release,
+	   "end-release=s" => \$end_release,
+	   "begin-worktree=s" => \$begin_worktree,
+	   "platform=s" => \$platform,
+	   "end-worktree=s" => \$end_worktree)
+    or die("Error in command line arguments");
+
+die "Required argument missing" unless ($begin_release and $end_release and $begin_worktree and $end_worktree);
+
+open(OUTPUT, "> RELEASE.md") || die "Failed to open RELEASE.md";
+
+print OUTPUT "# Release Notes for OpenPower Firmware $end_release\n";
+
+if (-e $begin_worktree) {
+    system("cd $begin_worktree && git checkout $begin_release && git submodule update --reference ../buildroot && git submodule update") and die "Could not update $begin_worktree";
+} else {
+    system("git worktree add $begin_worktree $begin_release && cd $begin_worktree&& git submodule update --reference ../buildroot && git submodule update && rm -rf dl && ln -s ../dl dl") and die "Couldn't init $begin_worktree";
+}
+
+if (-e $end_worktree) {
+    system("cd $end_worktree && git checkout $end_release  && git submodule update --reference ../buildroot && git submodule update") and die "Could not update $end_worktree";
+} else {
+    system("git worktree add $end_worktree $end_release && cd $end_worktree && git submodule update --reference ../buildroot && git submodule update && rm -rf dl && ln -s ../dl dl") and die "Couldn't init $end_worktree";
+}
+
+opendir (my $dh, "$begin_worktree/openpower/configs")
+    or die "can't scan $begin_worktree defconfigs";
+my @begin_platforms = grep { /.*_defconfig/ } readdir($dh);
+closedir $dh;
+
+opendir (my $dh, "$end_worktree/openpower/configs")
+    or die "can't scan $end_worktree defconfigs";
+my @end_platforms = grep { /.*_defconfig/ } readdir($dh);
+closedir $dh;
+
+s/_defconfig// foreach (@begin_platforms);
+s/_defconfig// foreach (@end_platforms);
+
+if ($platform) {
+    @begin_platforms = ($platform);
+    @end_platforms = ($platform);
+}
+
+$dbh->do("CREATE TABLE platforms (platform TEXT, version TEXT);") or die "$!";
 {
-    if (-e $repo)
-    {
-        system("cd $repo; git fetch") && die "Could not fetch $repo";
-    }
-    else
-    {
-        system("git clone $repos->{$repo}->{REPO} $repo") &&
-            die "Could not clone $repo";
-    }
+    my $q = "INSERT INTO platforms (platform,version) VALUES (?,?)";
+    my $sth = $dbh->prepare($q) or die "$!";
+    $sth->execute($_, $begin_release) foreach (@begin_platforms);
+    $sth->execute($_, $end_release) foreach (@end_platforms);
 }
 
-system("cd op-build; git checkout $end_release --force; git reset HEAD --hard");
+{
+    my $q = "SELECT platform FROM platforms WHERE version is ? AND platform NOT IN (SELECT platform FROM platforms WHERE version is ?)";
+    my $sth = $dbh->prepare($q) or die $!;
+    $sth->execute($begin_release, $end_release);
+    my $r;
+    print OUTPUT "## Removed platforms\n\n- ".$r->{platform}."\n" if $r = $sth->fetchrow_hashref;
+    print OUTPUT "- ".$_->{platform}."\n" while ($r = $sth->fetchrow_hashref);
 
+    $sth->execute($end_release, $begin_release);
+    print OUTPUT "## New platforms\n\n- ".$r->{platform}."\n" if $r = $sth->fetchrow_hashref;
+    print OUTPUT "- ".$_->{platform}."\n" while($r = $sth->fetchrow_hashref);
+}
+
+my @common_platforms;
+{
+    my $q = "SELECT platform FROM platforms WHERE version is ? AND EXISTS (select platform from platforms where version is ?)";
+    my $sth = $dbh->prepare($q) or die $!;
+    $sth->execute($begin_release, $end_release);
+    my $r;
+    push @common_platforms, $r->{platform} while ($r = $sth->fetchrow_hashref);
+}
+
+foreach my $p (@common_platforms) {
+    next if $p =~ /firenze/;
+    next if $p =~ /^zz$/;
+    next if $p =~ /mambo/;
+    $repos->{"$p-xml"} = { REPO => "http://github.com/open-power/$p-xml" ,
+			   DIR => "openpower/package/$p-xml" };
+}
+
+foreach my $p (@begin_platforms) {
+    system("bash -c '(cd $begin_worktree && . op-build-env && op-build ".$p."_defconfig && op-build legal-info)'");
+    # Forgive me for this....
+    system("sqlite3 release-notes.sqlite \".mode csv\" \".import $begin_worktree/output/legal-info/manifest.csv begin_".$p."_manifest\"");
+}
+
+foreach my $p (@end_platforms) {
+    system("bash -c '(cd $end_worktree && . op-build-env && op-build ".$p."_defconfig && op-build legal-info)'");
+    # Forgive me for this....
+    system("sqlite3 release-notes.sqlite \".mode csv\" \".import $end_worktree/output/legal-info/manifest.csv end_".$p."_manifest\"");
+}
 
-open(OP_SUMMARY, "cd op-build; ".
-        "git diff $begin_release...$end_release --name-only |".
-        "grep -e \"\.mk\" -e \"Config.in\" |".
-        "xargs git diff $begin_release...$end_release -- |".
-        "grep -e '^[+-][^[:space:]]*_VERSION' ".
-        "-e'^[+-][[:space:]]*default.*if.*LATEST_VERSION' |") || die;
+$dbh->do(<<'SQL') or die "$!";
+CREATE TABLE package_upgrades (
+  PACKAGE TEXT,
+  OLDVERSION TEXT,
+  NEWVERSION TEXT,
+  PLATFORM TEXT
+)
+SQL
+
+foreach my $p (@common_platforms) {
+    $dbh->do("INSERT INTO package_upgrades select b.package,b.version,e.version,'$p' from begin_".$p."_manifest as b LEFT JOIN end_".$p."_manifest AS e ON b.package=e.package WHERE b.version != e.version") or die $!;
+}
+
+$dbh->do(<<'SQL') or die "$!";
+CREATE TABLE new_package (
+  PACKAGE TEXT,
+  VERSION TEXT,
+  PLATFORM TEXT
+)
+SQL
+
+foreach my $p (@common_platforms) {
+    $dbh->do("INSERT INTO new_package select b.package,b.version,'$p' from end_".$p."_manifest as b WHERE NOT EXISTS(SELECT package FROM begin_".$p."_manifest AS e WHERE b.package=e.package)") or die $!;
+}
+
+$dbh->do(<<'SQL') or die "$!";
+CREATE TABLE removed_package (
+  PACKAGE TEXT,
+  VERSION TEXT,
+  PLATFORM TEXT
+)
+SQL
+
+foreach my $p (@common_platforms) {
+    $dbh->do("INSERT INTO removed_package select b.package,b.version,'$p' from begin_".$p."_manifest as b WHERE NOT EXISTS(SELECT package FROM end_".$p."_manifest AS e WHERE b.package=e.package)") or die $!;
+}
 
 my $old_level = {};
 my $new_level = {};
 
-while(my $line = <OP_SUMMARY>)
 {
-    chomp $line;
-    if ($line =~ m/\$/) # Sanity check there are not variables here.
-    {
-        next;
-    }
-    if ($line =~ m/-([^[:space:]]*)_VERSION([[:space:]]*\?*=[[:space:]]*)(.*)/)
-    {
-        print "Old $1:$3\n";
-        $old_level->{$1} = $3;
-    }
-    if ($line =~ m/-([^[:space:]]*)_VERSION_BRANCH_MASTER([[:space:]]*\?*=[[:space:]]*)(.*)/)
-    {
-        print "Old $1:$3\n";
-        $old_level->{$1} = $3;
+    my $q = <<'SQL';
+select package as pk ,oldversion as o ,newversion as n, 
+	GROUP_CONCAT(platform) as ps
+FROM package_upgrades 
+	GROUP BY package,oldversion,newversion
+ORDER BY package,platform
+SQL
+    my $sth = $dbh->prepare($q) or die $!;
+    $sth->execute();
+
+    print OUTPUT "## Updated Packages\n\n";
+    print OUTPUT "Package | Old Version | New Version | Platforms\n";
+    print OUTPUT "--- | --- | --- | ---\n";
+    while (my $r = $sth->fetchrow_hashref) {
+	print OUTPUT join(' | ',($r->{pk}, $r->{o}, $r->{n}, $r->{ps}))."\n" ;
+	if ($r->{pk} eq 'machine-xml') {
+	    $old_level->{$r->{ps}."-xml"} = $r->{o};
+	    $new_level->{$r->{ps}."-xml"} = $r->{n};
+	} else {
+	    $old_level->{$r->{pk}} = $r->{o};
+	    $new_level->{$r->{pk}} = $r->{n};
+	}
     }
-    if ($line =~ m/\+([^[:space:]]*)_VERSION([[:space:]]*\?*=[[:space:]]*)(.*)/)
-    {
-        print "New $1:$3\n";
-        $new_level->{$1} = $3;
+}
+
+{
+    my $q = <<'SQL';
+select package as pk ,version as v,
+	GROUP_CONCAT(platform) as ps
+FROM new_package
+	GROUP BY package,version
+ORDER BY package,platform
+SQL
+    my $sth = $dbh->prepare($q) or die $!;
+    $sth->execute();
+
+    print OUTPUT "\n\n## New Packages\n\n";
+    print OUTPUT "Package | Version | Platforms\n";
+    print OUTPUT "--- | --- | ---\n";
+    while (my $r = $sth->fetchrow_hashref) {
+	print OUTPUT join(' | ',($r->{pk}, $r->{v}, $r->{ps}))."\n" ;
+	if ($r->{pk} eq 'machine-xml') {
+	    $new_level->{$r->{ps}."-xml"} = $r->{v};
+	}
     }
-    if ($line =~ m/\+([^[:space:]]*)_VERSION_BRANCH_MASTER([[:space:]]*\?*=[[:space:]]*)(.*)/)
-    {
-        print "New $1:$3\n";
-        $new_level->{$1} = $3;
+}
+
+{
+    my $q = <<'SQL';
+select package as pk ,version as v,
+	GROUP_CONCAT(platform) as ps
+FROM removed_package
+	GROUP BY package,version
+ORDER BY package,platform
+SQL
+    my $sth = $dbh->prepare($q) or die $!;
+    $sth->execute();
+
+    print OUTPUT "\n\n## Removed Packages\n\n";
+    print OUTPUT "Package | Version | Platforms\n";
+    print OUTPUT "--- | --- | ---\n";
+    while (my $r = $sth->fetchrow_hashref) {
+	print OUTPUT join(' | ',($r->{pk}, $r->{v}, $r->{ps}))."\n" ;
+	if ($r->{pk} eq $r->{ps}."-xml") {
+	    $old_level->{$r->{ps}."-xml"} = $r->{v};
+	}
     }
+    print OUTPUT "\n\n";
+}
 
-    if ($line =~ m/-[[:space:]]*default[[:space:]]*"([^[:space:]]*)"[[:space:]]*if[[:space:]]BR2_([^[:space:]]*)_LATEST_VERSION/)
+foreach my $repo (keys %{$repos})
+{
+    if (-e $repo)
     {
-        print "Old $2:$1\n";
-        $old_level->{$2} = $1;
+        system("cd $repo; git fetch") && die "Could not fetch $repo";
     }
-    if ($line =~ m/\+[[:space:]]*default[[:space:]]*"([^[:space:]]*)"[[:space:]]*if[[:space:]]BR2_([^[:space:]]*)_LATEST_VERSION/)
+    else
     {
-        print "New $2:$1\n";
-        $new_level->{$2} = $1;
+        system("git clone $repos->{$repo}->{REPO} $repo") &&
+            die "Could not clone $repo";
     }
 }
 
-open(OUTPUT, "> RELEASE.md") || die "Failed to open RELEASE.md";
+system("cd op-build; git checkout $end_release --force; git reset HEAD --hard");
 
-print OUTPUT "# Release Notes for OpenPower Firmware $end_release\n";
 
 my $op_url = $repos->{'op-build'}->{REPO};
 
 foreach my $repo (sort keys %{$repos})
 {
-    next if (not exists $repos->{$repo}->{PACKAGE});
-
-    my $package = $repos->{$repo}->{PACKAGE};
+    my $package = $repo;
     my $url = $repos->{$repo}->{REPO};
     my $dir = $repos->{$repo}->{DIR};
 
@@ -135,8 +266,6 @@ foreach my $repo (sort keys %{$repos})
     print OUTPUT "[Repository]($url)\n";
     print OUTPUT "\n";
 
-    my $package = $repos->{$repo}->{PACKAGE};
-
     # Display patches.
     if (open(LSLOG, "ls op-build/$dir/*.patch | ".
                     "xargs -n1 --no-run-if-empty basename |"))
@@ -171,7 +300,6 @@ foreach my $repo (sort keys %{$repos})
         (exists $new_level->{$package}))
     {
         print "Changes in $repo...\n";
-
         open(GITLOG, "cd $repo; git shortlog $old_level->{$package}...".
                      "$new_level->{$package} --no-merges --format=".
                      "\"* [%h]($url/commit/%h) %s\" |");