#!/usr/bin/env python # Copyright 2014-2016 Gentoo Authors; Distributed under the GPL v2 from __future__ import unicode_literals import io import os import sys import time from xml.sax.saxutils import escape as escape_xml def usage(): sys.stderr.write(("usage: %s " " " " [whitelist dir]\n") % \ os.path.basename(sys.argv[0])) sys.stderr.flush() if len(sys.argv) < 7 or len(sys.argv) > 8: usage() sys.exit(1) if not os.access(sys.argv[1], os.R_OK): sys.stderr.write("couldn't read %s\n" % sys.argv[1]) sys.exit(2) fail_f=open(sys.argv[1], "r") suc_f=open(sys.argv[2],"r") if not os.path.isfile(sys.argv[3]): sys.stderr.write("%s isn't a file.\n" % sys.argv[3]) sys.exit(3) scheduled_deletions_log = sys.argv[3] log_date_format = sys.argv[4] try: outf = io.open(sys.argv[6], mode='w', encoding="utf_8") except OSError as e: sys.stderr.write("failed opening out file; lack write " "perms, dir exist? %s\n" % (e,)) sys.exit(4) def grab_whitelists(whitelists_dir): whitelists = {} for x in os.listdir(whitelists_dir): if x[:1] == ".": continue x = os.path.join(whitelists_dir, x) if not os.path.isfile(x): continue whitelists[x] = [] with io.open(x, mode='r', encoding='utf_8') as f: for entry in f: entry = entry.lstrip().rstrip() if len(entry) == 0 or entry.startswith("#"): continue whitelists[x].append(entry.lstrip().rstrip()) if not whitelists[x]: del whitelists[x] return whitelists if len(sys.argv) >= 8: whitelist = grab_whitelists(sys.argv[7]) else: whitelist = {} failed_ebuilds={} for l in fail_f: try: cpv, uri, reason = l.split("\t") except ValueError: continue cpv = cpv.lstrip().rstrip() uri = uri.lstrip().rstrip() reason = reason.lstrip().rstrip() #cpv, file, reason failed_ebuilds.setdefault(cpv, []).append((uri, reason)) fail_f.close() removed_files = {} added_files = {} for l in suc_f: cpv, file, comment = l.split("\t") if comment.rstrip().lstrip() == "removed": removed_files.setdefault(cpv,[]).append(file) elif comment.lstrip().startswith("added"): added_files.setdefault(cpv,[]).append(file) suc_f.close() if not os.path.isdir(sys.argv[5]): sys.stderr.write("%s isn't a dir, thus isn't distdir. die.\n" % sys.argv[5]) sys.exit(6) dist_files={} for f in os.listdir(sys.argv[5]): dist_files[f] = os.stat(sys.argv[5]+os.path.sep+f).st_size del_list={} del_totals={} del_count=0 t = None with io.open(scheduled_deletions_log, mode='r', encoding='utf_8') as f: for line in f: split = line.rstrip().split('\t') if len(split) == 1: try: t = time.strftime("%s", time.strptime(split[0], log_date_format)) except ValueError: continue # epoch time. del_totals[t] = 0 l = {} del_list[t] = l elif len(split) == 3 and not split[0] and t is not None: filename = split[1] cpv = split[2] l.setdefault(cpv, []).append(filename) try: del_totals[t] += dist_files[filename] del_count += 1 except KeyError: pass cur_ep = time.strftime("%s", time.strptime(time.strftime("%Y-%m-%d",time.gmtime()), "%Y-%m-%d")) l=list(del_list.keys()) l.sort() while len(l) and cur_ep > l[0]: del del_list[l[0]] l.pop(0) slated_for_death_total = 0 for x in l: slated_for_death_total += del_totals[x] f_len=max(len(str(len(list(dist_files.keys())))), len(str(del_count))) dist_sum=sum(dist_files.values()) d_len = max(len(str(dist_sum)), len(str(slated_for_death_total))) outf.write(""" Distfiles Mirroring Changes Report 1.0 """+time.asctime(time.localtime())+""" Master Stats

Report was generated on %s

Remember the tree could've changed since then.

%s bytes: %s files: Mirrored
%s bytes: %s files: Slated for deletion
Added %i files since beginning of day (UTC)
Removed %i files in last run

""" % \ (time.strftime("%A %b %d %Y, %H:%M:%S UTC", time.gmtime()), str(dist_sum).rjust(d_len), str(len(list(dist_files.keys()))).rjust(f_len), str(slated_for_death_total).rjust(d_len), str(del_count).rjust(f_len), sum([len(x) for x in list(added_files.values())]), sum([len(x) for x in list(removed_files.values())]))) dist_files.clear() outf.write(""" Failed Fetches
Why has a file failed to be fetched?

Upstream re-releases, a large failure in upstream mirrors/host, or flat out invalid URL's are typically the cause. Broken mirror specifications also (mirror:/gentoo is not valid for example, must be mirror://gentoo). Odd ports for the host can cause issues also (guidelines for ebuilds suggest standard ports for URI hosts).

In other words, there is something wrong with the ebuild referenced, so please check the ebuild. It's more likely then not broke in some fashion even if the maintainer isn't aware of it.

\n""") #quote stripping is needed. esort = list(failed_ebuilds.keys()) esort.sort() for cpv in esort: for f,r in failed_ebuilds[cpv]: outf.write("%s%s%s\n" % (escape_xml(cpv), escape_xml(f), escape_xml(r))) outf.write("
EbuildFileReason
\n") outf.write("""
 Files Added
\n""") added_s = list(added_files.keys()) added_s.sort() for cpv in added_s: added_files[cpv].sort() for f in added_files[cpv]: outf.write("%s%s\n" % (escape_xml(cpv), escape_xml(f))) outf.write("""
EbuildFile
 Files Removed
\n\n""") rem_s = list(removed_files.keys()) rem_s.sort() for cpv in rem_s: removed_files[cpv].sort() count = 0 for f in removed_files[cpv]: if count == 0: outf.write(""" %s""" % escape_xml(cpv)) count+=1 else: outf.write(""" """) outf.write("""%s\n""" % escape_xml(f)) outf.write("""
EbuildFile
 Scheduled Deletions
\n
    """) l=list(del_list.keys()) l.sort() for date in l: outf.write("""
  • %s, %i files for %i bytes
    • \n""" % (date, time.strftime("%B %d %Y", time.gmtime(float(date))), sum([len(x) for x in list(del_list[date].values())]), del_totals[date])) outf.write("
\n") for date in l: outf.write("""
Deletions for %s \n""" % (date, time.strftime("%A %B %d %Y", time.gmtime(float(date))))) cpvs = list(del_list[date].keys()) cpvs.sort() for cpv in cpvs: l=del_list[date][cpv] l.sort() outf.write(" %s%s\n" % (escape_xml(cpv), escape_xml(l[0]))) l.pop(0) for f in l: outf.write(" %s\n" % escape_xml(f)) outf.write("
Ebuild (If known)File
\n\n
\n") outf.write("") outf.write("White Lists") if not whitelist: outf.write("

Whitelist Report

") #outf.write("

No whitelists.

") l=list(whitelist.keys()) l.sort() for x in l: outf.write("
%s
    \n" % os.path.basename(x)) whitelist[x].sort() for y in whitelist[x]: outf.write("
  • %s
    • \n" % escape_xml(y)) outf.write("
\n") outf.write("") outf.close()