diff options
| -rw-r--r-- | buildbot_gentoo_ci/config/config.py | 3 | ||||
| -rw-r--r-- | buildbot_gentoo_ci/steps/logs.py | 141 | ||||
| -rw-r--r-- | gentooci.cfg | 20 |
3 files changed, 132 insertions, 32 deletions
diff --git a/buildbot_gentoo_ci/config/config.py b/buildbot_gentoo_ci/config/config.py index e2f33be..696ab0f 100644 --- a/buildbot_gentoo_ci/config/config.py +++ b/buildbot_gentoo_ci/config/config.py @@ -93,6 +93,7 @@ class GentooCiConfig(util.ComparableMixin): "db_url", "project", "worker_config", + "bug_config", "repository_basedir" ]) @@ -152,3 +153,5 @@ class GentooCiConfig(util.ComparableMixin): self.project['repository_basedir'] = DEFAULT_REPOSITORY_BASEDIR if 'worker_config' in config_dict: self.project['worker_config'] = config_dict['worker_config'] + if 'bug_config' in config_dict: + self.project['bug_config'] = config_dict['bug_config'] diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index e4fc951..ee11166 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -1,4 +1,4 @@ -# Copyright 2021 Gentoo Authors +# Copyright 2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os @@ -14,6 +14,7 @@ from twisted.internet import defer from twisted.python import log from buildbot.process.buildstep import BuildStep +from buildbot.process.properties import Properties from buildbot.process.results import SUCCESS from buildbot.process.results import FAILURE from buildbot.process.results import WARNINGS @@ -21,7 +22,7 @@ from buildbot.process.results import SKIPPED from buildbot.plugins import steps from buildbot.plugins import util -from buildbot_gentoo_ci.steps import minio +#from buildbot_gentoo_ci.steps import minio from buildbot_gentoo_ci.steps import master as master_steps from buildbot_gentoo_ci.steps import bugs @@ -91,13 +92,15 @@ class SetupPropertys(BuildStep): self.setProperty("default_project_data", default_project_data, 'default_project_data') self.setProperty("version_data", version_data, 'version_data') self.setProperty("status", 'completed', 'status') - if self.getProperty('faild_cpv'): + if isinstance(self.getProperty('faild_cpv'), str): log_cpv = self.getProperty('faild_cpv') else: log_cpv = self.getProperty('cpv') self.setProperty("log_cpv", log_cpv, 'log_cpv') - self.setProperty("bgo", dict( match=False), 'bgo') + self.setProperty("bgo", False, 'bgo') self.descriptionDone = 'Runing log checker on ' + log_cpv + logsdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id'])) + self.setProperty("logsdir", logsdir, 'logsdir') return SUCCESS class SetupParserBuildLoger(BuildStep): @@ -115,9 +118,9 @@ class SetupParserBuildLoger(BuildStep): @defer.inlineCallbacks def run(self): self.aftersteps_list = [] - workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id'])) log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')] - mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname']) + build_log_file_compressed = log_cpv['full_logname'] + '.xz' + mastersrc_log = yield os.path.join(self.getProperty('logsdir'), build_log_file_compressed) log_py = 'log_parser.py' config_log_py = 'logparser.json' mastersrc_py = yield os.path.join(self.master.basedir, log_py) @@ -125,7 +128,7 @@ class SetupParserBuildLoger(BuildStep): # Upload logfile to worker self.aftersteps_list.append(steps.FileDownload( mastersrc=mastersrc_log, - workerdest=log_cpv['full_logname'] + workerdest=build_log_file_compressed )) # Upload log parser py code self.aftersteps_list.append(steps.FileDownload( @@ -137,6 +140,16 @@ class SetupParserBuildLoger(BuildStep): mastersrc=mastersrc_config, workerdest=config_log_py )) + #Untar the log + shell_commad_list = [] + shell_commad_list.append('xz') + shell_commad_list.append('-dv') + shell_commad_list.append(build_log_file_compressed) + self.aftersteps_list.append( + steps.ShellCommand( + name = 'Decompress build log with xz', + command = shell_commad_list, + )) # Run the log parser code command = [] command.append('python3') @@ -171,23 +184,6 @@ class MakeIssue(BuildStep): def __init__(self, **kwargs): super().__init__(**kwargs) - @defer.inlineCallbacks - def logIssue(self): - separator1 = '\n' - separator2 = ' ' - log = yield self.addLog('issue') - self.error_dict['cpv'] = self.getProperty('log_cpv') - yield log.addStdout('Title:' + '\n') - yield log.addStdout(separator2.join([self.getProperty('log_cpv'), '-', self.error_dict['title']]) + separator1) - yield log.addStdout('Summary:' + '\n') - for line in self.summary_log_list: - yield log.addStdout(line + '\n') - yield log.addStdout('Attachments:' + '\n') - yield log.addStdout('emerge_info.log' + '\n') - log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')] - yield log.addStdout(log_cpv['full_logname'] + '\n') - yield log.addStdout('world.log' + '\n') - def getNiceErrorLine(self, line): # strip away hex addresses, loong path names, line and time numbers and other stuff # https://github.com/toralf/tinderbox/blob/main/bin/job.sh#L467 @@ -211,12 +207,16 @@ class MakeIssue(BuildStep): if s['type'] == self.error_dict['phase'] and s['status'] == 'error': text_issue_list.append(v['text']) # add the issue error + self.error_dict['cpv'] = self.getProperty('log_cpv') if text_issue_list != []: - self.error_dict['title_issue'] = text_issue_list[0].replace('*', '').strip() - self.error_dict['title_issue_nice'] = self.getNiceErrorLine(text_issue_list[0].replace('*', '').strip()) + title_line = text_issue_list[0].replace('*', '').strip() + self.error_dict['title_issue'] = title_line + self.error_dict['title_issue_nice'] = self.getNiceErrorLine(title_line) + self.error_dict['title_found'] = True else: self.error_dict['title_issue'] = 'title_issue : None' self.error_dict['title_nice'] = 'title_issue : None' + self.error_dict['title_found'] = False self.error_dict['title_phase'] = 'failed in '+ self.error_dict['phase'] #set the error title self.error_dict['title'] = self.error_dict['title_phase'] + ' - ' + self.error_dict['title_issue'] @@ -243,13 +243,14 @@ class MakeIssue(BuildStep): phase_error = v['text'].split(' (')[1].split(' phase')[0] self.error_dict['phase'] = phase_error error = True - # add build log + #FIXME: write summary_log_list to a file # add issue/bug/pr report if error: yield self.ClassifyIssue() print(self.error_dict) - yield self.logIssue() self.setProperty("status", 'failed', 'status') + #FIXME: nice description for post bug + self.error_dict['summary_log_nice'] = '' self.setProperty("error_dict", self.error_dict, 'error_dict') self.aftersteps_list.append(bugs.GetBugs()) if warning: @@ -295,14 +296,13 @@ class ReadEmergeInfoLog(BuildStep): def __init__(self, **kwargs): super().__init__(**kwargs) - @defer.inlineCallbacks + #@defer.inlineCallbacks def run(self): emerge_info_output = {} emerge_info_list = [] emerge_package_info = [] # Read the file and add it to a property - workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id'])) - with open(os.path.join(workdir, 'emerge_info.txt'), encoding='utf-8') as source: + with open(os.path.join(self.getProperty('logsdir'), 'emerge_info.txt'), encoding='utf-8') as source: emerge_info = source.read() # set emerge_info_output Property for line in emerge_info.split('\n'): @@ -379,7 +379,7 @@ class Upload(BuildStep): def run(self): log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')] bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs' - file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname']) + file_path = yield os.path.join(self.getProperty('logsdir') ,log_cpv['full_logname']) aftersteps_list = [] aftersteps_list.append(minio.putFileToMinio(file_path, log_cpv['full_logname'], bucket)) yield self.build.addStepsAfterCurrentStep(aftersteps_list) @@ -435,6 +435,83 @@ class ParserPkgCheckLog(BuildStep): returnstatus = WARNINGS return returnstatus +class SetupBugReportSteps(BuildStep): + name = 'SetupBugReportSteps' + description = 'Running' + descriptionDone = 'Ran' + descriptionSuffix = None + haltOnFailure = False + flunkOnFailure = True + warnOnWarnings = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + bug_config = self.gentooci.config.project['bug_config'] + # add bug enable profile and repo + if bug_config['enable'] and self.getProperty('status') == 'failed' and self.getProperty('error_dict')['title_found'] and not self.getProperty('bgo'): + aftersteps_list = [] + #Post + p = Properties() + p.master = self.master + separator = ' ' + separator2 = '\n' + bug_args = {} + bug_args['user'] = bug_config['user'] + bug_args['passwd'] = yield p.render(util.Secret("bugs_password")) + bug_params = {} + title = separator.join([bug_config['extra_summery'], self.getProperty('error_dict')['cpv'], self.getProperty('error_dict')['title']]) + # bug title max 170 + if len(title) >= 170: + title = title[:167] + '...' + bug_params['summary'] = title + # max 16384 + description_list = self.getProperty('summary_log_list') + del description_list[-6:] + description_text = separator2.join(description_list) + bug_params['description'] = description_text + separator + bug_params['assigned_to'] = bug_config['user'] + #params['cc'] = cc + bug_params['url'] = f"https://ci.dev.gentoo.org:8010/#/builders/7/builds/{self.getProperty('buildnumber')}" + print(bug_params) + # post + aftersteps_list.append(bugs.Post(bug_args, bug_params)) + # add buildbot info + bug_params2 = {} + bug_params2['comment'] = separator2.join(bug_config['buildbot_comment']) + aftersteps_list.append(bugs.Modify(bug_args, bug_params2)) + # add build info + comment_list = [] + comment_list.append(f"Was trying to build {self.getProperty('cpv')} Commit: {self.getProperty('revision')} Repository: {self.getProperty('repository')}") + if self.getProperty('faild_cpv') != self.getProperty('cpv'): + comment_list.append(f"Depend: {self.getProperty('faild_cpv')} fail to build") + comment_list.append(f"Package info:") + for line in self.getProperty('emerge_info_output')['emerge_package_info']: + comment_list.append(f"{line}") + comment_list.append(f"Emerge info:") + for line in self.getProperty('emerge_info_output')['emerge_info']: + comment_list.append(f"{line}") + bug_params3 = {} + bug_params3['comment'] = separator2.join(comment_list) + aftersteps_list.append(bugs.Modify(bug_args, bug_params3)) + # Attach build log + file_list = [] + with os.scandir(self.getProperty('logsdir')) as entries: + for entry in entries: + file_list.append(entry.name) + print(file_list) + for filename in file_list: + bug_params4 = {} + bug_params4['filename'] = yield os.path.join(self.getProperty('logsdir'), filename) + #bug_params4['comment'] = filename + aftersteps_list.append(bugs.Attach(bug_args, bug_params4)) + yield self.build.addStepsAfterCurrentStep(aftersteps_list) + return SUCCESS + return SKIPPED + class setBuildStatus(BuildStep): name = 'setBuildStatus' diff --git a/gentooci.cfg b/gentooci.cfg index cd4bfda..474da76 100644 --- a/gentooci.cfg +++ b/gentooci.cfg @@ -26,6 +26,9 @@ makeconf_list.append('PORTAGE_ELOG_CLASSES="*"') makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"') # use ansifilter on the logs makeconf_list.append('PORTAGE_LOG_FILTER_FILE_CMD="bash -c \'ansifilter --ignore-clear; exec cat\'"') +# use xz for Portage compression of both installed files and binary packages: +makeconf_list.append('PORTAGE_COMPRESS="xz"') +makeconf_list.append('BINPKG_COMPRESS="xz"') # This specifies what project buildbot uses for Gentoo Ci as default c['project'] = { @@ -43,3 +46,20 @@ c['worker_config'] = { 'repository_basedir' : 'repositorys', 'portage_repos_path' : '/var/db/repos', } +# This specifies bug configs +c['bug_config'] = { + 'enable' : True, + 'aout_assignee' : False, + 'user' : 'LarryTheCow@Cow.mu', + 'extra_summery' : '', + 'buildbot_comment' : [ + 'This was build on a tinderbox that use Buildbot API[1].', + 'We use docker images to build on.', + 'Web: https://ci.dev.gentoo.org:8010/#/', + 'Code: https://gitweb.gentoo.org/proj/tinderbox-cluster.git', + 'Wiki: https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster', + 'How to get more info: https://wiki.gentoo.org/wiki/Project:Tinderbox-cluster/something', + 'IRC: #gentoo-ci on libera', + '[1] https://buildbot.net/', + ], +} |