diff options
50 files changed, 276 insertions, 247 deletions
diff --git a/backend/backend.php b/backend/backend.php index 2874b1a..161b7b2 100755 --- a/backend/backend.php +++ b/backend/backend.php @@ -50,7 +50,7 @@ require_once(SHARED.'/include/dbinit.php'); while (true) { // TODO check first for builds that need to be resumed (and figure out how to resume things) while (true) { - $r=query('SELECT * FROM `builds` WHERE `backend`="'.$S['conf']['backend_id'].'" AND `status` IN ("queued","cancel","uploading","upload_failed","building","got_signal") ORDER BY `ctime` ASC LIMIT 1'); + $r=query('SELECT * FROM `builds` WHERE `backend`="'.$S['conf']['backend_id'].'" AND `status` IN ("queued","cancel","uploading","building","bundling") AND `failed`!="true" ORDER BY `ctime` ASC LIMIT 1'); if ($r->rowCount()) break; else { @@ -60,76 +60,37 @@ while (true) { } } $build=new sql_build($r->fetch(PDO::FETCH_ASSOC)); - if (!isset($build->start)) { - $build->start=time(); - $build->write(); - } debug('Starting build id='.$build->id); $file=null; $owner=$build->get_owner(); - $workdir=WORK."/build-$build->id"; - if (($image=$build->build($workdir)) !== false) { - try { - $bundler=$build->get_opt('bundler'); - $bundle_proc="bundle_$bundler"; - if (!function_exists($bundle_proc)) - throw_exception("No bundler function defined for bundler $bundler"); - $opts=$build->get_opts(); - $file=$bundle_proc($image, $workdir, $opts); - end_internal_task(0); // Just in case - } catch (Exception $e) { - log_msg('Caught exception: '.$e->getMessage()); - end_internal_task(1); - $build->status='failed'; - $build->write(); - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed in bundling stage. You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); - } - $build->finish=time(); - debug('Finished with build id='.$build->id); - if (isset($file)) { - debug("Completed build successfully"); - if ($S['conf']['split_setup']) { - $build->status='uploading'; - $build->write(); - $key=randstring(30); - $build->set_opt('uploadkey', $key); - $c=curl_init(url('backend/upload_image')); - curl_setopt($c, CURLOPT_POST, 1); - curl_setopt($c, CURLOPT_POSTFIELDS, array( - 'build' => $build->id, - 'key' => $key, - 'file' => "@$file" - )); - curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); - $result=curl_exec($c); - if ($S['conf']['debug'] && is_string($result)) { - debug($result); - } - if ($result === false || strpos($result, 'Upload successful') === false) { - $build->status='upload_failed'; - $build->write(); - } else { - debug("Transferred $file... unlinking it"); - unlink($file); - $build->status='complete'; - $build->write(); - shell_exec('rm -rf '.escapeshellarg($workdir)); - } - } else { - $build->status='complete'; - $base=basename($file); - $ext=substr($base, strpos($base, '.')); - rename($file, COMPLETED.'/build-'.$build->id.$ext); - $build->write(); - shell_exec('rm -rf '.escapeshellarg($workdir)); - } - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build finished', 'Your build has completed successfully. You can find more information and download the completed image at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); - } - } else { - $build->status='failed'; + $workdir=WORK."/$build->id"; + if ($build->status == 'cancel') { + shell_exec('rm -rf '.escapeshellarg($workdir)); + $build->status='canceled'; $build->write(); - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed. You can find more information at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); + continue; + } + if (isset($build->finish)) + $build->finish=null; + elseif (!isset($build->start)) + $build->start=time(); + $build->write(); + $success=$imagedir=$build->build($workdir); + if ($success && !$build->is_canceled()) { + $success=$file=$build->bundle($imagedir, $workdir); + if ($success && !$build->is_canceled()) { + $success=$build->upload($file); + } + } + if ($success) { + debug("Completed build successfully"); + execute_non_vital_command('Delete work directory', 'rm -rf '.escapeshellarg($workdir)); + $build->status='complete'; } + $build->finish=time(); + $build->write(); + debug('Finished with build id='.$build->id); + xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build finished', 'Your build has '.($success?'completed successfully':'failed').'. You can find more information and download the completed image at <a href="'.url("build/$build->id").'">'.url("build/$build->id").'</a>'); unset($build); } ?> diff --git a/backend/bundlers/cd-common.php b/backend/bundlers/cd-common.php new file mode 100644 index 0000000..0a7f3fb --- /dev/null +++ b/backend/bundlers/cd-common.php @@ -0,0 +1,6 @@ +<?php +makedir("$imagedir/boot"); +execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$imagedir/boot/kernel' extract /isolinux/gentoo.igz '$imagedir/boot/initrd' extract /image.squashfs '$workdir/'"); +//file_put_contents("$workdir/unsquashfs-files", "/lib64/modules\n/lib/modules\n"); +execute_command('Extract kernel modules from SquashFS to image', "unsquashfs -i -d '$workdir/modules' '$workdir/image.squashfs' /lib/modules /lib64/modules; cp -avT '$workdir/modules' '$imagedir'; rm -rf '$workdir/modules'"); +?> diff --git a/backend/bundlers/cd-head.php b/backend/bundlers/cd-head.php new file mode 100644 index 0000000..2e96bba --- /dev/null +++ b/backend/bundlers/cd-head.php @@ -0,0 +1,8 @@ +<?php +$profile=new sql_gentoo_profile($opts['profile']); +$headers=$profile->get_headers(); +if (strpos($headers['chost'], 'x86_64') === false) + $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; +else + $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; +?> diff --git a/backend/bundlers/cd.inc b/backend/bundlers/cd.inc deleted file mode 100644 index 6fb5aac..0000000 --- a/backend/bundlers/cd.inc +++ /dev/null @@ -1,12 +0,0 @@ -<?php -$profile=new sql_gentoo_profile($opts['profile']); -$headers=$profile->get_headers(); -if (strpos($headers['chost'], 'x86_64') === false) - $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; -else - $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; -makedir("$I/boot"); -execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'"); -//file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n"); -execute_command('Extract kernel modules from SquashFS to image', "unsquashfs -i -d '$W/modules' '$W/image.squashfs' /lib/modules /lib64/modules; cp -avT '$W/modules' '$I'; rm -rf '$W/modules'"); -?> diff --git a/backend/bundlers/ext2.php b/backend/bundlers/ext2.php deleted file mode 100644 index 239b5bb..0000000 --- a/backend/bundlers/ext2.php +++ /dev/null @@ -1,12 +0,0 @@ -<?php -function bundle_ext2($I, $W) { - execute_command('Make blank file for ext2 image', "dd if=/dev/zero of='$W/image.ext2' bs=1024 count=1048576"); - execute_command('Make ext2 filesystem', "mke2fs -t ext2 -F '$W/image.ext2'"); - makedir('ext2'); - execute_command('Mount ext2 image', "mount -o loop -t ext2 '$W/image.ext2' '$W/ext2'"); - execute_command('Copy files to ext2', "cp -va '$I/*' '$W/ext2/'"); - execute_command('Unmount ext2 image', "umount '$W/ext2'"); - execute_command('Compress ext2 image', "gzip '$W/image.ext2'"); - return "$W/image.ext2.gz"; -} -?> diff --git a/backend/bundlers/ext2/bundle.php b/backend/bundlers/ext2/bundle.php new file mode 100644 index 0000000..2418453 --- /dev/null +++ b/backend/bundlers/ext2/bundle.php @@ -0,0 +1,6 @@ +<?php +add_step('mkfs'); +add_step('cp'); +add_step('tar'); +return "$workdir/image.ext2.gz"; +?> diff --git a/backend/bundlers/ext2/cp.php b/backend/bundlers/ext2/cp.php new file mode 100644 index 0000000..befbd91 --- /dev/null +++ b/backend/bundlers/ext2/cp.php @@ -0,0 +1,6 @@ +<?php +makedir('ext2'); +execute_command('Mount ext2 image', "mount -o loop -t ext2 '$workdir/image.ext2' '$workdir/ext2'"); +execute_command('Copy files to ext2', "cp -va '$imagedir/*' '$workdir/ext2/'"); +execute_command('Unmount ext2 image', "umount '$workdir/ext2'"); +?> diff --git a/backend/bundlers/ext2/mkfs.php b/backend/bundlers/ext2/mkfs.php new file mode 100644 index 0000000..925c93f --- /dev/null +++ b/backend/bundlers/ext2/mkfs.php @@ -0,0 +1,4 @@ +<?php +execute_command('Make blank file for ext2 image', "dd if=/dev/zero of='$workdir/image.ext2' bs=1024 count=1048576"); +execute_command('Make ext2 filesystem', "mke2fs -t ext2 -F '$workdir/image.ext2'"); +?> diff --git a/backend/bundlers/ext2/tar.php b/backend/bundlers/ext2/tar.php new file mode 100644 index 0000000..5e51e1d --- /dev/null +++ b/backend/bundlers/ext2/tar.php @@ -0,0 +1,3 @@ +<?php +execute_command('Compress ext2 image', "gzip '$workdir/image.ext2'"); +?> diff --git a/backend/bundlers/installcd.php b/backend/bundlers/installcd.php deleted file mode 100644 index 63ffb5a..0000000 --- a/backend/bundlers/installcd.php +++ /dev/null @@ -1,8 +0,0 @@ -<?php -function bundle_installcd($I, $W, &$opts) { - require(dirname(__FILE__).'/cd.inc'); - execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' ."); - execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' add / '$W/image.tar.bz2' write '$W/image.iso'"); - return "$W/image.iso"; -} -?> diff --git a/backend/bundlers/installcd/assemble.php b/backend/bundlers/installcd/assemble.php new file mode 100644 index 0000000..c4cedc0 --- /dev/null +++ b/backend/bundlers/installcd/assemble.php @@ -0,0 +1,4 @@ +<?php +execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$workdir/image.tar.bz2' -C '$imagedir' ."); +execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' add / '$workdir/image.tar.bz2' write '$workdir/image.iso'"); +?> diff --git a/backend/bundlers/installcd/bundle.php b/backend/bundlers/installcd/bundle.php new file mode 100644 index 0000000..384490e --- /dev/null +++ b/backend/bundlers/installcd/bundle.php @@ -0,0 +1,6 @@ +<?php +require(dirname(__FILE__).'/cd-head.php'); +add_step('cd-common'); +add_step('assemble'); +return "$workdir/image.iso"; +?> diff --git a/backend/bundlers/installcd/cd-common.php b/backend/bundlers/installcd/cd-common.php new file mode 120000 index 0000000..70f9287 --- /dev/null +++ b/backend/bundlers/installcd/cd-common.php @@ -0,0 +1 @@ +../cd-common.php
\ No newline at end of file diff --git a/backend/bundlers/installcd/cd-head.php b/backend/bundlers/installcd/cd-head.php new file mode 120000 index 0000000..8a38a03 --- /dev/null +++ b/backend/bundlers/installcd/cd-head.php @@ -0,0 +1 @@ +../cd-head.php
\ No newline at end of file diff --git a/backend/bundlers/jffs2.php b/backend/bundlers/jffs2.php deleted file mode 100644 index c8e5de2..0000000 --- a/backend/bundlers/jffs2.php +++ /dev/null @@ -1,6 +0,0 @@ -<?php -function bundle_jffs2($I, $W) { - execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$I' -o '$W/image.jffs2'"); - return "$W/image.jffs2"; -} -?> diff --git a/backend/bundlers/jffs2/bundle.php b/backend/bundlers/jffs2/bundle.php new file mode 100644 index 0000000..de198a6 --- /dev/null +++ b/backend/bundlers/jffs2/bundle.php @@ -0,0 +1,4 @@ +<?php +add_step('mkfs'); +return "$workdir/image.jffs2"; +?> diff --git a/backend/bundlers/jffs2/mkfs.php b/backend/bundlers/jffs2/mkfs.php new file mode 100644 index 0000000..bf9e702 --- /dev/null +++ b/backend/bundlers/jffs2/mkfs.php @@ -0,0 +1,3 @@ +<?php +execute_command('Create JFFS2 image', "mkfs.jffs2 -x lzo -n -e 0x20000 -l -p -r '$imagedir' -o '$workdir/image.jffs2'"); +?> diff --git a/backend/bundlers/livecd.php b/backend/bundlers/livecd.php deleted file mode 100644 index 740c72b..0000000 --- a/backend/bundlers/livecd.php +++ /dev/null @@ -1,9 +0,0 @@ -<?php -function bundle_livecd($I, $W, &$opts) { - emerge('app-misc/livecd-toold', 'Install LiveCD utilities'); - require(dirname(__FILE__).'/cd.inc'); - execute_command('Compress finished image to squashfs', "mksquashfs '$I' '$W/image.squashfs' -noappend -info"); - execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' replace /image.squashfs '$W/image.squashfs' write '$W/image.iso'"); - return "$W/image.iso"; -} -?> diff --git a/backend/bundlers/livecd/assemble.php b/backend/bundlers/livecd/assemble.php new file mode 100644 index 0000000..0371103 --- /dev/null +++ b/backend/bundlers/livecd/assemble.php @@ -0,0 +1,4 @@ +<?php +execute_command('Compress finished image to squashfs', "mksquashfs '$imagedir' '$workdir/image.squashfs' -noappend -info"); +execute_command('Create ISO image', LIB."/bkisofs-cli '$minimaliso' replace /image.squashfs '$workdir/image.squashfs' write '$workdir/image.iso'"); +?> diff --git a/backend/bundlers/livecd/bundle.php b/backend/bundlers/livecd/bundle.php new file mode 100644 index 0000000..465404b --- /dev/null +++ b/backend/bundlers/livecd/bundle.php @@ -0,0 +1,7 @@ +<?php +require(dirname(__FILE__).'/cd-head.php'); +add_step('livecd-tools'); +add_step('cd-common'); +add_step('assemble'); +return "$workdir/image.iso"; +?> diff --git a/backend/bundlers/livecd/cd-common.php b/backend/bundlers/livecd/cd-common.php new file mode 120000 index 0000000..70f9287 --- /dev/null +++ b/backend/bundlers/livecd/cd-common.php @@ -0,0 +1 @@ +../cd-common.php
\ No newline at end of file diff --git a/backend/bundlers/livecd/cd-head.php b/backend/bundlers/livecd/cd-head.php new file mode 120000 index 0000000..8a38a03 --- /dev/null +++ b/backend/bundlers/livecd/cd-head.php @@ -0,0 +1 @@ +../cd-head.php
\ No newline at end of file diff --git a/backend/bundlers/livecd/livecd-tools.php b/backend/bundlers/livecd/livecd-tools.php new file mode 100644 index 0000000..13132a3 --- /dev/null +++ b/backend/bundlers/livecd/livecd-tools.php @@ -0,0 +1,3 @@ +<?php +emerge('app-misc/livecd-tools', 'Install LiveCD utilities'); +?> diff --git a/backend/bundlers/tbz2.php b/backend/bundlers/tbz2.php deleted file mode 100644 index ee69825..0000000 --- a/backend/bundlers/tbz2.php +++ /dev/null @@ -1,6 +0,0 @@ -<?php -function bundle_tbz2($I, $W) { - execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$I' ."); - return "$W/image.tar.bz2"; -} -?> diff --git a/backend/bundlers/tbz2/bundle.php b/backend/bundlers/tbz2/bundle.php new file mode 100644 index 0000000..6024c3e --- /dev/null +++ b/backend/bundlers/tbz2/bundle.php @@ -0,0 +1,4 @@ +<?php +add_step('tar'); +return "$workdir/image.tar.bz2"; +?> diff --git a/backend/bundlers/tbz2/tar.php b/backend/bundlers/tbz2/tar.php new file mode 100644 index 0000000..2aead89 --- /dev/null +++ b/backend/bundlers/tbz2/tar.php @@ -0,0 +1,3 @@ +<?php +execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$workdir/image.tar.bz2' -C '$imagedir' ."); +?> diff --git a/backend/bundlers/tgz.php b/backend/bundlers/tgz.php deleted file mode 100644 index 3b604d3..0000000 --- a/backend/bundlers/tgz.php +++ /dev/null @@ -1,6 +0,0 @@ -<?php -function bundle_tgz($I, $W) { - execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$I' ."); - return "$W/image.tar.gz"; -} -?> diff --git a/backend/bundlers/tgz/bundle.php b/backend/bundlers/tgz/bundle.php new file mode 100644 index 0000000..b283ee7 --- /dev/null +++ b/backend/bundlers/tgz/bundle.php @@ -0,0 +1,4 @@ +<?php +add_step('tar'); +return "$workdir/image.tar.gz"; +?> diff --git a/backend/bundlers/tgz/tar.php b/backend/bundlers/tgz/tar.php new file mode 100644 index 0000000..09fbf0c --- /dev/null +++ b/backend/bundlers/tgz/tar.php @@ -0,0 +1,3 @@ +<?php +execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$workdir/image.tar.gz' -C '$imagedir' ."); +?> diff --git a/backend/functions/api.php b/backend/functions/api.php index d8d648c..cffbf65 100644 --- a/backend/functions/api.php +++ b/backend/functions/api.php @@ -57,7 +57,6 @@ function log_msg($msg, $nl=true) { if (!isset($task)) { start_internal_task($msg); debug('log_msg creating task... this is bad'); - return; } $msg.=$nl?"\n":''; debug($msg); diff --git a/backend/functions/signals.php b/backend/functions/signals.php index 70231c7..62f5fe3 100644 --- a/backend/functions/signals.php +++ b/backend/functions/signals.php @@ -5,6 +5,7 @@ function handle_signal($sig=null) { unlink($pidfile); if (isset($build)) { if (isset($task)) { + log_msg("\nGot signal $sig\n"); $task->finish=time(); $task->exit=-$sig; $task->write(); @@ -13,10 +14,7 @@ function handle_signal($sig=null) { debug('$task not set'); } $build->finish=time(); - $build->status='got_signal'; - $build->details=$sig; $build->write(); - debug("build $build->id given status $build->status"); } debug("\nGot signal $sig - exiting"); exit; diff --git a/backend/include/includes.php b/backend/include/includes.php deleted file mode 100644 index d20c3ac..0000000 --- a/backend/include/includes.php +++ /dev/null @@ -1,4 +0,0 @@ -<?php -foreach (glob(BACKEND.'/bundlers/*.php') as $file) - require_once($file); -?> diff --git a/backend/modules/gentoo_portage/base-system.php b/backend/modules/gentoo_portage/base-system.php index c99447e..0c730e9 100644 --- a/backend/modules/gentoo_portage/base-system.php +++ b/backend/modules/gentoo_portage/base-system.php @@ -1,6 +1,6 @@ <?php $file=CACHE.'/stage3/'.$profile->stage3; -execute_command('Unpack base system', "tar -xvjpf '$file' -C '$I'"); +execute_command('Unpack base system', "tar -xvjpf '$file' -C '$imagedir'"); if ($opts['basesystem'] == 'user_prune' && $opts['prunepkgs']) { emerge($opts['prunepkgs'], 'Prune base system packages', '-C'); } elseif ($opts['basesystem'] == 'auto_prune') { diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php index 781aaef..2d2f32f 100644 --- a/backend/modules/gentoo_portage/build.php +++ b/backend/modules/gentoo_portage/build.php @@ -2,9 +2,9 @@ require_once(dirname(__FILE__).'/packages.php'); // __DIR__ 5.3.0 $profile=new sql_gentoo_profile($opts['profile']); $headers=$profile->get_headers(); -$I="$workdir/image"; -$C="$workdir/config_root"; -$S['prtg_cfgrt']=array('PORTAGE_CONFIGROOT' => $C); +$imagedir="$workdir/image"; +$confdir="$workdir/config_root"; +$S['prtg_cfgrt']=array('PORTAGE_CONFIGROOT' => $confdir); add_step('setup'); add_step('base-system'); $extra=explode(' ', $opts['options']); @@ -22,5 +22,5 @@ if (strlen($opts['pkgsets'])) add_step('pkgsets'); if (strlen($opts['install_packages'])) add_step('misc-pkgs'); -return $I; +return $imagedir; ?> diff --git a/backend/modules/gentoo_portage/hostname.php b/backend/modules/gentoo_portage/hostname.php index 11c7d20..e710560 100644 --- a/backend/modules/gentoo_portage/hostname.php +++ b/backend/modules/gentoo_portage/hostname.php @@ -1,4 +1,4 @@ <?php $hn=str_replace('@', '\@', $opts['hostname']); -execute_command('Set hostname in /etc/conf.d/hostname', "sed -i -r 's@^#?HOSTNAME=.*$@HOSTNAME=\"$hn\"@' '$I/etc/conf.d/hostname'"); +execute_command('Set hostname in /etc/conf.d/hostname', "sed -i -r 's@^#?HOSTNAME=.*$@HOSTNAME=\"$hn\"@' '$imagedir/etc/conf.d/hostname'"); ?> diff --git a/backend/modules/gentoo_portage/init.d.php b/backend/modules/gentoo_portage/init.d.php index deef101..9ab61e6 100644 --- a/backend/modules/gentoo_portage/init.d.php +++ b/backend/modules/gentoo_portage/init.d.php @@ -2,7 +2,7 @@ start_internal_task('Remove unwanted init scripts'); foreach (explode(' ', $opts['pruneinit']) as $init) { list($name, $runlevel)=explode(':', $init, 2); - log_status("$name ($runlevel)", unlink("$I/etc/runlevels/$runlevel/$name")); + log_status("$name ($runlevel)", unlink("$imagedir/etc/runlevels/$runlevel/$name")); } end_internal_task(0); ?> diff --git a/backend/modules/gentoo_portage/portage.php b/backend/modules/gentoo_portage/portage.php index a1eb3ed..bcfa7d7 100644 --- a/backend/modules/gentoo_portage/portage.php +++ b/backend/modules/gentoo_portage/portage.php @@ -3,14 +3,14 @@ $file=glob(CACHE.'/portage/*.tar.bz2'); if ($file) { $file=array_reverse($file); $file=$file[0]; - execute_command('Unpack portage snapshot', "tar -xvjpf '$file' -C '$I/usr'"); + execute_command('Unpack portage snapshot', "tar -xvjpf '$file' -C '$imagedir/usr'"); } else { start_internal_task('Copy local portage tree to image'); foreach (glob("{$S['conf']['portdir']}/*") as $from) { $file=substr($from, strlen($S['conf']['portdir'])+1); // Skip distfiles, binary packages, and potential overlay directories if ($file == 'distfiles' || $file == 'packages' || $file == 'local' || $file == 'overlay') continue; - $cmd="cp -av -t '$I/usr/portage/' '$from'"; + $cmd="cp -av -t '$imagedir/usr/portage/' '$from'"; error_get_last(); @shell_exec($cmd); end_internal_task((int)(bool)error_get_last()); @@ -33,8 +33,8 @@ foreach ($makeconf as $name => $val) $contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n"; unset($makeconf); log_msg("/etc/make.conf:\n$contents"); -log_status('Writing /etc/make.conf', file_put_contents("$I/etc/make.conf", $contents)); -log_status('Remove previous make.profile', unlink("$I/etc/make.profile"), false); -log_status("Symlink make.profile -> /usr/portage/profiles/{$headers['profile']}", symlink("/usr/portage/profiles/{$headers['profile']}", "$I/etc/make.profile")); +log_status('Writing /etc/make.conf', file_put_contents("$imagedir/etc/make.conf", $contents)); +log_status('Remove previous make.profile', unlink("$imagedir/etc/make.profile"), false); +log_status("Symlink make.profile -> /usr/portage/profiles/{$headers['profile']}", symlink("/usr/portage/profiles/{$headers['profile']}", "$imagedir/etc/make.profile")); end_internal_task(0); ?> diff --git a/backend/modules/gentoo_portage/setup.php b/backend/modules/gentoo_portage/setup.php index d96eb51..93dbad8 100644 --- a/backend/modules/gentoo_portage/setup.php +++ b/backend/modules/gentoo_portage/setup.php @@ -1,12 +1,12 @@ <?php start_internal_task('Create portage target environment'); -makedirs($I, $C, "$workdir/log", "$workdir/tmp"); -log_status("Make symlink $C/etc -> .", symlink('.', "$C/etc")); +makedirs($imagedir, $confdir, "$workdir/log", "$workdir/tmp"); +log_status("Make symlink $confdir/etc -> .", symlink('.', "$confdir/etc")); $makeconf=array( 'pkgdir' => $S['conf']['pkgdir_root'].'/'.$profile->pkgdir, 'chost' => $headers['chost'], 'accept_keywords' => $headers['accept_keywords'], - 'root' => $I, + 'root' => $imagedir, 'port_logdir' => "$workdir/log", 'emerge_log_dir' => "$workdir/log", 'portage_tmpdir' => "$workdir/tmp" @@ -15,8 +15,8 @@ $contents=''; foreach ($makeconf as $name => $val) $contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n"; unset($makeconf); -log_status("Write $C/etc/make.conf", file_put_contents("$C/etc/make.conf", $contents)); +log_status("Write $confdir/etc/make.conf", file_put_contents("$confdir/etc/make.conf", $contents)); unset($contents); -log_status('Make make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile')); +log_status('Make make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $confdir.'/etc/make.profile')); end_internal_task(0); ?> diff --git a/backend/modules/gentoo_portage/timezone.php b/backend/modules/gentoo_portage/timezone.php index e5cc196..b5d4e6e 100644 --- a/backend/modules/gentoo_portage/timezone.php +++ b/backend/modules/gentoo_portage/timezone.php @@ -1,9 +1,9 @@ <?php -if (glob("$I/var/db/pkg/sys-apps/baselayout-2*", GLOB_ONLYDIR)) - file_put_contents("$I/etc/timezone", "{$opts['timezone']}\n"); +if (glob("$imagedir/var/db/pkg/sys-apps/baselayout-2*", GLOB_ONLYDIR)) + file_put_contents("$imagedir/etc/timezone", "{$opts['timezone']}\n"); else { $tz=str_replace('@', '\@', $opts['timezone']); - execute_command('Set timezone in /etc/conf.d/clock', "sed -i -r 's@^#?TIMEZONE=.*$@TIMEZONE=\"$tz\"@' '$I/etc/conf.d/clock'"); + execute_command('Set timezone in /etc/conf.d/clock', "sed -i -r 's@^#?TIMEZONE=.*$@TIMEZONE=\"$tz\"@' '$imagedir/etc/conf.d/clock'"); } emerge('sys-libs/timezone-data', null, '-K -1 --root-deps=rdeps'); ?> diff --git a/frontend/pages/builds/delete.php b/frontend/pages/builds/delete.php index 6fc6ae3..b92d28b 100644 --- a/frontend/pages/builds/delete.php +++ b/frontend/pages/builds/delete.php @@ -13,11 +13,9 @@ function body_builds_delete(&$S) { case 'queued': if (isset($S['build']->backend)) die(print_warning('Oops', 'You tried to delete this build just as it was about to start being built. Please try to cancel it in a moment.')); - case 'upload_failed': case 'canceled': - case 'failed': + case 'queued': case 'complete': - case 'got_signal': $S['build']->delete(); echo print_success('Build deleted.'); break; @@ -27,6 +25,7 @@ function body_builds_delete(&$S) { case 'uploading': case 'building': default: + $S['build']->failed='false'; // Otherwise doesn't get noticed by backend $S['build']->status='cancel'; $S['build']->write(); echo print_success('Build queued for cancellation.'); diff --git a/frontend/pages/logout.php b/frontend/pages/logout.php index 71f8c11..a60ee2d 100644 --- a/frontend/pages/logout.php +++ b/frontend/pages/logout.php @@ -4,6 +4,7 @@ function init_logout(&$S) { $S['session']->delete(); } setcookie($S['conf']['cookiename'], '', 1, $S['cookie_dir'], '', false, true); + unset($S['user']); if (isset($_REQUEST['go'])) { header('Location: '.url($_REQUEST['go'])); } diff --git a/frontend/routing.csv b/frontend/routing.csv index 3151d27..7183a43 100644 --- a/frontend/routing.csv +++ b/frontend/routing.csv @@ -19,6 +19,7 @@ build/([a-z0-9]{6})/([0-9]+)/([0-9]+) builds/task build task page build/([a-zA-Z0-9]{6})/download builds/download build build/([a-zA-Z0-9]{6})/history builds/history build build/([a-zA-Z0-9]{6})/delete builds/delete build +build/([a-zA-Z0-9]{6})/cancel builds/delete build #build/([a-z0-9]{6})/live builds/live build # Configurations create configurations/wizard diff --git a/gentoo-steps b/gentoo-steps index d17d61b..e0e4fe6 100644 --- a/gentoo-steps +++ b/gentoo-steps @@ -2,7 +2,7 @@ X*Set the date Install a stage tarball Install a portage snapshot (from host sytem - support tarball snapshots?) -*Set up make.conf +Set up make.conf ?*Mirror selection Profile selection *Locale selection (selected from /usr/share/i18n/SUPPORTED) diff --git a/shared/classes/build.php b/shared/classes/build.php index 64730fe..10b2043 100644 --- a/shared/classes/build.php +++ b/shared/classes/build.php @@ -36,18 +36,25 @@ class sql_build extends conf_build_common { ), 'status' => array ( 'type' => 'ENUM', - 'length' => '\'queued\',\'uploading\',\'cancel\',\'complete\',\'upload_failed\',\'canceled\',\'failed\',\'got_signal\',\'building\',\'bundling\'', + 'length' => '\'queued\',\'uploading\',\'cancel\',\'complete\',\'canceled\',\'building\',\'bundling\'', 'not_null' => true ), - 'details' => array ( + 'build_step' => array ( 'type' => 'TINYINT', - 'length' => 4 + 'length' => 4, + 'unsigned' => true ), - 'build_step' => array ( + 'num_steps' => array ( 'type' => 'TINYINT', - 'length' => 3, + 'length' => 4, 'unsigned' => true ), + 'failed' => array ( + 'type' => 'ENUM', + 'length' => '\'false\',\'true\'', + 'not_null' => true, + 'default' => 'false' + ), 'ctime' => array ( 'type' => 'INT', 'length' => 10, @@ -71,49 +78,52 @@ class sql_build extends conf_build_common { $perms=$this->visibility == 'public' || owner_or_admin($this->id); $html='<div class="build"><span class="name">'.(isset($this->name) && strlen($this->name)?htmlentities($this->name):'Unnamed Build').'</span> '; $links=array(); - if ($this->status == 'queued') { + switch ($this->status) { + case 'queued': $total=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued"')->fetch(PDO::FETCH_COLUMN); $num=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued" AND `ctime` <= '.$this->ctime)->fetch(PDO::FETCH_COLUMN); $html.="<span class=\"status queued\">[queued ($num/$total)]</span>"; - } elseif ($this->status == 'uploading') { + break; + case 'uploading': $html.='<span class="status successful">[uploading]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'cancel') { + break; + case 'cancel': $html.='<span class="status queued">[pending cancellation]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'building') { - // TODO stage x/y - $html.='<span class="status building">[building ('.$this->build_step.'/'.$this->details.')]</span>'; + break; + case 'building': + case 'bundling': + $html.='<span class="status building">['.$this->status.' ('.$this->build_step.'/'.$this->num_steps.')]</span>'; if ($perms) { //$links['Watch']="build/$this->id/live"; $links['Build Log']="build/$this->id"; } - } elseif ($this->status == 'complete') { - $r=query('SELECT COUNT(*) as `count`, MAX(`time`) as `time` FROM `downloads` WHERE `build`="'.$this->id.'"')->fetch(PDO::FETCH_ASSOC); - $d=($perms && $r['count']?'<a href="'.url("build/$this->id/history").'">':'').$r['count'].' download'.($r['count'] != 1?'s':'').($r['count']?($perms?'</a>':'').'<br/><span class="time">(last at '.date($format, $r['time']).')</span>':''); + break; + case 'complete': + $url="build/$this->id/history"; + if ($perms && $S['request'] != $url) { + $r=query('SELECT COUNT(*) as `count`, MAX(`time`) as `time` FROM `downloads` WHERE `build`="'.$this->id.'"')->fetch(PDO::FETCH_ASSOC); + $d=($r['count']?'<a href="'.url($url).'">':'').$r['count'].' download'.($r['count'] != 1?'s':'').($r['count']?($perms?'</a>':'').'<br/><span class="time">(last at '.date($format, $r['time']).')</span>':''); + } else + $d=''; $html.='<span class="downloads">'.$d.'</span><span class="status successful">[successful]</span>'; $links['Download image']="build/$this->id/download"; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'upload_failed') { - $html.='<span class="status failed">[upload failed]</span>'; - if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'failed') { - $html.='<span class="status failed">[failed after step '.$this->build_step.']</span>'; - if ($perms) { - //$links['View output of failed command']="build/$this->id/failure"; - $links['Build log']="build/$this->id"; - } - } elseif ($this->status == 'canceled') { + break; + case 'canceled': $html.='<span class="status failed">[canceled]</span>'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'got_signal') { - $html.='<span class="status failed">[failed: got signal '.$this->details.' after step '.$this->build_step.']</span>'; - if ($perms) $links['Build log']="build/$this->id"; - } else { + break; + default: $html.='<span class="status failed">[UNKNOWN STATUS: '.$this->status.']</span>'; } - if ($perms && ($this->status == 'upload_failed' || $this->status == 'failed' || $this->status == 'canceled' || $this->status == 'queued' || $this->status == 'complete' || $this->status == 'got_signal')) - $links['Delete']="build/$this->id/delete"; + if ($perms) { + if ($this->status == 'canceled' || $this->status == 'queued' || $this->status == 'complete' || $this->failed == 'true') + $links['Delete']="build/$this->id/delete"; + elseif ($this->status != 'cancel') + $links['Cancel']="build/$this->id/cancel"; + } if ($links) { foreach ($links as $label => $url) { if ($S['request'] == $url) @@ -141,13 +151,6 @@ class sql_build extends conf_build_common { $html.='</div>'; return $html; } - public function queued_tasks() { - global $S; - static $cache; - if (!isset($cache)) - $cache=query('SELECT COUNT(`order`) FROM `tasks` WHERE `start` IS NULL AND `build`="'.$this->id.'"')->fetch(PDO::FETCH_COLUMN); - return $cache; - } public function delete() { global $S; query('DELETE FROM `buildlogs` WHERE `build`="'.$this->id.'"'); @@ -161,43 +164,108 @@ class sql_build extends conf_build_common { public function build($workdir) { global $S; try { - if (!is_dir($workdir)) - log_status('Create work directory '.$workdir, mkdir($workdir, 0700)); $opts=$this->get_opts(); $S['build_steps']=array(); if (!is_readable(BACKEND."/modules/$this->module/build.php")) throw_exception("No build script for module $this->module"); - $dir=require(BACKEND."/modules/$this->module/build.php"); + $imagedir=require(BACKEND."/modules/$this->module/build.php"); switch ($this->status) { case 'queued': - $this->build_step=0; - case 'got_signal': - case 'failed': $this->status='building'; - $this->details=count($S['build_steps']); + $this->build_step=0; + $this->num_steps=count($S['build_steps']); $this->write(); case 'building': $step=$this->build_step; break; - case 'uploading': - case 'upload_failed': - case 'cancel': - case 'bundling': default: - $step=count($S['build_steps']); + return $imagedir; } + if (!is_dir($workdir)) + log_status('Create work directory '.$workdir, mkdir($workdir, 0700)); while ($step < count($S['build_steps'])) { require(BACKEND."/modules/$this->module/{$S['build_steps'][$step]}.php"); $step++; $this->build_step=$step; $this->write(); + if ($this->is_canceled()) return false; } - return $dir; + return $imagedir; } catch(Exception $e) { log_msg('Caught exception: '.$e->getMessage()); end_internal_task(1); + $this->failed(); return false; } } + public function bundle($imagedir, $workdir) { + global $S; + try { + $opts=$this->get_opts(); + $bundler=$opts['bundler']; + if (!is_readable(BACKEND."/bundlers/$bundler/bundle.php")) + throw_exception("No bundle script for bundler $bundler"); + $S['build_steps']=array(); + $file=require(BACKEND."/bundlers/$bundler/bundle.php"); + switch($this->status) { + case 'building': + $this->status='bundling'; + $this->build_step=0; + $this->num_steps=count($S['build_steps']); + $this->write(); + case 'bundling': + $step=$this->build_step; + break; + default: + return $file; + } + print_r($S['build_steps']); + while ($step < count($S['build_steps'])) { + require(BACKEND."/bundlers/$bundler/{$S['build_steps'][$step]}.php"); + $step++; + $this->build_step=$step; + $this->write(); + if ($this->is_canceled()) return false; + } + return $file; + } catch (Exception $e) { + log_msg('Caught exception: '.$e->getMessage()); + end_internal_task(1); + $this->failed(); + return false; + } + } + public function upload($file) { + $this->status='uploading'; + $this->write(); + $key=randstring(30); + $this->set_opt('uploadkey', $key); + $c=curl_init(url('backend/upload_image')); + curl_setopt($c, CURLOPT_POST, 1); + curl_setopt($c, CURLOPT_POSTFIELDS, array( + 'build' => $this->id, + 'key' => $key, + 'file' => "@$file" + )); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + $result=curl_exec($c); + $result($result !== false && strpos($result, 'Upload successful') !== false); + if ($result) { + $this->status='complete'; + $this->finish=time(); + $this->write(); + } else + $this->failed(); + return $result; + } + private function failed() { + $this->failed='true'; + $this->finish=time(); + $this->write(); + } + public function is_canceled() { + $this->load(); + return ($this->status == 'cancel'); + } } ?> diff --git a/shared/classes/task.php b/shared/classes/task.php index d763107..e249e0f 100644 --- a/shared/classes/task.php +++ b/shared/classes/task.php @@ -51,30 +51,25 @@ class sql_task extends sql_row_obj { global $S; $link="build/$this->build/$this->order"; $html='<div class="task"><div class="description">'.htmlentities($this->description).'</div><div class="info">'.($S['request'] == $link || strpos($S['request'], "$link/") === 0?'':'[<a href="'.url($link).'">log</a>] ').($this->command?'<span class="command"'.($this->env?' title="'.htmlentities(str_replace("\n", '; ', $this->env)).'"':'').'>'.htmlentities($this->command).'</span> ':''); - if (isset($this->start)) { - if (isset($this->finish)) { - $html.='<span class="status '; - if ($this->exit === '0') { - $html.='successful">[successful'; - } else { - $html.='failed">['; - if (isset($this->exit)) { - if ($this->exit > 0) - $html.='exit status '.$this->exit; - elseif ($this->exit == -128) - $html.='got unknown signal'; - else - $html.='got signal '.-$this->exit; - } else - $html.='failed to execute'; - } - $html.=']</span> <span class="time">Finished in <span class="time">'.display_time($this->finish-$this->start).'</span></span>'; + if (isset($this->finish)) { + $html.='<span class="status '; + if ($this->exit === '0') { + $html.='successful">[successful'; } else { - $html.='<span class="status running">[running]</span> <span class="time">Running for <span class="time">'.display_time(time()-$this->start).'</span></span>'; + $html.='failed">['; + if (isset($this->exit)) { + if ($this->exit > 0) + $html.='exit status '.$this->exit; + elseif ($this->exit == -128) + $html.='got unknown signal'; + else + $html.='got signal '.-$this->exit; + } else + $html.='failed to execute'; } + $html.=']</span> <span class="time">Finished in <span class="time">'.display_time($this->finish-$this->start).'</span></span>'; } else { - $num=query('SELECT COUNT(*) FROM `tasks` WHERE `builds`="'.$this->build.'" AND `start` IS NULL AND `order` <= '.$this->order)->fetch(PDO::FETCH_ASSOC); - $html.="<span class=\"status queued\">[queued $num/".$build->queued_tasks()."]</span>"; + $html.='<span class="status running">[running]</span> <span class="time">Running for <span class="time">'.display_time(time()-$this->start).'</span></span>'; } $html.='</div></div>'; return $html; diff --git a/shared/config.php b/shared/config.php index 9270d2d..8a93454 100644 --- a/shared/config.php +++ b/shared/config.php @@ -8,14 +8,13 @@ $sqlpass='socpassword'; // MySQL password $sqldb='soc'; // MySQL database $debug=true; // Whether to print debugging information // $modules='All non-hidden dirs in frontend/modules'; // Space-separated list of modules to offer the user -// $bundlers='All non-hidden <bundler>.php files in backend/bundlers'; // Space-separated list of bundlers to offer the user +// $bundlers='All non-hidden dirs in backend/bundlers' that contain a bundle.php; // Space-separated list of bundlers to offer the user // $cookiename='ingenueid'; // Name of the cookie to send for keeping sessions // $sessionlength=1814400; // Time in seconds before sessions are purged // $mod_rewrite=true; // Use mod_rewrite for pretty URLs // $timezone_root='/usr/share/zoneinfo'; // Directory to search for timezone data (sys-libs/timezone-data) $emailfrom='noreply@gentoo.org'; // Used as the From: field in emails $check_email_dns=true; // Use DNS to check the domain of submitted emails for validity -// $split_setup=true; // Whether the frontend and backend are running on different hosts // Frontend options: // $registration=false; // Whether users can create new accounts without an invite // $invite='admin'; // Who can use the invite function: true or 'user'=users; admin=admins; false=nobody diff --git a/shared/functions/load_config.php b/shared/functions/load_config.php index a212b16..d719fb9 100644 --- a/shared/functions/load_config.php +++ b/shared/functions/load_config.php @@ -3,7 +3,7 @@ function load_config() { require(SHARED.'/include/defaults.php'); require(SHARED.'/config.php'); $modules=explode(' ', $modules); - foreach (explode(' ', 'title url sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite timezone_root emailfrom check_email_dns split_setup registration invite logview_max progressbar_width pkgdir_root emerge_default_opts portdir backend_id') as $var) { + foreach (explode(' ', 'title url sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite timezone_root emailfrom check_email_dns registration invite logview_max progressbar_width pkgdir_root emerge_default_opts portdir backend_id') as $var) { if (isset($$var)) { $GLOBALS['S']['conf'][$var]=$$var; } diff --git a/shared/include/defaults.php b/shared/include/defaults.php index ced0e01..95cbfc2 100644 --- a/shared/include/defaults.php +++ b/shared/include/defaults.php @@ -14,10 +14,8 @@ foreach (glob(FRONTEND.'/modules/*.info') as $module) { } $modules=implode(' ', $modules); $bundlers=array(); -foreach (glob(BACKEND.'/bundlers/*.php') as $bundler) { - $bundler=basename($bundler); - $bundlers[]=substr($bundler, 0, strlen($bundler)-4); -} +foreach (glob(BACKEND.'/bundlers/*/bundle.php') as $bundler) + $bundlers[]=basename(dirname($bundler)); $bundlers=implode(' ', $bundlers); $cookiename='ingenueid'; $sessionlength=1814400; @@ -25,7 +23,6 @@ $mod_rewrite=true; $timezone_root='/usr/share/zoneinfo'; $emailfrom='noreply@noreply.net'; $check_email_dns=false; -$split_setup=true; $registration=false; $invite='admin'; $logview_max=1000; diff --git a/shared/include/includes.php b/shared/include/includes.php index 410177f..8a75b58 100644 --- a/shared/include/includes.php +++ b/shared/include/includes.php @@ -9,9 +9,6 @@ foreach (array('functions', 'classes') as $type) { } } } -$includes=($_SERVER['DOCUMENT_ROOT']?FRONTEND:BACKEND).'/include/includes.php'; -if (is_readable($includes)) - require_once($includes); -unset($dir, $file, $type, $includes); +unset($dir, $file, $type); load_config(); ?> @@ -6,7 +6,6 @@ Add cleanup functions to the frontend and backend (tasks dir in backend containi Allow config viewing for builds, not just configurations Add `flags` column to configurations, builds, use it to implement public and private things Add safe shutdown to backend so it will stop once it gets to the end of a step that can be resumed -Add 'cancel', option to builds, allow deletion of currently running builds (have backend check if canceled before and after each task) Add build->configuration and configuration duplication Consider adding `configuration` col to builds Add map file for liveCD, load it into DB, etc. (currently hardcoded = evil) @@ -18,14 +17,10 @@ Add option to upload a kernel Add option to upload an arbitrary tar.gz/bz2 to be unzipped over the finished image *** Implement selected items from gentoo-steps *** *** Documentation *** -Ponder whether to remove the split-setup option and always upload to the frontend -Make backend not retry indefinitely on failure (`failures` column?) Offer FTP upload Offer SCP upload? Ask someone to add the necessary USE flags to php on tinderbox Add rollback to backend so it can resume after a partial task -Use 'bundling' status on builds, consider adding 'bundling_failed' -Handle 'cancel' status in backend, offer it in frontend -Change builds->display() to use a switch($this->status) +Offer option in frontend to submit a failed build for resume +Change builds->display() to handle `failed` column Confirm that backend stops after success -*** Tidy up backend, move the proper parts into build->build() |