Provide a fullupdate option instead of cleanup

The normal update works incrementally, the full update
can be used in case something went wrong during the
incremental update. It updates all versions, packages
and categories.

Signed-off-by: Max Magorsch <arzano@gentoo.org>

7 files changed, 118 insertions, 22 deletions

diff --git a/bin/cleanup.sh b/bin/fullupdate.sh
index b8d01f1..d71c895 100755
--- a/bin/cleanup.sh
+++ b/bin/fullupdate.sh
@@ -13,7 +13,7 @@ update_repository(){
 
 cleanup_database(){
   cd /mnt/packages-tree/gentoo/ || exit 1
-  /go/src/soko/bin/soko cleanup
+  /go/src/soko/bin/soko fullupdate
 }
 
 
diff --git a/pkg/portage/repository/category.go b/pkg/portage/repository/category.go
index e41d43a..8ea9413 100644
--- a/pkg/portage/repository/category.go
+++ b/pkg/portage/repository/category.go
@@ -28,6 +28,9 @@ func UpdateCategory(path string) {
 	splittedLine := strings.Split(path, "\t")
 
 	if len(splittedLine) != 2 {
+		if len(splittedLine) == 1 && isCategory(path) {
+			updateModifiedCategory(path)
+		}
 		return
 	}
 
diff --git a/pkg/portage/repository/package.go b/pkg/portage/repository/package.go
index e1d22ce..17f55b7 100644
--- a/pkg/portage/repository/package.go
+++ b/pkg/portage/repository/package.go
@@ -28,6 +28,9 @@ func UpdatePackage(path string) {
 	splittedLine := strings.Split(path, "\t")
 
 	if len(splittedLine) != 2 {
+		if len(splittedLine) == 1 && isPackage(path) {
+			updateModifiedPackage(path)
+		}
 		return
 	}
 
diff --git a/pkg/portage/repository/version.go b/pkg/portage/repository/version.go
index 9ed8f38..b752f18 100644
--- a/pkg/portage/repository/version.go
+++ b/pkg/portage/repository/version.go
@@ -26,6 +26,9 @@ func UpdateVersion(path string) {
 	line := strings.Split(path, "\t")
 
 	if len(line) != 2 {
+		if len(line) == 1 && isVersion(path) {
+			updateModifiedVersion(path)
+		}
 		return
 	}
 
diff --git a/pkg/portage/update.go b/pkg/portage/update.go
index 7ecd80a..031c656 100644
--- a/pkg/portage/update.go
+++ b/pkg/portage/update.go
@@ -113,16 +113,17 @@ func updateHistory() {
 }
 
 
-// CleanUp iterates through all ebuild versions in the database and
-// checks whether they are still present in main tree. Normally there
-// should be no version in the database anymore that is not present in
-// the main gentoo tree. However in case this does happen, it does indiciate
-// an error during the update process. In this case the version will be
-// logged and deleted from the database. That is, CleanUp is currently
-// used to a) find errors / outdated data and b) update the outdated data
-// This method will be removed as soon as it shows that there are no
-// errors present.
-func CleanUp() {
+// FullUpdate does - as the name applies - a full update. That is, it
+// iterates through *all* files in the tree and updates their records
+// in the database. Afterwards it is checked whether all records that
+// that are in the database, are present in the main tree. This way
+// files which already got deleted from the main tree, get deleted
+// from the database. All deleted files will be logged.
+// This method is mainly intended for cleaning up the data and finding
+// outdated data, which indicates bugs in the incremental update.
+// Once there is no outdated data found anymore this method may become
+// obsolete.
+func FullUpdate() {
 
 	database.Connect()
 	defer database.DBCon.Close()
@@ -131,8 +132,28 @@ func CleanUp() {
 		log.SetOutput(ioutil.Discard)
 	}
 
-	logger.Info.Println("Start clean up...")
+	logger.Info.Println("Full update up...")
 
+	// Add new entries & update existing
+	logger.Info.Println("Update all present files")
+	for _, path := range utils.AllFiles() {
+		repository.UpdateVersion(path)
+		repository.UpdatePackage(path)
+		repository.UpdateCategory(path)
+	}
+
+	// Delete removed entries
+	logger.Info.Println("Delete removed files from the database")
+	deleteRemovedVersions()
+	deleteRemovedPackages()
+	deleteRemovedCategories()
+
+	logger.Info.Println("Finished update up...")
+}
+
+// deleteRemovedVersions removes all versions from the database
+// that are present in the database but not in the main tree.
+func deleteRemovedVersions(){
 	var versions []*models.Version
 	database.DBCon.Model(&versions).Select()
 
@@ -142,18 +163,62 @@ func CleanUp() {
 
 			logger.Error.Println("Found ebuild version in the database that does not exist at:")
 			logger.Error.Println(path)
-			logger.Error.Println("The ebuild version got already deleted from the tree and should thus not exist in the database anymore:")
-			logger.Error.Println(version.Atom + "-" + version.Version)
 
 			_, err := database.DBCon.Model(version).WherePK().Delete()
 
 			if err != nil {
-				logger.Error.Println("Error deleting version")
-				logger.Error.Println(version.Atom + " - " + version.Version)
+				logger.Error.Println("Error deleting version " + version.Atom + " - " + version.Version)
+				logger.Error.Println(err)
+			}
+		}
+
+	}
+}
+
+// deleteRemovedPackages removes all packages from the database
+// that are present in the database but not in the main tree.
+func deleteRemovedPackages(){
+	var packages []*models.Package
+	database.DBCon.Model(&packages).Select()
+
+	for _, gpackage := range packages {
+		path := config.PortDir() + "/" + gpackage.Atom
+		if !utils.FileExists(path) {
+
+			logger.Error.Println("Found package in the database that does not exist at:")
+			logger.Error.Println(path)
+
+			_, err := database.DBCon.Model(gpackage).WherePK().Delete()
+
+			if err != nil {
+				logger.Error.Println("Error deleting package " + gpackage.Atom)
+				logger.Error.Println(err)
+			}
+		}
+
+	}
+}
+
+// deleteRemovedCategories removes all categories from the database
+// that are present in the database but not in the main tree.
+func deleteRemovedCategories(){
+	var categories []*models.Category
+	database.DBCon.Model(&categories).Select()
+
+	for _, category := range categories {
+		path := config.PortDir() + "/" + category.Name
+		if !utils.FileExists(path) {
+
+			logger.Error.Println("Found category in the database that does not exist at:")
+			logger.Error.Println(path)
+
+			_, err := database.DBCon.Model(category).WherePK().Delete()
+
+			if err != nil {
+				logger.Error.Println("Error deleting category " + category.Name)
 				logger.Error.Println(err)
 			}
 		}
 
 	}
-	logger.Info.Println("Finished clean up...")
 }
diff --git a/pkg/portage/utils/git.go b/pkg/portage/utils/git.go
index 07e3760..0af5429 100644
--- a/pkg/portage/utils/git.go
+++ b/pkg/portage/utils/git.go
@@ -11,6 +11,28 @@ import (
 	"strings"
 )
 
+
+// ChangedFiles returns a list of files that are
+// currently present in the master branch
+func AllFiles() []string {
+	var allFiles []string
+	cmd := exec.Command("git",
+		"ls-tree",
+		"-r", "master",
+		"--name-only")
+
+	cmd.Dir = config.PortDir()
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		logger.Error.Println("ERROR: cmd.Run() failed with:")
+		logger.Error.Println(err)
+		return allFiles
+	}
+
+	allFiles = strings.Split(string(out), "\n")
+	return allFiles
+}
+
 // ChangedFiles returns a list of files that have been changed
 // between the startCommit and the endCommit. The status of the
 // change as well as the path to the file is returned for each file
diff --git a/soko.go b/soko.go
index b0e96f2..46c56bc 100644
--- a/soko.go
+++ b/soko.go
@@ -14,9 +14,9 @@ import (
 
 func printHelp() {
 	fmt.Println("Please specific one of the following options:")
-	fmt.Println("  soko update  -- update the database")
-	fmt.Println("  soko cleanup -- log and update outdated data ")
-	fmt.Println("  soko serve   -- serve the application")
+	fmt.Println("  soko update     -- incrementally update the database")
+	fmt.Println("  soko fullupdate -- update the database ")
+	fmt.Println("  soko serve      -- serve the application")
 }
 
 func isCommand(command string) bool {
@@ -35,8 +35,8 @@ func main() {
 		app.Serve()
 	} else if isCommand("update") {
 		portage.Update()
-	} else if isCommand("cleanup") {
-		portage.CleanUp()
+	} else if isCommand("fullupdate") {
+		portage.FullUpdate()
 	} else {
 		printHelp()
 	}