1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
|
package cmd
import (
"fmt"
"sort"
"strings"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/utils"
"github.com/smira/commander"
)
// aptly db cleanup
func aptlyDBCleanup(cmd *commander.Command, args []string) error {
var err error
if len(args) != 0 {
cmd.Usage()
return commander.ErrCommandError
}
verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
collectionFactory := context.NewCollectionFactory()
// collect information about references packages...
existingPackageRefs := deb.NewPackageRefList()
// used only in verbose mode to report package use source
packageRefSources := map[string][]string{}
context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
if verbose {
context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
}
err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
}
e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
if e != nil {
return e
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
if verbose {
description := fmt.Sprintf("mirror %s", repo.Name)
_ = repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
collectionFactory.Flush()
if verbose {
context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
}
err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
}
e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
if e != nil {
return e
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
if verbose {
description := fmt.Sprintf("local repo %s", repo.Name)
_ = repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
collectionFactory.Flush()
if verbose {
context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
}
err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
}
e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
if e != nil {
return e
}
existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
if verbose {
description := fmt.Sprintf("snapshot %s", snapshot.Name)
_ = snapshot.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
return nil
})
if err != nil {
return err
}
collectionFactory.Flush()
if verbose {
context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
}
err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
}
if published.SourceKind != deb.SourceLocalRepo {
return nil
}
e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
if e != nil {
return e
}
for _, component := range published.Components() {
existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
if verbose {
description := fmt.Sprintf("published repository %s:%s/%s component %s",
published.Storage, published.Prefix, published.Distribution, component)
_ = published.RefList(component).ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
collectionFactory.Flush()
// ... and compare it to the list of all packages
context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
toDelete := allPackageRefs.Subtract(existingPackageRefs)
// delete packages that are no longer referenced
context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
// database can't err as collection factory already constructed
db, _ := context.Database()
if toDelete.Len() > 0 {
if verbose {
context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
err = toDelete.ForEach(func(ref []byte) error {
context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
return nil
})
if err != nil {
return err
}
}
if !dryRun {
batch := db.CreateBatch()
err = toDelete.ForEach(func(ref []byte) error {
return collectionFactory.PackageCollection().DeleteByKey(ref, batch)
})
if err != nil {
return fmt.Errorf("unable to delete by key: %s", err)
}
err = batch.Write()
if err != nil {
return fmt.Errorf("unable to write to DB: %s", err)
}
} else {
context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
}
}
collectionFactory.Flush()
// now, build a list of files that should be present in Repository (package pool)
context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
referencedFiles := make([]string, 0, existingPackageRefs.Len())
context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
err = existingPackageRefs.ForEach(func(key []byte) error {
pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
if err2 != nil {
tail := ""
if verbose {
tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
}
if dryRun {
context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
string(key), err2, tail)
return nil
}
return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
}
paths, err2 := pkg.FilepathList(context.PackagePool())
if err2 != nil {
return err2
}
referencedFiles = append(referencedFiles, paths...)
context.Progress().AddBar(1)
return nil
})
if err != nil {
return err
}
sort.Strings(referencedFiles)
context.Progress().ShutdownBar()
// build a list of files in the package pool
context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
existingFiles, err := context.PackagePool().FilepathList(context.Progress())
if err != nil {
return fmt.Errorf("unable to collect file paths: %s", err)
}
// find files which are in the pool but not referenced by packages
filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
// delete files that are no longer referenced
context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
if len(filesToDelete) > 0 {
if verbose {
context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
for _, file := range filesToDelete {
context.Progress().ColoredPrintf(" - @{r}%s@|", file)
}
}
if !dryRun {
context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
var size, totalSize int64
for _, file := range filesToDelete {
size, err = context.PackagePool().Remove(file)
if err != nil {
return err
}
context.Progress().AddBar(1)
totalSize += size
}
context.Progress().ShutdownBar()
context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
} else {
context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
}
}
if !dryRun {
context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
err = db.CompactDB()
} else {
context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
}
return err
}
func makeCmdDBCleanup() *commander.Command {
cmd := &commander.Command{
Run: aptlyDBCleanup,
UsageLine: "cleanup",
Short: "cleanup DB and package pool",
Long: `
Database cleanup removes information about unreferenced packages and removes
files in the package pool that aren't used by packages anymore
Example:
$ aptly db cleanup
`,
}
cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
cmd.Flag.Bool("dry-run", false, "don't delete anything")
return cmd
}
|