Skip to content
Snippets Groups Projects
Commit da7fe984 authored by Simon McVittie's avatar Simon McVittie
Browse files

populate-depot: Add an option to exclude redundant files


Regular files (with content) need to be shipped in the depot, but
directories, empty files and symlinks can be created on-demand by
pressure-vessel from the manifest, reducing the metadata burden on
Steampipe.

This might be done by default later, but for now it's just an option.

Resolves: T27914
Signed-off-by: default avatarSimon McVittie <smcv@collabora.com>
parent 6caeacf5
Branches
Tags
1 merge request!47populate-depot: Improve handling of depots containing unpacked files
......@@ -29,6 +29,7 @@ from just-built files or by downloading a previous build.
"""
import argparse
import errno
import gzip
import hashlib
import json
......@@ -423,6 +424,7 @@ class Main:
include_archives: bool = False,
include_sdk: bool = False,
layered: bool = False,
minimize: bool = False,
pressure_vessel: str = 'scout',
runtime: str = 'scout',
source_dir: str = HERE,
......@@ -486,6 +488,7 @@ class Main:
self.images_uri = images_uri
self.include_archives = include_archives
self.layered = layered
self.minimize = minimize
self.pressure_vessel = pressure_vessel
self.source_dir = source_dir
self.ssh_host = ssh_host
......@@ -805,6 +808,10 @@ class Main:
os.path.join(self.cache, runtime.tarball),
dest,
)
if self.minimize:
self.minimize_runtime(dest)
self.ensure_ref(dest)
if runtime.include_sdk:
......@@ -837,6 +844,10 @@ class Main:
os.path.join(self.cache, runtime.sdk_tarball),
dest,
)
if self.minimize:
self.minimize_runtime(dest)
self.ensure_ref(dest)
argv = [
......@@ -1334,6 +1345,38 @@ class Main:
for name in sorted(not_windows_friendly):
writer.write('# {}\n'.format(self.octal_escape(name)))
def minimize_runtime(self, root: str) -> None:
'''
Remove files that pressure-vessel can reconstitute from the manifest.
This is the equivalent of:
find $root/files -type l -delete
find $root/files -empty -delete
Note that this needs to be done before ensure_ref(), otherwise
it will delete files/.ref too.
'''
for (dirpath, dirnames, filenames) in os.walk(
os.path.join(root, 'files'),
topdown=False,
):
for f in filenames + dirnames:
path = os.path.join(dirpath, f)
try:
statinfo = os.lstat(path)
except FileNotFoundError:
continue
if stat.S_ISLNK(statinfo.st_mode) or statinfo.st_size == 0:
os.remove(path)
try:
os.rmdir(dirpath)
except OSError as e:
if e.errno != errno.ENOTEMPTY:
raise
def main() -> None:
logging.basicConfig()
......@@ -1450,6 +1493,21 @@ def main() -> None:
'--layered', default=False, action='store_true',
help='Produce a layered runtime that runs scout on soldier',
)
parser.add_argument(
'--minimize', action='store_true', default=False,
help=(
'Omit empty files, empty directories and symlinks from '
'runtime content, requiring pressure-vessel to fill them in '
'from the mtree manifest'
)
)
parser.add_argument(
'--no-minimize', action='store_false', dest='minimize',
help=(
'Include empty files, empty directories and symlinks in '
'runtime content [default]'
)
)
parser.add_argument(
'--source-dir', default=HERE,
help=(
......
......@@ -154,6 +154,7 @@ rm -fr depots/test-soldier-local
mkdir -p depots/test-soldier-local
python3 ./populate-depot.py \
--depot=depots/test-soldier-local \
--minimize \
--no-include-archives \
--toolmanifest \
--unpack-runtime \
......@@ -219,6 +220,24 @@ if ! [ -f "depots/test-soldier-local/$run_dir/files/.ref" ]; then
exit 1
fi
empties="$(find "depots/test-soldier-local/$run_dir/files" -name .ref -prune -o -empty -print)"
if [ -n "$empties" ]; then
echo "# Files not minimized:"
echo "$empties" | sed -e 's/^/# /'
echo "Bail out! Files not minimized"
exit 1
fi
symlinks="$(find "depots/test-soldier-local/$run_dir/files" -type l)"
if [ -n "$symlinks" ]; then
echo "# Symlinks not minimized:"
echo "$symlinks" | sed -e 's/^/# /'
echo "Bail out! Symlinks not minimized"
exit 1
fi
echo "ok 3 - soldier, running from local builds"
rm -fr depots/test-soldier-unversioned
......@@ -226,6 +245,7 @@ mkdir -p depots/test-soldier-unversioned
python3 ./populate-depot.py \
--depot=depots/test-soldier-unversioned \
--no-include-archives \
--no-minimize \
--toolmanifest \
--unpack-runtime \
--no-versioned-directories \
......@@ -277,6 +297,24 @@ for dir in depots/test-soldier-unversioned/soldier*; do
fi
done
empties="$(find "depots/test-soldier-unversioned/soldier/files" -name .ref -prune -o -empty | sed -e 's/^/# /')"
echo "# Empty files/dirs in runtime:"
echo "$empties"
if [ -z "$empties" ]; then
echo "Bail out! Runtime should include empty files"
exit 1
fi
symlinks="$(find "depots/test-soldier-unversioned/soldier/files" -type l | sed -e 's/^/# /')"
echo "# Symlinks in runtime:"
echo "$symlinks"
if [ -z "$symlinks" ]; then
echo "Bail out! Runtime should include symlinks"
exit 1
fi
echo "ok 4 - soldier, running from unpacked directory without version"
# vim:set sw=4 sts=4 et:
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment