Skip to content

Commit 67c8447

Browse files
dusty-nvdusty-nv
committed
updated jetson_containers
1 parent d42d265 commit 67c8447

File tree

8 files changed

+260
-141
lines changed

8 files changed

+260
-141
lines changed

‎build.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/usr/bin/env bash
22

33
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
#PYTHONPATH="$PYTHONPATH:$ROOT" python3 $ROOT/jetson_containers/build.py "$@"
45

5-
python3 $ROOT/jetson_containers/build.py "$@"
6+
PYTHONPATH="$PYTHONPATH:$ROOT" python3 -m jetson_containers.build "$@"

‎jetson_containers/__init__.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#!/usr/bin/env python3
22

33
from .base import *
4-
from .build import *
4+
from .logging import *
55
from .packages import *
6+
from .container import *
67
from .l4t_version import *
78

89

‎jetson_containers/base.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/env python3
2-
from l4t_version import L4T_VERSION
2+
from .l4t_version import L4T_VERSION
33

44
def get_l4t_base():
55
"""

‎jetson_containers/build.py

100755100644
+54-114
Original file line numberDiff line numberDiff line change
@@ -1,130 +1,70 @@
11
#!/usr/bin/env python3
22
import os
3+
import re
34
import sys
45
import pprint
6+
import fnmatch
57
import argparse
6-
import subprocess
78

8-
from packages import find_package, package_search_dirs, list_packages
9-
from l4t_version import L4T_VERSION
10-
from base import get_l4t_base
9+
from jetson_containers import build_container, find_packages, package_search_dirs, set_log_dir, L4T_VERSION
1110

1211

13-
def unroll_dependencies(packages):
14-
"""
15-
Expand the dependencies in the list of containers to build
16-
"""
17-
if isinstance(packages, str):
18-
packages = [packages]
19-
20-
while True:
21-
packages_org = packages.copy()
22-
23-
for package in packages_org:
24-
for dependency in find_package(package).get('depends', []):
25-
if dependency not in packages:
26-
packages.insert(packages.index(package), dependency)
27-
28-
if len(packages) == len(packages_org):
29-
break
30-
31-
return packages
32-
12+
parser = argparse.ArgumentParser()
13+
14+
parser.add_argument('packages', type=str, nargs='*', default=[], help='packages or containers to build (filterable by wildcards)')
3315

34-
def build_container(name, packages, base=get_l4t_base(), build_flags='', simulate=False):
35-
"""
36-
Build container chain of packages
37-
"""
38-
if isinstance(packages, str):
39-
packages = [packages]
40-
41-
if len(packages) == 0:
42-
raise ValueError("must specify at least one package to build")
43-
44-
# add all dependencies to the build tree
45-
packages = unroll_dependencies(packages)
46-
print('-- Building containers ', packages)
47-
48-
# make sure all packages can be found before building any
49-
for package in packages:
50-
find_package(package)
51-
52-
# assign default container name and tag if needed
53-
if len(name) == 0:
54-
name = packages[-1]
55-
56-
if name.find(':') < 0:
57-
name += f":r{L4T_VERSION}"
58-
59-
# build the chain of containers
60-
container_name = name
61-
62-
for idx, package in enumerate(packages):
63-
# if this isn't the final container in the chain, tag it with the sub-package
64-
if idx < len(packages) - 1:
65-
container_name = f"{name}-{package}"
66-
else:
67-
container_name = name
68-
69-
# build next container
70-
pkg = find_package(package)
71-
72-
if 'dockerfile' in pkg:
73-
cmd = f"sudo docker build --network=host --tag {container_name} \ \n"
74-
cmd += f"--file {os.path.join(pkg['path'], pkg['dockerfile'])} \ \n"
75-
cmd += f"--build-arg BASE_IMAGE={base} \ \n"
76-
77-
if 'build_args' in pkg:
78-
cmd += ''.join([f"--build-arg {key}=\"{value}\" \ \n" for key, value in pkg['build_args'].items()])
79-
80-
if 'build_flags' in pkg:
81-
cmd += pkg['build_flags'] + ' \ \n'
82-
83-
if build_flags:
84-
cmd += build_flags + ' \ \n'
85-
86-
cmd += pkg['path'] #" . "
87-
88-
print(f"-- Building container {container_name}")
89-
print(f"\n{cmd}\n")
90-
else:
91-
cmd = f"sudo docker tag {base} {container_name}"
92-
93-
print(f"-- Tagging container {container_name}")
94-
print(f"{cmd}\n")
95-
96-
if not simulate:
97-
subprocess.run(cmd.replace('\ \n', ''), shell=True, check=True) # remove the line breaks that were added for readability
16+
parser.add_argument('--name', type=str, default='', help='the name of the output container to build')
17+
parser.add_argument('--base', type=str, default='', help='the base container to use at the beginning of the build chain (default: l4t-jetpack)')
18+
parser.add_argument('--multi-stage', action='store_true', help='launch a multi-stage container build by chaining together the packages')
19+
parser.add_argument('--build-flags', type=str, default='', help="extra flags to pass to 'docker build'")
20+
parser.add_argument('--package-dirs', type=str, default='', help='additional package search directories (comma or colon-separated)')
21+
parser.add_argument('--list-packages', action='store_true', help='show the list of packages that were found under the search directories')
22+
parser.add_argument('--show-packages', action='store_true', help='show info about one or more packages (if none are specified, all will be listed')
23+
parser.add_argument('--skip-packages', type=str, default='', help='disable certain packages/containers (filterable by wildcards, comma/colon-separated)')
24+
parser.add_argument('--simulate', action='store_true', help='print out the build commands without actually building the containers')
25+
parser.add_argument('--logs', type=str, default='', help='sets the directory to save container build logs to (default: jetson-containers/logs)')
9826

99-
base = container_name
100-
101-
102-
103-
if __name__ == "__main__":
104-
parser = argparse.ArgumentParser()
105-
106-
parser.add_argument('packages', type=str, nargs='*', default=[], help='packages or configs to build')
107-
108-
parser.add_argument('--name', type=str, default='', help='the name of the output container to build')
109-
parser.add_argument('--base', type=str, default=get_l4t_base(), help='the base container image to use at the beginning of the build chain')
110-
parser.add_argument('--build-flags', type=str, default='', help="extra flags to pass to 'docker build'")
111-
parser.add_argument('--package-dirs', type=str, nargs='+', default=[], help='additional package search directories')
112-
parser.add_argument('--list-packages', action='store_true', help='list information about the found packages and exit')
113-
parser.add_argument('--simulate', action='store_true', help='print out the build commands without actually building the containers')
114-
115-
args = parser.parse_args()
116-
117-
print(args)
118-
print(f"-- L4T_VERSION={L4T_VERSION}")
119-
120-
# add package search directories from the user
27+
args = parser.parse_args()
28+
29+
# split multi-value keyword arguments
30+
args.package_dirs = re.split(',|;|:', args.package_dirs)
31+
args.skip_packages = re.split(',|;|:', args.skip_packages)
32+
33+
print(args)
34+
print(f"-- L4T_VERSION={L4T_VERSION}")
35+
36+
# add package directories
37+
if args.package_dirs:
12138
package_search_dirs(args.package_dirs)
39+
40+
# set logging directories
41+
if args.logs:
42+
set_log_dir(args.logs)
12243

123-
# list packages
44+
# list/show package info
45+
if args.list_packages or args.show_packages:
46+
packages = find_packages(args.packages, skip=args.skip_packages)
47+
12448
if args.list_packages:
125-
pprint.pprint(list_packages(scan=True))
126-
sys.exit(0)
49+
for package in sorted(packages.keys()):
50+
print(package)
51+
52+
if args.show_packages:
53+
pprint.pprint(packages)
12754

128-
# build container chain
55+
sys.exit(0)
56+
57+
# build one multi-stage container from chain of packages
58+
# or launch multiple independent container builds
59+
if args.multi_stage:
12960
build_container(args.name, args.packages, args.base, args.build_flags, args.simulate)
61+
else:
62+
if not args.packages: # build everything (for testing)
63+
args.packages = sorted(find_packages([]).keys())
64+
65+
packages = find_packages(args.packages, skip=args.skip_packages)
66+
print('-- Building containers', list(packages.keys()))
67+
68+
for package in packages: # build multiple containers
69+
build_container(args.name, package, args.base, args.build_flags, args.simulate)
13070

‎jetson_containers/container.py

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env python3
2+
import os
3+
import subprocess
4+
5+
from .packages import find_package, validate_dict
6+
from .l4t_version import L4T_VERSION
7+
from .base import get_l4t_base
8+
from .logging import log_dir
9+
10+
11+
def unroll_dependencies(packages):
12+
"""
13+
Expand the dependencies in the list of containers to build
14+
"""
15+
if isinstance(packages, str):
16+
packages = [packages]
17+
18+
while True:
19+
packages_org = packages.copy()
20+
21+
for package in packages_org:
22+
for dependency in find_package(package).get('depends', []):
23+
if dependency not in packages:
24+
packages.insert(packages.index(package), dependency)
25+
26+
if len(packages) == len(packages_org):
27+
break
28+
29+
return packages
30+
31+
32+
def build_container(name, packages, base=get_l4t_base(), build_flags='', simulate=False):
33+
"""
34+
Build container chain of packages
35+
"""
36+
if isinstance(packages, str):
37+
packages = [packages]
38+
elif validate_dict(packages):
39+
packages = [packages['name']]
40+
41+
if len(packages) == 0:
42+
raise ValueError("must specify at least one package to build")
43+
44+
if not base:
45+
base = get_l4t_base()
46+
47+
# add all dependencies to the build tree
48+
packages = unroll_dependencies(packages)
49+
print('-- Building containers ', packages)
50+
51+
# make sure all packages can be found before building any
52+
for package in packages:
53+
find_package(package)
54+
55+
# assign default container name and tag if needed
56+
if len(name) == 0:
57+
name = packages[-1]
58+
59+
if name.find(':') < 0:
60+
name += f":r{L4T_VERSION}"
61+
62+
for idx, package in enumerate(packages):
63+
# tag this build stage with the sub-package
64+
container_name = f"{name}-{package}"
65+
66+
# generate the logging file (without the extension)
67+
log_file = os.path.join(log_dir('build'), container_name).replace(':','_')
68+
69+
# build next container
70+
pkg = find_package(package)
71+
72+
if 'dockerfile' in pkg:
73+
cmd = f"sudo docker build --network=host --tag {container_name} \ \n"
74+
cmd += f"--file {os.path.join(pkg['path'], pkg['dockerfile'])} \ \n"
75+
cmd += f"--build-arg BASE_IMAGE={base} \ \n"
76+
77+
if 'build_args' in pkg:
78+
cmd += ''.join([f"--build-arg {key}=\"{value}\" \ \n" for key, value in pkg['build_args'].items()])
79+
80+
if 'build_flags' in pkg:
81+
cmd += pkg['build_flags'] + ' \ \n'
82+
83+
if build_flags:
84+
cmd += build_flags + ' \ \n'
85+
86+
cmd += pkg['path'] + ' \ \n' #" . "
87+
cmd += f"2>&1 | tee {log_file + '.txt'}"
88+
89+
print(f"-- Building container {container_name}")
90+
print(f"\n{cmd}\n")
91+
92+
if not simulate:
93+
with open(log_file + '.sh', 'w') as cmd_file: # save the build command to a shell script for future reference
94+
cmd_file.write('#!/usr/bin/env bash\n\n')
95+
cmd_file.write(cmd + '\n')
96+
97+
subprocess.run(cmd.replace('\ \n', ''), shell=True, check=True) # remove the line breaks that were added for readability
98+
99+
base = container_name
100+
101+
# tag the final container
102+
cmd = f"sudo docker tag {container_name} {name}"
103+
print(f"-- Tagging container {container_name} -> {name}")
104+
print(f"{cmd}\n")
105+
106+
if not simulate:
107+
subprocess.run(cmd, shell=True, check=True)
108+

‎jetson_containers/l4t_version.py

-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from packaging import version
66

77

8-
98
def get_l4t_version(version_file='/etc/nv_tegra_release'):
109
"""
1110
Returns the L4T_VERSION in a packaging.version.Version object

‎jetson_containers/logging.py

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env python3
2+
import os
3+
4+
from .packages import _PACKAGE_ROOT
5+
6+
7+
_LOG_DIRS = {}
8+
9+
10+
def log_dir(type='root'):
11+
"""
12+
Return the path to the logging directory.
13+
type can be: root, build, test, run
14+
"""
15+
return _LOG_DIRS[type]
16+
17+
18+
def set_log_dir(path, type='root', create=True):
19+
"""
20+
Set the path to the logging directory, and create it if needed.
21+
type can be: root, build, test, run
22+
"""
23+
_LOG_DIRS[type] = path
24+
25+
if create:
26+
os.makedirs(path, exist_ok=True)
27+
28+
if type == 'root':
29+
set_log_dir(os.path.join(path, 'build'), 'build', create)
30+
set_log_dir(os.path.join(path, 'test'), 'test', create)
31+
set_log_dir(os.path.join(path, 'run'), 'run', create)
32+
33+
34+
# default log dir is: jetson-containers/logs
35+
set_log_dir(os.path.join(_PACKAGE_ROOT, 'logs'))
36+

0 commit comments

Comments
 (0)