blob: e4cfd937d4be65cec3ea510914e6563ff7b51b5e [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
68 'rust', # Build only the rust components + copy artifacts to output dir
69 'test', # Run the unit tests
70 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000071]
72
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000073# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000074HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000075 # 'bluetooth_test_common',
76 # 'bluetoothtbd_test',
77 # 'net_test_avrcp',
78 # 'net_test_btcore',
79 # 'net_test_types',
80 # 'net_test_btm_iso',
81 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000082]
83
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070084BOOTSTRAP_GIT_REPOS = {
85 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
86 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
87 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
88}
89
90# List of packages required for linux build
91REQUIRED_APT_PACKAGES = [
92 'bison',
93 'build-essential',
94 'curl',
95 'debmake',
96 'flatbuffers-compiler',
97 'flex',
98 'g++-multilib',
99 'gcc-multilib',
100 'generate-ninja',
101 'gnupg',
102 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000103 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700104 'libc++-dev',
105 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000106 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700107 'libevent-dev',
108 'libevent-dev',
109 'libflatbuffers-dev',
110 'libflatbuffers1',
111 'libgl1-mesa-dev',
112 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000113 'libgtest-dev',
114 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700115 'liblz4-tool',
116 'libncurses5',
117 'libnss3-dev',
118 'libprotobuf-dev',
119 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000120 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700121 'libssl-dev',
122 'libtinyxml2-dev',
123 'libx11-dev',
124 'libxml2-utils',
125 'ninja-build',
126 'openssl',
127 'protobuf-compiler',
128 'unzip',
129 'x11proto-core-dev',
130 'xsltproc',
131 'zip',
132 'zlib1g-dev',
133]
134
135# List of cargo packages required for linux build
136REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
137
138APT_PKG_LIST = ['apt', '-qq', 'list']
139CARGO_PKG_LIST = ['cargo', 'install', '--list']
140
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000141
142class UseFlags():
143
144 def __init__(self, use_flags):
145 """ Construct the use flags.
146
147 Args:
148 use_flags: List of use flags parsed from the command.
149 """
150 self.flags = {}
151
152 # Import use flags required by common-mk
153 for use in COMMON_MK_USES:
154 self.set_flag(use, False)
155
156 # Set our defaults
157 for use, value in USE_DEFAULTS.items():
158 self.set_flag(use, value)
159
160 # Set use flags - value is set to True unless the use starts with -
161 # All given use flags always override the defaults
162 for use in use_flags:
163 value = not use.startswith('-')
164 self.set_flag(use, value)
165
166 def set_flag(self, key, value=True):
167 setattr(self, key, value)
168 self.flags[key] = value
169
170
171class HostBuild():
172
173 def __init__(self, args):
174 """ Construct the builder.
175
176 Args:
177 args: Parsed arguments from ArgumentParser
178 """
179 self.args = args
180
181 # Set jobs to number of cpus unless explicitly set
182 self.jobs = self.args.jobs
183 if not self.jobs:
184 self.jobs = multiprocessing.cpu_count()
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000185 print("Number of jobs = {}".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000186
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700187 # Normalize bootstrap dir and make sure it exists
188 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
189 os.makedirs(self.bootstrap_dir, exist_ok=True)
190
191 # Output and platform directories are based on bootstrap
192 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
193 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000194 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800196 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000197
198 # If default target isn't set, build everything
199 self.target = 'all'
200 if hasattr(self.args, 'target') and self.args.target:
201 self.target = self.args.target
202
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000203 target_use = self.args.use if self.args.use else []
204
205 # Unless set, always build test code
206 if not self.args.notest:
207 target_use.append('test')
208
209 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000210
211 # Validate platform directory
212 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
213 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
214
215 # Make sure output directory exists (or create it)
216 os.makedirs(self.output_dir, exist_ok=True)
217
218 # Set some default attributes
219 self.libbase_ver = None
220
221 self.configure_environ()
222
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000223 def _generate_rustflags(self):
224 """ Rustflags to include for the build.
225 """
226 rust_flags = [
227 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700228 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000229 '-C',
230 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700231 # exclude uninteresting warnings
232 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000233 ]
234
235 return ' '.join(rust_flags)
236
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000237 def configure_environ(self):
238 """ Configure environment variables for GN and Cargo.
239 """
240 self.env = os.environ.copy()
241
242 # Make sure cargo home dir exists and has a bin directory
243 cargo_home = os.path.join(self.output_dir, 'cargo_home')
244 os.makedirs(cargo_home, exist_ok=True)
245 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
246
247 # Configure Rust env variables
248 self.env['CARGO_TARGET_DIR'] = self.output_dir
249 self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000250 self.env['RUSTFLAGS'] = self._generate_rustflags()
Abhishek Pandit-Subedi1927afa2021-04-28 21:16:18 -0700251 self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700252 self.env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
Sonny Sasakac1bc8e12022-06-14 17:52:50 -0700253 self.env['CXX_OUTDIR'] = self._gn_default_output()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000254
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000255 def run_command(self, target, args, cwd=None, env=None):
256 """ Run command and stream the output.
257 """
258 # Set some defaults
259 if not cwd:
260 cwd = self.platform_dir
261 if not env:
262 env = self.env
263
264 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
265 with open(log_file, 'wb') as lf:
266 rc = 0
267 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
268 while True:
269 line = process.stdout.readline()
270 print(line.decode('utf-8'), end="")
271 lf.write(line)
272 if not line:
273 rc = process.poll()
274 if rc is not None:
275 break
276
277 time.sleep(0.1)
278
279 if rc != 0:
280 raise Exception("Return code is {}".format(rc))
281
282 def _get_basever(self):
283 if self.libbase_ver:
284 return self.libbase_ver
285
286 self.libbase_ver = os.environ.get('BASE_VER', '')
287 if not self.libbase_ver:
288 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
289 try:
290 with open(base_file, 'r') as f:
291 self.libbase_ver = f.read().strip('\n')
292 except:
293 self.libbase_ver = 'NOT-INSTALLED'
294
295 return self.libbase_ver
296
297 def _gn_default_output(self):
298 return os.path.join(self.output_dir, 'out/Default')
299
300 def _gn_configure(self):
301 """ Configure all required parameters for platform2.
302
303 Mostly copied from //common-mk/platform2.py
304 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700305 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000306
307 def to_gn_string(s):
308 return '"%s"' % s.replace('"', '\\"')
309
310 def to_gn_list(strs):
311 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
312
313 def to_gn_args_args(gn_args):
314 for k, v in gn_args.items():
315 if isinstance(v, bool):
316 v = str(v).lower()
317 elif isinstance(v, list):
318 v = to_gn_list(v)
319 elif isinstance(v, six.string_types):
320 v = to_gn_string(v)
321 else:
322 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
323 yield '%s=%s' % (k.replace('-', '_'), v)
324
325 gn_args = {
326 'platform_subdir': 'bt',
327 'cc': 'clang' if clang else 'gcc',
328 'cxx': 'clang++' if clang else 'g++',
329 'ar': 'llvm-ar' if clang else 'ar',
330 'pkg-config': 'pkg-config',
331 'clang_cc': clang,
332 'clang_cxx': clang,
333 'OS': 'linux',
334 'sysroot': self.sysroot,
335 'libdir': os.path.join(self.sysroot, self.libdir),
336 'build_root': self.output_dir,
337 'platform2_root': self.platform_dir,
338 'libbase_ver': self._get_basever(),
339 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
340 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800341 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000342 'enable_werror': False,
343 }
344
345 if clang:
346 # Make sure to mark the clang use flag as true
347 self.use.set_flag('clang', True)
348 gn_args['external_cxxflags'] += ['-I/usr/include/']
349
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000350 gn_args_args = list(to_gn_args_args(gn_args))
351 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
352 gn_args_args += ['use={%s}' % (' '.join(use_args))]
353
354 gn_args = [
355 'gn',
356 'gen',
357 ]
358
359 if self.args.verbose:
360 gn_args.append('-v')
361
362 gn_args += [
363 '--root=%s' % self.platform_dir,
364 '--args=%s' % ' '.join(gn_args_args),
365 self._gn_default_output(),
366 ]
367
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700368 if 'PKG_CONFIG_PATH' in self.env:
369 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000370
371 self.run_command('configure', gn_args)
372
373 def _gn_build(self, target):
374 """ Generate the ninja command for the target and run it.
375 """
376 args = ['%s:%s' % ('bt', target)]
377 ninja_args = ['ninja', '-C', self._gn_default_output()]
378 if self.jobs:
379 ninja_args += ['-j', str(self.jobs)]
380 ninja_args += args
381
382 if self.args.verbose:
383 ninja_args.append('-v')
384
385 self.run_command('build', ninja_args)
386
387 def _rust_configure(self):
388 """ Generate config file at cargo_home so we use vendored crates.
389 """
390 template = """
391 [source.systembt]
392 directory = "{}/external/rust/vendor"
393
394 [source.crates-io]
395 replace-with = "systembt"
396 local-registry = "/nonexistent"
397 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700398
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700399 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700400 contents = template.format(self.platform_dir)
401 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
402 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000403
404 def _rust_build(self):
405 """ Run `cargo build` from platform2/bt directory.
406 """
407 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
408
409 def _target_prepare(self):
410 """ Target to prepare the output directory for building.
411
412 This runs gn gen to generate all rquired files and set up the Rust
413 config properly. This will be run
414 """
415 self._gn_configure()
416 self._rust_configure()
417
418 def _target_tools(self):
419 """ Build the tools target in an already prepared environment.
420 """
421 self._gn_build('tools')
422
423 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
424 shutil.copy(
425 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
426
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800427 def _target_docs(self):
428 """Build the Rust docs."""
429 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
430
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000431 def _target_rust(self):
432 """ Build rust artifacts in an already prepared environment.
433 """
434 self._rust_build()
435
436 def _target_main(self):
437 """ Build the main GN artifacts in an already prepared environment.
438 """
439 self._gn_build('all')
440
441 def _target_test(self):
442 """ Runs the host tests.
443 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000444 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800445 rust_test_cmd = ['cargo', 'test']
446 if self.args.test_name:
447 rust_test_cmd = rust_test_cmd + [self.args.test_name]
448
449 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000450
451 # Host tests second based on host test list
452 for t in HOST_TESTS:
453 self.run_command(
454 'test', [os.path.join(self.output_dir, 'out/Default', t)],
455 cwd=os.path.join(self.output_dir),
456 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000457
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800458 def _target_install(self):
459 """ Installs files required to run Floss to install directory.
460 """
461 # First make the install directory
462 prefix = self.install_dir
463 os.makedirs(prefix, exist_ok=True)
464
465 # Next save the cwd and change to install directory
466 last_cwd = os.getcwd()
467 os.chdir(prefix)
468
469 bindir = os.path.join(self.output_dir, 'debug')
470 srcdir = os.path.dirname(__file__)
471
472 install_map = [
473 {
474 'src': os.path.join(bindir, 'btadapterd'),
475 'dst': 'usr/libexec/bluetooth/btadapterd',
476 'strip': True
477 },
478 {
479 'src': os.path.join(bindir, 'btmanagerd'),
480 'dst': 'usr/libexec/bluetooth/btmanagerd',
481 'strip': True
482 },
483 {
484 'src': os.path.join(bindir, 'btclient'),
485 'dst': 'usr/local/bin/btclient',
486 'strip': True
487 },
488 ]
489
490 for v in install_map:
491 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
492 dst = os.path.join(prefix, partial_dst)
493
494 # Create dst directory first and copy file there
495 os.makedirs(os.path.dirname(dst), exist_ok=True)
496 print('Installing {}'.format(dst))
497 shutil.copy(src, dst)
498
499 # Binary should be marked for strip and no-strip option shouldn't be
500 # set. No-strip is useful while debugging.
501 if strip and not self.args.no_strip:
502 self.run_command('install', ['llvm-strip', dst])
503
504 # Put all files into a tar.gz for easier installation
505 tar_location = os.path.join(prefix, 'floss.tar.gz')
506 with tarfile.open(tar_location, 'w:gz') as tar:
507 for v in install_map:
508 tar.add(v['dst'])
509
510 print('Tarball created at {}'.format(tar_location))
511
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000512 def _target_clean(self):
513 """ Delete the output directory entirely.
514 """
515 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800516
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700517 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800518 try:
519 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
520 except FileNotFoundError:
521 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000522
523 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800524 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000525 """
526 self._target_prepare()
527 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000528 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700529 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000530
531 def build(self):
532 """ Builds according to self.target
533 """
534 print('Building target ', self.target)
535
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800536 # Validate that the target is valid
537 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800538 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800539 return
540
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000541 if self.target == 'prepare':
542 self._target_prepare()
543 elif self.target == 'tools':
544 self._target_tools()
545 elif self.target == 'rust':
546 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800547 elif self.target == 'docs':
548 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000549 elif self.target == 'main':
550 self._target_main()
551 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000552 self._target_test()
553 elif self.target == 'clean':
554 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800555 elif self.target == 'install':
556 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000557 elif self.target == 'all':
558 self._target_all()
559
560
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700561class Bootstrap():
562
563 def __init__(self, base_dir, bt_dir):
564 """ Construct bootstrapper.
565
566 Args:
567 base_dir: Where to stage everything.
568 bt_dir: Where bluetooth source is kept (will be symlinked)
569 """
570 self.base_dir = os.path.abspath(base_dir)
571 self.bt_dir = os.path.abspath(bt_dir)
572
573 # Create base directory if it doesn't already exist
574 os.makedirs(self.base_dir, exist_ok=True)
575
576 if not os.path.isdir(self.bt_dir):
577 raise Exception('{} is not a valid directory'.format(self.bt_dir))
578
579 self.git_dir = os.path.join(self.base_dir, 'repos')
580 self.staging_dir = os.path.join(self.base_dir, 'staging')
581 self.output_dir = os.path.join(self.base_dir, 'output')
582 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
583
584 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
585
586 def _update_platform2(self):
587 """Updates repositories used for build."""
588 for repo in BOOTSTRAP_GIT_REPOS.keys():
589 cwd = os.path.join(self.git_dir, repo)
590 subprocess.check_call(['git', 'pull'], cwd=cwd)
591
592 def _setup_platform2(self):
593 """ Set up platform2.
594
595 This will check out all the git repos and symlink everything correctly.
596 """
597
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800598 # Create all directories we will need to use
599 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
600 os.makedirs(dirpath, exist_ok=True)
601
602 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700603 if os.path.isfile(self.dir_setup_complete):
604 print('{} already set-up. Updating instead.'.format(self.base_dir))
605 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800606 else:
607 # Check out all repos in git directory
608 for repo in BOOTSTRAP_GIT_REPOS.values():
609 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700610
611 # Symlink things
612 symlinks = [
613 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700614 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700615 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
616 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
617 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
618 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
619 ]
620
621 # Create symlinks
622 for pairs in symlinks:
623 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000624 try:
625 os.unlink(dst)
626 except Exception as e:
627 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700628 os.symlink(src, dst)
629
630 # Write to setup complete file so we don't repeat this step
631 with open(self.dir_setup_complete, 'w') as f:
632 f.write('Setup complete.')
633
634 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
635 """ Pretty print an install command.
636
637 Args:
638 install_cmd: Prefixed install command.
639 packages: Enumerate packages and append them to install command.
640 line_limit: Number of characters per line.
641
642 Return:
643 Array of lines to join and print.
644 """
645 install = [install_cmd]
646 line = ' '
647 # Remainder needed = space + len(pkg) + space + \
648 # Assuming 80 character lines, that's 80 - 3 = 77
649 line_limit = line_limit - 3
650 for pkg in packages:
651 if len(line) + len(pkg) < line_limit:
652 line = '{}{} '.format(line, pkg)
653 else:
654 install.append(line)
655 line = ' {} '.format(pkg)
656
657 if len(line) > 0:
658 install.append(line)
659
660 return install
661
662 def _check_package_installed(self, package, cmd, predicate):
663 """Check that the given package is installed.
664
665 Args:
666 package: Check that this package is installed.
667 cmd: Command prefix to check if installed (package appended to end)
668 predicate: Function/lambda to check if package is installed based
669 on output. Takes string output and returns boolean.
670
671 Return:
672 True if package is installed.
673 """
674 try:
675 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
676 is_installed = predicate(output.decode('utf-8'))
677 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
678
679 return is_installed
680 except Exception as e:
681 print(e)
682 return False
683
684 def _get_command_output(self, cmd):
685 """Runs the command and gets the output.
686
687 Args:
688 cmd: Command to run.
689
690 Return:
691 Tuple (Success, Output). Success represents if the command ran ok.
692 """
693 try:
694 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
695 return (True, output.decode('utf-8').split('\n'))
696 except Exception as e:
697 print(e)
698 return (False, "")
699
700 def _print_missing_packages(self):
701 """Print any missing packages found via apt.
702
703 This will find any missing packages necessary for build using apt and
704 print it out as an apt-get install printf.
705 """
706 print('Checking for any missing packages...')
707
708 (success, output) = self._get_command_output(APT_PKG_LIST)
709 if not success:
710 raise Exception("Could not query apt for packages.")
711
712 packages_installed = {}
713 for line in output:
714 if 'installed' in line:
715 split = line.split('/', 2)
716 packages_installed[split[0]] = True
717
718 need_packages = []
719 for pkg in REQUIRED_APT_PACKAGES:
720 if pkg not in packages_installed:
721 need_packages.append(pkg)
722
723 # No packages need to be installed
724 if len(need_packages) == 0:
725 print('+ All required packages are installed')
726 return
727
728 install = self._pretty_print_install('sudo apt-get install', need_packages)
729
730 # Print all lines so they can be run in cmdline
731 print('Missing system packages. Run the following command: ')
732 print(' \\\n'.join(install))
733
734 def _print_missing_rust_packages(self):
735 """Print any missing packages found via cargo.
736
737 This will find any missing packages necessary for build using cargo and
738 print it out as a cargo-install printf.
739 """
740 print('Checking for any missing cargo packages...')
741
742 (success, output) = self._get_command_output(CARGO_PKG_LIST)
743 if not success:
744 raise Exception("Could not query cargo for packages.")
745
746 packages_installed = {}
747 for line in output:
748 # Cargo installed packages have this format
749 # :
750 #
751 # We only care about the crates themselves
752 if ':' not in line:
753 continue
754
755 split = line.split(' ', 2)
756 packages_installed[split[0]] = True
757
758 need_packages = []
759 for pkg in REQUIRED_CARGO_PACKAGES:
760 if pkg not in packages_installed:
761 need_packages.append(pkg)
762
763 # No packages to be installed
764 if len(need_packages) == 0:
765 print('+ All required cargo packages are installed')
766 return
767
768 install = self._pretty_print_install('cargo install', need_packages)
769 print('Missing cargo packages. Run the following command: ')
770 print(' \\\n'.join(install))
771
772 def bootstrap(self):
773 """ Bootstrap the Linux build."""
774 self._setup_platform2()
775 self._print_missing_packages()
776 self._print_missing_rust_packages()
777
778
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000779if __name__ == '__main__':
780 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700781 parser.add_argument(
782 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
783 parser.add_argument(
784 '--run-bootstrap',
785 help='Run bootstrap code to verify build env is ok to build.',
786 default=False,
787 action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000788 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800789 parser.add_argument(
790 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000791 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800792 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
793 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000794 parser.add_argument('--target', help='Run specific build target')
795 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700796 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000797 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700798 parser.add_argument(
799 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000800 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000801 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700802
803 # Make sure we get absolute path + expanded path for bootstrap directory
804 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
805
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000806 # Possible values for machine() come from 'uname -m'
807 # Since this script only runs on Linux, x86_64 machines must have this value
808 if platform.machine() != 'x86_64':
809 raise Exception("Only x86_64 machines are currently supported by this build script.")
810
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700811 if args.run_bootstrap:
812 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
813 bootstrap.bootstrap()
814 else:
815 build = HostBuild(args)
816 build.build()