blob: 994e00a9daeb609333d5d7e952b8d47f16fc6632 [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
111 'libflatbuffers1',
112 'libgl1-mesa-dev',
113 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000114 'libgtest-dev',
115 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700116 'liblz4-tool',
117 'libncurses5',
118 'libnss3-dev',
119 'libprotobuf-dev',
120 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000121 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700122 'libssl-dev',
123 'libtinyxml2-dev',
124 'libx11-dev',
125 'libxml2-utils',
126 'ninja-build',
127 'openssl',
128 'protobuf-compiler',
129 'unzip',
130 'x11proto-core-dev',
131 'xsltproc',
132 'zip',
133 'zlib1g-dev',
134]
135
136# List of cargo packages required for linux build
137REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
138
139APT_PKG_LIST = ['apt', '-qq', 'list']
140CARGO_PKG_LIST = ['cargo', 'install', '--list']
141
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000142
143class UseFlags():
144
145 def __init__(self, use_flags):
146 """ Construct the use flags.
147
148 Args:
149 use_flags: List of use flags parsed from the command.
150 """
151 self.flags = {}
152
153 # Import use flags required by common-mk
154 for use in COMMON_MK_USES:
155 self.set_flag(use, False)
156
157 # Set our defaults
158 for use, value in USE_DEFAULTS.items():
159 self.set_flag(use, value)
160
161 # Set use flags - value is set to True unless the use starts with -
162 # All given use flags always override the defaults
163 for use in use_flags:
164 value = not use.startswith('-')
165 self.set_flag(use, value)
166
167 def set_flag(self, key, value=True):
168 setattr(self, key, value)
169 self.flags[key] = value
170
171
172class HostBuild():
173
174 def __init__(self, args):
175 """ Construct the builder.
176
177 Args:
178 args: Parsed arguments from ArgumentParser
179 """
180 self.args = args
181
182 # Set jobs to number of cpus unless explicitly set
183 self.jobs = self.args.jobs
184 if not self.jobs:
185 self.jobs = multiprocessing.cpu_count()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700186 sys.stderr.write("Number of jobs = {}\n".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000187
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700188 # Normalize bootstrap dir and make sure it exists
189 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
190 os.makedirs(self.bootstrap_dir, exist_ok=True)
191
192 # Output and platform directories are based on bootstrap
193 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
194 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Michael Sun4940f2b2022-09-15 16:08:24 -0700195 self.bt_dir = os.path.join(self.platform_dir, 'bt')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000197 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800198 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000199
Michael Sun4940f2b2022-09-15 16:08:24 -0700200 assert os.path.samefile(self.bt_dir,
201 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!"
202
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000203 # If default target isn't set, build everything
204 self.target = 'all'
205 if hasattr(self.args, 'target') and self.args.target:
206 self.target = self.args.target
207
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000208 target_use = self.args.use if self.args.use else []
209
210 # Unless set, always build test code
211 if not self.args.notest:
212 target_use.append('test')
213
214 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000215
216 # Validate platform directory
217 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
218 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
219
220 # Make sure output directory exists (or create it)
221 os.makedirs(self.output_dir, exist_ok=True)
222
223 # Set some default attributes
224 self.libbase_ver = None
225
226 self.configure_environ()
227
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000228 def _generate_rustflags(self):
229 """ Rustflags to include for the build.
230 """
231 rust_flags = [
232 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700233 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000234 '-C',
235 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700236 # exclude uninteresting warnings
237 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000238 ]
239
240 return ' '.join(rust_flags)
241
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000242 def configure_environ(self):
243 """ Configure environment variables for GN and Cargo.
244 """
245 self.env = os.environ.copy()
246
247 # Make sure cargo home dir exists and has a bin directory
248 cargo_home = os.path.join(self.output_dir, 'cargo_home')
249 os.makedirs(cargo_home, exist_ok=True)
250 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
251
252 # Configure Rust env variables
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700253 self.custom_env = {}
254 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir
255 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
256 self.custom_env['RUSTFLAGS'] = self._generate_rustflags()
257 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
258 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
259 self.custom_env['CXX_OUTDIR'] = self._gn_default_output()
260 self.env.update(self.custom_env)
261
262 def print_env(self):
263 """ Print the custom environment variables that are used in build.
264
265 Useful so that external tools can mimic the environment to be the same
266 as build.py, e.g. rust-analyzer.
267 """
268 for k, v in self.custom_env.items():
269 print("export {}='{}'".format(k, v))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000270
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000271 def run_command(self, target, args, cwd=None, env=None):
272 """ Run command and stream the output.
273 """
274 # Set some defaults
275 if not cwd:
276 cwd = self.platform_dir
277 if not env:
278 env = self.env
279
280 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
281 with open(log_file, 'wb') as lf:
282 rc = 0
283 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
284 while True:
285 line = process.stdout.readline()
286 print(line.decode('utf-8'), end="")
287 lf.write(line)
288 if not line:
289 rc = process.poll()
290 if rc is not None:
291 break
292
293 time.sleep(0.1)
294
295 if rc != 0:
296 raise Exception("Return code is {}".format(rc))
297
298 def _get_basever(self):
299 if self.libbase_ver:
300 return self.libbase_ver
301
302 self.libbase_ver = os.environ.get('BASE_VER', '')
303 if not self.libbase_ver:
304 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
305 try:
306 with open(base_file, 'r') as f:
307 self.libbase_ver = f.read().strip('\n')
308 except:
309 self.libbase_ver = 'NOT-INSTALLED'
310
311 return self.libbase_ver
312
313 def _gn_default_output(self):
314 return os.path.join(self.output_dir, 'out/Default')
315
316 def _gn_configure(self):
317 """ Configure all required parameters for platform2.
318
319 Mostly copied from //common-mk/platform2.py
320 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700321 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000322
323 def to_gn_string(s):
324 return '"%s"' % s.replace('"', '\\"')
325
326 def to_gn_list(strs):
327 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
328
329 def to_gn_args_args(gn_args):
330 for k, v in gn_args.items():
331 if isinstance(v, bool):
332 v = str(v).lower()
333 elif isinstance(v, list):
334 v = to_gn_list(v)
335 elif isinstance(v, six.string_types):
336 v = to_gn_string(v)
337 else:
338 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
339 yield '%s=%s' % (k.replace('-', '_'), v)
340
341 gn_args = {
342 'platform_subdir': 'bt',
343 'cc': 'clang' if clang else 'gcc',
344 'cxx': 'clang++' if clang else 'g++',
345 'ar': 'llvm-ar' if clang else 'ar',
346 'pkg-config': 'pkg-config',
347 'clang_cc': clang,
348 'clang_cxx': clang,
349 'OS': 'linux',
350 'sysroot': self.sysroot,
351 'libdir': os.path.join(self.sysroot, self.libdir),
352 'build_root': self.output_dir,
353 'platform2_root': self.platform_dir,
354 'libbase_ver': self._get_basever(),
355 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
356 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800357 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000358 'enable_werror': False,
359 }
360
361 if clang:
362 # Make sure to mark the clang use flag as true
363 self.use.set_flag('clang', True)
364 gn_args['external_cxxflags'] += ['-I/usr/include/']
365
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000366 gn_args_args = list(to_gn_args_args(gn_args))
367 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
368 gn_args_args += ['use={%s}' % (' '.join(use_args))]
369
370 gn_args = [
371 'gn',
372 'gen',
373 ]
374
375 if self.args.verbose:
376 gn_args.append('-v')
377
378 gn_args += [
379 '--root=%s' % self.platform_dir,
380 '--args=%s' % ' '.join(gn_args_args),
381 self._gn_default_output(),
382 ]
383
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700384 if 'PKG_CONFIG_PATH' in self.env:
385 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000386
387 self.run_command('configure', gn_args)
388
389 def _gn_build(self, target):
390 """ Generate the ninja command for the target and run it.
391 """
392 args = ['%s:%s' % ('bt', target)]
393 ninja_args = ['ninja', '-C', self._gn_default_output()]
394 if self.jobs:
395 ninja_args += ['-j', str(self.jobs)]
396 ninja_args += args
397
398 if self.args.verbose:
399 ninja_args.append('-v')
400
401 self.run_command('build', ninja_args)
402
403 def _rust_configure(self):
404 """ Generate config file at cargo_home so we use vendored crates.
405 """
406 template = """
407 [source.systembt]
408 directory = "{}/external/rust/vendor"
409
410 [source.crates-io]
411 replace-with = "systembt"
412 local-registry = "/nonexistent"
413 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700414
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700415 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700416 contents = template.format(self.platform_dir)
417 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
418 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000419
420 def _rust_build(self):
421 """ Run `cargo build` from platform2/bt directory.
422 """
423 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
424
425 def _target_prepare(self):
426 """ Target to prepare the output directory for building.
427
428 This runs gn gen to generate all rquired files and set up the Rust
429 config properly. This will be run
430 """
431 self._gn_configure()
432 self._rust_configure()
433
434 def _target_tools(self):
435 """ Build the tools target in an already prepared environment.
436 """
437 self._gn_build('tools')
438
439 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
440 shutil.copy(
441 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
442
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800443 def _target_docs(self):
444 """Build the Rust docs."""
445 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
446
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000447 def _target_rust(self):
448 """ Build rust artifacts in an already prepared environment.
449 """
450 self._rust_build()
451
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100452 def _target_rootcanal(self):
453 """ Build rust artifacts for RootCanal in an already prepared environment.
454 """
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700455 self.run_command(
456 'rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100457
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000458 def _target_main(self):
459 """ Build the main GN artifacts in an already prepared environment.
460 """
461 self._gn_build('all')
462
463 def _target_test(self):
464 """ Runs the host tests.
465 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000466 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800467 rust_test_cmd = ['cargo', 'test']
468 if self.args.test_name:
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700469 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800470
471 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100472 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000473
474 # Host tests second based on host test list
475 for t in HOST_TESTS:
476 self.run_command(
477 'test', [os.path.join(self.output_dir, 'out/Default', t)],
478 cwd=os.path.join(self.output_dir),
479 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000480
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800481 def _target_install(self):
482 """ Installs files required to run Floss to install directory.
483 """
484 # First make the install directory
485 prefix = self.install_dir
486 os.makedirs(prefix, exist_ok=True)
487
488 # Next save the cwd and change to install directory
489 last_cwd = os.getcwd()
490 os.chdir(prefix)
491
492 bindir = os.path.join(self.output_dir, 'debug')
493 srcdir = os.path.dirname(__file__)
494
495 install_map = [
496 {
497 'src': os.path.join(bindir, 'btadapterd'),
498 'dst': 'usr/libexec/bluetooth/btadapterd',
499 'strip': True
500 },
501 {
502 'src': os.path.join(bindir, 'btmanagerd'),
503 'dst': 'usr/libexec/bluetooth/btmanagerd',
504 'strip': True
505 },
506 {
507 'src': os.path.join(bindir, 'btclient'),
508 'dst': 'usr/local/bin/btclient',
509 'strip': True
510 },
511 ]
512
513 for v in install_map:
514 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
515 dst = os.path.join(prefix, partial_dst)
516
517 # Create dst directory first and copy file there
518 os.makedirs(os.path.dirname(dst), exist_ok=True)
519 print('Installing {}'.format(dst))
520 shutil.copy(src, dst)
521
522 # Binary should be marked for strip and no-strip option shouldn't be
523 # set. No-strip is useful while debugging.
524 if strip and not self.args.no_strip:
525 self.run_command('install', ['llvm-strip', dst])
526
527 # Put all files into a tar.gz for easier installation
528 tar_location = os.path.join(prefix, 'floss.tar.gz')
529 with tarfile.open(tar_location, 'w:gz') as tar:
530 for v in install_map:
531 tar.add(v['dst'])
532
533 print('Tarball created at {}'.format(tar_location))
534
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000535 def _target_clean(self):
536 """ Delete the output directory entirely.
537 """
538 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800539
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700540 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800541 try:
542 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
543 except FileNotFoundError:
544 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000545
546 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800547 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000548 """
549 self._target_prepare()
550 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000551 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700552 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000553
554 def build(self):
555 """ Builds according to self.target
556 """
557 print('Building target ', self.target)
558
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800559 # Validate that the target is valid
560 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800561 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800562 return
563
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000564 if self.target == 'prepare':
565 self._target_prepare()
566 elif self.target == 'tools':
567 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100568 elif self.target == 'rootcanal':
569 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000570 elif self.target == 'rust':
571 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800572 elif self.target == 'docs':
573 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000574 elif self.target == 'main':
575 self._target_main()
576 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000577 self._target_test()
578 elif self.target == 'clean':
579 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800580 elif self.target == 'install':
581 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000582 elif self.target == 'all':
583 self._target_all()
584
585
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700586class Bootstrap():
587
588 def __init__(self, base_dir, bt_dir):
589 """ Construct bootstrapper.
590
591 Args:
592 base_dir: Where to stage everything.
593 bt_dir: Where bluetooth source is kept (will be symlinked)
594 """
595 self.base_dir = os.path.abspath(base_dir)
596 self.bt_dir = os.path.abspath(bt_dir)
597
598 # Create base directory if it doesn't already exist
599 os.makedirs(self.base_dir, exist_ok=True)
600
601 if not os.path.isdir(self.bt_dir):
602 raise Exception('{} is not a valid directory'.format(self.bt_dir))
603
604 self.git_dir = os.path.join(self.base_dir, 'repos')
605 self.staging_dir = os.path.join(self.base_dir, 'staging')
606 self.output_dir = os.path.join(self.base_dir, 'output')
607 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
608
609 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
610
611 def _update_platform2(self):
612 """Updates repositories used for build."""
613 for repo in BOOTSTRAP_GIT_REPOS.keys():
614 cwd = os.path.join(self.git_dir, repo)
615 subprocess.check_call(['git', 'pull'], cwd=cwd)
616
617 def _setup_platform2(self):
618 """ Set up platform2.
619
620 This will check out all the git repos and symlink everything correctly.
621 """
622
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800623 # Create all directories we will need to use
624 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
625 os.makedirs(dirpath, exist_ok=True)
626
627 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700628 if os.path.isfile(self.dir_setup_complete):
629 print('{} already set-up. Updating instead.'.format(self.base_dir))
630 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800631 else:
632 # Check out all repos in git directory
633 for repo in BOOTSTRAP_GIT_REPOS.values():
634 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700635
636 # Symlink things
637 symlinks = [
638 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700639 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700640 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
641 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
642 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
643 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
644 ]
645
646 # Create symlinks
647 for pairs in symlinks:
648 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000649 try:
650 os.unlink(dst)
651 except Exception as e:
652 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700653 os.symlink(src, dst)
654
655 # Write to setup complete file so we don't repeat this step
656 with open(self.dir_setup_complete, 'w') as f:
657 f.write('Setup complete.')
658
659 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
660 """ Pretty print an install command.
661
662 Args:
663 install_cmd: Prefixed install command.
664 packages: Enumerate packages and append them to install command.
665 line_limit: Number of characters per line.
666
667 Return:
668 Array of lines to join and print.
669 """
670 install = [install_cmd]
671 line = ' '
672 # Remainder needed = space + len(pkg) + space + \
673 # Assuming 80 character lines, that's 80 - 3 = 77
674 line_limit = line_limit - 3
675 for pkg in packages:
676 if len(line) + len(pkg) < line_limit:
677 line = '{}{} '.format(line, pkg)
678 else:
679 install.append(line)
680 line = ' {} '.format(pkg)
681
682 if len(line) > 0:
683 install.append(line)
684
685 return install
686
687 def _check_package_installed(self, package, cmd, predicate):
688 """Check that the given package is installed.
689
690 Args:
691 package: Check that this package is installed.
692 cmd: Command prefix to check if installed (package appended to end)
693 predicate: Function/lambda to check if package is installed based
694 on output. Takes string output and returns boolean.
695
696 Return:
697 True if package is installed.
698 """
699 try:
700 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
701 is_installed = predicate(output.decode('utf-8'))
702 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
703
704 return is_installed
705 except Exception as e:
706 print(e)
707 return False
708
709 def _get_command_output(self, cmd):
710 """Runs the command and gets the output.
711
712 Args:
713 cmd: Command to run.
714
715 Return:
716 Tuple (Success, Output). Success represents if the command ran ok.
717 """
718 try:
719 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
720 return (True, output.decode('utf-8').split('\n'))
721 except Exception as e:
722 print(e)
723 return (False, "")
724
725 def _print_missing_packages(self):
726 """Print any missing packages found via apt.
727
728 This will find any missing packages necessary for build using apt and
729 print it out as an apt-get install printf.
730 """
731 print('Checking for any missing packages...')
732
733 (success, output) = self._get_command_output(APT_PKG_LIST)
734 if not success:
735 raise Exception("Could not query apt for packages.")
736
737 packages_installed = {}
738 for line in output:
739 if 'installed' in line:
740 split = line.split('/', 2)
741 packages_installed[split[0]] = True
742
743 need_packages = []
744 for pkg in REQUIRED_APT_PACKAGES:
745 if pkg not in packages_installed:
746 need_packages.append(pkg)
747
748 # No packages need to be installed
749 if len(need_packages) == 0:
750 print('+ All required packages are installed')
751 return
752
753 install = self._pretty_print_install('sudo apt-get install', need_packages)
754
755 # Print all lines so they can be run in cmdline
756 print('Missing system packages. Run the following command: ')
757 print(' \\\n'.join(install))
758
759 def _print_missing_rust_packages(self):
760 """Print any missing packages found via cargo.
761
762 This will find any missing packages necessary for build using cargo and
763 print it out as a cargo-install printf.
764 """
765 print('Checking for any missing cargo packages...')
766
767 (success, output) = self._get_command_output(CARGO_PKG_LIST)
768 if not success:
769 raise Exception("Could not query cargo for packages.")
770
771 packages_installed = {}
772 for line in output:
773 # Cargo installed packages have this format
774 # :
775 #
776 # We only care about the crates themselves
777 if ':' not in line:
778 continue
779
780 split = line.split(' ', 2)
781 packages_installed[split[0]] = True
782
783 need_packages = []
784 for pkg in REQUIRED_CARGO_PACKAGES:
785 if pkg not in packages_installed:
786 need_packages.append(pkg)
787
788 # No packages to be installed
789 if len(need_packages) == 0:
790 print('+ All required cargo packages are installed')
791 return
792
793 install = self._pretty_print_install('cargo install', need_packages)
794 print('Missing cargo packages. Run the following command: ')
795 print(' \\\n'.join(install))
796
797 def bootstrap(self):
798 """ Bootstrap the Linux build."""
799 self._setup_platform2()
800 self._print_missing_packages()
801 self._print_missing_rust_packages()
802
803
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000804if __name__ == '__main__':
805 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700806 parser.add_argument(
807 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
808 parser.add_argument(
809 '--run-bootstrap',
810 help='Run bootstrap code to verify build env is ok to build.',
811 default=False,
812 action='store_true')
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700813 parser.add_argument(
814 '--print-env', help='Print environment variables used for build.', default=False, action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000815 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800816 parser.add_argument(
817 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000818 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800819 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
820 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000821 parser.add_argument('--target', help='Run specific build target')
822 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700823 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000824 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700825 parser.add_argument(
826 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000827 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000828 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700829
830 # Make sure we get absolute path + expanded path for bootstrap directory
831 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
832
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000833 # Possible values for machine() come from 'uname -m'
834 # Since this script only runs on Linux, x86_64 machines must have this value
835 if platform.machine() != 'x86_64':
836 raise Exception("Only x86_64 machines are currently supported by this build script.")
837
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700838 if args.run_bootstrap:
839 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
840 bootstrap.bootstrap()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700841 elif args.print_env:
842 build = HostBuild(args)
843 build.print_env()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700844 else:
845 build = HostBuild(args)
846 build.build()