blob: 12fe4172b65ad40a1ecd15b8367f35f110f3c332 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Mike Frysingerb5d075d2021-03-01 00:56:38 -050015import multiprocessing
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070016import os
17import optparse
Colin Cross5acde752012-03-28 20:15:45 -070018import re
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import sys
20
David Rileye0684ad2017-04-05 00:02:59 -070021from event_log import EventLog
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070022from error import NoSuchProjectError
Colin Cross5acde752012-03-28 20:15:45 -070023from error import InvalidProjectGroupsError
Mike Frysingerb5d075d2021-03-01 00:56:38 -050024import progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025
David Pursehouseb148ac92012-11-16 09:33:39 +090026
Mike Frysingerdf8b1cb2021-07-26 15:59:20 -040027# Are we generating man-pages?
28GENERATE_MANPAGES = os.environ.get('_REPO_GENERATE_MANPAGES_') == ' indeed! '
29
30
Mike Frysinger7c871162021-02-16 01:45:39 -050031# Number of projects to submit to a single worker process at a time.
32# This number represents a tradeoff between the overhead of IPC and finer
33# grained opportunity for parallelism. This particular value was chosen by
34# iterating through powers of two until the overall performance no longer
35# improved. The performance of this batch size is not a function of the
36# number of cores on the system.
37WORKER_BATCH_SIZE = 32
38
39
Mike Frysinger6a2400a2021-02-16 01:43:31 -050040# How many jobs to run in parallel by default? This assumes the jobs are
41# largely I/O bound and do not hit the network.
42DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8)
43
44
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070045class Command(object):
46 """Base class for any command line action in repo.
47 """
48
Mike Frysingerd88b3692021-06-14 16:09:29 -040049 # Singleton for all commands to track overall repo command execution and
50 # provide event summary to callers. Only used by sync subcommand currently.
51 #
52 # NB: This is being replaced by git trace2 events. See git_trace2_event_log.
53 event_log = EventLog()
54
Mike Frysinger4f210542021-06-14 16:05:19 -040055 # Whether this command is a "common" one, i.e. whether the user would commonly
56 # use it or it's a more uncommon command. This is used by the help command to
57 # show short-vs-full summaries.
58 COMMON = False
59
Mike Frysinger6a2400a2021-02-16 01:43:31 -050060 # Whether this command supports running in parallel. If greater than 0,
61 # it is the number of parallel jobs to default to.
62 PARALLEL_JOBS = None
63
LaMont Jonescc879a92021-11-18 22:40:18 +000064 # Whether this command supports Multi-manifest. If False, then main.py will
65 # iterate over the manifests and invoke the command once per (sub)manifest.
66 # This is only checked after calling ValidateOptions, so that partially
67 # migrated subcommands can set it to False.
68 MULTI_MANIFEST_SUPPORT = True
69
Raman Tenneti784e16f2021-06-11 17:29:45 -070070 def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
LaMont Jonescc879a92021-11-18 22:40:18 +000071 git_event_log=None, outer_client=None, outer_manifest=None):
Mike Frysingerd58d0dd2021-06-14 16:17:27 -040072 self.repodir = repodir
73 self.client = client
LaMont Jonescc879a92021-11-18 22:40:18 +000074 self.outer_client = outer_client or client
Mike Frysingerd58d0dd2021-06-14 16:17:27 -040075 self.manifest = manifest
76 self.gitc_manifest = gitc_manifest
Raman Tenneti784e16f2021-06-11 17:29:45 -070077 self.git_event_log = git_event_log
LaMont Jonescc879a92021-11-18 22:40:18 +000078 self.outer_manifest = outer_manifest
Mike Frysingerd58d0dd2021-06-14 16:17:27 -040079
80 # Cache for the OptionParser property.
81 self._optparse = None
82
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -070083 def WantPager(self, _opt):
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070084 return False
85
David Pursehouseb148ac92012-11-16 09:33:39 +090086 def ReadEnvironmentOptions(self, opts):
87 """ Set options from environment variables. """
88
89 env_options = self._RegisteredEnvironmentOptions()
90
91 for env_key, opt_key in env_options.items():
92 # Get the user-set option value if any
93 opt_value = getattr(opts, opt_key)
94
95 # If the value is set, it means the user has passed it as a command
96 # line option, and we should use that. Otherwise we can try to set it
97 # with the value from the corresponding environment variable.
98 if opt_value is not None:
99 continue
100
101 env_value = os.environ.get(env_key)
102 if env_value is not None:
103 setattr(opts, opt_key, env_value)
104
105 return opts
106
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700107 @property
108 def OptionParser(self):
109 if self._optparse is None:
110 try:
111 me = 'repo %s' % self.NAME
112 usage = self.helpUsage.strip().replace('%prog', me)
113 except AttributeError:
114 usage = 'repo %s' % self.NAME
Mike Frysinger72ebf192020-02-19 01:20:18 -0500115 epilog = 'Run `repo help %s` to view the detailed manual.' % self.NAME
116 self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
Mike Frysinger9180a072021-04-13 14:57:40 -0400117 self._CommonOptions(self._optparse)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700118 self._Options(self._optparse)
119 return self._optparse
120
Mike Frysinger9180a072021-04-13 14:57:40 -0400121 def _CommonOptions(self, p, opt_v=True):
122 """Initialize the option parser with common options.
123
124 These will show up for *all* subcommands, so use sparingly.
125 NB: Keep in sync with repo:InitParser().
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700126 """
Mike Frysinger9180a072021-04-13 14:57:40 -0400127 g = p.add_option_group('Logging options')
128 opts = ['-v'] if opt_v else []
129 g.add_option(*opts, '--verbose',
130 dest='output_mode', action='store_true',
131 help='show all output')
132 g.add_option('-q', '--quiet',
133 dest='output_mode', action='store_false',
134 help='only show errors')
135
Mike Frysinger6a2400a2021-02-16 01:43:31 -0500136 if self.PARALLEL_JOBS is not None:
Mike Frysingerdf8b1cb2021-07-26 15:59:20 -0400137 default = 'based on number of CPU cores'
138 if not GENERATE_MANPAGES:
139 # Only include active cpu count if we aren't generating man pages.
140 default = f'%default; {default}'
Mike Frysinger6a2400a2021-02-16 01:43:31 -0500141 p.add_option(
142 '-j', '--jobs',
143 type=int, default=self.PARALLEL_JOBS,
Mike Frysingerdf8b1cb2021-07-26 15:59:20 -0400144 help=f'number of jobs to run in parallel (default: {default})')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700145
LaMont Jonescc879a92021-11-18 22:40:18 +0000146 m = p.add_option_group('Multi-manifest options')
147 m.add_option('--outer-manifest', action='store_true',
148 help='operate starting at the outermost manifest')
149 m.add_option('--no-outer-manifest', dest='outer_manifest',
150 action='store_false', default=None,
151 help='do not operate on outer manifests')
152 m.add_option('--this-manifest-only', action='store_true', default=None,
153 help='only operate on this (sub)manifest')
154 m.add_option('--no-this-manifest-only', '--all-manifests',
155 dest='this_manifest_only', action='store_false',
156 help='operate on this manifest and its submanifests')
157
Mike Frysinger9180a072021-04-13 14:57:40 -0400158 def _Options(self, p):
159 """Initialize the option parser with subcommand-specific options."""
160
David Pursehouseb148ac92012-11-16 09:33:39 +0900161 def _RegisteredEnvironmentOptions(self):
162 """Get options that can be set from environment variables.
163
164 Return a dictionary mapping environment variable name
165 to option key name that it can override.
166
167 Example: {'REPO_MY_OPTION': 'my_option'}
168
169 Will allow the option with key value 'my_option' to be set
170 from the value in the environment variable named 'REPO_MY_OPTION'.
171
172 Note: This does not work properly for options that are explicitly
173 set to None by the user, or options that are defined with a
174 default value other than None.
175
176 """
177 return {}
178
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700179 def Usage(self):
180 """Display usage and terminate.
181 """
182 self.OptionParser.print_usage()
183 sys.exit(1)
184
Mike Frysinger9180a072021-04-13 14:57:40 -0400185 def CommonValidateOptions(self, opt, args):
186 """Validate common options."""
187 opt.quiet = opt.output_mode is False
188 opt.verbose = opt.output_mode is True
189
Mike Frysingerae6cb082019-08-27 01:10:59 -0400190 def ValidateOptions(self, opt, args):
191 """Validate the user options & arguments before executing.
192
193 This is meant to help break the code up into logical steps. Some tips:
194 * Use self.OptionParser.error to display CLI related errors.
195 * Adjust opt member defaults as makes sense.
196 * Adjust the args list, but do so inplace so the caller sees updates.
197 * Try to avoid updating self state. Leave that to Execute.
198 """
199
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700200 def Execute(self, opt, args):
201 """Perform the action, after option parsing is complete.
202 """
203 raise NotImplementedError
Conley Owens971de8e2012-04-16 10:36:08 -0700204
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500205 @staticmethod
206 def ExecuteInParallel(jobs, func, inputs, callback, output=None, ordered=False):
207 """Helper for managing parallel execution boiler plate.
208
209 For subcommands that can easily split their work up.
210
211 Args:
212 jobs: How many parallel processes to use.
213 func: The function to apply to each of the |inputs|. Usually a
214 functools.partial for wrapping additional arguments. It will be run
215 in a separate process, so it must be pickalable, so nested functions
216 won't work. Methods on the subcommand Command class should work.
217 inputs: The list of items to process. Must be a list.
218 callback: The function to pass the results to for processing. It will be
219 executed in the main thread and process the results of |func| as they
220 become available. Thus it may be a local nested function. Its return
221 value is passed back directly. It takes three arguments:
222 - The processing pool (or None with one job).
223 - The |output| argument.
224 - An iterator for the results.
225 output: An output manager. May be progress.Progess or color.Coloring.
226 ordered: Whether the jobs should be processed in order.
227
228 Returns:
229 The |callback| function's results are returned.
230 """
231 try:
232 # NB: Multiprocessing is heavy, so don't spin it up for one job.
233 if len(inputs) == 1 or jobs == 1:
234 return callback(None, output, (func(x) for x in inputs))
235 else:
236 with multiprocessing.Pool(jobs) as pool:
237 submit = pool.imap if ordered else pool.imap_unordered
238 return callback(pool, output, submit(func, inputs, chunksize=WORKER_BATCH_SIZE))
239 finally:
240 if isinstance(output, progress.Progress):
241 output.end()
242
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800243 def _ResetPathToProjectMap(self, projects):
244 self._by_path = dict((p.worktree, p) for p in projects)
245
246 def _UpdatePathToProjectMap(self, project):
247 self._by_path[project.worktree] = project
248
Simran Basib9a1b732015-08-20 12:19:28 -0700249 def _GetProjectByPath(self, manifest, path):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800250 project = None
251 if os.path.exists(path):
252 oldpath = None
David Pursehouse5a2517f2020-02-12 14:55:01 +0900253 while (path and
254 path != oldpath and
255 path != manifest.topdir):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800256 try:
257 project = self._by_path[path]
258 break
259 except KeyError:
260 oldpath = path
261 path = os.path.dirname(path)
Mark E. Hamiltonf9fe3e12016-02-23 18:10:42 -0700262 if not project and path == manifest.topdir:
263 try:
264 project = self._by_path[path]
265 except KeyError:
266 pass
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800267 else:
268 try:
269 project = self._by_path[path]
270 except KeyError:
271 pass
272 return project
273
Simran Basib9a1b732015-08-20 12:19:28 -0700274 def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
LaMont Jonescc879a92021-11-18 22:40:18 +0000275 submodules_ok=False, all_manifests=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700276 """A list of projects that match the arguments.
277 """
LaMont Jonescc879a92021-11-18 22:40:18 +0000278 if all_manifests:
279 if not manifest:
280 manifest = self.manifest.outer_client
281 all_projects_list = manifest.all_projects
282 else:
283 if not manifest:
284 manifest = self.manifest
285 all_projects_list = manifest.projects
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700286 result = []
287
Graham Christensen0369a062015-07-29 17:02:54 -0500288 if not groups:
Raman Tenneti080877e2021-03-09 15:19:06 -0800289 groups = manifest.GetGroupsStr()
David Pursehouse1d947b32012-10-25 12:23:11 +0900290 groups = [x for x in re.split(r'[,\s]+', groups) if x]
Colin Cross5acde752012-03-28 20:15:45 -0700291
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700292 if not args:
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800293 derived_projects = {}
294 for project in all_projects_list:
295 if submodules_ok or project.sync_s:
296 derived_projects.update((p.name, p)
297 for p in project.GetDerivedSubprojects())
298 all_projects_list.extend(derived_projects.values())
299 for project in all_projects_list:
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700300 if (missing_ok or project.Exists) and project.MatchesGroups(groups):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700301 result.append(project)
302 else:
David James8d201162013-10-11 17:03:19 -0700303 self._ResetPathToProjectMap(all_projects_list)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700304
305 for arg in args:
Mike Frysingere778e572019-10-04 14:21:41 -0400306 # We have to filter by manifest groups in case the requested project is
307 # checked out multiple times or differently based on them.
LaMont Jonescc879a92021-11-18 22:40:18 +0000308 projects = [project for project in manifest.GetProjectsWithName(
309 arg, all_manifests=all_manifests)
Mike Frysingere778e572019-10-04 14:21:41 -0400310 if project.MatchesGroups(groups)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700311
David James8d201162013-10-11 17:03:19 -0700312 if not projects:
Anthony Newnamdf14a702011-01-09 17:31:57 -0800313 path = os.path.abspath(arg).replace('\\', '/')
LaMont Jonescc879a92021-11-18 22:40:18 +0000314 tree = manifest
315 if all_manifests:
316 # Look for the deepest matching submanifest.
317 for tree in reversed(list(manifest.all_manifests)):
318 if path.startswith(tree.topdir):
319 break
320 project = self._GetProjectByPath(tree, path)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700321
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800322 # If it's not a derived project, update path->project mapping and
323 # search again, as arg might actually point to a derived subproject.
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700324 if (project and not project.Derived and (submodules_ok or
325 project.sync_s)):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800326 search_again = False
327 for subproject in project.GetDerivedSubprojects():
328 self._UpdatePathToProjectMap(subproject)
329 search_again = True
330 if search_again:
Simran Basib9a1b732015-08-20 12:19:28 -0700331 project = self._GetProjectByPath(manifest, path) or project
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700332
David James8d201162013-10-11 17:03:19 -0700333 if project:
334 projects = [project]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700335
David James8d201162013-10-11 17:03:19 -0700336 if not projects:
337 raise NoSuchProjectError(arg)
338
339 for project in projects:
340 if not missing_ok and not project.Exists:
LaMont Jonescc879a92021-11-18 22:40:18 +0000341 raise NoSuchProjectError('%s (%s)' % (
342 arg, project.RelPath(local=not all_manifests)))
David James8d201162013-10-11 17:03:19 -0700343 if not project.MatchesGroups(groups):
344 raise InvalidProjectGroupsError(arg)
345
346 result.extend(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700347
348 def _getpath(x):
349 return x.relpath
350 result.sort(key=_getpath)
351 return result
352
LaMont Jonescc879a92021-11-18 22:40:18 +0000353 def FindProjects(self, args, inverse=False, all_manifests=False):
354 """Find projects from command line arguments.
355
356 Args:
357 args: a list of (case-insensitive) strings, projects to search for.
358 inverse: a boolean, if True, then projects not matching any |args| are
359 returned.
360 all_manifests: a boolean, if True then all manifests and submanifests are
361 used. If False, then only the local (sub)manifest is used.
362 """
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800363 result = []
David Pursehouse84c4d3c2013-04-30 10:57:37 +0900364 patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
LaMont Jonescc879a92021-11-18 22:40:18 +0000365 for project in self.GetProjects('', all_manifests=all_manifests):
366 paths = [project.name, project.RelPath(local=not all_manifests)]
David Pursehouse84c4d3c2013-04-30 10:57:37 +0900367 for pattern in patterns:
LaMont Jonescc879a92021-11-18 22:40:18 +0000368 match = any(pattern.search(x) for x in paths)
Takeshi Kanemoto1f056442016-01-26 14:11:35 +0900369 if not inverse and match:
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800370 result.append(project)
371 break
Takeshi Kanemoto1f056442016-01-26 14:11:35 +0900372 if inverse and match:
373 break
374 else:
375 if inverse:
376 result.append(project)
LaMont Jonescc879a92021-11-18 22:40:18 +0000377 result.sort(key=lambda project: (project.manifest.path_prefix,
378 project.relpath))
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800379 return result
380
LaMont Jonescc879a92021-11-18 22:40:18 +0000381 def ManifestList(self, opt):
382 """Yields all of the manifests to traverse.
383
384 Args:
385 opt: The command options.
386 """
387 top = self.outer_manifest
388 if opt.outer_manifest is False or opt.this_manifest_only:
389 top = self.manifest
390 yield top
391 if not opt.this_manifest_only:
392 for child in top.all_children:
393 yield child
394
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700395
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700396class InteractiveCommand(Command):
397 """Command which requires user interaction on the tty and
398 must not run within a pager, even if the user asks to.
399 """
David Pursehouse819827a2020-02-12 15:20:19 +0900400
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700401 def WantPager(self, _opt):
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700402 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700403
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700404
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700405class PagedCommand(Command):
406 """Command which defaults to output in a pager, as its
407 display tends to be larger than one screen full.
408 """
David Pursehouse819827a2020-02-12 15:20:19 +0900409
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700410 def WantPager(self, _opt):
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700411 return True
Shawn O. Pearcec95583b2009-03-03 17:47:06 -0800412
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700413
Shawn O. Pearcec95583b2009-03-03 17:47:06 -0800414class MirrorSafeCommand(object):
415 """Command permits itself to run within a mirror,
416 and does not require a working directory.
417 """
Dan Willemsen9ff2ece2015-08-31 15:45:06 -0700418
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700419
Dan Willemsen79360642015-08-31 15:45:06 -0700420class GitcAvailableCommand(object):
Dan Willemsen9ff2ece2015-08-31 15:45:06 -0700421 """Command that requires GITC to be available, but does
422 not require the local client to be a GITC client.
423 """
Dan Willemsen79360642015-08-31 15:45:06 -0700424
Mark E. Hamilton8ccfa742016-02-10 10:44:30 -0700425
Dan Willemsen79360642015-08-31 15:45:06 -0700426class GitcClientCommand(object):
427 """Command that requires the local client to be a GITC
428 client.
429 """