Changes in BitBake 1.8.x:
+ - Fix -f (force) in conjunction with -b
- Fix exit code for build failures in --continue mode
- Fix git branch tags fetching
- Change parseConfigurationFile so it works on real data, not a copy
the way bitbake schedules tasks
- Add BB_STAMP_POLICY variable/option ("perfile" or "full") controlling
how extensively stamps are looked at for validity
+ - When handling build target failures make sure idepends are checked and
+ failed where needed. Fixes --continue mode crashes.
+ - Fix problems with recrdeptask handling where some idepends weren't handled
+ correctly.
Changes in BitBake 1.8.10:
- Psyco is available only for x86 - do not use it on other architectures.
if old_mtime > newest_mtime:
newest_mtime = old_mtime
- if self.mtime(self.cachefile) >= newest_mtime:
+ if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
try:
p = pickle.Unpickler(file(self.cachefile, "rb"))
self.depends_cache, version_data = p.load()
except EOFError:
bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...")
self.depends_cache = {}
- except (ValueError, KeyError):
+ except:
bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
self.depends_cache = {}
else:
self.remove(fn)
return False
+ mtime = bb.parse.cached_mtime_noerror(fn)
+
# Check file still exists
- if self.mtime(fn) == 0:
+ if mtime == 0:
bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
self.remove(fn)
return False
# Check the file's timestamp
- if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
+ if mtime > self.getVar("CACHETIMESTAMP", fn, True):
bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
self.remove(fn)
return False
# Check dependencies are still valid
depends = self.getVar("__depends", fn, True)
- for f,old_mtime in depends:
- # Check if file still exists
- if self.mtime(f) == 0:
- self.remove(fn)
- return False
-
- new_mtime = bb.parse.cached_mtime(f)
- if (new_mtime > old_mtime):
- bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
- self.remove(fn)
- return False
+ if depends:
+ for f,old_mtime in depends:
+ fmtime = bb.parse.cached_mtime_noerror(f)
+ # Check if file still exists
+ if fmtime == 0:
+ self.remove(fn)
+ return False
+
+ if (fmtime > old_mtime):
+ bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
+ self.remove(fn)
+ return False
#bb.msg.debug(2, bb.msg.domain.Cache, "Depends Cache: %s is clean" % fn)
if not fn in self.clean:
pv = self.getVar('PV', file_name, True)
pr = self.getVar('PR', file_name, True)
dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
- provides = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split())
depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
packages = (self.getVar('PACKAGES', file_name, True) or "").split()
packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
cacheData.pkg_pepvpr[file_name] = (pe,pv,pr)
cacheData.pkg_dp[file_name] = dp
+ provides = [pn]
+ for provide in (self.getVar("PROVIDES", file_name, True) or "").split():
+ if provide not in provides:
+ provides.append(provide)
+
# Build forward and reverse provider hashes
# Forward: virtual -> [filenames]
# Reverse: PN -> [virtuals]
if pn not in cacheData.pn_provides:
- cacheData.pn_provides[pn] = Set()
- cacheData.pn_provides[pn] |= provides
+ cacheData.pn_provides[pn] = []
- cacheData.fn_provides[file_name] = Set()
+ cacheData.fn_provides[file_name] = provides
for provide in provides:
if provide not in cacheData.providers:
cacheData.providers[provide] = []
cacheData.providers[provide].append(file_name)
- cacheData.fn_provides[file_name].add(provide)
+ if not provide in cacheData.pn_provides[pn]:
+ cacheData.pn_provides[pn].append(provide)
- cacheData.deps[file_name] = Set()
+ cacheData.deps[file_name] = []
for dep in depends:
- cacheData.all_depends.add(dep)
- cacheData.deps[file_name].add(dep)
+ if not dep in cacheData.deps[file_name]:
+ cacheData.deps[file_name].append(dep)
+ if not dep in cacheData.all_depends:
+ cacheData.all_depends.append(dep)
# Build reverse hash for PACKAGES, so runtime dependencies
# can be be resolved (RDEPENDS, RRECOMMENDS etc.)
# Build hash of runtime depends and rececommends
- def add_dep(deplist, deps):
- for dep in deps:
- if not dep in deplist:
- deplist[dep] = ""
-
if not file_name in cacheData.rundeps:
cacheData.rundeps[file_name] = {}
if not file_name in cacheData.runrecs:
cacheData.runrecs[file_name] = {}
- rdepends = bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or "")
- rrecommends = bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or "")
+ rdepends = self.getVar('RDEPENDS', file_name, True) or ""
+ rrecommends = self.getVar('RRECOMMENDS', file_name, True) or ""
for package in packages + [pn]:
if not package in cacheData.rundeps[file_name]:
- cacheData.rundeps[file_name][package] = {}
+ cacheData.rundeps[file_name][package] = []
if not package in cacheData.runrecs[file_name]:
- cacheData.runrecs[file_name][package] = {}
+ cacheData.runrecs[file_name][package] = []
- add_dep(cacheData.rundeps[file_name][package], rdepends)
- add_dep(cacheData.runrecs[file_name][package], rrecommends)
- add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
- add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
+ cacheData.rundeps[file_name][package] = rdepends + " " + (self.getVar("RDEPENDS_%s" % package, file_name, True) or "")
+ cacheData.runrecs[file_name][package] = rrecommends + " " + (self.getVar("RRECOMMENDS_%s" % package, file_name, True) or "")
# Collect files we may need for possible world-dep
# calculations
data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
oldpath = os.path.abspath(os.getcwd())
- if self.mtime(bbfile_loc):
+ if bb.parse.cached_mtime_noerror(bbfile_loc):
os.chdir(bbfile_loc)
bb_data = data.init_db(config)
try:
self.pkg_dp = {}
self.pn_provides = {}
self.fn_provides = {}
- self.all_depends = Set()
+ self.all_depends = []
self.deps = {}
self.rundeps = {}
self.runrecs = {}
# Load data into the cache for fn
self.bb_cache = bb.cache.init(self)
- self.bb_cache.loadData(fn, self.configuration.data)
+ self.bb_cache.loadData(fn, self.configuration.data)
# Parse the loaded cache data
self.status = bb.cache.CacheData()
- self.bb_cache.handle_data(fn, self.status)
+ self.bb_cache.handle_data(fn, self.status)
# Tweak some variables
item = self.bb_cache.getVar('PN', fn, True)
# Remove stamp for target if force mode active
if self.configuration.force:
bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, fn))
- bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
+ bb.build.del_stamp('do_%s' % self.configuration.cmd, self.configuration.data)
# Setup taskdata structure
taskdata = bb.taskdata.TaskData(self.configuration.abort)
bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
(filelist, masked) = self.collect_bbfiles()
+ bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
self.parse_bbfiles(filelist, masked, self.myProgressCallback)
bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
depends = []
runq_build = []
+ recursive_tdepends = {}
taskData = self.taskData
# e.g. do_sometask[depends] = "targetname:do_someothertask"
# (makes sure sometask runs after targetname's someothertask)
idepends = taskData.tasks_idepends[task]
- for idepend in idepends:
- depid = int(idepend.split(":")[0])
+ for (depid, idependtask) in idepends:
if depid in taskData.build_targets:
# Won't be in build_targets if ASSUME_PROVIDED
depdata = taskData.build_targets[depid][0]
if depdata is not None:
dep = taskData.fn_index[depdata]
- depends.append(taskData.gettask_id(dep, idepend.split(":")[1]))
+ depends.append(taskData.gettask_id(dep, idependtask))
+
+ # Create a list of recursive dependent tasks (from tdepends) and cache
+ def get_recursive_tdepends(task):
+ if not task:
+ return []
+ if task in recursive_tdepends:
+ return recursive_tdepends[task]
+ rectdepends = [task]
+ nextdeps = [task]
+ while len(nextdeps) != 0:
+ newdeps = []
+ for nextdep in nextdeps:
+ for tdepend in taskData.tasks_tdepends[nextdep]:
+ if tdepend not in rectdepends:
+ rectdepends.append(tdepend)
+ newdeps.append(tdepend)
+ nextdeps = newdeps
+ recursive_tdepends[task] = rectdepends
+ return rectdepends
+
+ # Using the list of tdepends for this task create a list of
+ # the recursive idepends we have
+ def get_recursive_idepends(task):
+ if not task:
+ return []
+ rectdepends = get_recursive_tdepends(task)
+
+ recidepends = []
+ for tdepend in rectdepends:
+ for idepend in taskData.tasks_idepends[tdepend]:
+ recidepends.append(idepend)
+ return recidepends
def add_recursive_build(depid, depfnid):
"""
depdata = taskData.build_targets[depid][0]
if depdata is not None:
dep = taskData.fn_index[depdata]
- idepends = []
# Need to avoid creating new tasks here
taskid = taskData.gettask_id(dep, taskname, False)
if taskid is not None:
depends.append(taskid)
fnid = taskData.tasks_fnid[taskid]
- idepends = taskData.tasks_idepends[taskid]
#print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid])
else:
fnid = taskData.getfn_id(dep)
for nextdepid in taskData.rdepids[fnid]:
if nextdepid not in rdep_seen:
add_recursive_run(nextdepid, fnid)
- for idepend in idepends:
- nextdepid = int(idepend.split(":")[0])
- if nextdepid not in dep_seen:
- add_recursive_build(nextdepid, fnid)
+ for (idependid, idependtask) in get_recursive_idepends(taskid):
+ if idependid not in dep_seen:
+ add_recursive_build(idependid, fnid)
def add_recursive_run(rdepid, depfnid):
"""
depdata = taskData.run_targets[rdepid][0]
if depdata is not None:
dep = taskData.fn_index[depdata]
- idepends = []
# Need to avoid creating new tasks here
taskid = taskData.gettask_id(dep, taskname, False)
if taskid is not None:
depends.append(taskid)
fnid = taskData.tasks_fnid[taskid]
- idepends = taskData.tasks_idepends[taskid]
#print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid])
else:
fnid = taskData.getfn_id(dep)
for nextdepid in taskData.rdepids[fnid]:
if nextdepid not in rdep_seen:
add_recursive_run(nextdepid, fnid)
- for idepend in idepends:
- nextdepid = int(idepend.split(":")[0])
- if nextdepid not in dep_seen:
- add_recursive_build(nextdepid, fnid)
+ for (idependid, idependtask) in get_recursive_idepends(taskid):
+ if idependid not in dep_seen:
+ add_recursive_build(idependid, fnid)
# Resolve recursive 'recrdeptask' dependencies
#
add_recursive_build(depid, fnid)
for rdepid in taskData.rdepids[fnid]:
add_recursive_run(rdepid, fnid)
- for idepend in idepends:
- depid = int(idepend.split(":")[0])
- add_recursive_build(depid, fnid)
+ deptaskid = taskData.gettask_id(fn, taskname, False)
+ for (idependid, idependtask) in get_recursive_idepends(deptaskid):
+ add_recursive_build(idependid, fnid)
# Rmove all self references
if task in depends:
if len(self.runq_depends[task]) == 0:
buildable.append(task)
+ def check_buildable(self, task, buildable):
+ for revdep in self.runq_revdeps[task]:
+ alldeps = 1
+ for dep in self.runq_depends[revdep]:
+ if dep in unchecked:
+ alldeps = 0
+ if alldeps == 1:
+ if revdep in unchecked:
+ buildable.append(revdep)
+
for task in range(len(self.runq_fnid)):
if task not in unchecked:
continue
if not os.access(stampfile, os.F_OK):
del unchecked[task]
notcurrent.append(task)
+ check_buildable(self, task, buildable)
continue
# If its a 'nostamp' task, it's not current
taskdep = self.dataCache.task_deps[fn]
if 'nostamp' in taskdep and task in taskdep['nostamp']:
del unchecked[task]
notcurrent.append(task)
+ check_buildable(self, task, buildable)
continue
while (len(buildable) > 0):
else:
notcurrent.append(task)
- for revdep in self.runq_revdeps[task]:
- alldeps = 1
- for dep in self.runq_depends[revdep]:
- if dep in unchecked:
- alldeps = 0
- if alldeps == 1:
- if revdep in unchecked:
- nextbuildable.append(revdep)
+ check_buildable(self, task, nextbuildable)
buildable = nextbuildable
bb.fatal("check_stamps fatal internal error")
return current
+ def check_stamp(self, task):
+
+ if self.stamppolicy == "perfile":
+ fulldeptree = False
+ else:
+ fulldeptree = True
+
+ fn = self.taskData.fn_index[self.runq_fnid[task]]
+ taskname = self.runq_task[task]
+ stampfile = "%s.%s" % (self.dataCache.stamp[fn], taskname)
+ # If the stamp is missing its not current
+ if not os.access(stampfile, os.F_OK):
+ return False
+ # If its a 'nostamp' task, it's not current
+ taskdep = self.dataCache.task_deps[fn]
+ if 'nostamp' in taskdep and task in taskdep['nostamp']:
+ return False
+
+ iscurrent = True
+ t1 = os.stat(stampfile)[stat.ST_MTIME]
+ for dep in self.runq_depends[task]:
+ if iscurrent:
+ fn2 = self.taskData.fn_index[self.runq_fnid[dep]]
+ taskname2 = self.runq_task[dep]
+ stampfile2 = "%s.%s" % (self.dataCache.stamp[fn2], taskname2)
+ if fulldeptree or fn == fn2:
+ try:
+ t2 = os.stat(stampfile2)[stat.ST_MTIME]
+ if t1 < t2:
+ iscurrent = False
+ except:
+ iscurrent = False
+
+ return iscurrent
def execute_runqueue(self):
"""
event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata))
- # Find out which tasks have current stamps which we can skip when the
- # time comes
- currentstamps = self.check_stamps()
- self.stats.taskSkipped(len(currentstamps))
- self.stats.taskCompleted(len(currentstamps))
-
while True:
task = self.sched.next()
if task is not None:
fn = self.taskData.fn_index[self.runq_fnid[task]]
taskname = self.runq_task[task]
- if task in currentstamps:
- #if bb.build.stamp_is_current(taskname, self.dataCache, fn):
+ if self.check_stamp(task):
bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
self.runq_running[task] = 1
self.task_complete(task)
- #self.stats.taskCompleted()
- #self.stats.taskSkipped()
+ self.stats.taskCompleted()
+ self.stats.taskSkipped()
continue
bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task)))
ids = []
for dep in task_deps['depends'][task].split():
if dep:
- ids.append(str(self.getbuild_id(dep.split(":")[0])) + ":" + dep.split(":")[1])
+ ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
self.tasks_idepends[taskid].extend(ids)
# Work out build dependencies
rdepends = dataCache.rundeps[fn]
rrecs = dataCache.runrecs[fn]
for package in rdepends:
- for rdepend in rdepends[package]:
+ for rdepend in bb.utils.explode_deps(rdepends[package]):
bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
rdependids[self.getrun_id(rdepend)] = None
for package in rrecs:
- for rdepend in rrecs[package]:
+ for rdepend in bb.utils.explode_deps(rrecs[package]):
bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
rdependids[self.getrun_id(rdepend)] = None
self.rdepids[fnid] = rdependids.keys()
"""
if fnid in self.failed_fnids:
return
- if not missing_list:
- missing_list = [fnid]
bb.msg.debug(1, bb.msg.domain.Provider, "File '%s' is unbuildable, removing..." % self.fn_index[fnid])
self.failed_fnids.append(fnid)
for target in self.build_targets:
dependees = self.get_dependees(targetid)
for fnid in dependees:
self.fail_fnid(fnid, missing_list)
+ for taskid in range(len(self.tasks_idepends)):
+ idepends = self.tasks_idepends[taskid]
+ for (idependid, idependtask) in idepends:
+ if idependid == targetid:
+ self.fail_fnid(self.tasks_fnid[taskid], missing_list)
+
if self.abort and targetid in self.external_targets:
bb.msg.error(bb.msg.domain.Provider, "Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s" % (self.build_names_index[targetid], missing_list))
raise bb.providers.NoProvider
